From 2b58e6cde608c64ea90e6687cfa0a44f7f749ebd Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 11 Aug 2026 18:22:17 -0400 Subject: [PATCH 1/6] Evidence-tier release contract + publish path in microcosm-data (#506) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit validate_evidence_release_dir as a sibling of the certified contract: same required files and shape/provenance/hash checks, plus a mandatory non-empty known_failures block (verbatim gate-failure strings, each with an owner issue ref). Evidence manifests declare a distinct schema marker (1-evidence) and tier field, so the certified contract structurally rejects them in one direction and the evidence contract rejects certified-shape manifests in the other; validate_release_dir itself is untouched. Scoped to national-default releases: local-area roles (#398) and UK exact-k releases (#611) are refused pending their own adjudicated evidence semantics. publish_release grows evidence=True: validates via the evidence contract, publishes the immutable tag as usual, and moves only latest-evidence.json (payload mirrors latest_pointer_payload + tier) — the certified latest.json path never appears in an evidence commit. latest_evidence_release reads the new pointer; latest_release now refuses a pointer payload naming a foreign tier. The publish CLI grows --evidence. Slack alerts label the evidence tier so they can never read as a certified release announcement. Ported from the pre-rename populace tree (branch evidence-tier-506, commit 9012391d) onto the microcosm namespace. Co-Authored-By: Claude Fable 5 --- changelog.d/506-evidence-tier.added.md | 1 + .../src/microcosm/data/__init__.py | 16 + .../src/microcosm/data/contract.py | 239 +++++++++++- .../src/microcosm/data/publish_cli.py | 12 + .../src/microcosm/data/release.py | 192 +++++++-- .../src/microcosm/data/slack.py | 21 +- .../microcosm-data/tests/test_contract.py | 367 ++++++++++++++++++ .../tests/test_publish_guard.py | 28 ++ packages/microcosm-data/tests/test_release.py | 217 +++++++++++ 9 files changed, 1053 insertions(+), 40 deletions(-) create mode 100644 changelog.d/506-evidence-tier.added.md diff --git a/changelog.d/506-evidence-tier.added.md b/changelog.d/506-evidence-tier.added.md new file mode 100644 index 000000000..fb8810f0c --- /dev/null +++ b/changelog.d/506-evidence-tier.added.md @@ -0,0 +1 @@ +Evidence-tier publishing (microcosm#506): `validate_evidence_release_dir` as a structural sibling of the certified contract (same required files plus a mandatory non-empty `known_failures` block with owner issue refs), `publish_release(evidence=True)` moving only the new `latest-evidence.json` pointer (never `latest.json`), `latest_evidence_release` for consumers, and `microcosm-publish-release --evidence`. diff --git a/packages/microcosm-data/src/microcosm/data/__init__.py b/packages/microcosm-data/src/microcosm/data/__init__.py index 7c3f1feed..4bc0fabac 100644 --- a/packages/microcosm-data/src/microcosm/data/__init__.py +++ b/packages/microcosm-data/src/microcosm/data/__init__.py @@ -23,11 +23,14 @@ """ from microcosm.data.contract import ( + EVIDENCE_RELEASE_ID_SEGMENT, + EVIDENCE_RELEASE_MANIFEST_SCHEMA_VERSION, RELEASE_MANIFEST_SCHEMA_VERSION, REQUIRED_RELEASE_FILES, US_SOURCE_COVERAGE_DIAGNOSTICS_FILE, ReleaseContractError, required_release_files, + validate_evidence_release_dir, validate_release_dir, ) from microcosm.data.loader import ( @@ -40,9 +43,14 @@ ) from microcosm.data.registry import DEFAULT_VARIANT, REGISTRY, DatasetSpec, register from microcosm.data.release import ( + LATEST_EVIDENCE_POINTER_PATH, LATEST_POINTER_PATH, LATEST_POINTER_SCHEMA_VERSION, + RELEASE_TIER_CERTIFIED, + RELEASE_TIER_EVIDENCE, LatestPointer, + latest_evidence_pointer_payload, + latest_evidence_release, latest_pointer_payload, latest_release, publish_release, @@ -59,15 +67,23 @@ "DEFAULT_VARIANT", "REGISTRY", "register", + "EVIDENCE_RELEASE_ID_SEGMENT", + "EVIDENCE_RELEASE_MANIFEST_SCHEMA_VERSION", "RELEASE_MANIFEST_SCHEMA_VERSION", "REQUIRED_RELEASE_FILES", "US_SOURCE_COVERAGE_DIAGNOSTICS_FILE", "ReleaseContractError", "required_release_files", + "validate_evidence_release_dir", "validate_release_dir", + "LATEST_EVIDENCE_POINTER_PATH", "LATEST_POINTER_PATH", "LATEST_POINTER_SCHEMA_VERSION", + "RELEASE_TIER_CERTIFIED", + "RELEASE_TIER_EVIDENCE", "LatestPointer", + "latest_evidence_pointer_payload", + "latest_evidence_release", "latest_pointer_payload", "latest_release", "publish_release", diff --git a/packages/microcosm-data/src/microcosm/data/contract.py b/packages/microcosm-data/src/microcosm/data/contract.py index f48fae755..c5f238f52 100644 --- a/packages/microcosm-data/src/microcosm/data/contract.py +++ b/packages/microcosm-data/src/microcosm/data/contract.py @@ -14,6 +14,20 @@ naming **every** failure at once (a publisher should see the full repair list, not play whack-a-mole one failure per run). Publishing code calls it before any byte reaches the Hub. + +Two publication tiers share this module (microcosm#506). The **certified** +tier is :func:`validate_release_dir`, unchanged. The **evidence** tier is +:func:`validate_evidence_release_dir`, a sibling — not a relaxation — for +the best-available artifact when terminal gates failed: the same required +files, the same shape and provenance checks, plus a mandatory non-empty +``known_failures`` block carrying every recorded gate failure verbatim with +an owner issue. The tiers are structurally mutually exclusive: an evidence +manifest declares :data:`EVIDENCE_RELEASE_MANIFEST_SCHEMA_VERSION`, which +the certified contract rejects, and a certified manifest carries no +``known_failures``, which the evidence contract requires. (Distinct from +the UK terminal-gate *evidence receipts* checked below: those attest how a +certified verdict was reached; the evidence *tier* publishes an artifact +whose verdicts failed.) """ from __future__ import annotations @@ -43,6 +57,8 @@ ) __all__ = [ + "EVIDENCE_RELEASE_ID_SEGMENT", + "EVIDENCE_RELEASE_MANIFEST_SCHEMA_VERSION", "LOCAL_AREA_REQUIRED_RELEASE_FILES", "NATIONAL_DEFAULT_DATASET_ROLE", "NON_DEFAULT_LOCAL_AREA_DATASET_ROLE", @@ -52,6 +68,7 @@ "ReleaseContractError", "release_dataset_role", "required_release_files", + "validate_evidence_release_dir", "validate_release_dir", ] @@ -60,6 +77,18 @@ #: unversioned 1abddeb-era manifest is exactly the silence this guards against. RELEASE_MANIFEST_SCHEMA_VERSION = 1 +#: The release-manifest schema marker for EVIDENCE-tier releases +#: (microcosm#506). Deliberately a distinct value, not a superset flag on the +#: certified schema: the certified contract rejects any manifest carrying it, +#: so an evidence artifact can never be mistaken for (or promoted as) a +#: certified one, no matter which gates happened to fail. +EVIDENCE_RELEASE_MANIFEST_SCHEMA_VERSION = "1-evidence" + +#: Evidence release ids must carry this segment (e.g. +#: ``populace-us-2024-evidence--``) so the tier is visible in the +#: artifact name itself — in Hub tags, download paths, and logs. +EVIDENCE_RELEASE_ID_SEGMENT = "-evidence-" + #: Files a release directory must contain to count as published. A release #: missing any of these is invisible to :func:`validate_release_dir`-respecting #: publishers, by design. @@ -624,7 +653,11 @@ def _check_uk_terminal_build_manifest( def _check_release_manifest( - manifest: Mapping, release_id: str, failures: list[str] + manifest: Mapping, + release_id: str, + failures: list[str], + *, + expected_schema_version: object = RELEASE_MANIFEST_SCHEMA_VERSION, ) -> None: schema_version = manifest.get("schema_version") if schema_version is None: @@ -632,11 +665,11 @@ def _check_release_manifest( "release_manifest.json has no 'schema_version'; unversioned " "manifests (the 1abddeb-era shape) are not publishable." ) - elif schema_version != RELEASE_MANIFEST_SCHEMA_VERSION: + elif schema_version != expected_schema_version: failures.append( f"release_manifest.json 'schema_version' is {schema_version!r}; " f"this library publishes version " - f"{RELEASE_MANIFEST_SCHEMA_VERSION}." + f"{expected_schema_version!r}." ) build = manifest.get("build") if not isinstance(build, Mapping) or not build.get("build_id"): @@ -2679,7 +2712,10 @@ def _is_congressional_district_layout_target(target: Mapping) -> bool: def _check_source_coverage_diagnostics( - diagnostics: Mapping, failures: list[str] + diagnostics: Mapping, + failures: list[str], + *, + require_gate_passed: bool = True, ) -> None: schema_version = diagnostics.get("schema_version") if schema_version is None: @@ -2728,7 +2764,7 @@ def _check_source_coverage_diagnostics( f"{US_SOURCE_COVERAGE_DIAGNOSTICS_FILE} gate.name must be " "'us_source_coverage'." ) - if gate.get("passed") is not True: + if require_gate_passed and gate.get("passed") is not True: failures.append( f"{US_SOURCE_COVERAGE_DIAGNOSTICS_FILE} gate.passed must be true." ) @@ -2737,7 +2773,7 @@ def _check_source_coverage_diagnostics( failures.append( f"{US_SOURCE_COVERAGE_DIAGNOSTICS_FILE} gate.failures must be a list." ) - elif gate_failures: + elif gate_failures and require_gate_passed: failures.append( f"{US_SOURCE_COVERAGE_DIAGNOSTICS_FILE} gate.failures must be empty." ) @@ -3323,6 +3359,197 @@ def validate_release_dir(release_dir: Path | str) -> None: raise ReleaseContractError(release_dir, failures) +#: Owner refs in ``known_failures`` must point at a tracked issue — a +#: ``#NNN`` shorthand (optionally repo-qualified) or a GitHub issue/PR URL. +#: A name or a prose excuse is not an owner: the evidence tier ships a +#: failure only when somewhere is accountable for fixing it. +_ISSUE_REF_RE = re.compile(r"#\d+|github\.com/\S+/(?:issues|pull)/\d+") + + +def _check_evidence_release_manifest(manifest: Mapping, failures: list[str]) -> None: + """Evidence-only manifest requirements: the tier marker and the honest + non-empty ``known_failures`` record.""" + if manifest.get("tier") != "evidence": + failures.append( + "release_manifest.json 'tier' must be 'evidence' for an " + "evidence-tier release." + ) + known_failures = manifest.get("known_failures") + if not isinstance(known_failures, list) or not known_failures: + failures.append( + "release_manifest.json must declare a non-empty 'known_failures' " + "list; the evidence tier exists to carry recorded gate failures " + "honestly, never to hide them (an all-green artifact belongs on " + "the certified path)." + ) + return + for index, entry in enumerate(known_failures): + owner_prefix = f"release_manifest.json known_failures[{index}]" + if not isinstance(entry, Mapping): + failures.append( + f"{owner_prefix} must be an object with 'failure' and 'owner'." + ) + continue + failure_text = entry.get("failure") + if not isinstance(failure_text, str) or not failure_text.strip(): + failures.append( + f"{owner_prefix}.failure must be the recorded gate-failure " + "string, verbatim and non-empty." + ) + owner = entry.get("owner") + if not isinstance(owner, str) or not _ISSUE_REF_RE.search(owner): + failures.append( + f"{owner_prefix}.owner must carry an issue reference " + "(e.g. 'PolicyEngine/microcosm#487' or an issue URL)." + ) + + +def validate_evidence_release_dir(release_dir: Path | str) -> None: + """Check a local EVIDENCE-tier release directory against its contract. + + The sibling of :func:`validate_release_dir` for the best-available + artifact when terminal gates failed (microcosm#506): the same required + files and the same shape, provenance, and cross-manifest checks, with the + gate *verdict* requirements replaced by a recording requirement — the + release manifest must carry a non-empty ``known_failures`` block naming + every recorded gate failure verbatim, each with an owner issue. + + This is a different output contract, not a bypass of the certified one: + + - the release manifest must declare + :data:`EVIDENCE_RELEASE_MANIFEST_SCHEMA_VERSION` and ``tier: + "evidence"``, which the certified contract structurally rejects; + - the release id must carry the :data:`EVIDENCE_RELEASE_ID_SEGMENT`, so + the tier is visible in every tag and download path; + - critical-target fit and gate-passed requirements are not enforced — + their failures are exactly what ``known_failures`` records — but + everything that makes the artifact *auditable* (required files, build + provenance with a clean git commit, artifact hashes, cross-manifest + agreement) is enforced unchanged. + + Scope (microcosm#506, "dense first"): the tier exists for the US + national artifact. Non-default local-area releases and UK exact-k + releases are refused outright — each has its own certification lane + (microcosm#398, microcosm#611) and no evidence-tier semantics have been + adjudicated for them. + + Args: + release_dir: The local ``releases/`` directory about to be + published at the evidence tier. + + Raises: + ReleaseContractError: Naming every violation found. + """ + release_dir = Path(release_dir) + release_id = release_dir.name + failures: list[str] = [] + + if not release_dir.is_dir(): + raise ReleaseContractError(release_dir, [f"{release_dir} is not a directory."]) + + if EVIDENCE_RELEASE_ID_SEGMENT not in release_id: + failures.append( + f"evidence release ids must carry the " + f"{EVIDENCE_RELEASE_ID_SEGMENT!r} segment; {release_id!r} does " + "not name its tier." + ) + + if _is_uk_exact_k_release_id(release_id): + raise ReleaseContractError( + release_dir, + [ + "UK exact-k releases have no evidence-tier contract; the " + "gate-battery lane (microcosm#611) owns their verdicts and " + "microcosm#506 scoped the evidence tier to the US national " + "artifact." + ], + ) + + manifest_probe_path = release_dir / "release_manifest.json" + if manifest_probe_path.is_file(): + try: + manifest_probe = json.loads(manifest_probe_path.read_text()) + except (OSError, ValueError): + manifest_probe = None + if isinstance(manifest_probe, Mapping) and "dataset_role" in manifest_probe: + declared_role = manifest_probe["dataset_role"] + if declared_role != NATIONAL_DEFAULT_DATASET_ROLE: + raise ReleaseContractError( + release_dir, + [ + "the evidence tier supports only " + f"{NATIONAL_DEFAULT_DATASET_ROLE!r} releases " + f"(microcosm#506); dataset_role {declared_role!r} " + "has its own contract and no evidence-tier " + "semantics." + ], + ) + + build_manifest: Mapping | None = None + release_manifest: Mapping | None = None + calibration_diagnostics: Mapping | None = None + source_coverage_diagnostics: Mapping | None = None + + for filename in required_release_files(release_id): + if not (release_dir / filename).is_file(): + failures.append(f"required file {filename!r} is missing.") + + build_manifest_path = release_dir / "build_manifest.json" + if build_manifest_path.is_file(): + manifest = _load_json(build_manifest_path, failures) + if manifest is not None: + build_manifest = manifest + _check_build_manifest(manifest, release_id, failures) + + release_manifest_path = release_dir / "release_manifest.json" + if release_manifest_path.is_file(): + manifest = _load_json(release_manifest_path, failures) + if manifest is not None: + release_manifest = manifest + _check_release_manifest( + manifest, + release_id, + failures, + expected_schema_version=EVIDENCE_RELEASE_MANIFEST_SCHEMA_VERSION, + ) + _check_evidence_release_manifest(manifest, failures) + + calibration_diagnostics_path = release_dir / "calibration_diagnostics.json" + if calibration_diagnostics_path.is_file(): + diagnostics = _load_json(calibration_diagnostics_path, failures) + if diagnostics is not None: + calibration_diagnostics = diagnostics + _check_calibration_diagnostics(diagnostics, failures) + # No _check_us_critical_target_fit here: critical-fit breaches are + # the evidence tier's known_failures, not contract violations. + + _check_cross_manifest_consistency( + build_manifest, + release_manifest, + calibration_diagnostics, + failures, + ) + _check_local_artifact_hashes(release_dir, release_manifest, failures) + + source_coverage_path = release_dir / US_SOURCE_COVERAGE_DIAGNOSTICS_FILE + if release_id.startswith("populace-us-") and source_coverage_path.is_file(): + diagnostics = _load_json(source_coverage_path, failures) + if diagnostics is not None: + source_coverage_diagnostics = diagnostics + _check_source_coverage_diagnostics( + diagnostics, + failures, + require_gate_passed=False, + ) + + _check_us_fiscal_source_consistency( + calibration_diagnostics, source_coverage_diagnostics, failures + ) + + if failures: + raise ReleaseContractError(release_dir, failures) + + def _check_us_fiscal_source_consistency( calibration_diagnostics: Mapping | None, source_coverage_diagnostics: Mapping | None, diff --git a/packages/microcosm-data/src/microcosm/data/publish_cli.py b/packages/microcosm-data/src/microcosm/data/publish_cli.py index 16b3f5e50..f339d97b2 100644 --- a/packages/microcosm-data/src/microcosm/data/publish_cli.py +++ b/packages/microcosm-data/src/microcosm/data/publish_cli.py @@ -118,6 +118,17 @@ def main(argv: list[str] | None = None) -> int: "release never silently ships blank OBBBA validation." ), ) + parser.add_argument( + "--evidence", + action="store_true", + help=( + "Publish at the EVIDENCE tier (microcosm#506): validate against the " + "evidence release contract (which requires a non-empty " + "known_failures block) and move latest-evidence.json instead of " + "latest.json. Structurally never touches the certified pointer; a " + "certified-shape release is refused under this flag." + ), + ) args = parser.parse_args(argv) if args.tag_only and not args.no_latest: @@ -157,6 +168,7 @@ def main(argv: list[str] | None = None) -> int: updated_at=args.updated_at, update_latest=not args.no_latest, tag_only=args.tag_only, + evidence=args.evidence, ) print(json.dumps(pointer, indent=2)) diff --git a/packages/microcosm-data/src/microcosm/data/release.py b/packages/microcosm-data/src/microcosm/data/release.py index 961aa951d..790abdfa4 100644 --- a/packages/microcosm-data/src/microcosm/data/release.py +++ b/packages/microcosm-data/src/microcosm/data/release.py @@ -19,6 +19,11 @@ returns the typed pointer, the one-call answer to "which release is current?" for dashboards and scorers. +The EVIDENCE tier (microcosm#506) publishes through the same producer with +``evidence=True``: identical immutable-tag mechanics, but the pointer that +moves is ``latest-evidence.json`` — structurally never ``latest.json`` — +and :func:`latest_evidence_release` is its consumer. + The Hub client is injected (``api=``) everywhere it is used, so the suite exercises the real branch, commit, tag, and pointer ordering against a fake — no network and no mocking of our own internals. @@ -35,34 +40,53 @@ from typing import Any from microcosm.data.contract import ( + EVIDENCE_RELEASE_ID_SEGMENT, NATIONAL_DEFAULT_DATASET_ROLE, release_dataset_role, required_release_files, + validate_evidence_release_dir, validate_release_dir, ) from microcosm.data.slack import notify_release __all__ = [ + "LATEST_EVIDENCE_POINTER_PATH", "LATEST_POINTER_PATH", "LATEST_POINTER_SCHEMA_VERSION", + "RELEASE_TIER_CERTIFIED", + "RELEASE_TIER_EVIDENCE", "LatestPointer", + "latest_evidence_pointer_payload", "latest_pointer_payload", "publish_release", "latest_release", + "latest_evidence_release", ] #: Where the pointer lives in the dataset repo. The root, not a release #: directory: the pointer is repo state, not release state. LATEST_POINTER_PATH = "latest.json" +#: The evidence-tier pointer (microcosm#506): which evidence release is the +#: best *current* one. A separate file at the repo root, so certified +#: consumers reading ``latest.json`` (and the pe.py certification path built +#: on it) can never pick up an evidence artifact by accident. +LATEST_EVIDENCE_POINTER_PATH = "latest-evidence.json" + #: Version of the pointer payload itself, so the pointer can evolve without #: consumers guessing (the same discipline the release manifest learned). LATEST_POINTER_SCHEMA_VERSION = 1 +#: Publication tiers (microcosm#506). Certified is the default everywhere; +#: the evidence tier is opted into explicitly and carries its tier in the +#: pointer payload, the release id, and the release manifest. +RELEASE_TIER_CERTIFIED = "certified" +RELEASE_TIER_EVIDENCE = "evidence" + @dataclass(frozen=True) class LatestPointer: - """The parsed ``latest.json``: which release is current, and where. + """A parsed release pointer: which release is current, and where. Attributes: release_id: The current build id (the ``releases/`` directory name). @@ -71,11 +95,16 @@ class LatestPointer: (``"build_manifest"``, ``"release_manifest"``, ``"calibration_diagnostics"``, plus country-specific contract files such as US source coverage). + tier: The publication tier the pointer names — + :data:`RELEASE_TIER_CERTIFIED` for ``latest.json`` (whose payload + predates tiers and carries no field), or + :data:`RELEASE_TIER_EVIDENCE` for ``latest-evidence.json``. """ release_id: str updated_at: str paths: dict[str, str] + tier: str = RELEASE_TIER_CERTIFIED def latest_pointer_payload(release_id: str, *, updated_at: str | None = None) -> dict: @@ -102,6 +131,21 @@ def latest_pointer_payload(release_id: str, *, updated_at: str | None = None) -> } +def latest_evidence_pointer_payload( + release_id: str, *, updated_at: str | None = None +) -> dict: + """The ``latest-evidence.json`` payload for ``release_id``. + + Mirrors :func:`latest_pointer_payload` exactly, plus a ``tier`` field — + so evidence consumers reuse certified pointer tooling, while a reader + that lands on the wrong file sees the tier immediately. + """ + return { + **latest_pointer_payload(release_id, updated_at=updated_at), + "tier": RELEASE_TIER_EVIDENCE, + } + + def _hf_api(): try: from huggingface_hub import HfApi @@ -126,6 +170,7 @@ def publish_release( update_latest: bool = True, tag_only: bool = False, notify: bool = True, + evidence: bool = False, ) -> dict: """Publish a release directory and optionally point ``latest.json`` at it. @@ -173,18 +218,31 @@ def publish_release( fires when ``update_latest`` is set — a non-default publish moves no pointer, so there is no "new latest release" to announce. Set ``False`` to suppress it (tests, dry-runs, re-publishes). + evidence: Publish at the EVIDENCE tier (microcosm#506). The release + is validated against + :func:`~microcosm.data.contract.validate_evidence_release_dir` + instead of the certified contract, and the pointer that moves is + ``latest-evidence.json`` — this path is structurally incapable of + writing ``latest.json``, so an evidence artifact can never become + the certified default or feed pe.py certification. Tag and upload + mechanics are otherwise identical. ``update_latest`` then governs + the evidence pointer. Returns: - The release's ``latest.json`` payload. It is uploaded only when + The release's pointer payload (``latest.json`` shape, plus a ``tier`` + field at the evidence tier). It is uploaded only when ``update_latest=True``. Raises: - ReleaseContractError: If the release directory violates the + ReleaseContractError: If the release directory violates its tier's contract. Nothing is uploaded in that case. FileNotFoundError: If an ``extra_files`` entry does not exist. """ release_dir = Path(release_dir) - validate_release_dir(release_dir) + if evidence: + validate_evidence_release_dir(release_dir) + else: + validate_release_dir(release_dir) release_id = release_dir.name role = release_dataset_role(release_dir) if role != NATIONAL_DEFAULT_DATASET_ROLE and update_latest: @@ -262,7 +320,10 @@ def publish_release( if api is None: api = _hf_api() - payload = latest_pointer_payload(release_id, updated_at=updated_at) + if evidence: + payload = latest_evidence_pointer_payload(release_id, updated_at=updated_at) + else: + payload = latest_pointer_payload(release_id, updated_at=updated_at) if create_tag and not callable(getattr(api, "create_tag", None)): raise TypeError( "publish_release requires a Hub backend with create_tag support; " @@ -287,14 +348,17 @@ def publish_release( create_tag=create_tag, update_latest=update_latest, tag_only=tag_only, + evidence=evidence, ) # The pointer is live: announce it. Best-effort and coupled to the promotion # so every publish path alerts; warn (don't fail) if the webhook is unset. # Skip when no pointer moved — a non-default publish is not a new release. if notify and update_latest: - notify_release( - repo_id, release_id, payload.get("updated_at"), warn_if_unset=True - ) + notify_kwargs: dict = {"warn_if_unset": True} + if evidence: + # The alert must never read as a certified release announcement. + notify_kwargs["tier"] = RELEASE_TIER_EVIDENCE + notify_release(repo_id, release_id, payload.get("updated_at"), **notify_kwargs) return payload @@ -313,6 +377,7 @@ def _commit_operations( filenames: list[str], root_artifacts: Mapping[str, str], pointer: bytes | None = None, + pointer_path: str = LATEST_POINTER_PATH, ) -> list: try: from huggingface_hub import CommitOperationAdd @@ -340,7 +405,7 @@ def _commit_operations( if pointer is not None: operations.append( CommitOperationAdd( - path_in_repo=LATEST_POINTER_PATH, + path_in_repo=pointer_path, path_or_fileobj=pointer, ) ) @@ -373,6 +438,7 @@ def _publish_atomic( create_tag: bool, update_latest: bool = True, tag_only: bool = False, + evidence: bool = False, ) -> None: staging_branch = f"release-staging/{release_id}" main_revision = _repo_revision(api, repo_id=repo_id) @@ -419,11 +485,16 @@ def _publish_atomic( # candidates deliberately stop here: neither canonical root artifacts # nor release-directory copies are written to main. return + # The evidence tier writes ONLY its own pointer file: the certified + # ``latest.json`` path never appears in an evidence commit, so no bug in + # flag-plumbing can promote an evidence artifact to certified default. + tier_label = "evidence release" if evidence else "release" + pointer_path = LATEST_EVIDENCE_POINTER_PATH if evidence else LATEST_POINTER_PATH if update_latest: - message = f"Update latest release to {release_id}" + message = f"Update latest {tier_label} to {release_id}" pointer = json.dumps(payload, indent=1).encode() else: - message = f"Publish non-default release {release_id}" + message = f"Publish non-default {tier_label} {release_id}" pointer = None api.create_commit( repo_id=repo_id, @@ -437,6 +508,7 @@ def _publish_atomic( filenames=filenames, root_artifacts=root_artifacts, pointer=pointer, + pointer_path=pointer_path, ), ) @@ -586,38 +658,27 @@ def _create_release_tag(api: object, *, repo_id: str, tag: str, revision: str | return create_tag(**kwargs) -def latest_release(repo_id: str, *, api=None) -> LatestPointer: - """Read ``latest.json`` from a dataset repo: which release is current. - - Args: - repo_id: Hub dataset repo, e.g. ``"policyengine/populace-us"``. - api: A ``huggingface_hub.HfApi``-shaped object (anything with - ``hf_hub_download(repo_id=, filename=, repo_type=)``); - constructed lazily when omitted. - - Raises: - ValueError: If the pointer is malformed or its schema version is - newer than this library understands. - """ +def _read_pointer(repo_id: str, api, *, pointer_path: str) -> dict: + """Download and structurally validate a release pointer file.""" if api is None: api = _hf_api() local = api.hf_hub_download( - repo_id=repo_id, filename=LATEST_POINTER_PATH, repo_type="dataset" + repo_id=repo_id, filename=pointer_path, repo_type="dataset" ) payload = json.loads(Path(local).read_text()) schema_version = payload.get("schema_version") if schema_version != LATEST_POINTER_SCHEMA_VERSION: raise ValueError( - f"{LATEST_POINTER_PATH} in {repo_id} has schema_version " + f"{pointer_path} in {repo_id} has schema_version " f"{schema_version!r}; this microcosm-data reads version " f"{LATEST_POINTER_SCHEMA_VERSION}. Upgrade microcosm-data." ) release_id = payload.get("release_id") if not release_id: - raise ValueError(f"{LATEST_POINTER_PATH} in {repo_id} has no 'release_id'.") + raise ValueError(f"{pointer_path} in {repo_id} has no 'release_id'.") paths = payload.get("paths") if not isinstance(paths, dict): - raise ValueError(f"{LATEST_POINTER_PATH} in {repo_id} has no 'paths' object.") + raise ValueError(f"{pointer_path} in {repo_id} has no 'paths' object.") expected_paths = latest_pointer_payload(str(release_id), updated_at="")["paths"] observed_paths = {str(key): value for key, value in paths.items()} missing_paths = sorted(set(expected_paths) - set(observed_paths)) @@ -629,12 +690,81 @@ def latest_release(repo_id: str, *, api=None) -> LatestPointer: ) if missing_paths or unexpected_paths or malformed_paths: raise ValueError( - f"{LATEST_POINTER_PATH} in {repo_id} has incomplete paths: " + f"{pointer_path} in {repo_id} has incomplete paths: " f"missing={missing_paths}, unexpected={unexpected_paths}, " f"malformed={malformed_paths}." ) + return payload + + +def latest_release(repo_id: str, *, api=None) -> LatestPointer: + """Read ``latest.json`` from a dataset repo: which release is current. + + Args: + repo_id: Hub dataset repo, e.g. ``"policyengine/populace-us"``. + api: A ``huggingface_hub.HfApi``-shaped object (anything with + ``hf_hub_download(repo_id=, filename=, repo_type=)``); + constructed lazily when omitted. + + Raises: + ValueError: If the pointer is malformed, its schema version is newer + than this library understands, or it names a non-certified tier + (an evidence payload in ``latest.json`` is a publication bug and + must never be consumed as the certified default). + """ + payload = _read_pointer(repo_id, api, pointer_path=LATEST_POINTER_PATH) + tier = payload.get("tier") + if tier not in (None, RELEASE_TIER_CERTIFIED): + raise ValueError( + f"{LATEST_POINTER_PATH} in {repo_id} declares tier {tier!r}; the " + "certified pointer must never name another tier — evidence " + f"releases live at {LATEST_EVIDENCE_POINTER_PATH}." + ) + return LatestPointer( + release_id=str(payload["release_id"]), + updated_at=str(payload.get("updated_at", "")), + paths={str(k): str(v) for k, v in payload["paths"].items()}, + tier=RELEASE_TIER_CERTIFIED, + ) + + +def latest_evidence_release(repo_id: str, *, api=None) -> LatestPointer: + """Read ``latest-evidence.json``: the best *current* evidence release. + + The evidence-tier sibling of :func:`latest_release` (microcosm#506) — how + consumers discover the best available artifact when no certified release + carries it yet. Each evidence publish supersedes the last, so this + pointer always names the current one. + + Args: + repo_id: Hub dataset repo, e.g. ``"policyengine/populace-us"``. + api: A ``huggingface_hub.HfApi``-shaped object (anything with + ``hf_hub_download(repo_id=, filename=, repo_type=)``); + constructed lazily when omitted. + + Raises: + ValueError: If the pointer is malformed, does not declare the + evidence tier, or names a release id without the + ``-evidence-`` segment. + """ + payload = _read_pointer(repo_id, api, pointer_path=LATEST_EVIDENCE_POINTER_PATH) + tier = payload.get("tier") + if tier != RELEASE_TIER_EVIDENCE: + raise ValueError( + f"{LATEST_EVIDENCE_POINTER_PATH} in {repo_id} declares tier " + f"{tier!r}; the evidence pointer must declare " + f"{RELEASE_TIER_EVIDENCE!r}." + ) + release_id = str(payload["release_id"]) + if EVIDENCE_RELEASE_ID_SEGMENT not in release_id: + raise ValueError( + f"{LATEST_EVIDENCE_POINTER_PATH} in {repo_id} names release " + f"{release_id!r}, which does not carry the " + f"{EVIDENCE_RELEASE_ID_SEGMENT!r} segment." + ) return LatestPointer( - release_id=str(release_id), + release_id=release_id, updated_at=str(payload.get("updated_at", "")), - paths={str(k): str(v) for k, v in paths.items()}, + paths={str(k): str(v) for k, v in payload["paths"].items()}, + tier=RELEASE_TIER_EVIDENCE, ) diff --git a/packages/microcosm-data/src/microcosm/data/slack.py b/packages/microcosm-data/src/microcosm/data/slack.py index 1fd453ce8..b9fd01646 100644 --- a/packages/microcosm-data/src/microcosm/data/slack.py +++ b/packages/microcosm-data/src/microcosm/data/slack.py @@ -49,6 +49,7 @@ def notify_release( webhook: str | None = None, post: Callable[[str, dict[str, Any]], None] | None = None, warn_if_unset: bool = False, + tier: str = "certified", ) -> bool: """Announce a published release to the country's Slack channel. @@ -56,7 +57,9 @@ def notify_release( env var is unset. Never raises — a failed post is logged, not fatal. When ``warn_if_unset`` is set, an unset webhook logs a warning instead of returning silently, so a release that publishes without an alert is visible - in the log rather than a mystery. + in the log rather than a mystery. ``tier="evidence"`` labels the alert as + an evidence-tier publish (microcosm#506) so it can never read as a new + certified release; the certified message is unchanged. """ country = country_for_repo(repo_id) url = webhook or os.environ.get(CHANNEL_ENV[country]) @@ -69,23 +72,35 @@ def notify_release( return False label = "🇬🇧 UK" if country == "uk" else "🇺🇸 US" + is_evidence = tier == "evidence" context = " · ".join( part for part in ( repo_id, + ( + "evidence tier — known failures recorded in the release manifest" + if is_evidence + else "" + ), f"published {updated_at}" if updated_at else "", f"<{DASHBOARD_URL}|calibration diagnostics>", ) if part ) + if is_evidence: + text = f"New Microcosm {country.upper()} EVIDENCE release: {release_id}" + header = f":warning: *New Microcosm {label} EVIDENCE release*\n`{release_id}`" + else: + text = f"New Microcosm {country.upper()} release: {release_id}" + header = f":rocket: *New Microcosm {label} release*\n`{release_id}`" payload = { - "text": f"New Microcosm {country.upper()} release: {release_id}", + "text": text, "blocks": [ { "type": "section", "text": { "type": "mrkdwn", - "text": f":rocket: *New Microcosm {label} release*\n`{release_id}`", + "text": header, }, }, { diff --git a/packages/microcosm-data/tests/test_contract.py b/packages/microcosm-data/tests/test_contract.py index 50801705e..819ee27c1 100644 --- a/packages/microcosm-data/tests/test_contract.py +++ b/packages/microcosm-data/tests/test_contract.py @@ -17,10 +17,13 @@ import pytest from microcosm.data import ( + EVIDENCE_RELEASE_ID_SEGMENT, + EVIDENCE_RELEASE_MANIFEST_SCHEMA_VERSION, RELEASE_MANIFEST_SCHEMA_VERSION, US_SOURCE_COVERAGE_DIAGNOSTICS_FILE, ReleaseContractError, required_release_files, + validate_evidence_release_dir, validate_release_dir, ) @@ -3449,3 +3452,367 @@ def test_exact_k_uk_terminal_rejects_invalid_qrf_observable_values( failures = "\n".join(excinfo.value.failures) assert match in failures assert "attestation.signature does not authenticate" not in failures + + +# --------------------------------------------------------------------------- +# Evidence-tier release contract (microcosm#506) +# --------------------------------------------------------------------------- +# +# The evidence contract is a SIBLING of the certified one, not a relaxation: +# same required files, plus a mandatory non-empty known_failures block naming +# every recorded gate failure verbatim with an owner issue. The two tiers must +# never be confusable — an evidence manifest fails certified validation +# structurally (distinct schema marker), and a certified manifest fails +# evidence validation (no tier, no known_failures). + +EVIDENCE_RELEASE_ID = "populace-us-2024-evidence-9f1260b-20260611" + + +def _known_failures() -> list[dict]: + return [ + { + "failure": ( + "SOI Table 1.4 national dollar fit failed: target " + "'irs_soi.ty2023.table_1_4.all.capital_gain_distributions_amount" + "@2024' has relative_error=-0.302, exceeding 0.25." + ), + "owner": "PolicyEngine/microcosm#487", + }, + { + "failure": ( + "QRF tail concentration failed: 7 sparse QRF-imputed columns " + "concentrate past the top-k weighted-mass share bound." + ), + "owner": "PolicyEngine/microcosm#481", + }, + ] + + +def _evidence_release_manifest( + *, + diagnostics_sha: str, + source_coverage_sha: str, + known_failures: list[dict] | None = None, +) -> dict: + manifest = _release_manifest( + EVIDENCE_RELEASE_ID, + diagnostics_sha=diagnostics_sha, + source_coverage_sha=source_coverage_sha, + ) + manifest["schema_version"] = EVIDENCE_RELEASE_MANIFEST_SCHEMA_VERSION + manifest["tier"] = "evidence" + manifest["known_failures"] = ( + _known_failures() if known_failures is None else known_failures + ) + return manifest + + +@pytest.fixture +def evidence_release_dir(tmp_path: Path) -> Path: + """A complete, evidence-contract-valid release directory.""" + directory = tmp_path / "releases" / EVIDENCE_RELEASE_ID + directory.mkdir(parents=True) + (directory / "build_manifest.json").write_text( + json.dumps(_build_manifest(EVIDENCE_RELEASE_ID)) + ) + (directory / "calibration_diagnostics.json").write_text( + json.dumps(_calibration_diagnostics()) + ) + (directory / US_SOURCE_COVERAGE_DIAGNOSTICS_FILE).write_text( + json.dumps(_source_coverage_diagnostics()) + ) + (directory / "release_manifest.json").write_text( + json.dumps( + _evidence_release_manifest( + diagnostics_sha=_sha256(directory / "calibration_diagnostics.json"), + source_coverage_sha=_sha256( + directory / US_SOURCE_COVERAGE_DIAGNOSTICS_FILE + ), + ) + ) + ) + return directory + + +def _rewrite_evidence_manifest(evidence_release_dir: Path, mutate) -> None: + manifest_path = evidence_release_dir / "release_manifest.json" + manifest = json.loads(manifest_path.read_text()) + mutate(manifest) + manifest_path.write_text(json.dumps(manifest)) + + +def test_a_complete_evidence_release_passes(evidence_release_dir: Path) -> None: + validate_evidence_release_dir(evidence_release_dir) + + +def test_evidence_release_fails_the_certified_contract( + evidence_release_dir: Path, +) -> None: + """The structural guarantee: whatever gates failed, an evidence manifest + can never certify — its schema marker alone refuses the certified tier.""" + with pytest.raises(ReleaseContractError) as excinfo: + validate_release_dir(evidence_release_dir) + failures = "\n".join(excinfo.value.failures) + assert f"'schema_version' is {EVIDENCE_RELEASE_MANIFEST_SCHEMA_VERSION!r}" in ( + failures + ) + + +def test_certified_release_fails_the_evidence_contract(release_dir: Path) -> None: + with pytest.raises(ReleaseContractError) as excinfo: + validate_evidence_release_dir(release_dir) + failures = "\n".join(excinfo.value.failures) + assert "known_failures" in failures + assert "tier" in failures + assert EVIDENCE_RELEASE_ID_SEGMENT in failures + + +def test_evidence_release_requires_the_id_segment(tmp_path: Path) -> None: + """An evidence-shaped manifest under a certified-shaped id is refused: the + tier must be visible in the release id itself.""" + directory = tmp_path / "releases" / RELEASE_ID + directory.mkdir(parents=True) + (directory / "build_manifest.json").write_text(json.dumps(_build_manifest())) + (directory / "calibration_diagnostics.json").write_text( + json.dumps(_calibration_diagnostics()) + ) + (directory / US_SOURCE_COVERAGE_DIAGNOSTICS_FILE).write_text( + json.dumps(_source_coverage_diagnostics()) + ) + manifest = _release_manifest( + diagnostics_sha=_sha256(directory / "calibration_diagnostics.json"), + source_coverage_sha=_sha256(directory / US_SOURCE_COVERAGE_DIAGNOSTICS_FILE), + ) + manifest["schema_version"] = EVIDENCE_RELEASE_MANIFEST_SCHEMA_VERSION + manifest["tier"] = "evidence" + manifest["known_failures"] = _known_failures() + (directory / "release_manifest.json").write_text(json.dumps(manifest)) + + with pytest.raises(ReleaseContractError) as excinfo: + validate_evidence_release_dir(directory) + failures = "\n".join(excinfo.value.failures) + assert EVIDENCE_RELEASE_ID_SEGMENT in failures + + +def test_evidence_release_refuses_uk_exact_k_ids(tmp_path: Path) -> None: + """UK exact-k verdicts belong to the gate-battery lane (microcosm#611); + the evidence tier is scoped to the US national artifact.""" + directory = tmp_path / "releases" / "populace-uk-2023-evidence-frs-k535080" + directory.mkdir(parents=True) + + with pytest.raises(ReleaseContractError) as excinfo: + validate_evidence_release_dir(directory) + assert "no evidence-tier contract" in "\n".join(excinfo.value.failures) + + +def test_evidence_release_refuses_non_default_roles( + evidence_release_dir: Path, +) -> None: + """Local-area releases have their own contract (microcosm#398) and no + adjudicated evidence-tier semantics.""" + _rewrite_evidence_manifest( + evidence_release_dir, + lambda manifest: manifest.update(dataset_role="non_default_local_area"), + ) + with pytest.raises(ReleaseContractError) as excinfo: + validate_evidence_release_dir(evidence_release_dir) + failures = "\n".join(excinfo.value.failures) + assert "evidence tier supports only" in failures + + +def test_evidence_release_rejects_empty_known_failures( + evidence_release_dir: Path, +) -> None: + """The tier exists to carry failures honestly; an empty block is invalid.""" + _rewrite_evidence_manifest( + evidence_release_dir, + lambda manifest: manifest.update(known_failures=[]), + ) + with pytest.raises(ReleaseContractError) as excinfo: + validate_evidence_release_dir(evidence_release_dir) + failures = "\n".join(excinfo.value.failures) + assert "non-empty 'known_failures'" in failures + + +def test_evidence_release_rejects_missing_known_failures( + evidence_release_dir: Path, +) -> None: + def _drop(manifest: dict) -> None: + del manifest["known_failures"] + + _rewrite_evidence_manifest(evidence_release_dir, _drop) + with pytest.raises(ReleaseContractError) as excinfo: + validate_evidence_release_dir(evidence_release_dir) + assert "non-empty 'known_failures'" in "\n".join(excinfo.value.failures) + + +def test_evidence_known_failures_require_verbatim_failure_text( + evidence_release_dir: Path, +) -> None: + _rewrite_evidence_manifest( + evidence_release_dir, + lambda manifest: manifest.update( + known_failures=[{"failure": "", "owner": "PolicyEngine/microcosm#487"}] + ), + ) + with pytest.raises(ReleaseContractError) as excinfo: + validate_evidence_release_dir(evidence_release_dir) + failures = "\n".join(excinfo.value.failures) + assert "known_failures[0].failure" in failures + + +def test_evidence_known_failures_require_an_owner_issue_ref( + evidence_release_dir: Path, +) -> None: + _rewrite_evidence_manifest( + evidence_release_dir, + lambda manifest: manifest.update( + known_failures=[ + {"failure": "QRF tail concentration failed: ...", "owner": "Max"} + ] + ), + ) + with pytest.raises(ReleaseContractError) as excinfo: + validate_evidence_release_dir(evidence_release_dir) + failures = "\n".join(excinfo.value.failures) + assert "known_failures[0].owner" in failures + assert "issue reference" in failures + + +def test_evidence_known_failures_accept_issue_url_owners( + evidence_release_dir: Path, +) -> None: + _rewrite_evidence_manifest( + evidence_release_dir, + lambda manifest: manifest.update( + known_failures=[ + { + "failure": "QRF tail concentration failed: ...", + "owner": "https://github.com/PolicyEngine/microcosm/issues/481", + } + ] + ), + ) + validate_evidence_release_dir(evidence_release_dir) + + +def test_evidence_release_requires_the_evidence_tier_field( + evidence_release_dir: Path, +) -> None: + def _drop(manifest: dict) -> None: + del manifest["tier"] + + _rewrite_evidence_manifest(evidence_release_dir, _drop) + with pytest.raises(ReleaseContractError) as excinfo: + validate_evidence_release_dir(evidence_release_dir) + assert "'tier'" in "\n".join(excinfo.value.failures) + + +def test_evidence_release_rejects_certified_schema_version( + evidence_release_dir: Path, +) -> None: + _rewrite_evidence_manifest( + evidence_release_dir, + lambda manifest: manifest.update( + schema_version=RELEASE_MANIFEST_SCHEMA_VERSION + ), + ) + with pytest.raises(ReleaseContractError) as excinfo: + validate_evidence_release_dir(evidence_release_dir) + failures = "\n".join(excinfo.value.failures) + assert f"{EVIDENCE_RELEASE_MANIFEST_SCHEMA_VERSION!r}" in failures + + +def test_evidence_release_tolerates_critical_target_breaches( + evidence_release_dir: Path, +) -> None: + """The tier's point: the Build N medical-class breach ships as evidence, + carried in known_failures, instead of blocking the artifact.""" + diagnostics = _calibration_diagnostics() + target = next( + row + for row in diagnostics["targets"] + if row["name"] == "irs_soi.ty2022.historic_table_2.us.all." + "medical_dental_expense_amount@2024" + ) + target["final_estimate"] = target["target"] * 1.21 + target["relative_error"] = 0.21 + _write_json_and_refresh_manifest_hash( + evidence_release_dir, + filename="calibration_diagnostics.json", + artifact_key="calibration_diagnostics", + payload=diagnostics, + ) + + validate_evidence_release_dir(evidence_release_dir) + + with pytest.raises(ReleaseContractError): + validate_release_dir(evidence_release_dir) + + +def test_evidence_release_tolerates_failed_source_coverage_gate( + evidence_release_dir: Path, +) -> None: + payload = _source_coverage_diagnostics() + payload["gate"] = { + "name": "us_source_coverage", + "passed": False, + "failures": ["social_security_ssi/ssa-ssi-table-7b1-2024 missing"], + } + _write_json_and_refresh_manifest_hash( + evidence_release_dir, + filename=US_SOURCE_COVERAGE_DIAGNOSTICS_FILE, + artifact_key="us_source_coverage", + payload=payload, + ) + + validate_evidence_release_dir(evidence_release_dir) + + +def test_evidence_release_still_enforces_required_files( + evidence_release_dir: Path, +) -> None: + (evidence_release_dir / "calibration_diagnostics.json").unlink() + with pytest.raises(ReleaseContractError) as excinfo: + validate_evidence_release_dir(evidence_release_dir) + failures = "\n".join(excinfo.value.failures) + assert "required file 'calibration_diagnostics.json' is missing." in failures + + +def test_evidence_release_still_enforces_artifact_hashes( + evidence_release_dir: Path, +) -> None: + (evidence_release_dir / "calibration_diagnostics.json").write_text( + json.dumps(_calibration_diagnostics() | {"options": {"epochs": 121}}) + ) + with pytest.raises(ReleaseContractError) as excinfo: + validate_evidence_release_dir(evidence_release_dir) + failures = "\n".join(excinfo.value.failures) + assert "declares sha256" in failures + + +def test_evidence_release_still_enforces_build_id_match( + evidence_release_dir: Path, +) -> None: + build_manifest = _build_manifest(EVIDENCE_RELEASE_ID) + build_manifest["build_id"] = "populace-us-2024-evidence-other-20260611" + (evidence_release_dir / "build_manifest.json").write_text( + json.dumps(build_manifest) + ) + with pytest.raises(ReleaseContractError) as excinfo: + validate_evidence_release_dir(evidence_release_dir) + assert "the directory name IS the" in "\n".join(excinfo.value.failures) + + +def test_evidence_release_still_enforces_dirty_git_refusal( + evidence_release_dir: Path, +) -> None: + """Evidence tier relaxes gate verdicts, never provenance.""" + build_manifest = _build_manifest(EVIDENCE_RELEASE_ID) + build_manifest["code"]["git_dirty"] = True + (evidence_release_dir / "build_manifest.json").write_text( + json.dumps(build_manifest) + ) + with pytest.raises(ReleaseContractError) as excinfo: + validate_evidence_release_dir(evidence_release_dir) + assert "'code.git_dirty' must be false" in "\n".join(excinfo.value.failures) diff --git a/packages/microcosm-data/tests/test_publish_guard.py b/packages/microcosm-data/tests/test_publish_guard.py index 761066502..a950f50e2 100644 --- a/packages/microcosm-data/tests/test_publish_guard.py +++ b/packages/microcosm-data/tests/test_publish_guard.py @@ -117,3 +117,31 @@ def unexpected_publish(*args, **kwargs): assert message in capsys.readouterr().err assert called is False + + +def _capture_publish(monkeypatch) -> list: + import microcosm.data.publish_cli as cli + + calls: list = [] + + def _record(release_dir, repo_id, **kwargs): + calls.append((release_dir, repo_id, kwargs)) + return {"release_id": "r", "updated_at": None} + + monkeypatch.setattr(cli, "publish_release", _record) + return calls + + +def test_publish_cli_evidence_flag_wires_the_evidence_tier(tmp_path, monkeypatch): + calls = _capture_publish(monkeypatch) + rc = main([str(tmp_path), "--evidence"]) + assert rc == 0 + assert len(calls) == 1 + assert calls[0][2]["evidence"] is True + + +def test_publish_cli_defaults_to_the_certified_tier(tmp_path, monkeypatch): + calls = _capture_publish(monkeypatch) + rc = main([str(tmp_path)]) + assert rc == 0 + assert calls[0][2]["evidence"] is False diff --git a/packages/microcosm-data/tests/test_release.py b/packages/microcosm-data/tests/test_release.py index 74c773fb2..5a44b37e9 100644 --- a/packages/microcosm-data/tests/test_release.py +++ b/packages/microcosm-data/tests/test_release.py @@ -16,12 +16,16 @@ from microcosm.data import ReleaseContractError from microcosm.data.contract import ( + EVIDENCE_RELEASE_MANIFEST_SCHEMA_VERSION, US_SOURCE_COVERAGE_DIAGNOSTICS_FILE, required_release_files, ) from microcosm.data.release import ( + LATEST_EVIDENCE_POINTER_PATH, LATEST_POINTER_PATH, LATEST_POINTER_SCHEMA_VERSION, + latest_evidence_pointer_payload, + latest_evidence_release, latest_pointer_payload, latest_release, publish_release, @@ -1214,3 +1218,216 @@ def test_pointer_with_swapped_contract_path_is_refused(hub: FakeHub) -> None: with pytest.raises(ValueError, match="malformed=\\['build_manifest'\\]"): latest_release("policyengine/populace-us", api=hub) + + +# --------------------------------------------------------------------------- +# Evidence-tier publishing (microcosm#506) +# --------------------------------------------------------------------------- +# +# Evidence releases publish exactly like certified ones — contract-gated, +# immutable tag first — except the pointer: they move latest-evidence.json, +# and are structurally incapable of touching latest.json (the certified +# pointer pe.py certification reads). + +EVIDENCE_RELEASE_ID = "populace-us-2024-evidence-9f1260b-20260611" + + +@pytest.fixture +def evidence_release_dir(release_dir: Path) -> Path: + """The certified fixture re-tiered: evidence id, evidence schema marker, + and a non-empty known_failures block.""" + directory = release_dir.parent / EVIDENCE_RELEASE_ID + directory.mkdir() + for name in ("calibration_diagnostics.json", US_SOURCE_COVERAGE_DIAGNOSTICS_FILE): + (directory / name).write_text((release_dir / name).read_text()) + build_manifest = json.loads((release_dir / "build_manifest.json").read_text()) + build_manifest["build_id"] = EVIDENCE_RELEASE_ID + (directory / "build_manifest.json").write_text(json.dumps(build_manifest)) + manifest = json.loads((release_dir / "release_manifest.json").read_text()) + manifest["schema_version"] = EVIDENCE_RELEASE_MANIFEST_SCHEMA_VERSION + manifest["tier"] = "evidence" + manifest["known_failures"] = [ + { + "failure": ( + "SOI Table 1.4 national dollar fit failed: target " + "'irs_soi.ty2023.table_1_4.all.capital_gain_distributions_" + "amount@2024' has relative_error=-0.302, exceeding 0.25." + ), + "owner": "PolicyEngine/microcosm#487", + } + ] + manifest["build"]["build_id"] = EVIDENCE_RELEASE_ID + for artifact in manifest["artifacts"].values(): + artifact["revision"] = EVIDENCE_RELEASE_ID + (directory / "release_manifest.json").write_text(json.dumps(manifest)) + return directory + + +def test_evidence_pointer_payload_mirrors_the_certified_payload() -> None: + updated_at = "2026-07-22T13:53:15+00:00" + certified_shape = latest_pointer_payload(EVIDENCE_RELEASE_ID, updated_at=updated_at) + payload = latest_evidence_pointer_payload( + EVIDENCE_RELEASE_ID, updated_at=updated_at + ) + assert payload == {**certified_shape, "tier": "evidence"} + + +def test_publish_evidence_release_never_touches_the_certified_pointer( + hub: FakeHub, evidence_release_dir: Path, artifact_root: Path +) -> None: + payload = publish_release( + evidence_release_dir, + "policyengine/populace-us", + api=hub, + artifact_root=artifact_root, + updated_at="2026-07-22T13:53:15+00:00", + evidence=True, + ) + assert payload["tier"] == "evidence" + assert payload["release_id"] == EVIDENCE_RELEASE_ID + # Immutable tag flow unchanged: the tag is the evidence release id. + assert [tag["tag"] for tag in hub.tags] == [EVIDENCE_RELEASE_ID] + # The evidence pointer lands last, in the final main commit. + final_event, final_commit = hub.events[-1] + assert final_event == "create_commit" + assert final_commit["paths"][-1] == LATEST_EVIDENCE_POINTER_PATH + # The certified pointer is never written, anywhere in the flow. + assert all(path != LATEST_POINTER_PATH for path, _ in hub.uploads) + published = json.loads(dict(hub.uploads)[LATEST_EVIDENCE_POINTER_PATH]) + assert published["tier"] == "evidence" + assert published["release_id"] == EVIDENCE_RELEASE_ID + + +def test_publish_evidence_release_refuses_a_certified_release_dir( + hub: FakeHub, release_dir: Path, artifact_root: Path +) -> None: + """--evidence on a certified-shape release: refused, nothing uploaded. + The tier must be declared by the artifact, not chosen at publish time.""" + with pytest.raises(ReleaseContractError): + publish_release( + release_dir, + "policyengine/populace-us", + api=hub, + artifact_root=artifact_root, + evidence=True, + ) + assert hub.uploads == [] + + +def test_certified_publish_refuses_an_evidence_release_dir( + hub: FakeHub, evidence_release_dir: Path, artifact_root: Path +) -> None: + with pytest.raises(ReleaseContractError): + publish_release( + evidence_release_dir, + "policyengine/populace-us", + api=hub, + artifact_root=artifact_root, + ) + assert hub.uploads == [] + + +def test_publish_evidence_no_latest_skips_the_evidence_pointer( + hub: FakeHub, evidence_release_dir: Path, artifact_root: Path +) -> None: + publish_release( + evidence_release_dir, + "policyengine/populace-us", + api=hub, + artifact_root=artifact_root, + update_latest=False, + evidence=True, + ) + final_event, final_commit = hub.events[-1] + assert final_event == "create_commit" + assert LATEST_EVIDENCE_POINTER_PATH not in final_commit["paths"] + assert all(path != LATEST_POINTER_PATH for path, _ in hub.uploads) + assert ( + final_commit["message"] + == f"Publish non-default evidence release {EVIDENCE_RELEASE_ID}" + ) + + +def test_publish_evidence_release_announces_the_tier( + hub: FakeHub, evidence_release_dir: Path, artifact_root: Path, monkeypatch +) -> None: + calls: list = [] + monkeypatch.setattr( + "microcosm.data.release.notify_release", + lambda repo_id, release_id, updated_at, **kw: calls.append( + (repo_id, release_id, updated_at, kw) + ), + ) + publish_release( + evidence_release_dir, + "policyengine/populace-us", + api=hub, + artifact_root=artifact_root, + updated_at="2026-07-22T13:53:15+00:00", + evidence=True, + ) + assert calls == [ + ( + "policyengine/populace-us", + EVIDENCE_RELEASE_ID, + "2026-07-22T13:53:15+00:00", + {"warn_if_unset": True, "tier": "evidence"}, + ) + ] + + +def test_publish_then_latest_evidence_release_round_trips( + hub: FakeHub, evidence_release_dir: Path, artifact_root: Path +) -> None: + publish_release( + evidence_release_dir, + "policyengine/populace-us", + api=hub, + artifact_root=artifact_root, + updated_at="2026-07-22T13:53:15+00:00", + evidence=True, + ) + pointer = latest_evidence_release("policyengine/populace-us", api=hub) + assert pointer.release_id == EVIDENCE_RELEASE_ID + assert pointer.tier == "evidence" + assert pointer.updated_at == "2026-07-22T13:53:15+00:00" + assert ( + pointer.paths["build_manifest"] + == f"releases/{EVIDENCE_RELEASE_ID}/build_manifest.json" + ) + + +def test_latest_release_refuses_an_evidence_tier_pointer(hub: FakeHub) -> None: + """Tier defense on the certified consumer: if an evidence payload ever + lands in latest.json, readers refuse it rather than certify it.""" + payload = latest_evidence_pointer_payload(EVIDENCE_RELEASE_ID) + hub.seed_main_file(LATEST_POINTER_PATH, json.dumps(payload).encode()) + + with pytest.raises(ValueError, match="tier"): + latest_release("policyengine/populace-us", api=hub) + + +def test_latest_evidence_release_requires_the_evidence_tier(hub: FakeHub) -> None: + payload = latest_pointer_payload(EVIDENCE_RELEASE_ID) + hub.seed_main_file(LATEST_EVIDENCE_POINTER_PATH, json.dumps(payload).encode()) + + with pytest.raises(ValueError, match="tier"): + latest_evidence_release("policyengine/populace-us", api=hub) + + +def test_latest_evidence_release_requires_the_id_segment(hub: FakeHub) -> None: + payload = latest_evidence_pointer_payload(RELEASE_ID) + hub.seed_main_file(LATEST_EVIDENCE_POINTER_PATH, json.dumps(payload).encode()) + + with pytest.raises(ValueError, match="-evidence-"): + latest_evidence_release("policyengine/populace-us", api=hub) + + +def test_certified_latest_pointer_keeps_its_certified_shape(hub: FakeHub) -> None: + """The certified pointer payload gains no tier field — its bytes are the + pre-#506 shape, so existing consumers see no drift.""" + payload = latest_pointer_payload(RELEASE_ID, updated_at="2026-06-11T00:00:00+00:00") + assert "tier" not in payload + hub.seed_main_file(LATEST_POINTER_PATH, json.dumps(payload).encode()) + pointer = latest_release("policyengine/populace-us", api=hub) + assert pointer.tier == "certified" From 9d7beb342d658fce6107573533b1293711af626c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 13 Aug 2026 21:16:07 -0400 Subject: [PATCH 2/6] Builder --evidence-release mode: export on terminal-gate failure with owned known_failures (#506) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On terminal-gate failure the certified path is unchanged: batched pre-export raise, #568 weight-evidence sidecar, no H5, no manifests. Under --evidence-release the same recorded failures instead ride into the release manifest's known_failures block and the run continues through the H5 write, reform smoke, take-up contract, and manifests. Every conversion point resolves owners immediately — a failure matching neither the standing US_EVIDENCE_FAILURE_OWNERS register (the two #506-adjudicated dense families: SOI Table 1.4 dollar fit -> #487, QRF tail concentration -> #481/#487) nor the --evidence-failure-owners per-run file refuses the export. An all-green run under the flag is refused: the flag is structurally incapable of minting a certified-shape manifest (the release manifest gets the 1-evidence schema marker + tier + known_failures via _evidence_release_manifest_fields, which raises on an empty record). Release ids: --evidence-release auto-ids carry the -evidence- segment; certified builds refuse ids that squat it. Incompatible with --exact-k (the ladder lane keeps its own tag-only contract). Preflight and mid-build source-stage gates, --audit-export-targets, and the dirty-worktree refusal abort in both modes: the tier relaxes terminal gate verdicts, never artifact auditability. Co-Authored-By: Claude Fable 5 --- .../tests/test_us_fiscal_refresh_builder.py | 234 +++++++++++++ tools/build_us_fiscal_refresh_release.py | 325 ++++++++++++++++-- 2 files changed, 527 insertions(+), 32 deletions(-) diff --git a/packages/microcosm-build/tests/test_us_fiscal_refresh_builder.py b/packages/microcosm-build/tests/test_us_fiscal_refresh_builder.py index 0b6bc629d..c05691122 100644 --- a/packages/microcosm-build/tests/test_us_fiscal_refresh_builder.py +++ b/packages/microcosm-build/tests/test_us_fiscal_refresh_builder.py @@ -10803,3 +10803,237 @@ def test_calibration_diagnostics_schema_lockstep() -> None: ) assert WRITER_SCHEMA_VERSION == CONTRACT_SCHEMA_VERSION + + +# --------------------------------------------------------------------------- +# Evidence-tier builder mode (microcosm#506) +# --------------------------------------------------------------------------- + + +def test_evidence_release_id_reserves_the_segment_for_the_tier() -> None: + builder = _load_builder_module() + + builder._assert_us_release_id( + "populace-us-2024-evidence-abc1234-20260812T000000Z", + evidence_release=True, + ) + builder._assert_us_release_id("populace-us-2024-abc1234-20260812T000000Z") + + with pytest.raises(ValueError, match="must carry"): + builder._assert_us_release_id( + "populace-us-2024-abc1234-20260812T000000Z", + evidence_release=True, + ) + with pytest.raises(ValueError, match="reserved for --evidence-release"): + builder._assert_us_release_id( + "populace-us-2024-evidence-abc1234-20260812T000000Z" + ) + + +def test_default_release_id_carries_the_evidence_segment() -> None: + builder = _load_builder_module() + from datetime import UTC, datetime + + stamp = datetime(2026, 8, 12, 1, 2, 3, tzinfo=UTC) + certified = builder._default_release_id( + SimpleNamespace(exact_k=None, evidence_release=False), + digest="abcdef0", + commit="123456789abc", + build_timestamp=stamp, + ) + evidence = builder._default_release_id( + SimpleNamespace(exact_k=None, evidence_release=True), + digest="abcdef0", + commit="123456789abc", + build_timestamp=stamp, + ) + assert certified == "populace-us-2024-abcdef0-123456789abc-20260812T010203Z" + assert evidence == "populace-us-2024-evidence-abcdef0-123456789abc-20260812T010203Z" + # The two ids differ ONLY by the tier segment, and each passes its own + # tier's assertion while failing the other's. + assert evidence.replace("-evidence-", "-") == certified + builder._assert_us_release_id(certified) + builder._assert_us_release_id(evidence, evidence_release=True) + + +def test_evidence_known_failures_map_the_adjudicated_owner_register() -> None: + builder = _load_builder_module() + failures = [ + ( + "SOI Table 1.4 national dollar fit failed: target " + "'irs_soi.ty2023.table_1_4.all.capital_gain_distributions_amount" + "@2024' has relative_error=-0.302, exceeding 0.25." + ), + ( + "QRF tail concentration failed: 7 sparse QRF-imputed columns " + "past the top-k share bound." + ), + ] + + entries = builder._evidence_known_failures( + failures, builder.US_EVIDENCE_FAILURE_OWNERS + ) + + assert [entry["failure"] for entry in entries] == failures + assert entries[0]["owner"] == "PolicyEngine/microcosm#487" + assert "PolicyEngine/microcosm#481" in entries[1]["owner"] + + +def test_evidence_known_failures_refuse_unowned_failures() -> None: + builder = _load_builder_module() + with pytest.raises(RuntimeError, match="match no\\s+owner") as excinfo: + builder._evidence_known_failures( + ["Some novel gate failed: it broke."], + builder.US_EVIDENCE_FAILURE_OWNERS, + ) + # The refusal names the unowned failure verbatim so the operator can + # adjudicate exactly what the run recorded. + assert "Some novel gate failed: it broke." in str(excinfo.value) + + +def test_evidence_failure_owner_file_takes_precedence(tmp_path) -> None: + builder = _load_builder_module() + owners_path = tmp_path / "owners.json" + owners_path.write_text( + json.dumps( + { + "QRF tail concentration failed:": "PolicyEngine/microcosm#900", + "Input coverage failed:": "PolicyEngine/microcosm#368", + } + ) + ) + + patterns = builder._load_evidence_failure_owner_patterns(owners_path) + entries = builder._evidence_known_failures( + [ + "QRF tail concentration failed: whatever.", + "Input coverage failed: tip_income missing.", + ], + patterns, + ) + + assert entries[0]["owner"] == "PolicyEngine/microcosm#900" + assert entries[1]["owner"] == "PolicyEngine/microcosm#368" + + +def test_evidence_failure_owner_file_requires_issue_refs(tmp_path) -> None: + builder = _load_builder_module() + owners_path = tmp_path / "owners.json" + owners_path.write_text(json.dumps({"Input coverage failed:": "Max"})) + with pytest.raises(ValueError, match="issue reference"): + builder._load_evidence_failure_owner_patterns(owners_path) + + owners_path.write_text(json.dumps({" ": "PolicyEngine/microcosm#1"})) + with pytest.raises(ValueError, match="empty pattern"): + builder._load_evidence_failure_owner_patterns(owners_path) + + +def test_evidence_manifest_fields_are_structurally_uncertifiable() -> None: + """The flag can never mint a certified-shape manifest: the fields carry + the evidence schema marker, which the certified contract rejects, and an + empty failure record is refused outright.""" + builder = _load_builder_module() + from microcosm.data.contract import ( + EVIDENCE_RELEASE_MANIFEST_SCHEMA_VERSION, + RELEASE_MANIFEST_SCHEMA_VERSION, + ) + + entries = [ + {"failure": "SOI Table 1.4 national dollar fit failed: x.", "owner": "#487"} + ] + fields = builder._evidence_release_manifest_fields(entries) + + assert fields["schema_version"] == EVIDENCE_RELEASE_MANIFEST_SCHEMA_VERSION + assert fields["schema_version"] != RELEASE_MANIFEST_SCHEMA_VERSION + assert fields["tier"] == "evidence" + assert fields["known_failures"] == entries + + with pytest.raises(ValueError, match="all-green artifact"): + builder._evidence_release_manifest_fields([]) + + +def test_evidence_release_flags_parse(monkeypatch, tmp_path) -> None: + builder = _load_builder_module() + owners_path = tmp_path / "owners.json" + owners_path.write_text("{}") + monkeypatch.setattr( + sys, + "argv", + [ + "build_us_fiscal_refresh_release.py", + "--ledger-facts", + "facts.jsonl", + "--out", + "release", + ], + ) + args = builder._parse_args() + assert args.evidence_release is False + assert args.evidence_failure_owners is None + + monkeypatch.setattr( + sys, + "argv", + [ + "build_us_fiscal_refresh_release.py", + "--ledger-facts", + "facts.jsonl", + "--out", + "release", + "--evidence-release", + "--evidence-failure-owners", + str(owners_path), + ], + ) + args = builder._parse_args() + assert args.evidence_release is True + assert args.evidence_failure_owners == owners_path + + +def test_evidence_owner_file_without_evidence_release_is_refused( + monkeypatch, capsys, tmp_path +) -> None: + builder = _load_builder_module() + owners_path = tmp_path / "owners.json" + owners_path.write_text("{}") + monkeypatch.setattr( + sys, + "argv", + [ + "build_us_fiscal_refresh_release.py", + "--ledger-facts", + "facts.jsonl", + "--out", + str(tmp_path), + "--evidence-failure-owners", + str(owners_path), + ], + ) + + with pytest.raises(SystemExit, match="2"): + builder._parse_args() + assert "requires --evidence-release" in capsys.readouterr().err + + +def test_evidence_release_is_incompatible_with_exact_k( + monkeypatch, capsys, tmp_path +) -> None: + builder = _load_builder_module() + monkeypatch.setattr( + sys, + "argv", + [ + "build_us_fiscal_refresh_release.py", + "--ledger-facts", + "facts.jsonl", + "--out", + str(tmp_path), + "--evidence-release", + "--exact-k", + "20000", + ], + ) + + with pytest.raises(SystemExit, match="2"): + builder._parse_args() + assert "incompatible with --exact-k" in capsys.readouterr().err diff --git a/tools/build_us_fiscal_refresh_release.py b/tools/build_us_fiscal_refresh_release.py index 3681975ad..149208cb0 100644 --- a/tools/build_us_fiscal_refresh_release.py +++ b/tools/build_us_fiscal_refresh_release.py @@ -36,6 +36,7 @@ import json import math import platform +import re import shutil import subprocess import sys @@ -274,6 +275,10 @@ diagnostics_payload, write_calibration_diagnostics, ) +from microcosm.data.contract import ( + EVIDENCE_RELEASE_ID_SEGMENT, + EVIDENCE_RELEASE_MANIFEST_SCHEMA_VERSION, +) from microcosm.data.us_critical_targets import ( US_CRITICAL_TARGET_IMPROVEMENT_MAX_ABS_RELATIVE_ERROR, US_EXACT_CRITICAL_TARGET_FIT_REQUIREMENTS, @@ -1060,6 +1065,38 @@ def _parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: "probe scores ~$0. Release builds must leave this unset." ), ) + parser.add_argument( + "--evidence-release", + action="store_true", + help=( + "Build an EVIDENCE-tier release (microcosm#506): on terminal-gate " + "failure, continue to write the H5 and full manifests with the " + "recorded failures carried verbatim in the release manifest's " + "known_failures block (each with an owner issue), instead of " + "refusing to export. NOT a gate bypass: the release id carries an " + "'-evidence-' segment, the release manifest declares the evidence " + "schema marker the certified contract structurally rejects, and a " + "run whose terminal gates all pass is refused under this flag " + "(rerun without it). Preflight and mid-build source-stage gates, " + "artifact-integrity assertions (e.g. --audit-export-targets), and " + "the dirty-worktree refusal still abort: the evidence tier " + "relaxes terminal gate verdicts, never artifact auditability. " + "Incompatible with --exact-k (the ladder candidate lane has its " + "own tag-only publication contract)." + ), + ) + parser.add_argument( + "--evidence-failure-owners", + type=Path, + help=( + "JSON file mapping failure-substring patterns to owner issue refs " + '(e.g. {"Input coverage failed:": "PolicyEngine/microcosm#368"}) ' + "for --evidence-release. Checked ahead of the standing " + "US_EVIDENCE_FAILURE_OWNERS register; a recorded failure matching " + "neither refuses the evidence export — every shipped failure " + "needs an owner." + ), + ) parser.add_argument( "--epochs", type=int, @@ -1506,6 +1543,14 @@ def _parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: "--refit-l2-lambda requires the sparse L0+refit default dataset; " "--dense-default-dataset has no refit stage (use --l2-lambda)." ) + if args.evidence_failure_owners is not None and not args.evidence_release: + parser.error("--evidence-failure-owners requires --evidence-release.") + if args.evidence_release and args.exact_k is not None: + parser.error( + "--evidence-release is incompatible with --exact-k: ladder " + "candidates publish tag-only under their own contract; the " + "evidence tier (microcosm#506) covers the national artifact." + ) ladder_values = ( args.exact_k, args.exact_k_pi_hi, @@ -7278,6 +7323,7 @@ def _build_manifests( calibration_key: str = "populace_us_2024_calibration", calibration_filename: str = CALIBRATION_FILENAME, exact_k_ladder: Mapping[str, object] | None = None, + evidence_known_failures: Sequence[Mapping[str, str]] | None = None, ) -> None: dataset_path = artifact_root / dataset_filename calibration_path = artifact_root / calibration_filename @@ -7723,6 +7769,14 @@ def _build_manifests( ), }, } + if evidence_known_failures is not None: + # Evidence tier (microcosm#506): replace the certified schema marker + # with the evidence one and carry the owned failure record. The + # certified branch (evidence_known_failures None) writes the exact + # dict above, byte-identical to a build without the flag. + release_manifest.update( + _evidence_release_manifest_fields(evidence_known_failures) + ) (release_dir / "release_manifest.json").write_text( json.dumps(release_manifest, indent=1, allow_nan=False) ) @@ -7851,12 +7905,158 @@ def _fiscal_target_source_provenance( } -def _assert_us_release_id(release_id: str) -> None: +#: Standing failure-pattern -> owner register for --evidence-release +#: (microcosm#506). Only owner-adjudicated families belong here: the two dense +#: blockers named in the #506 brief — the SOI Table 1.4 dollar blanket (the +#: −30.2% capital-gain-distributions row, PUF donor uprating, #487) and the +#: dense QRF tail-concentration set (bootstrap seed-lottery tail draws, #481, +#: with the #487 uprating interaction). Any other recorded failure needs a +#: per-run adjudication via --evidence-failure-owners; an unowned failure +#: refuses the evidence export. +US_EVIDENCE_FAILURE_OWNERS: tuple[tuple[str, str], ...] = ( + ( + "SOI Table 1.4 national dollar fit failed:", + "PolicyEngine/microcosm#487", + ), + ( + "QRF tail concentration failed:", + "PolicyEngine/microcosm#481, PolicyEngine/microcosm#487", + ), +) + +# Kept in lockstep with microcosm.data.contract._ISSUE_REF_RE; the evidence +# publish contract re-validates every owner ref, so drift here fails at +# publish rather than silently. +_EVIDENCE_ISSUE_REF_RE = re.compile(r"#\d+|github\.com/\S+/(?:issues|pull)/\d+") + + +def _load_evidence_failure_owner_patterns( + path: Path | None, +) -> tuple[tuple[str, str], ...]: + """Failure-pattern -> owner pairs: per-run adjudications first, then the + standing register, so a run-specific entry can sharpen a standing owner.""" + per_run: list[tuple[str, str]] = [] + if path is not None: + payload = json.loads(path.read_text()) + if not isinstance(payload, dict): + raise ValueError( + f"--evidence-failure-owners {path} must be a JSON object of " + "failure-substring pattern -> owner issue ref." + ) + for pattern, owner in payload.items(): + if not isinstance(pattern, str) or not pattern.strip(): + raise ValueError( + f"--evidence-failure-owners {path} has an empty pattern key." + ) + if not isinstance(owner, str) or not _EVIDENCE_ISSUE_REF_RE.search(owner): + raise ValueError( + f"--evidence-failure-owners {path} owner for pattern " + f"{pattern!r} must carry an issue reference (e.g. " + f"'PolicyEngine/microcosm#487'), got {owner!r}." + ) + per_run.append((pattern, owner)) + return (*per_run, *US_EVIDENCE_FAILURE_OWNERS) + + +def _evidence_known_failures( + failures: Sequence[str], + owner_patterns: Sequence[tuple[str, str]], +) -> list[dict[str, str]]: + """Recorded gate failures -> owned known_failures entries, verbatim. + + Every failure string must match an owner pattern (substring, first match + wins); an unowned failure refuses the evidence export — the tier ships + failures only when somewhere is accountable for fixing them. + """ + entries: list[dict[str, str]] = [] + unowned: list[str] = [] + for failure in failures: + owner = next( + (owner for pattern, owner in owner_patterns if pattern in failure), + None, + ) + if owner is None: + unowned.append(failure) + else: + entries.append({"failure": failure, "owner": owner}) + if unowned: + raise RuntimeError( + "Evidence release refused: recorded gate failure(s) match no " + "owner in the standing US_EVIDENCE_FAILURE_OWNERS register or " + "the --evidence-failure-owners file. Adjudicate an owner issue " + "for each before shipping (microcosm#506): " + "; ".join(unowned) + ) + return entries + + +def _evidence_release_manifest_fields( + evidence_known_failures: Sequence[Mapping[str, str]], +) -> dict[str, object]: + """The release-manifest fields that mark the evidence tier. + + The schema marker is the structural guarantee (microcosm#506): the + certified contract rejects any manifest carrying it, so no flag-plumbing + bug can mint a certified-shape manifest from an evidence build. An empty + failure record is refused here too — the tier exists to carry failures, + and an all-green run belongs on the certified path. + """ + if not evidence_known_failures: + raise ValueError( + "an evidence release manifest requires at least one known " + "failure; an all-green artifact belongs on the certified path." + ) + return { + "schema_version": EVIDENCE_RELEASE_MANIFEST_SCHEMA_VERSION, + "tier": "evidence", + "known_failures": [dict(entry) for entry in evidence_known_failures], + } + + +def _default_release_id( + args: argparse.Namespace, + *, + digest: str, + commit: str, + build_timestamp: datetime, +) -> str: + """The auto-generated release id for a run without --release-id. + + Exact-k candidates carry the k segment; evidence-tier builds carry the + ``evidence`` segment (microcosm#506) so the tier is visible in every tag + and download path; certified national builds keep the historical shape. + """ + if args.exact_k is not None: + return ( + f"populace-us-2024-k{args.exact_k}-{digest}-{commit}-" + f"{build_timestamp:%Y%m%dT%H%M%SZ}" + ) + if args.evidence_release: + return ( + f"populace-us-2024-evidence-{digest}-{commit}-" + f"{build_timestamp:%Y%m%dT%H%M%SZ}" + ) + return f"populace-us-2024-{digest}-{commit}-{build_timestamp:%Y%m%dT%H%M%SZ}" + + +def _assert_us_release_id(release_id: str, *, evidence_release: bool = False) -> None: if not release_id.startswith("populace-us-"): raise ValueError( "US fiscal refresh release ids must start with 'populace-us-' so " "the US release contract requires source coverage diagnostics." ) + if evidence_release and EVIDENCE_RELEASE_ID_SEGMENT not in release_id: + raise ValueError( + "--evidence-release ids must carry the " + f"{EVIDENCE_RELEASE_ID_SEGMENT!r} segment (e.g. " + "populace-us-2024-evidence--) so the tier is visible " + "in every tag and download path." + ) + if not evidence_release and EVIDENCE_RELEASE_ID_SEGMENT in release_id: + raise ValueError( + f"release ids containing {EVIDENCE_RELEASE_ID_SEGMENT!r} are " + "reserved for --evidence-release builds; a certified build must " + "not squat the evidence namespace." + ) def _assert_exact_k_release_id(release_id: str, k: int) -> None: @@ -8107,6 +8307,15 @@ def main(argv: Sequence[str] | None = None) -> None: args = _parse_args(argv) if _git_dirty(): raise SystemExit("Refusing to build a release from a dirty git worktree.") + # Loaded (and validated) up front so a malformed owners file dies in + # seconds, not after the multi-hour build. The flag-combination rules + # (--evidence-failure-owners requires the tier flag; --exact-k is + # incompatible with it) live in _parse_args with the other combos. + evidence_failure_owner_patterns = ( + _load_evidence_failure_owner_patterns(args.evidence_failure_owners) + if args.evidence_release + else () + ) build_started = time.perf_counter() timing: dict[str, float] = {} @@ -8170,13 +8379,13 @@ def main(argv: Sequence[str] | None = None) -> None: build_timestamp = datetime.now(UTC) full_commit = _git_output("rev-parse", "HEAD") commit = _git_output("rev-parse", "--short=12", "HEAD") - release_id = args.release_id or ( - f"populace-us-2024-k{args.exact_k}-{digest}-{commit}-" - f"{build_timestamp:%Y%m%dT%H%M%SZ}" - if args.exact_k is not None - else (f"populace-us-2024-{digest}-{commit}-{build_timestamp:%Y%m%dT%H%M%SZ}") + release_id = args.release_id or _default_release_id( + args, + digest=digest, + commit=commit, + build_timestamp=build_timestamp, ) - _assert_us_release_id(release_id) + _assert_us_release_id(release_id, evidence_release=args.evidence_release) if args.exact_k is not None: _assert_exact_k_release_id(release_id, args.exact_k) # The immutable release id is the dataset's exact-count name. Keep the @@ -10953,33 +11162,55 @@ def main(argv: Sequence[str] | None = None) -> None: # internally, and both require the written H5 / export artifacts that a # gate-failed run must not produce. if terminal_gate_failures: - # Gate-failure path ONLY (microcosm#568 review): a batched pre-export - # failure mints no H5, so the exact calibrated weight vector — with - # the ordered household ids it aligns to, bound to the target-frame - # identity — is persisted here as the run's only record-level weight - # evidence. Green runs never write these files (the certified H5 - # carries the weights); late gates (reform smoke, take-up contract) - # raise after the H5 write, so their failed runs retain weights in - # the written dataset itself. - _write_final_household_weight_evidence( - release_dir, - export_frame, - identity=target_frame_checkpoint_identity, + if not args.evidence_release: + # Gate-failure path ONLY (microcosm#568 review): a batched + # pre-export failure mints no H5, so the exact calibrated weight + # vector — with the ordered household ids it aligns to, bound to + # the target-frame identity — is persisted here as the run's only + # record-level weight evidence. Green runs never write these files + # (the certified H5 carries the weights); late gates (reform + # smoke, take-up contract) raise after the H5 write, so their + # failed runs retain weights in the written dataset itself. + _write_final_household_weight_evidence( + release_dir, + export_frame, + identity=target_frame_checkpoint_identity, + ) + terminal_batch_telemetry.stage( + "release_gates", + status="failed", + message="Release gates failed (batched pre-export report).", + failures=terminal_gate_failures, + force_upload=True, + ) + raise RuntimeError( + "Release gates failed: " + "; ".join(terminal_gate_failures) + ) + # Evidence tier (microcosm#506): the recorded terminal failures ride + # into the release manifest's known_failures block instead of + # aborting the export. Owners are resolved NOW so an unowned failure + # refuses the export before the H5 and manifest work below; the H5 + # itself carries the calibrated weights, so the #568 weight-evidence + # sidecar is not written on this path. + _evidence_known_failures( + terminal_gate_failures, evidence_failure_owner_patterns ) terminal_batch_telemetry.stage( "release_gates", status="failed", - message="Release gates failed (batched pre-export report).", + message=( + "Release gates failed; --evidence-release continues to " + "export with the failures recorded (microcosm#506)." + ), failures=terminal_gate_failures, force_upload=True, ) - raise RuntimeError("Release gates failed: " + "; ".join(terminal_gate_failures)) # A green run must not inherit a prior failed attempt's weight evidence # (microcosm#568 round 2): with --out/--release-id reuse, stale evidence # files would coexist with a certified release whose manifest knows - # nothing about them. The batched gates have passed, so any evidence - # present here belongs to a superseded attempt — remove it before the - # certified artifacts are written. + # nothing about them. The batched gates have passed (or --evidence-release + # is recording their failures), so any evidence present here belongs to a + # superseded attempt — remove it before the release artifacts are written. for stale_evidence in ( release_dir / FINAL_HOUSEHOLD_WEIGHTS_FILENAME, release_dir / FINAL_HOUSEHOLD_WEIGHT_IDS_FILENAME, @@ -11045,12 +11276,18 @@ def main(argv: Sequence[str] | None = None) -> None: failures=list(reform_coverage_smoke_gate.failures), force_upload=True, ) - raise RuntimeError( - "Release gates failed: " - + "; ".join( - f"Reform coverage smoke failed: {failure}" - for failure in reform_coverage_smoke_gate.failures - ) + smoke_failures = [ + f"Reform coverage smoke failed: {failure}" + for failure in reform_coverage_smoke_gate.failures + ] + if not args.evidence_release: + raise RuntimeError("Release gates failed: " + "; ".join(smoke_failures)) + # Evidence tier: the smoke verdict joins the recorded terminal + # set (owner-checked immediately, so an unowned failure aborts + # before further export work). + terminal_gate_failures.extend(smoke_failures) + _evidence_known_failures( + terminal_gate_failures, evidence_failure_owner_patterns ) if args.audit_export_targets: if telemetry is not None: @@ -11186,11 +11423,21 @@ def main(argv: Sequence[str] | None = None) -> None: and row.get("ships_at_engine_default") ] if stale_count_calibrated: - raise RuntimeError( - "Release gates failed: count-calibrated take-up column(s) " + stale_count_calibrated_failure = ( + "count-calibrated take-up column(s) " f"{stale_count_calibrated} ship at the engine default on the " "export frame despite the stage having run." ) + if not args.evidence_release: + raise RuntimeError( + "Release gates failed: " + stale_count_calibrated_failure + ) + # Evidence tier: recorded like every other terminal verdict, with + # the same immediate owner check. + terminal_gate_failures.append(stale_count_calibrated_failure) + _evidence_known_failures( + terminal_gate_failures, evidence_failure_owner_patterns + ) if telemetry is not None: telemetry.attach_artifact( "us_take_up_participation", @@ -11210,6 +11457,19 @@ def main(argv: Sequence[str] | None = None) -> None: ) telemetry.stage("manifests", message="Writing release manifests.") timing["total_build_seconds"] = time.perf_counter() - build_started + evidence_known_failures = None + if args.evidence_release: + if not terminal_gate_failures: + raise RuntimeError( + "Evidence release refused: every terminal gate passed. This " + "artifact qualifies for the certified path — rerun without " + "--evidence-release (the flag cannot mint a certified-shape " + "manifest, and an evidence manifest with no known failures " + "is invalid by contract)." + ) + evidence_known_failures = _evidence_known_failures( + terminal_gate_failures, evidence_failure_owner_patterns + ) _build_manifests( release_id=release_id, release_dir=release_dir, @@ -11247,6 +11507,7 @@ def main(argv: Sequence[str] | None = None) -> None: calibration_key=calibration_key, calibration_filename=calibration_filename, exact_k_ladder=exact_k_ladder_provenance, + evidence_known_failures=evidence_known_failures, ) if telemetry is not None: telemetry.attach_artifact("build_manifest", release_dir / "build_manifest.json") From b92beabdafc8a0e73ad9cd65377a9791a2d3dbfc Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 13 Aug 2026 21:23:12 -0400 Subject: [PATCH 3/6] Pin the converse pointer isolation: certified publish never writes latest-evidence.json (#506) Co-Authored-By: Claude Fable 5 --- packages/microcosm-data/tests/test_release.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/packages/microcosm-data/tests/test_release.py b/packages/microcosm-data/tests/test_release.py index 5a44b37e9..66e0120ef 100644 --- a/packages/microcosm-data/tests/test_release.py +++ b/packages/microcosm-data/tests/test_release.py @@ -1431,3 +1431,19 @@ def test_certified_latest_pointer_keeps_its_certified_shape(hub: FakeHub) -> Non hub.seed_main_file(LATEST_POINTER_PATH, json.dumps(payload).encode()) pointer = latest_release("policyengine/populace-us", api=hub) assert pointer.tier == "certified" + + +def test_certified_publish_never_touches_the_evidence_pointer( + hub: FakeHub, release_dir: Path, artifact_root: Path +) -> None: + """The converse of the evidence-pointer isolation: a certified publish + moves latest.json only, never latest-evidence.json.""" + publish_release( + release_dir, + "policyengine/populace-us", + api=hub, + artifact_root=artifact_root, + updated_at="2026-06-11T13:53:15+00:00", + ) + assert any(path == LATEST_POINTER_PATH for path, _ in hub.uploads) + assert all(path != LATEST_EVIDENCE_POINTER_PATH for path, _ in hub.uploads) From 7032f3add5d868f49e96d885e422bb36081c0339 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 13 Aug 2026 23:21:18 -0400 Subject: [PATCH 4/6] Sol review round 1: bind known_failures to the record, reserve pointer paths, harden scope (#506) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Findings fixed from the cross-family adversarial review of the committed diff: - (HIGH) Pointer isolation was bypassable through manifest-declared root artifacts: an artifact entry whose path IS latest.json (or latest-evidence.json) uploaded at that root path. publish_release now refuses reserved pointer paths as root artifacts on BOTH tiers, pinned in both directions. - (HIGH) known_failures was unbound from the artifact's own record. The evidence contract now requires every build_manifest gates.calibration failure verbatim in known_failures, and recomputes the certified critical-target verdicts requiring each breach acknowledged by name — a softened or dropped entry refuses. (Over-disclosure stays legal: the tier can under-promise, never under-disclose.) - (MEDIUM) The evidence contract now refuses non-populace-us release ids: a generic id with the segment must not buy a weaker contract by deactivating the US-specific requirements. - (MEDIUM) latest_release now refuses ANY tier field in latest.json — explicit 'certified' and null included; no certified producer writes one. - (PLAUSIBLE, confirmed) An evidence attempt refused for an unowned failure now falls through to the same failed-run path as a certified gate failure, #568 weight-evidence sidecar included, before re-raising the refusal. - (COVERAGE) New AST guard pins the main() conversion shape: every terminal 'Release gates failed' raise after the #548 accumulator is conditioned on --evidence-release, every one before it is unconditional, the all-green refusal precedes the manifest write, and _build_manifests receives the owned record. Adjudicated without change: the certified-path refusal of '-evidence-' ids is the intentional reserved-namespace guard (disclosed in the PR body); LatestPointer.tier is a deliberate additive API field, now in the changelog fragment. Co-Authored-By: Claude Fable 5 --- changelog.d/506-evidence-tier.added.md | 2 +- .../tests/test_us_fiscal_refresh_builder.py | 93 +++++++++++++++ .../src/microcosm/data/contract.py | 100 +++++++++++++++- .../src/microcosm/data/release.py | 31 +++-- .../microcosm-data/tests/test_contract.py | 110 ++++++++++++++++-- packages/microcosm-data/tests/test_release.py | 56 +++++++++ tools/build_us_fiscal_refresh_release.py | 30 +++-- 7 files changed, 390 insertions(+), 32 deletions(-) diff --git a/changelog.d/506-evidence-tier.added.md b/changelog.d/506-evidence-tier.added.md index fb8810f0c..0e196956f 100644 --- a/changelog.d/506-evidence-tier.added.md +++ b/changelog.d/506-evidence-tier.added.md @@ -1 +1 @@ -Evidence-tier publishing (microcosm#506): `validate_evidence_release_dir` as a structural sibling of the certified contract (same required files plus a mandatory non-empty `known_failures` block with owner issue refs), `publish_release(evidence=True)` moving only the new `latest-evidence.json` pointer (never `latest.json`), `latest_evidence_release` for consumers, and `microcosm-publish-release --evidence`. +Evidence-tier publishing (microcosm#506): `validate_evidence_release_dir` as a structural sibling of the certified contract — same required files plus a mandatory non-empty `known_failures` block whose entries carry owner issue refs and are bound to the artifact's own record (every `gates.calibration` failure verbatim, every recomputed critical-target breach acknowledged by name); `publish_release(evidence=True)` moving only the new `latest-evidence.json` pointer (never `latest.json`, with both pointer paths reserved against manifest-declared artifacts on both tiers); `latest_evidence_release` for consumers plus a `tier` field on `LatestPointer` (defaulting to `certified`); `microcosm-publish-release --evidence`; and builder `--evidence-release` exporting on terminal-gate failure with the recorded failures owned or refused. diff --git a/packages/microcosm-build/tests/test_us_fiscal_refresh_builder.py b/packages/microcosm-build/tests/test_us_fiscal_refresh_builder.py index c05691122..ff89fc57b 100644 --- a/packages/microcosm-build/tests/test_us_fiscal_refresh_builder.py +++ b/packages/microcosm-build/tests/test_us_fiscal_refresh_builder.py @@ -11037,3 +11037,96 @@ def test_evidence_release_is_incompatible_with_exact_k( with pytest.raises(SystemExit, match="2"): builder._parse_args() assert "incompatible with --exact-k" in capsys.readouterr().err + + +def test_evidence_mode_conversion_is_pinned_structurally() -> None: + """microcosm#506 control-flow guard (the #443/#568 AST pattern): in + main(), every terminal 'Release gates failed' raise after the #548 + accumulator forms must be conditioned on --evidence-release (the + conversion sites), every one BEFORE it must be unconditional (preflight + and mid-build gates never convert), the all-green evidence refusal must + sit before the manifest write, and _build_manifests must receive the + owned failure record.""" + import ast + + builder = _load_builder_module() + source = Path(builder.__file__).read_text() + tree = ast.parse(source) + main_fn = next( + node + for node in ast.walk(tree) + if isinstance(node, ast.FunctionDef) and node.name == "main" + ) + + parents: dict[ast.AST, ast.AST] = {} + for parent in ast.walk(main_fn): + for child in ast.iter_child_nodes(parent): + parents[child] = parent + + def _ancestor_if_tests(node: ast.AST) -> list[str]: + tests = [] + cursor = node + while cursor in parents: + cursor = parents[cursor] + if isinstance(cursor, ast.If): + tests.append(ast.unparse(cursor.test)) + return tests + + accumulator = next( + node + for node in ast.walk(main_fn) + if isinstance(node, ast.AnnAssign) + and isinstance(node.target, ast.Name) + and node.target.id == "terminal_gate_failures" + ) + + gate_raises = [ + node + for node in ast.walk(main_fn) + if isinstance(node, ast.Raise) + and "Release gates failed: " in (ast.get_source_segment(source, node) or "") + ] + assert gate_raises, "main() lost its release-gate raises" + for raise_node in gate_raises: + conditioned = any( + "args.evidence_release" in test or "evidence_refusal" in test + for test in _ancestor_if_tests(raise_node) + ) + if raise_node.lineno > accumulator.lineno: + assert conditioned, ( + f"terminal raise at line {raise_node.lineno} is not " + "conditioned on --evidence-release; the conversion site " + "regressed" + ) + else: + assert not conditioned, ( + f"pre-terminal raise at line {raise_node.lineno} is " + "conditioned on --evidence-release; preflight/mid-build " + "gates must abort in both modes" + ) + + refusals = [ + node + for node in ast.walk(main_fn) + if isinstance(node, ast.Raise) + and "Evidence release refused: every terminal gate passed" + in (ast.get_source_segment(source, node) or "") + ] + assert len(refusals) == 1, "the all-green evidence refusal must exist once" + assert any( + test == "args.evidence_release" for test in _ancestor_if_tests(refusals[0]) + ) + + manifest_calls = [ + node + for node in ast.walk(main_fn) + if isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "_build_manifests" + ] + assert len(manifest_calls) == 1 + assert refusals[0].lineno < manifest_calls[0].lineno, ( + "the all-green refusal must precede the manifest write" + ) + keywords = {keyword.arg for keyword in manifest_calls[0].keywords} + assert "evidence_known_failures" in keywords diff --git a/packages/microcosm-data/src/microcosm/data/contract.py b/packages/microcosm-data/src/microcosm/data/contract.py index c5f238f52..d1ef3ded9 100644 --- a/packages/microcosm-data/src/microcosm/data/contract.py +++ b/packages/microcosm-data/src/microcosm/data/contract.py @@ -3366,6 +3366,79 @@ def validate_release_dir(release_dir: Path | str) -> None: _ISSUE_REF_RE = re.compile(r"#\d+|github\.com/\S+/(?:issues|pull)/\d+") +#: First quoted token in a contract-recomputed critical-fit failure — the +#: target name (or requirement id) the breach is about, used to bind the +#: breach to its ``known_failures`` acknowledgment. +_QUOTED_TOKEN_RE = re.compile(r"'([^']+)'") + + +def _check_evidence_known_failures_binding( + *, + release_manifest: Mapping | None, + build_manifest: Mapping | None, + recomputed_critical_failures: list[str], + failures: list[str], +) -> None: + """Bind ``known_failures`` to what the artifact itself records. + + The evidence tier's honesty cannot rest on trusting the manifest author: + everything a local validator can recompute or read back must be + acknowledged. Two bindings (a hand-edited record that softens or drops a + recorded failure fails here): + + - every failure string the build manifest records under + ``gates.calibration.failures`` must appear in ``known_failures`` + verbatim; + - every critical-target breach recomputed from + ``calibration_diagnostics.json`` (the same check the certified + contract enforces as a hard refusal) must be acknowledged by name in + some ``known_failures`` entry. + + The converse direction is deliberately open: ``known_failures`` may + carry entries beyond what is locally recomputable (post-battery gate + groups record into side artifacts, not contract files), so the tier can + over-disclose but never under-disclose. + """ + if release_manifest is None: + return + known_failures = release_manifest.get("known_failures") + if not isinstance(known_failures, list): + return # the shape failure is already recorded + recorded_texts = [ + entry.get("failure") + for entry in known_failures + if isinstance(entry, Mapping) and isinstance(entry.get("failure"), str) + ] + combined = "\n".join(recorded_texts) + if build_manifest is not None: + gates = build_manifest.get("gates") + calibration = gates.get("calibration") if isinstance(gates, Mapping) else None + recorded_gate_failures = ( + calibration.get("failures") if isinstance(calibration, Mapping) else None + ) + if isinstance(recorded_gate_failures, list): + missing = [ + failure + for failure in recorded_gate_failures + if isinstance(failure, str) and failure not in recorded_texts + ] + if missing: + failures.append( + "release_manifest.json known_failures must carry every " + "build_manifest.json gates.calibration failure verbatim; " + f"missing {_sample_values(missing)}." + ) + for recomputed in recomputed_critical_failures: + match = _QUOTED_TOKEN_RE.search(recomputed) + token = match.group(1) if match else None + if token is None or token not in combined: + failures.append( + "release_manifest.json known_failures must acknowledge the " + f"critical-target breach naming {token or recomputed!r}; the " + "evidence tier records failures, it never hides them." + ) + + def _check_evidence_release_manifest(manifest: Mapping, failures: list[str]) -> None: """Evidence-only manifest requirements: the tier marker and the honest non-empty ``known_failures`` record.""" @@ -3454,6 +3527,15 @@ def validate_evidence_release_dir(release_dir: Path | str) -> None: "not name its tier." ) + if not release_id.startswith("populace-us-"): + # Without the US prefix every country-specific requirement above the + # generic three files silently deactivates — an out-of-scope id must + # not buy a weaker contract. + failures.append( + "the evidence tier is scoped to US national releases " + f"(microcosm#506); release id {release_id!r} is out of scope." + ) + if _is_uk_exact_k_release_id(release_id): raise ReleaseContractError( release_dir, @@ -3514,14 +3596,28 @@ def validate_evidence_release_dir(release_dir: Path | str) -> None: ) _check_evidence_release_manifest(manifest, failures) + recomputed_critical_failures: list[str] = [] calibration_diagnostics_path = release_dir / "calibration_diagnostics.json" if calibration_diagnostics_path.is_file(): diagnostics = _load_json(calibration_diagnostics_path, failures) if diagnostics is not None: calibration_diagnostics = diagnostics _check_calibration_diagnostics(diagnostics, failures) - # No _check_us_critical_target_fit here: critical-fit breaches are - # the evidence tier's known_failures, not contract violations. + # Critical-fit breaches are permitted at the evidence tier — but + # never silently. Recompute the certified verdicts into a scratch + # list and require each breach to be acknowledged in + # known_failures (binding check below). + if release_id.startswith("populace-us-"): + _check_us_critical_target_fit( + diagnostics, recomputed_critical_failures + ) + + _check_evidence_known_failures_binding( + release_manifest=release_manifest, + build_manifest=build_manifest, + recomputed_critical_failures=recomputed_critical_failures, + failures=failures, + ) _check_cross_manifest_consistency( build_manifest, diff --git a/packages/microcosm-data/src/microcosm/data/release.py b/packages/microcosm-data/src/microcosm/data/release.py index 790abdfa4..d87beddd6 100644 --- a/packages/microcosm-data/src/microcosm/data/release.py +++ b/packages/microcosm-data/src/microcosm/data/release.py @@ -285,6 +285,21 @@ def publish_release( f"extra release file {filename!r} not found in {release_dir}." ) root_artifacts = _release_manifest_root_artifacts(release_dir) + # Root artifacts upload at their manifest-declared repo paths — the one + # surface where a manifest author could smuggle a pointer write past the + # tier's pointer selection (an artifact literally named latest.json or + # latest-evidence.json). Both pointer paths are reserved on BOTH tiers: + # pointers move only via the publisher's own pointer operation. + pointer_collisions = sorted( + {LATEST_POINTER_PATH, LATEST_EVIDENCE_POINTER_PATH} & set(root_artifacts) + ) + if pointer_collisions: + raise ValueError( + "release_manifest.json declares root artifact(s) at reserved " + f"pointer path(s) {pointer_collisions}; latest.json and " + "latest-evidence.json are written only by the publisher itself, " + "never as release artifacts." + ) artifact_revisions = _release_manifest_artifact_revisions(release_dir) tag = tag_name or release_id if release_id in artifact_revisions and not create_tag: @@ -708,17 +723,17 @@ def latest_release(repo_id: str, *, api=None) -> LatestPointer: Raises: ValueError: If the pointer is malformed, its schema version is newer - than this library understands, or it names a non-certified tier - (an evidence payload in ``latest.json`` is a publication bug and - must never be consumed as the certified default). + than this library understands, or it carries a ``tier`` field at + all — the certified payload predates tiers and no certified + producer writes one, so any tier field (even ``"certified"``) is + foreign and must never be consumed as the certified default. """ payload = _read_pointer(repo_id, api, pointer_path=LATEST_POINTER_PATH) - tier = payload.get("tier") - if tier not in (None, RELEASE_TIER_CERTIFIED): + if "tier" in payload: raise ValueError( - f"{LATEST_POINTER_PATH} in {repo_id} declares tier {tier!r}; the " - "certified pointer must never name another tier — evidence " - f"releases live at {LATEST_EVIDENCE_POINTER_PATH}." + f"{LATEST_POINTER_PATH} in {repo_id} carries a 'tier' field " + f"({payload.get('tier')!r}); the certified pointer never does — " + f"evidence releases live at {LATEST_EVIDENCE_POINTER_PATH}." ) return LatestPointer( release_id=str(payload["release_id"]), diff --git a/packages/microcosm-data/tests/test_contract.py b/packages/microcosm-data/tests/test_contract.py index 819ee27c1..bf57050ec 100644 --- a/packages/microcosm-data/tests/test_contract.py +++ b/packages/microcosm-data/tests/test_contract.py @@ -3723,20 +3723,20 @@ def test_evidence_release_rejects_certified_schema_version( assert f"{EVIDENCE_RELEASE_MANIFEST_SCHEMA_VERSION!r}" in failures -def test_evidence_release_tolerates_critical_target_breaches( - evidence_release_dir: Path, -) -> None: - """The tier's point: the Build N medical-class breach ships as evidence, - carried in known_failures, instead of blocking the artifact.""" +BREACHED_CRITICAL_TARGET = ( + "irs_soi.ty2022.historic_table_2.us.all.income_tax_liability_amount@2024" +) + + +def _breach_critical_target(evidence_release_dir: Path) -> None: + """Push the federal income-tax row far past its blocking tolerance (the + same breach the certified suite uses to prove a hard refusal).""" diagnostics = _calibration_diagnostics() target = next( - row - for row in diagnostics["targets"] - if row["name"] == "irs_soi.ty2022.historic_table_2.us.all." - "medical_dental_expense_amount@2024" + row for row in diagnostics["targets"] if row["name"] == BREACHED_CRITICAL_TARGET ) - target["final_estimate"] = target["target"] * 1.21 - target["relative_error"] = 0.21 + target["final_estimate"] = 735_173_331_468.564 + target["relative_error"] = -0.6508063496056629 _write_json_and_refresh_manifest_hash( evidence_release_dir, filename="calibration_diagnostics.json", @@ -3744,12 +3744,100 @@ def test_evidence_release_tolerates_critical_target_breaches( payload=diagnostics, ) + +def test_evidence_release_requires_breaches_to_be_acknowledged( + evidence_release_dir: Path, +) -> None: + """A critical breach the known_failures record does not name is refused: + the tier records failures, it never hides them.""" + _breach_critical_target(evidence_release_dir) + + with pytest.raises(ReleaseContractError) as excinfo: + validate_evidence_release_dir(evidence_release_dir) + failures = "\n".join(excinfo.value.failures) + assert "must acknowledge the critical-target breach" in failures + assert BREACHED_CRITICAL_TARGET in failures + + +def test_evidence_release_tolerates_acknowledged_critical_breaches( + evidence_release_dir: Path, +) -> None: + """The tier's point: the Build N-class breach ships as evidence once + known_failures names it, instead of blocking the artifact — while the + certified contract still refuses the same directory.""" + _breach_critical_target(evidence_release_dir) + _rewrite_evidence_manifest( + evidence_release_dir, + lambda manifest: manifest.update( + known_failures=[ + *manifest["known_failures"], + { + "failure": ( + "Fiscal critical target " + f"'{BREACHED_CRITICAL_TARGET}' breached its blocking " + "tolerance with relative_error=-0.6508." + ), + "owner": "PolicyEngine/microcosm#487", + }, + ] + ), + ) + validate_evidence_release_dir(evidence_release_dir) with pytest.raises(ReleaseContractError): validate_release_dir(evidence_release_dir) +def test_evidence_release_requires_recorded_gate_failures_verbatim( + evidence_release_dir: Path, +) -> None: + """Every failure the build manifest records must ride into known_failures + unmodified — a softened or dropped copy is refused.""" + recorded = ( + "SOI Table 1.4 national dollar fit failed: target " + "'irs_soi.ty2023.table_1_4.all.capital_gain_distributions_amount@2024' " + "has relative_error=-0.302, exceeding 0.25." + ) + build_manifest = _build_manifest(EVIDENCE_RELEASE_ID) + build_manifest["gates"]["calibration"] = {"passed": False, "failures": [recorded]} + (evidence_release_dir / "build_manifest.json").write_text( + json.dumps(build_manifest) + ) + + # The default fixture's first entry IS that verbatim string, so the + # binding holds as-is. + validate_evidence_release_dir(evidence_release_dir) + + # Softening one character of the recorded string breaks the binding. + _rewrite_evidence_manifest( + evidence_release_dir, + lambda manifest: manifest.update( + known_failures=[ + { + "failure": recorded.replace("-0.302", "-0.03"), + "owner": "PolicyEngine/microcosm#487", + } + ] + ), + ) + with pytest.raises(ReleaseContractError) as excinfo: + validate_evidence_release_dir(evidence_release_dir) + failures = "\n".join(excinfo.value.failures) + assert "gates.calibration failure verbatim" in failures + + +def test_evidence_release_scope_requires_the_us_prefix(tmp_path: Path) -> None: + """A generic id with the segment must not buy a weaker contract by + deactivating the US-specific requirements.""" + directory = tmp_path / "releases" / "acme-evidence-build" + directory.mkdir(parents=True) + + with pytest.raises(ReleaseContractError) as excinfo: + validate_evidence_release_dir(directory) + assert "out of scope" in "\n".join(excinfo.value.failures) + + def test_evidence_release_tolerates_failed_source_coverage_gate( evidence_release_dir: Path, ) -> None: diff --git a/packages/microcosm-data/tests/test_release.py b/packages/microcosm-data/tests/test_release.py index 66e0120ef..ffd8bc2f4 100644 --- a/packages/microcosm-data/tests/test_release.py +++ b/packages/microcosm-data/tests/test_release.py @@ -1447,3 +1447,59 @@ def test_certified_publish_never_touches_the_evidence_pointer( ) assert any(path == LATEST_POINTER_PATH for path, _ in hub.uploads) assert all(path != LATEST_EVIDENCE_POINTER_PATH for path, _ in hub.uploads) + + +def test_latest_release_refuses_any_tier_field(hub: FakeHub) -> None: + """No certified producer writes a tier field; even 'certified' or null is + foreign and refused rather than consumed as the default.""" + for tier_value in ("certified", None): + payload = latest_pointer_payload(RELEASE_ID) + payload["tier"] = tier_value + hub.seed_main_file(LATEST_POINTER_PATH, json.dumps(payload).encode()) + with pytest.raises(ValueError, match="tier"): + latest_release("policyengine/populace-us", api=hub) + + +def _declare_root_artifact(release_dir: Path, *, key: str, path: str) -> None: + manifest_path = release_dir / "release_manifest.json" + manifest = json.loads(manifest_path.read_text()) + manifest["artifacts"][key] = { + "kind": "diagnostics", + "path": path, + "repo_id": "policyengine/populace-us", + "revision": release_dir.name, + "sha256": "1" * 64, + } + manifest_path.write_text(json.dumps(manifest)) + + +def test_publish_refuses_root_artifacts_at_pointer_paths( + hub: FakeHub, release_dir: Path, evidence_release_dir: Path, artifact_root: Path +) -> None: + """A manifest-declared root artifact must not be able to smuggle a + pointer write past the tier's pointer selection — in either direction.""" + _declare_root_artifact( + evidence_release_dir, key="smuggled_pointer", path=LATEST_POINTER_PATH + ) + with pytest.raises(ValueError, match="reserved"): + publish_release( + evidence_release_dir, + "policyengine/populace-us", + api=hub, + artifact_root=artifact_root, + update_latest=False, + evidence=True, + ) + assert hub.uploads == [] + + _declare_root_artifact( + release_dir, key="smuggled_pointer", path=LATEST_EVIDENCE_POINTER_PATH + ) + with pytest.raises(ValueError, match="reserved"): + publish_release( + release_dir, + "policyengine/populace-us", + api=hub, + artifact_root=artifact_root, + ) + assert hub.uploads == [] diff --git a/tools/build_us_fiscal_refresh_release.py b/tools/build_us_fiscal_refresh_release.py index 149208cb0..0bd2a8ad9 100644 --- a/tools/build_us_fiscal_refresh_release.py +++ b/tools/build_us_fiscal_refresh_release.py @@ -11162,7 +11162,21 @@ def main(argv: Sequence[str] | None = None) -> None: # internally, and both require the written H5 / export artifacts that a # gate-failed run must not produce. if terminal_gate_failures: - if not args.evidence_release: + # Evidence tier (microcosm#506): owners are resolved NOW so an + # unowned failure refuses the export before the H5 and manifest work + # below. A refused evidence attempt then falls through to the SAME + # failed-run path as a certified gate failure — #568 weight-evidence + # sidecar included — so no failed run ever loses its record-level + # weight evidence. + evidence_refusal: RuntimeError | None = None + if args.evidence_release: + try: + _evidence_known_failures( + terminal_gate_failures, evidence_failure_owner_patterns + ) + except RuntimeError as error: + evidence_refusal = error + if not args.evidence_release or evidence_refusal is not None: # Gate-failure path ONLY (microcosm#568 review): a batched # pre-export failure mints no H5, so the exact calibrated weight # vector — with the ordered household ids it aligns to, bound to @@ -11183,18 +11197,14 @@ def main(argv: Sequence[str] | None = None) -> None: failures=terminal_gate_failures, force_upload=True, ) + if evidence_refusal is not None: + raise evidence_refusal raise RuntimeError( "Release gates failed: " + "; ".join(terminal_gate_failures) ) - # Evidence tier (microcosm#506): the recorded terminal failures ride - # into the release manifest's known_failures block instead of - # aborting the export. Owners are resolved NOW so an unowned failure - # refuses the export before the H5 and manifest work below; the H5 - # itself carries the calibrated weights, so the #568 weight-evidence - # sidecar is not written on this path. - _evidence_known_failures( - terminal_gate_failures, evidence_failure_owner_patterns - ) + # The owned failures ride into the release manifest's known_failures + # block instead of aborting the export; the H5 written below carries + # the calibrated weights, so the sidecar is not written on this path. terminal_batch_telemetry.stage( "release_gates", status="failed", From a3b87b892332f0f32cea3f3ef2fe317948bff2a1 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 13 Aug 2026 23:32:05 -0400 Subject: [PATCH 5/6] Document the evidence-tier publish lane in the README (#506) Co-Authored-By: Claude Fable 5 --- README.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/README.md b/README.md index bdef84bd1..632ec532a 100644 --- a/README.md +++ b/README.md @@ -128,6 +128,25 @@ by its explicit release id or tag until a separate promotion updates `latest.json`. Because Slack alerts are coupled to that production pointer update, tag-only publication sends no release alert. +Evidence-tier releases (microcosm#506) are the third lane: the best available +artifact when terminal gates failed, built with +`tools/build_us_fiscal_refresh_release.py --evidence-release` (which records +every gate failure with an owner issue in the release manifest's +`known_failures` block, or refuses) and published with + +```bash +tools/publish_release.sh releases/ --repo-id policyengine/populace-us --evidence +``` + +The `--evidence` flag validates against the evidence release contract — a +certified-shape release is refused under it and vice versa — tags the +immutable release as usual, and moves only `latest-evidence.json`; the +certified `latest.json` pointer and the pe.py certification path never see +evidence artifacts. Each evidence publish supersedes the last, so +`latest-evidence.json` always names the best current evidence artifact +(consumers: `microcosm.data.latest_evidence_release`). Its Slack alert is +labeled as an evidence-tier publish. + The alert is a **no-op unless the channel's incoming-webhook URL is set**, so configure it once on the build machine: From 3850350cb6c1fe8c4d859a7b53d3203e9a86a6f9 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 18 Aug 2026 22:03:43 -0400 Subject: [PATCH 6/6] Sol review round 2: normalize pointer paths, fail-closed bindings, delimited acknowledgment (#506) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round-2 adversarial findings on the round-1 fixes, all confirmed and fixed: - (HIGH) './latest.json' dodge: root-artifact paths must now already be in canonical clean relative POSIX form before the reserved-pointer check — a path the Hub would canonicalize differently is refused outright; bare file names are enforced for every release-dir upload (extra_files traversal included). - (HIGH) Fail-open binding: the evidence contract now REQUIRES both locally checkable records to exist and be list-of-strings shaped — build_manifest gates.calibration.failures and calibration_diagnostics build.release_gates.failures (the run's merged terminal record) — and binds both into known_failures verbatim. Deleting a record now refuses instead of disabling its binding. - (HIGH) Coverage-gate failures are now bound: us_source_coverage gate.failures must each ride within a known_failures entry, and the builder records them (prefixed, owner-checked) at the evidence tier so a real build with a failing coverage gate stays publishable-by-construction. - (MEDIUM) Acknowledgment matching is now name-delimited against the diagnostics' own target names (ctc_amount can no longer be satisfied by an actc_amount entry), with the quoted-token fallback only when a recomputed failure names no target. - (MEDIUM) The AST guard now pins polarity (exact certified-guard forms), the exact guard chain of the all-green refusal, the forwarded evidence_known_failures Name (not a constant), the owned-record assignment, and all five owner-resolution sites. - (MEDIUM, adjudicated) A telemetry-crash failure appended after the batched owner check exports the H5 before the next owner check refuses manifests — consistent with the #568 late-gate doctrine (weights retained in the written dataset); now documented at the conversion site. Co-Authored-By: Claude Fable 5 --- .../tests/test_us_fiscal_refresh_builder.py | 74 +++++-- .../src/microcosm/data/contract.py | 186 ++++++++++++++---- .../src/microcosm/data/release.py | 28 ++- .../microcosm-data/tests/test_contract.py | 165 ++++++++++++++-- packages/microcosm-data/tests/test_release.py | 71 ++++++- tools/build_us_fiscal_refresh_release.py | 22 +++ 6 files changed, 468 insertions(+), 78 deletions(-) diff --git a/packages/microcosm-build/tests/test_us_fiscal_refresh_builder.py b/packages/microcosm-build/tests/test_us_fiscal_refresh_builder.py index ff89fc57b..0f6b42fce 100644 --- a/packages/microcosm-build/tests/test_us_fiscal_refresh_builder.py +++ b/packages/microcosm-build/tests/test_us_fiscal_refresh_builder.py @@ -11080,6 +11080,12 @@ def _ancestor_if_tests(node: ast.AST) -> list[str]: and node.target.id == "terminal_gate_failures" ) + # The exact certified-guard forms — polarity included, so a reversed or + # aliased condition cannot pass (round-2 sol mutation probe). + certified_guard_tests = { + "not args.evidence_release", + "not args.evidence_release or evidence_refusal is not None", + } gate_raises = [ node for node in ast.walk(main_fn) @@ -11088,18 +11094,20 @@ def _ancestor_if_tests(node: ast.AST) -> list[str]: ] assert gate_raises, "main() lost its release-gate raises" for raise_node in gate_raises: - conditioned = any( + ancestor_tests = _ancestor_if_tests(raise_node) + guarded = any(test in certified_guard_tests for test in ancestor_tests) + mentions_flag = any( "args.evidence_release" in test or "evidence_refusal" in test - for test in _ancestor_if_tests(raise_node) + for test in ancestor_tests ) if raise_node.lineno > accumulator.lineno: - assert conditioned, ( - f"terminal raise at line {raise_node.lineno} is not " - "conditioned on --evidence-release; the conversion site " - "regressed" + assert guarded, ( + f"terminal raise at line {raise_node.lineno} must sit under " + f"one of {sorted(certified_guard_tests)}; the conversion " + "site regressed or changed polarity" ) else: - assert not conditioned, ( + assert not mentions_flag, ( f"pre-terminal raise at line {raise_node.lineno} is " "conditioned on --evidence-release; preflight/mid-build " "gates must abort in both modes" @@ -11113,9 +11121,13 @@ def _ancestor_if_tests(node: ast.AST) -> list[str]: in (ast.get_source_segment(source, node) or "") ] assert len(refusals) == 1, "the all-green evidence refusal must exist once" - assert any( - test == "args.evidence_release" for test in _ancestor_if_tests(refusals[0]) - ) + # Exact guard chain, innermost first: reachable precisely when the flag + # is set and no terminal failure was recorded — an extra wrapper (the + # unreachability mutation) or a polarity flip breaks the equality. + assert _ancestor_if_tests(refusals[0]) == [ + "not terminal_gate_failures", + "args.evidence_release", + ] manifest_calls = [ node @@ -11128,5 +11140,43 @@ def _ancestor_if_tests(node: ast.AST) -> list[str]: assert refusals[0].lineno < manifest_calls[0].lineno, ( "the all-green refusal must precede the manifest write" ) - keywords = {keyword.arg for keyword in manifest_calls[0].keywords} - assert "evidence_known_failures" in keywords + manifest_keyword = next( + keyword + for keyword in manifest_calls[0].keywords + if keyword.arg == "evidence_known_failures" + ) + # The kwarg must forward the resolved record, not a constant (the + # evidence_known_failures=None mutation). + assert isinstance(manifest_keyword.value, ast.Name) + assert manifest_keyword.value.id == "evidence_known_failures" + owned_assignments = [ + node + for node in ast.walk(main_fn) + if isinstance(node, ast.Assign) + and any( + isinstance(target, ast.Name) and target.id == "evidence_known_failures" + for target in node.targets + ) + and isinstance(node.value, ast.Call) + and isinstance(node.value.func, ast.Name) + and node.value.func.id == "_evidence_known_failures" + ] + assert len(owned_assignments) == 1, ( + "the owned record must be assigned from _evidence_known_failures" + ) + assert "args.evidence_release" in _ancestor_if_tests(owned_assignments[0]) + + # Every owner-resolution site: the batched conversion, the reform-smoke + # and take-up recordings, the coverage-gate recording, and the final + # assignment before the manifest write. A dropped site weakens the + # unowned-failure refusal. + owner_check_calls = [ + node + for node in ast.walk(main_fn) + if isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "_evidence_known_failures" + ] + assert len(owner_check_calls) == 5, ( + f"expected 5 owner-resolution sites in main(), found {len(owner_check_calls)}" + ) diff --git a/packages/microcosm-data/src/microcosm/data/contract.py b/packages/microcosm-data/src/microcosm/data/contract.py index d1ef3ded9..bf14874a4 100644 --- a/packages/microcosm-data/src/microcosm/data/contract.py +++ b/packages/microcosm-data/src/microcosm/data/contract.py @@ -3367,15 +3367,67 @@ def validate_release_dir(release_dir: Path | str) -> None: #: First quoted token in a contract-recomputed critical-fit failure — the -#: target name (or requirement id) the breach is about, used to bind the -#: breach to its ``known_failures`` acknowledgment. +#: fallback binding token when the failure names no diagnostics target. _QUOTED_TOKEN_RE = re.compile(r"'([^']+)'") +def _token_appears_delimited(token: str, text: str) -> bool: + """True if ``token`` appears in ``text`` as a whole name, not merely as a + substring of a longer one (``ctc_amount@2024`` must not be satisfied by + an entry about ``actc_amount@2024``).""" + return ( + re.search( + rf"(? None: + """Require every recorded failure string to ride into ``known_failures``. + + ``verbatim`` demands exact-string membership; otherwise substring + containment suffices (the builder records some families with a + gate-family prefix around the raw string). Fail-closed on shape: a + record that is not a list of strings is itself a violation — absence + must never silently disable the binding. + """ + if not isinstance(recorded, list) or any( + not isinstance(failure, str) for failure in recorded + ): + failures.append( + f"{source} must be a list of strings for an evidence release; " + "the known_failures binding cannot be verified otherwise." + ) + return + combined = "\n".join(recorded_texts) + if verbatim: + missing = [failure for failure in recorded if failure not in recorded_texts] + requirement = "verbatim" + else: + missing = [failure for failure in recorded if failure not in combined] + requirement = "within an entry" + if missing: + failures.append( + f"release_manifest.json known_failures must carry every {source} " + f"entry {requirement}; missing {_sample_values(missing)}." + ) + + def _check_evidence_known_failures_binding( *, release_manifest: Mapping | None, build_manifest: Mapping | None, + calibration_diagnostics: Mapping | None, + source_coverage_diagnostics: Mapping | None, recomputed_critical_failures: list[str], failures: list[str], ) -> None: @@ -3383,21 +3435,26 @@ def _check_evidence_known_failures_binding( The evidence tier's honesty cannot rest on trusting the manifest author: everything a local validator can recompute or read back must be - acknowledged. Two bindings (a hand-edited record that softens or drops a - recorded failure fails here): - - - every failure string the build manifest records under - ``gates.calibration.failures`` must appear in ``known_failures`` - verbatim; - - every critical-target breach recomputed from - ``calibration_diagnostics.json`` (the same check the certified - contract enforces as a hard refusal) must be acknowledged by name in - some ``known_failures`` entry. + acknowledged, and every locally checkable record is REQUIRED to be + present and well-shaped (fail-closed — deleting a record must never + disable its binding). A hand-edited manifest that softens or drops a + recorded failure fails here: + + - ``build_manifest.json`` ``gates.calibration.failures`` — verbatim + membership in ``known_failures``; + - ``calibration_diagnostics.json`` ``build.release_gates.failures`` + (the run's merged terminal record) — verbatim membership; + - ``us_source_coverage.json`` ``gate.failures`` — containment (the + builder records these with a gate-family prefix); + - every critical-target breach recomputed from the diagnostics (the + same check the certified contract enforces as a hard refusal) — + acknowledged by delimited name in the record. The converse direction is deliberately open: ``known_failures`` may - carry entries beyond what is locally recomputable (post-battery gate - groups record into side artifacts, not contract files), so the tier can - over-disclose but never under-disclose. + carry entries beyond what is locally recomputable, so the tier can + over-disclose but never under-disclose. The binding guarantees each + recorded failure is NAMED with an owner — prose sentiment around it is + for the human review the #490 register pattern already requires. """ if release_manifest is None: return @@ -3413,32 +3470,75 @@ def _check_evidence_known_failures_binding( if build_manifest is not None: gates = build_manifest.get("gates") calibration = gates.get("calibration") if isinstance(gates, Mapping) else None - recorded_gate_failures = ( - calibration.get("failures") if isinstance(calibration, Mapping) else None - ) - if isinstance(recorded_gate_failures, list): - missing = [ - failure - for failure in recorded_gate_failures - if isinstance(failure, str) and failure not in recorded_texts - ] - if missing: - failures.append( - "release_manifest.json known_failures must carry every " - "build_manifest.json gates.calibration failure verbatim; " - f"missing {_sample_values(missing)}." - ) + _recorded_failure_subset_binding( + calibration.get("failures") if isinstance(calibration, Mapping) else None, + recorded_texts=recorded_texts, + source="build_manifest.json gates.calibration.failures", + failures=failures, + verbatim=True, + ) + if calibration_diagnostics is not None: + build = calibration_diagnostics.get("build") + release_gates = ( + build.get("release_gates") if isinstance(build, Mapping) else None + ) + _recorded_failure_subset_binding( + ( + release_gates.get("failures") + if isinstance(release_gates, Mapping) + else None + ), + recorded_texts=recorded_texts, + source="calibration_diagnostics.json build.release_gates.failures", + failures=failures, + verbatim=True, + ) + if source_coverage_diagnostics is not None: + gate = source_coverage_diagnostics.get("gate") + _recorded_failure_subset_binding( + gate.get("failures") if isinstance(gate, Mapping) else None, + recorded_texts=recorded_texts, + source=f"{US_SOURCE_COVERAGE_DIAGNOSTICS_FILE} gate.failures", + failures=failures, + verbatim=False, + ) + diagnostic_target_names = _diagnostic_target_names(calibration_diagnostics) for recomputed in recomputed_critical_failures: - match = _QUOTED_TOKEN_RE.search(recomputed) - token = match.group(1) if match else None - if token is None or token not in combined: + tokens = [ + name + for name in diagnostic_target_names + if _token_appears_delimited(name, recomputed) + ] + if not tokens: + match = _QUOTED_TOKEN_RE.search(recomputed) + tokens = [match.group(1)] if match else [] + unacknowledged = [ + token + for token in tokens + if not _token_appears_delimited(token, combined) + ] + if not tokens or unacknowledged: failures.append( "release_manifest.json known_failures must acknowledge the " - f"critical-target breach naming {token or recomputed!r}; the " - "evidence tier records failures, it never hides them." + "critical-target breach naming " + f"{unacknowledged or [recomputed]}; the evidence tier " + "records failures, it never hides them." ) +def _diagnostic_target_names(calibration_diagnostics: Mapping | None) -> list[str]: + if calibration_diagnostics is None: + return [] + targets = calibration_diagnostics.get("targets") + if not isinstance(targets, list): + return [] + return [ + str(target["name"]) + for target in targets + if isinstance(target, Mapping) and target.get("name") + ] + + def _check_evidence_release_manifest(manifest: Mapping, failures: list[str]) -> None: """Evidence-only manifest requirements: the tier marker and the honest non-empty ``known_failures`` record.""" @@ -3612,13 +3712,6 @@ def validate_evidence_release_dir(release_dir: Path | str) -> None: diagnostics, recomputed_critical_failures ) - _check_evidence_known_failures_binding( - release_manifest=release_manifest, - build_manifest=build_manifest, - recomputed_critical_failures=recomputed_critical_failures, - failures=failures, - ) - _check_cross_manifest_consistency( build_manifest, release_manifest, @@ -3638,6 +3731,15 @@ def validate_evidence_release_dir(release_dir: Path | str) -> None: require_gate_passed=False, ) + _check_evidence_known_failures_binding( + release_manifest=release_manifest, + build_manifest=build_manifest, + calibration_diagnostics=calibration_diagnostics, + source_coverage_diagnostics=source_coverage_diagnostics, + recomputed_critical_failures=recomputed_critical_failures, + failures=failures, + ) + _check_us_fiscal_source_consistency( calibration_diagnostics, source_coverage_diagnostics, failures ) diff --git a/packages/microcosm-data/src/microcosm/data/release.py b/packages/microcosm-data/src/microcosm/data/release.py index d87beddd6..de1dc2f99 100644 --- a/packages/microcosm-data/src/microcosm/data/release.py +++ b/packages/microcosm-data/src/microcosm/data/release.py @@ -33,6 +33,7 @@ import hashlib import json +import posixpath from collections.abc import Mapping from dataclasses import dataclass from datetime import UTC, datetime @@ -279,6 +280,14 @@ def publish_release( release_artifacts = _release_manifest_release_artifacts(release_dir) filenames = _ordered_unique((*contract_files, *release_artifacts, *extra_files)) for filename in filenames: + # Every release-dir upload lands at releases// — a name + # carrying path components ('../../latest.json') could escape that + # prefix once the Hub canonicalizes the path. Bare file names only. + if filename in {".", ".."} or "/" in filename or "\\" in filename: + raise ValueError( + f"release file name {filename!r} must be a bare file name; " + "path components cannot ride into the release upload." + ) local = release_dir / filename if not local.is_file(): raise FileNotFoundError( @@ -287,9 +296,22 @@ def publish_release( root_artifacts = _release_manifest_root_artifacts(release_dir) # Root artifacts upload at their manifest-declared repo paths — the one # surface where a manifest author could smuggle a pointer write past the - # tier's pointer selection (an artifact literally named latest.json or - # latest-evidence.json). Both pointer paths are reserved on BOTH tiers: - # pointers move only via the publisher's own pointer operation. + # tier's pointer selection. Two layers, both on BOTH tiers: the declared + # path must already be in canonical clean relative form (so './latest.json' + # or 'x/../latest.json' cannot dodge a raw-string comparison and be + # canonicalized by the Hub afterwards), and the canonical pointer paths + # are reserved outright — pointers move only via the publisher's own + # pointer operation. + for path_in_repo in root_artifacts: + if ( + "\\" in path_in_repo + or path_in_repo.startswith("/") + or posixpath.normpath(path_in_repo) != path_in_repo + ): + raise ValueError( + f"release_manifest.json root artifact path {path_in_repo!r} " + "is not a clean relative POSIX path; refusing to upload it." + ) pointer_collisions = sorted( {LATEST_POINTER_PATH, LATEST_EVIDENCE_POINTER_PATH} & set(root_artifacts) ) diff --git a/packages/microcosm-data/tests/test_contract.py b/packages/microcosm-data/tests/test_contract.py index bf57050ec..29b2fdd31 100644 --- a/packages/microcosm-data/tests/test_contract.py +++ b/packages/microcosm-data/tests/test_contract.py @@ -3507,16 +3507,40 @@ def _evidence_release_manifest( return manifest +def _evidence_build_manifest() -> dict: + """The build manifest an --evidence-release run writes: the battery + verdict records the same failure strings known_failures carries.""" + manifest = _build_manifest(EVIDENCE_RELEASE_ID) + manifest["gates"]["calibration"] = { + "passed": False, + "failures": [entry["failure"] for entry in _known_failures()], + } + return manifest + + +def _evidence_calibration_diagnostics() -> dict: + """Diagnostics as the evidence builder writes them: the merged terminal + record rides in build.release_gates.""" + diagnostics = _calibration_diagnostics() + diagnostics["build"] = { + "release_gates": { + "passed": False, + "failures": [entry["failure"] for entry in _known_failures()], + } + } + return diagnostics + + @pytest.fixture def evidence_release_dir(tmp_path: Path) -> Path: """A complete, evidence-contract-valid release directory.""" directory = tmp_path / "releases" / EVIDENCE_RELEASE_ID directory.mkdir(parents=True) (directory / "build_manifest.json").write_text( - json.dumps(_build_manifest(EVIDENCE_RELEASE_ID)) + json.dumps(_evidence_build_manifest()) ) (directory / "calibration_diagnostics.json").write_text( - json.dumps(_calibration_diagnostics()) + json.dumps(_evidence_calibration_diagnostics()) ) (directory / US_SOURCE_COVERAGE_DIAGNOSTICS_FILE).write_text( json.dumps(_source_coverage_diagnostics()) @@ -3687,9 +3711,10 @@ def test_evidence_known_failures_accept_issue_url_owners( lambda manifest: manifest.update( known_failures=[ { - "failure": "QRF tail concentration failed: ...", + "failure": entry["failure"], "owner": "https://github.com/PolicyEngine/microcosm/issues/481", } + for entry in _known_failures() ] ), ) @@ -3728,15 +3753,18 @@ def test_evidence_release_rejects_certified_schema_version( ) -def _breach_critical_target(evidence_release_dir: Path) -> None: - """Push the federal income-tax row far past its blocking tolerance (the - same breach the certified suite uses to prove a hard refusal).""" - diagnostics = _calibration_diagnostics() - target = next( - row for row in diagnostics["targets"] if row["name"] == BREACHED_CRITICAL_TARGET +def _breach_critical_target( + evidence_release_dir: Path, *, name: str = BREACHED_CRITICAL_TARGET +) -> None: + """Push a critical row far past its blocking tolerance (the same breach + the certified suite uses to prove a hard refusal), keeping the fixture's + recorded build.release_gates block intact.""" + diagnostics = json.loads( + (evidence_release_dir / "calibration_diagnostics.json").read_text() ) - target["final_estimate"] = 735_173_331_468.564 - target["relative_error"] = -0.6508063496056629 + target = next(row for row in diagnostics["targets"] if row["name"] == name) + target["final_estimate"] = target["target"] * 0.35 + target["relative_error"] = -0.65 _write_json_and_refresh_manifest_hash( evidence_release_dir, filename="calibration_diagnostics.json", @@ -3824,7 +3852,7 @@ def test_evidence_release_requires_recorded_gate_failures_verbatim( with pytest.raises(ReleaseContractError) as excinfo: validate_evidence_release_dir(evidence_release_dir) failures = "\n".join(excinfo.value.failures) - assert "gates.calibration failure verbatim" in failures + assert "gates.calibration.failures entry verbatim" in failures def test_evidence_release_scope_requires_the_us_prefix(tmp_path: Path) -> None: @@ -3838,9 +3866,12 @@ def test_evidence_release_scope_requires_the_us_prefix(tmp_path: Path) -> None: assert "out of scope" in "\n".join(excinfo.value.failures) -def test_evidence_release_tolerates_failed_source_coverage_gate( +def test_evidence_release_binds_failed_source_coverage_gate( evidence_release_dir: Path, ) -> None: + """A failed coverage gate no longer blocks the evidence tier, but each of + its recorded failures must ride into known_failures (the builder records + them with a gate-family prefix) — unacknowledged, the release refuses.""" payload = _source_coverage_diagnostics() payload["gate"] = { "name": "us_source_coverage", @@ -3854,6 +3885,99 @@ def test_evidence_release_tolerates_failed_source_coverage_gate( payload=payload, ) + with pytest.raises(ReleaseContractError) as excinfo: + validate_evidence_release_dir(evidence_release_dir) + assert "gate.failures entry within an entry" in "\n".join(excinfo.value.failures) + + _rewrite_evidence_manifest( + evidence_release_dir, + lambda manifest: manifest.update( + known_failures=[ + *manifest["known_failures"], + { + "failure": ( + "Source coverage failed: " + "social_security_ssi/ssa-ssi-table-7b1-2024 missing" + ), + "owner": "PolicyEngine/microcosm#470", + }, + ] + ), + ) + validate_evidence_release_dir(evidence_release_dir) + + +def test_evidence_release_requires_the_battery_record_to_exist( + evidence_release_dir: Path, +) -> None: + """Deleting a locally checkable record must fail CLOSED, not silently + disable its binding (sol round-2 finding).""" + build_manifest = _evidence_build_manifest() + del build_manifest["gates"]["calibration"] + (evidence_release_dir / "build_manifest.json").write_text( + json.dumps(build_manifest) + ) + with pytest.raises(ReleaseContractError) as excinfo: + validate_evidence_release_dir(evidence_release_dir) + failures = "\n".join(excinfo.value.failures) + assert "gates.calibration.failures must be a list of strings" in failures + + +def test_evidence_release_requires_the_terminal_record_to_exist( + evidence_release_dir: Path, +) -> None: + diagnostics = _evidence_calibration_diagnostics() + del diagnostics["build"] + _write_json_and_refresh_manifest_hash( + evidence_release_dir, + filename="calibration_diagnostics.json", + artifact_key="calibration_diagnostics", + payload=diagnostics, + ) + with pytest.raises(ReleaseContractError) as excinfo: + validate_evidence_release_dir(evidence_release_dir) + failures = "\n".join(excinfo.value.failures) + assert "build.release_gates.failures must be a list of strings" in failures + + +def test_evidence_breach_acknowledgment_rejects_substring_collisions( + evidence_release_dir: Path, +) -> None: + """An entry naming actc_amount must not satisfy a ctc_amount breach: the + acknowledgment match is name-delimited, not substring (sol round-2).""" + ctc_name = "irs_soi.ty2022.historic_table_2.us.all.ctc_amount@2024" + actc_name = "irs_soi.ty2022.historic_table_2.us.all.actc_amount@2024" + _breach_critical_target(evidence_release_dir, name=ctc_name) + _rewrite_evidence_manifest( + evidence_release_dir, + lambda manifest: manifest.update( + known_failures=[ + *manifest["known_failures"], + { + "failure": f"Fiscal critical target '{actc_name}' drifted.", + "owner": "PolicyEngine/microcosm#487", + }, + ] + ), + ) + + with pytest.raises(ReleaseContractError) as excinfo: + validate_evidence_release_dir(evidence_release_dir) + failures = "\n".join(excinfo.value.failures) + assert "must acknowledge the critical-target breach" in failures + + _rewrite_evidence_manifest( + evidence_release_dir, + lambda manifest: manifest.update( + known_failures=[ + *manifest["known_failures"], + { + "failure": f"Fiscal critical target '{ctc_name}' breached.", + "owner": "PolicyEngine/microcosm#487", + }, + ] + ), + ) validate_evidence_release_dir(evidence_release_dir) @@ -3904,3 +4028,18 @@ def test_evidence_release_still_enforces_dirty_git_refusal( with pytest.raises(ReleaseContractError) as excinfo: validate_evidence_release_dir(evidence_release_dir) assert "'code.git_dirty' must be false" in "\n".join(excinfo.value.failures) + + +def test_breach_acknowledgment_matching_is_name_delimited() -> None: + from microcosm.data import contract as contract_module + + assert not contract_module._token_appears_delimited( + "ctc_amount@2024", "an entry about actc_amount@2024 only" + ) + assert contract_module._token_appears_delimited( + "ctc_amount@2024", "the ctc_amount@2024 row breached" + ) + # A dotted-name suffix is not the name. + assert not contract_module._token_appears_delimited( + "b.c@2024", "this names a.b.c@2024" + ) diff --git a/packages/microcosm-data/tests/test_release.py b/packages/microcosm-data/tests/test_release.py index ffd8bc2f4..28ca91f09 100644 --- a/packages/microcosm-data/tests/test_release.py +++ b/packages/microcosm-data/tests/test_release.py @@ -1231,32 +1231,49 @@ def test_pointer_with_swapped_contract_path_is_refused(hub: FakeHub) -> None: EVIDENCE_RELEASE_ID = "populace-us-2024-evidence-9f1260b-20260611" +EVIDENCE_KNOWN_FAILURE = ( + "SOI Table 1.4 national dollar fit failed: target " + "'irs_soi.ty2023.table_1_4.all.capital_gain_distributions_" + "amount@2024' has relative_error=-0.302, exceeding 0.25." +) + @pytest.fixture def evidence_release_dir(release_dir: Path) -> Path: """The certified fixture re-tiered: evidence id, evidence schema marker, - and a non-empty known_failures block.""" + a non-empty known_failures block, and the recorded gate results the + known_failures binding reads back (build gates.calibration and + diagnostics build.release_gates).""" directory = release_dir.parent / EVIDENCE_RELEASE_ID directory.mkdir() - for name in ("calibration_diagnostics.json", US_SOURCE_COVERAGE_DIAGNOSTICS_FILE): - (directory / name).write_text((release_dir / name).read_text()) + (directory / US_SOURCE_COVERAGE_DIAGNOSTICS_FILE).write_text( + (release_dir / US_SOURCE_COVERAGE_DIAGNOSTICS_FILE).read_text() + ) + diagnostics = json.loads((release_dir / "calibration_diagnostics.json").read_text()) + diagnostics["build"] = { + "release_gates": {"passed": False, "failures": [EVIDENCE_KNOWN_FAILURE]} + } + (directory / "calibration_diagnostics.json").write_text(json.dumps(diagnostics)) build_manifest = json.loads((release_dir / "build_manifest.json").read_text()) build_manifest["build_id"] = EVIDENCE_RELEASE_ID + build_manifest["gates"]["calibration"] = { + "passed": False, + "failures": [EVIDENCE_KNOWN_FAILURE], + } (directory / "build_manifest.json").write_text(json.dumps(build_manifest)) manifest = json.loads((release_dir / "release_manifest.json").read_text()) manifest["schema_version"] = EVIDENCE_RELEASE_MANIFEST_SCHEMA_VERSION manifest["tier"] = "evidence" manifest["known_failures"] = [ { - "failure": ( - "SOI Table 1.4 national dollar fit failed: target " - "'irs_soi.ty2023.table_1_4.all.capital_gain_distributions_" - "amount@2024' has relative_error=-0.302, exceeding 0.25." - ), + "failure": EVIDENCE_KNOWN_FAILURE, "owner": "PolicyEngine/microcosm#487", } ] manifest["build"]["build_id"] = EVIDENCE_RELEASE_ID + manifest["artifacts"]["calibration_diagnostics"]["sha256"] = _sha256( + directory / "calibration_diagnostics.json" + ) for artifact in manifest["artifacts"].values(): artifact["revision"] = EVIDENCE_RELEASE_ID (directory / "release_manifest.json").write_text(json.dumps(manifest)) @@ -1503,3 +1520,41 @@ def test_publish_refuses_root_artifacts_at_pointer_paths( artifact_root=artifact_root, ) assert hub.uploads == [] + + +def test_publish_refuses_unclean_root_artifact_paths( + hub: FakeHub, evidence_release_dir: Path, artifact_root: Path +) -> None: + """'./latest.json' must not dodge the reserved-path comparison and get + canonicalized to the pointer by the Hub afterwards (sol round-2).""" + _declare_root_artifact( + evidence_release_dir, key="smuggled_pointer", path=f"./{LATEST_POINTER_PATH}" + ) + with pytest.raises(ValueError, match="clean relative POSIX path"): + publish_release( + evidence_release_dir, + "policyengine/populace-us", + api=hub, + artifact_root=artifact_root, + update_latest=False, + evidence=True, + ) + assert hub.uploads == [] + + +def test_publish_refuses_path_components_in_extra_files( + hub: FakeHub, release_dir: Path, artifact_root: Path +) -> None: + """extra_files land under releases// — a traversal name could escape + that prefix once the service canonicalizes the path.""" + outside = release_dir.parent / "escape.json" + outside.write_text("{}") + with pytest.raises(ValueError, match="bare file name"): + publish_release( + release_dir, + "policyengine/populace-us", + api=hub, + artifact_root=artifact_root, + extra_files=("../escape.json",), + ) + assert hub.uploads == [] diff --git a/tools/build_us_fiscal_refresh_release.py b/tools/build_us_fiscal_refresh_release.py index 0bd2a8ad9..4ca97a283 100644 --- a/tools/build_us_fiscal_refresh_release.py +++ b/tools/build_us_fiscal_refresh_release.py @@ -11205,6 +11205,12 @@ def main(argv: Sequence[str] | None = None) -> None: # The owned failures ride into the release manifest's known_failures # block instead of aborting the export; the H5 written below carries # the calibrated weights, so the sidecar is not written on this path. + # Failures appended AFTER this point (a _TerminalBatchTelemetry crash + # line, the smoke/take-up/coverage recordings) are owner-checked at + # their own append sites and again before the manifest write; if one + # is unowned the run dies post-H5 — weights retained in the written + # dataset, per the #568 doctrine for late-gate failures — and no + # manifest is minted. terminal_batch_telemetry.stage( "release_gates", status="failed", @@ -11397,6 +11403,22 @@ def main(argv: Sequence[str] | None = None) -> None: write_us_source_coverage_diagnostics( coverage, release_dir / "us_source_coverage.json" ) + if args.evidence_release: + # The certified path surfaces a failed source-coverage gate at publish + # (the contract requires gate.passed); the evidence contract relaxes + # that verdict but BINDS gate.failures into known_failures — so an + # evidence build must record them here, owner-checked immediately, or + # its manifest would be unpublishable by construction. + coverage_gate = coverage.get("gate") or {} + coverage_gate_failures = [ + f"Source coverage failed: {failure}" + for failure in (coverage_gate.get("failures") or ()) + ] + if coverage_gate_failures: + terminal_gate_failures.extend(coverage_gate_failures) + _evidence_known_failures( + terminal_gate_failures, evidence_failure_owner_patterns + ) if telemetry is not None: telemetry.attach_artifact( "us_source_coverage",