From 6bd4e309a262e0fa4f189720c84bdfbdae5812d7 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 21 Aug 2026 15:06:56 +0300 Subject: [PATCH] Add Logbook dataset families and cardinality --- .../637-logbook-family-actions.added.md | 1 + .../637-logbook-family-discovery.added.md | 1 + .../641-logbook-build-cardinality.added.md | 1 + logbook/README.md | 120 ++ .../src/microcosm/build/logbook.py | 288 ++++- .../src/microcosm/build/logbook_family.py | 1053 +++++++++++++++++ .../tests/fixtures/logbook_row_versions.json | 112 ++ .../microcosm-build/tests/test_logbook.py | 157 ++- .../tests/test_logbook_chain_scopes_pg.py | 386 +++++- .../microcosm-build/tests/test_logbook_cli.py | 351 +++++- .../tests/test_logbook_family.py | 510 ++++++++ .../tests/test_us_exact_k_ladder_launcher.py | 291 ++++- .../20260821000000_logbook_family_model.sql | 726 ++++++++++++ tools/build_us_exact_k_ladder_release.py | 313 ++++- tools/logbook.py | 474 +++++++- 15 files changed, 4703 insertions(+), 81 deletions(-) create mode 100644 changelog.d/637-logbook-family-actions.added.md create mode 100644 changelog.d/637-logbook-family-discovery.added.md create mode 100644 changelog.d/641-logbook-build-cardinality.added.md create mode 100644 packages/microcosm-build/src/microcosm/build/logbook_family.py create mode 100644 packages/microcosm-build/tests/fixtures/logbook_row_versions.json create mode 100644 packages/microcosm-build/tests/test_logbook_family.py create mode 100644 supabase/migrations/20260821000000_logbook_family_model.sql diff --git a/changelog.d/637-logbook-family-actions.added.md b/changelog.d/637-logbook-family-actions.added.md new file mode 100644 index 000000000..311b80a25 --- /dev/null +++ b/changelog.d/637-logbook-family-actions.added.md @@ -0,0 +1 @@ +Add append-only family actions for withdrawing a build or recording one compatible family member as the direct replacement for another. diff --git a/changelog.d/637-logbook-family-discovery.added.md b/changelog.d/637-logbook-family-discovery.added.md new file mode 100644 index 000000000..91418d866 --- /dev/null +++ b/changelog.d/637-logbook-family-discovery.added.md @@ -0,0 +1 @@ +Add Logbook families and two-column family membership records so builds made from the same verified prepared input can be discovered without treating build order as a release relationship. diff --git a/changelog.d/641-logbook-build-cardinality.added.md b/changelog.d/641-logbook-build-cardinality.added.md new file mode 100644 index 000000000..1a8793a17 --- /dev/null +++ b/changelog.d/641-logbook-build-cardinality.added.md @@ -0,0 +1 @@ +Add versioned Logbook build records with typed requested and realized dataset cardinality and a normalized record unit. Exact-k builds use a null sampling-fraction field, while every legacy archive record and checksum remains unchanged. diff --git a/logbook/README.md b/logbook/README.md index ff9697585..4112712c3 100644 --- a/logbook/README.md +++ b/logbook/README.md @@ -9,6 +9,9 @@ public git archives of those rows. ``` logbook/us.jsonl # grandfathered mixed US chain logbook//.jsonl # one new hash chain per dataset line +logbook/families/.jsonl # family identity and source checksum +logbook/family_members/.jsonl # family-to-build associations +logbook/family_actions/.jsonl # revocations and replacements logbook-spool/// # committed spool mirror, same shape ``` @@ -28,6 +31,46 @@ base-data name. Different base data need different scopes, and scope is also where builds serialize: if two lines must append concurrently, they need separate chains. +The three family-data directories are not build sequences. Their files use +the same scope paths for validation and selection, but their records have no +predecessor checksum. Build validation and rendering explicitly skip these +directories. + +## Build record versions and dataset families + +Committed build records created before typed cardinality support retain their +original JSON fields and checksum. Current writers use +`row_format_version = 2`. That representation adds `requested_k`, +`realized_k`, and `record_unit` to the checksum input. A full-size request such +as `N` is resolved before recording, so the stored values are positive numbers; +for example, a full input with 100,000 households records 100000 in both +cardinality columns and `household` as the unit. A failed build may retain a +known request and leave the realized value null. + +The existing `rung` field stores a sampling-fraction category such as `f010`. +An exact-k request specifies an absolute household count, not a fraction, so +the exact-k launcher writes SQL/JSON null in `rung`. Version-2 validation +allows that null value and includes it in the build checksum. Legacy rows and +current fraction-based writers continue to require one of the established +fraction values. + +The relational model has three parts: + +- `families` assigns a caller-created UUID to one Logbook scope and one + verified prepared-input manifest checksum. The UUID and checksum are + separate: the UUID is the database identifier, while the checksum describes + the input used by the family. +- `family_members` contains only `family_id` and `build_id`. It states that the + build used the family's prepared input. Dataset properties such as size, + random seed, file location, and build status remain on the build record. +- `family_actions` records either `revokes`, which withdraws one family member, + or `supersedes`, which says one same-size family member directly replaces + another. Revocation and replacement remain separate facts. + +A build may belong to at most one family. A family may contain any number of +builds. These associations do not change the per-scope predecessor sequence, +and there is no additional release entity or general relationship graph. + ## The vocabulary is closed-world The ratified scopes are exactly `us` and `uk/frs` — deliberately minimal: a @@ -89,6 +132,69 @@ python tools/logbook.py render --archive logbook/us.jsonl `render` is the public-safe projection: it shows `artifact_location` only for `published` and `certified` rows. +Family records are exported from the durable spool into all three files for +one scope: + +```bash +python tools/logbook.py family-export --scope us --source /logbook-spool +python tools/logbook.py family-export --scope uk/frs --remote +``` + +To restore archives, first copy the scope's build archive and three family +archives into a local spool, then send the spool. The import rejects a member +whose archived build belongs to another scope. Reconciliation sends queued +builds first, families second, memberships third, and actions last. If a +request fails, its file and every dependent file remain available for the same +command to retry. + +```bash +python tools/logbook.py family-import --scope us --spool logbook-spool +python tools/logbook.py reconcile --spool logbook-spool +``` + +The exact-k launcher uses configuration format version 2 and requires the +caller to create the family UUID: + +```json +{ + "schema_version": 2, + "family": {"id": "12345678-1234-4234-9234-123456789abc"}, + "pool": { + "release_id": "prepared-pool-release", + "manifest_sha256": "" + } +} +``` + +Use the same `family.id` for every exact-k build that uses that prepared input. +The launcher verifies `pool.manifest_sha256`, stores that same checksum as the +family's `source_pool_sha256`, resolves `N` to a numeric household count, and +writes the build, family, and membership under `/logbook-spool/`. It then +attempts remote insertion in dependency order. If credentials or database +access are unavailable, all files remain local; run `tools/logbook.py +reconcile --spool /logbook-spool` later. A family UUID already associated +with another source checksum is rejected, and the conflicting local files are +retained for inspection. + +Pass the current US predecessor checksum with +`--logbook-prev-row-digest` or `POPULACE_LOGBOOK_PREV_ROW_DIGEST`. The exact-k +launcher still only writes a package and a manual publication command; it does +not publish, certify, or update a published-release pointer. + +The archive queries emit compact JSON, suitable for reading directly or +piping to `jq`: + +```bash +python tools/logbook.py list-families +python tools/logbook.py list-family-builds --family-id +python tools/logbook.py show-family-history --family-id +``` + +This implementation completes the typed exact-count fields requested by +issue #641. It supplies the relational family, membership, and action storage +needed by issue #637; workflow-specific discovery and user interfaces remain +separate work. + ## The live store The best-effort Supabase insert (`POPULACE_LEDGER_URL` + @@ -111,6 +217,20 @@ only the grandfathered US pipelines; for `logbook//.jsonl`, it requests matching `--*` pipelines and then verifies the scope again client-side before ordering the chain. +Family export also reads one scope at a time. It filters `families`, the public +member-build view, and the public action view by the stored family scope, then +validates all family and member references before changing an archive. + +## Database migration boundary + +The family migration must be applied only after the base Logbook, prediction, +fraction-category, and `20260818000000_logbook_chain_scopes.sql` migrations. +Before deployment, identify the intended Supabase project and owning +organization, record its project reference, and compare its migration history +with this order. Do not link this repository to a project based only on a name +match. This repository change does not deploy the migration; deployment is a +separate, explicitly authorized operation. + ## UK M1 receipt The campaign's M1 row remains a local receipt only. It is structurally diff --git a/packages/microcosm-build/src/microcosm/build/logbook.py b/packages/microcosm-build/src/microcosm/build/logbook.py index 5047bb77a..e29360ffa 100644 --- a/packages/microcosm-build/src/microcosm/build/logbook.py +++ b/packages/microcosm-build/src/microcosm/build/logbook.py @@ -58,8 +58,13 @@ __all__ = [ "BUILD_DISPOSITIONS", + "DECLARED_LOGBOOK_SCOPES", + "LEGACY_LOGBOOK_ROW_FIELDS", + "LEGACY_US_PIPELINES", "LOGBOOK_ROW_FIELDS", "LOGBOOK_RUNGS", + "REMOTE_LOGBOOK_ROW_FIELDS", + "VERSION_2_LOGBOOK_ROW_FIELDS", "LogbookExportResult", "LogbookRow", "LogbookWriteResult", @@ -71,9 +76,11 @@ "load_spool_rows", "load_logbook_row", "order_rows_by_chain", + "logbook_chain_scope", "reconcile_spool", "record_build_attempt", "render_markdown", + "spool_build_rows", ] @@ -90,9 +97,18 @@ ) _DIGEST_PATTERN = re.compile(r"^[0-9a-f]{64}$") _BUILD_ID_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,254}$") +_PIPELINE_SCOPE_PATTERN = re.compile( + r"^(?P[a-z]{2})-(?P[a-z0-9_]+)(?:-[a-z0-9_-]+)?$" +) LOGBOOK_RUNGS = frozenset({"f001", "f004", "f010", "f025", "f100"}) +LEGACY_US_PIPELINES = ( + "us-2024-release", + "us-pool-inc2", + "us-stacked-pool", +) +DECLARED_LOGBOOK_SCOPES = frozenset({"us", "uk/frs"}) LEDGER_API_KEY_ENV = "POPULACE_LEDGER_API_KEY" -LOGBOOK_ROW_FIELDS = frozenset( +LEGACY_LOGBOOK_ROW_FIELDS = frozenset( { "build_id", "ts", @@ -113,7 +129,21 @@ "row_digest", } ) +_VERSION_2_EXTENSION_FIELDS = frozenset( + { + "row_format_version", + "requested_k", + "realized_k", + "record_unit", + } +) +VERSION_2_LOGBOOK_ROW_FIELDS = LEGACY_LOGBOOK_ROW_FIELDS | _VERSION_2_EXTENSION_FIELDS +REMOTE_LOGBOOK_ROW_FIELDS = VERSION_2_LOGBOOK_ROW_FIELDS +# Current writers emit version 2. Keep the established public name as the +# current writer field set while exposing the legacy set explicitly. +LOGBOOK_ROW_FIELDS = VERSION_2_LOGBOOK_ROW_FIELDS _HASH_EXCLUDED_FIELDS = frozenset({"prev_row_digest", "row_digest"}) +_RECORD_UNIT_PATTERN = re.compile(r"^[a-z][a-z0-9_]*$") _ARCHIVE_THREAD_LOCKS: dict[Path, threading.Lock] = {} _ARCHIVE_THREAD_LOCKS_GUARD = threading.Lock() @@ -125,7 +155,7 @@ class LogbookRow: build_id: str ts: str pipeline: str - rung: str + rung: str | None seed: int | None code_pin: str input_pins_digest: str @@ -137,6 +167,10 @@ class LogbookRow: artifact_location: str | None disposition: str prediction_id: str | None + row_format_version: int | None + requested_k: int | None + realized_k: int | None + record_unit: str | None prev_row_digest: str | None row_digest: str @@ -147,7 +181,7 @@ def create( build_id: str, ts: str | datetime, pipeline: str, - rung: str, + rung: str | None, seed: int | None, code_pin: str, input_pins_digest: str, @@ -160,6 +194,10 @@ def create( disposition: str, prediction_id: str | None, prev_row_digest: str | None, + row_format_version: int | None = None, + requested_k: int | None = None, + realized_k: int | None = None, + record_unit: str | None = None, row_digest: str | None = None, ) -> LogbookRow: """Validate, normalize, and hash a complete attempt receipt.""" @@ -170,7 +208,6 @@ def create( "build_id must use only letters, digits, '.', '_', ':', or '-'." ) normalized_pipeline = _nonempty_text(pipeline, "pipeline") - normalized_rung = _validate_rung(rung) normalized_seed = _validate_seed(seed) normalized_code_pin = _nonempty_text(code_pin, "code_pin") normalized_input_digest = _validate_digest( @@ -198,6 +235,22 @@ def create( f"artifact_location is required for disposition {disposition!r}." ) normalized_prediction = _optional_text(prediction_id, "prediction_id") + ( + normalized_version, + normalized_requested, + normalized_realized, + normalized_record_unit, + ) = _validate_row_cardinality( + row_format_version=row_format_version, + requested_k=requested_k, + realized_k=realized_k, + record_unit=record_unit, + disposition=disposition, + ) + normalized_rung = _validate_rung( + rung, + row_format_version=normalized_version, + ) normalized_prev = _validate_digest( prev_row_digest, "prev_row_digest", @@ -222,6 +275,15 @@ def create( "prediction_id": normalized_prediction, "prev_row_digest": normalized_prev, } + if normalized_version == 2: + values.update( + { + "row_format_version": normalized_version, + "requested_k": normalized_requested, + "realized_k": normalized_realized, + "record_unit": normalized_record_unit, + } + ) calculated = compute_row_digest(values) if row_digest is not None: supplied = _validate_digest(row_digest, "row_digest", nullable=False) @@ -247,6 +309,10 @@ def create( artifact_location=normalized_artifact, disposition=disposition, prediction_id=normalized_prediction, + row_format_version=normalized_version, + requested_k=normalized_requested, + realized_k=normalized_realized, + record_unit=normalized_record_unit, prev_row_digest=normalized_prev, row_digest=calculated, ) @@ -260,14 +326,61 @@ def from_mapping(cls, value: Mapping[str, Any]) -> LogbookRow: f"Logbook row must be an object, got {type(value).__name__}." ) keys = frozenset(value) - if keys != LOGBOOK_ROW_FIELDS: - missing = sorted(LOGBOOK_ROW_FIELDS - keys) - extra = sorted(keys - LOGBOOK_ROW_FIELDS) + if keys == LEGACY_LOGBOOK_ROW_FIELDS: + expected = LEGACY_LOGBOOK_ROW_FIELDS + elif keys == VERSION_2_LOGBOOK_ROW_FIELDS: + expected = VERSION_2_LOGBOOK_ROW_FIELDS + else: + expected = ( + VERSION_2_LOGBOOK_ROW_FIELDS + if keys & _VERSION_2_EXTENSION_FIELDS + else LEGACY_LOGBOOK_ROW_FIELDS + ) + missing = sorted(expected - keys) + extra = sorted(keys - expected) raise ValueError( f"Logbook row schema mismatch; missing={missing}, extra={extra}." ) return cls.create(**dict(value)) + @classmethod + def from_database_mapping(cls, value: Mapping[str, Any]) -> LogbookRow: + """Normalize a PostgREST build row into its archived representation.""" + + if not isinstance(value, Mapping): + raise ValueError( + f"Logbook database row must be an object, got {type(value).__name__}." + ) + keys = frozenset(value) + if keys == LEGACY_LOGBOOK_ROW_FIELDS: + return cls.from_mapping(value) + if keys != REMOTE_LOGBOOK_ROW_FIELDS: + missing = sorted(REMOTE_LOGBOOK_ROW_FIELDS - keys) + extra = sorted(keys - REMOTE_LOGBOOK_ROW_FIELDS) + raise ValueError( + "Logbook database row schema mismatch; " + f"missing={missing}, extra={extra}." + ) + normalized = dict(value) + if normalized["row_format_version"] is None: + populated = { + field: normalized[field] + for field in ( + "requested_k", + "realized_k", + "record_unit", + ) + if normalized[field] is not None + } + if populated: + raise ValueError( + "Legacy Logbook database row has version-2 cardinality " + f"values: {sorted(populated)}." + ) + for field in _VERSION_2_EXTENSION_FIELDS: + normalized.pop(field) + return cls.from_mapping(normalized) + def to_mapping(self) -> dict[str, Any]: """Return the normalized JSON row in #628 database-column order.""" @@ -290,6 +403,15 @@ def to_mapping(self) -> dict[str, Any]: "prev_row_digest": self.prev_row_digest, "row_digest": self.row_digest, } + if self.row_format_version == 2: + mapping.update( + { + "row_format_version": self.row_format_version, + "requested_k": self.requested_k, + "realized_k": self.realized_k, + "record_unit": self.record_unit, + } + ) # ``frozen=True`` prevents attribute replacement but a caller can # still mutate nested JSON containers. Re-authenticate immediately # before every serialization boundary so neither such a mutation nor @@ -355,19 +477,43 @@ def canonical_json_bytes(value: Any) -> bytes: raise ValueError(f"Value is not canonical JSON: {exc}.") from exc +def logbook_chain_scope(pipeline: str) -> str | None: + """Return the database sequence scope derived from a pipeline name.""" + + normalized = _nonempty_text(pipeline, "pipeline") + if normalized in LEGACY_US_PIPELINES: + return "us" + match = _PIPELINE_SCOPE_PATTERN.fullmatch(normalized) + if match is None: + return None + return f"{match.group('country')}/{match.group('line')}" + + def compute_row_digest(value: Mapping[str, Any]) -> str: """Compute SHA-256(canonical non-chain fields || predecessor).""" - missing = (LOGBOOK_ROW_FIELDS - {"row_digest"}) - frozenset(value) - if missing: - raise ValueError(f"Cannot hash Logbook row; missing fields: {sorted(missing)}.") + fields = ( + VERSION_2_LOGBOOK_ROW_FIELDS + if "row_format_version" in value + else LEGACY_LOGBOOK_ROW_FIELDS + ) + keys = frozenset(value) + missing = (fields - {"row_digest"}) - keys + extra = keys - fields + if missing or extra: + raise ValueError( + "Cannot hash Logbook row; " + f"missing={sorted(missing)}, extra={sorted(extra)}." + ) predecessor = _validate_digest( value.get("prev_row_digest"), "prev_row_digest", nullable=True, ) payload = { - key: value[key] for key in sorted(value) if key not in _HASH_EXCLUDED_FIELDS + key: value[key] + for key in sorted(fields) + if key in value and key not in _HASH_EXCLUDED_FIELDS } material = canonical_json_bytes(payload) + (predecessor or "").encode("ascii") return hashlib.sha256(material).hexdigest() @@ -378,7 +524,7 @@ def record_build_attempt( build_id: str, ts: str | datetime, pipeline: str, - rung: str, + rung: str | None, seed: int | None, code_pin: str, input_pins_digest: str, @@ -391,9 +537,14 @@ def record_build_attempt( disposition: str, prediction_id: str | None, prev_row_digest: str | None, + row_format_version: int = 2, + requested_k: int | None = None, + realized_k: int | None = None, + record_unit: str | None = None, row_digest: str | None = None, spool_dir: str | Path = "logbook-spool", timeout: float = 10.0, + post_remote: bool = True, ) -> LogbookWriteResult: """Validate, durably spool, then best-effort insert one terminal attempt. @@ -420,12 +571,16 @@ def record_build_attempt( disposition=disposition, prediction_id=prediction_id, prev_row_digest=prev_row_digest, + row_format_version=row_format_version, + requested_k=requested_k, + realized_k=realized_k, + record_unit=record_unit, row_digest=row_digest, ) spool_path = Path(spool_dir) / f"{row.row_digest}.json" _atomic_write_row(spool_path, row) config = _remote_config() - if config is None: + if config is None or not post_remote: return LogbookWriteResult(row=row, spool_path=spool_path) posted, error = _post_build_row( row, @@ -471,6 +626,22 @@ def load_spool_rows(spool_dir: str | Path) -> tuple[LogbookRow, ...]: return order_rows_by_chain(rows) +def spool_build_rows( + rows: Sequence[LogbookRow], + *, + spool_dir: str | Path = "logbook-spool", +) -> tuple[Path, ...]: + """Copy authenticated build rows into a durable reconciliation spool.""" + + ordered = order_rows_by_chain(rows) + paths: list[Path] = [] + for row in ordered: + path = Path(spool_dir) / f"{row.row_digest}.json" + _atomic_write_row(path, row) + paths.append(path) + return tuple(paths) + + def order_rows_by_chain( rows: Sequence[LogbookRow], ) -> tuple[LogbookRow, ...]: @@ -695,7 +866,7 @@ def render_markdown( row.ts, row.build_id, row.pipeline, - row.rung, + row.rung or "—", row.disposition, public_artifact or "—", ) @@ -961,7 +1132,15 @@ def _normalize_timestamp(value: str | datetime, field: str) -> str: return parsed.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%S.%fZ") -def _validate_rung(value: str) -> str: +def _validate_rung( + value: str | None, + *, + row_format_version: int | None = None, +) -> str | None: + if value is None: + if row_format_version == 2: + return None + raise ValueError("rung may be null only for a version-2 Logbook row.") if not isinstance(value, str) or value not in LOGBOOK_RUNGS: raise ValueError( "rung must be a #624 fraction token: 'f001', 'f004', 'f010', 'f025', or 'f100'." @@ -1040,6 +1219,85 @@ def _validate_nonnegative_number( return value +def _validate_row_cardinality( + *, + row_format_version: int | None, + requested_k: int | None, + realized_k: int | None, + record_unit: str | None, + disposition: str, +) -> tuple[int | None, int | None, int | None, str | None]: + if row_format_version is None: + populated = { + name: value + for name, value in ( + ("requested_k", requested_k), + ("realized_k", realized_k), + ("record_unit", record_unit), + ) + if value is not None + } + if populated: + raise ValueError( + "Legacy Logbook rows cannot contain version-2 cardinality " + f"values: {sorted(populated)}." + ) + return None, None, None, None + if isinstance(row_format_version, bool) or row_format_version != 2: + raise ValueError( + "row_format_version must be 2 for current rows or null for " + f"legacy rows, got {row_format_version!r}." + ) + + normalized_values: list[int | None] = [] + for field, value in ( + ("requested_k", requested_k), + ("realized_k", realized_k), + ): + if value is None: + normalized_values.append(None) + continue + if ( + isinstance(value, bool) + or not isinstance(value, int) + or value <= 0 + or value > 2**63 - 1 + ): + raise ValueError( + f"{field} must be a positive signed 64-bit integer or null, " + f"got {value!r}." + ) + normalized_values.append(value) + + requested, realized = normalized_values + if requested is None and realized is None: + if record_unit is not None: + raise ValueError( + "record_unit must be null when requested_k and realized_k " + "are both null." + ) + normalized_unit = None + else: + normalized_unit = _nonempty_text(record_unit, "record_unit") + if ( + normalized_unit != normalized_unit.lower() + or not _RECORD_UNIT_PATTERN.fullmatch(normalized_unit) + ): + raise ValueError( + "record_unit must be a normalized lowercase identifier " + "using letters, digits, or underscores." + ) + + if disposition in {"published", "certified"} and requested is not None: + if realized != requested: + raise ValueError( + f"{disposition} exact-k build requires realized_k to equal " + f"requested_k; got requested_k={requested!r}, " + f"realized_k={realized!r}." + ) + return 2, requested, realized, normalized_unit + + def _validate_digest( value: Any, field: str, diff --git a/packages/microcosm-build/src/microcosm/build/logbook_family.py b/packages/microcosm-build/src/microcosm/build/logbook_family.py new file mode 100644 index 000000000..a0c7e2f07 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/logbook_family.py @@ -0,0 +1,1053 @@ +"""Typed, durable records for Logbook dataset families and later decisions.""" + +from __future__ import annotations + +import json +import re +import uuid +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Protocol +from urllib.error import HTTPError +from urllib.parse import urlencode +from urllib.request import Request + +from .logbook import ( + DECLARED_LOGBOOK_SCOPES, + LogbookRow, + ReconcileResult, + _atomic_write_bytes, + _fsync_file_and_parent, + _fsync_parent_directory, + _nonempty_text, + _normalize_timestamp, + _optional_text, + _remote_config, + _validate_digest, + _validate_remote_url, + logbook_chain_scope, + reconcile_spool, + urlopen, +) + +__all__ = [ + "FAMILY_ACTION_TYPES", + "FamilyAction", + "FamilyArchiveRecords", + "FamilyExportResult", + "FamilyMember", + "FamilyReconcileResult", + "FamilyWriteResult", + "LogbookFamily", + "LogbookReconcileResult", + "export_family_records", + "export_family_scope", + "family_archive_path", + "import_family_scope", + "load_family_actions", + "load_family_archive_records", + "load_family_members", + "load_family_spool", + "load_families", + "reconcile_family_spool", + "reconcile_logbook_spool", + "record_family", + "record_family_action", + "record_family_member", + "validate_family_action", + "validate_family_membership", + "validate_family_source", +] + + +FAMILY_ACTION_TYPES = frozenset({"revokes", "supersedes"}) +_BUILD_ID_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,254}$") +_SPOOL_DIRECTORIES = { + "families": "families", + "family_members": "family_members", + "family_actions": "family_actions", +} + + +class FamilyRecord(Protocol): + def to_mapping(self) -> dict[str, Any]: ... + + def to_json_line(self) -> str: ... + + +@dataclass(frozen=True) +class LogbookFamily: + family_id: str + chain_scope: str + source_pool_sha256: str + + @classmethod + def create( + cls, + *, + family_id: str, + chain_scope: str, + source_pool_sha256: str, + ) -> LogbookFamily: + parsed_id = _canonical_uuid(family_id, "family_id") + parsed_scope = _nonempty_text(chain_scope, "chain_scope") + if parsed_scope not in DECLARED_LOGBOOK_SCOPES: + raise ValueError( + f"chain_scope must be one of {sorted(DECLARED_LOGBOOK_SCOPES)}, " + f"got {parsed_scope!r}." + ) + parsed_source = _validate_digest( + source_pool_sha256, + "source_pool_sha256", + nullable=False, + ) + assert parsed_source is not None + return cls( + family_id=parsed_id, + chain_scope=parsed_scope, + source_pool_sha256=parsed_source, + ) + + @classmethod + def from_mapping(cls, value: Mapping[str, Any]) -> LogbookFamily: + _exact_keys( + value, + {"family_id", "chain_scope", "source_pool_sha256"}, + "family", + ) + return cls.create(**dict(value)) + + def to_mapping(self) -> dict[str, Any]: + mapping = { + "family_id": self.family_id, + "chain_scope": self.chain_scope, + "source_pool_sha256": self.source_pool_sha256, + } + LogbookFamily.from_mapping(mapping) + return mapping + + def to_json_line(self) -> str: + return _json_line(self.to_mapping()) + + +@dataclass(frozen=True) +class FamilyMember: + family_id: str + build_id: str + + @classmethod + def create(cls, *, family_id: str, build_id: str) -> FamilyMember: + parsed_build = _nonempty_text(build_id, "build_id") + if not _BUILD_ID_PATTERN.fullmatch(parsed_build): + raise ValueError( + "build_id must use only letters, digits, '.', '_', ':', or '-'." + ) + return cls( + family_id=_canonical_uuid(family_id, "family_id"), + build_id=parsed_build, + ) + + @classmethod + def from_mapping(cls, value: Mapping[str, Any]) -> FamilyMember: + _exact_keys(value, {"family_id", "build_id"}, "family member") + return cls.create(**dict(value)) + + def to_mapping(self) -> dict[str, Any]: + mapping = { + "family_id": self.family_id, + "build_id": self.build_id, + } + FamilyMember.from_mapping(mapping) + return mapping + + def to_json_line(self) -> str: + return _json_line(self.to_mapping()) + + +@dataclass(frozen=True) +class FamilyAction: + action_id: str + family_id: str + build_id: str + action_type: str + related_build_id: str | None + recorded_at: str + actor: str + reason: str + evidence_location: str | None + + @classmethod + def create( + cls, + *, + action_id: str, + family_id: str, + build_id: str, + action_type: str, + related_build_id: str | None, + recorded_at: str, + actor: str, + reason: str, + evidence_location: str | None, + ) -> FamilyAction: + parsed_type = _nonempty_text(action_type, "action_type") + if parsed_type not in FAMILY_ACTION_TYPES: + raise ValueError( + f"action_type must be one of {sorted(FAMILY_ACTION_TYPES)}, " + f"got {parsed_type!r}." + ) + parsed_build = FamilyMember.create( + family_id=family_id, + build_id=build_id, + ) + parsed_related = ( + None + if related_build_id is None + else FamilyMember.create( + family_id=family_id, + build_id=related_build_id, + ).build_id + ) + if parsed_type == "revokes" and parsed_related is not None: + raise ValueError("revokes action must not contain related_build_id.") + if parsed_type == "supersedes": + if parsed_related is None: + raise ValueError("supersedes action requires related_build_id.") + if parsed_related == parsed_build.build_id: + raise ValueError("A build cannot supersede itself.") + return cls( + action_id=_canonical_uuid(action_id, "action_id"), + family_id=parsed_build.family_id, + build_id=parsed_build.build_id, + action_type=parsed_type, + related_build_id=parsed_related, + recorded_at=_normalize_timestamp(recorded_at, "recorded_at"), + actor=_nonempty_text(actor, "actor"), + reason=_nonempty_text(reason, "reason"), + evidence_location=_optional_text( + evidence_location, + "evidence_location", + ), + ) + + @classmethod + def from_mapping(cls, value: Mapping[str, Any]) -> FamilyAction: + _exact_keys( + value, + { + "action_id", + "family_id", + "build_id", + "action_type", + "related_build_id", + "recorded_at", + "actor", + "reason", + "evidence_location", + }, + "family action", + ) + return cls.create(**dict(value)) + + def to_mapping(self) -> dict[str, Any]: + mapping = { + "action_id": self.action_id, + "family_id": self.family_id, + "build_id": self.build_id, + "action_type": self.action_type, + "related_build_id": self.related_build_id, + "recorded_at": self.recorded_at, + "actor": self.actor, + "reason": self.reason, + "evidence_location": self.evidence_location, + } + FamilyAction.from_mapping(mapping) + return mapping + + def to_json_line(self) -> str: + return _json_line(self.to_mapping()) + + +@dataclass(frozen=True) +class FamilyWriteResult: + record: LogbookFamily | FamilyMember | FamilyAction + spool_path: Path + posted: bool = False + remote_error: str | None = None + + +@dataclass(frozen=True) +class FamilyReconcileResult: + attempted: int + posted: int + retained: int + errors: tuple[str, ...] + + +@dataclass(frozen=True) +class LogbookReconcileResult: + builds: ReconcileResult + families: FamilyReconcileResult + + +@dataclass(frozen=True) +class FamilyExportResult: + existing: int + appended: int + + +@dataclass(frozen=True) +class FamilyArchiveRecords: + families: tuple[LogbookFamily, ...] + family_members: tuple[FamilyMember, ...] + family_actions: tuple[FamilyAction, ...] + + +def validate_family_source( + family: LogbookFamily, + source_pool_sha256: str, +) -> None: + supplied = _validate_digest( + source_pool_sha256, + "source_pool_sha256", + nullable=False, + ) + if supplied != family.source_pool_sha256: + raise ValueError( + f"Family {family.family_id} identifies source " + f"{family.source_pool_sha256}, got {supplied}." + ) + + +def validate_family_membership( + family: LogbookFamily, + member: FamilyMember, + build: LogbookRow, +) -> None: + if member.family_id != family.family_id: + raise ValueError( + f"Membership family {member.family_id} does not match " + f"family {family.family_id}." + ) + if member.build_id != build.build_id: + raise ValueError( + f"Membership build {member.build_id} does not match build {build.build_id}." + ) + build_scope = logbook_chain_scope(build.pipeline) + if build_scope != family.chain_scope: + raise ValueError( + f"Build {build.build_id} scope {build_scope!r} does not match " + f"family scope {family.chain_scope!r}." + ) + + +def validate_family_action( + action: FamilyAction, + *, + members: Sequence[FamilyMember], + builds: Mapping[str, LogbookRow], +) -> None: + member_ids = { + member.build_id for member in members if member.family_id == action.family_id + } + if action.build_id not in member_ids: + raise ValueError( + f"Build {action.build_id} is not a member of family {action.family_id}." + ) + if action.action_type == "revokes": + return + assert action.related_build_id is not None + if action.related_build_id not in member_ids: + raise ValueError( + f"Build {action.related_build_id} is not a member of family " + f"{action.family_id}." + ) + try: + replacement = builds[action.build_id] + replaced = builds[action.related_build_id] + except KeyError as exc: + raise ValueError( + f"Missing build data for family action: {exc.args[0]}." + ) from exc + if ( + replacement.requested_k != replaced.requested_k + or replacement.record_unit != replaced.record_unit + ): + raise ValueError( + "Superseding builds must have matching requested_k and record_unit." + ) + + +def record_family( + family: LogbookFamily, + *, + spool_dir: str | Path = "logbook-spool", + timeout: float = 10.0, + post_remote: bool = True, +) -> FamilyWriteResult: + for existing in load_family_spool(spool_dir).families: + if existing.family_id == family.family_id and existing != family: + raise ValueError( + f"Family spool contains a divergent retry for {family.family_id}." + ) + if ( + existing.family_id != family.family_id + and existing.chain_scope == family.chain_scope + and existing.source_pool_sha256 == family.source_pool_sha256 + ): + raise ValueError( + f"Source {family.source_pool_sha256} in scope " + f"{family.chain_scope} already belongs to family " + f"{existing.family_id}." + ) + return _record_family_value( + family, + record_type="families", + key=family.family_id, + conflict_fields=("family_id",), + spool_dir=spool_dir, + timeout=timeout, + post_remote=post_remote, + ) + + +def record_family_member( + member: FamilyMember, + *, + spool_dir: str | Path = "logbook-spool", + timeout: float = 10.0, + post_remote: bool = True, +) -> FamilyWriteResult: + return _record_family_value( + member, + record_type="family_members", + key=f"{member.family_id}--{member.build_id}", + conflict_fields=("family_id", "build_id"), + spool_dir=spool_dir, + timeout=timeout, + post_remote=post_remote, + ) + + +def record_family_action( + action: FamilyAction, + *, + spool_dir: str | Path = "logbook-spool", + timeout: float = 10.0, + post_remote: bool = True, +) -> FamilyWriteResult: + return _record_family_value( + action, + record_type="family_actions", + key=action.action_id, + conflict_fields=("action_id",), + spool_dir=spool_dir, + timeout=timeout, + post_remote=post_remote, + ) + + +def reconcile_logbook_spool( + spool_dir: str | Path = "logbook-spool", + *, + timeout: float = 10.0, +) -> LogbookReconcileResult: + build_result = reconcile_spool(spool_dir, timeout=timeout) + if build_result.errors: + family_result = _retained_family_result(spool_dir) + else: + family_result = reconcile_family_spool(spool_dir, timeout=timeout) + return LogbookReconcileResult( + builds=build_result, + families=family_result, + ) + + +def reconcile_family_spool( + spool_dir: str | Path = "logbook-spool", + *, + timeout: float = 10.0, +) -> FamilyReconcileResult: + directory = Path(spool_dir) + queued = _load_spooled_family_records(directory) + retained = sum(len(records) for _, records in queued) + config = _remote_config() + if config is None or retained == 0: + return FamilyReconcileResult(0, 0, retained, ()) + + attempted = 0 + posted = 0 + removed = 0 + errors: list[str] = [] + for record_type, records in queued: + for path, record in records: + attempted += 1 + success, error = _post_family_value( + record, + record_type=record_type, + conflict_fields=_conflict_fields(record_type), + ledger_url=config[0], + ledger_key=config[1], + ledger_api_key=config[2], + timeout=timeout, + ) + if not success: + errors.append( + f"{_record_key(record)}: {error or 'remote insert failed'}" + ) + return FamilyReconcileResult( + attempted, + posted, + retained - removed, + tuple(errors), + ) + posted += 1 + try: + path.unlink() + removed += 1 + _fsync_parent_directory(path.parent) + except OSError as exc: + errors.append( + f"{_record_key(record)}: remote insert succeeded but " + f"spool cleanup failed: {exc}" + ) + return FamilyReconcileResult( + attempted, + posted, + retained - removed, + tuple(errors), + ) + return FamilyReconcileResult( + attempted, + posted, + retained - removed, + tuple(errors), + ) + + +def load_families(path: str | Path) -> tuple[LogbookFamily, ...]: + return _load_family_file(path, LogbookFamily) + + +def load_family_members(path: str | Path) -> tuple[FamilyMember, ...]: + return _load_family_file(path, FamilyMember) + + +def load_family_actions(path: str | Path) -> tuple[FamilyAction, ...]: + return _load_family_file(path, FamilyAction) + + +def load_family_spool(spool_dir: str | Path) -> FamilyArchiveRecords: + queued = dict(_load_spooled_family_records(Path(spool_dir))) + return FamilyArchiveRecords( + families=tuple( + record + for _, record in queued["families"] + if isinstance(record, LogbookFamily) + ), + family_members=tuple( + record + for _, record in queued["family_members"] + if isinstance(record, FamilyMember) + ), + family_actions=tuple( + record + for _, record in queued["family_actions"] + if isinstance(record, FamilyAction) + ), + ) + + +def export_family_records[RecordT: (LogbookFamily, FamilyMember, FamilyAction)]( + path: str | Path, + candidates: Sequence[RecordT], +) -> FamilyExportResult: + archive = Path(path) + if archive.exists(): + original = archive.read_bytes() + if candidates: + record_class = type(candidates[0]) + existing = _load_family_file(archive, record_class) + elif original: + raise ValueError( + "Cannot infer family record type for a nonempty archive " + "without candidates." + ) + else: + existing = () + else: + original = b"" + existing = () + if original and not original.endswith(b"\n"): + raise ValueError(f"Family archive {archive} does not end with a newline.") + + existing_by_key = {_record_key(record): record for record in existing} + appended: list[RecordT] = [] + seen_new: dict[str, RecordT] = {} + for record in candidates: + key = _record_key(record) + previous = existing_by_key.get(key) + if previous is not None: + if previous != record: + raise ValueError( + f"Family archive record {key} conflicts with existing content." + ) + continue + current = seen_new.get(key) + if current is not None: + if current != record: + raise ValueError( + f"Family archive candidates reuse {key} with different content." + ) + continue + seen_new[key] = record + appended.append(record) + + if appended: + addition = "".join(record.to_json_line() for record in appended).encode("utf-8") + _atomic_write_bytes(archive, original + addition) + elif archive.exists(): + _fsync_file_and_parent(archive) + return FamilyExportResult( + existing=len(existing), + appended=len(appended), + ) + + +def family_archive_path( + archive_root: str | Path, + record_type: str, + scope: str, +) -> Path: + if record_type not in _SPOOL_DIRECTORIES: + raise ValueError(f"Unknown family record type: {record_type}.") + parsed_scope = _nonempty_text(scope, "scope") + if parsed_scope not in DECLARED_LOGBOOK_SCOPES: + raise ValueError( + f"scope must be one of {sorted(DECLARED_LOGBOOK_SCOPES)}, " + f"got {parsed_scope!r}." + ) + scope_parts = parsed_scope.split("/") + return Path(archive_root) / record_type / Path(*scope_parts).with_suffix(".jsonl") + + +def load_family_archive_records( + archive_root: str | Path, + scope: str, +) -> FamilyArchiveRecords: + paths = { + record_type: family_archive_path(archive_root, record_type, scope) + for record_type in _SPOOL_DIRECTORIES + } + records = FamilyArchiveRecords( + families=_load_optional_family_file(paths["families"], LogbookFamily), + family_members=_load_optional_family_file( + paths["family_members"], + FamilyMember, + ), + family_actions=_load_optional_family_file( + paths["family_actions"], + FamilyAction, + ), + ) + _validate_archive_records(records, scope=scope) + return records + + +def export_family_scope( + archive_root: str | Path, + *, + scope: str, + families: Sequence[LogbookFamily] = (), + family_members: Sequence[FamilyMember] = (), + family_actions: Sequence[FamilyAction] = (), +) -> dict[str, FamilyExportResult]: + existing = load_family_archive_records(archive_root, scope) + combined = FamilyArchiveRecords( + families=_merge_archive_records(existing.families, families), + family_members=_merge_archive_records( + existing.family_members, + family_members, + ), + family_actions=_merge_archive_records( + existing.family_actions, + family_actions, + ), + ) + _validate_archive_records(combined, scope=scope) + candidates_by_type = { + "families": families, + "family_members": family_members, + "family_actions": family_actions, + } + existing_counts = { + "families": len(existing.families), + "family_members": len(existing.family_members), + "family_actions": len(existing.family_actions), + } + return { + record_type: ( + export_family_records( + family_archive_path(archive_root, record_type, scope), + candidates, + ) + if candidates + else FamilyExportResult(existing_counts[record_type], 0) + ) + for record_type, candidates in candidates_by_type.items() + } + + +def import_family_scope( + archive_root: str | Path, + *, + scope: str, + spool_dir: str | Path = "logbook-spool", +) -> FamilyArchiveRecords: + records = load_family_archive_records(archive_root, scope) + for family in records.families: + record_family(family, spool_dir=spool_dir, post_remote=False) + for member in records.family_members: + record_family_member(member, spool_dir=spool_dir, post_remote=False) + for action in records.family_actions: + record_family_action(action, spool_dir=spool_dir, post_remote=False) + return records + + +def _record_family_value( + record: LogbookFamily | FamilyMember | FamilyAction, + *, + record_type: str, + key: str, + conflict_fields: tuple[str, ...], + spool_dir: str | Path, + timeout: float, + post_remote: bool, +) -> FamilyWriteResult: + path = Path(spool_dir) / _SPOOL_DIRECTORIES[record_type] / f"{key}.json" + _atomic_write_family_record(path, record) + config = _remote_config() + if config is None or not post_remote: + return FamilyWriteResult(record=record, spool_path=path) + posted, error = _post_family_value( + record, + record_type=record_type, + conflict_fields=conflict_fields, + ledger_url=config[0], + ledger_key=config[1], + ledger_api_key=config[2], + timeout=timeout, + ) + return FamilyWriteResult( + record=record, + spool_path=path, + posted=posted, + remote_error=error, + ) + + +def _post_family_value( + record: LogbookFamily | FamilyMember | FamilyAction, + *, + record_type: str, + conflict_fields: tuple[str, ...], + ledger_url: str, + ledger_key: str, + ledger_api_key: str, + timeout: float, +) -> tuple[bool, str | None]: + if timeout <= 0: + return False, "timeout must be greater than zero" + try: + endpoint = _table_endpoint( + ledger_url, + record_type, + conflict_fields=conflict_fields, + ) + request = Request( + endpoint, + data=record.to_json_line().rstrip("\n").encode("utf-8"), + method="POST", + headers={ + "apikey": ledger_api_key, + "Authorization": f"Bearer {ledger_key}", + "Content-Profile": "logbook", + "Content-Type": "application/json", + "Prefer": "resolution=ignore-duplicates,return=minimal", + }, + ) + with urlopen(request, timeout=timeout) as response: + status = getattr(response, "status", 200) + if not 200 <= status < 300: + return False, f"Supabase returned HTTP {status}" + except HTTPError as exc: + try: + body = exc.read(1_024).decode("utf-8", errors="replace").strip() + except OSError: + body = "" + detail = f": {body}" if body else "" + return False, f"Supabase returned HTTP {exc.code}{detail}" + except OSError as exc: + return False, f"Supabase insert failed: {exc}" + except Exception as exc: # pragma: no cover + return False, f"Supabase insert failed: {type(exc).__name__}: {exc}" + return True, None + + +def _table_endpoint( + url: str, + table: str, + *, + conflict_fields: tuple[str, ...], +) -> str: + _validate_remote_url(url) + base = url.rstrip("/") + if base.endswith("/rest/v1/builds"): + base = base[: -len("/builds")] + elif not base.endswith("/rest/v1"): + base = f"{base}/rest/v1" + query = urlencode({"on_conflict": ",".join(conflict_fields)}) + return f"{base}/{table}?{query}" + + +def _atomic_write_family_record( + path: Path, + record: LogbookFamily | FamilyMember | FamilyAction, +) -> None: + content = record.to_json_line().encode("utf-8") + if path.exists(): + try: + existing = type(record).from_mapping( + json.loads(path.read_text(encoding="utf-8")) + ) + except (OSError, json.JSONDecodeError, ValueError) as exc: + raise ValueError(f"Invalid family spool record {path}: {exc}.") from exc + if existing != record: + raise ValueError( + f"Family spool key collision or divergent retry at {path}." + ) + _fsync_file_and_parent(path) + return + _atomic_write_bytes(path, content) + + +def _load_spooled_family_records( + spool_dir: Path, +) -> tuple[ + tuple[ + str, + tuple[ + tuple[Path, LogbookFamily | FamilyMember | FamilyAction], + ..., + ], + ], + ..., +]: + definitions = ( + ("families", LogbookFamily), + ("family_members", FamilyMember), + ("family_actions", FamilyAction), + ) + result = [] + for record_type, record_class in definitions: + directory = spool_dir / _SPOOL_DIRECTORIES[record_type] + records = [] + if directory.exists(): + for path in sorted(directory.glob("*.json")): + try: + value = json.loads(path.read_text(encoding="utf-8")) + record = record_class.from_mapping(value) + except (OSError, json.JSONDecodeError, ValueError) as exc: + raise ValueError( + f"Invalid family spool record {path}: {exc}." + ) from exc + if path.stem != _record_key(record): + raise ValueError( + f"Family spool filename does not match record key: {path}." + ) + records.append((path, record)) + result.append((record_type, tuple(records))) + return tuple(result) + + +def _retained_family_result(spool_dir: str | Path) -> FamilyReconcileResult: + queued = _load_spooled_family_records(Path(spool_dir)) + retained = sum(len(records) for _, records in queued) + return FamilyReconcileResult(0, 0, retained, ()) + + +def _load_family_file[RecordT: (LogbookFamily, FamilyMember, FamilyAction)]( + path: str | Path, + record_class: type[RecordT], +) -> tuple[RecordT, ...]: + archive = Path(path) + if not archive.exists(): + raise ValueError(f"Family archive does not exist: {archive}.") + try: + text = archive.read_text(encoding="utf-8") + except OSError as exc: + raise ValueError(f"Cannot read family archive {archive}: {exc}.") from exc + if not text: + return () + lines = text.splitlines() + records: list[RecordT] = [] + seen: dict[str, RecordT] = {} + for position, line in enumerate(lines, start=1): + if not line.strip(): + raise ValueError( + f"Invalid blank family record at line {position} in {archive}." + ) + try: + record = record_class.from_mapping(json.loads(line)) + except (json.JSONDecodeError, ValueError) as exc: + raise ValueError( + f"Invalid family record at line {position} in {archive}: {exc}." + ) from exc + key = _record_key(record) + previous = seen.get(key) + if previous is not None: + if previous != record: + raise ValueError(f"Family archive reuses {key} with different content.") + raise ValueError(f"Family archive repeats record {key}.") + seen[key] = record + records.append(record) + return tuple(records) + + +def _load_optional_family_file[RecordT: (LogbookFamily, FamilyMember, FamilyAction)]( + path: Path, + record_class: type[RecordT], +) -> tuple[RecordT, ...]: + if not path.exists(): + return () + return _load_family_file(path, record_class) + + +def _merge_archive_records[RecordT: (LogbookFamily, FamilyMember, FamilyAction)]( + existing: Sequence[RecordT], + candidates: Sequence[RecordT], +) -> tuple[RecordT, ...]: + merged = list(existing) + by_key = {_record_key(record): record for record in existing} + for record in candidates: + key = _record_key(record) + previous = by_key.get(key) + if previous is not None: + if previous != record: + raise ValueError( + f"Family archive record {key} conflicts with existing content." + ) + continue + by_key[key] = record + merged.append(record) + return tuple(merged) + + +def _validate_archive_records( + records: FamilyArchiveRecords, + *, + scope: str, +) -> None: + expected_scope = _nonempty_text(scope, "scope") + if expected_scope not in DECLARED_LOGBOOK_SCOPES: + raise ValueError( + f"scope must be one of {sorted(DECLARED_LOGBOOK_SCOPES)}, " + f"got {expected_scope!r}." + ) + families = {family.family_id: family for family in records.families} + wrong_scope = sorted( + family.family_id + for family in records.families + if family.chain_scope != expected_scope + ) + if wrong_scope: + raise ValueError( + f"Family archive for scope {expected_scope} contains families " + f"from another scope: {', '.join(wrong_scope)}." + ) + + members = { + (member.family_id, member.build_id): member for member in records.family_members + } + for member in records.family_members: + if member.family_id not in families: + raise ValueError( + f"Family member {member.build_id} references missing family " + f"{member.family_id}." + ) + for action in records.family_actions: + if (action.family_id, action.build_id) not in members: + raise ValueError( + f"Family action {action.action_id} references missing member " + f"{action.family_id}/{action.build_id}." + ) + if ( + action.related_build_id is not None + and ( + action.family_id, + action.related_build_id, + ) + not in members + ): + raise ValueError( + f"Family action {action.action_id} references missing related " + f"member {action.family_id}/{action.related_build_id}." + ) + + +def _record_key(record: LogbookFamily | FamilyMember | FamilyAction) -> str: + if isinstance(record, LogbookFamily): + return record.family_id + if isinstance(record, FamilyMember): + return f"{record.family_id}--{record.build_id}" + return record.action_id + + +def _conflict_fields(record_type: str) -> tuple[str, ...]: + if record_type == "families": + return ("family_id",) + if record_type == "family_members": + return ("family_id", "build_id") + if record_type == "family_actions": + return ("action_id",) + raise ValueError(f"Unknown family record type: {record_type}.") + + +def _canonical_uuid(value: Any, field: str) -> str: + if not isinstance(value, str): + raise ValueError(f"{field} must be a canonical UUID string.") + try: + parsed = uuid.UUID(value) + except (ValueError, AttributeError): + raise ValueError(f"{field} must be a canonical UUID string.") from None + canonical = str(parsed) + if value != canonical: + raise ValueError(f"{field} must use canonical lowercase UUID text.") + return canonical + + +def _exact_keys( + value: Mapping[str, Any], + expected: set[str], + label: str, +) -> None: + if not isinstance(value, Mapping): + raise ValueError(f"{label} must be an object.") + missing = sorted(expected - set(value)) + extra = sorted(set(value) - expected) + if missing or extra: + raise ValueError(f"{label} schema mismatch; missing={missing}, extra={extra}.") + + +def _json_line(value: Mapping[str, Any]) -> str: + return ( + json.dumps( + value, + allow_nan=False, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ) + + "\n" + ) diff --git a/packages/microcosm-build/tests/fixtures/logbook_row_versions.json b/packages/microcosm-build/tests/fixtures/logbook_row_versions.json new file mode 100644 index 000000000..545d990b4 --- /dev/null +++ b/packages/microcosm-build/tests/fixtures/logbook_row_versions.json @@ -0,0 +1,112 @@ +{ + "legacy": { + "expected_row_digest": "80a01b5cdefeeed6a8acd36dfa06b1e4506f4853c2786101d3c3ba414cd8a927", + "row": { + "artifact_location": null, + "build_id": "fixture-build-1", + "code_pin": "1c1fc717", + "cost_usd": 1.0, + "disposition": "failed", + "gate_verdicts": { + "agreement": { + "diagnostics": { + "ratio": 1.0, + "unicode": "café" + }, + "receipt": "receipt://fixture/agreement.json", + "verdict": "failed" + } + }, + "identity_digest": "2222222222222222222222222222222222222222222222222222222222222222", + "input_pins_digest": "1111111111111111111111111111111111111111111111111111111111111111", + "phases_reached": [ + "assembled", + "simulated" + ], + "pipeline": "fixture-pipeline", + "prediction_id": "p001", + "prev_row_digest": null, + "row_digest": "80a01b5cdefeeed6a8acd36dfa06b1e4506f4853c2786101d3c3ba414cd8a927", + "rung": "f010", + "seed": 628, + "ts": "2026-08-04T19:06:00.000000Z", + "wall_seconds": 12.5 + } + }, + "version_2": { + "expected_row_digest": "00845773dfd0093e30eb873655137619b181f235b998394a7b9d0db4020f030b", + "row": { + "artifact_location": null, + "build_id": "fixture-build-1", + "code_pin": "1c1fc717", + "cost_usd": 1.0, + "disposition": "failed", + "gate_verdicts": { + "agreement": { + "diagnostics": { + "ratio": 1.0, + "unicode": "café" + }, + "receipt": "receipt://fixture/agreement.json", + "verdict": "failed" + } + }, + "identity_digest": "2222222222222222222222222222222222222222222222222222222222222222", + "input_pins_digest": "1111111111111111111111111111111111111111111111111111111111111111", + "phases_reached": [ + "assembled", + "simulated" + ], + "pipeline": "fixture-pipeline", + "prediction_id": "p001", + "prev_row_digest": null, + "realized_k": 20000, + "record_unit": "household", + "requested_k": 20000, + "row_digest": "00845773dfd0093e30eb873655137619b181f235b998394a7b9d0db4020f030b", + "row_format_version": 2, + "rung": "f010", + "seed": 628, + "ts": "2026-08-04T19:06:00.000000Z", + "wall_seconds": 12.5 + } + }, + "version_2_exact_k": { + "expected_row_digest": "1390cb69de86116232b0c27782d298c8bc4e25df746957c689e4d0813c9e45a8", + "row": { + "artifact_location": null, + "build_id": "fixture-build-1", + "code_pin": "1c1fc717", + "cost_usd": 1.0, + "disposition": "failed", + "gate_verdicts": { + "agreement": { + "diagnostics": { + "ratio": 1.0, + "unicode": "café" + }, + "receipt": "receipt://fixture/agreement.json", + "verdict": "failed" + } + }, + "identity_digest": "2222222222222222222222222222222222222222222222222222222222222222", + "input_pins_digest": "1111111111111111111111111111111111111111111111111111111111111111", + "phases_reached": [ + "assembled", + "simulated" + ], + "pipeline": "fixture-pipeline", + "prediction_id": "p001", + "prev_row_digest": null, + "realized_k": 20000, + "record_unit": "household", + "requested_k": 20000, + "row_digest": "1390cb69de86116232b0c27782d298c8bc4e25df746957c689e4d0813c9e45a8", + "row_format_version": 2, + "rung": null, + "seed": 628, + "ts": "2026-08-04T19:06:00.000000Z", + "wall_seconds": 12.5 + } + } +} diff --git a/packages/microcosm-build/tests/test_logbook.py b/packages/microcosm-build/tests/test_logbook.py index b27580914..a2022c318 100644 --- a/packages/microcosm-build/tests/test_logbook.py +++ b/packages/microcosm-build/tests/test_logbook.py @@ -12,12 +12,16 @@ import microcosm.build.logbook as logbook from microcosm.build.logbook import ( + LEGACY_LOGBOOK_ROW_FIELDS, LOGBOOK_ROW_FIELDS, + REMOTE_LOGBOOK_ROW_FIELDS, + VERSION_2_LOGBOOK_ROW_FIELDS, LogbookRow, canonical_json_bytes, load_logbook_row, reconcile_spool, record_build_attempt, + render_markdown, ) ROOT = Path(__file__).resolve().parents[3] @@ -25,6 +29,12 @@ CHAIN_SCOPE_MIGRATION = ( ROOT / "supabase/migrations/20260818000000_logbook_chain_scopes.sql" ) +FAMILY_MODEL_MIGRATION = ( + ROOT / "supabase/migrations/20260821000000_logbook_family_model.sql" +) +ROW_VERSION_FIXTURES = ( + ROOT / "packages/microcosm-build/tests/fixtures/logbook_row_versions.json" +) @pytest.fixture(autouse=True) @@ -71,17 +81,36 @@ def test_row_schema_json_round_trip_matches_628_golden() -> None: restored = LogbookRow.from_mapping(json.loads(row.to_json_line())) assert restored == row - assert frozenset(row.to_mapping()) == LOGBOOK_ROW_FIELDS + assert frozenset(row.to_mapping()) == LEGACY_LOGBOOK_ROW_FIELDS assert restored.ts == "2026-08-04T19:06:00.000000Z" assert restored.row_digest == ( "80a01b5cdefeeed6a8acd36dfa06b1e4506f4853c2786101d3c3ba414cd8a927" ) +def test_version_2_row_matches_shared_golden() -> None: + fixtures = json.loads(ROW_VERSION_FIXTURES.read_text(encoding="utf-8")) + expected = fixtures["version_2"] + + row = LogbookRow.create( + **_row_kwargs(), + row_format_version=2, + requested_k=20_000, + realized_k=20_000, + record_unit="household", + ) + + assert row.to_mapping() == expected["row"] + assert row.row_digest == expected["expected_row_digest"] + assert frozenset(row.to_mapping()) == VERSION_2_LOGBOOK_ROW_FIELDS + assert LOGBOOK_ROW_FIELDS == VERSION_2_LOGBOOK_ROW_FIELDS + assert REMOTE_LOGBOOK_ROW_FIELDS == VERSION_2_LOGBOOK_ROW_FIELDS + + def test_sql_schema_round_trip_matches_python_hash_surface() -> None: sql = MIGRATION.read_text(encoding="utf-8") builds = sql.split("CREATE TABLE logbook.builds (", 1)[1].split("\n);", 1)[0] - for field in LOGBOOK_ROW_FIELDS: + for field in LEGACY_LOGBOOK_ROW_FIELDS: assert re.search(rf"^ {field}\s", builds, flags=re.MULTILINE), field payload = sql.split("CREATE OR REPLACE FUNCTION logbook.build_hash_payload", 1)[ @@ -89,7 +118,7 @@ def test_sql_schema_round_trip_matches_python_hash_surface() -> None: ].split("$function$;", 1)[0] payload_fields = set(re.findall(r"'([a-z_]+)',\s+p_build\.", payload)) payload_fields.add("ts") - assert payload_fields == LOGBOOK_ROW_FIELDS - { + assert payload_fields == LEGACY_LOGBOOK_ROW_FIELDS - { "prev_row_digest", "row_digest", } @@ -163,6 +192,24 @@ def test_sql_schema_round_trip_matches_python_hash_surface() -> None: assert "CREATE POLICY predictions_exporter_select" not in sql assert "GRANT logbook_writer, logbook_exporter TO authenticator" in sql + extension = FAMILY_MODEL_MIGRATION.read_text(encoding="utf-8") + for field in ( + "row_format_version", + "requested_k", + "realized_k", + "record_unit", + ): + assert re.search(rf"ADD COLUMN {field}\s", extension), field + assert "'row_format_version', p_build.row_format_version" in extension + assert "'requested_k', p_build.requested_k" in extension + assert "'realized_k', p_build.realized_k" in extension + assert "'record_unit', p_build.record_unit" in extension + assert "ALTER COLUMN rung DROP NOT NULL" in extension + assert "builds_rung_by_row_format" in extension + assert "CREATE OR REPLACE FUNCTION logbook.enforce_build_chain" not in extension + assert "CREATE OR REPLACE FUNCTION logbook.chain_scope" not in extension + assert "CREATE OR REPLACE FUNCTION logbook.scope_declared" not in extension + def test_logbook_chain_scope_migration_contract() -> None: sql = CHAIN_SCOPE_MIGRATION.read_text(encoding="utf-8") @@ -257,6 +304,108 @@ def test_schema_rejects_missing_and_extra_fields_by_name() -> None: LogbookRow.from_mapping(mapping) +def test_remote_legacy_null_columns_restore_exact_legacy_shape() -> None: + legacy = LogbookRow.create(**_row_kwargs()).to_mapping() + database_row = { + **legacy, + "row_format_version": None, + "requested_k": None, + "realized_k": None, + "record_unit": None, + } + + restored = LogbookRow.from_database_mapping(database_row) + + assert restored.to_mapping() == legacy + assert frozenset(restored.to_mapping()) == LEGACY_LOGBOOK_ROW_FIELDS + + +def test_remote_partially_versioned_legacy_row_is_rejected() -> None: + database_row = { + **LogbookRow.create(**_row_kwargs()).to_mapping(), + "row_format_version": None, + "requested_k": 20_000, + "realized_k": None, + "record_unit": "household", + } + + with pytest.raises(ValueError, match="version-2 cardinality"): + LogbookRow.from_database_mapping(database_row) + + +@pytest.mark.parametrize( + ("overrides", "message"), + [ + ({"row_format_version": 3}, "row_format_version"), + ( + {"row_format_version": 2, "requested_k": 0, "record_unit": "household"}, + "requested_k", + ), + ( + {"row_format_version": 2, "realized_k": True, "record_unit": "household"}, + "realized_k", + ), + ({"row_format_version": 2, "requested_k": 20_000}, "record_unit"), + ({"row_format_version": 2, "record_unit": "household"}, "record_unit"), + ( + { + "row_format_version": 2, + "requested_k": 20_000, + "record_unit": "Household", + }, + "normalized lowercase", + ), + ( + { + "row_format_version": 2, + "requested_k": 20_000, + "realized_k": 19_999, + "record_unit": "household", + "disposition": "published", + "artifact_location": "hf://fixture", + }, + "requires realized_k to equal", + ), + ], +) +def test_version_2_cardinality_validation_fails_closed( + overrides: dict[str, object], + message: str, +) -> None: + with pytest.raises(ValueError, match=message): + LogbookRow.create(**_row_kwargs(**overrides)) + + +def test_failed_version_2_row_can_retain_request_without_output() -> None: + row = LogbookRow.create( + **_row_kwargs(), + row_format_version=2, + requested_k=20_000, + realized_k=None, + record_unit="household", + ) + + assert row.requested_k == 20_000 + assert row.realized_k is None + + +def test_version_2_exact_k_row_accepts_null_rung() -> None: + fixture = json.loads(ROW_VERSION_FIXTURES.read_text(encoding="utf-8"))[ + "version_2_exact_k" + ] + row = LogbookRow.from_mapping(fixture["row"]) + + assert row.rung is None + assert row.row_digest == fixture["expected_row_digest"] + assert json.loads(row.to_json_line())["rung"] is None + assert "| — |" in render_markdown([row]) + + +def test_legacy_row_rejects_null_rung() -> None: + with pytest.raises(ValueError, match="only for a version-2"): + LogbookRow.create(**_row_kwargs(rung=None)) + + def test_row_rejects_nul_in_nested_gate_diagnostics() -> None: gate_verdicts = { "agreement": { @@ -301,6 +450,8 @@ def test_predecessor_is_bound_once_into_the_row_digest() -> None: def test_spool_round_trip_authenticates_contents_and_filename(tmp_path: Path) -> None: result = record_build_attempt(**_row_kwargs(), spool_dir=tmp_path) + assert result.row.row_format_version == 2 + assert frozenset(result.row.to_mapping()) == VERSION_2_LOGBOOK_ROW_FIELDS assert result.spool_path == tmp_path / f"{result.row.row_digest}.json" assert load_logbook_row(result.spool_path) == result.row diff --git a/packages/microcosm-build/tests/test_logbook_chain_scopes_pg.py b/packages/microcosm-build/tests/test_logbook_chain_scopes_pg.py index 32bddc801..a1f0c974a 100644 --- a/packages/microcosm-build/tests/test_logbook_chain_scopes_pg.py +++ b/packages/microcosm-build/tests/test_logbook_chain_scopes_pg.py @@ -31,7 +31,11 @@ ROOT = Path(__file__).resolve().parents[3] MIGRATIONS = ROOT / "supabase/migrations" NEW_MIGRATION = MIGRATIONS / "20260818000000_logbook_chain_scopes.sql" +FAMILY_MIGRATION = MIGRATIONS / "20260821000000_logbook_family_model.sql" ROWS = ROOT / "logbook/us.jsonl" +ROW_VERSION_FIXTURES = ( + ROOT / "packages/microcosm-build/tests/fixtures/logbook_row_versions.json" +) BASE_MIGRATIONS = [ "20260805000000_logbook.sql", "20260805000001_logbook_predictions.sql", @@ -66,12 +70,28 @@ "prev_row_digest", "row_digest", ] +VERSION_2_COLUMNS = [ + *COLUMNS, + "row_format_version", + "requested_k", + "realized_k", + "record_unit", +] def _connect(uri: str): return psycopg.connect(uri, autocommit=True) +def _postgres_server(): + runtime = Path(tempfile.mkdtemp(prefix="microcosm-pgserver-runtime-")) + server_class = pgserver.PostgresServer + server_class.runtime_path = runtime + server_class.lock_path = runtime / ".lockfile" + server_class._lock = server_class.fasteners.InterProcessLock(server_class.lock_path) + return pgserver.get_server(Path(tempfile.mkdtemp(prefix="microcosm-pgdata-"))) + + def _apply_sql(connection, sql: str) -> None: with connection.cursor() as cursor: cursor.execute(sql) @@ -117,9 +137,10 @@ def _build_row(build_id: str, *, pipeline: str, predecessor: str | None) -> dict } -def _insert(connection, row: dict) -> None: +def _insert(connection, row: dict, *, versioned: bool = False) -> None: + columns = VERSION_2_COLUMNS if versioned else COLUMNS values = [] - for column in COLUMNS: + for column in columns: value = row.get(column) if isinstance(value, (dict, list)): value = json.dumps(value) @@ -128,10 +149,10 @@ def _insert(connection, row: dict) -> None: # would acquire digits on the cast and change the row digest. value = repr(value) values.append(value) - placeholders = ", ".join(["%s"] * len(COLUMNS)) + placeholders = ", ".join(["%s"] * len(columns)) with connection.cursor() as cursor: cursor.execute( - f"INSERT INTO logbook.builds ({', '.join(COLUMNS)}) " + f"INSERT INTO logbook.builds ({', '.join(columns)}) " f"VALUES ({placeholders});", values, ) @@ -161,8 +182,27 @@ def _refuses(connection, row: dict, needle: str) -> None: _insert(connection, row) +def _execute(connection, statement: str, values: tuple = ()) -> None: + with connection.cursor() as cursor: + cursor.execute(statement, values) + + +def _fetchone(connection, statement: str, values: tuple = ()) -> tuple: + with connection.cursor() as cursor: + cursor.execute(statement, values) + found = cursor.fetchone() + assert found is not None + return found + + +def _fetchall(connection, statement: str, values: tuple = ()) -> list[tuple]: + with connection.cursor() as cursor: + cursor.execute(statement, values) + return cursor.fetchall() + + def test_logbook_chain_scopes_migration_preserves_and_scopes_live_rows() -> None: - server = pgserver.get_server(tempfile.mkdtemp()) + server = _postgres_server() connection = _connect(server.get_uri()) _apply_migrations(connection) archived = _archived_rows() @@ -205,9 +245,7 @@ def test_logbook_chain_scopes_migration_preserves_and_scopes_live_rows() -> None # independence is already proven above: uk/frs opened while us had rows. _refuses( connection, - _build_row( - "uk-locals-genesis", pipeline="uk-locals-rowwise", predecessor=None - ), + _build_row("uk-locals-genesis", pipeline="uk-locals-rowwise", predecessor=None), "not in the ratified scope list", ) _refuses( @@ -269,3 +307,335 @@ def test_logbook_chain_scopes_migration_preserves_and_scopes_live_rows() -> None if _digest_of(connection, build_id) != digest ] assert changed == [] + + +def test_family_model_migration_enforces_versioning_relationships_and_access() -> None: + server = _postgres_server() + connection = _connect(server.get_uri()) + _apply_migrations(connection) + archived = _archived_rows() + for row in archived: + _insert(connection, row) + _apply_sql(connection, NEW_MIGRATION.read_text(encoding="utf-8")) + + # Plain PostgreSQL does not provide Supabase's API roles. Create them + # before applying this migration so its conditional public-view grants + # can be exercised in the harness. + _execute(connection, "CREATE ROLE anon NOLOGIN;") + _apply_sql(connection, FAMILY_MIGRATION.read_text(encoding="utf-8")) + + assert [ + row[0] + for row in _fetchall( + connection, + "SELECT row_format_version FROM logbook.builds ORDER BY ts, build_id;", + ) + ] == [None] * len(archived) + assert { + row[0]: row[1] + for row in _fetchall( + connection, + "SELECT build_id, row_digest FROM logbook.builds;", + ) + } == {row["build_id"]: row["row_digest"] for row in archived} + + fixtures = json.loads(ROW_VERSION_FIXTURES.read_text(encoding="utf-8")) + for fixture_name in ("legacy", "version_2", "version_2_exact_k"): + fixture = fixtures[fixture_name] + payload = json.dumps(fixture["row"]) + observed = _fetchone( + connection, + "SELECT logbook.expected_build_row_digest(" + "json_populate_record(NULL::logbook.builds, %s::json));", + (payload,), + )[0] + assert observed == fixture["expected_row_digest"] + + us_tail = archived[-1]["row_digest"] + us_first = _build_row( + "family-us-20000-a", + pipeline="us-stacked-pool", + predecessor=us_tail, + ) + us_first.update( + row_format_version=2, + requested_k=20_000, + realized_k=20_000, + record_unit="household", + ) + _insert(connection, us_first, versioned=True) + us_second = _build_row( + "family-us-20000-b", + pipeline="us-stacked-pool", + predecessor=_digest_of(connection, us_first["build_id"]), + ) + us_second.update( + row_format_version=2, + requested_k=20_000, + realized_k=20_000, + record_unit="household", + ) + _insert(connection, us_second, versioned=True) + us_mismatch = _build_row( + "family-us-57240", + pipeline="us-stacked-pool", + predecessor=_digest_of(connection, us_second["build_id"]), + ) + us_mismatch.update( + row_format_version=2, + requested_k=57_240, + realized_k=57_240, + record_unit="household", + ) + _insert(connection, us_mismatch, versioned=True) + + legacy_null_rung = _build_row( + "legacy-null-rung", + pipeline="us-stacked-pool", + predecessor=_digest_of(connection, us_mismatch["build_id"]), + ) + legacy_null_rung["rung"] = None + with pytest.raises(psycopg.errors.CheckViolation): + _insert(connection, legacy_null_rung) + + exact_k_null_rung = _build_row( + "exact-k-null-rung", + pipeline="us-stacked-pool", + predecessor=_digest_of(connection, us_mismatch["build_id"]), + ) + exact_k_null_rung.update( + rung=None, + row_format_version=2, + requested_k=20_000, + realized_k=20_000, + record_unit="household", + ) + _insert(connection, exact_k_null_rung, versioned=True) + assert _fetchone( + connection, + "SELECT rung FROM logbook.builds WHERE build_id = %s;", + (exact_k_null_rung["build_id"],), + ) == (None,) + + uk_build = _build_row( + "family-uk", + pipeline="uk-frs-staging", + predecessor=None, + ) + uk_build.update( + row_format_version=2, + requested_k=10, + realized_k=10, + record_unit="household", + ) + _insert(connection, uk_build, versioned=True) + + invalid_cardinality = _build_row( + "invalid-cardinality", + pipeline="us-stacked-pool", + predecessor=_digest_of(connection, exact_k_null_rung["build_id"]), + ) + invalid_cardinality.update( + row_format_version=2, + requested_k=0, + realized_k=None, + record_unit="household", + ) + with pytest.raises(psycopg.errors.CheckViolation): + _insert(connection, invalid_cardinality, versioned=True) + + invalid_publication = dict(invalid_cardinality) + invalid_publication.update( + build_id="invalid-publication", + requested_k=20_000, + realized_k=19_999, + disposition="published", + artifact_location="hf://datasets/fixture/release", + ) + with pytest.raises(psycopg.errors.CheckViolation): + _insert(connection, invalid_publication, versioned=True) + + family_id = "12345678-1234-4234-9234-123456789abc" + other_family_id = "22345678-1234-4234-9234-123456789abc" + second_us_family_id = "62345678-1234-4234-9234-123456789abc" + source_sha = "a" * 64 + family_upsert = ( + "INSERT INTO logbook.families " + "(family_id, chain_scope, source_pool_sha256) VALUES (%s, %s, %s) " + "ON CONFLICT (family_id) DO NOTHING;" + ) + _execute(connection, family_upsert, (family_id, "us", source_sha)) + _execute(connection, family_upsert, (family_id, "us", source_sha)) + with pytest.raises(psycopg.errors.UniqueViolation, match="divergent content"): + _execute(connection, family_upsert, (family_id, "uk/frs", "b" * 64)) + with pytest.raises(psycopg.errors.UniqueViolation, match="already belongs"): + _execute(connection, family_upsert, (other_family_id, "us", source_sha)) + + member_insert = ( + "INSERT INTO logbook.family_members (family_id, build_id) " + "VALUES (%s, %s) ON CONFLICT (family_id, build_id) DO NOTHING;" + ) + for build_id in ( + us_first["build_id"], + us_second["build_id"], + us_mismatch["build_id"], + ): + _execute(connection, member_insert, (family_id, build_id)) + _execute(connection, member_insert, (family_id, us_first["build_id"])) + with pytest.raises(psycopg.errors.CheckViolation, match="does not match"): + _execute(connection, member_insert, (family_id, uk_build["build_id"])) + _execute(connection, family_upsert, (other_family_id, "uk/frs", "b" * 64)) + _execute( + connection, + family_upsert, + (second_us_family_id, "us", "c" * 64), + ) + with pytest.raises(psycopg.errors.UniqueViolation): + _execute( + connection, + member_insert, + (second_us_family_id, us_first["build_id"]), + ) + + revocation_id = "32345678-1234-4234-9234-123456789abc" + replacement_id = "42345678-1234-4234-9234-123456789abc" + action_insert = ( + "INSERT INTO logbook.family_actions " + "(action_id, family_id, build_id, action_type, related_build_id, " + "recorded_at, actor, reason, evidence_location) " + "VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s) " + "ON CONFLICT (action_id) DO NOTHING;" + ) + revocation = ( + revocation_id, + family_id, + us_mismatch["build_id"], + "revokes", + None, + "2026-08-21T12:00:00Z", + "fixture", + "Invalid output", + None, + ) + _execute(connection, action_insert, revocation) + _execute(connection, action_insert, revocation) + replacement = ( + replacement_id, + family_id, + us_second["build_id"], + "supersedes", + us_first["build_id"], + "2026-08-21T12:01:00Z", + "fixture", + "Corrected output", + None, + ) + _execute(connection, action_insert, replacement) + with pytest.raises(psycopg.errors.UniqueViolation, match="divergent content"): + _execute( + connection, + action_insert, + (*replacement[:-2], "Different reason", replacement[-1]), + ) + with pytest.raises(psycopg.errors.CheckViolation): + _execute( + connection, + action_insert, + ( + "72345678-1234-4234-9234-123456789abc", + family_id, + us_first["build_id"], + "supersedes", + us_first["build_id"], + "2026-08-21T12:01:30Z", + "fixture", + "Self replacement", + None, + ), + ) + with pytest.raises(psycopg.errors.UniqueViolation): + _execute( + connection, + action_insert, + ( + "82345678-1234-4234-9234-123456789abc", + family_id, + us_second["build_id"], + "supersedes", + us_first["build_id"], + "2026-08-21T12:01:45Z", + "fixture", + "Conflicting direct replacement", + None, + ), + ) + with pytest.raises(psycopg.errors.CheckViolation, match="matching requested_k"): + _execute( + connection, + action_insert, + ( + "52345678-1234-4234-9234-123456789abc", + family_id, + us_mismatch["build_id"], + "supersedes", + us_second["build_id"], + "2026-08-21T12:02:00Z", + "fixture", + "Wrong size", + None, + ), + ) + + public_build = _fetchone( + connection, + "SELECT requested_k, realized_k, record_unit " + "FROM logbook.family_members_public WHERE build_id = %s;", + (us_first["build_id"],), + ) + assert public_build == (20_000, 20_000, "household") + public_cardinalities = _fetchone( + connection, + "SELECT array_agg(requested_k ORDER BY requested_k, build_id) " + "FROM logbook.family_members_public WHERE family_id = %s;", + (family_id,), + )[0] + assert public_cardinalities == [20_000, 20_000, 57_240] + status = _fetchone( + connection, + "SELECT revoked, superseded_by_build_id " + "FROM logbook.family_member_status_public WHERE build_id = %s;", + (us_first["build_id"],), + ) + assert status == (False, us_second["build_id"]) + + _execute(connection, "SET ROLE anon;") + assert ( + _fetchone(connection, "SELECT count(*) FROM logbook.families_public;")[0] == 3 + ) + with pytest.raises(psycopg.errors.UndefinedColumn): + _execute(connection, "SELECT cost_usd FROM logbook.family_members_public;") + with pytest.raises(psycopg.errors.InsufficientPrivilege): + _execute(connection, "SELECT * FROM logbook.family_actions;") + with pytest.raises(psycopg.errors.InsufficientPrivilege): + _execute(connection, "SELECT * FROM logbook.predictions;") + _execute(connection, "RESET ROLE;") + + _execute(connection, "SET ROLE logbook_writer;") + _execute( + connection, + family_upsert, + ("92345678-1234-4234-9234-123456789abc", "us", "d" * 64), + ) + with pytest.raises(psycopg.errors.InsufficientPrivilege): + _execute( + connection, + "UPDATE logbook.families SET chain_scope = 'uk/frs' WHERE family_id = %s;", + (family_id,), + ) + with pytest.raises(psycopg.errors.InsufficientPrivilege): + _execute( + connection, + "DELETE FROM logbook.family_members WHERE build_id = %s;", + (us_first["build_id"],), + ) + _execute(connection, "RESET ROLE;") diff --git a/packages/microcosm-build/tests/test_logbook_cli.py b/packages/microcosm-build/tests/test_logbook_cli.py index 15c3a56f2..d88b02a3e 100644 --- a/packages/microcosm-build/tests/test_logbook_cli.py +++ b/packages/microcosm-build/tests/test_logbook_cli.py @@ -11,6 +11,17 @@ import pytest from microcosm.build.logbook import LogbookRow, load_logbook_file +from microcosm.build.logbook_family import ( + FamilyAction, + FamilyMember, + LogbookFamily, + load_families, + load_family_actions, + load_family_members, + record_family, + record_family_action, + record_family_member, +) ROOT = Path(__file__).resolve().parents[3] CLI_PATH = ROOT / "tools/logbook.py" @@ -32,6 +43,7 @@ def _row( pipeline: str = "uk-frs-staging", rung: str = "f010", disposition: str = "failed", + requested_k: int | None = None, ) -> LogbookRow: artifact = ( f"hf://datasets/policyengine/populace-us@{build_id}" @@ -60,6 +72,10 @@ def _row( disposition=disposition, prediction_id=None, prev_row_digest=predecessor, + row_format_version=2 if requested_k is not None else None, + requested_k=requested_k, + realized_k=requested_k, + record_unit="household" if requested_k is not None else None, ) @@ -165,9 +181,7 @@ def test_export_refuses_a_directory_archive( spool.mkdir() cli = _load_cli() - assert ( - cli.main(["export", "--archive", str(tmp_path), "--source", str(spool)]) == 1 - ) + assert cli.main(["export", "--archive", str(tmp_path), "--source", str(spool)]) == 1 assert "extends exactly one scope chain" in capsys.readouterr().err @@ -213,6 +227,247 @@ def test_cli_validate_and_filtered_render( assert "cost_usd" not in output +def test_build_archive_discovery_excludes_family_record_directories( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + root = tmp_path / "logbook" + root.mkdir() + _write_jsonl(root / "us.jsonl", _chain(pipeline="us-stacked-pool")) + for directory in ("families", "family_members", "family_actions"): + archive = root / directory / "us.jsonl" + archive.parent.mkdir(parents=True) + archive.write_text('{"not":"a build row"}\n', encoding="utf-8") + cli = _load_cli() + + assert cli.main(["validate", "--archive", str(root)]) == 0 + output = capsys.readouterr().out + assert output.count("validated 3 Logbook rows") == 1 + + assert ( + cli.main( + [ + "validate", + "--archive", + str(root / "families"), + ] + ) + == 1 + ) + assert "No Logbook build archives" in capsys.readouterr().err + assert ( + cli.main( + [ + "validate", + "--archive", + str(root / "families/us.jsonl"), + ] + ) + == 1 + ) + assert "No Logbook build archives" in capsys.readouterr().err + + +def test_family_archive_commands_and_queries( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + family_id = "12345678-1234-4234-9234-123456789abc" + action_id = "22345678-1234-4234-9234-123456789abc" + family = LogbookFamily.create( + family_id=family_id, + chain_scope="us", + source_pool_sha256="a" * 64, + ) + large = _row( + "family-large", + predecessor=None, + minute=1, + pipeline="us-stacked-pool", + requested_k=57_240, + ) + small = _row( + "family-small", + predecessor=large.row_digest, + minute=2, + pipeline="us-stacked-pool", + requested_k=20_000, + ) + members = ( + FamilyMember.create(family_id=family_id, build_id=large.build_id), + FamilyMember.create(family_id=family_id, build_id=small.build_id), + ) + action = FamilyAction.create( + action_id=action_id, + family_id=family_id, + build_id=small.build_id, + action_type="supersedes", + related_build_id=large.build_id, + recorded_at="2026-08-21T12:00:00Z", + actor="fixture", + reason="Corrected build", + evidence_location=None, + ) + source_spool = tmp_path / "source-spool" + record_family(family, spool_dir=source_spool, post_remote=False) + for member in members: + record_family_member(member, spool_dir=source_spool, post_remote=False) + record_family_action(action, spool_dir=source_spool, post_remote=False) + archive_root = tmp_path / "logbook" + archive_root.mkdir() + _write_jsonl(archive_root / "us.jsonl", (large, small)) + cli = _load_cli() + + assert ( + cli.main( + [ + "family-export", + "--scope", + "us", + "--archive-root", + str(archive_root), + "--source", + str(source_spool), + ] + ) + == 0 + ) + assert load_families(archive_root / "families/us.jsonl") == (family,) + assert load_family_members(archive_root / "family_members/us.jsonl") == members + assert load_family_actions(archive_root / "family_actions/us.jsonl") == (action,) + + imported_spool = tmp_path / "imported-spool" + assert ( + cli.main( + [ + "family-import", + "--scope", + "us", + "--archive-root", + str(archive_root), + "--spool", + str(imported_spool), + ] + ) + == 0 + ) + assert len(list(imported_spool.rglob("*.json"))) == 6 + capsys.readouterr() + + assert ( + cli.main( + [ + "list-families", + "--archive-root", + str(archive_root), + ] + ) + == 0 + ) + assert json.loads(capsys.readouterr().out)["family_id"] == family_id + + assert ( + cli.main( + [ + "list-family-builds", + "--family-id", + family_id, + "--archive-root", + str(archive_root), + ] + ) + == 0 + ) + build_rows = [ + json.loads(line) for line in capsys.readouterr().out.splitlines() if line + ] + assert [row["build_id"] for row in build_rows] == [ + "family-small", + "family-large", + ] + assert [row["requested_k"] for row in build_rows] == [20_000, 57_240] + assert all( + "gate_verdicts" not in row and "cost_usd" not in row for row in build_rows + ) + + assert ( + cli.main( + [ + "show-family-history", + "--family-id", + family_id, + "--archive-root", + str(archive_root), + ] + ) + == 0 + ) + assert json.loads(capsys.readouterr().out) == action.to_mapping() + + +def test_family_import_rejects_member_whose_build_has_another_scope( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + family_id = "12345678-1234-4234-9234-123456789abc" + family = LogbookFamily.create( + family_id=family_id, + chain_scope="us", + source_pool_sha256="a" * 64, + ) + member = FamilyMember.create(family_id=family_id, build_id="uk-build") + source_spool = tmp_path / "source-spool" + record_family(family, spool_dir=source_spool, post_remote=False) + record_family_member(member, spool_dir=source_spool, post_remote=False) + archive_root = tmp_path / "logbook" + archive_root.mkdir() + cli = _load_cli() + assert ( + cli.main( + [ + "family-export", + "--scope", + "us", + "--archive-root", + str(archive_root), + "--source", + str(source_spool), + ] + ) + == 0 + ) + _write_jsonl( + archive_root / "us.jsonl", + ( + _row( + "uk-build", + predecessor=None, + minute=1, + pipeline="uk-frs-staging", + ), + ), + ) + capsys.readouterr() + + destination = tmp_path / "destination" + assert ( + cli.main( + [ + "family-import", + "--scope", + "us", + "--archive-root", + str(archive_root), + "--spool", + str(destination), + ] + ) + == 1 + ) + assert "does not match family scope" in capsys.readouterr().err + assert not destination.exists() + + def test_cli_export_appends_jsonl_source_suffix_idempotently( tmp_path: Path, capsys: pytest.CaptureFixture[str], @@ -256,9 +511,7 @@ def test_cli_local_export_refuses_wrong_scope_rows( _write_jsonl(source, _chain(pipeline="uk-locals-rowwise")) cli = _load_cli() - exit_code = cli.main( - ["export", "--archive", str(archive), "--source", str(source)] - ) + exit_code = cli.main(["export", "--archive", str(archive), "--source", str(source)]) assert exit_code == 1 err = capsys.readouterr().err @@ -280,9 +533,7 @@ def test_cli_export_refuses_an_unratified_scope_archive( _write_jsonl(source, _chain(pipeline="uk-firms-staging")) cli = _load_cli() - exit_code = cli.main( - ["export", "--archive", str(archive), "--source", str(source)] - ) + exit_code = cli.main(["export", "--archive", str(archive), "--source", str(source)]) assert exit_code == 1 assert "not in the ratified scope list" in capsys.readouterr().err @@ -381,6 +632,23 @@ def __exit__(self, *_args: object) -> None: return None +class _RemoteMappingResponse: + status = 206 + + def __init__(self, rows: list[dict[str, object]]) -> None: + self._payload = json.dumps(rows).encode() + self.headers = {"Content-Range": f"0-{len(rows) - 1}/{len(rows)}"} + + def read(self) -> bytes: + return self._payload + + def __enter__(self) -> _RemoteMappingResponse: + return self + + def __exit__(self, *_args: object) -> None: + return None + + def test_cli_remote_export_uses_distinct_read_only_key( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, @@ -419,6 +687,71 @@ def fake_urlopen(request: object, *, timeout: float) -> _RemoteResponse: ] +def test_cli_remote_family_export_filters_each_table_by_stored_scope( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + family_id = "12345678-1234-4234-9234-123456789abc" + action_id = "22345678-1234-4234-9234-123456789abc" + family = LogbookFamily.create( + family_id=family_id, + chain_scope="us", + source_pool_sha256="a" * 64, + ) + member = FamilyMember.create(family_id=family_id, build_id="family-build") + action = FamilyAction.create( + action_id=action_id, + family_id=family_id, + build_id=member.build_id, + action_type="revokes", + related_build_id=None, + recorded_at="2026-08-21T12:00:00Z", + actor="fixture", + reason="Invalid output", + evidence_location=None, + ) + rows_by_table = { + "families": [family.to_mapping()], + "family_members_public": [member.to_mapping()], + "family_actions_public": [action.to_mapping()], + } + cli = _load_cli() + monkeypatch.setenv("POPULACE_LEDGER_URL", "https://fixture.supabase.co") + monkeypatch.setenv("POPULACE_LEDGER_EXPORT_KEY", "exporter-jwt") + monkeypatch.setenv("POPULACE_LEDGER_API_KEY", "project-api-key") + requests: list[object] = [] + + def fake_urlopen(request: object, *, timeout: float) -> _RemoteMappingResponse: + assert timeout == 30.0 + requests.append(request) + table = urlparse(request.full_url).path.rsplit("/", 1)[-1] + return _RemoteMappingResponse(rows_by_table[table]) + + monkeypatch.setattr(cli, "urlopen", fake_urlopen) + archive_root = tmp_path / "logbook" + + assert ( + cli.main( + [ + "family-export", + "--scope", + "us", + "--archive-root", + str(archive_root), + "--remote", + ] + ) + == 0 + ) + + assert load_families(archive_root / "families/us.jsonl") == (family,) + assert len(requests) == 3 + for request in requests: + query = parse_qs(urlparse(request.full_url).query) + assert query["chain_scope"] == ["eq.us"] + assert request.headers["Authorization"] == "Bearer exporter-jwt" + + def test_cli_remote_export_filters_nested_scope( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, diff --git a/packages/microcosm-build/tests/test_logbook_family.py b/packages/microcosm-build/tests/test_logbook_family.py new file mode 100644 index 000000000..cb2029b41 --- /dev/null +++ b/packages/microcosm-build/tests/test_logbook_family.py @@ -0,0 +1,510 @@ +"""Contracts for Logbook dataset-family records.""" + +from __future__ import annotations + +import json +from pathlib import Path +from urllib.parse import parse_qs, urlparse + +import pytest + +import microcosm.build.logbook as logbook +import microcosm.build.logbook_family as family_module +from microcosm.build.logbook import LogbookRow, record_build_attempt +from microcosm.build.logbook_family import ( + FamilyAction, + FamilyMember, + LogbookFamily, + export_family_records, + export_family_scope, + family_archive_path, + import_family_scope, + load_families, + load_family_archive_records, + reconcile_family_spool, + reconcile_logbook_spool, + record_family, + record_family_action, + record_family_member, + validate_family_action, + validate_family_membership, + validate_family_source, +) + +FAMILY_ID = "12345678-1234-4234-9234-123456789abc" +ACTION_ID = "87654321-4321-4321-8321-cba987654321" + + +@pytest.fixture(autouse=True) +def _spool_only(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("POPULACE_LEDGER_URL", raising=False) + monkeypatch.delenv("POPULACE_LEDGER_KEY", raising=False) + monkeypatch.delenv("POPULACE_LEDGER_API_KEY", raising=False) + + +def _family(**overrides: object) -> LogbookFamily: + values = { + "family_id": FAMILY_ID, + "chain_scope": "us", + "source_pool_sha256": "a" * 64, + } + values.update(overrides) + return LogbookFamily.create(**values) + + +def _build( + build_id: str = "family-build-1", + *, + pipeline: str = "us-stacked-pool", + requested_k: int = 20_000, +) -> LogbookRow: + return LogbookRow.create( + build_id=build_id, + ts="2026-08-21T12:00:00Z", + pipeline=pipeline, + rung="f100", + seed=17, + code_pin="abc1234", + input_pins_digest="1" * 64, + identity_digest="2" * 64, + phases_reached=["built"], + gate_verdicts={ + "build_validation": { + "verdict": "passed", + "receipt": "receipt://fixture.json", + } + }, + wall_seconds=1.0, + cost_usd=None, + artifact_location=None, + disposition="iterating", + prediction_id=None, + prev_row_digest=None, + row_format_version=2, + requested_k=requested_k, + realized_k=requested_k, + record_unit="household", + ) + + +def _action(**overrides: object) -> FamilyAction: + values = { + "action_id": ACTION_ID, + "family_id": FAMILY_ID, + "build_id": "family-build-2", + "action_type": "supersedes", + "related_build_id": "family-build-1", + "recorded_at": "2026-08-21T12:30:00+00:00", + "actor": "anthony", + "reason": "Corrected build", + "evidence_location": None, + } + values.update(overrides) + return FamilyAction.create(**values) + + +def test_family_and_membership_have_exact_fields() -> None: + family = _family() + member = FamilyMember.create( + family_id=family.family_id, + build_id="family-build-1", + ) + + assert family.to_mapping() == { + "family_id": FAMILY_ID, + "chain_scope": "us", + "source_pool_sha256": "a" * 64, + } + assert member.to_mapping() == { + "family_id": FAMILY_ID, + "build_id": "family-build-1", + } + + +@pytest.mark.parametrize( + ("overrides", "message"), + [ + ({"family_id": "not-a-uuid"}, "canonical UUID"), + ({"family_id": FAMILY_ID.upper()}, "canonical lowercase"), + ({"chain_scope": "uk/locals"}, "chain_scope"), + ({"source_pool_sha256": "A" * 64}, "source_pool_sha256"), + ], +) +def test_family_validation_rejects_invalid_identity( + overrides: dict[str, object], + message: str, +) -> None: + with pytest.raises(ValueError, match=message): + _family(**overrides) + + +def test_membership_validates_family_build_and_scope() -> None: + family = _family() + member = FamilyMember.create( + family_id=FAMILY_ID, + build_id="family-build-1", + ) + + validate_family_source(family, "a" * 64) + validate_family_membership(family, member, _build()) + + with pytest.raises(ValueError, match="identifies source"): + validate_family_source(family, "b" * 64) + with pytest.raises(ValueError, match="does not match family scope"): + validate_family_membership( + family, + member, + _build(pipeline="uk-frs-staging"), + ) + + +@pytest.mark.parametrize( + ("overrides", "message"), + [ + ({"action_type": "publishes"}, "action_type"), + ( + {"action_type": "revokes", "related_build_id": "family-build-1"}, + "must not contain", + ), + ({"related_build_id": None}, "requires related_build_id"), + ({"related_build_id": "family-build-2"}, "cannot supersede itself"), + ({"actor": " "}, "actor"), + ({"recorded_at": "2026-08-21T12:30:00"}, "UTC offset"), + ], +) +def test_family_action_shape_rejects_invalid_values( + overrides: dict[str, object], + message: str, +) -> None: + with pytest.raises(ValueError, match=message): + _action(**overrides) + + +def test_supersession_requires_members_and_matching_cardinality() -> None: + action = _action() + members = ( + FamilyMember.create(family_id=FAMILY_ID, build_id="family-build-1"), + FamilyMember.create(family_id=FAMILY_ID, build_id="family-build-2"), + ) + builds = { + "family-build-1": _build("family-build-1"), + "family-build-2": _build("family-build-2"), + } + + validate_family_action(action, members=members, builds=builds) + + mismatched = { + **builds, + "family-build-2": _build("family-build-2", requested_k=57_240), + } + with pytest.raises(ValueError, match="matching requested_k"): + validate_family_action(action, members=members, builds=mismatched) + + +def test_family_spool_is_durable_and_idempotent(tmp_path: Path) -> None: + family = _family() + + first = record_family(family, spool_dir=tmp_path) + second = record_family(family, spool_dir=tmp_path) + + assert first == second + assert first.spool_path == tmp_path / "families" / f"{FAMILY_ID}.json" + assert json.loads(first.spool_path.read_text()) == family.to_mapping() + + with pytest.raises(ValueError, match="divergent retry"): + record_family( + _family(source_pool_sha256="b" * 64), + spool_dir=tmp_path, + ) + + +def test_family_spool_retry_completes_interrupted_parent_sync( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + real_sync = logbook._fsync_parent_directory + calls = 0 + + def fail_first(path: Path) -> None: + nonlocal calls + calls += 1 + if calls == 1: + raise OSError("injected directory sync failure") + real_sync(path) + + monkeypatch.setattr(logbook, "_fsync_parent_directory", fail_first) + + with pytest.raises(OSError, match="injected directory sync failure"): + record_family(_family(), spool_dir=tmp_path) + persisted = tmp_path / "families" / f"{FAMILY_ID}.json" + assert persisted.exists() + + retry = record_family(_family(), spool_dir=tmp_path) + + assert retry.spool_path == persisted + assert calls == 2 + + +class _Response: + status = 201 + + def __enter__(self) -> _Response: + return self + + def __exit__(self, *_args: object) -> None: + return None + + +def test_remote_family_insert_occurs_after_local_spool( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + monkeypatch.setenv("POPULACE_LEDGER_URL", "https://fixture.supabase.co") + monkeypatch.setenv("POPULACE_LEDGER_KEY", "writer-jwt") + monkeypatch.setenv("POPULACE_LEDGER_API_KEY", "project-key") + requests = [] + + def fake_urlopen(request: object, *, timeout: float) -> _Response: + assert timeout == 3.0 + assert (tmp_path / "families" / f"{FAMILY_ID}.json").exists() + requests.append(request) + return _Response() + + monkeypatch.setattr(family_module, "urlopen", fake_urlopen) + + result = record_family(_family(), spool_dir=tmp_path, timeout=3.0) + + assert result.posted is True + request = requests[0] + parsed = urlparse(request.full_url) + assert parsed.path == "/rest/v1/families" + assert parse_qs(parsed.query)["on_conflict"] == ["family_id"] + assert request.headers["Content-profile"] == "logbook" + assert json.loads(request.data) == _family().to_mapping() + + +def test_combined_reconciliation_preserves_dependency_order( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + build_result = record_build_attempt( + **{ + key: value + for key, value in _build().to_mapping().items() + if key not in {"row_digest", "row_format_version"} + }, + row_format_version=2, + spool_dir=tmp_path, + ) + family = _family() + member = FamilyMember.create( + family_id=FAMILY_ID, + build_id=build_result.row.build_id, + ) + action = FamilyAction.create( + action_id=ACTION_ID, + family_id=FAMILY_ID, + build_id=build_result.row.build_id, + action_type="revokes", + related_build_id=None, + recorded_at="2026-08-21T12:30:00Z", + actor="anthony", + reason="Invalid artifact", + evidence_location=None, + ) + record_family(family, spool_dir=tmp_path, post_remote=False) + record_family_member(member, spool_dir=tmp_path, post_remote=False) + record_family_action(action, spool_dir=tmp_path, post_remote=False) + monkeypatch.setenv("POPULACE_LEDGER_URL", "https://fixture.supabase.co") + monkeypatch.setenv("POPULACE_LEDGER_KEY", "writer-jwt") + posted_paths: list[str] = [] + + def fake_urlopen(request: object, *, timeout: float) -> _Response: + del timeout + posted_paths.append(urlparse(request.full_url).path) + return _Response() + + monkeypatch.setattr(logbook, "urlopen", fake_urlopen) + monkeypatch.setattr(family_module, "urlopen", fake_urlopen) + + receipt = reconcile_logbook_spool(tmp_path) + + assert receipt.builds.errors == () + assert receipt.families.errors == () + assert posted_paths == [ + "/rest/v1/builds", + "/rest/v1/families", + "/rest/v1/family_members", + "/rest/v1/family_actions", + ] + assert list(tmp_path.rglob("*.json")) == [] + + +def test_family_reconciliation_stops_and_retains_dependents( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + family = _family() + member = FamilyMember.create( + family_id=FAMILY_ID, + build_id="family-build-1", + ) + record_family(family, spool_dir=tmp_path, post_remote=False) + record_family_member(member, spool_dir=tmp_path, post_remote=False) + monkeypatch.setenv("POPULACE_LEDGER_URL", "https://fixture.supabase.co") + monkeypatch.setenv("POPULACE_LEDGER_KEY", "writer-jwt") + + def fail(*_args: object, **_kwargs: object) -> _Response: + raise OSError("offline") + + monkeypatch.setattr(family_module, "urlopen", fail) + + receipt = reconcile_family_spool(tmp_path) + + assert receipt.attempted == 1 + assert receipt.posted == 0 + assert receipt.retained == 2 + assert "offline" in receipt.errors[0] + assert len(list(tmp_path.rglob("*.json"))) == 2 + + +def test_family_archive_append_is_idempotent_and_detects_conflict( + tmp_path: Path, +) -> None: + archive = tmp_path / "families" / "us.jsonl" + family = _family() + + first = export_family_records(archive, [family]) + second = export_family_records(archive, [family]) + + assert (first.existing, first.appended) == (0, 1) + assert (second.existing, second.appended) == (1, 0) + assert load_families(archive) == (family,) + + with pytest.raises(ValueError, match="conflicts with existing"): + export_family_records( + archive, + [_family(source_pool_sha256="b" * 64)], + ) + + +@pytest.mark.parametrize( + ("scope", "relative_path"), + [ + ("us", "families/us.jsonl"), + ("uk/frs", "families/uk/frs.jsonl"), + ], +) +def test_family_scope_archives_use_scope_specific_paths( + tmp_path: Path, + scope: str, + relative_path: str, +) -> None: + assert family_archive_path(tmp_path, "families", scope) == ( + tmp_path / relative_path + ) + + +def test_family_scope_archive_exports_and_imports_in_dependency_order( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + family = _family() + members = ( + FamilyMember.create(family_id=FAMILY_ID, build_id="family-build-1"), + FamilyMember.create(family_id=FAMILY_ID, build_id="family-build-2"), + ) + action = _action() + archive_root = tmp_path / "logbook" + + first = export_family_scope( + archive_root, + scope="us", + families=(family,), + family_members=members, + family_actions=(action,), + ) + second = export_family_scope( + archive_root, + scope="us", + families=(family,), + family_members=members, + family_actions=(action,), + ) + no_new_actions = export_family_scope( + archive_root, + scope="us", + families=(family,), + family_members=members, + ) + + assert {key: value.appended for key, value in first.items()} == { + "families": 1, + "family_members": 2, + "family_actions": 1, + } + assert all(value.appended == 0 for value in second.values()) + assert no_new_actions["family_actions"] == family_module.FamilyExportResult( + existing=1, + appended=0, + ) + assert load_family_archive_records(archive_root, "us").families == (family,) + + calls: list[str] = [] + real_family = family_module.record_family + real_member = family_module.record_family_member + real_action = family_module.record_family_action + + def track_family(*args: object, **kwargs: object): + calls.append("families") + return real_family(*args, **kwargs) + + def track_member(*args: object, **kwargs: object): + calls.append("family_members") + return real_member(*args, **kwargs) + + def track_action(*args: object, **kwargs: object): + calls.append("family_actions") + return real_action(*args, **kwargs) + + monkeypatch.setattr(family_module, "record_family", track_family) + monkeypatch.setattr(family_module, "record_family_member", track_member) + monkeypatch.setattr(family_module, "record_family_action", track_action) + + imported = import_family_scope( + archive_root, + scope="us", + spool_dir=tmp_path / "spool", + ) + + assert imported.family_actions == (action,) + assert calls == [ + "families", + "family_members", + "family_members", + "family_actions", + ] + + +def test_family_scope_archive_rejects_cross_scope_and_missing_dependencies( + tmp_path: Path, +) -> None: + with pytest.raises(ValueError, match="another scope"): + export_family_scope( + tmp_path, + scope="us", + families=(_family(chain_scope="uk/frs"),), + ) + assert not list(tmp_path.rglob("*.jsonl")) + + member = FamilyMember.create( + family_id=FAMILY_ID, + build_id="family-build-1", + ) + with pytest.raises(ValueError, match="missing family"): + export_family_scope( + tmp_path, + scope="us", + family_members=(member,), + ) + assert not list(tmp_path.rglob("*.jsonl")) diff --git a/packages/microcosm-build/tests/test_us_exact_k_ladder_launcher.py b/packages/microcosm-build/tests/test_us_exact_k_ladder_launcher.py index 5f600b70c..3a318bcbf 100644 --- a/packages/microcosm-build/tests/test_us_exact_k_ladder_launcher.py +++ b/packages/microcosm-build/tests/test_us_exact_k_ladder_launcher.py @@ -5,9 +5,29 @@ import json import sys from pathlib import Path +from urllib.parse import urlparse import pytest +import microcosm.build.logbook as logbook_module +import microcosm.build.logbook_family as family_module +from microcosm.build.logbook import load_spool_rows +from microcosm.build.logbook_family import ( + LogbookFamily, + load_family_spool, + record_family, +) + +FAMILY_ID = "12345678-1234-4234-9234-123456789abc" + + +@pytest.fixture(autouse=True) +def _isolated_logbook_environment(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("POPULACE_LEDGER_URL", raising=False) + monkeypatch.delenv("POPULACE_LEDGER_KEY", raising=False) + monkeypatch.delenv("POPULACE_LEDGER_API_KEY", raising=False) + monkeypatch.delenv("POPULACE_LOGBOOK_PREV_ROW_DIGEST", raising=False) + def _launcher_module(): tools = Path(__file__).resolve().parents[3] / "tools" @@ -27,7 +47,8 @@ def _config_payload( release_id: str = "populace-us-2024-k8-fixture", ) -> dict[str, object]: return { - "schema_version": 1, + "schema_version": 2, + "family": {"id": FAMILY_ID}, "pool": { "release_id": "fixture-publication", "manifest_sha256": pool_manifest_sha256, @@ -82,6 +103,12 @@ def test_config_requires_explicit_seed_and_ratified_k(tmp_path: Path) -> None: with pytest.raises(ValueError, match="and its SHA-256 pin"): launcher._read_config(path) + invalid_family = _config_payload() + invalid_family["family"]["id"] = "not-a-uuid" + path = _write_config(tmp_path, invalid_family) + with pytest.raises(ValueError, match="family.id must be a canonical UUID"): + launcher._read_config(path) + def test_config_rejects_k_larger_than_manifest_pool( tmp_path: Path, monkeypatch: pytest.MonkeyPatch @@ -298,10 +325,272 @@ def fake_builder(argv): ] assert "--artifact-root" in result["publish_argv"] assert "--repo-id policyengine/populace-us" in result["publish_command"] + assert result["family_id"] == FAMILY_ID + assert result["requested_k"] == 8 + assert result["realized_k"] == 8 + assert result["record_unit"] == "household" + assert result["rung"] is None assert json.loads((tmp_path / "out" / "package_result.json").read_text()) == ( result ) + rows = load_spool_rows(tmp_path / "out" / "logbook-spool") + assert len(rows) == 1 + assert rows[0].rung is None + assert rows[0].requested_k == 8 + assert rows[0].realized_k == 8 + assert rows[0].record_unit == "household" + assert rows[0].disposition == "iterating" + family_records = load_family_spool(tmp_path / "out" / "logbook-spool") + assert family_records.families == ( + LogbookFamily.create( + family_id=FAMILY_ID, + chain_scope="us", + source_pool_sha256=_sha256(manifest), + ), + ) + assert [member.build_id for member in family_records.family_members] == [ + "populace-us-2024-k8-fixture" + ] + + +def test_reduced_build_records_numeric_cardinality( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + launcher = _launcher_module() + manifest = tmp_path / "pool.manifest.json" + manifest.write_text("fixture", encoding="utf-8") + config_path = _write_config( + tmp_path, + _config_payload( + pool_manifest_sha256=_sha256(manifest), + requested_k=20_000, + release_id="populace-us-2024-k20000-fixture", + ), + ) + monkeypatch.setattr( + launcher, + "_validate_pins_and_resolve_k", + lambda **_: (20_000, {}), + ) + + result = launcher.launch( + pool_manifest=manifest, + config_path=config_path, + out=tmp_path / "out", + release_builder=lambda _argv: None, + ) + + row = load_spool_rows(tmp_path / "out" / "logbook-spool")[0] + assert (result["requested_k"], result["realized_k"]) == (20_000, 20_000) + assert (row.requested_k, row.realized_k, row.rung) == (20_000, 20_000, None) + + +def test_failure_before_n_resolution_records_null_cardinality( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + launcher = _launcher_module() + manifest = tmp_path / "pool.manifest.json" + manifest.write_text("fixture", encoding="utf-8") + config_path = _write_config( + tmp_path, + _config_payload(pool_manifest_sha256=_sha256(manifest)), + ) + + def fail_validation(**_kwargs): + raise ValueError("fixture manifest failure") + + monkeypatch.setattr(launcher, "_validate_pins_and_resolve_k", fail_validation) + + with pytest.raises(ValueError, match="fixture manifest failure"): + launcher.launch( + pool_manifest=manifest, + config_path=config_path, + out=tmp_path / "out", + release_builder=lambda _argv: pytest.fail("builder must not run"), + ) + + row = load_spool_rows(tmp_path / "out" / "logbook-spool")[0] + assert (row.rung, row.requested_k, row.realized_k, row.record_unit) == ( + None, + None, + None, + None, + ) + assert row.disposition == "failed" + family_records = load_family_spool(tmp_path / "out" / "logbook-spool") + assert family_records.families == () + assert family_records.family_members == () + + +def test_failure_after_numeric_resolution_retains_request_without_membership( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + launcher = _launcher_module() + manifest = tmp_path / "pool.manifest.json" + manifest.write_text("fixture", encoding="utf-8") + config_path = _write_config( + tmp_path, + _config_payload( + pool_manifest_sha256=_sha256(manifest), + requested_k=20_000, + release_id="populace-us-2024-k20000-fixture", + ), + ) + monkeypatch.setattr( + launcher, + "_validate_pins_and_resolve_k", + lambda **_: (20_000, {}), + ) + + def fail_builder(_argv): + raise RuntimeError("fixture build failure") + + with pytest.raises(RuntimeError, match="fixture build failure"): + launcher.launch( + pool_manifest=manifest, + config_path=config_path, + out=tmp_path / "out", + release_builder=fail_builder, + ) + + row = load_spool_rows(tmp_path / "out" / "logbook-spool")[0] + assert (row.rung, row.requested_k, row.realized_k, row.record_unit) == ( + None, + 20_000, + None, + "household", + ) + family_records = load_family_spool(tmp_path / "out" / "logbook-spool") + assert len(family_records.families) == 1 + assert family_records.family_members == () + + +def test_matching_family_retry_is_accepted_and_mismatched_source_is_rejected( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + launcher = _launcher_module() + manifest = tmp_path / "pool.manifest.json" + manifest.write_text("fixture", encoding="utf-8") + source_sha256 = _sha256(manifest) + config_path = _write_config( + tmp_path, + _config_payload(pool_manifest_sha256=source_sha256), + ) + monkeypatch.setattr( + launcher, + "_validate_pins_and_resolve_k", + lambda **_: (8, {}), + ) + matching_out = tmp_path / "matching" + record_family( + LogbookFamily.create( + family_id=FAMILY_ID, + chain_scope="us", + source_pool_sha256=source_sha256, + ), + spool_dir=matching_out / "logbook-spool", + post_remote=False, + ) + + launcher.launch( + pool_manifest=manifest, + config_path=config_path, + out=matching_out, + release_builder=lambda _argv: None, + ) + assert len(load_family_spool(matching_out / "logbook-spool").families) == 1 + + mismatched_out = tmp_path / "mismatched" + record_family( + LogbookFamily.create( + family_id=FAMILY_ID, + chain_scope="us", + source_pool_sha256="f" * 64, + ), + spool_dir=mismatched_out / "logbook-spool", + post_remote=False, + ) + builder_called = False + + def unexpected_builder(_argv): + nonlocal builder_called + builder_called = True + + with pytest.raises(ValueError, match="divergent retry"): + launcher.launch( + pool_manifest=manifest, + config_path=config_path, + out=mismatched_out, + release_builder=unexpected_builder, + ) + + assert builder_called is False + mismatched_records = load_family_spool(mismatched_out / "logbook-spool") + assert mismatched_records.families[0].source_pool_sha256 == "f" * 64 + assert mismatched_records.family_members == () + + +def test_exact_k_spools_all_records_before_dependency_ordered_remote_delivery( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + launcher = _launcher_module() + manifest = tmp_path / "pool.manifest.json" + manifest.write_text("fixture", encoding="utf-8") + config_path = _write_config( + tmp_path, + _config_payload(pool_manifest_sha256=_sha256(manifest)), + ) + monkeypatch.setattr( + launcher, + "_validate_pins_and_resolve_k", + lambda **_: (8, {}), + ) + monkeypatch.setenv("POPULACE_LEDGER_URL", "https://fixture.supabase.co") + monkeypatch.setenv("POPULACE_LEDGER_KEY", "writer-jwt") + spool = tmp_path / "out" / "logbook-spool" + posted_paths: list[str] = [] + + class Response: + status = 201 + + def __enter__(self) -> Response: + return self + + def __exit__(self, *_args: object) -> None: + return None + + def fake_urlopen(request: object, *, timeout: float) -> Response: + assert timeout == 10.0 + if not posted_paths: + assert len(list(spool.glob("*.json"))) == 1 + assert len(list((spool / "families").glob("*.json"))) == 1 + assert len(list((spool / "family_members").glob("*.json"))) == 1 + posted_paths.append(urlparse(request.full_url).path) + return Response() + + monkeypatch.setattr(logbook_module, "urlopen", fake_urlopen) + monkeypatch.setattr(family_module, "urlopen", fake_urlopen) + + launcher.launch( + pool_manifest=manifest, + config_path=config_path, + out=tmp_path / "out", + release_builder=lambda _argv: None, + ) + + assert posted_paths == [ + "/rest/v1/builds", + "/rest/v1/families", + "/rest/v1/family_members", + ] + assert list(spool.rglob("*.json")) == [] + def test_pool_release_id_must_match_authenticated_manifest_identity( tmp_path: Path, diff --git a/supabase/migrations/20260821000000_logbook_family_model.sql b/supabase/migrations/20260821000000_logbook_family_model.sql new file mode 100644 index 000000000..46702d53f --- /dev/null +++ b/supabase/migrations/20260821000000_logbook_family_model.sql @@ -0,0 +1,726 @@ +-- Add authenticated build cardinality and relational dataset-family records. +-- +-- This migration depends on 20260818000000_logbook_chain_scopes.sql. It does +-- not replace chain_scope(), scope_declared(), or enforce_build_chain(): the +-- current one-predecessor-per-scope behavior remains the build insertion rule. + +ALTER TABLE logbook.builds + ADD COLUMN row_format_version smallint, + ADD COLUMN requested_k bigint, + ADD COLUMN realized_k bigint, + ADD COLUMN record_unit text; + +ALTER TABLE logbook.builds + ALTER COLUMN rung DROP NOT NULL, + DROP CONSTRAINT builds_rung_fraction_token; + +ALTER TABLE logbook.builds + ADD CONSTRAINT builds_rung_by_row_format CHECK ( + ( + row_format_version IS NULL + AND rung IS NOT NULL + AND rung IN ('f001', 'f004', 'f010', 'f025', 'f100') + ) + OR ( + row_format_version IS NOT DISTINCT FROM 2 + AND ( + rung IS NULL + OR rung IN ('f001', 'f004', 'f010', 'f025', 'f100') + ) + ) + ), + ADD CONSTRAINT builds_row_format_and_cardinality CHECK ( + ( + row_format_version IS NULL + AND requested_k IS NULL + AND realized_k IS NULL + AND record_unit IS NULL + ) + OR ( + row_format_version IS NOT DISTINCT FROM 2 + AND (requested_k IS NULL OR requested_k > 0) + AND (realized_k IS NULL OR realized_k > 0) + AND ( + ( + requested_k IS NULL + AND realized_k IS NULL + AND record_unit IS NULL + ) + OR ( + (requested_k IS NOT NULL OR realized_k IS NOT NULL) + AND record_unit IS NOT NULL + AND logbook.nonempty_trimmed_text(record_unit) + AND record_unit = lower(record_unit) + AND record_unit ~ '^[a-z][a-z0-9_]*$' + ) + ) + ) + ), + ADD CONSTRAINT builds_published_cardinality_matches CHECK ( + disposition NOT IN ('published', 'certified') + OR requested_k IS NULL + OR ( + realized_k IS NOT NULL + AND realized_k = requested_k + ) + ); + +CREATE OR REPLACE FUNCTION logbook.build_hash_payload( + p_build logbook.builds +) +RETURNS jsonb +LANGUAGE plpgsql +STABLE +STRICT +SET search_path = pg_catalog, logbook +AS $function$ +DECLARE + payload jsonb; +BEGIN + payload := jsonb_build_object( + 'artifact_location', p_build.artifact_location, + 'build_id', p_build.build_id, + 'code_pin', p_build.code_pin, + 'cost_usd', p_build.cost_usd, + 'disposition', p_build.disposition::text, + 'gate_verdicts', p_build.gate_verdicts, + 'identity_digest', p_build.identity_digest, + 'input_pins_digest', p_build.input_pins_digest, + 'phases_reached', p_build.phases_reached, + 'pipeline', p_build.pipeline, + 'prediction_id', p_build.prediction_id, + 'rung', p_build.rung, + 'seed', p_build.seed, + 'ts', to_char( + p_build.ts AT TIME ZONE 'UTC', + 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"' + ), + 'wall_seconds', p_build.wall_seconds + ); + + IF p_build.row_format_version IS NULL THEN + IF p_build.requested_k IS NOT NULL + OR p_build.realized_k IS NOT NULL + OR p_build.record_unit IS NOT NULL + THEN + RAISE EXCEPTION + 'Legacy Logbook build % has version-2 cardinality values', + p_build.build_id + USING ERRCODE = '23514'; + END IF; + RETURN payload; + END IF; + + IF p_build.row_format_version = 2 THEN + RETURN payload || jsonb_build_object( + 'realized_k', p_build.realized_k, + 'record_unit', p_build.record_unit, + 'requested_k', p_build.requested_k, + 'row_format_version', p_build.row_format_version + ); + END IF; + + RAISE EXCEPTION + 'Unsupported Logbook row_format_version % for build %', + p_build.row_format_version, + p_build.build_id + USING ERRCODE = '23514'; +END; +$function$; + +CREATE OR REPLACE FUNCTION logbook.expected_build_row_digest( + p_build logbook.builds +) +RETURNS logbook.sha256_hex +LANGUAGE sql +STABLE +STRICT +SET search_path = pg_catalog, logbook, extensions +AS $function$ + SELECT encode( + extensions.digest( + convert_to( + logbook.canonical_json_text( + logbook.build_hash_payload(p_build) + ) || coalesce(p_build.prev_row_digest::text, ''), + 'UTF8' + ), + 'sha256' + ), + 'hex' + )::logbook.sha256_hex +$function$; + +CREATE OR REPLACE VIEW logbook.builds_public +WITH (security_barrier = true) +AS +SELECT + build_id, + ts, + pipeline, + rung, + seed, + code_pin, + input_pins_digest, + identity_digest, + phases_reached, + wall_seconds, + CASE + WHEN disposition IN ('published', 'certified') + THEN artifact_location + ELSE NULL + END AS artifact_location, + disposition, + prediction_id, + prev_row_digest, + row_digest, + requested_k, + realized_k, + record_unit +FROM logbook.builds; + +CREATE TYPE logbook.family_action_type AS ENUM ( + 'revokes', + 'supersedes' +); + +CREATE TABLE logbook.families ( + family_id uuid PRIMARY KEY, + chain_scope text NOT NULL CHECK ( + logbook.scope_declared(chain_scope) + ), + source_pool_sha256 logbook.sha256_hex NOT NULL, + CONSTRAINT families_scope_source_unique + UNIQUE (chain_scope, source_pool_sha256) +); + +CREATE TABLE logbook.family_members ( + family_id uuid NOT NULL, + build_id text NOT NULL, + CONSTRAINT family_members_pk PRIMARY KEY (family_id, build_id), + CONSTRAINT family_members_build_unique UNIQUE (build_id), + CONSTRAINT family_members_family_fk FOREIGN KEY (family_id) + REFERENCES logbook.families (family_id) + ON UPDATE RESTRICT + ON DELETE RESTRICT, + CONSTRAINT family_members_build_fk FOREIGN KEY (build_id) + REFERENCES logbook.builds (build_id) + ON UPDATE RESTRICT + ON DELETE RESTRICT +); + +CREATE TABLE logbook.family_actions ( + action_id uuid PRIMARY KEY, + family_id uuid NOT NULL, + build_id text NOT NULL, + action_type logbook.family_action_type NOT NULL, + related_build_id text, + recorded_at timestamptz NOT NULL CHECK (isfinite(recorded_at)), + actor text NOT NULL CHECK (logbook.nonempty_trimmed_text(actor)), + reason text NOT NULL CHECK (logbook.nonempty_trimmed_text(reason)), + evidence_location text CHECK ( + evidence_location IS NULL + OR logbook.nonempty_trimmed_text(evidence_location) + ), + CONSTRAINT family_actions_shape CHECK ( + ( + action_type = 'revokes' + AND related_build_id IS NULL + ) + OR ( + action_type = 'supersedes' + AND related_build_id IS NOT NULL + AND related_build_id <> build_id + ) + ), + CONSTRAINT family_actions_member_fk + FOREIGN KEY (family_id, build_id) + REFERENCES logbook.family_members (family_id, build_id) + ON UPDATE RESTRICT + ON DELETE RESTRICT, + CONSTRAINT family_actions_related_member_fk + FOREIGN KEY (family_id, related_build_id) + REFERENCES logbook.family_members (family_id, build_id) + ON UPDATE RESTRICT + ON DELETE RESTRICT +); + +CREATE UNIQUE INDEX family_actions_one_direct_replacement + ON logbook.family_actions (family_id, related_build_id) + WHERE action_type = 'supersedes'; + +CREATE INDEX families_scope_idx + ON logbook.families (chain_scope, family_id); +CREATE INDEX families_source_idx + ON logbook.families (source_pool_sha256, family_id); +CREATE INDEX family_members_build_idx + ON logbook.family_members (build_id, family_id); +CREATE INDEX family_actions_revocation_idx + ON logbook.family_actions (family_id, build_id, recorded_at) + WHERE action_type = 'revokes'; +CREATE INDEX family_actions_replacement_idx + ON logbook.family_actions (family_id, related_build_id, recorded_at) + WHERE action_type = 'supersedes'; + +CREATE OR REPLACE FUNCTION logbook.enforce_family_insert() +RETURNS trigger +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog, logbook +AS $function$ +DECLARE + by_id logbook.families%ROWTYPE; + by_source logbook.families%ROWTYPE; + found_id boolean := false; + found_source boolean := false; +BEGIN + PERFORM pg_advisory_xact_lock( + 628, + hashtext('family:' || NEW.family_id::text) + ); + + SELECT * + INTO by_id + FROM logbook.families + WHERE family_id = NEW.family_id; + found_id := FOUND; + + SELECT * + INTO by_source + FROM logbook.families + WHERE chain_scope = NEW.chain_scope + AND source_pool_sha256 = NEW.source_pool_sha256; + found_source := FOUND; + + IF found_id AND ( + by_id.chain_scope IS DISTINCT FROM NEW.chain_scope + OR by_id.source_pool_sha256 + IS DISTINCT FROM NEW.source_pool_sha256 + ) THEN + RAISE EXCEPTION + 'Logbook family_id % already exists with divergent content', + NEW.family_id + USING ERRCODE = '23505'; + END IF; + + IF found_source + AND by_source.family_id IS DISTINCT FROM NEW.family_id + THEN + RAISE EXCEPTION + 'Logbook source % in scope % already belongs to family %', + NEW.source_pool_sha256, + NEW.chain_scope, + by_source.family_id + USING ERRCODE = '23505'; + END IF; + + RETURN NEW; +END; +$function$; + +CREATE TRIGGER families_enforce_insert_before_insert +BEFORE INSERT ON logbook.families +FOR EACH ROW +EXECUTE FUNCTION logbook.enforce_family_insert(); + +CREATE OR REPLACE FUNCTION logbook.enforce_family_membership() +RETURNS trigger +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog, logbook +AS $function$ +DECLARE + family_scope text; + build_scope text; +BEGIN + SELECT family.chain_scope + INTO family_scope + FROM logbook.families AS family + WHERE family.family_id = NEW.family_id; + + IF NOT FOUND THEN + RAISE EXCEPTION + 'Logbook family % does not exist', + NEW.family_id + USING ERRCODE = '23503'; + END IF; + + SELECT logbook.chain_scope(build.pipeline) + INTO build_scope + FROM logbook.builds AS build + WHERE build.build_id = NEW.build_id; + + IF NOT FOUND THEN + RAISE EXCEPTION + 'Logbook build % does not exist', + NEW.build_id + USING ERRCODE = '23503'; + END IF; + + IF build_scope IS DISTINCT FROM family_scope THEN + RAISE EXCEPTION + 'Logbook build % scope % does not match family % scope %', + NEW.build_id, + coalesce(build_scope, 'missing'), + NEW.family_id, + family_scope + USING ERRCODE = '23514'; + END IF; + + RETURN NEW; +END; +$function$; + +CREATE TRIGGER family_members_enforce_before_insert +BEFORE INSERT ON logbook.family_members +FOR EACH ROW +EXECUTE FUNCTION logbook.enforce_family_membership(); + +CREATE OR REPLACE FUNCTION logbook.enforce_family_action() +RETURNS trigger +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog, logbook +AS $function$ +DECLARE + existing_action logbook.family_actions%ROWTYPE; + replacement_requested bigint; + replacement_unit text; + replaced_requested bigint; + replaced_unit text; +BEGIN + PERFORM pg_advisory_xact_lock( + 628, + hashtext('family-action:' || NEW.action_id::text) + ); + + SELECT * + INTO existing_action + FROM logbook.family_actions + WHERE action_id = NEW.action_id; + + IF FOUND THEN + IF existing_action.family_id IS DISTINCT FROM NEW.family_id + OR existing_action.build_id IS DISTINCT FROM NEW.build_id + OR existing_action.action_type IS DISTINCT FROM NEW.action_type + OR existing_action.related_build_id + IS DISTINCT FROM NEW.related_build_id + OR existing_action.recorded_at IS DISTINCT FROM NEW.recorded_at + OR existing_action.actor IS DISTINCT FROM NEW.actor + OR existing_action.reason IS DISTINCT FROM NEW.reason + OR existing_action.evidence_location + IS DISTINCT FROM NEW.evidence_location + THEN + RAISE EXCEPTION + 'Logbook action_id % already exists with divergent content', + NEW.action_id + USING ERRCODE = '23505'; + END IF; + RETURN NEW; + END IF; + + IF NEW.action_type = 'supersedes' THEN + SELECT build.requested_k, build.record_unit + INTO replacement_requested, replacement_unit + FROM logbook.family_members AS member + JOIN logbook.builds AS build + ON build.build_id = member.build_id + WHERE member.family_id = NEW.family_id + AND member.build_id = NEW.build_id; + + IF NOT FOUND THEN + RAISE EXCEPTION + 'Replacement build % is not a member of family %', + NEW.build_id, + NEW.family_id + USING ERRCODE = '23503'; + END IF; + + SELECT build.requested_k, build.record_unit + INTO replaced_requested, replaced_unit + FROM logbook.family_members AS member + JOIN logbook.builds AS build + ON build.build_id = member.build_id + WHERE member.family_id = NEW.family_id + AND member.build_id = NEW.related_build_id; + + IF NOT FOUND THEN + RAISE EXCEPTION + 'Replaced build % is not a member of family %', + NEW.related_build_id, + NEW.family_id + USING ERRCODE = '23503'; + END IF; + + IF replacement_requested IS DISTINCT FROM replaced_requested + OR replacement_unit IS DISTINCT FROM replaced_unit + THEN + RAISE EXCEPTION + 'Superseding builds must have matching requested_k and record_unit' + USING ERRCODE = '23514'; + END IF; + END IF; + + RETURN NEW; +END; +$function$; + +CREATE TRIGGER family_actions_enforce_before_insert +BEFORE INSERT ON logbook.family_actions +FOR EACH ROW +EXECUTE FUNCTION logbook.enforce_family_action(); + +CREATE VIEW logbook.families_public +WITH (security_barrier = true) +AS +SELECT + family_id, + chain_scope, + source_pool_sha256 +FROM logbook.families; + +CREATE VIEW logbook.family_members_public +WITH (security_barrier = true) +AS +SELECT + family.family_id, + family.chain_scope, + family.source_pool_sha256, + build.build_id, + build.ts, + build.pipeline, + build.rung, + build.seed, + build.code_pin, + build.input_pins_digest, + build.identity_digest, + build.phases_reached, + build.wall_seconds, + build.artifact_location, + build.disposition, + build.prediction_id, + build.prev_row_digest, + build.row_digest, + build.requested_k, + build.realized_k, + build.record_unit +FROM logbook.family_members AS member +JOIN logbook.families_public AS family + ON family.family_id = member.family_id +JOIN logbook.builds_public AS build + ON build.build_id = member.build_id; + +CREATE VIEW logbook.family_actions_public +WITH (security_barrier = true) +AS +SELECT + action.action_id, + action.family_id, + family.chain_scope, + action.build_id, + action.action_type, + action.related_build_id, + action.recorded_at, + action.actor, + action.reason, + action.evidence_location +FROM logbook.family_actions AS action +JOIN logbook.families AS family + ON family.family_id = action.family_id; + +CREATE VIEW logbook.family_member_status_public +WITH (security_barrier = true) +AS +SELECT + member.family_id, + member.build_id, + EXISTS ( + SELECT 1 + FROM logbook.family_actions AS action + WHERE action.family_id = member.family_id + AND action.build_id = member.build_id + AND action.action_type = 'revokes' + ) AS revoked, + ( + SELECT action.build_id + FROM logbook.family_actions AS action + WHERE action.family_id = member.family_id + AND action.related_build_id = member.build_id + AND action.action_type = 'supersedes' + ) AS superseded_by_build_id +FROM logbook.family_members AS member; + +REVOKE ALL ON + logbook.families, + logbook.family_members, + logbook.family_actions, + logbook.families_public, + logbook.family_members_public, + logbook.family_actions_public, + logbook.family_member_status_public +FROM PUBLIC; +REVOKE ALL ON FUNCTION + logbook.enforce_family_insert(), + logbook.enforce_family_membership(), + logbook.enforce_family_action() +FROM PUBLIC; + +GRANT USAGE ON TYPE logbook.family_action_type + TO logbook_writer, logbook_exporter, logbook_break_glass_admin; +GRANT INSERT ON + logbook.families, + logbook.family_members, + logbook.family_actions +TO logbook_writer; +GRANT SELECT (family_id) ON logbook.families + TO logbook_writer; +GRANT SELECT (family_id, build_id) ON logbook.family_members + TO logbook_writer; +GRANT SELECT (action_id) ON logbook.family_actions + TO logbook_writer; +GRANT SELECT ON + logbook.families, + logbook.family_members, + logbook.family_actions, + logbook.families_public, + logbook.family_members_public, + logbook.family_actions_public, + logbook.family_member_status_public +TO logbook_exporter; +GRANT SELECT, INSERT, UPDATE, DELETE ON + logbook.families, + logbook.family_members, + logbook.family_actions +TO logbook_break_glass_admin; +GRANT SELECT ON + logbook.families_public, + logbook.family_members_public, + logbook.family_actions_public, + logbook.family_member_status_public +TO logbook_break_glass_admin; + +ALTER TABLE logbook.families ENABLE ROW LEVEL SECURITY; +ALTER TABLE logbook.families FORCE ROW LEVEL SECURITY; +ALTER TABLE logbook.family_members ENABLE ROW LEVEL SECURITY; +ALTER TABLE logbook.family_members FORCE ROW LEVEL SECURITY; +ALTER TABLE logbook.family_actions ENABLE ROW LEVEL SECURITY; +ALTER TABLE logbook.family_actions FORCE ROW LEVEL SECURITY; + +CREATE POLICY families_writer_insert + ON logbook.families + FOR INSERT + TO logbook_writer + WITH CHECK (true); +CREATE POLICY families_writer_conflict_select + ON logbook.families + FOR SELECT + TO logbook_writer + USING (true); +CREATE POLICY family_members_writer_insert + ON logbook.family_members + FOR INSERT + TO logbook_writer + WITH CHECK (true); +CREATE POLICY family_members_writer_conflict_select + ON logbook.family_members + FOR SELECT + TO logbook_writer + USING (true); +CREATE POLICY family_actions_writer_insert + ON logbook.family_actions + FOR INSERT + TO logbook_writer + WITH CHECK (true); +CREATE POLICY family_actions_writer_conflict_select + ON logbook.family_actions + FOR SELECT + TO logbook_writer + USING (true); + +CREATE POLICY families_exporter_select + ON logbook.families + FOR SELECT + TO logbook_exporter + USING (true); +CREATE POLICY family_members_exporter_select + ON logbook.family_members + FOR SELECT + TO logbook_exporter + USING (true); +CREATE POLICY family_actions_exporter_select + ON logbook.family_actions + FOR SELECT + TO logbook_exporter + USING (true); + +CREATE POLICY families_break_glass_all + ON logbook.families + FOR ALL + TO logbook_break_glass_admin + USING (true) + WITH CHECK (true); +CREATE POLICY family_members_break_glass_all + ON logbook.family_members + FOR ALL + TO logbook_break_glass_admin + USING (true) + WITH CHECK (true); +CREATE POLICY family_actions_break_glass_all + ON logbook.family_actions + FOR ALL + TO logbook_break_glass_admin + USING (true) + WITH CHECK (true); + +DO $supabase_family_roles$ +DECLARE + api_role text; +BEGIN + FOREACH api_role IN ARRAY ARRAY['anon', 'authenticated'] + LOOP + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = api_role) THEN + EXECUTE format( + 'GRANT USAGE ON SCHEMA logbook TO %I', api_role + ); + EXECUTE format( + 'GRANT SELECT ON logbook.families_public, ' + 'logbook.family_members_public, ' + 'logbook.family_actions_public, ' + 'logbook.family_member_status_public TO %I', + api_role + ); + END IF; + END LOOP; + + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'service_role') THEN + EXECUTE + 'REVOKE ALL ON logbook.families, logbook.family_members, ' + 'logbook.family_actions, logbook.families_public, ' + 'logbook.family_members_public, ' + 'logbook.family_actions_public, ' + 'logbook.family_member_status_public FROM service_role'; + END IF; +END; +$supabase_family_roles$; + +COMMENT ON COLUMN logbook.builds.row_format_version IS + 'Logbook row representation version; NULL is the unchanged legacy shape.'; +COMMENT ON COLUMN logbook.builds.rung IS + 'Sampling-fraction category; NULL in version 2 for absolute-count builds.'; +COMMENT ON COLUMN logbook.builds.requested_k IS + 'Requested record cardinality after resolving any symbolic request.'; +COMMENT ON COLUMN logbook.builds.realized_k IS + 'Validated record cardinality in the completed dataset.'; +COMMENT ON COLUMN logbook.builds.record_unit IS + 'Normalized entity counted by requested_k and realized_k.'; +COMMENT ON TABLE logbook.families IS + 'Dataset families identified separately from their prepared-input checksum.'; +COMMENT ON TABLE logbook.family_members IS + 'Two-column association from a dataset family to an immutable build row.'; +COMMENT ON TABLE logbook.family_actions IS + 'Append-only revocation and direct-replacement decisions for family members.'; + +-- Reserved for the spec-engine run identity and deliberately absent here: +-- identity_generation, source_grammar_receipt, spec_binding, +-- authority_versions, code_inventory_digest, artifact_protocol_inventory, +-- run_request, execution_receipt, and schema_version. diff --git a/tools/build_us_exact_k_ladder_release.py b/tools/build_us_exact_k_ladder_release.py index 17668c5e6..e272b2fcf 100644 --- a/tools/build_us_exact_k_ladder_release.py +++ b/tools/build_us_exact_k_ladder_release.py @@ -13,10 +13,11 @@ --config configs/us_exact_k_57240.json \ --out build/us-exact-k -Schema-v1 configuration (paths are resolved relative to the config file):: +Schema-v2 configuration (paths are resolved relative to the config file):: { - "schema_version": 1, + "schema_version": 2, + "family": {"id": "12345678-1234-4234-9234-123456789abc"}, "pool": {"release_id": "...", "manifest_sha256": ""}, "ladder": {"k": 57240, "seed": 17, "pi_hi": 0.95}, "targets": { @@ -56,8 +57,11 @@ import re import shlex import sys +import time +import uuid from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass +from datetime import UTC, datetime from pathlib import Path _TOOLS_DIR = Path(__file__).resolve().parent @@ -65,13 +69,39 @@ sys.path.insert(0, str(_TOOLS_DIR)) import build_us_fiscal_refresh_release as fiscal_release +from microcosm.build.logbook import ( + LogbookWriteResult, + canonical_json_bytes, + record_build_attempt, +) +from microcosm.build.logbook_adoption import ( + AttemptState, + append_phase, + attempt_receipt_dir, + atomic_write_json, + error_receipt_path, + git_code_pin, + local_artifact_reference, + resolve_predecessor, + sha256_argument, + write_error_receipt, +) +from microcosm.build.logbook_family import ( + FamilyMember, + LogbookFamily, + reconcile_logbook_spool, + record_family, + record_family_member, +) from microcosm.build.us_runtime.h5_io import ( load_simulation_ready_us_multispine_pool_manifest, ) -CONFIG_SCHEMA_VERSION = 1 +CONFIG_SCHEMA_VERSION = 2 RATIFIED_SPARSE_K = fiscal_release.RATIFIED_EXACT_K_COUNTS US_RELEASE_REPO_ID = "policyengine/populace-us" +_LOGBOOK_PIPELINE = "us-2024-release" +_REPOSITORY_ROOT = Path(__file__).resolve().parents[1] _LOWERCASE_SHA256 = re.compile(r"[0-9a-f]{64}") _RELEASE_ID = re.compile(r"[A-Za-z0-9-]+") @@ -80,6 +110,7 @@ class LadderReleaseConfig: """Validated, path-resolved launcher configuration.""" + family_id: str pool_release_id: str pool_manifest_sha256: str requested_k: str | int @@ -115,9 +146,17 @@ def _parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: "--config", type=Path, required=True, - help="Strict schema-v1 JSON release configuration.", + help="Strict schema-v2 JSON release configuration.", ) parser.add_argument("--out", type=Path, required=True) + parser.add_argument( + "--logbook-prev-row-digest", + type=sha256_argument, + help=( + "Current US Logbook row checksum. May instead be supplied through " + "POPULACE_LOGBOOK_PREV_ROW_DIGEST." + ), + ) return parser.parse_args(argv) @@ -131,6 +170,7 @@ def _read_config(path: Path) -> LadderReleaseConfig: root, required={ "schema_version", + "family", "pool", "ladder", "targets", @@ -148,6 +188,8 @@ def _read_config(path: Path) -> LadderReleaseConfig: f"{CONFIG_SCHEMA_VERSION}, got {root['schema_version']!r}." ) + family = _object(root["family"], label="family") + _keys(family, required={"id"}, label="family") pool = _object(root["pool"], label="pool") _keys(pool, required={"release_id", "manifest_sha256"}, label="pool") ladder = _object(root["ladder"], label="ladder") @@ -264,6 +306,7 @@ def _read_config(path: Path) -> LadderReleaseConfig: ) return LadderReleaseConfig( + family_id=_uuid_value(family["id"], label="family.id"), pool_release_id=pool_release_id, pool_manifest_sha256=_sha256_value( pool["manifest_sha256"], label="pool.manifest_sha256" @@ -455,35 +498,245 @@ def _builder_argv( return argv +def _logbook_input_pins_digest(config: LadderReleaseConfig) -> str: + payload = { + "pool_manifest_sha256": config.pool_manifest_sha256, + "ledger_facts_sha256": config.ledger_facts_sha256, + "ledger_manifest_sha256": config.ledger_manifest_sha256, + "incumbent_diagnostics_sha256": config.incumbent_diagnostics_sha256, + "target_surface_sha256": config.target_surface_sha256, + "ssi_take_up_prior_weight_basis_sha256": ( + config.ssi_take_up_prior_weight_basis_sha256 + ), + } + return hashlib.sha256(canonical_json_bytes(payload)).hexdigest() + + +def _logbook_identity_digest( + config: LadderReleaseConfig, + *, + resolved_k: int | None, +) -> str: + payload = { + "pipeline": _LOGBOOK_PIPELINE, + "build_id": config.release_id, + "family_id": config.family_id, + "source_pool_sha256": config.pool_manifest_sha256, + "requested_k_input": config.requested_k, + "requested_k_resolved": resolved_k, + "record_unit": "household" if resolved_k is not None else None, + "seed": config.seed, + "pi_hi": config.pi_hi, + "calibration": { + "epochs": config.epochs, + "learning_rate": config.learning_rate, + "max_weight_ratio": config.max_weight_ratio, + "l0_refit_lambda_share": config.l0_refit_lambda_share, + "l2_lambda": config.l2_lambda, + "refit_l2_lambda": config.refit_l2_lambda, + }, + } + return hashlib.sha256(canonical_json_bytes(payload)).hexdigest() + + +def _record_exact_k_attempt( + *, + state: AttemptState, + started_at: float, + started_ts: datetime, + code_pin: str, + seed: int, + predecessor: str | None, + spool_dir: Path, + requested_k: int | None, + realized_k: int | None, + disposition: str, +) -> LogbookWriteResult: + return record_build_attempt( + build_id=state.build_id, + ts=started_ts, + pipeline=_LOGBOOK_PIPELINE, + rung=None, + seed=seed, + code_pin=code_pin, + input_pins_digest=state.input_pins_digest, + identity_digest=state.identity_digest, + phases_reached=state.phases_reached, + gate_verdicts=state.gate_verdicts, + wall_seconds=time.perf_counter() - started_at, + cost_usd=None, + artifact_location=state.artifact_location, + disposition=disposition, + prediction_id=None, + prev_row_digest=predecessor, + row_format_version=2, + requested_k=requested_k, + realized_k=realized_k, + record_unit="household" if requested_k is not None else None, + spool_dir=spool_dir, + post_remote=False, + ) + + def launch( *, pool_manifest: Path, config_path: Path, out: Path, + logbook_prev_row_digest: str | None = None, release_builder: Callable[[Sequence[str] | None], object] = fiscal_release.main, ) -> dict[str, object]: - """Validate pins, run the house release path, and write a publish receipt.""" + """Validate pins, build, persist Logbook records, and write a receipt.""" + started_at = time.perf_counter() + started_ts = datetime.now(UTC) config = _read_config(config_path) resolved_pool_manifest = pool_manifest.resolve() resolved_out = out.resolve() - k, _ = _validate_pins_and_resolve_k( - config=config, - pool_manifest_path=resolved_pool_manifest, + spool_dir = resolved_out / "logbook-spool" + predecessor = resolve_predecessor(logbook_prev_row_digest) + code_pin = git_code_pin(_REPOSITORY_ROOT) + requested_k = ( + int(config.requested_k) if isinstance(config.requested_k, int) else None + ) + success_receipt = ( + attempt_receipt_dir(resolved_out, build_id=config.release_id) / "release.json" + ) + success_receipt_reference = local_artifact_reference( + success_receipt, + repository_hint=_REPOSITORY_ROOT, ) - release_builder( - _builder_argv( + state = AttemptState( + build_id=config.release_id, + identity_digest=_logbook_identity_digest( config=config, - pool_manifest=resolved_pool_manifest, - out=resolved_out, - k=config.requested_k, + resolved_k=requested_k, + ), + input_pins_digest=_logbook_input_pins_digest(config), + phases_reached=["attempt_started"], + gate_verdicts={ + "exact_k_build": { + "verdict": "running", + "receipt": success_receipt_reference, + } + }, + ) + family = LogbookFamily.create( + family_id=config.family_id, + chain_scope="us", + source_pool_sha256=config.pool_manifest_sha256, + ) + + try: + k, _ = _validate_pins_and_resolve_k( + config=config, + pool_manifest_path=resolved_pool_manifest, + ) + requested_k = k + state.identity_digest = _logbook_identity_digest( + config, + resolved_k=requested_k, + ) + append_phase(state, "source_pool_verified") + record_family(family, spool_dir=spool_dir, post_remote=False) + append_phase(state, "family_record_spooled") + release_builder( + _builder_argv( + config=config, + pool_manifest=resolved_pool_manifest, + out=resolved_out, + k=config.requested_k, + ) + ) + append_phase(state, "dataset_built") + build = { + "release_id": config.release_id, + "release_dir": str(resolved_out / "releases" / config.release_id), + "artifact_root": str(resolved_out / "artifacts"), + } + atomic_write_json( + success_receipt, + { + "artifact_kind": "populace_exact_k_release_receipt", + "schema_version": 1, + **build, + "family_id": config.family_id, + "source_pool_sha256": config.pool_manifest_sha256, + "requested_k": requested_k, + "realized_k": requested_k, + "record_unit": "household", + "rung": None, + }, + ) + state.gate_verdicts = { + "exact_k_build": { + "verdict": "passed", + "receipt": success_receipt_reference, + } + } + state.artifact_location = local_artifact_reference( + Path(build["release_dir"]), + repository_hint=_REPOSITORY_ROOT, ) + except BaseException as error: + try: + failure_path = write_error_receipt( + error_receipt_path(resolved_out, build_id=config.release_id), + state=state, + pipeline=_LOGBOOK_PIPELINE, + error=error, + ) + failure_reference = local_artifact_reference( + failure_path, + repository_hint=_REPOSITORY_ROOT, + ) + state.gate_verdicts = { + "exact_k_build": { + "verdict": "error", + "receipt": failure_reference, + } + } + state.artifact_location = failure_reference + append_phase(state, "error") + _record_exact_k_attempt( + state=state, + started_at=started_at, + started_ts=started_ts, + code_pin=code_pin, + seed=config.seed, + predecessor=predecessor, + spool_dir=spool_dir, + requested_k=requested_k, + realized_k=None, + disposition="failed", + ) + reconcile_logbook_spool(spool_dir) + except Exception as recording_error: + error.add_note( + "Exact-k Logbook failure recording also failed: " + f"{type(recording_error).__name__}: {recording_error}" + ) + raise + + assert requested_k is not None + write_result = _record_exact_k_attempt( + state=state, + started_at=started_at, + started_ts=started_ts, + code_pin=code_pin, + seed=config.seed, + predecessor=predecessor, + spool_dir=spool_dir, + requested_k=requested_k, + realized_k=requested_k, + disposition="iterating", ) - build = { - "release_id": config.release_id, - "release_dir": str(resolved_out / "releases" / config.release_id), - "artifact_root": str(resolved_out / "artifacts"), - } + member = FamilyMember.create( + family_id=config.family_id, + build_id=config.release_id, + ) + record_family_member(member, spool_dir=spool_dir, post_remote=False) + reconcile_logbook_spool(spool_dir) publish_argv = [ "tools/publish_release.sh", @@ -499,6 +752,12 @@ def launch( result: dict[str, object] = { **build, "k": k, + "family_id": config.family_id, + "requested_k": requested_k, + "realized_k": requested_k, + "record_unit": "household", + "rung": None, + "logbook_row_digest": write_result.row.row_digest, "seed": config.seed, "automatic_publish": False, "pointer_update": False, @@ -518,12 +777,8 @@ def launch( "publish_argv": publish_argv, "publish_command": shlex.join(publish_argv), } - resolved_out.mkdir(parents=True, exist_ok=True) package_result = resolved_out / "package_result.json" - package_result.write_text( - json.dumps(result, indent=2, sort_keys=True, allow_nan=False) + "\n", - encoding="utf-8", - ) + atomic_write_json(package_result, result) print(json.dumps(result, indent=2, sort_keys=True, allow_nan=False)) return result @@ -563,6 +818,17 @@ def _sha256_value(value: object, *, label: str) -> str: return parsed +def _uuid_value(value: object, *, label: str) -> str: + parsed = _nonempty_string(value, label=label) + try: + normalized = str(uuid.UUID(parsed)) + except ValueError as exc: + raise ValueError(f"{label} must be a canonical UUID.") from exc + if parsed != normalized: + raise ValueError(f"{label} must use canonical lowercase UUID text.") + return normalized + + def _nonnegative_int(value: object, *, label: str) -> int: if isinstance(value, bool) or not isinstance(value, int) or value < 0: raise ValueError(f"{label} must be a non-negative integer, got {value!r}.") @@ -619,6 +885,7 @@ def main(argv: Sequence[str] | None = None) -> dict[str, object]: pool_manifest=args.pool_manifest, config_path=args.config, out=args.out, + logbook_prev_row_digest=args.logbook_prev_row_digest, ) diff --git a/tools/logbook.py b/tools/logbook.py index 4956799fd..7d312a51d 100644 --- a/tools/logbook.py +++ b/tools/logbook.py @@ -26,16 +26,32 @@ from urllib.request import Request from microcosm.build.logbook import ( + DECLARED_LOGBOOK_SCOPES, + LEGACY_US_PIPELINES, LOGBOOK_ROW_FIELDS, LogbookRow, _validate_remote_url, export_rows, load_logbook_file, load_spool_rows, + logbook_chain_scope, order_rows_by_chain, render_markdown, + spool_build_rows, urlopen, ) +from microcosm.build.logbook_family import ( + FamilyAction, + FamilyArchiveRecords, + FamilyMember, + LogbookFamily, + export_family_scope, + import_family_scope, + load_family_archive_records, + load_family_spool, + reconcile_logbook_spool, + validate_family_membership, +) ROOT = Path(__file__).resolve().parents[1] #: Archives live at ``logbook//.jsonl`` — one hash chain per @@ -49,22 +65,15 @@ REMOTE_EXPORT_KEY_ENV = "POPULACE_LEDGER_EXPORT_KEY" REMOTE_API_KEY_ENV = "POPULACE_LEDGER_API_KEY" REMOTE_PAGE_SIZE = 500 +FAMILY_ARCHIVE_DIRECTORIES = frozenset({"families", "family_members", "family_actions"}) # Mirror of logbook.chain_scope() in # supabase/migrations/20260818000000_logbook_chain_scopes.sql. The legacy US # rows predate the scope split and must continue one mixed `us` chain forever. -LEGACY_US_PIPELINES = ( - "us-2024-release", - "us-pool-inc2", - "us-stacked-pool", -) -_PIPELINE_SCOPE_PATTERN = re.compile( - r"^(?P[a-z]{2})-(?P[a-z0-9_]+)(?:-[a-z0-9_-]+)?$" -) #: Mirror of logbook.scope_declared() in the same migration: the ratified #: scope vocabulary, closed-world. Opening a scope is a reviewed diff here, #: in the migration, and in logbook/README.md -- never a side effect of a #: well-formed pipeline name. -DECLARED_SCOPES = frozenset({"us", "uk/frs"}) +DECLARED_SCOPES = DECLARED_LOGBOOK_SCOPES def _parser() -> argparse.ArgumentParser: @@ -144,6 +153,91 @@ def _parser() -> argparse.ArgumentParser: action="append", help="Include this disposition; repeat to include more than one.", ) + + family_export = subparsers.add_parser( + "family-export", + help="Append family records to the three archives for one scope.", + ) + family_export.add_argument( + "--scope", choices=sorted(DECLARED_SCOPES), required=True + ) + family_export.add_argument( + "--archive-root", + type=Path, + default=DEFAULT_ARCHIVE_ROOT, + ) + family_source = family_export.add_mutually_exclusive_group(required=True) + family_source.add_argument( + "--source", + type=Path, + help="A Logbook spool containing family record subdirectories.", + ) + family_source.add_argument( + "--remote", + action="store_true", + help="Read family records for the scope from the live store.", + ) + + family_import = subparsers.add_parser( + "family-import", + help="Copy one scope's family archives into a durable local spool.", + ) + family_import.add_argument( + "--scope", choices=sorted(DECLARED_SCOPES), required=True + ) + family_import.add_argument( + "--archive-root", + type=Path, + default=DEFAULT_ARCHIVE_ROOT, + ) + family_import.add_argument( + "--spool", + type=Path, + default=DEFAULT_SPOOL_ROOT, + ) + + reconcile = subparsers.add_parser( + "reconcile", + help="Send queued builds and family records in dependency order.", + ) + reconcile.add_argument( + "--spool", + type=Path, + default=DEFAULT_SPOOL_ROOT, + ) + + list_families = subparsers.add_parser( + "list-families", + help="List archived dataset families.", + ) + list_families.add_argument( + "--archive-root", + type=Path, + default=DEFAULT_ARCHIVE_ROOT, + ) + list_families.add_argument("--scope", choices=sorted(DECLARED_SCOPES)) + + list_builds = subparsers.add_parser( + "list-family-builds", + help="List archived builds associated with one family.", + ) + list_builds.add_argument("--family-id", required=True) + list_builds.add_argument( + "--archive-root", + type=Path, + default=DEFAULT_ARCHIVE_ROOT, + ) + + show_history = subparsers.add_parser( + "show-family-history", + help="Show archived revocations and replacements for one family.", + ) + show_history.add_argument("--family-id", required=True) + show_history.add_argument( + "--archive-root", + type=Path, + default=DEFAULT_ARCHIVE_ROOT, + ) return parser @@ -161,14 +255,35 @@ def _archive_files(path: Path) -> tuple[Path, ...]: one invocation without ever merging the chains themselves. """ + if _within_family_archive_directory(path): + raise ValueError(f"No Logbook build archives found under {path}.") if not path.is_dir(): return (path,) - files = tuple(sorted(path.rglob("*.jsonl"))) + files = tuple( + sorted( + candidate + for candidate in path.rglob("*.jsonl") + if not _is_family_archive(candidate, root=path) + ) + ) if not files: raise ValueError(f"No Logbook scope archives found under {path}.") return files +def _is_family_archive(path: Path, *, root: Path) -> bool: + try: + relative = path.relative_to(root) + except ValueError: + return False + return bool(relative.parts) and relative.parts[0] in FAMILY_ARCHIVE_DIRECTORIES + + +def _within_family_archive_directory(path: Path) -> bool: + candidates = (path, *path.parents) + return any(candidate.name in FAMILY_ARCHIVE_DIRECTORIES for candidate in candidates) + + def _scope_label(archive: Path, root: Path | None = None) -> str: """Name a chain by its scope path (``us/pool``), not its filename. @@ -192,12 +307,7 @@ def _scope_label(archive: Path, root: Path | None = None) -> str: def _chain_scope(pipeline: str) -> str | None: """Return the Logbook chain scope declared by a pipeline name.""" - if pipeline in LEGACY_US_PIPELINES: - return "us" - match = _PIPELINE_SCOPE_PATTERN.fullmatch(pipeline) - if match is None: - return None - return f"{match.group('country')}/{match.group('line')}" + return logbook_chain_scope(pipeline) def _archive_scope(archive: Path) -> str: @@ -276,7 +386,7 @@ def _remote_rows(scope: str) -> tuple[LogbookRow, ...]: raise RuntimeError(f"Logbook live store returned HTTP {status}") page = _decode_remote_page(response.read()) total = _content_range_total(getattr(response, "headers", {})) - rows.extend(LogbookRow.from_mapping(item) for item in page) + rows.extend(LogbookRow.from_database_mapping(item) for item in page) offset += len(page) if total is not None and offset >= total: break @@ -287,11 +397,7 @@ def _remote_rows(scope: str) -> tuple[LogbookRow, ...]: ) break wrong_scope = sorted( - { - row.pipeline - for row in rows - if _chain_scope(row.pipeline) != scope - } + {row.pipeline for row in rows if _chain_scope(row.pipeline) != scope} ) if wrong_scope: raise ValueError( @@ -333,6 +439,126 @@ def _remote_builds_endpoint( return f"{endpoint}?{query}" +def _remote_family_records(scope: str) -> FamilyArchiveRecords: + families = tuple( + LogbookFamily.from_mapping(row) + for row in _remote_table_rows( + table="families", + fields=("family_id", "chain_scope", "source_pool_sha256"), + scope=scope, + ) + ) + members = tuple( + FamilyMember.from_mapping(row) + for row in _remote_table_rows( + table="family_members_public", + fields=("family_id", "build_id"), + scope=scope, + ) + ) + actions = tuple( + FamilyAction.from_mapping(row) + for row in _remote_table_rows( + table="family_actions_public", + fields=( + "action_id", + "family_id", + "build_id", + "action_type", + "related_build_id", + "recorded_at", + "actor", + "reason", + "evidence_location", + ), + scope=scope, + ) + ) + return FamilyArchiveRecords(families, members, actions) + + +def _remote_table_rows( + *, + table: str, + fields: tuple[str, ...], + scope: str, +) -> tuple[dict[str, Any], ...]: + ledger_url = os.environ.get("POPULACE_LEDGER_URL") + export_key = os.environ.get(REMOTE_EXPORT_KEY_ENV) + api_key = os.environ.get(REMOTE_API_KEY_ENV) + if not ledger_url or not export_key or not api_key: + raise ValueError( + "remote export requires POPULACE_LEDGER_URL, " + f"{REMOTE_EXPORT_KEY_ENV}, and {REMOTE_API_KEY_ENV}" + ) + + rows: list[dict[str, Any]] = [] + offset = 0 + while True: + endpoint = _remote_table_endpoint( + ledger_url, + table=table, + fields=fields, + scope=scope, + offset=offset, + limit=REMOTE_PAGE_SIZE, + ) + request = Request( + endpoint, + headers={ + "Accept": "application/json", + "Accept-Profile": "logbook", + "apikey": api_key, + "Authorization": f"Bearer {export_key}", + "Prefer": "count=exact", + }, + ) + with urlopen(request, timeout=30.0) as response: + status = getattr(response, "status", 200) + if not 200 <= status < 300: + raise RuntimeError(f"Logbook live store returned HTTP {status}") + page = _decode_remote_page(response.read()) + total = _content_range_total(getattr(response, "headers", {})) + rows.extend(page) + offset += len(page) + if total is not None and offset >= total: + break + if not page: + if total is not None: + raise RuntimeError( + "Logbook live store ended before its declared row count" + ) + break + return tuple(rows) + + +def _remote_table_endpoint( + url: str, + *, + table: str, + fields: tuple[str, ...], + scope: str, + offset: int, + limit: int, +) -> str: + _validate_remote_url(url) + base = url.rstrip("/") + if base.endswith("/rest/v1/builds"): + base = base[: -len("/builds")] + elif not base.endswith("/rest/v1"): + base = f"{base}/rest/v1" + query = urlencode( + { + "select": ",".join(fields), + "chain_scope": f"eq.{scope}", + "order": ",".join(f"{field}.asc" for field in fields[:2]), + "limit": str(limit), + "offset": str(offset), + } + ) + return f"{base}/{table}?{query}" + + def _decode_remote_page(payload: bytes) -> list[dict[str, Any]]: try: value = json.loads(payload) @@ -363,11 +589,215 @@ def _content_range_total(headers: Any) -> int | None: return parsed +def _archived_family_records( + archive_root: Path, + *, + scope: str | None = None, +) -> FamilyArchiveRecords: + scopes = (scope,) if scope is not None else tuple(sorted(DECLARED_SCOPES)) + families: list[LogbookFamily] = [] + members: list[FamilyMember] = [] + actions: list[FamilyAction] = [] + for candidate_scope in scopes: + records = load_family_archive_records(archive_root, candidate_scope) + families.extend(records.families) + members.extend(records.family_members) + actions.extend(records.family_actions) + family_ids: dict[str, LogbookFamily] = {} + for family in families: + previous = family_ids.get(family.family_id) + if previous is not None and previous != family: + raise ValueError( + f"Family {family.family_id} has conflicting archived records." + ) + family_ids[family.family_id] = family + return FamilyArchiveRecords(tuple(families), tuple(members), tuple(actions)) + + +def _archived_builds(archive_root: Path) -> dict[str, LogbookRow]: + return { + row.build_id: row + for archive in _archive_files(archive_root) + for row in load_logbook_file(archive) + } + + +def _scope_build_archive(archive_root: Path, scope: str) -> Path: + return archive_root / Path(*scope.split("/")).with_suffix(".jsonl") + + +def _print_json_lines(values: list[dict[str, Any]]) -> None: + for value in values: + print( + json.dumps( + value, + allow_nan=False, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ) + ) + + +def _public_build_mapping(row: LogbookRow) -> dict[str, Any]: + mapping = row.to_mapping() + mapping.pop("gate_verdicts") + mapping.pop("cost_usd") + mapping.pop("row_format_version", None) + if row.disposition not in {"published", "certified"}: + mapping["artifact_location"] = None + mapping.setdefault("requested_k", None) + mapping.setdefault("realized_k", None) + mapping.setdefault("record_unit", None) + return mapping + + def main(argv: list[str] | None = None) -> int: """Run one Logbook command and return its process exit code.""" args = _parser().parse_args(argv) try: + if args.command == "family-export": + records = ( + _remote_family_records(args.scope) + if args.remote + else load_family_spool(args.source) + ) + receipts = export_family_scope( + args.archive_root, + scope=args.scope, + families=records.families, + family_members=records.family_members, + family_actions=records.family_actions, + ) + print( + "exported family records for " + f"{args.scope}: " + + ", ".join( + f"{record_type}={receipt.appended} new/{receipt.existing} existing" + for record_type, receipt in receipts.items() + ) + ) + return 0 + + if args.command == "family-import": + records = load_family_archive_records( + args.archive_root, + args.scope, + ) + build_archive = _scope_build_archive(args.archive_root, args.scope) + if records.family_members and not build_archive.is_file(): + raise ValueError( + f"Family members for {args.scope} require build archive " + f"{build_archive}." + ) + builds = load_logbook_file(build_archive) if build_archive.is_file() else () + builds_by_id = {build.build_id: build for build in builds} + families_by_id = {family.family_id: family for family in records.families} + for member in records.family_members: + try: + family = families_by_id[member.family_id] + build = builds_by_id[member.build_id] + except KeyError as exc: + raise ValueError( + f"Family member import is missing archived record " + f"{exc.args[0]}." + ) from exc + validate_family_membership(family, member, build) + spool_build_rows(builds, spool_dir=args.spool) + imported = import_family_scope( + args.archive_root, + scope=args.scope, + spool_dir=args.spool, + ) + print( + f"imported family records for {args.scope}: " + f"builds={len(builds)}, families={len(imported.families)}, " + f"members={len(imported.family_members)}, " + f"actions={len(imported.family_actions)}" + ) + return 0 + + if args.command == "reconcile": + receipt = reconcile_logbook_spool(args.spool) + print( + "reconciled Logbook spool: " + f"builds={receipt.builds.posted} posted/" + f"{receipt.builds.retained} retained; " + f"family records={receipt.families.posted} posted/" + f"{receipt.families.retained} retained" + ) + errors = (*receipt.builds.errors, *receipt.families.errors) + if errors: + raise RuntimeError("; ".join(errors)) + return 0 + + if args.command == "list-families": + records = _archived_family_records( + args.archive_root, + scope=args.scope, + ) + _print_json_lines( + [ + family.to_mapping() + for family in sorted( + records.families, + key=lambda value: (value.chain_scope, value.family_id), + ) + ] + ) + return 0 + + if args.command == "list-family-builds": + records = _archived_family_records(args.archive_root) + family = next( + ( + value + for value in records.families + if value.family_id == args.family_id + ), + None, + ) + if family is None: + raise ValueError(f"Unknown family_id {args.family_id}.") + builds = _archived_builds(args.archive_root) + selected: list[LogbookRow] = [] + for member in records.family_members: + if member.family_id != family.family_id: + continue + try: + selected.append(builds[member.build_id]) + except KeyError as exc: + raise ValueError( + f"Family member {member.build_id} has no archived build." + ) from exc + selected.sort( + key=lambda value: ( + value.requested_k is None, + value.requested_k or 0, + value.build_id, + ) + ) + _print_json_lines([_public_build_mapping(row) for row in selected]) + return 0 + + if args.command == "show-family-history": + records = _archived_family_records(args.archive_root) + if not any( + family.family_id == args.family_id for family in records.families + ): + raise ValueError(f"Unknown family_id {args.family_id}.") + actions = sorted( + ( + action + for action in records.family_actions + if action.family_id == args.family_id + ), + key=lambda value: (value.recorded_at, value.action_id), + ) + _print_json_lines([action.to_mapping() for action in actions]) + return 0 + if args.command == "validate": archives = _archive_files(args.archive) root = args.archive if args.archive.is_dir() else None