diff --git a/app/public/data/lanes.json b/app/public/data/lanes.json index d2b4a03..398650a 100644 --- a/app/public/data/lanes.json +++ b/app/public/data/lanes.json @@ -265,10 +265,10 @@ "source": "UKMOD/EUROMOD", "area": "household cases via JRC connector", "mode": 3, - "stage": "registered", + "stage": "cataloged", "running": false, - "updated": "2026-08-01", - "note": "Connector runnable per axiom-oracles#264 (#5)" + "updated": "2026-08-14", + "note": "Case schema + 1st battery landed (#41); connector run pending UKMOD environment; schema shared with taxsim-cases (#5)" }, { "id": "cbo-baseline", diff --git a/data/lanes.json b/data/lanes.json index d2b4a03..398650a 100644 --- a/data/lanes.json +++ b/data/lanes.json @@ -265,10 +265,10 @@ "source": "UKMOD/EUROMOD", "area": "household cases via JRC connector", "mode": 3, - "stage": "registered", + "stage": "cataloged", "running": false, - "updated": "2026-08-01", - "note": "Connector runnable per axiom-oracles#264 (#5)" + "updated": "2026-08-14", + "note": "Case schema + 1st battery landed (#41); connector run pending UKMOD environment; schema shared with taxsim-cases (#5)" }, { "id": "cbo-baseline", diff --git a/scorecard_db/__init__.py b/scorecard_db/__init__.py index eae73f5..21e902a 100644 --- a/scorecard_db/__init__.py +++ b/scorecard_db/__init__.py @@ -1,3 +1,14 @@ +from .case_diffs import ( + ADJUDICATED_ONLY, + DEFAULT_TOLERANCES, + CaseResult, + CaseSpec, + DiffClassification, + Oracle, + VariableClass, + classify, + load_battery, +) from .db import ScorecardDB from .models import ( BASELINE, @@ -17,19 +28,28 @@ ) __all__ = [ + "ADJUDICATED_ONLY", "BASELINE", "CURRENT_LAW_DESCRIPTOR", + "DEFAULT_TOLERANCES", "STANDARD_CONDITIONS", "BenchmarkClass", "CalibrationRelationship", + "CaseResult", + "CaseSpec", "ComparisonStatus", "DiagnosisClass", + "DiffClassification", "ExternalScore", "Metric", + "Oracle", "PEResult", "ReformRef", "ScorecardDB", "TimeBasis", "UnitConcept", + "VariableClass", "baseline_key", + "classify", + "load_battery", ] diff --git a/scorecard_db/case_diffs.py b/scorecard_db/case_diffs.py new file mode 100644 index 0000000..09db78e --- /dev/null +++ b/scorecard_db/case_diffs.py @@ -0,0 +1,706 @@ +"""Mode-3 case-diff models: record-level comparisons against household +oracles (UKMOD/EUROMOD via the JRC connector, NBER TAXSIM). + +One schema for every mode-3 lane (#5, #41) — the contract lives at +sources/ukmod-cases/SCHEMA.md and is deliberately engine-agnostic: the +``country`` field on a case and the ``oracle`` field on a result are what +keep ``ukmod-cases`` and ``taxsim-cases`` on a single schema instead of two. + +Design, same doctrine as :mod:`scorecard_db.models`: + +1. **Closed vocabularies fail loudly.** Oracles, diff classifications, + variable classes, household/person input keys — all closed sets; unknown + values raise instead of passing through. +2. **Cases are inputs only.** A ``CaseSpec`` never embeds expected output + values; both sides of every comparison come from engine runs, so the + battery cannot smuggle in hand-computed truth. +3. **The classifier never flatters.** ``classify`` emits match buckets, the + two null-side scope buckets, or ``unclassified``; every other outcome + exists only as an ADJUDICATION of an ``unclassified`` row, with a + traceable writeup — misses stay visible until someone explains them. +4. **The seam is enforced, not conventional.** ``CaseResult`` RECOMPUTES + ``classify()`` and compares: a stored classification either equals what + the classifier says, or it is an adjudication of a row the classifier + left ``unclassified`` and carries a writeup. Nothing can persist + ``100`` vs ``100`` as ``pe_gap``, a boolean ``1`` vs ``0`` as a + tolerance match, or a numeric mismatch as ``policy_scope_mismatch`` + without an explanation — the failure modes the round-2 probes found. + Tolerances come from :data:`DEFAULT_TOLERANCES` alone; a caller cannot + hand in a wider one. +5. **Identities are closed at the edges too.** Regions, focus variables, + battery schema paths and baseline worlds route through registries; NaN + and infinity are not numbers a case may carry, and a boolean-class + comparison may only hold 0 or 1. + +Not yet here: the case/result TABLE, writer, exporter and build_db step. +Mode-3 rows therefore do not persist to the DB in this PR, and the +epistemic wiring that rides on those columns (calibration_relationship, +run/bundle pins, connector revision, citable adjudication links) lands +with them. That design has repo-wide surface and is being paired on +rather than decided unilaterally here; SCHEMA.md records the open shape. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from datetime import date +from enum import Enum +from pathlib import Path +from typing import Optional + +# Contract version for the battery file and result rows. Bump on any +# breaking change to the vocabularies or row shape so stored artifacts +# can be migrated explicitly instead of silently reinterpreted. +SCHEMA_VERSION = 1 + + +class Oracle(str, Enum): + UKMOD = "ukmod" + TAXSIM = "taxsim" + + +class VariableClass(str, Enum): + CURRENCY = "currency" + BOOLEAN = "boolean" + + +class DiffClassification(str, Enum): + MATCH_EXACT = "match_exact" + MATCH_WITHIN_TOLERANCE = "match_within_tolerance" + PE_GAP = "pe_gap" + ORACLE_DIFFERENCE = "oracle_difference" + POLICY_SCOPE_MISMATCH = "policy_scope_mismatch" + ROUNDING = "rounding" + UNCLASSIFIED = "unclassified" + + +# Classifications the automatic classifier may emit. +CLASSIFIER_EMITTED = frozenset( + { + DiffClassification.MATCH_EXACT, + DiffClassification.MATCH_WITHIN_TOLERANCE, + DiffClassification.PE_GAP, + DiffClassification.POLICY_SCOPE_MISMATCH, + DiffClassification.UNCLASSIFIED, + } +) + +# Outcomes a human may adjudicate an ``unclassified`` row INTO, each +# requiring a traceable writeup in ``annotations``. Note that pe_gap and +# policy_scope_mismatch appear both here and in CLASSIFIER_EMITTED: the +# classifier emits them for a NULL side, where they are mechanical, and a +# human may also assign them to a numeric-vs-numeric difference, where +# they are a judgement and need explaining. The round-2 finding was that +# only oracle_difference and rounding demanded a writeup, so a +# numeric-vs-numeric row could be labelled pe_gap silently. +ADJUDICATABLE = frozenset( + { + DiffClassification.ORACLE_DIFFERENCE, + DiffClassification.ROUNDING, + DiffClassification.PE_GAP, + DiffClassification.POLICY_SCOPE_MISMATCH, + } +) + +# Back-compatible alias for the adjudicated-only pair. +ADJUDICATED_ONLY = frozenset( + { + DiffClassification.ORACLE_DIFFERENCE, + DiffClassification.ROUNDING, + } +) + +# Currency tolerance: benefit rules are stated weekly to the penny and +# comparisons are annual, so 52 x GBP/USD 0.01. Booleans agree or they +# don't. Wider slack is never a tolerance bump — a documented oracle +# rounding rule adjudicates to ROUNDING instead (SCHEMA.md). +DEFAULT_TOLERANCES: dict[VariableClass, float] = { + VariableClass.CURRENCY: 0.52, + VariableClass.BOOLEAN: 0.0, +} + +VALID_COUNTRIES = frozenset({"UK", "US"}) + +VALID_TENURES = frozenset( + {"owned_outright", "owned_mortgage", "rented_social", "rented_private"} +) + +# The engine-agnostic household input vocabulary (SCHEMA.md). Each mode-3 +# connector owns the mapping to its engines' variables; the battery speaks +# only these keys. Extending the vocabulary means extending these sets AND +# documenting the key in SCHEMA.md. +PERSON_NUMERIC_KEYS = frozenset( + { + "employment_income", + "self_employment_income", + "pension_income", + "state_pension", + "savings_income", + "capital", + "employee_pension_contributions", + "hours_worked_per_week", + } +) +PERSON_BOOL_KEYS = frozenset( + {"salary_sacrifice", "is_disabled", "is_carer", "gainfully_self_employed"} +) +PERSON_KEYS = ( + frozenset({"age", "date_of_birth"}) | PERSON_NUMERIC_KEYS | PERSON_BOOL_KEYS +) + +HOUSEHOLD_KEYS = frozenset( + {"people", "benefit_units", "region", "tenure", "rent", "council_tax", "brma"} +) + +# Regions are a CLOSED registry per country, not free text: region routes +# devolved policy and location-dependent amounts, so `region="MARS"` must +# not reach a connector that will silently fall back to a default rate. +VALID_REGIONS = { + # UK ITL-1 + "UK": frozenset( + { + "NORTH_EAST", + "NORTH_WEST", + "YORKSHIRE", + "EAST_MIDLANDS", + "WEST_MIDLANDS", + "EAST_OF_ENGLAND", + "LONDON", + "SOUTH_EAST", + "SOUTH_WEST", + "WALES", + "SCOTLAND", + "NORTHERN_IRELAND", + } + ), + # US state codes (+ DC), for the taxsim-cases lane + "US": frozenset( + "AL AK AZ AR CA CO CT DE DC FL GA HI ID IL IN IA KS KY LA ME MD MA " + "MI MN MS MO MT NE NV NH NJ NM NY NC ND OH OK OR PA RI SC SD TN TX " + "UT VT VA WA WV WI WY".split() + ), +} + +# Broad Rental Market Areas the private-rent cases may pin. A UK +# private-rent case MUST name one: LHA rates are set per BRMA, not per +# ITL-1 region, so "Yorkshire" does not fix a world — the round-2 finding +# that the rent clears the cap only in "most" Yorkshire BRMAs means the +# two engines were not guaranteed the same LHA rate. Extending the list +# is deliberate, and each entry names the region it sits in so a case +# cannot pin a BRMA outside its own region. +BRMA_REGION = { + "Leeds": "YORKSHIRE", + "Sheffield": "YORKSHIRE", + "Inner London": "LONDON", + "Outer London": "LONDON", + "Manchester": "NORTH_WEST", + "Nottingham": "EAST_MIDLANDS", + "Birmingham": "WEST_MIDLANDS", + "Southampton": "SOUTH_EAST", +} + +# expected_focus is claim-shaped: it names the output variables a case is +# curated to exercise, and a connector maps each to both engines. An +# invented name would silently produce no comparison at all, so the +# vocabulary is closed per country. +FOCUS_VARIABLES = { + "UK": frozenset( + { + "universal_credit", + "child_benefit", + "pension_credit", + "carers_allowance", + "benefit_cap", + "income_tax", + "national_insurance", + "scottish_child_payment", + "council_tax_reduction", + "housing_benefit", + } + ), + "US": frozenset( + { + "federal_income_tax", + "state_income_tax", + "payroll_tax", + "eitc", + "ctc", + "snap", + } + ), +} + +# Battery `schema` values: the contract document a battery declares +# itself against. An arbitrary path let a battery claim a schema nobody +# wrote. +VALID_BATTERY_SCHEMAS = frozenset({"sources/ukmod-cases/SCHEMA.md"}) + + +def _finite(value) -> bool: + """NaN and infinity are not amounts a case or a comparison may carry: + NaN silently defeats every comparison operator below, and an infinite + rent produces an infinite entitlement rather than an error.""" + return value == value and value not in (float("inf"), float("-inf")) + + +def _validate_person(person_id: str, person: dict) -> None: + if not isinstance(person, dict): + raise ValueError(f"person {person_id!r} must be a mapping") + unknown = set(person) - PERSON_KEYS + if unknown: + raise ValueError(f"person {person_id!r} has unknown keys: {sorted(unknown)}") + age = person.get("age") + if not isinstance(age, int) or isinstance(age, bool) or age < 0: + raise ValueError(f"person {person_id!r} needs an integer age >= 0") + dob = person.get("date_of_birth") + if dob is not None: + # a real calendar date, not just the YYYY-MM-DD shape + # ("2026-13-40" must fail) + try: + if not (isinstance(dob, str) and len(dob) == 10): + raise ValueError + date.fromisoformat(dob) + except ValueError: + raise ValueError( + f"person {person_id!r} date_of_birth must be a real " + f"YYYY-MM-DD date, got {dob!r}" + ) from None + for key in PERSON_NUMERIC_KEYS & set(person): + v = person[key] + if isinstance(v, bool) or not isinstance(v, (int, float)) or not _finite(v): + raise ValueError(f"person {person_id!r} {key} must be a finite number >= 0") + if v < 0: + raise ValueError(f"person {person_id!r} {key} must be a number >= 0") + for key in PERSON_BOOL_KEYS & set(person): + if not isinstance(person[key], bool): + raise ValueError(f"person {person_id!r} {key} must be a boolean") + + +def _validate_household(household: dict, country: str) -> None: + if not isinstance(household, dict): + raise ValueError("household must be a mapping") + unknown = set(household) - HOUSEHOLD_KEYS + if unknown: + raise ValueError(f"household has unknown keys: {sorted(unknown)}") + people = household.get("people") + if not isinstance(people, dict) or not people: + raise ValueError("household.people must be a non-empty mapping") + for person_id, person in people.items(): + _validate_person(person_id, person) + units = household.get("benefit_units") + if not isinstance(units, list) or not units: + raise ValueError("household.benefit_units must be a non-empty list") + assigned: list[str] = [] + for unit in units: + if not isinstance(unit, dict) or set(unit) != {"adults", "children"}: + raise ValueError("each benefit unit needs exactly adults + children lists") + for role in ("adults", "children"): + members = unit[role] + if not isinstance(members, list) or not all( + isinstance(m, str) and m for m in members + ): + raise ValueError( + f"benefit unit {role} must be a list of person ids, got {members!r}" + ) + if not unit["adults"]: + raise ValueError("each benefit unit needs at least one adult") + assigned.extend(unit["adults"]) + assigned.extend(unit["children"]) + if sorted(assigned) != sorted(people): + raise ValueError( + "benefit units must assign every person exactly once " + f"(people={sorted(people)}, assigned={sorted(assigned)})" + ) + tenure = household.get("tenure") + if tenure is not None and tenure not in VALID_TENURES: + raise ValueError(f"unknown tenure: {tenure!r}") + for key in ("rent", "council_tax"): + v = household.get(key) + if v is not None and ( + isinstance(v, bool) + or not isinstance(v, (int, float)) + or not _finite(v) + or v < 0 + ): + raise ValueError(f"household.{key} must be a finite number >= 0") + if household.get("rent", 0) and tenure in ("owned_outright", "owned_mortgage"): + raise ValueError("owner-occupier households carry no rent") + region = household.get("region") + if region is not None and region not in VALID_REGIONS[country]: + raise ValueError( + f"unregistered {country} region {region!r} — region routes " + "devolved policy and location-dependent amounts, so it is a " + "closed registry (VALID_REGIONS), never free text" + ) + brma = household.get("brma") + if country == "UK" and tenure == "rented_private": + if brma is None: + raise ValueError( + "a UK private-rent case must pin a BRMA: LHA rates are set " + "per Broad Rental Market Area, not per ITL-1 region, so a " + "region alone does not fix the world both engines run" + ) + if brma not in BRMA_REGION: + raise ValueError(f"unregistered BRMA: {brma!r}") + if region is not None and BRMA_REGION[brma] != region: + raise ValueError(f"BRMA {brma!r} sits in {BRMA_REGION[brma]}, not {region}") + elif brma is not None: + raise ValueError("brma is recorded only on UK private-rent cases") + + +def _validate_baseline(descriptor) -> None: + """A case may only name a baseline world the registry describes. + + Registration is what makes a counterfactual auditable: an unnamed + world is exactly the "which law did this run?" ambiguity mode 3 + exists to remove. Imported lazily — baselines.py reaches the DB + layer, and this module must stay importable on its own. + """ + if descriptor is None: + return + if not isinstance(descriptor, dict) or not descriptor.get("policy"): + raise ValueError( + f"case baseline must be a descriptor with a policy key: {descriptor!r}" + ) + from .baselines import BASELINES + from .models import baseline_key + + known = {baseline_key(d) for d, *_ in BASELINES} + if baseline_key(descriptor) not in known: + raise ValueError( + f"unregistered baseline world {descriptor!r} — register it in " + "scorecard_db/baselines.py before a case may be evaluated in it" + ) + + +@dataclass +class CaseSpec: + """One curated hypothetical household: inputs + expected focus only.""" + + case_id: str + description: str + policy_year: int + country: str + household: dict + expected_focus: list + rationale: str = "" + # The policy world the case is evaluated in. None = current law. + # A descriptor of the same shape ReformRef.baseline carries, and it + # must already be REGISTERED in baselines.py — this is how a case + # pins a same-year counterfactual (e.g. {"policy": "pre_ab2025"}, + # the two-child limit reinstated) instead of leaning on a + # year-on-year delta that moves ages, rates and policy year at once. + baseline: Optional[dict] = None + + def __post_init__(self): + if not self.case_id or not isinstance(self.case_id, str): + raise ValueError("case_id must be a non-empty string") + if self.country not in VALID_COUNTRIES: + raise ValueError(f"unknown country: {self.country!r}") + if not self.case_id.startswith(self.country.lower() + "-"): + raise ValueError( + f"case_id {self.case_id!r} must be prefixed '{self.country.lower()}-'" + ) + if ( + not isinstance(self.policy_year, int) + or not 1990 <= self.policy_year <= 2100 + ): + raise ValueError(f"implausible policy_year: {self.policy_year!r}") + if ( + not isinstance(self.expected_focus, list) + or not self.expected_focus + or not all(isinstance(v, str) and v for v in self.expected_focus) + ): + raise ValueError("expected_focus must be a non-empty list of variables") + unknown_focus = sorted(set(self.expected_focus) - FOCUS_VARIABLES[self.country]) + if unknown_focus: + raise ValueError( + f"unregistered {self.country} focus variables: {unknown_focus} — " + "a name no connector maps produces no comparison at all" + ) + if len(set(self.expected_focus)) != len(self.expected_focus): + raise ValueError("expected_focus must not repeat a variable") + _validate_baseline(self.baseline) + _validate_household(self.household, self.country) + + +@dataclass +class CaseResult: + """PE vs oracle for one case x output variable x run (history kept).""" + + case_id: str + variable: str + pe_value: Optional[float] + oracle_value: Optional[float] + oracle: Oracle + engine_version: str + oracle_version: str + computed_at: str # ISO timestamp, caller-supplied + classification: DiffClassification + # REQUIRED, not optional: the validator re-runs classify() against it, + # so a row without a variable class cannot be checked at all. (It was + # Optional, and the shipped valid-row test omitted it.) + variable_class: VariableClass + abs_diff: Optional[float] = None + tolerance: Optional[float] = None + annotations: list = field(default_factory=list) + # REQUIRED: an omitted version used to default to the current one, + # which is precisely the silent reinterpretation the field exists to + # prevent. `None` raises rather than assuming 1. + schema_version: Optional[int] = None + + def __post_init__(self): + self.oracle = Oracle(self.oracle) + self.classification = DiffClassification(self.classification) + self.variable_class = VariableClass(self.variable_class) + if self.schema_version is None: + raise ValueError( + "schema_version is required — an omitted version silently " + f"reinterprets a stored row; write {SCHEMA_VERSION} " + "explicitly" + ) + if self.schema_version != SCHEMA_VERSION: + raise ValueError( + f"schema_version {self.schema_version!r} is not the current " + f"contract version {SCHEMA_VERSION}; migrate explicitly" + ) + if not self.case_id or not self.variable: + raise ValueError("case_id and variable are required") + if not str(self.engine_version).strip() or not str(self.oracle_version).strip(): + raise ValueError( + "engine_version and oracle_version are required — a blank " + "version makes a stored comparison unreproducible" + ) + for name in ("pe_value", "oracle_value"): + v = getattr(self, name) + if v is None: + continue + if isinstance(v, bool) or not isinstance(v, (int, float)) or not _finite(v): + raise ValueError(f"{name} must be a finite number or None, got {v!r}") + if self.variable_class is VariableClass.BOOLEAN and v not in (0, 1): + raise ValueError( + f"boolean-class {name} must be 0 or 1, got {v!r} — a " + "boolean comparison carrying 7 is not a boolean" + ) + if not isinstance(self.annotations, list) or not all( + isinstance(a, str) and a.strip() for a in self.annotations + ): + raise ValueError("annotations must be a list of non-empty strings") + both_numeric = self.pe_value is not None and self.oracle_value is not None + if both_numeric: + expected = abs(self.pe_value - self.oracle_value) + if self.abs_diff is None or abs(self.abs_diff - expected) > 1e-9: + raise ValueError( + f"abs_diff must equal |pe_value - oracle_value| " + f"(got {self.abs_diff}, expected {expected})" + ) + elif self.abs_diff is not None: + raise ValueError("abs_diff requires both values") + if self.pe_value is None and self.classification != DiffClassification.PE_GAP: + raise ValueError("null pe_value requires classification=pe_gap") + if ( + self.oracle_value is None + and self.pe_value is not None + and self.classification != DiffClassification.POLICY_SCOPE_MISMATCH + ): + raise ValueError( + "null oracle_value requires classification=policy_scope_mismatch" + ) + if self.classification == DiffClassification.MATCH_EXACT and ( + not both_numeric or self.abs_diff != 0.0 + ): + raise ValueError("match_exact requires identical numeric values") + if self.classification == DiffClassification.MATCH_WITHIN_TOLERANCE: + if ( + isinstance(self.tolerance, bool) + or not isinstance(self.tolerance, (int, float)) + or self.tolerance <= 0 + ): + raise ValueError( + "match_within_tolerance requires the numeric tolerance " + "the row was judged against (tolerance > 0)" + ) + if not both_numeric or not 0 < self.abs_diff <= self.tolerance: + raise ValueError( + "match_within_tolerance requires 0 < abs_diff <= tolerance " + f"(abs_diff={self.abs_diff}, tolerance={self.tolerance})" + ) + elif self.tolerance is not None: + raise ValueError( + "tolerance is recorded only on match_within_tolerance rows" + ) + # THE SEAM. Recompute rather than trust the caller: the factory + # was a convention, and round-2 probes persisted 100-vs-100 as + # pe_gap, a numeric mismatch as policy_scope_mismatch, and a + # boolean 1-vs-0 as a tolerance match with a caller-supplied + # tolerance of 100. A stored classification is valid only if it + # is what the classifier says, or an explained adjudication of a + # row the classifier left unclassified. + computed = classify(self.pe_value, self.oracle_value, self.variable_class) + if self.classification != computed: + if computed != DiffClassification.UNCLASSIFIED: + raise ValueError( + f"classification {self.classification.value!r} contradicts " + f"the classifier, which says {computed.value!r} for " + f"pe={self.pe_value!r} oracle={self.oracle_value!r} " + f"({self.variable_class.value}). Only an `unclassified` " + "row may be adjudicated." + ) + if self.classification not in ADJUDICATABLE: + raise ValueError( + f"{self.classification.value!r} is not an outcome an " + f"unclassified row may be adjudicated into " + f"({sorted(c.value for c in ADJUDICATABLE)})" + ) + if not self.annotations: + raise ValueError( + f"{self.classification.value} is an adjudicated outcome " + "and requires a writeup in annotations (SCHEMA.md)" + ) + # Tolerance comes from the registry, never from the caller: a + # hand-passed tolerance is how a boolean mismatch became a match. + if self.classification == DiffClassification.MATCH_WITHIN_TOLERANCE: + registered = float(DEFAULT_TOLERANCES[self.variable_class]) + if abs(float(self.tolerance) - registered) > 1e-12: + raise ValueError( + f"tolerance {self.tolerance!r} is not the registered " + f"tolerance for {self.variable_class.value} " + f"({registered}) — tolerances are a registry decision, " + "never a per-row argument" + ) + + @classmethod + def from_classification( + cls, + *, + case_id: str, + variable: str, + pe_value: Optional[float], + oracle_value: Optional[float], + variable_class: VariableClass, + oracle: Oracle, + engine_version: str, + oracle_version: str, + computed_at: str, + annotations: Optional[list] = None, + classification: Optional[DiffClassification] = None, + ) -> "CaseResult": + """The one wiring path from ``classify`` to a stored row. + + Derives abs_diff, threads the registered tolerance (only onto + match_within_tolerance rows, per the field contract), and + persists the variable class. + + ``annotations`` is accepted here because some lanes REQUIRE one — + the calculator oracles (#64) must carry an archive annotation, so + a factory that could not take annotations made every calculator + call raise. ``classification`` is accepted only to record an + adjudication of a row the classifier leaves unclassified; the + constructor re-checks it either way. + """ + variable_class = VariableClass(variable_class) + computed = classify(pe_value, oracle_value, variable_class) + both = pe_value is not None and oracle_value is not None + abs_diff = abs(float(pe_value) - float(oracle_value)) if both else None + tolerance = ( + float(DEFAULT_TOLERANCES[variable_class]) + if computed == DiffClassification.MATCH_WITHIN_TOLERANCE + else None + ) + return cls( + case_id=case_id, + variable=variable, + pe_value=pe_value, + oracle_value=oracle_value, + oracle=oracle, + engine_version=engine_version, + oracle_version=oracle_version, + computed_at=computed_at, + classification=computed if classification is None else classification, + abs_diff=abs_diff, + tolerance=tolerance, + variable_class=variable_class, + annotations=list(annotations or []), + schema_version=SCHEMA_VERSION, + ) + + +def classify( + pe_value: Optional[float], + oracle_value: Optional[float], + variable_class: VariableClass, + tolerance_table: dict = DEFAULT_TOLERANCES, +) -> DiffClassification: + """Automatic first-pass classification of one (pe, oracle) pair. + + Emits only ``CLASSIFIER_EMITTED`` buckets: null sides map to the scope + buckets, exact/tolerance matches to the match buckets, and everything + above tolerance to ``unclassified`` — the adjudication queue, never a + silently flattering default. + """ + variable_class = VariableClass(variable_class) + if pe_value is None: + return DiffClassification.PE_GAP + if oracle_value is None: + return DiffClassification.POLICY_SCOPE_MISMATCH + if variable_class not in tolerance_table: + raise ValueError(f"no tolerance for variable class {variable_class.value!r}") + diff = abs(float(pe_value) - float(oracle_value)) + if diff == 0.0: + return DiffClassification.MATCH_EXACT + if diff <= tolerance_table[variable_class]: + return DiffClassification.MATCH_WITHIN_TOLERANCE + return DiffClassification.UNCLASSIFIED + + +BATTERY_KEYS = frozenset({"schema", "schema_version", "description", "cases"}) +CASE_KEYS = frozenset( + { + "case_id", + "description", + "policy_year", + "country", + "household", + "expected_focus", + "rationale", + "baseline", + } +) + + +def load_battery(path) -> list[CaseSpec]: + """Load and validate a case battery file; every defect raises.""" + raw = json.loads(Path(path).read_text()) + if not isinstance(raw, dict): + raise ValueError("battery must be a JSON object") + unknown = set(raw) - BATTERY_KEYS + if unknown: + raise ValueError(f"battery has unknown keys: {sorted(unknown)}") + if raw.get("schema") not in VALID_BATTERY_SCHEMAS: + raise ValueError( + f"battery schema {raw.get('schema')!r} is not a contract this " + f"repo defines ({sorted(VALID_BATTERY_SCHEMAS)})" + ) + if raw.get("schema_version") != SCHEMA_VERSION: + raise ValueError( + f"battery schema_version must be {SCHEMA_VERSION}, " + f"got {raw.get('schema_version')!r}" + ) + if not isinstance(raw.get("cases"), list) or not raw["cases"]: + raise ValueError("battery.cases must be a non-empty list") + cases = [] + for entry in raw["cases"]: + if not isinstance(entry, dict): + raise ValueError("each case must be a JSON object") + unknown = set(entry) - CASE_KEYS + if unknown: + raise ValueError( + f"case {entry.get('case_id')!r} has unknown keys: {sorted(unknown)}" + ) + cases.append(CaseSpec(**entry)) + ids = [c.case_id for c in cases] + dupes = sorted({i for i in ids if ids.count(i) > 1}) + if dupes: + raise ValueError(f"duplicate case_ids: {dupes}") + return cases diff --git a/scorecard_db/ingest_harvest.py b/scorecard_db/ingest_harvest.py index 52a9c55..4e04169 100644 --- a/scorecard_db/ingest_harvest.py +++ b/scorecard_db/ingest_harvest.py @@ -100,7 +100,18 @@ def sync_lane_feed( ) -> int: """Merge DB lane rows into the committed feed. `lanes` maps lane id -> display meta; defaults to this module's harvest registry (other - exporters pass their own — the merge only touches listed ids).""" + exporters pass their own — the merge only touches listed ids). + + ``updated`` is a LITERAL the caller supplies, not a derived value. + That is the contract, stated here because it is easy to misread: the + feed's top-level `updated` stamp is whatever the LAST caller in a + build passes, so every ingest that syncs this feed must pass the SAME + constant. If two callers disagree, the committed data/lanes.json + drifts depending on ingest order and the no-drift gate fails — the + exact breakage that had to be repaired once already. A lane's own + `updated_at` comes from its DB row and is per-lane; only this + top-level stamp is the shared literal. + """ feed = json.loads(feed_path.read_text()) by_id = {lane["id"]: lane for lane in feed["lanes"]} n = 0 diff --git a/sources/ukmod-cases/SCHEMA.md b/sources/ukmod-cases/SCHEMA.md new file mode 100644 index 0000000..ba157ee --- /dev/null +++ b/sources/ukmod-cases/SCHEMA.md @@ -0,0 +1,287 @@ +# Mode-3 case-diff schema (shared: UKMOD/EUROMOD and TAXSIM lanes) + +The scorecard's third claim class: **record-level case diffs**. Instead of a +published aggregate, the external model is run as an *oracle* on curated +hypothetical households, and PolicyEngine's per-variable outputs are compared +case by case. One schema serves every mode-3 lane — this document is the +single contract for both `ukmod-cases` (#41) and `taxsim-cases` (#5); the +`country` and `oracle` fields are what keep it engine-agnostic. Do not fork a +second schema for a new oracle: extend the closed vocabularies here, fail-loud +style, and note the extension. + +Code: `scorecard_db/case_diffs.py` (dataclasses, battery loader, classifier). +Battery: `sources/ukmod-cases/battery/cases.json`. Tests: +`tests/test_case_schema.py`. + +## Case (input) — `CaseSpec` + +A case is a fully specified hypothetical household plus the variables it is +designed to exercise. **Inputs only** — a case never embeds expected output +values; both sides of every comparison come from engine runs (PE and the +oracle), so the battery cannot smuggle in hand-computed "truth". + +```json +{ + "case_id": "uk-uc-single-unemployed", + "description": "Single unemployed adult, social rent, on UC", + "policy_year": 2026, + "country": "UK", + "household": { ... }, + "expected_focus": ["universal_credit", "income_tax"], + "rationale": "Why this case is in the battery, and what edge it pins" +} +``` + +- `case_id` — globally unique slug, prefixed with the country + (`uk-…`, `us-…`); stable forever once results reference it. +- `policy_year` — the tax-benefit year the case is evaluated in (for the UK, + the fiscal year starting 6 April of that calendar year). +- `country` — closed set, currently `{"UK", "US"}`; extend when a new + mode-3 lane lands (EUROMOD EU countries, state calculators). +- `expected_focus` — the output variables (canonical names, below) the case + was designed to exercise. The runner computes and diffs *all* mapped + variables; `expected_focus` drives coverage accounting (every battery must + exercise each focus area at least once) and diff triage order. +- `rationale` — human audit trail: why these numbers, what boundary they sit + on. + +### Household spec (engine-agnostic) + +The household is described in a small, closed, engine-neutral input +vocabulary. Each lane's connector owns the mapping from this vocabulary to +its engines' variables (PE UK / UKMOD policy spine; PE US / TAXSIM v35 +columns); the mapping table lives with the connector, never in the battery. +Unknown keys are a hard error — adding an input concept means extending the +vocabulary in `case_diffs.py` and documenting it here. + +```json +{ + "people": { + "adult_1": {"age": 35, "employment_income": 12000}, + "child_1": {"age": 8} + }, + "benefit_units": [ + {"adults": ["adult_1"], "children": ["child_1"]} + ], + "region": "LONDON", + "tenure": "rented_private", + "rent": 13000 +} +``` + +Conventions: + +- **All monetary amounts are annual, in the country's currency** (GBP for + UK, USD for US). Weekly-quoted UK amounts enter as weekly × 52 (round, + auditable numbers preferred — e.g. £250/week rent = 13000). +- Every person appears in exactly one benefit unit. A benefit unit is the + assessment unit (UK: single/couple + dependent children; US: the tax + unit). Multi-benefit-unit households are allowed (e.g. a non-dependant + adult) but every person must be assigned. +- Omitted person keys default to 0 / false. Only non-defaults are written, + so each case reads as exactly its rationale. + +Person keys (closed set): + +| key | type | meaning | +|---|---|---| +| `age` | int, required | age at the start of the policy year | +| `date_of_birth` | `YYYY-MM-DD`, a real calendar date (`2026-13-40` fails) | only when the exact date is load-bearing (two-child-limit protection, state pension age) | +| `employment_income` | number ≥ 0 | gross annual employee earnings | +| `self_employment_income` | number ≥ 0 | annual trading profit | +| `pension_income` | number ≥ 0 | private/occupational pension in payment | +| `state_pension` | number ≥ 0 | annual state pension in payment | +| `savings_income` | number ≥ 0 | annual interest income | +| `capital` | number ≥ 0 | liquid capital / savings stock | +| `employee_pension_contributions` | number ≥ 0 | annual employee pension contributions; `salary_sacrifice` true means they reduce gross pay for tax and NI | +| `salary_sacrifice` | bool | pension contributions are via salary sacrifice | +| `is_disabled` | bool | disabled for benefit purposes (drives disability elements/premia; the connector maps to each engine's disability concept and records the mapping) | +| `is_carer` | bool | provides ≥ 35 hours/week care for a disabled person (Carer's Allowance / UC carer element eligibility) | +| `gainfully_self_employed` | bool | in gainful self-employment for UC (minimum income floor applies after the start-up period) | +| `hours_worked_per_week` | number ≥ 0 | contracted weekly hours | + +Household keys (closed set): `people`, `benefit_units`, `region`, `tenure` +(`owned_outright` | `owned_mortgage` | `rented_social` | `rented_private`), +`rent` (annual), `council_tax` (annual, optional), `brma`. `region` uses +the target country's own geography vocabulary (UK: ITL-1 slugs like +`LONDON`, `SCOTLAND`; US: state codes) — it is what routes devolved policy +(Scottish income tax, Scottish Child Payment; US state taxes) and the +benefit-cap tier. It is a CLOSED registry (`VALID_REGIONS`), never free +text: an unrecognised region would otherwise reach a connector that +silently falls back to a default. + +`brma` is REQUIRED on a UK `rented_private` case and forbidden elsewhere. +Local Housing Allowance rates are set per Broad Rental Market Area, not +per ITL-1 region, so a region alone does not fix the world the two engines +run — "the rent clears the cap in most Yorkshire BRMAs" is not a pinned +comparison. `BRMA_REGION` is the registry, and each entry names the region +it sits in, so a case cannot pin a BRMA outside its own region. + +`expected_focus` is likewise closed per country (`FOCUS_VARIABLES`): it +names the output variables the case is curated to exercise, and a name no +connector maps produces no comparison at all rather than an error. + +Amounts must be FINITE. NaN silently defeats every comparison operator, +and an infinite rent produces an infinite entitlement rather than a +failure. + +### `baseline` — the policy world a case is evaluated in + +Optional; absent means current law. A descriptor of the same shape +`ReformRef.baseline` carries, and it must already be REGISTERED in +`scorecard_db/baselines.py` — an unnamed counterfactual is exactly the +"which law did this run?" ambiguity mode 3 exists to remove. + +This is how a case pins a SAME-YEAR counterfactual instead of leaning on a +year-on-year delta. The two-child-limit family uses it: three 2026 cases, +`uk-uc-two-child-limit-binding` and `uk-uc-two-child-limit-multiple-birth` +in the registered `pre_ab2025` world (limit reinstated at two children) +and `uk-uc-two-child-limit-abolished` under current law. The binding / +abolished pair differs only in the world, so it attributes the abolition; +the binding / multiple-birth pair differs only in `child_3`'s date of +birth, so it attributes the exception. An earlier design paired a 2025 +case against a 2026 one, which moved ages, uprated rates and the policy +year together, and used twins in the "limit in force" case — where an +engine that wrongly kept the limit but rightly applied the multiple-birth +exception still paid three elements and passed. + +## Result row — `CaseResult` + +One row per case × output variable × run. History is preserved: re-running +on a new engine or oracle version appends, never overwrites (same doctrine +as `pe_results`). + +`variable_class` and `schema_version` are REQUIRED. Both used to be +omissible, and an omitted `schema_version` defaulted to the current one — +precisely the silent reinterpretation the field exists to prevent. +`engine_version` and `oracle_version` may not be blank. A boolean-class +comparison may only carry 0 or 1 on either side. + +### The classify → CaseResult seam is enforced, not conventional + +`CaseResult.__post_init__` RECOMPUTES `classify()` and compares. A stored +classification is valid only if it is what the classifier says, or if it +is an ADJUDICATION of a row the classifier left `unclassified`, drawn from +`ADJUDICATABLE` (`oracle_difference`, `rounding`, `pe_gap`, +`policy_scope_mismatch`) and carrying a writeup in `annotations`. A row +the classifier has already decided is not up for reinterpretation, and +nothing may be adjudicated INTO a match. + +Note the two-sidedness of `pe_gap` and `policy_scope_mismatch`: the +classifier emits them mechanically for a null side, and a human may also +assign them to a numeric-vs-numeric difference — where they are a +judgement and need explaining. Previously only `oracle_difference` and +`rounding` demanded a writeup, so `100` vs `200` could be filed as a +`pe_gap` with nothing said. + +Tolerances come from `DEFAULT_TOLERANCES` alone. `from_classification` +no longer accepts a tolerance table and a hand-written row carrying a +tolerance other than the registered one is rejected — a caller-supplied +tolerance is how a boolean `1` vs `0` became a `match_within_tolerance`. +`from_classification` does accept `annotations`, because some lanes +require one (the #64 calculator rows must carry an archive annotation). + +### Not yet built + +There is no case/result TABLE, writer, exporter or `build_db` step, so +mode-3 rows do not persist to the DB yet. The epistemic wiring that rides +on those columns — `calibration_relationship`, the executed baseline key, +case/battery digests, engine bundle and run pins, the JRC connector +revision, and a citable adjudication link — lands with that table, as does +splitting the descriptive comparison status from the normative diagnosis +the way `external_scores` and `diagnoses` already do. That design has +repo-wide surface and is being paired on rather than decided here. + +```json +{ + "case_id": "uk-uc-single-unemployed", + "variable": "universal_credit", + "pe_value": 4796.48, + "oracle_value": 4796.52, + "oracle": "ukmod", + "engine_version": "policyengine-uk 2.x.y", + "oracle_version": "UKMOD B2026.08 / EUROMOD I7.0+", + "computed_at": "2026-08-14T12:00:00Z", + "abs_diff": 0.04, + "tolerance": 0.52, + "variable_class": "currency", + "classification": "match_within_tolerance", + "schema_version": 1 +} +``` + +- `variable` — canonical output name in PE's vocabulary for the case's + country (`universal_credit`, `income_tax`, `national_insurance`, + `child_benefit`, `pension_credit`, …). The connector's mapping table + pairs it with the oracle's variable (UKMOD `bsauc_s`, etc.) and records + any construction (e.g. summing UKMOD monthly output × 12). +- `oracle` — closed set `{"ukmod", "taxsim"}`; grows with #5. +- `pe_value` / `oracle_value` — annual amounts (booleans as 0/1). `null` + means that side cannot produce the variable: `pe_value: null` ⇒ + `pe_gap`; `oracle_value: null` ⇒ `policy_scope_mismatch` (the oracle + does not model it — e.g. TAXSIM has no benefits; UKMOD's UK model omits + some devolved payments depending on version). +- `abs_diff` — `|pe_value − oracle_value|` when both sides are numeric, + else `null`. Stored, not derived at read time, so the miss table is + self-contained. +- `tolerance` — required on (and only on) `match_within_tolerance` rows: + the numeric tolerance the row was judged against, satisfying + `0 < abs_diff ≤ tolerance`. Stored so a tolerance-table change can never + silently re-bless old rows. +- `variable_class` — which tolerance rule applied (`currency` / + `boolean`), persisted so a stored row can be re-classified and + audited without inferring the class from the free-text `variable`. +- `schema_version` — this contract's version (currently `1`), on the + battery file and every result row; a mismatched version raises, so + a future breaking change migrates stored artifacts explicitly. +- Connectors build rows via `CaseResult.from_classification(...)` — + the one wiring path from `classify` to a stored row. It derives + `abs_diff`, threads the exact tolerance `classify` judged against, + and persists `variable_class`, so no caller re-derives the + tolerance by hand. +- `annotations` — a list of non-empty strings. Required (non-empty) on the + adjudicated-only classifications `oracle_difference` and `rounding`: the + traceable writeup naming the oracle defect / documented rounding rule. + A row claiming either without a writeup fails validation. + +## Classification (closed set — fail loud) + +| classification | meaning | assigned by | +|---|---|---| +| `match_exact` | values identical | classifier | +| `match_within_tolerance` | nonzero diff ≤ the variable class's tolerance | classifier | +| `pe_gap` | PE cannot produce the variable, or adjudication found PE wrong | classifier (null side) / adjudication | +| `oracle_difference` | adjudication found the oracle wrong (upstream report filed) | adjudication | +| `policy_scope_mismatch` | the two engines model different policy scope for this variable (documented) | classifier (null side) / adjudication | +| `rounding` | diff fully explained by the oracle's documented rounding rules (e.g. UKMOD monthly rounding × 12) | adjudication | +| `unclassified` | diff above tolerance, not yet adjudicated | classifier (default) | + +The classifier (`case_diffs.classify`) only ever emits `match_exact`, +`match_within_tolerance`, `pe_gap` / `policy_scope_mismatch` (null sides), +or `unclassified`. Every `unclassified` row is a work item: adjudication — +the diagnosis stage, human-or-agent, with a traceable writeup — moves it to +`pe_gap`, `oracle_difference`, `policy_scope_mismatch`, or `rounding`. +Nothing defaults to a flattering bucket; misses stay visible, exactly as in +modes 1–2. Unknown classification strings raise. The writeup requirement is +enforced at validation: `oracle_difference` and `rounding` rows raise +without a non-empty `annotations` writeup, and `match_within_tolerance` +rows raise without the `tolerance` they were judged against. + +## Tolerances (per variable class) + +| variable class | tolerance | rationale | +|---|---|---| +| `currency` | £0.01/week ⇒ **0.52/year** (same rule in USD for TAXSIM) | benefit rules are stated weekly to the penny; comparisons are annual, so 52 × 0.01. A wider tolerance is allowed only with a documented oracle rounding rule, and then the adjudicated class is `rounding`, not a silent tolerance bump. | +| `boolean` | exact (0) | eligibility flags either agree or they don't | + +The tolerance table is data (`DEFAULT_TOLERANCES`), passed to `classify` +explicitly; a variable class missing from the table raises rather than +guessing. + +## What this lane still needs (out of scope here) + +The JRC connector run (runnable per axiom-oracles#264, needs the UKMOD +environment): map the vocabulary to UKMOD input variables, execute the +battery on both engines, append `CaseResult` rows, publish the miss table, +and advance the lane. Until then the battery is inputs + focus only, by +design. diff --git a/sources/ukmod-cases/battery/cases.json b/sources/ukmod-cases/battery/cases.json new file mode 100644 index 0000000..7667b30 --- /dev/null +++ b/sources/ukmod-cases/battery/cases.json @@ -0,0 +1,629 @@ +{ + "schema": "sources/ukmod-cases/SCHEMA.md", + "schema_version": 1, + "description": "Initial UK mode-3 case battery for the UKMOD/EUROMOD lane (#41): standard EUROMOD-style hypothetical households plus the edge cases PE UK carries regression coverage for. Inputs + expected focus only \u2014 expected output values come from engine runs (PE and the oracle), never from this file. All monetary amounts are annual GBP; weekly-quoted amounts enter as weekly x 52.", + "cases": [ + { + "case_id": "uk-uc-single-unemployed", + "description": "Single unemployed adult on UC, social rent", + "policy_year": 2026, + "country": "UK", + "household": { + "people": { + "adult_1": { + "age": 30 + } + }, + "benefit_units": [ + { + "adults": [ + "adult_1" + ], + "children": [] + } + ], + "region": "NORTH_EAST", + "tenure": "rented_social", + "rent": 6240 + }, + "expected_focus": [ + "universal_credit", + "income_tax", + "national_insurance" + ], + "rationale": "The simplest UC case: standard allowance (single 25+) plus social-rent housing element with no deductions. Rent GBP 120/week = 6,240/year. Pins the standard allowance and housing element before any taper, cap, or element interacts." + }, + { + "case_id": "uk-uc-lone-parent-two-children-rent", + "description": "Workless lone parent, two children, private rent", + "policy_year": 2026, + "country": "UK", + "household": { + "people": { + "adult_1": { + "age": 32 + }, + "child_1": { + "age": 8 + }, + "child_2": { + "age": 4 + } + }, + "benefit_units": [ + { + "adults": [ + "adult_1" + ], + "children": [ + "child_1", + "child_2" + ] + } + ], + "region": "YORKSHIRE", + "tenure": "rented_private", + "rent": 9360, + "brma": "Leeds" + }, + "expected_focus": [ + "universal_credit", + "child_benefit" + ], + "rationale": "UC child elements (two children, both post-April-2017 so no protected rate) plus the private-rent housing element via LHA (2-bed rate, Yorkshire BRMA) and Child Benefit for two. Rent GBP 180/week = 9,360/year, below the LHA cap in most Yorkshire BRMAs so the LHA-vs-actual-rent minimum is exercised without binding." + }, + { + "case_id": "uk-uc-two-child-limit-binding", + "description": "Workless couple, three singleton-birth children, limit in force", + "policy_year": 2026, + "country": "UK", + "household": { + "people": { + "adult_1": { + "age": 37 + }, + "adult_2": { + "age": 35 + }, + "child_1": { + "age": 7, + "date_of_birth": "2018-06-01" + }, + "child_2": { + "age": 3, + "date_of_birth": "2022-09-01" + }, + "child_3": { + "age": 3, + "date_of_birth": "2022-11-15" + } + }, + "benefit_units": [ + { + "adults": [ + "adult_1", + "adult_2" + ], + "children": [ + "child_1", + "child_2", + "child_3" + ] + } + ], + "region": "WEST_MIDLANDS", + "tenure": "rented_social", + "rent": 8320 + }, + "expected_focus": [ + "universal_credit", + "child_benefit" + ], + "baseline": { + "policy": "pre_ab2025" + }, + "rationale": "The limit itself, with nothing else to hide behind. Three post-April-2017 children, all SEPARATE births (child_2 and child_3 have different dates of birth), so no exception of any kind applies: an engine applying the two-child limit pays two child elements and an engine that has dropped it pays three. The earlier version of this case used twins, which meant an engine that wrongly retained the limit but rightly applied the multiple-birth exception still paid three elements \u2014 it could not catch the failure its own rationale named. Evaluated in the registered pre_ab2025 world (limit reinstated at two children), so the case does not depend on a pre-abolition policy year." + }, + { + "case_id": "uk-uc-two-child-limit-abolished", + "description": "The same 2026 household under current law (limit abolished)", + "policy_year": 2026, + "country": "UK", + "household": { + "people": { + "adult_1": { + "age": 37 + }, + "adult_2": { + "age": 35 + }, + "child_1": { + "age": 7, + "date_of_birth": "2018-06-01" + }, + "child_2": { + "age": 3, + "date_of_birth": "2022-09-01" + }, + "child_3": { + "age": 3, + "date_of_birth": "2022-11-15" + } + }, + "benefit_units": [ + { + "adults": [ + "adult_1", + "adult_2" + ], + "children": [ + "child_1", + "child_2", + "child_3" + ] + } + ], + "region": "WEST_MIDLANDS", + "tenure": "rented_social", + "rent": 8320 + }, + "expected_focus": [ + "universal_credit", + "child_benefit" + ], + "rationale": "The abolition counterpart to uk-uc-two-child-limit-binding: byte-identical household, SAME policy year, same rates, same ages \u2014 the only difference is the policy world (current law, where AB2025 removed the limit from 6 April 2026, versus the registered pre_ab2025 counterfactual). The pair therefore isolates the abolition itself. The earlier year-on-year pair moved ages, uprated rates and the policy year together, so its delta could not attribute anything." + }, + { + "case_id": "uk-uc-two-child-limit-multiple-birth", + "description": "Same household with a multiple birth, limit in force", + "policy_year": 2026, + "country": "UK", + "household": { + "people": { + "adult_1": { + "age": 37 + }, + "adult_2": { + "age": 35 + }, + "child_1": { + "age": 7, + "date_of_birth": "2018-06-01" + }, + "child_2": { + "age": 3, + "date_of_birth": "2022-09-01" + }, + "child_3": { + "age": 3, + "date_of_birth": "2022-09-01" + } + }, + "benefit_units": [ + { + "adults": [ + "adult_1", + "adult_2" + ], + "children": [ + "child_1", + "child_2", + "child_3" + ] + } + ], + "region": "WEST_MIDLANDS", + "tenure": "rented_social", + "rent": 8320 + }, + "expected_focus": [ + "universal_credit", + "child_benefit" + ], + "baseline": { + "policy": "pre_ab2025" + }, + "rationale": "The multiple-birth exception, isolated against uk-uc-two-child-limit-binding: identical household, identical ages, identical pre_ab2025 world, differing in exactly one input \u2014 child_3's date of birth, which makes children 2 and 3 a multiple birth. The exception sits on a third-or-later child, so it strictly changes entitlement (three elements vs the binding case's two), and the one-field difference is what attributes the change to the exception rather than to the limit. Also pins that Child Benefit ignores the limit in both." + }, + { + "case_id": "uk-benefit-cap-london", + "description": "Workless couple, two children, high London private rent", + "policy_year": 2026, + "country": "UK", + "household": { + "people": { + "adult_1": { + "age": 33 + }, + "adult_2": { + "age": 31 + }, + "child_1": { + "age": 6 + }, + "child_2": { + "age": 3 + } + }, + "benefit_units": [ + { + "adults": [ + "adult_1", + "adult_2" + ], + "children": [ + "child_1", + "child_2" + ] + } + ], + "region": "LONDON", + "tenure": "rented_private", + "rent": 20800, + "brma": "Inner London" + }, + "expected_focus": [ + "benefit_cap", + "universal_credit", + "child_benefit" + ], + "rationale": "Benefit cap binding: rent GBP 400/week = 20,800/year drives pre-cap UC (standard + 2 child elements + housing) above the Greater London couple cap, so the cap reduction is nonzero. Exercises the London cap tier and the cap-applies-to-UC-not-Child-Benefit accounting." + }, + { + "case_id": "uk-uc-taper-part-time-earner", + "description": "Lone parent working part time, UC taper and work allowance", + "policy_year": 2026, + "country": "UK", + "household": { + "people": { + "adult_1": { + "age": 29, + "employment_income": 12000, + "hours_worked_per_week": 20 + }, + "child_1": { + "age": 5 + } + }, + "benefit_units": [ + { + "adults": [ + "adult_1" + ], + "children": [ + "child_1" + ] + } + ], + "region": "NORTH_WEST", + "tenure": "rented_private", + "rent": 7800, + "brma": "Manchester" + }, + "expected_focus": [ + "universal_credit", + "income_tax", + "national_insurance", + "child_benefit" + ], + "rationale": "The 55% UC earnings taper above the lower (with-housing) work allowance. Earnings GBP 12,000 sit below the personal allowance (income tax GBP 0) but above the NI primary threshold pro-rata, so net-earnings-for-UC differs from gross \u2014 the taper base (net of tax and NI) is the thing under test. Rent GBP 150/week = 7,800/year." + }, + { + "case_id": "uk-uc-minimum-income-floor", + "description": "Gainfully self-employed single adult below the MIF", + "policy_year": 2026, + "country": "UK", + "household": { + "people": { + "adult_1": { + "age": 40, + "self_employment_income": 6000, + "gainfully_self_employed": true + } + }, + "benefit_units": [ + { + "adults": [ + "adult_1" + ], + "children": [] + } + ], + "region": "EAST_MIDLANDS", + "tenure": "rented_private", + "rent": 7280, + "brma": "Nottingham" + }, + "expected_focus": [ + "universal_credit" + ], + "rationale": "Minimum income floor: actual trading profit GBP 6,000/year is far below the assumed floor (35 hours x NLW net of notional tax/NI), so UC is tapered on the floor, not on actual income. Past the 12-month start-up period by assumption (gainfully_self_employed = true). Pins the floor construction, a known cross-model divergence spot." + }, + { + "case_id": "uk-pension-credit-single", + "description": "Single pensioner, state pension only, guarantee credit", + "policy_year": 2026, + "country": "UK", + "household": { + "people": { + "adult_1": { + "age": 70, + "state_pension": 9000 + } + }, + "benefit_units": [ + { + "adults": [ + "adult_1" + ], + "children": [] + } + ], + "region": "SOUTH_WEST", + "tenure": "owned_outright", + "rent": 0 + }, + "expected_focus": [ + "pension_credit", + "income_tax" + ], + "rationale": "Guarantee credit tops a GBP 9,000 state pension up to the single-person standard minimum guarantee; income below the personal allowance so income tax is zero. Deliberately below the full new state pension so the top-up is strictly positive." + }, + { + "case_id": "uk-pension-credit-couple", + "description": "Pensioner couple, mixed state and private pension income", + "policy_year": 2026, + "country": "UK", + "household": { + "people": { + "adult_1": { + "age": 72, + "state_pension": 9500, + "pension_income": 1000 + }, + "adult_2": { + "age": 70, + "state_pension": 4500 + } + }, + "benefit_units": [ + { + "adults": [ + "adult_1", + "adult_2" + ], + "children": [] + } + ], + "region": "WALES", + "tenure": "owned_outright", + "rent": 0 + }, + "expected_focus": [ + "pension_credit", + "income_tax" + ], + "rationale": "Couple guarantee credit with joint income assessment: GBP 15,000 combined (two state pensions of 9,500 and 4,500 plus a 1,000 private pension) against the couple standard minimum guarantee. Pins joint aggregation and that private pension income counts in full." + }, + { + "case_id": "uk-mixed-age-couple-uc", + "description": "Mixed-age couple routed to UC, not Pension Credit", + "policy_year": 2026, + "country": "UK", + "household": { + "people": { + "adult_1": { + "age": 68, + "state_pension": 9500 + }, + "adult_2": { + "age": 60 + } + }, + "benefit_units": [ + { + "adults": [ + "adult_1", + "adult_2" + ], + "children": [] + } + ], + "region": "NORTH_EAST", + "tenure": "rented_social", + "rent": 6760 + }, + "expected_focus": [ + "universal_credit", + "pension_credit" + ], + "rationale": "Since May 2019 a mixed-age couple (one over, one under state pension age) claims UC, not Pension Credit. Expect pension_credit = 0 and UC assessed with the state pension counted as unearned income (GBP-for-GBP deduction). A known routing rule models get wrong in both directions. Rent GBP 130/week = 6,760/year." + }, + { + "case_id": "uk-scotland-earner-scp", + "description": "Scottish couple, one earner, two young children, on UC", + "policy_year": 2026, + "country": "UK", + "household": { + "people": { + "adult_1": { + "age": 35, + "employment_income": 22000, + "hours_worked_per_week": 35 + }, + "adult_2": { + "age": 33 + }, + "child_1": { + "age": 7 + }, + "child_2": { + "age": 2 + } + }, + "benefit_units": [ + { + "adults": [ + "adult_1", + "adult_2" + ], + "children": [ + "child_1", + "child_2" + ] + } + ], + "region": "SCOTLAND", + "tenure": "rented_social", + "rent": 8840 + }, + "expected_focus": [ + "income_tax", + "universal_credit", + "scottish_child_payment", + "child_benefit" + ], + "rationale": "Devolution double-check: GBP 22,000 earnings span the Scottish starter, basic, and intermediate bands (different liability than rUK at the same gross), while UC entitlement plus two under-16s triggers Scottish Child Payment for both children. Exercises whether the oracle models devolved Scottish payments at all \u2014 a candidate policy_scope_mismatch, which is exactly why it is in the battery." + }, + { + "case_id": "uk-hicbc-partial-taper", + "description": "Higher-rate earner at GBP 70,000 with Child Benefit (HICBC mid-taper)", + "policy_year": 2026, + "country": "UK", + "household": { + "people": { + "adult_1": { + "age": 41, + "employment_income": 70000 + }, + "adult_2": { + "age": 39 + }, + "child_1": { + "age": 10 + }, + "child_2": { + "age": 7 + } + }, + "benefit_units": [ + { + "adults": [ + "adult_1", + "adult_2" + ], + "children": [ + "child_1", + "child_2" + ] + } + ], + "region": "SOUTH_EAST", + "tenure": "owned_mortgage", + "rent": 0 + }, + "expected_focus": [ + "income_tax", + "child_benefit" + ], + "rationale": "High Income Child Benefit Charge at the midpoint of the post-2024 GBP 60,000-80,000 taper: adjusted net income 70,000 claws back exactly half the two-child Child Benefit through the earner's income tax. Round midpoint chosen so the expected clawback fraction (1/2) is auditable by inspection." + }, + { + "case_id": "uk-ni-thresholds-salary-sacrifice", + "description": "Couple pinned at the NI primary threshold and UEL, with salary sacrifice", + "policy_year": 2026, + "country": "UK", + "household": { + "people": { + "adult_1": { + "age": 45, + "employment_income": 12570 + }, + "adult_2": { + "age": 44, + "employment_income": 50270, + "employee_pension_contributions": 2000, + "salary_sacrifice": true + } + }, + "benefit_units": [ + { + "adults": [ + "adult_1", + "adult_2" + ], + "children": [] + } + ], + "region": "EAST_OF_ENGLAND", + "tenure": "owned_mortgage", + "rent": 0 + }, + "expected_focus": [ + "national_insurance", + "income_tax" + ], + "rationale": "Threshold edges: adult_1 sits exactly at GBP 12,570 (personal allowance = NI primary threshold; both liabilities should be exactly zero). Adult_2 sits at the GBP 50,270 UEL with a GBP 2,000 salary sacrifice, so NI-able and taxable pay drop to 48,270 \u2014 pinning that sacrifice reduces both tax and Class 1 NI, and that the main-vs-additional NI rate boundary is applied to post-sacrifice pay." + }, + { + "case_id": "uk-carer-ca-uc", + "description": "Single carer on Carer's Allowance and UC with carer element", + "policy_year": 2026, + "country": "UK", + "household": { + "people": { + "adult_1": { + "age": 45, + "is_carer": true + } + }, + "benefit_units": [ + { + "adults": [ + "adult_1" + ], + "children": [] + } + ], + "region": "NORTH_WEST", + "tenure": "rented_social", + "rent": 6240 + }, + "expected_focus": [ + "carers_allowance", + "universal_credit" + ], + "rationale": "Carer's Allowance plus UC carer element interaction: CA is unearned income deducted from UC GBP-for-GBP, while the carer element is added \u2014 the net gain should equal the carer element alone. The cared-for person (receiving a qualifying disability benefit) lives outside the household, per the is_carer flag's definition. A classic double-counting trap between models." + }, + { + "case_id": "uk-pa-taper-110k", + "description": "Single earner at GBP 110,000, personal allowance taper", + "policy_year": 2026, + "country": "UK", + "household": { + "people": { + "adult_1": { + "age": 48, + "employment_income": 110000 + } + }, + "benefit_units": [ + { + "adults": [ + "adult_1" + ], + "children": [] + } + ], + "region": "LONDON", + "tenure": "owned_mortgage", + "rent": 0 + }, + "expected_focus": [ + "income_tax", + "national_insurance" + ], + "rationale": "The GBP 100,000 cliff: at 110,000 the personal allowance is tapered by GBP 1 per 2 over 100,000 (5,000 of allowance lost), producing the 60% effective marginal band. Round 10,000 excess makes the expected allowance (12,570 - 5,000 = 7,570) auditable by inspection." + } + ] +} diff --git a/tests/test_case_schema.py b/tests/test_case_schema.py new file mode 100644 index 0000000..6a668e2 --- /dev/null +++ b/tests/test_case_schema.py @@ -0,0 +1,615 @@ +"""The mode-3 case-diff schema (#41, shared with taxsim-cases per #5): +the UK battery validates, ids are unique and country-prefixed, cases stay +inputs-only, and the classifier's closed vocabulary fails loudly.""" + +import json +from pathlib import Path + +import pytest + +from scorecard_db.case_diffs import ( + BRMA_REGION, + SCHEMA_VERSION, + DEFAULT_TOLERANCES, + CaseResult, + CaseSpec, + DiffClassification, + Oracle, + VariableClass, + classify, + load_battery, +) + +BATTERY = Path(__file__).parent.parent / "sources/ukmod-cases/battery/cases.json" + + +def case_kwargs(**kw): + base = dict( + case_id="uk-test-case", + description="test", + policy_year=2026, + country="UK", + household={ + "people": {"adult_1": {"age": 30}}, + "benefit_units": [{"adults": ["adult_1"], "children": []}], + "tenure": "rented_social", + "rent": 6240, + }, + expected_focus=["universal_credit"], + ) + base.update(kw) + return base + + +class TestBattery: + def test_loads_and_validates(self): + cases = load_battery(BATTERY) + assert 12 <= len(cases) <= 20 + assert all(isinstance(c, CaseSpec) for c in cases) + + def test_unique_ids(self): + cases = load_battery(BATTERY) + ids = [c.case_id for c in cases] + assert len(ids) == len(set(ids)) + + def test_uk_battery_shape(self): + for c in load_battery(BATTERY): + assert c.country == "UK" + assert c.case_id.startswith("uk-") + # Every case is a 2026 case now: the two-child family reaches + # the pre-abolition world through the registered pre_ab2025 + # baseline instead of through an earlier policy year, so no + # comparison has to straddle two uprating rounds. + assert c.policy_year == 2026 + assert c.rationale # every case explains itself + + def test_two_child_limit_cases_isolate_one_mechanism_each(self): + """Three cases, two same-year pairs, each differing in exactly + one thing — the round-2 finding was that the old pair changed the + policy world, the ages and the uprating round all at once, and + that its 'limit in force' case used twins, so an engine that + wrongly kept the limit but rightly applied the multiple-birth + exception still paid three elements and passed.""" + by_id = {c.case_id: c for c in load_battery(BATTERY)} + binding = by_id["uk-uc-two-child-limit-binding"] + abolished = by_id["uk-uc-two-child-limit-abolished"] + twins = by_id["uk-uc-two-child-limit-multiple-birth"] + + for case in (binding, abolished, twins): + children = case.household["benefit_units"][0]["children"] + assert len(children) == 3 + # Every child post-April-2017, so no pre-2017 protection can + # stand in for the mechanism under test. + for child in children: + assert case.household["people"][child]["date_of_birth"] > "2017-04-06" + assert case.policy_year == 2026 # one uprating round for all three + + # Pair 1 — abolition: same household, same year, different world. + assert binding.household == abolished.household + assert binding.baseline == {"policy": "pre_ab2025"} + assert abolished.baseline is None + + # Pair 2 — the exception: same world, same year, ONE input apart. + assert twins.baseline == binding.baseline + differing = [ + pid + for pid in binding.household["people"] + if binding.household["people"][pid] != twins.household["people"][pid] + ] + assert differing == ["child_3"] + assert ( + binding.household["people"]["child_3"]["age"] + == twins.household["people"]["child_3"]["age"] + ) + + # The binding case has no exception to hide behind: three + # separate births, so the limit itself decides the entitlement. + b_people = binding.household["people"] + assert len({b_people[c]["date_of_birth"] for c in ("child_2", "child_3")}) == 2 + # ...and the multiple birth spans a third-or-later child. + t_people = twins.household["people"] + assert ( + t_people["child_2"]["date_of_birth"] == t_people["child_3"]["date_of_birth"] + ) + + def test_battery_is_inputs_only(self): + # No case embeds expected output values (SCHEMA.md doctrine). + raw = json.loads(BATTERY.read_text()) + for entry in raw["cases"]: + assert set(entry) <= { + "case_id", + "description", + "policy_year", + "country", + "household", + "expected_focus", + "rationale", + "baseline", + } + + def test_focus_coverage(self): + focus = {v for c in load_battery(BATTERY) for v in c.expected_focus} + assert { + "universal_credit", + "income_tax", + "national_insurance", + "child_benefit", + "pension_credit", + "benefit_cap", + "scottish_child_payment", + "carers_allowance", + } <= focus + + def test_duplicate_ids_rejected(self, tmp_path): + raw = json.loads(BATTERY.read_text()) + raw["cases"].append(dict(raw["cases"][0])) + p = tmp_path / "cases.json" + p.write_text(json.dumps(raw)) + with pytest.raises(ValueError, match="duplicate case_ids"): + load_battery(p) + + def test_unknown_case_key_rejected(self, tmp_path): + raw = json.loads(BATTERY.read_text()) + raw["cases"][0]["expected_value"] = 123.0 + p = tmp_path / "cases.json" + p.write_text(json.dumps(raw)) + with pytest.raises(ValueError, match="unknown keys"): + load_battery(p) + + +class TestCaseSpec: + def test_unknown_country_rejected(self): + with pytest.raises(ValueError, match="unknown country"): + CaseSpec(**case_kwargs(country="FR")) + + def test_country_prefix_enforced(self): + with pytest.raises(ValueError, match="prefixed"): + CaseSpec(**case_kwargs(case_id="single-unemployed")) + + def test_unknown_person_key_rejected(self): + kw = case_kwargs() + kw["household"]["people"]["adult_1"]["wages"] = 1000 + with pytest.raises(ValueError, match="unknown keys"): + CaseSpec(**kw) + + def test_unassigned_person_rejected(self): + kw = case_kwargs() + kw["household"]["people"]["adult_2"] = {"age": 40} + with pytest.raises(ValueError, match="exactly once"): + CaseSpec(**kw) + + def test_owner_with_rent_rejected(self): + kw = case_kwargs() + kw["household"]["tenure"] = "owned_outright" + with pytest.raises(ValueError, match="no rent"): + CaseSpec(**kw) + + def test_missing_age_rejected(self): + kw = case_kwargs() + kw["household"]["people"]["adult_1"] = {"employment_income": 1000} + with pytest.raises(ValueError, match="age"): + CaseSpec(**kw) + + def test_empty_focus_rejected(self): + with pytest.raises(ValueError, match="expected_focus"): + CaseSpec(**case_kwargs(expected_focus=[])) + + def test_string_focus_rejected(self): + # A bare string iterates as characters; the schema demands a list. + with pytest.raises(ValueError, match="expected_focus"): + CaseSpec(**case_kwargs(expected_focus="universal_credit")) + + def test_string_unit_members_rejected(self): + kw = case_kwargs() + kw["household"]["benefit_units"] = [{"adults": "adult_1", "children": []}] + with pytest.raises(ValueError, match="list of person ids"): + CaseSpec(**kw) + kw["household"]["benefit_units"] = [ + {"adults": ["adult_1"], "children": "child_1"} + ] + with pytest.raises(ValueError, match="list of person ids"): + CaseSpec(**kw) + + +class TestClassify: + def test_exact(self): + assert classify(100.0, 100.0, VariableClass.CURRENCY) == ( + DiffClassification.MATCH_EXACT + ) + + def test_within_tolerance(self): + # 52 x GBP 0.01 = 0.52/year currency tolerance. + assert classify(100.0, 100.52, VariableClass.CURRENCY) == ( + DiffClassification.MATCH_WITHIN_TOLERANCE + ) + + def test_above_tolerance_defaults_unclassified(self): + assert classify(100.0, 100.53, VariableClass.CURRENCY) == ( + DiffClassification.UNCLASSIFIED + ) + + def test_bool_exact_only(self): + assert classify(1.0, 1.0, VariableClass.BOOLEAN) == ( + DiffClassification.MATCH_EXACT + ) + assert classify(1.0, 0.0, VariableClass.BOOLEAN) == ( + DiffClassification.UNCLASSIFIED + ) + + def test_null_sides(self): + assert classify(None, 5.0, VariableClass.CURRENCY) == ( + DiffClassification.PE_GAP + ) + assert classify(5.0, None, VariableClass.CURRENCY) == ( + DiffClassification.POLICY_SCOPE_MISMATCH + ) + + def test_unknown_variable_class_rejected(self): + with pytest.raises(ValueError): + classify(1.0, 1.0, "percentage") + + def test_missing_tolerance_rejected(self): + with pytest.raises(ValueError, match="no tolerance"): + classify(1.0, 2.0, VariableClass.CURRENCY, tolerance_table={}) + + def test_default_tolerances_closed(self): + assert set(DEFAULT_TOLERANCES) == set(VariableClass) + + +class TestCaseResult: + def result_kwargs(self, **kw): + base = dict( + case_id="uk-uc-single-unemployed", + variable="universal_credit", + pe_value=4796.48, + oracle_value=4796.52, + oracle="ukmod", + engine_version="policyengine-uk 2.0.0", + oracle_version="UKMOD B2026.08", + computed_at="2026-08-14T12:00:00Z", + classification="match_within_tolerance", + variable_class="currency", + abs_diff=0.04, + tolerance=0.52, + schema_version=SCHEMA_VERSION, + ) + base.update(kw) + return base + + def test_valid_row(self): + r = CaseResult(**self.result_kwargs()) + assert r.oracle is Oracle.UKMOD + assert r.classification is DiffClassification.MATCH_WITHIN_TOLERANCE + + def test_unknown_oracle_rejected(self): + with pytest.raises(ValueError): + CaseResult(**self.result_kwargs(oracle="euromod-lite")) + + def test_unknown_classification_rejected(self): + with pytest.raises(ValueError): + CaseResult(**self.result_kwargs(classification="close_enough")) + + def test_abs_diff_must_be_consistent(self): + with pytest.raises(ValueError, match="abs_diff"): + CaseResult(**self.result_kwargs(abs_diff=1.0)) + + def test_null_pe_value_requires_pe_gap(self): + with pytest.raises(ValueError, match="pe_gap"): + CaseResult(**self.result_kwargs(pe_value=None, abs_diff=None)) + r = CaseResult( + **self.result_kwargs( + pe_value=None, abs_diff=None, tolerance=None, classification="pe_gap" + ) + ) + assert r.classification is DiffClassification.PE_GAP + + def test_null_oracle_value_requires_scope_mismatch(self): + with pytest.raises(ValueError, match="policy_scope_mismatch"): + CaseResult(**self.result_kwargs(oracle_value=None, abs_diff=None)) + + def test_match_exact_requires_identity(self): + with pytest.raises(ValueError, match="match_exact"): + CaseResult(**self.result_kwargs(classification="match_exact")) + + def test_within_tolerance_requires_tolerance(self): + # An empty-annotation, no-tolerance row must not validate. + with pytest.raises(ValueError, match="tolerance"): + CaseResult(**self.result_kwargs(tolerance=None)) + + def test_within_tolerance_bounds_enforced(self): + with pytest.raises(ValueError, match="abs_diff <= tolerance"): + CaseResult(**self.result_kwargs(oracle_value=4797.48, abs_diff=1.0)) + with pytest.raises(ValueError, match="tolerance > 0"): + CaseResult(**self.result_kwargs(tolerance=0.0)) + with pytest.raises(ValueError, match="abs_diff <= tolerance"): + # A zero diff is match_exact, never match_within_tolerance. + CaseResult(**self.result_kwargs(oracle_value=4796.48, abs_diff=0.0)) + + def test_tolerance_only_on_within_tolerance_rows(self): + with pytest.raises(ValueError, match="only on match_within_tolerance"): + CaseResult( + **self.result_kwargs( + pe_value=None, abs_diff=None, classification="pe_gap" + ) + ) + + def test_adjudicated_classes_require_writeup(self): + """An adjudication applies to a row the classifier left + unclassified, and it needs a writeup.""" + # 4796.48 vs 4900.00 is well above the currency tolerance, so the + # classifier says `unclassified` — the adjudication queue. + unclassified = dict(oracle_value=4900.0, abs_diff=103.52, tolerance=None) + for cls in ("oracle_difference", "rounding", "pe_gap"): + with pytest.raises(ValueError, match="adjudicated"): + CaseResult(**self.result_kwargs(classification=cls, **unclassified)) + r = CaseResult( + **self.result_kwargs( + classification=cls, + annotations=[ + "UKMOD rounds monthly amounts to the penny; " + "x12 explains the difference (writeup ref)" + ], + **unclassified, + ) + ) + assert r.classification is DiffClassification(cls) + + def test_adjudication_cannot_overrule_the_classifier(self): + """Only an `unclassified` row may be adjudicated: a row the + classifier has already decided is not up for reinterpretation.""" + with pytest.raises(ValueError, match="contradicts the classifier"): + # a within-tolerance row relabelled as an oracle difference + CaseResult( + **self.result_kwargs( + classification="oracle_difference", + tolerance=None, + annotations=["because I say so"], + ) + ) + + def test_a_match_is_never_an_adjudicated_outcome(self): + """`unclassified` may be explained, never flattered into a + match.""" + with pytest.raises(ValueError, match="abs_diff <= tolerance"): + CaseResult( + **self.result_kwargs( + oracle_value=4900.0, + abs_diff=103.52, + classification="match_within_tolerance", + annotations=["close enough"], + ) + ) + + def test_annotations_must_be_nonempty_strings(self): + with pytest.raises(ValueError, match="annotations"): + CaseResult(**self.result_kwargs(annotations="a writeup")) + with pytest.raises(ValueError, match="annotations"): + CaseResult(**self.result_kwargs(annotations=[""])) + + +class TestDateOfBirthRealDate: + def test_impossible_date_rejected(self): + for bad in ("2026-13-40", "0000-00-00", "2025-02-30"): + hh = { + "people": {"adult_1": {"age": 30, "date_of_birth": bad}}, + "benefit_units": [{"adults": ["adult_1"], "children": []}], + } + with pytest.raises(ValueError, match="real"): + CaseSpec(**case_kwargs(household=hh)) + + def test_real_date_accepted(self): + hh = { + "people": {"adult_1": {"age": 30, "date_of_birth": "1996-02-29"}}, + "benefit_units": [{"adults": ["adult_1"], "children": []}], + } + CaseSpec(**case_kwargs(household=hh)) + + +class TestSchemaVersion: + def test_battery_carries_current_version(self): + raw = json.loads(BATTERY.read_text()) + assert raw["schema_version"] == SCHEMA_VERSION + + def test_battery_with_wrong_version_rejected(self, tmp_path): + raw = json.loads(BATTERY.read_text()) + raw["schema_version"] = SCHEMA_VERSION + 1 + p = tmp_path / "cases.json" + p.write_text(json.dumps(raw)) + with pytest.raises(ValueError, match="schema_version"): + load_battery(p) + + def test_battery_without_version_rejected(self, tmp_path): + raw = json.loads(BATTERY.read_text()) + del raw["schema_version"] + p = tmp_path / "cases.json" + p.write_text(json.dumps(raw)) + with pytest.raises(ValueError, match="schema_version"): + load_battery(p) + + def test_result_row_rejects_foreign_version(self): + kw = TestCaseResult().result_kwargs(schema_version=SCHEMA_VERSION + 1) + with pytest.raises(ValueError, match="schema_version"): + CaseResult(**kw) + + +class TestClassifyToResultSeam: + """The classify -> CaseResult wiring path (review: previously untested; + callers had to re-derive the tolerance by hand).""" + + def wired(self, pe, oracle_value, vclass=VariableClass.CURRENCY, **kw): + base = dict( + case_id="uk-uc-single-unemployed", + variable="universal_credit", + pe_value=pe, + oracle_value=oracle_value, + variable_class=vclass, + oracle=Oracle.UKMOD, + engine_version="policyengine-uk 2.0.0", + oracle_version="UKMOD B2026.08", + computed_at="2026-08-14T12:00:00Z", + ) + base.update(kw) + return CaseResult.from_classification(**base) + + def test_tolerance_match_threads_the_applied_tolerance(self): + r = self.wired(100.0, 100.30) + assert r.classification is DiffClassification.MATCH_WITHIN_TOLERANCE + assert r.tolerance == DEFAULT_TOLERANCES[VariableClass.CURRENCY] + assert r.variable_class is VariableClass.CURRENCY + assert abs(r.abs_diff - 0.30) < 1e-9 + + def test_exact_match_carries_no_tolerance(self): + r = self.wired(100.0, 100.0) + assert r.classification is DiffClassification.MATCH_EXACT + assert r.tolerance is None and r.abs_diff == 0.0 + + def test_null_pe_side_wires_to_pe_gap(self): + r = self.wired(None, 5.0) + assert r.classification is DiffClassification.PE_GAP + assert r.abs_diff is None and r.tolerance is None + + def test_a_caller_cannot_widen_the_tolerance(self): + """Tolerances are a registry decision. The factory no longer + takes a table, and a hand-written row carrying a wider tolerance + is rejected — the probe that turned a boolean 1-vs-0 into a + `match_within_tolerance` with tolerance 100.""" + with pytest.raises(TypeError): + self.wired(100.0, 100.9, tolerance_table={VariableClass.CURRENCY: 1.0}) + with pytest.raises(ValueError, match="registered tolerance"): + CaseResult( + case_id="uk-uc-single-unemployed", + variable="universal_credit", + pe_value=100.0, + oracle_value=100.3, + oracle=Oracle.UKMOD, + engine_version="policyengine-uk 2.0.0", + oracle_version="UKMOD B2026.08", + computed_at="2026-08-14T12:00:00Z", + classification="match_within_tolerance", + variable_class=VariableClass.CURRENCY, + abs_diff=0.3, + tolerance=1.0, + schema_version=SCHEMA_VERSION, + ) + + def test_a_boolean_mismatch_can_never_be_a_match(self): + r = self.wired(1.0, 0.0, vclass=VariableClass.BOOLEAN) + assert r.classification is DiffClassification.UNCLASSIFIED + with pytest.raises(ValueError, match="not an outcome"): + CaseResult( + case_id="uk-uc-single-unemployed", + variable="is_uc_eligible", + pe_value=1.0, + oracle_value=0.0, + oracle=Oracle.UKMOD, + engine_version="policyengine-uk 2.0.0", + oracle_version="UKMOD B2026.08", + computed_at="2026-08-14T12:00:00Z", + classification="match_within_tolerance", + variable_class=VariableClass.BOOLEAN, + abs_diff=1.0, + tolerance=100.0, + schema_version=SCHEMA_VERSION, + ) + + def test_the_factory_carries_annotations(self): + """#64's calculator rows REQUIRE an archive annotation; a factory + that could not take one made every calculator call raise.""" + r = self.wired(100.0, 100.0, annotations=["archive:https://example/x"]) + assert r.annotations == ["archive:https://example/x"] + + def test_above_tolerance_lands_unclassified(self): + r = self.wired(100.0, 200.0) + assert r.classification is DiffClassification.UNCLASSIFIED + assert r.tolerance is None + + def test_variable_class_is_a_closed_vocabulary(self): + with pytest.raises(ValueError): + self.wired(100.0, 100.0, vclass="percentage") + + +class TestIdentityClosureAtTheEdges: + """Round-2 probes accepted region="MARS", invented focus variables, + NaN incomes, infinite rent, boolean values outside {0,1} and an + arbitrary battery schema. Each is now a closed decision.""" + + def test_region_is_a_closed_registry(self): + hh = dict(case_kwargs()["household"], region="MARS") + with pytest.raises(ValueError, match="unregistered UK region"): + CaseSpec(**case_kwargs(household=hh)) + CaseSpec(**case_kwargs(household=dict(hh, region="SCOTLAND"))) + + def test_focus_variables_are_closed_per_country(self): + with pytest.raises(ValueError, match="unregistered UK focus"): + CaseSpec(**case_kwargs(expected_focus=["vibes_credit"])) + # a US variable is not a UK variable + with pytest.raises(ValueError, match="unregistered UK focus"): + CaseSpec(**case_kwargs(expected_focus=["snap"])) + + def test_focus_variables_do_not_repeat(self): + with pytest.raises(ValueError, match="must not repeat"): + CaseSpec( + **case_kwargs(expected_focus=["universal_credit", "universal_credit"]) + ) + + def test_nan_and_infinity_are_not_amounts(self): + for bad in (float("nan"), float("inf")): + hh = case_kwargs()["household"] + people = dict(hh["people"], adult_1={"age": 30, "employment_income": bad}) + with pytest.raises(ValueError, match="finite"): + CaseSpec(**case_kwargs(household=dict(hh, people=people))) + with pytest.raises(ValueError, match="finite"): + CaseSpec(**case_kwargs(household=dict(hh, rent=bad))) + + def test_boolean_class_values_must_be_zero_or_one(self): + kw = TestCaseResult().result_kwargs( + variable_class="boolean", + pe_value=7.0, + oracle_value=7.0, + abs_diff=0.0, + tolerance=None, + classification="match_exact", + ) + with pytest.raises(ValueError, match="must be 0 or 1"): + CaseResult(**kw) + + def test_blank_engine_versions_rejected(self): + for field in ("engine_version", "oracle_version"): + with pytest.raises(ValueError, match="required"): + CaseResult(**TestCaseResult().result_kwargs(**{field: " "})) + + def test_battery_schema_path_is_closed(self, tmp_path): + raw = json.loads(BATTERY.read_text()) + raw["schema"] = "sources/made-up/SCHEMA.md" + p = tmp_path / "cases.json" + p.write_text(json.dumps(raw)) + with pytest.raises(ValueError, match="not a contract this repo defines"): + load_battery(p) + + def test_private_rent_cases_pin_a_brma(self): + """LHA rates are per BRMA, not per ITL-1 region: without one, the + two engines are not guaranteed the same world.""" + hh = dict( + case_kwargs()["household"], + tenure="rented_private", + rent=9360, + region="YORKSHIRE", + ) + with pytest.raises(ValueError, match="must pin a BRMA"): + CaseSpec(**case_kwargs(household=hh)) + with pytest.raises(ValueError, match="unregistered BRMA"): + CaseSpec(**case_kwargs(household=dict(hh, brma="Atlantis"))) + with pytest.raises(ValueError, match="sits in"): + CaseSpec(**case_kwargs(household=dict(hh, brma="Inner London"))) + CaseSpec(**case_kwargs(household=dict(hh, brma="Leeds"))) + + def test_every_private_rent_case_in_the_battery_pins_one(self): + for c in load_battery(BATTERY): + if c.household.get("tenure") == "rented_private": + assert c.household["brma"] in BRMA_REGION + + def test_case_baselines_must_be_registered(self): + with pytest.raises(ValueError, match="unregistered baseline world"): + CaseSpec(**case_kwargs(baseline={"policy": "a_world_nobody_named"})) + # the registered reinstated-limit world is accepted + CaseSpec(**case_kwargs(baseline={"policy": "pre_ab2025"}))