diff --git a/data/lanes.json b/data/lanes.json index ff8b4b4..fd156c3 100644 --- a/data/lanes.json +++ b/data/lanes.json @@ -304,6 +304,16 @@ "running": false, "updated": "2026-08-04", "note": "205 claims, 675 per-release results (5 releases, 180 attached to harvest claims); 11 unscoreable rows skipped" + }, + { + "id": "uk-deductions-frr", + "source": "HMT + DWP", + "area": "UC deductions (FRR)", + "mode": 2, + "stage": "ingested", + "running": false, + "updated": "2026-08-19", + "note": "7 claims" } ] } diff --git a/data/scorecard.db b/data/scorecard.db index 4684f85..4c82772 100644 Binary files a/data/scorecard.db and b/data/scorecard.db differ diff --git a/scorecard_db/baselines.py b/scorecard_db/baselines.py index 99f5aaa..6303e19 100644 --- a/scorecard_db/baselines.py +++ b/scorecard_db/baselines.py @@ -153,6 +153,20 @@ "set in scorecard_db/ingest_uk_externals.py " "(_RECKONER_BASELINE).", ), + ( + {"policy": "pre_frr_uc_deductions"}, + "pre_frr_uc_deductions", + "UK law before the Fair Repayment Rate: UC deductions capped at " + "25% of the standard allowance (the FRR lowered the cap to 15% " + "from 2025-04-30). The counterfactual AB2024's FRR figures score " + "against — NOT today's current law, which includes the FRR; a PE " + "counterpart must construct the 25%-cap world explicitly.", + "policy_ref", + "AB2024 para 5.134 p.142 + DWP press release 2025-04-30 (both " + "vendored/linked in sources/harvest-uk-deductions/frr/" + "VERIFICATION.md); descriptor set in " + "scorecard_db/ingest_uk_deductions.py (PRE_FRR_BASELINE).", + ), ( {"policy": "pre_obbba_law"}, "pre_obbba_law", diff --git a/scorecard_db/db.py b/scorecard_db/db.py index 2951df9..30c5bb6 100644 --- a/scorecard_db/db.py +++ b/scorecard_db/db.py @@ -262,9 +262,24 @@ def __init__(self, path: str | Path): self.conn.row_factory = sqlite3.Row self.conn.executescript(DDL) self._migrate() - self.conn.executescript( - VIEW_DDL.replace("__CURRENT_LAW_KEY__", CURRENT_LAW_KEY) - ) + self._ensure_view() + + def _ensure_view(self): + """(Re)create the comparisons view ONLY when its stored + definition differs from the code's. The old unconditional + DROP+CREATE wrote a schema change on EVERY open, so a process + that merely READ the committed DB (tests, exporters) bumped its + header counters and left the binary dirty — the recurring + "harmless header churn" the #48/#52 gates kept flagging.""" + ddl = VIEW_DDL.replace("__CURRENT_LAW_KEY__", CURRENT_LAW_KEY) + index_sql, create = ddl.split("DROP VIEW IF EXISTS comparisons;", 1) + self.conn.executescript(index_sql) # IF NOT EXISTS — no-op when present + expected = create.strip().removesuffix(";") + stored = self.conn.execute( + "SELECT sql FROM sqlite_master WHERE type='view' AND name='comparisons'" + ).fetchone() + if stored is None or stored["sql"] != expected: + self.conn.executescript("DROP VIEW IF EXISTS comparisons;" + create) def _migrate(self): """Bring a pre-existing file up to the current schema. diff --git a/scorecard_db/ingest_uk_deductions.py b/scorecard_db/ingest_uk_deductions.py new file mode 100644 index 0000000..f53ac1a --- /dev/null +++ b/scorecard_db/ingest_uk_deductions.py @@ -0,0 +1,289 @@ +"""Ingest the UC-deductions FRR claim family (#39, staged per #21). + +Reads sources/harvest-uk-deductions/frr/claims_staged.jsonl — a +re-harvest of issue #21's Fair Repayment Rate family from primary +sources (the original ~/scorecard-harvest/uk_deductions staging is +machine-local). Every staged row carries a verbatim quote and a page +locator, re-verified 2026-08-14; see VERIFICATION.md alongside. + +Mapping, under the harvest fail-loud contract: + - reform rows ride policy_ref {"policy": "uc_fair_repayment_rate"} + (cap 25% -> 15% of the UC standard allowance, effective + 2025-04-30, carried as reform detail) AGAINST the registered + pre_frr_uc_deductions baseline world: the costing's own + counterfactual is the 25%-cap law it replaced, not today's + current law (which includes the FRR) — descriptor honesty the + cross-baseline view guard depends on. conditions + ["baseline_policy"] mirrors it for queryability. + - the PSNCR line lands as CASH_REQUIREMENT_CHANGE with + conditions["fiscal_measure"]="psncr" — deliberately NOT + revenue_change, per #21's PSNCR-never-PSNB rule + - the £420 average annual gain is a per-household statistic -> + GBP_PER_HOUSEHOLD, never bare GBP (an average a query could sum) + - identity values route through the closed registry (uk_aliases); + period must equal the fy END year (the live claim convention), + the fy label must be a well-formed YYYY-YY, and staged conditions + may never set the generated identity keys (country, geography, + fy, baseline_policy) — each asserted per row, never trusted from + staging; exact per-source accounting gates the staging wholesale + - everything is held_out; the DWP quarterly deductions outturn + tables PE's parameters consume are calibration territory and are + not staged here at all. The press release's "as many as 2.8 + million households seeing deductions" is that same administrative + quantity republished — it was staged here once as a held-out + FY2025-26 level and REMOVED at gate (the release states no + measurement vintage and the earlier staging paraphrased it); it + belongs to the future Ledger lane with the DWP deductions + statistics publication as provenance (VERIFICATION.md). + +The write path mirrors ingest_uk_externals: one transaction replaces +this module's two sources wholesale AND runs the deliberate- +registration gate (#13) inside it — commit or nothing; the lane-feed +mirror is rewritten only after the commit. + +Usage: + PYTHONPATH=. python -m scorecard_db.ingest_uk_deductions data/scorecard.db +""" + +from __future__ import annotations + +import json +import re +from pathlib import Path + +from .db import LANE_SQL, SCORES_SQL, ScorecardDB +from .harvest import REPO, finish +from .uk_aliases import canon +from .models import ( + CalibrationRelationship, + ExternalScore, + Metric, + ReformRef, + TimeBasis, + UnitConcept, +) + +STAGED = REPO / "sources" / "harvest-uk-deductions" / "frr" / "claims_staged.jsonl" +DEDUCTION_SOURCES = ("hm_treasury", "dwp") + +KNOWN_FIELDS = frozenset( + { + "source", + "metric", + "unit_concept", + "period", + "fy", + "value", + "conditions", + "reform_policy", + "publication", + "quote", + "verified", + } +) + +# staged metric -> (Metric, expected staged unit, DB unit concept, +# value_kind). The staged unit label is validated against the expected +# one — a drifted re-stage fails loudly, never re-maps silently. +METRICS = { + "cash_requirement_change": ( + Metric.CASH_REQUIREMENT_CHANGE, + "gbp", + UnitConcept.GBP, + "gbp", + ), + "gainer_count": ( + Metric.GAINER_COUNT, + "households", + UnitConcept.HOUSEHOLDS, + "count", + ), + "average_annual_gain": ( + Metric.AVERAGE_ANNUAL_GAIN, + "gbp", + UnitConcept.GBP_PER_HOUSEHOLD, + "gbp", + ), +} + +# Staged conditions may NEVER carry these: the stager generates them +# (country/geography constants, fy from the row's own field, the +# baseline mirror from the reform routing). A staged row setting one +# could silently override the generated identity — gate finding: a +# poison row with geography="Mars" and a baseline_policy contradicting +# ReformRef.baseline was accepted under the old merge order. +RESERVED_CONDITIONS = frozenset({"country", "geography", "fy", "baseline_policy"}) + +# Exact per-source accounting (the ingest_uk_externals _EXPECTED +# pattern): a drifted or truncated staging fails wholesale — without +# this, an empty file would wholesale-delete both sources and commit +# zero rows. +EXPECTED_COUNTS = {"hm_treasury": 4, "dwp": 3} + +PRE_FRR_BASELINE = {"policy": "pre_frr_uc_deductions"} + +FRR_REFORM = ReformRef( + framework="policy_ref", + reform={ + "policy": "uc_fair_repayment_rate", + "change": "uc_deductions_cap_25pct_to_15pct_of_standard_allowance", + "effective": "2025-04-30", + }, + baseline=PRE_FRR_BASELINE, +) + + +def _fy_end(label: str) -> int: + """Validated YYYY-YY financial-year label -> end year. The suffix + must be the start year + 1 (gate finding: '2029-99' parsed to 2030 + under a bare int(label[:4]) + 1).""" + m = re.fullmatch(r"(\d{4})-(\d{2})", label) + if not m: + raise ValueError(f"uk_deductions: malformed fy label {label!r}") + start = int(m.group(1)) + if (start + 1) % 100 != int(m.group(2)): + raise ValueError( + f"uk_deductions: fy label {label!r} suffix is not start year + 1" + ) + return start + 1 + + +def stage_scores() -> list[ExternalScore]: + if not STAGED.exists(): + raise FileNotFoundError(f"staged claims missing: {STAGED}") + scores = [] + counts: dict[str, int] = {} + for line in STAGED.read_text().splitlines(): + if not line.strip(): + continue + row = json.loads(line) + unknown = set(row) - KNOWN_FIELDS + if unknown: + raise ValueError( + f"uk_deductions: unhandled staged fields {sorted(unknown)}" + ) + if row["source"] not in DEDUCTION_SOURCES: + raise ValueError(f"uk_deductions: unknown source {row['source']!r}") + reserved = RESERVED_CONDITIONS & set(row["conditions"]) + if reserved: + raise ValueError( + "uk_deductions: staged conditions may not set generated " + f"identity keys {sorted(reserved)}" + ) + # period keys the FY END year (the live claim-side convention; + # models.py fy note) — asserted, never inherited from staging. + end = _fy_end(row["fy"]) + if row["period"] != end: + raise ValueError( + f"uk_deductions: period {row['period']} is not fy " + f"{row['fy']}'s end year {end}" + ) + if row["metric"] not in METRICS: + raise ValueError(f"uk_deductions: unknown metric {row['metric']!r}") + metric, staged_unit, unit, value_kind = METRICS[row["metric"]] + canon(row["source"], "unit", row["unit_concept"]) + if row["unit_concept"] != staged_unit: + raise ValueError( + f"uk_deductions: {row['metric']} staged with unit " + f"{row['unit_concept']!r}, expected {staged_unit!r}" + ) + if row["reform_policy"] == "uc_fair_repayment_rate": + reform = FRR_REFORM + # baseline variants are load-bearing: mirrored in conditions + # for queryability (models.py COLLATION worklist item 3) + mirror = {"baseline_policy": PRE_FRR_BASELINE["policy"]} + else: + # No baseline-framework rows remain in this family — the 2.8m + # deductions level was removed at gate (see the module + # docstring); a future level row is a deliberate decision + # with its own routing, never a silent default. + raise ValueError(f"uk_deductions: unknown reform {row['reform_policy']!r}") + # Staged conditions first, canonicalized; generated identity + # fields LAST so nothing staged can override them (the reserved- + # key check above makes an attempt loud, this makes it inert). + conditions = dict(row["conditions"]) + if "program" in conditions: + conditions["program"] = canon( + row["source"], "program", conditions["program"] + ) + if "subgroup" in conditions: + conditions["subgroup"] = canon( + row["source"], "subgroup", conditions["subgroup"] + ) + conditions |= { + "country": "UK", + "geography": canon(row["source"], "geography", "GB"), + "fy": row["fy"], + **mirror, + } + counts[row["source"]] = counts.get(row["source"], 0) + 1 + scores.append( + ExternalScore( + source=row["source"], + metric=metric, + unit_concept=unit, + period=row["period"], + time_basis=TimeBasis.FISCAL_YEAR, + value=float(row["value"]), + conditions=conditions, + reform=reform, + calibration_relationship=CalibrationRelationship.HELD_OUT, + source_column=row["quote"], + publication=row["publication"], + value_kind=value_kind, + ) + ) + if counts != EXPECTED_COUNTS: + raise ValueError( + f"uk_deductions: staging accounting drifted: {counts} != " + f"{EXPECTED_COUNTS} — a truncated or regrown staging must be " + "a deliberate re-pin, never a silent wholesale replace" + ) + return finish(scores, "uk_deductions") + + +def ingest(db_path: Path) -> dict: + """One transaction replaces this module's two sources wholesale and + runs every persistence gate inside it (the ingest_uk_externals + contract): delete, insert, the deliberate-registration gate (#13), + the lane row — commit or nothing. The lane-feed mirror is rewritten + only after the commit (idempotent merge keyed by lane id).""" + scores = stage_scores() + db = ScorecardDB(db_path) + rows = [ScorecardDB.score_row(s) for s in scores] + placeholders = ",".join("?" * len(DEDUCTION_SOURCES)) + from .baselines import register_baselines_txn + from .ingest_harvest import sync_lane_feed + + with db.conn: + db.conn.execute( + f"DELETE FROM external_scores WHERE source IN ({placeholders})", + DEDUCTION_SOURCES, + ) + db.conn.executemany(SCORES_SQL, rows) + register_baselines_txn(db) + db.conn.execute( + LANE_SQL, + ("uk-deductions-frr", "ingested", f"{len(rows)} claims", "2026-08-19"), + ) + sync_lane_feed( + db, + REPO / "data" / "lanes.json", + "2026-08-19", + lanes={ + "uk-deductions-frr": { + "source": "HMT + DWP", + "area": "UC deductions (FRR)", + "mode": 2, + } + }, + ) + db.close() + return {"claims": len(rows)} + + +if __name__ == "__main__": + import sys + + out = Path(sys.argv[1] if len(sys.argv) > 1 else "data/scorecard.db") + print(json.dumps(ingest(out), indent=1)) diff --git a/scorecard_db/ingest_uk_externals.py b/scorecard_db/ingest_uk_externals.py index 19e1a66..2403545 100644 --- a/scorecard_db/ingest_uk_externals.py +++ b/scorecard_db/ingest_uk_externals.py @@ -131,16 +131,21 @@ def _fy(label: str) -> tuple[int, str]: 'FYE 2024' -> (2024, '2023-24'); '2023-24' -> (2024, '2023-24'); '2024/25' -> (2025, '2024-25'). """ - m = re.match(r"^FYE (\d{4})$", label) + # fullmatch, not match-with-$: Python's $ also matches before a + # trailing newline, so "2029-30\n" would parse (round-2 gate). + m = re.fullmatch(r"FYE (\d{4})", label) if m: end = int(m.group(1)) return end, f"{end - 1}-{str(end)[2:]}" - m = re.match(r"^(\d{4})-(\d{2})$", label) - if m: - return int(m.group(1)) + 1, label - m = re.match(r"^(\d{4})/(\d{2})$", label) + m = re.fullmatch(r"(\d{4})[-/](\d{2})", label) if m: start = int(m.group(1)) + # the suffix must be the start year + 1 — '2029-99' is malformed, + # never year 2030 (round-1 gate on the deductions family) + if (start + 1) % 100 != int(m.group(2)): + raise ValueError( + f"UK financial-year label {label!r}: suffix is not start year + 1" + ) return start + 1, f"{start}-{m.group(2)}" raise ValueError(f"unparseable UK financial-year label: {label!r}") @@ -420,7 +425,7 @@ def _absolute_anchor(start_year: int, end_year: int) -> str: "modified_oecd_companion_ahc" if housing == "ahc" else "modified_oecd" ), } - span = _SPAN.match(row["period"]) + span = _SPAN.fullmatch(row["period"]) if span: start, end = int(span.group(1)) + 1, int(span.group(3)) + 1 cond["window_kind"] = "annual_average" @@ -742,7 +747,7 @@ def stage_ukmod() -> tuple[list[ExternalScore], list[dict], dict]: cond["equivalisation"] = "modified_oecd" cond.pop("program", None) cond["quantile"] = cond.pop("subgroup") - elif _UKMOD_POVERTY.match(row["metric"]): + elif _UKMOD_POVERTY.fullmatch(row["metric"]): metric, unit, value_kind = ( Metric.POVERTY_RATE, UnitConcept.SHARE, @@ -752,7 +757,7 @@ def stage_ukmod() -> tuple[list[ExternalScore], list[dict], dict]: cond["poverty_line"] = canon( "ukmod", "poverty_line", - _UKMOD_POVERTY.match(row["metric"]).group(1), + _UKMOD_POVERTY.fullmatch(row["metric"]).group(1), ) cond["housing_costs"] = canon("ukmod", "housing_costs", "bhc") cond["equivalisation"] = "modified_oecd" diff --git a/scorecard_db/models.py b/scorecard_db/models.py index fa32cec..96432cc 100644 --- a/scorecard_db/models.py +++ b/scorecard_db/models.py @@ -82,6 +82,14 @@ class Metric(str, Enum): GINI = "gini" INCOME_STATISTIC = "income_statistic" INCOME_SHARE = "income_share" + # UK UC-deductions harvest (#39/#21). cash_requirement_change is a + # PSNCR effect — a cash measure, deliberately distinct from + # revenue_change (PSNB): the FRR has no PSNB impact and the boundary + # must be unconfusable. gainer_count / average_annual_gain are the + # distributional-impact vocabulary of UK fiscal-event documents. + CASH_REQUIREMENT_CHANGE = "cash_requirement_change" + GAINER_COUNT = "gainer_count" + AVERAGE_ANNUAL_GAIN = "average_annual_gain" class UnitConcept(str, Enum): @@ -120,6 +128,10 @@ class UnitConcept(str, Enum): GBP_PER_WEEK = "gbp_per_week" GBP_PER_MONTH = "gbp_per_month" INDEX_0_1 = "index_0_1" + # Per-household GBP statistic (the UK mirror of USD_PER_HOUSEHOLD, + # same rule: averages must never be summable as aggregates). The + # FRR family's £420 average annual gain is per household per year. + GBP_PER_HOUSEHOLD = "gbp_per_household" # Standardized conditions vocabulary (COLLATION worklist item 4). @@ -212,6 +224,11 @@ class UnitConcept(str, Enum): # line uses: "fye_2011" | "fye_2025" | # "mixed_fye2011_fye2025" (a multi-year window # straddling the FYE-2025 re-anchor) + # fiscal_measure which fiscal aggregate a change claim moves: + # "psncr" on the FRR family (a cash-requirement + # effect, deliberately NOT PSNB — PQ UIN 3751) + "measure", + "fiscal_measure", "country", "fy", "housing_costs", diff --git a/scorecard_db/uk_aliases.py b/scorecard_db/uk_aliases.py index 579043b..33d5a49 100644 --- a/scorecard_db/uk_aliases.py +++ b/scorecard_db/uk_aliases.py @@ -264,6 +264,13 @@ def _alias(source: str, axis: str, source_value: str, canonical: str) -> None: ], ) +# --- UC-deductions FRR family (#39; sources are fiscal-event documents, +# not statistical adapters — same closed-registry rule) ---------------------- +for _src in ("hm_treasury", "dwp"): + _identity(_src, "program", ["universal_credit"]) + _identity(_src, "subgroup", ["families_with_children"]) + _identity(_src, "geography", ["GB"]) + # --- units (adapter unit_concept vocabulary; DB units mapped in ingest) ------ _identity( "dwp_takeup", @@ -288,6 +295,8 @@ def _alias(source: str, axis: str, source_value: str, canonical: str) -> None: "gbp_per_month_equivalised", ], ) +for _src in ("hm_treasury", "dwp"): + _identity(_src, "unit", ["gbp", "households"]) def canon(source: str, axis: str, value: str) -> str: diff --git a/sources/harvest-uk-deductions/frr/VERIFICATION.md b/sources/harvest-uk-deductions/frr/VERIFICATION.md new file mode 100644 index 0000000..28ef835 --- /dev/null +++ b/sources/harvest-uk-deductions/frr/VERIFICATION.md @@ -0,0 +1,80 @@ +# FRR claim family — page verification (2026-08-14) + +Re-harvest of the Fair Repayment Rate family from issue #21, rebuilt +from primary sources because the original `~/scorecard-harvest/ +uk_deductions/` staging is machine-local and not vendored. Every figure +below was re-verified against the vendored PDFs / live page on +2026-08-14; the `quote` field on each staged row is verbatim. + +## Sources vendored here + +- `autumn_budget_2024.pdf` — HM Treasury, Autumn Budget 2024 print + (2024-10-30). FRR passages: para 2.30 (p.47), para 4.111 (p.110), + para 5.134 (p.142). +- `autumn_budget_2024_policy_costings.pdf` — AB2024 policy costings + document. **Verified absence**: no FRR costing line (searched + 'Repayment'/'deduction' across all 93 pages) — consistent with issue + #21's note that the FRR is a financial transaction with no PSNB + impact; the only fiscal quantity is the PSNCR line in para 5.134. + The absence is machine-checked, not prose-only: + `tests/test_uk_deductions_ingest.py::test_frr_costing_absence_is_machine_checked` + extracts the vendored costings PDF and asserts zero occurrences of + the measure name (skips where pypdf is unavailable, e.g. bare CI). +- DWP press release 2025-04-30 (URL in the staged rows): effective + date 30 April 2025, cap 25% -> 15% of the UC standard allowance. + +## Figures + +| claim | value | where | +|---|---|---| +| PSNCR increase, 2029-30 | +£385m | AB2024 para 5.134 | +| households better off | 1.2m | AB2024 5.134/2.30; DWP PR | +| average annual gain | £420 | AB2024 5.134/2.30; DWP PR | +| of which families with children | 700k | AB2024 2.30; DWP PR | + +### Removed at gate (2026-08-19): the 2.8m deductions level + +The 2026-08-14 staging carried a fifth row — 2.8m "households with +deductions (pre-FRR level)", keyed FY2025-26, held_out. Removed on the +#52 review round, three defects: + +1. **The staged quote was a paraphrase.** The release's actual sentence + (re-verified 2026-08-19 against the live page): "With as many as + 2.8 million households seeing deductions made to their Universal + Credit award to pay off debt each month, the new rate is designed to + ensure money is repaid where it is owed…" — the staged "2.8 million + households currently experience deductions monthly" appears nowhere. +2. **The release states no measurement vintage**, so FY2025-26 / + period 2026 / FISCAL_YEAR was an unsupported temporal identity; the + reviewer traced the figure to DWP's administrative deductions + statistics (the December 2023 to November 2024 publication — + pre-FRR data). +3. **Boundary rule**: that administrative quantity is exactly the DWP + quarterly deductions OUTTURN family relationships.py routes away + from external scores ("deliberately not ingested"); a press-release + republication does not make it independent. + +Disposition: the figure belongs to the future Ledger lane, staged from +the DWP deductions statistics publication itself (exact cell + vintage ++ the consuming pe-uk parameter named), never from the press release. + +## Hygiene rules carried from #21 + +- **PSNCR, never PSNB**: the £385m is a public-sector-net-cash- + requirement effect. A PSNB comparison row would compare against a + number that does not exist (PQ UIN 3751). +- **DWP quarterly deductions outturns route to calibration, not + external_scores** — the Stat-Xplare deductions tables PE's parameters + consume are consumed_as_target territory and are NOT staged here. +- The gainer counts/average gains have no stated fiscal year in the + sources; they describe the steady state after the 30 April 2025 + start, staged here as FY 2025-26 with the effective date on the + reform descriptor. Flag on comparison if PE's year convention + differs. + +## Not yet staged (rest of #21) + +JRF protected-minimum floor (pre/post-FRR baselines), historical cap +costings (Budget 2018/2020), DWP/DfC screening counts, Policy in +Practice maximums, Citizens Advice costings — these need their own +page-verified pass against their primary PDFs. diff --git a/sources/harvest-uk-deductions/frr/autumn_budget_2024.pdf b/sources/harvest-uk-deductions/frr/autumn_budget_2024.pdf new file mode 100644 index 0000000..86c7570 Binary files /dev/null and b/sources/harvest-uk-deductions/frr/autumn_budget_2024.pdf differ diff --git a/sources/harvest-uk-deductions/frr/autumn_budget_2024_policy_costings.pdf b/sources/harvest-uk-deductions/frr/autumn_budget_2024_policy_costings.pdf new file mode 100644 index 0000000..06e1584 Binary files /dev/null and b/sources/harvest-uk-deductions/frr/autumn_budget_2024_policy_costings.pdf differ diff --git a/sources/harvest-uk-deductions/frr/claims_staged.jsonl b/sources/harvest-uk-deductions/frr/claims_staged.jsonl new file mode 100644 index 0000000..1fe9489 --- /dev/null +++ b/sources/harvest-uk-deductions/frr/claims_staged.jsonl @@ -0,0 +1,7 @@ +{"source": "hm_treasury", "metric": "cash_requirement_change", "unit_concept": "gbp", "period": 2030, "fy": "2029-30", "value": 385000000, "conditions": {"program": "universal_credit", "fiscal_measure": "psncr"}, "reform_policy": "uc_fair_repayment_rate", "publication": {"title": "Autumn Budget 2024", "url": "https://assets.publishing.service.gov.uk/media/672b98bb40f7da695c921c61/Autumn_Budget_2024_Print.pdf", "date": "2024-10-30", "locator": "para 5.134, p.142"}, "quote": "This measure increases the public sector net cash requirement by £385 million in 2029-30.", "verified": "2026-08-14"} +{"source": "hm_treasury", "metric": "gainer_count", "unit_concept": "households", "period": 2026, "fy": "2025-26", "value": 1200000, "conditions": {"program": "universal_credit"}, "reform_policy": "uc_fair_repayment_rate", "publication": {"title": "Autumn Budget 2024", "url": "https://assets.publishing.service.gov.uk/media/672b98bb40f7da695c921c61/Autumn_Budget_2024_Print.pdf", "date": "2024-10-30", "locator": "para 5.134, p.142; para 2.30, p.47"}, "quote": "This will mean 1.2 million households will be better off by £420 per year on average as a result of this change.", "verified": "2026-08-14"} +{"source": "hm_treasury", "metric": "average_annual_gain", "unit_concept": "gbp", "period": 2026, "fy": "2025-26", "value": 420, "conditions": {"program": "universal_credit", "statistic": "mean"}, "reform_policy": "uc_fair_repayment_rate", "publication": {"title": "Autumn Budget 2024", "url": "https://assets.publishing.service.gov.uk/media/672b98bb40f7da695c921c61/Autumn_Budget_2024_Print.pdf", "date": "2024-10-30", "locator": "para 5.134, p.142; para 2.30, p.47"}, "quote": "households expected to be better off by £420 a year on average", "verified": "2026-08-14"} +{"source": "hm_treasury", "metric": "gainer_count", "unit_concept": "households", "period": 2026, "fy": "2025-26", "value": 700000, "conditions": {"program": "universal_credit", "subgroup": "families_with_children"}, "reform_policy": "uc_fair_repayment_rate", "publication": {"title": "Autumn Budget 2024", "url": "https://assets.publishing.service.gov.uk/media/672b98bb40f7da695c921c61/Autumn_Budget_2024_Print.pdf", "date": "2024-10-30", "locator": "para 2.30, p.47"}, "quote": "Around 700,000 of the poorest families with children will benefit as a result of this change", "verified": "2026-08-14"} +{"source": "dwp", "metric": "gainer_count", "unit_concept": "households", "period": 2026, "fy": "2025-26", "value": 1200000, "conditions": {"program": "universal_credit"}, "reform_policy": "uc_fair_repayment_rate", "publication": {"title": "Universal Credit change brings £420 boost to over a million households (press release)", "url": "https://www.gov.uk/government/news/universal-credit-change-brings-420-boost-to-over-a-million-households", "date": "2025-04-30", "locator": "press release"}, "quote": "Around 1.2 million of the poorest households - including 700,000 with children - will keep an extra £420 a year on average", "verified": "2026-08-14"} +{"source": "dwp", "metric": "average_annual_gain", "unit_concept": "gbp", "period": 2026, "fy": "2025-26", "value": 420, "conditions": {"program": "universal_credit", "statistic": "mean"}, "reform_policy": "uc_fair_repayment_rate", "publication": {"title": "Universal Credit change brings £420 boost to over a million households (press release)", "url": "https://www.gov.uk/government/news/universal-credit-change-brings-420-boost-to-over-a-million-households", "date": "2025-04-30", "locator": "press release"}, "quote": "an average £420 extra a year for 1.2 million of the poorest households", "verified": "2026-08-14"} +{"source": "dwp", "metric": "gainer_count", "unit_concept": "households", "period": 2026, "fy": "2025-26", "value": 700000, "conditions": {"program": "universal_credit", "subgroup": "families_with_children"}, "reform_policy": "uc_fair_repayment_rate", "publication": {"title": "Universal Credit change brings £420 boost to over a million households (press release)", "url": "https://www.gov.uk/government/news/universal-credit-change-brings-420-boost-to-over-a-million-households", "date": "2025-04-30", "locator": "press release"}, "quote": "including 700,000 with children", "verified": "2026-08-14"} diff --git a/tests/test_scorecard_db.py b/tests/test_scorecard_db.py index f5adef1..64fa1d9 100644 --- a/tests/test_scorecard_db.py +++ b/tests/test_scorecard_db.py @@ -849,3 +849,36 @@ def test_disavowed_referenced_row_refuses(self, tmp_path): with pytest.raises(ValueError, match="re-key the data"): register_baselines(db) db.close() + + +def test_open_close_never_rewrites_the_file(tmp_path): + """Opening the DB read-only-in-spirit must be read-only in bytes: + the old unconditional DROP+CREATE of the comparisons view wrote a + schema change on every open, so tests and exporters that merely + READ the committed DB dirtied its header counters (the recurring + binary churn the #48/#52 gates flagged).""" + p = tmp_path / "t.db" + ScorecardDB(p).close() + before = p.read_bytes() + for _ in range(3): + ScorecardDB(p).close() + assert p.read_bytes() == before + + +def test_stale_view_definition_is_rebuilt(tmp_path): + """The conditional recreate still upgrades a drifted view.""" + import sqlite3 + + p = tmp_path / "t.db" + ScorecardDB(p).close() + c = sqlite3.connect(p) + with c: + c.executescript( + "DROP VIEW comparisons;" + " CREATE VIEW comparisons AS SELECT 1 AS not_the_view;" + ) + c.close() + db = ScorecardDB(p) + cols = {r[1] for r in db.conn.execute("PRAGMA table_info(comparisons)")} + db.close() + assert "claim_id" in cols and "not_the_view" not in cols diff --git a/tests/test_uk_deductions_ingest.py b/tests/test_uk_deductions_ingest.py new file mode 100644 index 0000000..4eca35b --- /dev/null +++ b/tests/test_uk_deductions_ingest.py @@ -0,0 +1,253 @@ +"""Tests for the FRR claim-family ingest (#39, staged per #21).""" + +import json +from pathlib import Path + +import pytest + +from scorecard_db import Metric, ScorecardDB, UnitConcept +from scorecard_db.ingest_uk_deductions import STAGED, ingest, stage_scores + +pytestmark = pytest.mark.skipif(not STAGED.exists(), reason="FRR staging not present") + + +def _one(scores, metric, **conds): + hits = [ + s + for s in scores + if s.metric is metric + and all(s.conditions.get(k) == v for k, v in conds.items()) + ] + assert len(hits) == 1, (metric, conds, len(hits)) + return hits[0] + + +def test_stage_headline_values(): + scores = stage_scores() + assert len(scores) == 7 + psncr = _one(scores, Metric.CASH_REQUIREMENT_CHANGE) + assert psncr.value == 385_000_000 + assert psncr.period == 2030 + assert psncr.conditions["fiscal_measure"] == "psncr" + assert psncr.reform.reform["policy"] == "uc_fair_repayment_rate" + assert psncr.value_kind == "gbp" # never usd on a UK claim + + hmt_gainers = [ + s + for s in scores + if s.metric is Metric.GAINER_COUNT + and s.source == "hm_treasury" + and "subgroup" not in s.conditions + ] + assert len(hmt_gainers) == 1 and hmt_gainers[0].value == 1_200_000 + + with_children = [ + s + for s in scores + if s.metric is Metric.GAINER_COUNT + and s.conditions.get("subgroup") == "families_with_children" + ] + assert {s.value for s in with_children} == {700_000} + assert {s.source for s in with_children} == {"hm_treasury", "dwp"} + + gain = [s for s in scores if s.metric is Metric.AVERAGE_ANNUAL_GAIN] + assert {s.value for s in gain} == {420.0} + # a per-household average is never bare GBP (a query could sum it) + assert {s.unit_concept for s in gain} == {UnitConcept.GBP_PER_HOUSEHOLD} + + +def test_frr_rows_score_against_the_pre_frr_world(): + """The costing's own counterfactual is the 25%-cap law the FRR + replaced — NOT today's current law, which includes the FRR. Every + row in the family is a reform claim carrying the registered pre-FRR + baseline descriptor, mirrored in conditions (load-bearing, + COLLATION worklist item 3). The family's one baseline-framework row + (the 2.8m deductions level) was removed at gate: it is the DWP + administrative deductions quantity relationships.py routes away + from external scores, republished in a press release with no + measurement vintage.""" + scores = stage_scores() + assert len(scores) == 7 + for s in scores: + assert s.reform.framework == "policy_ref" + assert s.reform.baseline == {"policy": "pre_frr_uc_deductions"} + assert s.conditions["baseline_policy"] == "pre_frr_uc_deductions" + assert all(s.metric is not Metric.PARTICIPANT_COUNT for s in scores) + + +def test_reserved_condition_keys_are_rejected(tmp_path, monkeypatch): + """Gate probe (sol, #52 round 1): staged conditions used to be + merged AFTER the generated identity fields, so geography='Mars' and + a baseline_policy contradicting ReformRef.baseline were accepted. + Reserved keys now raise.""" + import scorecard_db.ingest_uk_deductions as mod + + for poison in ({"geography": "Mars"}, {"baseline_policy": "current_law"}): + rows = [ + json.loads(line) for line in STAGED.read_text().splitlines() if line.strip() + ] + rows[0]["conditions"] |= poison + bad = tmp_path / "claims_staged.jsonl" + bad.write_text("\n".join(json.dumps(r) for r in rows) + "\n") + monkeypatch.setattr(mod, "STAGED", bad) + with pytest.raises(ValueError, match="generated identity keys"): + stage_scores() + + +def test_staging_accounting_gates_wholesale_replace(tmp_path, monkeypatch): + """An empty or truncated staging must fail loudly — without the + exact per-source accounting, ingest would wholesale-delete both + sources and commit zero rows.""" + import scorecard_db.ingest_uk_deductions as mod + + lines = [line for line in STAGED.read_text().splitlines() if line.strip()] + for content in ("", "\n".join(lines[:-1]) + "\n"): + bad = tmp_path / "claims_staged.jsonl" + bad.write_text(content) + monkeypatch.setattr(mod, "STAGED", bad) + with pytest.raises(ValueError, match="accounting drifted"): + stage_scores() + + +def test_malformed_fy_label_raises(tmp_path, monkeypatch): + """'2029-99' must never parse (the suffix is not start year + 1).""" + import scorecard_db.ingest_uk_deductions as mod + + rows = [ + json.loads(line) for line in STAGED.read_text().splitlines() if line.strip() + ] + rows[0]["fy"] = "2029-99" + rows[0]["period"] = 2030 + bad = tmp_path / "claims_staged.jsonl" + bad.write_text("\n".join(json.dumps(r) for r in rows) + "\n") + monkeypatch.setattr(mod, "STAGED", bad) + with pytest.raises(ValueError, match="suffix is not start year"): + stage_scores() + + +def test_period_must_be_fy_end_year(tmp_path, monkeypatch): + """The live claim convention (models.py): integer period = FY END + year. A staged row keyed by start year must fail, never ingest.""" + import scorecard_db.ingest_uk_deductions as mod + + rows = [ + json.loads(line) for line in STAGED.read_text().splitlines() if line.strip() + ] + rows[0]["period"] = rows[0]["period"] - 1 # start-year keying + bad = tmp_path / "claims_staged.jsonl" + bad.write_text("\n".join(json.dumps(r) for r in rows) + "\n") + monkeypatch.setattr(mod, "STAGED", bad) + with pytest.raises(ValueError, match="end year"): + stage_scores() + + +def test_unknown_identity_values_raise(tmp_path, monkeypatch): + """Identity values route through the closed registry: an unregistered + subgroup raises, never passes through.""" + import scorecard_db.ingest_uk_deductions as mod + + rows = [ + json.loads(line) for line in STAGED.read_text().splitlines() if line.strip() + ] + target = next(r for r in rows if r["conditions"].get("subgroup")) + target["conditions"]["subgroup"] = "lone_parents" + bad = tmp_path / "claims_staged.jsonl" + bad.write_text("\n".join(json.dumps(r) for r in rows) + "\n") + monkeypatch.setattr(mod, "STAGED", bad) + with pytest.raises(ValueError, match="unregistered subgroup"): + stage_scores() + + +def test_no_psnb_shaped_claims(): + # PSNCR-never-PSNB rule (#21): the FRR family must not stage any + # revenue_change row a PSNB comparison could silently join against. + scores = stage_scores() + assert all(s.metric is not Metric.REVENUE_CHANGE for s in scores) + + +def test_every_claim_carries_quote_and_publication(): + for s in stage_scores(): + assert s.source_column # the verbatim quote + assert s.publication.get("url") and s.publication.get("date") + assert s.conditions["country"] == "UK" + assert s.unit_concept in ( + UnitConcept.GBP, + UnitConcept.GBP_PER_HOUSEHOLD, + UnitConcept.HOUSEHOLDS, + ) + + +def test_round_trip(tmp_path): + from scorecard_db.models import baseline_key + + assert ingest(tmp_path / "t.db") == {"claims": 7} + db = ScorecardDB(tmp_path / "t.db") + n = db.conn.execute( + "SELECT COUNT(*) FROM external_scores WHERE metric='cash_requirement_change'" + ).fetchone()[0] + assert n == 1 + # the pre-FRR world is REGISTERED (the deliberate-registration gate + # ran inside the write transaction) and every reform row is keyed + # to it + pre_frr = baseline_key({"policy": "pre_frr_uc_deductions"}) + assert ( + db.conn.execute( + "SELECT COUNT(*) FROM baselines WHERE baseline_key = ?", (pre_frr,) + ).fetchone()[0] + == 1 + ) + assert ( + db.conn.execute( + "SELECT COUNT(*) FROM external_scores WHERE baseline_key = ?", (pre_frr,) + ).fetchone()[0] + == 7 + ) + # no usd value kinds on UK sources; lane row set by the ingest + assert ( + db.conn.execute( + "SELECT COUNT(*) FROM external_scores WHERE value_kind='usd'" + " AND source IN ('hm_treasury','dwp')" + ).fetchone()[0] + == 0 + ) + lane = db.conn.execute( + "SELECT stage, detail FROM lanes WHERE lane='uk-deductions-frr'" + ).fetchone() + assert lane["stage"] == "ingested" and lane["detail"] == "7 claims" + db.close() + + +FRR_DIR = Path(__file__).resolve().parent.parent / "sources" / "harvest-uk-deductions" + + +def _pdf_text(path): + pypdf = pytest.importorskip("pypdf") + import re + + reader = pypdf.PdfReader(path) + text = " ".join((p.extract_text() or "") for p in reader.pages) + return re.sub(r"\s+", " ", text) + + +def test_420_quotes_are_two_distinct_verbatim_sentences(): + """Rows 2 and 3 cite the SAME figure from two DIFFERENT passages of + AB2024 (para 4.111's "£420 per year" sentence vs the para 2.30/5.134 + "£420 a year" sentence). Both must appear verbatim — neither is a + reworded fragment of the other.""" + text = _pdf_text(FRR_DIR / "frr" / "autumn_budget_2024.pdf") + assert ( + "This will mean 1.2 million households will be better off by " + "£420 per year on average as a result of this change." in text + ) + assert "with households expected to be better off by £420 a year on average" in text + + +def test_frr_costing_absence_is_machine_checked(): + """The 'verified absence' of an FRR costing line is not trust-me + prose: the AB2024 policy costings document (93 pages) contains zero + occurrences of the measure name. The FRR is a financial transaction + (PSNCR, not PSNB), so a costing line appearing in a future re-vendor + would mean the wrong document or a mis-staged claim.""" + text = _pdf_text(FRR_DIR / "frr" / "autumn_budget_2024_policy_costings.pdf") + assert "Fair Repayment Rate" not in text + assert "repayment rate" not in text.lower() diff --git a/tests/test_uk_externals_ingest.py b/tests/test_uk_externals_ingest.py index 16dd9d9..b12d90a 100644 --- a/tests/test_uk_externals_ingest.py +++ b/tests/test_uk_externals_ingest.py @@ -41,8 +41,59 @@ def test_fy_parsing(): assert _fy("FYE 2024") == (2024, "2023-24") assert _fy("2023-24") == (2024, "2023-24") assert _fy("2024/25") == (2025, "2024-25") + assert _fy("1999-00") == (2000, "1999-00") with pytest.raises(ValueError): _fy("FY2024") + # a mismatched suffix is malformed, never silently start + 1 + with pytest.raises(ValueError, match="suffix"): + _fy("2029-99") + with pytest.raises(ValueError, match="suffix"): + _fy("2029/31") + # fullmatch: $ would accept a trailing newline + for tainted in ("2029-30\n", "2029/30\n", "FYE 2030\n"): + with pytest.raises(ValueError): + _fy(tainted) + + +def test_tainted_span_and_ukmod_metric_raise(monkeypatch): + """The other two $-anchored call sites (round-3 gate): a trailing- + newline HBAI window period and UKMOD poverty metric were both + ingested under .match(); fullmatch makes them fail loudly.""" + from scorecard_db.ingest_uk_externals import stage_hbai + + hbai_row = _row( + source="dwp_hbai", + program="hbai_low_income", + metric="relative_low_income_rate", + subgroup="total", + variant="bhc", + geography="UK", + unit_concept="persons", + period="2020/21-2022/23\n", + value=0.17, + ) + monkeypatch.setattr( + "scorecard_db.ingest_uk_externals._load", lambda name: [hbai_row] + ) + with pytest.raises(ValueError, match="unparseable"): + stage_hbai() + + ukmod_row = _row( + source="ukmod", + program="poverty", + metric="poverty_rate_below_60pct_median\n", + subgroup="total", + variant="ukmod", # the primary variant — anything else is dropped + geography="UK", + unit_concept="fraction", + period="2026", + value=0.18, + ) + monkeypatch.setattr( + "scorecard_db.ingest_uk_externals._load", lambda name: [ukmod_row] + ) + with pytest.raises(ValueError, match="unknown metric"): + stage_ukmod() def test_dwp_mapping_and_drops(monkeypatch):