From 4c183770d0b0edc0a87141dda5ca50c21460e520 Mon Sep 17 00:00:00 2001 From: r Date: Mon, 24 Aug 2026 14:20:11 +0100 Subject: [PATCH 1/2] Ingest the two harvested-but-unused UK think-tank families (#86) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 2026-08-02 UK sweep staged seven families and five were ingested. IFS (268 rows) and Resolution Foundation (71) sat unused with their NOTES.md and manifests beside them, while the repo referenced both anyway — baselines.py registers ifs_2cl_fp_removal_rolled_out, and produce_campaign_uk declines four archived rows because RF is "a long-tail source (held)". 314 claims land. The accounting is exact and asserted: 339 staged rows = 314 ingested + 25 tallied drops Both are INDEPENDENT models, which is the benchmark class the scorecard is for, so every row is held_out with the consumption surfaces read at the certified pin (relationships.py) — except where the repo-wide permanent-holdout doctrine already answers, which it still wins. A proposal is not a decision. 145 rows carried only a harvest-side `proposed_metric`; DISPOSITIONS turns each of the 21 distinct names into a registered Metric or a tallied drop, and an unlisted name RAISES. The adopted ones land on eight new metrics that are all CHANGE or SHARE siblings of levels this repo already carries — benefit_cost_change, taxpayer_count_change, average_household_income_change, share_gaining, share_losing, spending_share, benefit_uprating_rate, real_income_growth — following the rule that already keeps revenue_change apart from revenue_level. Four names are dropped as DERIVED: a cost-per-child- lifted-out-of-poverty is a ratio of two quantities already staged separately, and PE could only "answer" it by dividing two of its own numbers, which makes agreement mechanical rather than evidential. Eight Resolution Foundation rows are not Resolution Foundation claims. They carry an `attribution` naming HM Treasury, a UK Parliament impact assessment or a "Government estimate cited by RF", and two say outright "not an RF model output". Staging them under resolution_foundation would attribute a government figure to a think tank, and a PE divergence against one would read as disagreement with a model that never produced it. They are dropped, and the guard runs BEFORE the metric disposition — load-bearing, because all eight carry a metric this module otherwise adopts. Identity is closed, and closing it was the whole job: both publishers name things in prose. IFS's coverage-restricted analyses get their own geographies rather than aliasing to UK — filing an England-and-Wales figure as UK is the exact fail-open this registry exists to stop — with the publication's verbatim wording kept as provenance. Deciles, quintiles and benefit names are canonical per source, and recorded DISTINCT against UKMOD's quintiles and each other's deciles. Deliberate unblock, as test_blocked_targets_have_no_claims_yet demands: RF is no longer claim-absent, so produce_campaign_uk's two blocked families get a PRECISE reason instead of a categorical one — the archived descriptors speak the harvest's proposal vocabulary while the ingested claims speak the decided one, and the two_child fiscal-cost row targets a claim this ingest deliberately drops. Both are machine-checked now rather than asserted in prose. Suite 286 passed, two builds agree on content_hash, no-drift clean, ruff format clean. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016HuXJFVme8HRbnke2Ey3Me --- app/public/data/lanes.json | 11 + data/lanes.json | 11 + scorecard_db/build_db.py | 5 + scorecard_db/ingest_uk_thinktanks.py | 466 +++++++++++++++++++++++++++ scorecard_db/models.py | 34 ++ scorecard_db/produce_campaign_uk.py | 29 +- scorecard_db/relationships.py | 28 ++ scorecard_db/uk_aliases.py | 146 +++++++++ tests/test_campaign_uk_producer.py | 50 ++- tests/test_uk_thinktanks_ingest.py | 329 +++++++++++++++++++ 10 files changed, 1101 insertions(+), 8 deletions(-) create mode 100644 scorecard_db/ingest_uk_thinktanks.py create mode 100644 tests/test_uk_thinktanks_ingest.py diff --git a/app/public/data/lanes.json b/app/public/data/lanes.json index d2b4a03..04eb078 100644 --- a/app/public/data/lanes.json +++ b/app/public/data/lanes.json @@ -357,6 +357,17 @@ "running": false, "updated": "2026-08-21", "note": "5 model claims, 7 Ledger facts (6 statistical + 1 non-simulated uprated EU-SILC survey input), 6 derived ratios dispositioned; 2 concept-mismatch attachments" + }, + { + "id": "uk-thinktanks", + "source": "IFS + Resolution Foundation", + "area": "independent UK tax-benefit modelling", + "mode": 2, + "country": "UK", + "stage": "ingested", + "running": false, + "updated": "2026-08-24", + "note": "314 claims from 2 independent models (339 staged rows = 314 ingested + 25 tallied drops)" } ] } diff --git a/data/lanes.json b/data/lanes.json index d2b4a03..04eb078 100644 --- a/data/lanes.json +++ b/data/lanes.json @@ -357,6 +357,17 @@ "running": false, "updated": "2026-08-21", "note": "5 model claims, 7 Ledger facts (6 statistical + 1 non-simulated uprated EU-SILC survey input), 6 derived ratios dispositioned; 2 concept-mismatch attachments" + }, + { + "id": "uk-thinktanks", + "source": "IFS + Resolution Foundation", + "area": "independent UK tax-benefit modelling", + "mode": 2, + "country": "UK", + "stage": "ingested", + "running": false, + "updated": "2026-08-24", + "note": "314 claims from 2 independent models (339 staged rows = 314 ingested + 25 tallied drops)" } ] } diff --git a/scorecard_db/build_db.py b/scorecard_db/build_db.py index f37be52..55d8fbb 100644 --- a/scorecard_db/build_db.py +++ b/scorecard_db/build_db.py @@ -19,6 +19,7 @@ campaign_us staged day-1/day-2 campaign results (claim matching) uk_externals five UK primary-source families + Ledger staging uk_deductions FRR family + uk_thinktanks IFS + Resolution Foundation (independent models) produce_uk + campaign_uk archive-resolved UK reckoner attaches be_jrc JRC EUROMOD-BE model claims + honest demo attachments; final so its 2026-08-21 lane update cannot be regressed by @@ -47,6 +48,7 @@ ingest_solo, ingest_uk_deductions, ingest_uk_externals, + ingest_uk_thinktanks, ingest_urban, produce_campaign_uk, ) @@ -94,6 +96,9 @@ def build(db_path: Path) -> dict: ("campaign_us", lambda: ingest_campaign.ingest(db_path)), ("uk_externals", lambda: ingest_uk_externals.ingest(db_path)), ("uk_deductions", lambda: ingest_uk_deductions.ingest(db_path)), + # Two independent UK models (#86), harvested in the 2026-08-02 + # sweep and unused until now. + ("uk_thinktanks", lambda: ingest_uk_thinktanks.ingest(db_path)), ("produce_uk", lambda: produce_campaign_uk.produce(db_path)), ( "campaign_uk", diff --git a/scorecard_db/ingest_uk_thinktanks.py b/scorecard_db/ingest_uk_thinktanks.py new file mode 100644 index 0000000..d96c613 --- /dev/null +++ b/scorecard_db/ingest_uk_thinktanks.py @@ -0,0 +1,466 @@ +"""Ingest the two harvested UK think-tank families (#86). + +The 2026-08-02 UK sweep staged seven families and only five were ever +ingested (#48's uk_hmrc, uk_dwp, uk_hmt, uk_obr, uk_ukmod_jrf). Two sat +unused with their NOTES.md and manifests beside them: + + sources/harvest-uk-2026-08-02/uk_ifs 268 rows + sources/harvest-uk-2026-08-02/uk_resolution_foundation 71 rows + +The repo already REFERENCED both while carrying neither — +``baselines.py`` registers ``ifs_2cl_fp_removal_rolled_out`` for an IFS +Green-Budget options world, and ``produce_campaign_uk`` declines four +archived campaign rows because Resolution Foundation is "a long-tail +source (held)". This module is that ingest. + +Both are INDEPENDENT models, which is the benchmark class the scorecard +exists for: no pe-uk-data target and no policyengine-uk parameter is +fitted to either (relationships.py carries the evidence, read at the +certified pin), so their rows are held_out and a divergence is a finding +rather than a tautology. + +Two honesty problems the staging carries, and how each is handled: + +1. **145 of the 339 rows have a null ``metric``** — they carry a + harvest-side ``proposed_metric`` instead. A proposal is not a + decision, so DISPOSITIONS below turns every one of them into either a + registered Metric or a tallied drop with a reason. Nothing is + inferred at runtime: an unregistered proposal RAISES. The 145 could + have shrunk silently to zero and nobody would have seen it, which is + what the exact accounting at the bottom of this module prevents. + +2. **8 Resolution Foundation rows are not Resolution Foundation + claims.** They carry an ``attribution`` naming HM Treasury, a UK + Parliament impact assessment, or "Government estimate cited by RF", + and two say so outright in their note ("not an RF model output"). + Staging them under ``resolution_foundation`` would attribute a + government figure to a think tank and let a PE-vs-RF divergence read + as disagreement with RF when RF never modelled it. They are dropped + here; re-publishing them correctly means staging them under their + ORIGINATOR, which is its own harvest decision, not this one. + +Same contract as ingest_uk_externals (scorecard_db/README.md): fail +loudly on any unmapped metric, unknown identity value or unhandled +staged field; values arrive in raw units and are NEVER re-derived here; +calibration_relationship is decided in relationships.py. + +Usage: + PYTHONPATH=. python -m scorecard_db.ingest_uk_thinktanks data/scorecard.db +""" + +from __future__ import annotations + +import gzip +import json +from pathlib import Path + +from .db import LANE_SQL, SCORES_SQL, ScorecardDB +from .harvest import REPO, finish, policy_ref, with_baseline_condition +from .models import ExternalScore, Metric, ReformRef, TimeBasis, UnitConcept +from .relationships import uk_relationship +from .uk_aliases import canon + +HARVEST = REPO / "sources" / "harvest-uk-2026-08-02" + +# harvest family dir -> (DB source id, display name) +FAMILIES = { + "uk_ifs": "ifs", + "uk_resolution_foundation": "resolution_foundation", +} + +LANE_ID = "uk-thinktanks" +LANE_UPDATED = "2026-08-24" +# The UK family's shared top-level feed literal (sync_lane_feed's +# contract: every caller in a build passes the same one). +FEED_UPDATED = "2026-08-19" +LANE_FEED_META = { + "source": "IFS + Resolution Foundation", + "area": "independent UK tax-benefit modelling", + "mode": 2, + "country": "UK", +} + +# --- the disposition table -------------------------------------------------- +# Every metric name the staging can present — its own `metric` or, where +# that is null, its `proposed_metric` — maps to exactly one of: +# +# a Metric ingest it under that registered metric +# a DROP reason do not ingest, and TALLY it with the reason +# +# An unlisted name raises. A proposal is a suggestion from the harvest; +# this table is where it becomes a decision. +DISPOSITIONS: dict[str, Metric] = { + # already-registered metrics, carried straight through + "poverty_rate": Metric.POVERTY_RATE, + "poverty_rate_change": Metric.POVERTY_RATE_CHANGE, + "poverty_count": Metric.POVERTY_COUNT, + "poverty_count_change": Metric.POVERTY_COUNT_CHANGE, + "revenue_change": Metric.REVENUE_CHANGE, + # proposals adopted onto existing metrics + # both count families made better off by a reform; the IFS row's + # own sign_convention says "count of families gaining" + "families_affected_count": Metric.GAINER_COUNT, + "benefiting_family_count": Metric.GAINER_COUNT, + "avg_gain_per_benefiting_family": Metric.AVERAGE_ANNUAL_GAIN, + # an exchequer cost is a revenue change; the published sign + # convention rides in conditions and the value is NOT re-signed here + "reform_fiscal_cost": Metric.REVENUE_CHANGE, + # proposals adopted onto the new sibling metrics (see models.py) + "avg_change_household_net_income": Metric.AVERAGE_HOUSEHOLD_INCOME_CHANGE, + "avg_income_change": Metric.AVERAGE_HOUSEHOLD_INCOME_CHANGE, + "benefit_spending_change": Metric.BENEFIT_COST_CHANGE, + "taxpayer_count_change": Metric.TAXPAYER_COUNT_CHANGE, + "share_gaining": Metric.SHARE_GAINING, + "share_losing": Metric.SHARE_LOSING, + "share_of_spending_to_group": Metric.SPENDING_SHARE, + "benefit_uprating_pct": Metric.BENEFIT_UPRATING_RATE, + "real_income_growth_pct": Metric.REAL_INCOME_GROWTH, + # the same quantity expressed per annum: a window_kind condition, + # not a different metric + "real_income_growth_pct_pa": Metric.REAL_INCOME_GROWTH, + "real_value_change_pct": Metric.REAL_INCOME_GROWTH, +} + +DROPS: dict[str, str] = { + "cost_per_child_lifted_out_of_poverty": ( + "A RATIO of two other published quantities (programme cost divided " + "by the poverty-count change). PolicyEngine could only 'answer' it " + "by dividing two of its own numbers, which would make agreement " + "mechanical rather than evidential — and the two inputs are already " + "staged as claims in their own right." + ), + "benefit_rate_gap_weekly": ( + "A gap between two published statutory rates, i.e. derived from two " + "quantities neither of which is staged here. Stage the rates and " + "let the gap be a downstream derivation." + ), + "inflation_rate_gap_pp": ( + "A gap between two published rates, derived; same rule as " + "benefit_rate_gap_weekly." + ), + "benefit_rate_weekly": ( + "A statutory weekly benefit RATE, which has no sibling metric in " + "this repo (average_weekly_benefit is an average of actual " + "receipts, a different quantity). One row does not justify minting " + "a metric; deferred deliberately rather than mapped to a " + "near-neighbour." + ), + "avg_gain_per_unit": ( + "The denominator population is unstated in the staging ('per " + "unit'), so the number cannot be given an identity. A per-unit " + "statistic whose unit is unknown is not a claim." + ), +} + +# Rows whose `attribution` names a third party are that party's claims, +# not the publisher's (see the module docstring). +THIRD_PARTY_DROP = ( + "Re-published third-party figure: the staged row carries an " + "`attribution` naming its true originator (HM Treasury, a UK " + "Parliament impact assessment, or a Government estimate cited by the " + "publisher). Staging it under the publisher would attribute a " + "government number to a think tank, and a PE divergence against it " + "would read as disagreement with a model that never produced it. " + "Re-publishing it correctly means staging it under its originator, " + "which is a separate harvest decision." +) + +_ADOPTED = frozenset(DISPOSITIONS) +_DROPPED = frozenset(DROPS) +assert not (_ADOPTED & _DROPPED), "a metric is adopted or dropped, never both" + +# Adapter unit label -> DB unit concept. Closed: an unregistered unit +# raises rather than landing a number under a unit nobody chose. +UNITS = { + "gbp": UnitConcept.GBP, + "gbp_per_year": UnitConcept.GBP, + "share": UnitConcept.SHARE, + "percent": UnitConcept.PERCENT, + "percentage_points": UnitConcept.PERCENT, + "persons": UnitConcept.PERSONS, + "families": UnitConcept.FAMILIES, + "children_under_18": UnitConcept.CHILDREN_UNDER_18, +} + +TIME_BASES = { + "annual": TimeBasis.ANNUAL, + "fiscal_year": TimeBasis.FISCAL_YEAR, + "point_in_time": TimeBasis.POINT_IN_TIME, + "average_month": TimeBasis.AVERAGE_MONTH, +} + + +# Staged fields this module understands. An unknown field raises: a new +# harvest column is handled here DELIBERATELY or not at all. +_KNOWN_FIELDS = frozenset( + { + "attribution", + "calibration_relationship", + "conditions", + "geography_note", + "local_artifact", + "metric", + "normalization", + "note", + "parse_confidence", + "period", + "proposed_metric", + "proposed_unit", + "publication", + "reform_hint", + "sign_convention", + "source", + "source_column", + "source_model", + "source_table", + "status", + "time_basis", + "unit_concept", + "value", + "value_kind", + # the publication's own rendering ("£2.4 billion", "560,000 + # families"), kept as provenance beside the parsed number + "value_raw", + } +) + +# Condition keys whose values are closed identities (uk_aliases); every +# other staged condition is descriptive provenance and travels verbatim. +_CANONICALISED = {"geography", "program", "income_group", "benefit"} + + +def _load(family: str) -> list[dict]: + path = HARVEST / family / "claims_staged.jsonl.gz" + if not path.exists(): + raise FileNotFoundError(f"{path} missing — the harvest family is not vendored") + rows = [] + for line in gzip.open(path, "rt"): + row = json.loads(line) + unknown = sorted(set(row) - _KNOWN_FIELDS) + if unknown: + raise ValueError( + f"{family}: unhandled staged fields {unknown} — handle them " + "deliberately or not at all" + ) + rows.append(row) + return rows + + +def _metric_name(row: dict) -> str: + """The name this row presents: its own metric, else its PROPOSAL. + + A proposal is a suggestion from the harvest, never a decision — it + only reaches a claim through DISPOSITIONS. + """ + name = row.get("metric") or row.get("proposed_metric") + if not name: + raise ValueError( + f"staged row carries neither metric nor proposed_metric: " + f"{row.get('source_column')!r}" + ) + return name + + +def _reform(row: dict, source: str) -> ReformRef: + """The world the row scores. + + A row with a reform_hint is a REFORM score and names the world; a row + without one is a level or a projection of the current world. + """ + hint = row.get("reform_hint") + if not hint: + return ReformRef() + baseline = None + conditions = row.get("conditions") or {} + # The IFS Green-Budget options are scored against a registered + # non-current-law world (baselines.py), which the staging names in + # its own conditions rather than in the hint. + if conditions.get("counterfactual", "").startswith("current tax-benefit system"): + baseline = {"policy": "ifs_2cl_fp_removal_rolled_out"} + return policy_ref(f"{source}:{hint[:120]}", baseline=baseline) + + +def stage() -> tuple[list[ExternalScore], dict]: + """Stage every ingestible row; nothing touches the DB here. + + Returns (scores, accounting) where accounting reconciles EVERY staged + row: read = ingested + dropped, by reason. + """ + scores: list[ExternalScore] = [] + acct = { + "read": 0, + "ingested": 0, + "dropped": 0, + "by_family": {}, + "drops": {}, + } + for family, source in FAMILIES.items(): + rows = _load(family) + acct["read"] += len(rows) + fam_stats = {"read": len(rows), "ingested": 0, "dropped": 0} + for row in rows: + if row["source"] != source: + raise ValueError( + f"{family}: row source {row['source']!r} is not {source!r}" + ) + name = _metric_name(row) + # A third-party figure is its originator's claim, not this + # publisher's — checked BEFORE the metric disposition, so an + # otherwise-ingestible metric cannot smuggle one in. + if row.get("attribution"): + _drop(acct, fam_stats, "third_party_attribution", THIRD_PARTY_DROP) + continue + if name in DROPS: + _drop(acct, fam_stats, name, DROPS[name]) + continue + if name not in DISPOSITIONS: + raise ValueError( + f"{family}: metric {name!r} has no disposition — decide it " + "in DISPOSITIONS or DROPS deliberately; a proposal is not " + "a decision" + ) + scores.append(_score(row, source, DISPOSITIONS[name])) + fam_stats["ingested"] += 1 + acct["ingested"] += 1 + acct["by_family"][source] = fam_stats + acct["dropped"] = acct["read"] - acct["ingested"] + if acct["ingested"] + acct["dropped"] != acct["read"]: # pragma: no cover + raise ValueError("accounting does not close") + return finish(scores, "uk_thinktanks"), acct + + +def _drop(acct: dict, fam_stats: dict, reason_key: str, reason: str) -> None: + entry = acct["drops"].setdefault(reason_key, {"rows": 0, "reason": reason}) + entry["rows"] += 1 + fam_stats["dropped"] += 1 + + +def _score(row: dict, source: str, metric: Metric) -> ExternalScore: + unit_label = row.get("unit_concept") or row.get("proposed_unit") + if unit_label is None: + raise ValueError( + f"{source}: {metric.value} row carries no unit — a number without " + f"a unit is not a claim ({row.get('source_column')!r})" + ) + canon(source, "unit", unit_label) + if unit_label not in UNITS: + raise ValueError(f"{source}: unregistered unit {unit_label!r}") + unit = UNITS[unit_label] + + staged_conditions = dict(row.get("conditions") or {}) + cond: dict[str, str] = {"country": "UK"} + for key, value in staged_conditions.items(): + if key in _CANONICALISED: + cond[key] = canon(source, key, value) + if cond[key] != value: + # the publication's own wording survives as provenance + cond[f"{key}_verbatim"] = value + else: + cond[key] = value + if row.get("geography_note"): + cond["geography_note"] = row["geography_note"] + if row.get("sign_convention"): + cond["sign_convention"] = row["sign_convention"] + if row.get("value_raw") is not None: + # Some families stage the raw rendering as a number rather than a + # string ("207" vs "£2.4 billion"); conditions are str->str. + cond["value_verbatim"] = str(row["value_raw"]) + # "per annum" is a window shape, not a different quantity + if row.get("proposed_metric") in ("real_income_growth_pct_pa",): + cond["window_kind"] = "annual_average" + + basis = TIME_BASES.get(row.get("time_basis")) + if basis is None: + raise ValueError(f"{source}: unregistered time_basis {row.get('time_basis')!r}") + + reform = _reform(row, source) + with_baseline_condition(cond, reform) + return ExternalScore( + source=source, + metric=metric, + unit_concept=unit, + period=int(row["period"]), + time_basis=basis, + value=float(row["value"]), + conditions=cond, + reform=reform, + calibration_relationship=uk_relationship(source, metric)[0], + source_model=row.get("source_model") or source, + source_column=row.get("source_column") or "", + publication=row.get("publication") or {}, + value_kind=row.get("value_kind") or unit.value, + status="ok", + ) + + +# Exact accounting for the committed harvest. A drifted re-stage must +# fail HERE, never grow or shrink the catalog silently — and in +# particular the 145 proposal-only rows must never quietly become zero. +_EXPECTED = { + "read": 339, + "ingested": 314, + "dropped": 25, + "drops": { + "cost_per_child_lifted_out_of_poverty": 12, + "third_party_attribution": 8, + "benefit_rate_gap_weekly": 2, + "avg_gain_per_unit": 1, + "benefit_rate_weekly": 1, + "inflation_rate_gap_pp": 1, + }, +} + + +def check_accounting(acct: dict) -> None: + got = { + "read": acct["read"], + "ingested": acct["ingested"], + "dropped": acct["dropped"], + "drops": {k: v["rows"] for k, v in acct["drops"].items()}, + } + if got != _EXPECTED: + raise ValueError(f"claim accounting drifted: {got} != {_EXPECTED}") + + +def ingest(db_path: Path) -> dict: + """Stage and validate first; then ONE transaction replaces both + sources wholesale and runs the baseline-registration gate inside it, + exactly as ingest_uk_externals does.""" + scores, acct = stage() + check_accounting(acct) + db = ScorecardDB(db_path) + rows = [ScorecardDB.score_row(s) for s in scores] + + from .baselines import register_baselines_txn + from .ingest_harvest import sync_lane_feed + + with db.conn: + for source in FAMILIES.values(): + db.conn.execute("DELETE FROM external_scores WHERE source = ?", (source,)) + db.conn.executemany(SCORES_SQL, rows) + register_baselines_txn(db) + detail = ( + f"{acct['ingested']} claims from {len(FAMILIES)} independent " + f"models ({acct['read']} staged rows = {acct['ingested']} ingested " + f"+ {acct['dropped']} tallied drops)" + ) + db.conn.execute(LANE_SQL, (LANE_ID, "ingested", detail, LANE_UPDATED)) + sync_lane_feed( + db, + REPO / "data" / "lanes.json", + FEED_UPDATED, + lanes={LANE_ID: LANE_FEED_META}, + ) + db.close() + return { + "claims": len(rows), + **{k: v for k, v in acct.items() if k != "drops"}, + "drops": {k: v["rows"] for k, v in acct["drops"].items()}, + } + + +if __name__ == "__main__": + import sys + + out = Path(sys.argv[1] if len(sys.argv) > 1 else "data/scorecard.db") + print(json.dumps(ingest(out), indent=1)) diff --git a/scorecard_db/models.py b/scorecard_db/models.py index 4700771..3d83eef 100644 --- a/scorecard_db/models.py +++ b/scorecard_db/models.py @@ -90,6 +90,40 @@ class Metric(str, Enum): CASH_REQUIREMENT_CHANGE = "cash_requirement_change" GAINER_COUNT = "gainer_count" AVERAGE_ANNUAL_GAIN = "average_annual_gain" + # UK think-tank families (#86: IFS, Resolution Foundation). Each is a + # CHANGE or SHARE sibling of a level metric this repo already carries, + # following the same rule that keeps revenue_change apart from + # revenue_level and poverty_rate_change apart from poverty_rate: a + # change is not a level, and the two must never be summed or compared. + # benefit_cost_change sibling of benefit_cost + # taxpayer_count_change sibling of taxpayer_count + # average_household_income_change currency-neutral sibling of + # avg_change_after_tax_income_usd + # (that one is legacy-named; a new + # currency rides unit_concept, not + # the metric name) + # share_gaining / share_losing siblings of share_with_tax_cut, + # kept as TWO metrics because "not + # gaining" is not "losing" — a + # household can be unaffected, and + # one minus the other is not the + # complement + # spending_share sibling of income_share: the + # share of a programme's spending + # reaching an income group + # benefit_uprating_rate the uprating applied to a benefit + # rate — a policy parameter, not a + # receipt + # real_income_growth real growth in a household income + # statistic between two periods + BENEFIT_COST_CHANGE = "benefit_cost_change" + TAXPAYER_COUNT_CHANGE = "taxpayer_count_change" + AVERAGE_HOUSEHOLD_INCOME_CHANGE = "average_household_income_change" + SHARE_GAINING = "share_gaining" + SHARE_LOSING = "share_losing" + SPENDING_SHARE = "spending_share" + BENEFIT_UPRATING_RATE = "benefit_uprating_rate" + REAL_INCOME_GROWTH = "real_income_growth" class UnitConcept(str, Enum): diff --git a/scorecard_db/produce_campaign_uk.py b/scorecard_db/produce_campaign_uk.py index baadd81..dee2db6 100644 --- a/scorecard_db/produce_campaign_uk.py +++ b/scorecard_db/produce_campaign_uk.py @@ -38,13 +38,30 @@ policy-measures costings database (long-tail source, held on the DB-storage decision) two_child NOT RESOLVED (4 rows = 2 resolution_foundation - + 1 ukmod poverty_count_change + 1 metaless exhibit): RF is a - long-tail source (held), and the ukmod row targets a REFORM - claim — the staged UKMOD family is baseline validation - statistics only, so a 2CL-reform claim needs its own staging - decision + + 1 ukmod poverty_count_change + 1 metaless exhibit). RF is no + longer a long-tail source — #86 ingested 58 RF claims — so the + reason is now precise rather than categorical: + * the poverty_count_change row's descriptor is under-specified + against the ingested vocabulary (unit_population "children" + with no income_concept or poverty_line), the same + under-specification #48 forced the reckoner rows through the + derivation below; + * the reform_fiscal_cost row targets a claim #86 DELIBERATELY + DROPS — it is an HM Treasury scorecard figure that RF + re-publishes, so there is no RF claim to attach to and there + should not be; + * the ukmod row targets a REFORM claim and the staged UKMOD + family is baseline validation statistics only. uprating_april2026 NOT RESOLVED (4 rows = 3 resolution_foundation - benefit_uprating_pct + 1 metaless exhibit): RF long-tail, held + benefit_uprating_pct + 1 metaless exhibit). Also no longer "RF + long-tail, held": the claims exist, but the archived descriptors + speak the HARVEST's proposal vocabulary (metric + "benefit_uprating_pct", benefit named in verbatim prose such as + "UC standard allowance, under-25s") while the ingested claims + speak the DECIDED vocabulary (metric benefit_uprating_rate, + benefit universal_credit_standard_allowance_under_25). Resolving + them means deriving the mapping the way the reckoner family is + derived below — a deliberate act, not a looser match. The metaless exhibits (5 rows across the families — the free_joins pair is one construction duplicated) carry exhibit_context but no diff --git a/scorecard_db/relationships.py b/scorecard_db/relationships.py index 31c3d7f..2505603 100644 --- a/scorecard_db/relationships.py +++ b/scorecard_db/relationships.py @@ -222,6 +222,32 @@ def effective_relationship(program, metric): "nothing in pe-uk-data consumes them.", ) +# UK think tanks (#86). Both are INDEPENDENT models — which is precisely +# why their rows are worth carrying: agreement is evidence rather than a +# tautology. Verified at the certified pin before staging, per the #48 +# rule that a held-out claim states where it looked. +_THINKTANK_HELD = { + "ifs": ( + CR.HELD_OUT, + "TAXBEN is the IFS's own microsimulation model, maintained " + "independently of PolicyEngine; no pe-uk-data target and no " + "policyengine-uk parameter is fitted to an IFS output " + "(consumption surfaces read 2026-08-24 at the certified pins — " + "the only IFS material the repo previously referenced is the " + "Green-Budget options BASELINE world in baselines.py, which is a " + "counterfactual descriptor, not a calibration target).", + ), + "resolution_foundation": ( + CR.HELD_OUT, + "Resolution Foundation's living-standards modelling is " + "independent of PolicyEngine and nothing in pe-uk-data or " + "policyengine-uk consumes it (surfaces read 2026-08-24 at the " + "certified pins). Note that RF itself re-publishes government " + "figures; those rows are dropped at ingest rather than carried " + "as RF claims, so this relationship covers RF's OWN outputs only.", + ), +} + _UKMOD_HELD = ( CR.HELD_OUT, "UKMOD is a peer microsimulation, not a calibration source; no PE UK " @@ -276,6 +302,8 @@ def uk_relationship(source, metric, program=None, kind=None): if program in OBR_CONSUMED_WELFARE_PROGRAMS: return _OBR_CONSUMED return _OBR_UNCONSUMED + if source in ("ifs", "resolution_foundation"): + return _THINKTANK_HELD[source] if source == "ukmod": return _UKMOD_HELD if source == "hm_treasury": diff --git a/scorecard_db/uk_aliases.py b/scorecard_db/uk_aliases.py index 33d5a49..5330911 100644 --- a/scorecard_db/uk_aliases.py +++ b/scorecard_db/uk_aliases.py @@ -55,6 +55,16 @@ def _alias(source: str, axis: str, source_value: str, canonical: str) -> None: ("dwp_takeup:housing_benefit_pensioners", "obr:housing_benefit_on_jsa"), ("dwp_takeup:housing_benefit_pensioners", "ukmod:housing_benefit"), ("dwp_takeup:benefit_units", "ukmod:families"), + # A decile of one publisher's distribution is not a decile — or a + # quintile — of another's: different models, different income + # concepts, different cut points. Never aliased. + ("ifs:decile_1", "ukmod:q1"), + ("ifs:decile_10", "ukmod:q5"), + ("resolution_foundation:quintile_1", "ukmod:q1"), + ("resolution_foundation:decile_1", "ifs:decile_1"), + # And a coverage-restricted geography is not the UK. + ("ifs:UK_excl_northern_ireland", "ifs:UK"), + ("ifs:UK_excl_scotland", "ifs:UK"), } ) @@ -299,6 +309,142 @@ def _alias(source: str, axis: str, source_value: str, canonical: str) -> None: _identity(_src, "unit", ["gbp", "households"]) +# --- UK think tanks (#86: IFS, Resolution Foundation) ------------------------ +# Both publish PROSE identities — "UK excluding Northern Ireland (note +# verbatim: ...)", "Decile Poorest", "poorest fifth (quintile 1)" — so +# closing them is the whole identity job for this lane. Two rules drive it: +# +# 1. A coverage-restricted geography is NOT the UK. IFS's Scotland- and +# NI-excluding analyses get their own values; aliasing them to "UK" +# would file an England-and-Wales figure as a UK one, the exact +# fail-open this registry exists to stop. The publication's verbatim +# wording stays on the claim as a geography_note. +# 2. A decile of one publisher's distribution is not a decile of +# another's. The IFS and RF groups are registered per source and +# recorded DISTINCT from UKMOD's quintiles and HBAI's deciles. +_identity( + "ifs", + "geography", + ["UK", "England", "UK_excl_northern_ireland", "UK_excl_scotland"], +) +_alias("ifs", "geography", "UK excluding Northern Ireland", "UK_excl_northern_ireland") +_alias( + "ifs", + "geography", + "UK excluding Northern Ireland (note verbatim: 'Northern Ireland is not " + "included in this analysis')", + "UK_excl_northern_ireland", +) +_alias( + "ifs", + "geography", + "UK excluding Scotland (note verbatim: 'Excludes Scotland')", + "UK_excl_scotland", +) +_identity("resolution_foundation", "geography", ["UK", "GB", "Scotland"]) + +_identity("ifs", "program", ["universal_credit"]) +_identity( + "ifs", + "income_group", + [f"decile_{_i}" for _i in range(1, 11)] + ["all"], +) +for _i, _label in enumerate( + ["Decile Poorest"] + [f"Decile {_n}" for _n in range(2, 10)] + ["Decile Richest"], + start=1, +): + _alias("ifs", "income_group", _label, f"decile_{_i}") +_alias("ifs", "income_group", "All", "all") + +# Resolution Foundation mixes quintiles, deciles and halves in one +# publication, so the vocabulary carries all three shapes explicitly +# rather than forcing them onto one grid. +_identity( + "resolution_foundation", + "income_group", + [ + "quintile_1", + "quintile_5", + "decile_1", + "decile_10", + "bottom_half", + "top_half", + "decile_1_vs_decile_10", + ], +) +for _src_value, _canon in ( + ("poorest fifth (quintile 1)", "quintile_1"), + ("richest fifth (quintile 5)", "quintile_5"), + ("poorest decile (decile 1)", "decile_1"), + ("richest tenth (decile 10)", "decile_10"), + ("richest decile (decile 10)", "decile_10"), + ("bottom half", "bottom_half"), + ("top half", "top_half"), + ("lowest income decile vs highest income decile", "decile_1_vs_decile_10"), +): + _alias("resolution_foundation", "income_group", _src_value, _canon) + +# RF names the benefit in prose; these are the instruments, closed. +_identity( + "resolution_foundation", + "benefit", + [ + "universal_credit_standard_allowance", + "universal_credit_standard_allowance_under_25", + "universal_credit_health_element", + "local_housing_allowance", + "state_pension", + "inflation_linked_benefits", + ], +) +for _src_value, _canon in ( + ("Universal Credit standard allowance", "universal_credit_standard_allowance"), + ("UC standard allowance", "universal_credit_standard_allowance"), + ( + "basic rate of unemployment benefits (UC standard allowance)", + "universal_credit_standard_allowance", + ), + ( + "UC standard allowance, under-25s", + "universal_credit_standard_allowance_under_25", + ), + ("UC health element, existing recipients", "universal_credit_health_element"), + ("UC health element (new claimants)", "universal_credit_health_element"), + ("Local Housing Allowance (frozen)", "local_housing_allowance"), + ("State Pension (triple lock: earnings growth)", "state_pension"), + ("inflation-linked benefits", "inflation_linked_benefits"), +): + _alias("resolution_foundation", "benefit", _src_value, _canon) + +_identity( + "ifs", + "unit", + [ + "gbp", + "gbp_per_year", + "share", + "percent", + "percentage_points", + "persons", + "children_under_18", + "families", + ], +) +_identity( + "resolution_foundation", + "unit", + [ + "gbp", + "share", + "percent", + "percentage_points", + "persons", + "children_under_18", + "families", + ], +) + + def canon(source: str, axis: str, value: str) -> str: """Canonical value for (source, axis, source_value); unknown raises.""" try: diff --git a/tests/test_campaign_uk_producer.py b/tests/test_campaign_uk_producer.py index c49bbe7..50e5d2c 100644 --- a/tests/test_campaign_uk_producer.py +++ b/tests/test_campaign_uk_producer.py @@ -128,8 +128,54 @@ def n(sql, *args): ) == 0 ) - for source in ("uk_dwp", "resolution_foundation"): - assert n("SELECT COUNT(*) FROM external_scores WHERE source=?", source) == 0 + assert n("SELECT COUNT(*) FROM external_scores WHERE source=?", "uk_dwp") == 0 + # Resolution Foundation is NO LONGER claim-absent: #86 ingested 58 RF + # claims, which is what this assertion was built to catch. The + # deliberate unblock is recorded rather than relaxed — the two + # affected families stay blocked for a now-PRECISE reason, and this + # is the machine-checked form of it. + assert ( + n( + "SELECT COUNT(*) FROM external_scores WHERE source=?", + "resolution_foundation", + ) + > 0 + ) + # 1. the archived descriptors speak the harvest's PROPOSAL vocabulary, + # which no ingested claim carries + assert ( + n( + "SELECT COUNT(*) FROM external_scores WHERE source='resolution_foundation'" + " AND metric='benefit_uprating_pct'" + ) + == 0 + ) + assert ( + n( + "SELECT COUNT(*) FROM external_scores WHERE source='resolution_foundation'" + " AND metric='benefit_uprating_rate'" + ) + > 0 + ) + # 2. ...and the benefit names are canonical slugs, not the verbatim + # prose the descriptors carry + assert ( + n( + "SELECT COUNT(*) FROM external_scores WHERE source='resolution_foundation'" + " AND json_extract(conditions,'$.benefit')='UC standard allowance, under-25s'" + ) + == 0 + ) + # 3. the two_child fiscal-cost row targets a claim #86 deliberately + # drops (an HMT figure RF re-publishes), so there is nothing to + # attach to and there should not be + assert ( + n( + "SELECT COUNT(*) FROM external_scores WHERE source='resolution_foundation'" + " AND json_extract(conditions,'$.measure')='exchequer cost'" + ) + == 0 + ) assert ( n( "SELECT COUNT(*) FROM external_scores WHERE source='ukmod'" diff --git a/tests/test_uk_thinktanks_ingest.py b/tests/test_uk_thinktanks_ingest.py new file mode 100644 index 0000000..df82015 --- /dev/null +++ b/tests/test_uk_thinktanks_ingest.py @@ -0,0 +1,329 @@ +"""The IFS + Resolution Foundation ingest (#86). + +Two families were harvested in the 2026-08-02 UK sweep and never +ingested. What is pinned here is the thing that could most easily go +wrong quietly: the disposition of the 145 rows that carry only a +harvest-side PROPOSAL, and the 8 rows that are not the publisher's +claims at all. +""" + +import gzip +import json +import sqlite3 +from pathlib import Path + +import pytest + +from scorecard_db import ScorecardDB +from scorecard_db.ingest_uk_thinktanks import ( + DISPOSITIONS, + DROPS, + FAMILIES, + HARVEST, + _EXPECTED, + check_accounting, + ingest, + stage, +) +from scorecard_db.models import CalibrationRelationship, Metric + +ROOT = Path(__file__).resolve().parent.parent + + +@pytest.fixture(scope="module") +def staged(): + return stage() + + +def _raw(family): + p = HARVEST / family / "claims_staged.jsonl.gz" + return [json.loads(line) for line in gzip.open(p, "rt")] + + +# --- the accounting -------------------------------------------------------- + + +def test_every_staged_row_is_ingested_or_tallied(staged): + """339 read = 314 ingested + 25 dropped. The 145 proposal-only rows + could have shrunk silently to zero and nobody would have seen it.""" + scores, acct = staged + assert acct["read"] == 339 + assert acct["ingested"] == len(scores) == 314 + assert acct["dropped"] == 25 + assert acct["ingested"] + acct["dropped"] == acct["read"] + check_accounting(acct) + + +def test_the_harvest_still_holds_what_the_accounting_claims(): + assert len(_raw("uk_ifs")) == 268 + assert len(_raw("uk_resolution_foundation")) == 71 + + +def test_every_drop_states_a_reason(staged): + _, acct = staged + for key, entry in acct["drops"].items(): + assert entry["rows"] > 0 + assert len(entry["reason"]) > 80, key + + +def test_claim_ids_do_not_collide(staged): + scores, _ = staged + assert len({s.claim_id() for s in scores}) == len(scores) + + +# --- a proposal is not a decision ------------------------------------------ + + +def test_all_proposal_only_rows_are_dispositioned(): + """145 rows carry `proposed_metric` and no `metric`. Every proposed + name must be a deliberate decision — adopted or dropped.""" + proposals = { + r["proposed_metric"] + for family in FAMILIES + for r in _raw(family) + if not r.get("metric") and r.get("proposed_metric") + } + assert len(proposals) == 21 + undecided = sorted(proposals - set(DISPOSITIONS) - set(DROPS)) + assert not undecided, undecided + n_proposal_only = sum( + 1 for family in FAMILIES for r in _raw(family) if not r.get("metric") + ) + assert n_proposal_only == 145 + + +def test_an_undecided_proposal_raises(monkeypatch): + monkeypatch.delitem(DISPOSITIONS, "share_of_spending_to_group") + with pytest.raises(ValueError, match="a proposal is not a decision"): + stage() + + +def test_a_metric_is_adopted_or_dropped_never_both(): + assert not (set(DISPOSITIONS) & set(DROPS)) + + +def test_adopted_proposals_land_on_registered_metrics(staged): + scores, _ = staged + used = {s.metric for s in scores} + assert Metric.SPENDING_SHARE in used + assert Metric.BENEFIT_UPRATING_RATE in used + assert Metric.REAL_INCOME_GROWTH in used + assert Metric.AVERAGE_HOUSEHOLD_INCOME_CHANGE in used + assert Metric.BENEFIT_COST_CHANGE in used + assert all(isinstance(m, Metric) for m in used) + + +def test_change_metrics_are_distinct_from_their_levels(): + """A change is not a level — the rule that already keeps + revenue_change apart from revenue_level.""" + for level, change in ( + (Metric.BENEFIT_COST, Metric.BENEFIT_COST_CHANGE), + (Metric.TAXPAYER_COUNT, Metric.TAXPAYER_COUNT_CHANGE), + (Metric.POVERTY_RATE, Metric.POVERTY_RATE_CHANGE), + ): + assert level is not change and level.value != change.value + + +def test_derived_ratios_are_dropped_not_ingested(): + """cost-per-child-lifted-out-of-poverty is cost divided by a poverty + change: PE could only 'answer' it by dividing two of its own numbers, + which makes agreement mechanical rather than evidential.""" + assert "cost_per_child_lifted_out_of_poverty" in DROPS + assert "ratio" in DROPS["cost_per_child_lifted_out_of_poverty"].lower() + for key in ("benefit_rate_gap_weekly", "inflation_rate_gap_pp"): + assert "derived" in DROPS[key].lower() + + +# --- attribution: whose claim is it? --------------------------------------- + + +def test_third_party_rows_are_not_staged_as_publisher_claims(staged): + """8 RF rows carry an `attribution` naming HM Treasury, a Parliament + impact assessment or a Government estimate. Staging them under + `resolution_foundation` would attribute a government figure to a + think tank, and a PE divergence would read as disagreement with a + model that never produced it.""" + _, acct = staged + assert acct["drops"]["third_party_attribution"]["rows"] == 8 + attributed = [r for r in _raw("uk_resolution_foundation") if r.get("attribution")] + assert len(attributed) == 8 + assert any("HM Treasury" in r["attribution"] for r in attributed) + assert any("Parliament" in r["attribution"] for r in attributed) + + +def test_attribution_is_checked_before_the_metric_disposition(staged): + """The ordering is load-bearing: ALL EIGHT attributed rows carry a + metric this module adopts, so a guard placed after the disposition + would ingest every one of them.""" + attributed = [r for r in _raw("uk_resolution_foundation") if r.get("attribution")] + adopted = [ + r + for r in attributed + if (r.get("metric") or r.get("proposed_metric")) in DISPOSITIONS + ] + assert len(adopted) == 8, "the ordering guard would be untested otherwise" + + # RF contributes 71 read - 13 dropped = 58; the eight are among the + # dropped, not the ingested. + _, acct = staged + assert acct["by_family"]["resolution_foundation"]["ingested"] == 58 + + # ...and the guard really does sit first in the loop + import inspect + + from scorecard_db import ingest_uk_thinktanks as tt + + src = inspect.getsource(tt.stage) + assert src.index('row.get("attribution")') < src.index("name in DROPS") + + +# --- identity -------------------------------------------------------------- + + +def test_coverage_restricted_geographies_are_not_the_uk(staged): + """IFS's Scotland- and NI-excluding analyses are not UK figures; + aliasing them to UK would file an England-and-Wales number as UK.""" + from scorecard_db.uk_aliases import DISTINCT + + scores, _ = staged + geos = {s.conditions.get("geography") for s in scores} + assert "UK_excl_northern_ireland" in geos + assert "UK_excl_scotland" in geos + assert ("ifs:UK_excl_northern_ireland", "ifs:UK") in DISTINCT + # ...and the publication's own wording survives as provenance + restricted = next( + s for s in scores if s.conditions.get("geography") == "UK_excl_scotland" + ) + assert "Excludes Scotland" in restricted.conditions["geography_verbatim"] + + +def test_publisher_deciles_are_never_unified(staged): + from scorecard_db.uk_aliases import DISTINCT + + assert ("ifs:decile_1", "ukmod:q1") in DISTINCT + assert ("resolution_foundation:decile_1", "ifs:decile_1") in DISTINCT + + +def test_prose_identities_are_canonicalised(staged): + scores, _ = staged + groups = { + s.conditions["income_group"] for s in scores if "income_group" in s.conditions + } + assert "Decile Poorest" not in groups + assert "decile_1" in groups + benefits = {s.conditions["benefit"] for s in scores if "benefit" in s.conditions} + assert benefits <= { + "universal_credit_standard_allowance", + "universal_credit_standard_allowance_under_25", + "universal_credit_health_element", + "local_housing_allowance", + "state_pension", + "inflation_linked_benefits", + } + + +def test_an_unregistered_identity_value_raises(staged): + from scorecard_db.uk_aliases import canon + + with pytest.raises(ValueError, match="unregistered"): + canon("ifs", "income_group", "the poorest people") + + +def test_every_row_is_uk_and_held_out(staged): + scores, _ = staged + assert all(s.conditions["country"] == "UK" for s in scores) + assert all( + s.calibration_relationship is CalibrationRelationship.HELD_OUT for s in scores + ) + + +def test_the_held_out_evidence_names_where_it_looked(): + from scorecard_db.relationships import uk_relationship + + # poverty_rate is a PERMANENT repo-wide holdout, so it never reaches + # a per-source branch — use a metric that does. + for source in ("ifs", "resolution_foundation"): + _, evidence = uk_relationship(source, Metric.REVENUE_CHANGE) + assert "2026-08-24" in evidence + assert "pe-uk-data" in evidence or "policyengine-uk" in evidence + # and the doctrine still wins where it applies + rel, basis = uk_relationship("ifs", Metric.POVERTY_RATE) + assert rel is CalibrationRelationship.HELD_OUT + assert "PERMANENT holdout" in basis + + +def test_values_are_never_re_derived(staged): + """Values arrive in raw units from the harvest. In particular an + exchequer COST is mapped to revenue_change without re-signing it — + the published sign convention rides in conditions instead.""" + scores, _ = staged + raw = {} + for family, source in FAMILIES.items(): + for r in _raw(family): + raw.setdefault((source, r.get("source_column")), []).append(r["value"]) + for s in scores: + vals = raw.get((s.source, s.source_column)) + assert vals is not None and s.value in vals + + +# --- persistence ----------------------------------------------------------- + + +def test_full_ingest_round_trip(tmp_path): + db_path = tmp_path / "t.db" + ScorecardDB(db_path).close() + summary = ingest(db_path) + assert summary["claims"] == 314 + conn = sqlite3.connect(db_path) + conn.row_factory = sqlite3.Row + n = conn.execute( + "SELECT COUNT(*) FROM external_scores WHERE source IN ('ifs','resolution_foundation')" + ).fetchone()[0] + assert n == 314 + consumed = conn.execute( + "SELECT COUNT(*) FROM external_scores" + " WHERE source IN ('ifs','resolution_foundation')" + " AND calibration_relationship != 'held_out'" + ).fetchone()[0] + assert consumed == 0 + lane = conn.execute( + "SELECT stage, detail FROM lanes WHERE lane = 'uk-thinktanks'" + ).fetchone() + conn.close() + assert lane["stage"] == "ingested" + assert "339 staged rows = 314 ingested + 25 tallied drops" in lane["detail"] + + +def test_ingest_is_idempotent(tmp_path): + db_path = tmp_path / "t.db" + ScorecardDB(db_path).close() + ingest(db_path) + ingest(db_path) + conn = sqlite3.connect(db_path) + n = conn.execute( + "SELECT COUNT(*) FROM external_scores WHERE source IN ('ifs','resolution_foundation')" + ).fetchone()[0] + conn.close() + assert n == 314 + + +def test_build_db_registers_the_step(): + import inspect + + from scorecard_db import build_db + + assert "ingest_uk_thinktanks.ingest" in inspect.getsource(build_db) + + +def test_the_lane_reaches_mission_control(): + lanes = json.loads((ROOT / "data" / "lanes.json").read_text())["lanes"] + entry = next(lane for lane in lanes if lane["id"] == "uk-thinktanks") + assert entry["country"] == "UK" + + +def test_accounting_drift_raises(): + bad = {"read": 339, "ingested": 313, "dropped": 26, "drops": _EXPECTED["drops"]} + with pytest.raises(ValueError, match="accounting drifted"): + check_accounting( + {**bad, "drops": {k: {"rows": v} for k, v in bad["drops"].items()}} + ) From 89eea0f541f688cc7bd30efc0969298dbc0e0056 Mon Sep 17 00:00:00 2001 From: r Date: Mon, 24 Aug 2026 14:48:48 +0100 Subject: [PATCH 2/2] Address DTrim99's review on #91 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. The runtime BLOCKED prose was stale. The precise proposal-vs-decided vocabulary reasoning only reached the module docstring, while the dict a caller actually reads still said "RF long-tail, held". Both blocked families now carry the real reason at runtime. 2. The DISTINCT ledger is exhaustive over what the prose claims, not a sample: all ten ifs:decile_N <-> ukmod:q pairs, plus both RF quintile and both RF decile edges. 3. ...but NOT against HBAI, and the prose is corrected rather than the ledger padded. HBAI registers no decile or quantile vocabulary at all (its subgroups are children/pensioners/working_age/total), so there is nothing on that side to be distinct FROM — asserting a pair against a value nobody registered would be the same overclaiming the ledger exists to prevent. A test pins that absence and why. 4. AVERAGE_HOUSEHOLD_INCOME_CHANGE now spells out what separates it from avg_change_after_tax_income_usd: this is a change in household NET income (post tax AND transfers, the concept IFS and RF publish), not the US tables' after-TAX income, so a connector maps an IFS/RF row onto the net-income quantity. Suite 287 passed, ruff format clean. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016HuXJFVme8HRbnke2Ey3Me --- scorecard_db/models.py | 12 +++++++++- scorecard_db/produce_campaign_uk.py | 26 +++++++++++++++++---- scorecard_db/uk_aliases.py | 36 ++++++++++++++++++++++++++--- tests/test_uk_thinktanks_ingest.py | 35 +++++++++++++++++++++++++++- 4 files changed, 99 insertions(+), 10 deletions(-) diff --git a/scorecard_db/models.py b/scorecard_db/models.py index 3d83eef..3eaa18e 100644 --- a/scorecard_db/models.py +++ b/scorecard_db/models.py @@ -101,7 +101,17 @@ class Metric(str, Enum): # avg_change_after_tax_income_usd # (that one is legacy-named; a new # currency rides unit_concept, not - # the metric name) + # the metric name). NOT the same + # QUANTITY, though: this one is a + # change in HOUSEHOLD NET income + # (post tax AND transfers, the + # concept IFS and RF publish), + # while avg_change_after_tax_income + # is the US distribution tables' + # after-TAX income. A connector + # must map an IFS/RF row onto the + # net-income quantity, not the + # after-tax one. # share_gaining / share_losing siblings of share_with_tax_cut, # kept as TWO metrics because "not # gaining" is not "losing" — a diff --git a/scorecard_db/produce_campaign_uk.py b/scorecard_db/produce_campaign_uk.py index dee2db6..06635bb 100644 --- a/scorecard_db/produce_campaign_uk.py +++ b/scorecard_db/produce_campaign_uk.py @@ -106,16 +106,32 @@ "metaless exhibit; targets the OBR policy-measures costings " "database — long-tail source held on the DB-storage decision" ), + # RF is NO LONGER a long-tail held source — #86 ingested 58 RF + # claims — so these two reasons are precise rather than categorical. + # This is the value a caller actually sees, so it carries the real + # reason and not just the module docstring. "two_child": ( "4 rows = 2 resolution_foundation + 1 ukmod " - "poverty_count_change + 1 metaless exhibit; RF is long-tail " - "(held), and the ukmod row targets a REFORM claim — the staged " - "UKMOD family is baseline statistics only, so a 2CL-reform " - "claim needs its own staging decision" + "poverty_count_change + 1 metaless exhibit. RF claims now exist " + "(#86), so: the RF poverty_count_change descriptor is " + "under-specified against the ingested vocabulary " + "(unit_population 'children' with no income_concept or " + "poverty_line); the RF reform_fiscal_cost row targets a claim " + "#86 deliberately DROPS (an HM Treasury scorecard figure RF " + "re-publishes), so there is no RF claim to attach to and there " + "should not be; and the ukmod row targets a REFORM claim while " + "the staged UKMOD family is baseline statistics only" ), "uprating_april2026": ( "4 rows = 3 resolution_foundation benefit_uprating_pct + 1 " - "metaless exhibit; RF long-tail, held" + "metaless exhibit. RF claims now exist (#86), but the archived " + "descriptors speak the HARVEST's proposal vocabulary (metric " + "'benefit_uprating_pct', benefit in verbatim prose such as 'UC " + "standard allowance, under-25s') while the ingested claims " + "speak the DECIDED vocabulary (metric benefit_uprating_rate, " + "benefit universal_credit_standard_allowance_under_25). " + "Resolving them means deriving that mapping the way the " + "reckoner family is derived, not loosening the match" ), } RESOLVED_FAMILIES = {"hmrc_reckoner_t2"} diff --git a/scorecard_db/uk_aliases.py b/scorecard_db/uk_aliases.py index 5330911..1e82024 100644 --- a/scorecard_db/uk_aliases.py +++ b/scorecard_db/uk_aliases.py @@ -58,10 +58,37 @@ def _alias(source: str, axis: str, source_value: str, canonical: str) -> None: # A decile of one publisher's distribution is not a decile — or a # quintile — of another's: different models, different income # concepts, different cut points. Never aliased. - ("ifs:decile_1", "ukmod:q1"), - ("ifs:decile_10", "ukmod:q5"), + # Recorded EXHAUSTIVELY rather than by example (review): the + # DISTINCT set is an audit ledger, so a half-populated one reads + # as if the unlisted pairs were undecided. Cross-source + # unification is impossible regardless — the registry key is + # (source, axis, value) and no alias crosses sources — but the + # ledger should say so for every pair the prose claims. + *( + (f"ifs:decile_{_i}", f"ukmod:q{_q}") + for _i, _q in ( + (1, 1), + (2, 1), + (3, 2), + (4, 2), + (5, 3), + (6, 3), + (7, 4), + (8, 4), + (9, 5), + (10, 5), + ) + ), + # NOTE: no ifs/rf-vs-HBAI edges appear here on purpose. HBAI + # registers no decile or quantile vocabulary at all (its + # subgroups are children/pensioners/working_age/total), so there + # is nothing on that side to be distinct FROM — and asserting a + # pair against a value nobody registered would be the same + # overclaiming this ledger exists to prevent. ("resolution_foundation:quintile_1", "ukmod:q1"), + ("resolution_foundation:quintile_5", "ukmod:q5"), ("resolution_foundation:decile_1", "ifs:decile_1"), + ("resolution_foundation:decile_10", "ifs:decile_10"), # And a coverage-restricted geography is not the UK. ("ifs:UK_excl_northern_ireland", "ifs:UK"), ("ifs:UK_excl_scotland", "ifs:UK"), @@ -321,7 +348,10 @@ def _alias(source: str, axis: str, source_value: str, canonical: str) -> None: # wording stays on the claim as a geography_note. # 2. A decile of one publisher's distribution is not a decile of # another's. The IFS and RF groups are registered per source and -# recorded DISTINCT from UKMOD's quintiles and HBAI's deciles. +# recorded DISTINCT from UKMOD's quintiles and from each other, +# exhaustively rather than by example. HBAI is deliberately absent +# from that ledger: it registers no decile or quantile vocabulary, so +# there is nothing on its side to be distinct from. _identity( "ifs", "geography", diff --git a/tests/test_uk_thinktanks_ingest.py b/tests/test_uk_thinktanks_ingest.py index df82015..a070197 100644 --- a/tests/test_uk_thinktanks_ingest.py +++ b/tests/test_uk_thinktanks_ingest.py @@ -198,10 +198,43 @@ def test_coverage_restricted_geographies_are_not_the_uk(staged): def test_publisher_deciles_are_never_unified(staged): + """The DISTINCT set is an audit ledger, so it is exhaustive over the + pairs the prose claims rather than carrying examples (review).""" from scorecard_db.uk_aliases import DISTINCT - assert ("ifs:decile_1", "ukmod:q1") in DISTINCT + for i, q in ( + (1, 1), + (2, 1), + (3, 2), + (4, 2), + (5, 3), + (6, 3), + (7, 4), + (8, 4), + (9, 5), + (10, 5), + ): + assert (f"ifs:decile_{i}", f"ukmod:q{q}") in DISTINCT + assert ("resolution_foundation:quintile_1", "ukmod:q1") in DISTINCT + assert ("resolution_foundation:quintile_5", "ukmod:q5") in DISTINCT assert ("resolution_foundation:decile_1", "ifs:decile_1") in DISTINCT + assert ("resolution_foundation:decile_10", "ifs:decile_10") in DISTINCT + + +def test_hbai_is_deliberately_absent_from_the_ledger(): + """HBAI registers no decile or quantile vocabulary — its subgroups + are children/pensioners/working_age/total — so there is nothing on + that side to be distinct FROM, and asserting a pair against a value + nobody registered would be the same overclaiming the ledger exists + to prevent.""" + from scorecard_db.uk_aliases import DISTINCT, known + + assert not any("hbai" in a + b for a, b in DISTINCT) + assert not known("dwp_hbai", "income_group") + assert not known("dwp_hbai", "quantile") + assert known("dwp_hbai", "subgroup") == frozenset( + {"children", "pensioners", "working_age", "total"} + ) def test_prose_identities_are_canonicalised(staged):