Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions app/public/data/lanes.json
Original file line number Diff line number Diff line change
Expand Up @@ -357,6 +357,17 @@
"running": false,
"updated": "2026-08-21",
"note": "5 model claims, 7 Chronicle facts (6 statistical + 1 non-simulated uprated EU-SILC survey input), 6 derived ratios dispositioned; 2 concept-mismatch attachments"
},
{
"id": "dwp-pension-participation",
"source": "DWP",
"area": "workplace pension participation",
"mode": 1,
"country": "UK",
"stage": "ingested",
"running": false,
"updated": "2026-08-25",
"note": "1377 claims \u2014 the first external validation of any pensions quantity in the UK scorecard (participation among auto-enrolment ELIGIBLE employees, GB, ASHE-derived)"
}
]
}
24,788 changes: 24,788 additions & 0 deletions data/externals/dwp-pension-participation.json

Large diffs are not rendered by default.

11 changes: 11 additions & 0 deletions data/lanes.json
Original file line number Diff line number Diff line change
Expand Up @@ -357,6 +357,17 @@
"running": false,
"updated": "2026-08-21",
"note": "5 model claims, 7 Chronicle facts (6 statistical + 1 non-simulated uprated EU-SILC survey input), 6 derived ratios dispositioned; 2 concept-mismatch attachments"
},
{
"id": "dwp-pension-participation",
"source": "DWP",
"area": "workplace pension participation",
"mode": 1,
"country": "UK",
"stage": "ingested",
"running": false,
"updated": "2026-08-25",
"note": "1377 claims \u2014 the first external validation of any pensions quantity in the UK scorecard (participation among auto-enrolment ELIGIBLE employees, GB, ASHE-derived)"
}
]
}
5 changes: 5 additions & 0 deletions scorecard_db/build_db.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
campaign_us staged day-1/day-2 campaign results (claim matching)
uk_externals five UK primary-source families + Chronicle staging
uk_deductions FRR family
dwp_pensions workplace pension participation (ASHE, GB)
produce_uk + campaign_uk archive-resolved UK reckoner attaches
be_jrc JRC EUROMOD-BE model claims + honest demo attachments;
final so its 2026-08-21 lane update cannot be regressed by
Expand Down Expand Up @@ -47,6 +48,7 @@
ingest_solo,
ingest_uk_deductions,
ingest_uk_externals,
ingest_dwp_pensions,
ingest_urban,
produce_campaign_uk,
)
Expand Down Expand Up @@ -94,6 +96,9 @@ def build(db_path: Path) -> dict:
("campaign_us", lambda: ingest_campaign.ingest(db_path)),
("uk_externals", lambda: ingest_uk_externals.ingest(db_path)),
("uk_deductions", lambda: ingest_uk_deductions.ingest(db_path)),
# DWP workplace pension participation (#98): the first external
# validation of any pensions quantity in the UK scorecard.
("dwp_pensions", lambda: ingest_dwp_pensions.ingest(db_path)),
("produce_uk", lambda: produce_campaign_uk.produce(db_path)),
(
"campaign_uk",
Expand Down
198 changes: 198 additions & 0 deletions scorecard_db/ingest_dwp_pensions.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,198 @@
"""Ingest DWP workplace pension participation (#98).

PolicyEngine-UK models pensions — contributions, their relief, the age
limit, the salary-sacrifice interaction — and until now not one of the
15,858 UK external claims said anything about pensions. This is the
external side: 1,377 participation rates by earnings band, age band and
region, split public/private/overall, 2009-2025.

Two things ride on every claim because leaving them in a methodology
note would make the rows uninterpretable:

denominator The rate is a share of employees ELIGIBLE for automatic
enrolment — an earnings-trigger and age-range definition
that has moved over the series. A participation rate
whose denominator is unstated says nothing.
survey_axis DWP derives these from ONS ASHE, an employer survey of
jobs; the certified PE-UK world is FRS-based. A
PE-vs-DWP gap is a survey-population difference before
it is an engine question — the same axis the LPC lane
(#88) carries.

Geography is GB, not UK: ASHE excludes Northern Ireland, and the
registry keeps the two apart for the same reason #91 keeps IFS's
coverage-restricted analyses apart.

PYTHONPATH=. python -m scorecard_db.ingest_dwp_pensions data/scorecard.db
"""

from __future__ import annotations

import json
from pathlib import Path

from .db import LANE_SQL, SCORES_SQL, ScorecardDB
from .harvest import REPO, finish, require_fields
from .models import ExternalScore, Metric, TimeBasis, UnitConcept
from .relationships import uk_relationship
from .uk_aliases import canon

EXTERNALS = REPO / "data" / "externals"

ADAPTER_SOURCE = "dwp-pension-participation"
SOURCE = "dwp_pensions"
LANE_ID = "dwp-pension-participation"
LANE_UPDATED = "2026-08-25"
FEED_UPDATED = "2026-08-19"
LANE_FEED_META = {
"source": "DWP",
"area": "workplace pension participation",
"mode": 1,
"country": "UK",
}

PUBLICATION = {
"title": "DWP Workplace pension participation and savings trends: 2009 to 2025",
"url": (
"https://www.gov.uk/government/statistics/"
"workplace-pension-participation-and-savings-trends-2009-to-2025"
),
"date": "2026-07-30",
"country": "UK",
}

_KNOWN_FIELDS = frozenset(
{
"axis",
"country",
"denominator",
"edition",
"geography",
"metric",
"period",
"program",
"sector",
"source",
"source_column",
"status",
"subgroup",
"survey_axis",
"unit_concept",
"value",
}
)

_METRICS = {
"participation_rate": (Metric.PARTICIPATION_RATE, UnitConcept.SHARE, "share")
}
# the adapter's axis name -> the uk_aliases axis its values are closed on
_AXES = {"earnings_band": "earnings_band", "age_band": "age_band", "region": "region"}


def _load() -> list[dict]:
path = EXTERNALS / f"{ADAPTER_SOURCE}.json"
if not path.exists():
raise FileNotFoundError(
f"{path} missing — run sources/dwp-pension-participation/adapter.py first"
)
rows = json.loads(path.read_text())
for row in rows:
require_fields(row, _KNOWN_FIELDS, SOURCE)
if row["source"] != ADAPTER_SOURCE:
raise ValueError(f"{SOURCE}: row source {row['source']!r} is wrong")
if row["country"] != "UK":
raise ValueError(f"{SOURCE}: row country {row['country']!r} is not 'UK'")
return rows


def stage() -> tuple[list[ExternalScore], dict]:
scores: list[ExternalScore] = []
counts: dict[str, int] = {}
for row in _load():
metric, unit, value_kind = _METRICS[row["metric"]]
canon(SOURCE, "unit", row["unit_concept"])
axis = row["axis"]
if axis not in _AXES:
raise ValueError(f"{SOURCE}: unregistered axis {axis!r}")
value = float(row["value"])
if not 0.0 <= value <= 1.0:
raise ValueError(
f"{SOURCE}: participation rate {value} is not a share — the "
"publication gives fractions, and a percent slipping in here "
"would be a hundredfold error nobody could see downstream"
)
cond = {
"country": "UK",
# GB, not UK: ASHE excludes Northern Ireland
"geography": canon(SOURCE, "geography", row["geography"]),
"program": canon(SOURCE, "program", row["program"]),
"sector": canon(SOURCE, "sector", row["sector"]),
# which axis this row cuts on, so an earnings band and an age
# band never collide on one subgroup key
"axis": axis,
"subgroup": canon(SOURCE, _AXES[axis], row["subgroup"]),
# the two facts that make the row interpretable at all
"denominator": row["denominator"],
"survey_axis": row["survey_axis"],
"edition": row["edition"],
}
scores.append(
ExternalScore(
source=SOURCE,
metric=metric,
unit_concept=unit,
period=int(row["period"]),
# ASHE April reference year, carried as a calendar year
time_basis=TimeBasis.ANNUAL,
value=value,
conditions=cond,
calibration_relationship=uk_relationship(SOURCE, metric)[0],
source_model="dwp_ashe",
source_column=row["source_column"],
publication=PUBLICATION,
value_kind=value_kind,
status=row["status"],
)
)
counts[axis] = counts.get(axis, 0) + 1
return finish(scores, SOURCE), counts


_EXPECTED = {"earnings_band": 357, "age_band": 459, "region": 561}


def ingest(db_path: Path) -> dict:
scores, counts = stage()
if counts != _EXPECTED:
raise ValueError(f"claim accounting drifted: {counts} != {_EXPECTED}")
db = ScorecardDB(db_path)
rows = [ScorecardDB.score_row(s) for s in scores]

from .baselines import register_baselines_txn
from .ingest_harvest import sync_lane_feed

with db.conn:
db.conn.execute("DELETE FROM external_scores WHERE source = ?", (SOURCE,))
db.conn.executemany(SCORES_SQL, rows)
register_baselines_txn(db)
detail = (
f"{len(rows)} claims — the first external validation of any "
"pensions quantity in the UK scorecard (participation among "
"auto-enrolment ELIGIBLE employees, GB, ASHE-derived)"
)
db.conn.execute(LANE_SQL, (LANE_ID, "ingested", detail, LANE_UPDATED))
sync_lane_feed(
db,
REPO / "data" / "lanes.json",
FEED_UPDATED,
lanes={LANE_ID: LANE_FEED_META},
)
db.close()
return {"claims": len(rows), "by_axis": dict(sorted(counts.items()))}


if __name__ == "__main__":
import sys

out = Path(sys.argv[1] if len(sys.argv) > 1 else "data/scorecard.db")
print(json.dumps(ingest(out), indent=1))
16 changes: 16 additions & 0 deletions scorecard_db/relationships.py
Original file line number Diff line number Diff line change
Expand Up @@ -222,6 +222,20 @@ def effective_relationship(program, metric):
"nothing in pe-uk-data consumes them.",
)

# DWP workplace pension participation (#98). ASHE-derived, like the LPC
# lane, so the survey difference is the first divergence axis.
_DWP_PENSIONS_HELD = (
CR.HELD_OUT,
"DWP's workplace pension participation estimates are derived from ONS "
"ASHE, an employer survey of jobs, where the certified "
"policyengine-uk world is FRS-based; no pe-uk-data target and no "
"policyengine-uk parameter is fitted to a participation rate "
"(consumption surfaces read 2026-08-25 at the certified pins). Note "
"the engine models pension contributions and their relief but has no "
"pension commencement lump sum at all, so the pensions question this "
"scorecard can ask is narrower than the pensions system.",
)

_UKMOD_HELD = (
CR.HELD_OUT,
"UKMOD is a peer microsimulation, not a calibration source; no PE UK "
Expand Down Expand Up @@ -276,6 +290,8 @@ def uk_relationship(source, metric, program=None, kind=None):
if program in OBR_CONSUMED_WELFARE_PROGRAMS:
return _OBR_CONSUMED
return _OBR_UNCONSUMED
if source == "dwp_pensions":
return _DWP_PENSIONS_HELD
if source == "ukmod":
return _UKMOD_HELD
if source == "hm_treasury":
Expand Down
57 changes: 57 additions & 0 deletions scorecard_db/uk_aliases.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,21 @@ def _alias(source: str, axis: str, source_value: str, canonical: str) -> None:
("dwp_takeup:housing_benefit_pensioners", "obr:housing_benefit_on_jsa"),
("dwp_takeup:housing_benefit_pensioners", "ukmod:housing_benefit"),
("dwp_takeup:benefit_units", "ukmod:families"),
# DWP's auto-enrolment ELIGIBILITY earnings bands are defined by
# the enrolment trigger, not by an income distribution, so they
# are not UKMOD's quantiles nor anyone's deciles.
("dwp_pensions:gbp_10k_20k", "ukmod:q1"),
("dwp_pensions:gbp_70k_plus", "ukmod:q5"),
# DEFERRED, deliberately: DWP's age bands are also not LPC's
# minimum-wage age bands (one is an auto-enrolment range, the
# other a wage-rate category), but the `lpc` vocabulary is
# registered on the #93 branch and not here. Asserting a pair
# against a source nobody has registered is the same overclaiming
# the review caught on #92 — the ledger would read as if the
# distinction had been checked when there is nothing on the other
# side to check against. The pairs go in once #93 lands; until
# then the distinction lives in prose, in
# sources/dwp-pension-participation/source.json.
}
)

Expand Down Expand Up @@ -299,6 +314,48 @@ def _alias(source: str, axis: str, source_value: str, canonical: str) -> None:
_identity(_src, "unit", ["gbp", "households"])


# --- DWP workplace pension participation (#98) -------------------------------
# Geography is GB, not UK: ASHE excludes Northern Ireland, and the two
# are kept apart for the same reason #91 keeps IFS's coverage-restricted
# analyses apart from the UK. The earnings and age bands are the
# publication's own and are closed per source — DWP's auto-enrolment
# eligibility bands are not UKMOD's income groups and not LPC's age
# bands, even where the numbers coincide.
_identity("dwp_pensions", "geography", ["GB"])
_identity("dwp_pensions", "program", ["workplace_pension"])
_identity("dwp_pensions", "sector", ["public", "private", "all"])
_identity(
"dwp_pensions",
"earnings_band",
[
"gbp_10k_20k",
"gbp_20k_30k",
"gbp_30k_40k",
"gbp_40k_50k",
"gbp_50k_60k",
"gbp_60k_70k",
"gbp_70k_plus",
],
)
_identity(
"dwp_pensions",
"age_band",
[
"age_22_25",
"age_26_30",
"age_31_35",
"age_36_40",
"age_41_45",
"age_46_50",
"age_51_55",
"age_56_60",
"age_61_65",
],
)
_identity("dwp_pensions", "region", _UK_REGIONS)
_identity("dwp_pensions", "unit", ["share"])


def canon(source: str, axis: str, value: str) -> str:
"""Canonical value for (source, axis, source_value); unknown raises."""
try:
Expand Down
Loading
Loading