From 80827f62ce1e2eb739959f91d5f85cc50dc4f719 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Fri, 21 Aug 2026 19:47:10 +0200 Subject: [PATCH 1/5] Add the E8 UK spine stages: CGT structure, salary sacrifice, student loans Five manifest stages between the SPI channel and the certified pair (roster 21 -> 26, both manifest projections): cgt_incidence_clone (clone_records equal mass split w/2 + Advani-Summers prior amounts on the oldest adult, identity-keyed), cgt_band_donors (270 = 30 x 9 HMRC Table 2.1a bands at band-exact never-zero weights, seed 1), hmrc_cgt_gains_spine (the merged #560 Table 3 redraw reused unchanged on the spine path; the certified-line wrapper and national-driver injection stay untouched until seed 42, + conversion to the 5.4m staging target, identity-keyed seed 2024), and student_loans (cohort rules at calibration year 2025 + PLAN_5- first top-up to SLC liable stocks, identity-keyed seed 42; PLAN_4 never imputed, documented). Thirteen operation kinds registered; five committed cited resources (Table 2.1a with publisher-ODS sha pin, A&S CAGE WP 465 distribution with the row-30 anomaly documented, HMRC 7.7m salary-sacrifice anchor with the 0.70 staging ratio, SLC Table 6a liable stocks flagged chronicle_candidate, donor support bounds recorded as inapplicable-at-terminal for review); per-stage reviewed-constants drift asserts and executed-effect receipts per the #730/#684 two-arm rule, now documented at the spi_spine reviewed-constants block; StagePlan gains explicit declared rewrites; family coverage closes the capital_gains and student_loan_plan producers; student_loan_plan enum-domain gate (BRMA generalization); export surface gains household_is_cgt_band_donor; spine driver wires --cgt-ods and harvests every new seed into the build sidecar. Closes #684. Co-Authored-By: Claude Fable 5 --- ...84-uk-e8-cgt-salsac-student-loans.added.md | 1 + .../src/microcosm/build/country_spec.py | 13 +- .../src/microcosm/build/plan.py | 18 +- .../src/microcosm/build/source_manifest.py | 13 + .../spec_engine/schema/sources.schema.json | 126 +++- ...ni_summers_capital_gains_distribution.json | 76 +++ .../uk/cgt_band_donor_support_bounds.json | 25 + .../microcosm/build/uk/country_package.json | 25 + .../src/microcosm/build/uk/gates.json | 13 + .../build/uk/hmrc_cgt_size_bands.json | 111 +++ .../uk/release_input_coverage_manifest.json | 133 +++- .../build/uk/salary_sacrifice_anchor.json | 26 + .../microcosm/build/uk/slc_liable_stocks.json | 22 + .../src/microcosm/build/uk/source_stages.json | 345 ++++++++++ .../src/microcosm/build/uk/spec/sources.yaml | 301 +++++++++ .../build/uk_runtime/battery_bindings.py | 29 +- .../build/uk_runtime/cgt_imputation.py | 67 ++ .../build/uk_runtime/cgt_structure.py | 633 ++++++++++++++++++ .../build/uk_runtime/national_build.py | 9 +- .../uk_runtime/release_input_coverage.py | 43 +- .../build/uk_runtime/salary_sacrifice.py | 340 ++++++++++ .../build/uk_runtime/source_runtime.py | 10 + .../microcosm/build/uk_runtime/spi_spine.py | 5 +- .../build/uk_runtime/student_loans.py | 369 ++++++++++ .../build/uk_runtime/terminal_gates.py | 1 + .../tests/test_country_spec.py | 15 +- packages/microcosm-build/tests/test_plan.py | 19 + .../tests/test_spec_engine_country_bundles.py | 2 +- .../tests/test_uk_battery_bindings.py | 6 +- .../tests/test_uk_cgt_source_manifest.py | 18 + .../tests/test_uk_cgt_structure.py | 278 ++++++++ .../tests/test_uk_frs_spine.py | 27 +- .../tests/test_uk_national_build.py | 1 + .../tests/test_uk_release_input_coverage.py | 5 + .../tests/test_uk_salary_sacrifice.py | 224 +++++++ .../tests/test_uk_source_runtime.py | 10 + .../tests/test_uk_source_stages.py | 96 ++- .../tests/test_uk_student_loans.py | 238 +++++++ .../tests/test_uk_take_up_gate.py | 24 + .../src/microcosm/data/contract.py | 2 + .../microcosm-data/tests/test_contract.py | 5 + tools/build_uk_frs_spine.py | 60 ++ ...uild_uk_release_input_coverage_manifest.py | 132 +++- 43 files changed, 3864 insertions(+), 52 deletions(-) create mode 100644 changelog.d/684-uk-e8-cgt-salsac-student-loans.added.md create mode 100644 packages/microcosm-build/src/microcosm/build/uk/advani_summers_capital_gains_distribution.json create mode 100644 packages/microcosm-build/src/microcosm/build/uk/cgt_band_donor_support_bounds.json create mode 100644 packages/microcosm-build/src/microcosm/build/uk/hmrc_cgt_size_bands.json create mode 100644 packages/microcosm-build/src/microcosm/build/uk/salary_sacrifice_anchor.json create mode 100644 packages/microcosm-build/src/microcosm/build/uk/slc_liable_stocks.json create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_structure.py create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/salary_sacrifice.py create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/student_loans.py create mode 100644 packages/microcosm-build/tests/test_uk_cgt_structure.py create mode 100644 packages/microcosm-build/tests/test_uk_salary_sacrifice.py create mode 100644 packages/microcosm-build/tests/test_uk_student_loans.py diff --git a/changelog.d/684-uk-e8-cgt-salsac-student-loans.added.md b/changelog.d/684-uk-e8-cgt-salsac-student-loans.added.md new file mode 100644 index 000000000..e9b9c9e96 --- /dev/null +++ b/changelog.d/684-uk-e8-cgt-salsac-student-loans.added.md @@ -0,0 +1 @@ +Add UK spine stages for capital-gains structure and redraws, salary-sacrifice support, and student-loan plan cohorts. diff --git a/packages/microcosm-build/src/microcosm/build/country_spec.py b/packages/microcosm-build/src/microcosm/build/country_spec.py index 495f4c740..e3fcde952 100644 --- a/packages/microcosm-build/src/microcosm/build/country_spec.py +++ b/packages/microcosm-build/src/microcosm/build/country_spec.py @@ -1644,11 +1644,12 @@ def country_stage_plan( f"Country {spec.country!r} declares no source_stages.json; there " "is no stage plan to assemble." ) - declared: list[tuple[str, DonorSpec | None, tuple[str, ...]]] = [ + declared: list[tuple[str, DonorSpec | None, tuple[str, ...], tuple[str, ...]]] = [ ( stage.stage, DonorSpec(survey=stage.survey, source=stage.source, notes=stage.notes), stage.outputs, + stage.rewrites, ) for stage in spec.sources.stages ] @@ -1663,9 +1664,10 @@ def country_stage_plan( notes=spine.assignment_source.notes, ), (spine.code_column,), + (), ) ) - declared_names = [name for name, _, _ in declared] + declared_names = [name for name, _, _, _ in declared] selected_names: tuple[str, ...] if stage_names is None: selected_names = tuple(declared_names) @@ -1702,8 +1704,8 @@ def country_stage_plan( f"{declared_names}." ) selected = [ - (name, donor, outputs) - for name, donor, outputs in declared + (name, donor, outputs, rewrites) + for name, donor, outputs, rewrites in declared if name in set(selected_names) ] return StagePlan( @@ -1711,7 +1713,8 @@ def country_stage_plan( name=name, transform=implementations[name], produces=outputs, + rewrites=rewrites, donor=donor, ) - for name, donor, outputs in selected + for name, donor, outputs, rewrites in selected ) diff --git a/packages/microcosm-build/src/microcosm/build/plan.py b/packages/microcosm-build/src/microcosm/build/plan.py index 8d8274e63..969ee3e88 100644 --- a/packages/microcosm-build/src/microcosm/build/plan.py +++ b/packages/microcosm-build/src/microcosm/build/plan.py @@ -71,6 +71,8 @@ class Stage: happens inside and any failure aborts the build. produces: Columns the stage must add (validated after the transform). Empty is allowed for assert/report-only stages. + rewrites: Declared outputs that may have an earlier canonical producer + and are intentionally replaced by this stage. consumes: Columns that must exist (on any entity) before the stage runs. donor: The donor survey, when the stage imputes. ``None`` for @@ -80,6 +82,7 @@ class Stage: name: str transform: Callable[[Frame], Frame] produces: tuple[str, ...] = () + rewrites: tuple[str, ...] = () consumes: tuple[str, ...] = () donor: DonorSpec | None = None @@ -143,12 +146,15 @@ def __init__(self, stages: Iterable[Stage]) -> None: names.add(stage.name) for column in stage.produces: if column in producers: - raise ValueError( - f"Column {column!r} is declared by two stages " - f"({producers[column]!r} and {stage.name!r}); every " - "column has one canonical producer." - ) - producers[column] = stage.name + if column not in stage.rewrites: + raise ValueError( + f"Column {column!r} is declared by two stages " + f"({producers[column]!r} and {stage.name!r}); every " + "column has one canonical producer unless a later " + "stage explicitly declares it as a rewrite." + ) + else: + producers[column] = stage.name self._stages = materialized @property diff --git a/packages/microcosm-build/src/microcosm/build/source_manifest.py b/packages/microcosm-build/src/microcosm/build/source_manifest.py index c4dafc795..fca25b48d 100644 --- a/packages/microcosm-build/src/microcosm/build/source_manifest.py +++ b/packages/microcosm-build/src/microcosm/build/source_manifest.py @@ -47,6 +47,7 @@ "assign_binary_from_rate", "assign_binary_with_anchored_residual", "assign_clipped_normal", + "assign_student_loan_plan_cohorts", "assign_uniform_draw", "aggregate_person_to_benunit", "allocate_per_capita_from_cell_table", @@ -58,7 +59,10 @@ "bridge_donor_column_via_qrf", "calibrate_binary_assignment", "calibrate_binary_assignment_joint_targets", + "classify_cgt_band_facts_with_reviewed_fence", "classify_hmrc_income_facts_with_reviewed_fences", + "clone_records", + "convert_donors_to_target_stock", "convert_interest_to_structural_mortgage_inputs", "compute_ratio", "declare_income_reference_offset", @@ -88,6 +92,7 @@ "derive_weeks_unemployed", "derive_wic_claim", "disaggregate_aggregate_records", + "draw_capital_gains_prior_from_banded_quantiles", "fit_labor_market_models", "fit_tip_income_model", "fit_weighted_acs_rent_qrf", @@ -119,22 +124,30 @@ "map_coded_amounts", "materialize_hmrc_income_bands_fail_closed", "materialize_rules_engine_predictors", + "rank_preserving_allocation", "read_table", "read_tables", "read_acs_rent_donor", "redraw_columns_from_fitted_qrf", + "record_mass_conservation_receipt", "replace_zero_weight_spi_support", "retain_adjudicated_frs_hmrc_leaves", "sample_categorical_from_count_table", "replace_sentinels", "split_component_by_share", + "stack_band_donor_households", "stack_zero_weight_donors", "strict_read_private_table", "support_clip", + "sub_aea_remainder", + "taxable_income_proxy", + "top_up_to_stock", "uprate", "uprate_to_regional_reference", "verify_certified_candidate", + "verify_pinned_cgt_ods", "verify_pinned_hmrc_source_pair", + "within_band_draws", "zero_when_false", } ) diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json index 1c7d9d1d5..3a9ae12d5 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json @@ -5872,7 +5872,13 @@ "targets": {"type": "array", "items": {"type": "string"}}, "weights": {"type": "string"}, "n_estimators": {"type": "integer", "minimum": 1}, - "seed": {"type": "integer"} + "seed": {"type": "integer"}, + "training_population": {"type": "string"}, + "target_population": {"type": "string"}, + "weight_mapping": {"type": "string"}, + "clamp_minimum": {"type": "number"}, + "preserve_asked_rows": {"type": "boolean"}, + "cache": {"type": "boolean"} } }, { @@ -6554,6 +6560,121 @@ } } }, + { + "type": "object", + "additionalProperties": false, + "required": ["kind", "entity", "copies", "flag_column", "mass_split", "weight_kind_out", "conservation", "id_remapping", "declared_factor", "reason"], + "properties": { + "kind": {"const": "clone_records"}, + "entity": {"type": "string"}, + "copies": {"type": "integer", "minimum": 2}, + "flag_column": {"type": "string"}, + "original_flag": {"type": "boolean"}, + "clone_flag": {"type": "boolean"}, + "mass_split": {"type": "number"}, + "weight_kind_out": {"type": "string"}, + "conservation": {"type": "string"}, + "id_remapping": {"type": "string"}, + "declared_factor": {"type": "number"}, + "reason": {"type": "string"} + } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["kind", "resource", "income_proxy_components", "allowance_subtraction", "carrier", "adult_minimum_age", "quantile_points", "spline_degree", "extrapolation", "keep_negative_draws", "seed", "salt"], + "properties": { + "kind": {"const": "draw_capital_gains_prior_from_banded_quantiles"}, + "resource": {"type": "string"}, + "income_proxy_components": {"type": "array", "items": {"type": "string"}}, + "allowance_subtraction": {"type": "boolean"}, + "carrier": {"type": "string"}, + "adult_minimum_age": {"type": "integer"}, + "quantile_points": {"type": "array", "items": {"type": "number"}}, + "spline_degree": {"type": "integer"}, + "extrapolation": {"type": "string"}, + "keep_negative_draws": {"type": "boolean"}, + "seed": {"type": "integer"}, + "salt": {"type": "string"} + } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["kind", "size_band_resource", "incidence_resource", "minimum_band_lower", "donors_per_band", "expected_band_count", "expected_donor_count", "candidate_order", "draw", "propensity", "seed", "flag_column", "carrier", "initial_weight", "never_zero_weight", "weight_kind_out", "reason"], + "properties": { + "kind": {"const": "stack_band_donor_households"}, + "size_band_resource": {"type": "string"}, + "incidence_resource": {"type": "string"}, + "minimum_band_lower": {"type": "number"}, + "donors_per_band": {"type": "integer", "minimum": 1}, + "expected_band_count": {"type": "integer", "minimum": 1}, + "expected_donor_count": {"type": "integer", "minimum": 1}, + "candidate_order": {"type": "string"}, + "draw": {"type": "string"}, + "propensity": {"type": "string"}, + "seed": {"type": "integer"}, + "flag_column": {"type": "string"}, + "carrier": {"type": "string"}, + "initial_weight": {"type": "string"}, + "never_zero_weight": {"type": "boolean"}, + "weight_kind_out": {"type": "string"}, + "reason": {"type": "string"} + } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["kind", "resource", "target", "donor_pool", "rate_cap", "move", "seed", "salt", "receipt"], + "properties": { + "kind": {"const": "convert_donors_to_target_stock"}, + "resource": {"type": "string"}, + "target": {"type": "number"}, + "donor_pool": {"type": "string"}, + "rate_cap": {"type": "number"}, + "move": {"type": "string"}, + "seed": {"type": "integer"}, + "salt": {"type": "string"}, + "receipt": {"type": "string"} + } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["kind", "year_rule", "start_year_formula", "reported_repayment_test", "reported_country_gate", "plan_1_before", "plan_5_from", "enum_domain", "plan_4_imputation"], + "properties": { + "kind": {"const": "assign_student_loan_plan_cohorts"}, + "year_rule": {"type": "string"}, + "start_year_formula": {"type": "string"}, + "reported_repayment_test": {"type": "string"}, + "reported_country_gate": {"type": "boolean"}, + "plan_1_before": {"type": "integer"}, + "plan_5_from": {"type": "integer"}, + "enum_domain": {"type": "array", "items": {"type": "string"}}, + "plan_4_imputation": {"type": "boolean"} + } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["kind", "plan", "priority", "resource", "stock_series", "year_rule", "age_min", "age_max", "cohort_start_min", "eligible_region_exclusions", "highest_education", "seed", "salt"], + "properties": { + "kind": {"const": "top_up_to_stock"}, + "plan": {"type": "string"}, + "priority": {"type": "integer"}, + "resource": {"type": "string"}, + "stock_series": {"type": "string"}, + "year_rule": {"type": "string"}, + "age_min": {"type": "integer"}, + "age_max": {"type": "integer"}, + "cohort_start_min": {"type": "integer"}, + "cohort_start_max_exclusive": {"type": "integer"}, + "eligible_region_exclusions": {"type": "array", "items": {"type": "string"}}, + "highest_education": {"type": "string"}, + "seed": {"type": "integer"}, + "salt": {"type": "string"} + } + }, { "type": "object", "additionalProperties": false, @@ -6625,6 +6746,9 @@ }, "format": { "type": "string" + }, + "runtime_sha256_required": { + "type": "boolean" } } } diff --git a/packages/microcosm-build/src/microcosm/build/uk/advani_summers_capital_gains_distribution.json b/packages/microcosm-build/src/microcosm/build/uk/advani_summers_capital_gains_distribution.json new file mode 100644 index 000000000..35dd075ff --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk/advani_summers_capital_gains_distribution.json @@ -0,0 +1,76 @@ +{ + "version": 1, + "country": "uk", + "source": { + "citation": "Advani, Arun and Andy Summers (May 2020), Capital Gains and UK Inequality, CAGE Working Paper 465, University of Warwick.", + "url": "https://warwick.ac.uk/fac/soc/economics/research/centres/cage/manage/publications/wp465.2020.pdf", + "incumbent_csv_sha256": "7cb73f8c0a35aeb07c7f13bb0476fb48e9d3bd9b31735f3229554289906297a0", + "columns": ["percentile", "minimum_total_income", "percent_with_gains", "mean_gains_given_gains", "p05", "p10", "p25", "p50", "p75", "p90", "p95"] + }, + "known_anomalies": [ + "Percentile 69 has p95=15190 below p90=78200 in the incumbent vintage, likely a dropped digit. It is replicated unchanged because the vintage source table was not available to verify a correction; the later CGT redraw limits its footprint." + ], + "rows": [ + {"percentile":"<40","minimum_total_income":0,"percent_with_gains":0.0031,"mean_gains_given_gains":45600,"p05":-16400,"p10":-4400,"p25":3800,"p50":14400,"p75":38400,"p90":92500,"p95":165000}, + {"percentile":40,"minimum_total_income":10000,"percent_with_gains":0.006,"mean_gains_given_gains":50300,"p05":-12600,"p10":-3100,"p25":4000,"p50":13500,"p75":37100,"p90":86900,"p95":156800}, + {"percentile":41,"minimum_total_income":10300,"percent_with_gains":0.0065,"mean_gains_given_gains":49000,"p05":-17900,"p10":-6600,"p25":3200,"p50":13300,"p75":36700,"p90":87700,"p95":142500}, + {"percentile":42,"minimum_total_income":10600,"percent_with_gains":0.0062,"mean_gains_given_gains":41100,"p05":-16200,"p10":-5500,"p25":3300,"p50":13000,"p75":31900,"p90":74000,"p95":125700}, + {"percentile":43,"minimum_total_income":11000,"percent_with_gains":0.0062,"mean_gains_given_gains":42600,"p05":-15100,"p10":-4700,"p25":3700,"p50":13200,"p75":34400,"p90":79500,"p95":143000}, + {"percentile":44,"minimum_total_income":11400,"percent_with_gains":0.0061,"mean_gains_given_gains":41900,"p05":-15900,"p10":-4100,"p25":2800,"p50":12800,"p75":34500,"p90":79400,"p95":144500}, + {"percentile":45,"minimum_total_income":11800,"percent_with_gains":0.0062,"mean_gains_given_gains":41800,"p05":-14500,"p10":-4000,"p25":3500,"p50":12500,"p75":34700,"p90":86900,"p95":147300}, + {"percentile":46,"minimum_total_income":12100,"percent_with_gains":0.0065,"mean_gains_given_gains":47700,"p05":-13800,"p10":-3800,"p25":3300,"p50":12500,"p75":35500,"p90":88400,"p95":158800}, + {"percentile":47,"minimum_total_income":12500,"percent_with_gains":0.0062,"mean_gains_given_gains":52400,"p05":-13500,"p10":-3800,"p25":3100,"p50":13200,"p75":35400,"p90":84100,"p95":158400}, + {"percentile":48,"minimum_total_income":12900,"percent_with_gains":0.0061,"mean_gains_given_gains":43600,"p05":-14200,"p10":-4000,"p25":3800,"p50":12800,"p75":33800,"p90":82200,"p95":137800}, + {"percentile":49,"minimum_total_income":13300,"percent_with_gains":0.0063,"mean_gains_given_gains":38600,"p05":-13700,"p10":-3700,"p25":3200,"p50":12800,"p75":34200,"p90":77000,"p95":129000}, + {"percentile":50,"minimum_total_income":13700,"percent_with_gains":0.0062,"mean_gains_given_gains":48200,"p05":-14100,"p10":-4300,"p25":2900,"p50":12300,"p75":33300,"p90":79400,"p95":150200}, + {"percentile":51,"minimum_total_income":14100,"percent_with_gains":0.006,"mean_gains_given_gains":40300,"p05":-14700,"p10":-4300,"p25":3100,"p50":12300,"p75":32900,"p90":79800,"p95":137700}, + {"percentile":52,"minimum_total_income":14500,"percent_with_gains":0.0063,"mean_gains_given_gains":46700,"p05":-10700,"p10":-3300,"p25":3600,"p50":12900,"p75":33500,"p90":74400,"p95":145500}, + {"percentile":53,"minimum_total_income":14900,"percent_with_gains":0.0062,"mean_gains_given_gains":49900,"p05":-15300,"p10":-5000,"p25":2400,"p50":11700,"p75":32800,"p90":78300,"p95":141700}, + {"percentile":54,"minimum_total_income":15300,"percent_with_gains":0.0062,"mean_gains_given_gains":38700,"p05":-15800,"p10":-4800,"p25":2800,"p50":11900,"p75":32400,"p90":80000,"p95":153800}, + {"percentile":55,"minimum_total_income":15700,"percent_with_gains":0.0063,"mean_gains_given_gains":40300,"p05":-12100,"p10":-4200,"p25":3000,"p50":12500,"p75":33000,"p90":79100,"p95":137000}, + {"percentile":56,"minimum_total_income":16100,"percent_with_gains":0.0063,"mean_gains_given_gains":42600,"p05":-13400,"p10":-3900,"p25":3400,"p50":12300,"p75":33800,"p90":82200,"p95":152800}, + {"percentile":57,"minimum_total_income":16600,"percent_with_gains":0.0062,"mean_gains_given_gains":50800,"p05":-12700,"p10":-3500,"p25":3700,"p50":13300,"p75":34900,"p90":88900,"p95":157600}, + {"percentile":58,"minimum_total_income":17000,"percent_with_gains":0.0065,"mean_gains_given_gains":49100,"p05":-16100,"p10":-4700,"p25":3100,"p50":11900,"p75":31900,"p90":80900,"p95":146800}, + {"percentile":59,"minimum_total_income":17400,"percent_with_gains":0.0064,"mean_gains_given_gains":42100,"p05":-15700,"p10":-4300,"p25":3100,"p50":11900,"p75":32500,"p90":81700,"p95":152100}, + {"percentile":60,"minimum_total_income":17900,"percent_with_gains":0.0065,"mean_gains_given_gains":50800,"p05":-13100,"p10":-4300,"p25":3100,"p50":12300,"p75":32600,"p90":79400,"p95":157000}, + {"percentile":61,"minimum_total_income":18300,"percent_with_gains":0.0062,"mean_gains_given_gains":42200,"p05":-13800,"p10":-4100,"p25":3300,"p50":12400,"p75":32300,"p90":78100,"p95":146700}, + {"percentile":62,"minimum_total_income":18800,"percent_with_gains":0.0065,"mean_gains_given_gains":39900,"p05":-13400,"p10":-4600,"p25":3200,"p50":12200,"p75":32500,"p90":80700,"p95":138500}, + {"percentile":63,"minimum_total_income":19200,"percent_with_gains":0.0069,"mean_gains_given_gains":46900,"p05":-12700,"p10":-3400,"p25":3500,"p50":12100,"p75":32100,"p90":84000,"p95":167000}, + {"percentile":64,"minimum_total_income":19700,"percent_with_gains":0.0069,"mean_gains_given_gains":43600,"p05":-13800,"p10":-4000,"p25":3200,"p50":12400,"p75":31000,"p90":85700,"p95":152000}, + {"percentile":65,"minimum_total_income":20200,"percent_with_gains":0.0069,"mean_gains_given_gains":52600,"p05":-14200,"p10":-4200,"p25":3600,"p50":12100,"p75":34800,"p90":88900,"p95":176400}, + {"percentile":66,"minimum_total_income":20700,"percent_with_gains":0.0069,"mean_gains_given_gains":39500,"p05":-11800,"p10":-3800,"p25":4000,"p50":12700,"p75":33300,"p90":78500,"p95":143300}, + {"percentile":67,"minimum_total_income":21200,"percent_with_gains":0.0072,"mean_gains_given_gains":47900,"p05":-11000,"p10":-3300,"p25":3900,"p50":12500,"p75":32700,"p90":79200,"p95":155400}, + {"percentile":68,"minimum_total_income":21800,"percent_with_gains":0.0074,"mean_gains_given_gains":40900,"p05":-12800,"p10":-4200,"p25":3300,"p50":11800,"p75":31200,"p90":72700,"p95":122300}, + {"percentile":69,"minimum_total_income":22300,"percent_with_gains":0.0077,"mean_gains_given_gains":46300,"p05":-13700,"p10":-4100,"p25":2900,"p50":11800,"p75":32100,"p90":78200,"p95":15190}, + {"percentile":70,"minimum_total_income":22900,"percent_with_gains":0.0077,"mean_gains_given_gains":50600,"p05":-10600,"p10":-3400,"p25":3400,"p50":12200,"p75":31700,"p90":75800,"p95":148000}, + {"percentile":71,"minimum_total_income":23500,"percent_with_gains":0.0082,"mean_gains_given_gains":41200,"p05":-11600,"p10":-3500,"p25":3500,"p50":12100,"p75":32600,"p90":83400,"p95":146800}, + {"percentile":72,"minimum_total_income":24100,"percent_with_gains":0.0084,"mean_gains_given_gains":42000,"p05":-12400,"p10":-4600,"p25":2900,"p50":11300,"p75":29900,"p90":77400,"p95":154000}, + {"percentile":73,"minimum_total_income":24800,"percent_with_gains":0.0086,"mean_gains_given_gains":49000,"p05":-15300,"p10":-5100,"p25":3100,"p50":11900,"p75":31700,"p90":83800,"p95":161800}, + {"percentile":74,"minimum_total_income":25400,"percent_with_gains":0.0084,"mean_gains_given_gains":41400,"p05":-14700,"p10":-4000,"p25":3300,"p50":11800,"p75":30000,"p90":82600,"p95":154200}, + {"percentile":75,"minimum_total_income":26100,"percent_with_gains":0.0087,"mean_gains_given_gains":42600,"p05":-16400,"p10":-5000,"p25":2900,"p50":11500,"p75":30900,"p90":83200,"p95":167800}, + {"percentile":76,"minimum_total_income":26800,"percent_with_gains":0.0092,"mean_gains_given_gains":42400,"p05":-13800,"p10":-3900,"p25":3300,"p50":11800,"p75":31500,"p90":84500,"p95":162700}, + {"percentile":77,"minimum_total_income":27500,"percent_with_gains":0.0094,"mean_gains_given_gains":42600,"p05":-13200,"p10":-3800,"p25":3800,"p50":12300,"p75":33200,"p90":82600,"p95":145000}, + {"percentile":78,"minimum_total_income":28300,"percent_with_gains":0.0097,"mean_gains_given_gains":51900,"p05":-14900,"p10":-3800,"p25":4100,"p50":12500,"p75":31900,"p90":82400,"p95":148500}, + {"percentile":79,"minimum_total_income":29100,"percent_with_gains":0.0102,"mean_gains_given_gains":44700,"p05":-12600,"p10":-4000,"p25":3000,"p50":11700,"p75":29200,"p90":75700,"p95":145400}, + {"percentile":80,"minimum_total_income":30000,"percent_with_gains":0.0103,"mean_gains_given_gains":49800,"p05":-15400,"p10":-4200,"p25":4000,"p50":12500,"p75":32700,"p90":93600,"p95":184700}, + {"percentile":81,"minimum_total_income":30900,"percent_with_gains":0.0112,"mean_gains_given_gains":50400,"p05":-11900,"p10":-3300,"p25":3200,"p50":11900,"p75":32300,"p90":87500,"p95":164400}, + {"percentile":82,"minimum_total_income":31800,"percent_with_gains":0.0113,"mean_gains_given_gains":46200,"p05":-13900,"p10":-4300,"p25":3100,"p50":11500,"p75":30100,"p90":86700,"p95":165700}, + {"percentile":83,"minimum_total_income":32800,"percent_with_gains":0.0118,"mean_gains_given_gains":47800,"p05":-12800,"p10":-3600,"p25":3300,"p50":11700,"p75":31700,"p90":91900,"p95":174100}, + {"percentile":84,"minimum_total_income":33900,"percent_with_gains":0.0121,"mean_gains_given_gains":46000,"p05":-13700,"p10":-4400,"p25":3600,"p50":11800,"p75":30700,"p90":93800,"p95":178300}, + {"percentile":85,"minimum_total_income":35000,"percent_with_gains":0.0136,"mean_gains_given_gains":46800,"p05":-14300,"p10":-4100,"p25":3700,"p50":12200,"p75":30800,"p90":90000,"p95":181200}, + {"percentile":86,"minimum_total_income":36200,"percent_with_gains":0.0147,"mean_gains_given_gains":55100,"p05":-13200,"p10":-4200,"p25":3500,"p50":12000,"p75":33300,"p90":100800,"p95":206800}, + {"percentile":87,"minimum_total_income":37400,"percent_with_gains":0.0172,"mean_gains_given_gains":59400,"p05":-14500,"p10":-4500,"p25":3600,"p50":11800,"p75":33800,"p90":108100,"p95":212600}, + {"percentile":88,"minimum_total_income":38700,"percent_with_gains":0.0191,"mean_gains_given_gains":52100,"p05":-15000,"p10":-5300,"p25":2300,"p50":11400,"p75":35100,"p90":110800,"p95":221600}, + {"percentile":89,"minimum_total_income":39900,"percent_with_gains":0.02,"mean_gains_given_gains":55600,"p05":-16700,"p10":-4900,"p25":2900,"p50":11500,"p75":34000,"p90":111700,"p95":226300}, + {"percentile":90,"minimum_total_income":41500,"percent_with_gains":0.0204,"mean_gains_given_gains":59000,"p05":-17200,"p10":-5500,"p25":2400,"p50":11400,"p75":32500,"p90":99800,"p95":209800}, + {"percentile":91,"minimum_total_income":43300,"percent_with_gains":0.021,"mean_gains_given_gains":53200,"p05":-16500,"p10":-5100,"p25":2800,"p50":11400,"p75":30700,"p90":99900,"p95":213000}, + {"percentile":92,"minimum_total_income":45500,"percent_with_gains":0.0226,"mean_gains_given_gains":58500,"p05":-16100,"p10":-5000,"p25":3200,"p50":11600,"p75":32700,"p90":100900,"p95":208600}, + {"percentile":93,"minimum_total_income":48300,"percent_with_gains":0.0247,"mean_gains_given_gains":68300,"p05":-14900,"p10":-4700,"p25":3800,"p50":11700,"p75":34600,"p90":106800,"p95":226200}, + {"percentile":94,"minimum_total_income":51900,"percent_with_gains":0.0281,"mean_gains_given_gains":50600,"p05":-15500,"p10":-4800,"p25":3400,"p50":11700,"p75":35500,"p90":113500,"p95":250000}, + {"percentile":95,"minimum_total_income":56700,"percent_with_gains":0.0326,"mean_gains_given_gains":73800,"p05":-17800,"p10":-5800,"p25":3300,"p50":11700,"p75":35000,"p90":120500,"p95":265700}, + {"percentile":96,"minimum_total_income":63400,"percent_with_gains":0.0391,"mean_gains_given_gains":83600,"p05":-18300,"p10":-5600,"p25":3600,"p50":12000,"p75":38100,"p90":134400,"p95":310900}, + {"percentile":97,"minimum_total_income":73400,"percent_with_gains":0.0503,"mean_gains_given_gains":96100,"p05":-20200,"p10":-5900,"p25":3600,"p50":12500,"p75":41900,"p90":161500,"p95":373900}, + {"percentile":98,"minimum_total_income":90100,"percent_with_gains":0.071,"mean_gains_given_gains":120400,"p05":-24100,"p10":-7500,"p25":2900,"p50":12700,"p75":48000,"p90":200000,"p95":470000}, + {"percentile":99,"minimum_total_income":128200,"percent_with_gains":0.1508,"mean_gains_given_gains":306800,"p05":-42800,"p10":-12400,"p25":200,"p50":13600,"p75":74900,"p90":431600,"p95":1162400} + ] +} diff --git a/packages/microcosm-build/src/microcosm/build/uk/cgt_band_donor_support_bounds.json b/packages/microcosm-build/src/microcosm/build/uk/cgt_band_donor_support_bounds.json new file mode 100644 index 000000000..3e6e4dda0 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk/cgt_band_donor_support_bounds.json @@ -0,0 +1,25 @@ +{ + "version": 1, + "country": "uk", + "policy": "Disclosure-free support intervals derived from published HMRC Table 2.1a gain bands retained by the CGT donor stage. Intervals are lower-inclusive and upper-exclusive.", + "source": { + "resource": "hmrc_cgt_size_bands.json", + "table": "2.1a", + "minimum_lower_limit": 12300 + }, + "bounds": { + "capital_gains": [12300, null] + }, + "bands": [ + {"lower": 12300, "upper": 25000}, + {"lower": 25000, "upper": 50000}, + {"lower": 50000, "upper": 100000}, + {"lower": 100000, "upper": 250000}, + {"lower": 250000, "upper": 500000}, + {"lower": 500000, "upper": 1000000}, + {"lower": 1000000, "upper": 2000000}, + {"lower": 2000000, "upper": 5000000}, + {"lower": 5000000, "upper": null} + ], + "semantic_fit_note": "These intervals describe donor-stage initial support. The later Table 3 redraw can move amounts within a different band surface, so terminal support-gate applicability requires reviewer confirmation." +} diff --git a/packages/microcosm-build/src/microcosm/build/uk/country_package.json b/packages/microcosm-build/src/microcosm/build/uk/country_package.json index 9bc16e859..2c04f97d3 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/country_package.json +++ b/packages/microcosm-build/src/microcosm/build/uk/country_package.json @@ -67,6 +67,31 @@ "kind": "legacy_json", "schema_id": "legacy_json" }, + { + "path": "hmrc_cgt_size_bands.json", + "kind": "legacy_json", + "schema_id": "legacy_json" + }, + { + "path": "advani_summers_capital_gains_distribution.json", + "kind": "legacy_json", + "schema_id": "legacy_json" + }, + { + "path": "salary_sacrifice_anchor.json", + "kind": "legacy_json", + "schema_id": "legacy_json" + }, + { + "path": "slc_liable_stocks.json", + "kind": "legacy_json", + "schema_id": "legacy_json" + }, + { + "path": "cgt_band_donor_support_bounds.json", + "kind": "legacy_json", + "schema_id": "legacy_json" + }, { "path": "hmrc_income_release_gate_report.json", "kind": "legacy_json", diff --git a/packages/microcosm-build/src/microcosm/build/uk/gates.json b/packages/microcosm-build/src/microcosm/build/uk/gates.json index 5b33c56ce..39a384ded 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/gates.json +++ b/packages/microcosm-build/src/microcosm/build/uk/gates.json @@ -208,6 +208,7 @@ "household.gas_consumption", "household.has_fuel_consumption", "household.household_is_capital_gains_clone", + "household.household_is_cgt_band_donor", "household.household_is_spi_synthetic", "household.la_code_oa", "household.lsoa_code", @@ -277,6 +278,18 @@ }, "notes": "The household BRMA assignment must remain inside the PolicyEngine-UK brma enum domain." }, + { + "id": "uk_student_loan_plan_enum_domain", + "gate": "enum_domain", + "phase": "terminal", + "criticality": "release_blocking", + "parameters": { + "columns": [ + "student_loan_plan" + ] + }, + "notes": "The person student-loan plan assignment must remain inside the PolicyEngine-UK StudentLoanPlan enum domain." + }, { "id": "uk_calibration_reference_coverage", "gate": "calibration_reference_coverage", diff --git a/packages/microcosm-build/src/microcosm/build/uk/hmrc_cgt_size_bands.json b/packages/microcosm-build/src/microcosm/build/uk/hmrc_cgt_size_bands.json new file mode 100644 index 000000000..f17ec7fa1 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk/hmrc_cgt_size_bands.json @@ -0,0 +1,111 @@ +{ + "version": 1, + "country": "uk", + "source": { + "citation": "HMRC, Capital Gains Tax statistics, Table 2.1a: estimated number of taxpayers, amounts of gains and tax liabilities by size of gain, individuals, 2023 to 2024 (provisional).", + "collection_url": "https://www.gov.uk/government/statistics/capital-gains-tax-statistics", + "url": "https://assets.publishing.service.gov.uk/media/6878ac562bad77c3dae4dcef/Table_2_2025_Size_of_gain.ods", + "artifact": "Table_2_2025_Size_of_gain.ods", + "sheet": "2_1a_2023-24", + "published": "2025-07-24", + "ods_sha256": "696456114f408edd84c004fbe6175d918c5a511b6aaa7a53a95ca8936db22491", + "ods_size_bytes": 10658, + "ods_retrieved": "2026-08-21", + "ods_content_check": "Sheet 2_1a_2023-24 present; taxpayer (79/74/53 thousand) and gains (1418/2645/22714 GBP million) cell values match the committed extraction.", + "extracted_csv_sha256": "97e6bfc60d35eec230a62f4412e3adc034f58c46b3699570415a547f59628ecb", + "mapped_build_period": "2024", + "period_mapping": "latest_published_tax_year", + "units": { + "taxpayers_thousands": "thousand taxpayers", + "gains_gbp_millions": "GBP million", + "tax_gbp_millions": "GBP million" + }, + "definition_note": "Published figures include only taxpayers with a CGT liability; they are narrower than all people with positive gains or gains above the annual exempt amount." + }, + "rows": [ + { + "lower_limit": 0, + "taxpayers_thousands": 2, + "gains_gbp_millions": 1, + "tax_gbp_millions": 9 + }, + { + "lower_limit": 3000, + "taxpayers_thousands": 0, + "gains_gbp_millions": 1, + "tax_gbp_millions": 0 + }, + { + "lower_limit": 6000, + "taxpayers_thousands": 61, + "gains_gbp_millions": 461, + "tax_gbp_millions": 18 + }, + { + "lower_limit": 10000, + "taxpayers_thousands": 23, + "gains_gbp_millions": 251, + "tax_gbp_millions": 20 + }, + { + "lower_limit": 12300, + "taxpayers_thousands": 79, + "gains_gbp_millions": 1418, + "tax_gbp_millions": 169 + }, + { + "lower_limit": 25000, + "taxpayers_thousands": 74, + "gains_gbp_millions": 2645, + "tax_gbp_millions": 423 + }, + { + "lower_limit": 50000, + "taxpayers_thousands": 53, + "gains_gbp_millions": 3731, + "tax_gbp_millions": 706 + }, + { + "lower_limit": 100000, + "taxpayers_thousands": 37, + "gains_gbp_millions": 5649, + "tax_gbp_millions": 1077 + }, + { + "lower_limit": 250000, + "taxpayers_thousands": 14, + "gains_gbp_millions": 4766, + "tax_gbp_millions": 826 + }, + { + "lower_limit": 500000, + "taxpayers_thousands": 8, + "gains_gbp_millions": 5705, + "tax_gbp_millions": 895 + }, + { + "lower_limit": 1000000, + "taxpayers_thousands": 5, + "gains_gbp_millions": 6390, + "tax_gbp_millions": 1067 + }, + { + "lower_limit": 2000000, + "taxpayers_thousands": 3, + "gains_gbp_millions": 9189, + "tax_gbp_millions": 1718 + }, + { + "lower_limit": 5000000, + "taxpayers_thousands": 2, + "gains_gbp_millions": 22714, + "tax_gbp_millions": 4532 + } + ], + "retained_band_checksums": { + "minimum_lower_limit": 12300, + "band_count": 9, + "taxpayers": 275000, + "gains_gbp": 62207000000 + } +} diff --git a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json index 7c166b3cb..0f9c19d58 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json @@ -458,10 +458,57 @@ "weight_source": "household_weight" }, "family_coverage": { + "cgt_band_donors": { + "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", + "base_candidate_tier": "frs", + "effective_mass_requirements": {}, + "mass_change_semantics": "mass_increasing_support", + "output_weight_kind": "importance", + "outputs": [ + "household_is_cgt_band_donor", + "capital_gains" + ], + "required_mass_change_reason": "Stack 30 positive-weight HMRC Table 2.1a support households per retained gain band; published donor mass is added explicitly.", + "rewrites": [ + "capital_gains" + ], + "source_manifest": "source_stages.json", + "source_manifest_sha256": "9beda5ab8d9f16ddacecfa1530e6679f7cb00a49eae01b671f318be04ce1bef6", + "source_vintages": { + "source": "HMRC Capital Gains Tax statistics, July 2025, Table 2.1a", + "survey": "HMRC Capital Gains Tax statistics Table 2.1a and Advani-Summers capital-gains incidence" + }, + "stage": "cgt_band_donors", + "status": "required_at_build" + }, + "cgt_incidence_clone": { + "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", + "base_candidate_tier": "frs", + "effective_mass_requirements": {}, + "mass_change_semantics": "mass_conserving", + "output_weight_kind": "importance", + "outputs": [ + "household_is_capital_gains_clone", + "capital_gains" + ], + "required_mass_change_reason": "Capital-gains incidence clone splits every household's mass equally across original and clone records; total household mass is conserved.", + "rewrites": [ + "capital_gains" + ], + "source_manifest": "source_stages.json", + "source_manifest_sha256": "9beda5ab8d9f16ddacecfa1530e6679f7cb00a49eae01b671f318be04ce1bef6", + "source_vintages": { + "source": "Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465", + "survey": "Family Resources Survey 2024-25, SPI synthetic support, and Advani-Summers capital-gains incidence" + }, + "stage": "cgt_incidence_clone", + "status": "required_at_build" + }, "etb_services": { "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", "base_candidate_tier": "frs", "effective_mass_requirements": {}, + "mass_change_semantics": "mass_conserving", "output_weight_kind": "importance", "outputs": [ "dfe_education_spending", @@ -478,7 +525,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "31818338ef62a19d9aebad7cdcf79af7cf05c2f0657ecc5e53b5954390ffa93b", + "source_manifest_sha256": "9beda5ab8d9f16ddacecfa1530e6679f7cb00a49eae01b671f318be04ce1bef6", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab, DfT rail fare index, and public NHS activity/cost table.", "survey": "Effects of Taxes and Benefits 1977-2024 and NHS age-gender public table" @@ -490,6 +537,7 @@ "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", "base_candidate_tier": "frs", "effective_mass_requirements": {}, + "mass_change_semantics": "mass_conserving", "output_weight_kind": "importance", "outputs": [ "full_rate_vat_expenditure_rate" @@ -497,7 +545,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "31818338ef62a19d9aebad7cdcf79af7cf05c2f0657ecc5e53b5954390ffa93b", + "source_manifest_sha256": "9beda5ab8d9f16ddacecfa1530e6679f7cb00a49eae01b671f318be04ce1bef6", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab and cited VAT anchor resource.", "survey": "Effects of Taxes and Benefits 1977-2024" @@ -526,13 +574,37 @@ "stage": "hmrc_cgt_gains", "status": "required_at_build" }, + "hmrc_cgt_gains_spine": { + "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", + "base_candidate_tier": "frs", + "calibration_permitted": false, + "effective_mass_requirements": {}, + "fact_fence_id": "cgt_band_facts_policy_endogenous_proxy_conditioned", + "fenced_fact_count": 76, + "output_weight_kind": "importance", + "outputs": [ + "capital_gains" + ], + "required_mass_change_reason": "Amounts-only capital gains redraw: household weights pass through unchanged and total household mass is conserved.", + "rewrites": [ + "capital_gains" + ], + "source_manifest": "source_stages.json", + "source_manifest_sha256": "9beda5ab8d9f16ddacecfa1530e6679f7cb00a49eae01b671f318be04ce1bef6", + "source_vintages": { + "hmrc_surface": "2023-24", + "mapped_build_period": "2024" + }, + "stage": "hmrc_cgt_gains_spine", + "status": "required_at_build" + }, "hmrc_spi_income": { "band_measure": "hmrc_spi_assessable_income", "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", "base_candidate_tier": "frs", "calibration_permitted": false, "canonical_source_manifest": "source_stages.json", - "canonical_source_manifest_sha256": "31818338ef62a19d9aebad7cdcf79af7cf05c2f0657ecc5e53b5954390ffa93b", + "canonical_source_manifest_sha256": "9beda5ab8d9f16ddacecfa1530e6679f7cb00a49eae01b671f318be04ce1bef6", "effective_mass_requirements": { "charitable_investment_gifts": { "mass_share_denominator": "all_person_effective_mass", @@ -617,6 +689,7 @@ "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", "base_candidate_tier": "frs", "effective_mass_requirements": {}, + "mass_change_semantics": "mass_conserving", "output_weight_kind": "importance", "outputs": [ "food_and_non_alcoholic_beverages_consumption", @@ -642,7 +715,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "31818338ef62a19d9aebad7cdcf79af7cf05c2f0657ecc5e53b5954390ffa93b", + "source_manifest_sha256": "9beda5ab8d9f16ddacecfa1530e6679f7cb00a49eae01b671f318be04ce1bef6", "source_vintages": { "source": "UK Data Service SN 9468 Living Costs and Food Survey 2023-24 household/person tabs, NEED 2023 headline energy tables, Ofgem Q2 2026 unit rates, and WAS round-8 bridge donor.", "survey": "Living Costs and Food Survey 2023-24" @@ -654,6 +727,7 @@ "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", "base_candidate_tier": "frs", "effective_mass_requirements": {}, + "mass_change_semantics": "mass_conserving", "output_weight_kind": "importance", "outputs": [], "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", @@ -662,7 +736,7 @@ "property_wealth" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "31818338ef62a19d9aebad7cdcf79af7cf05c2f0657ecc5e53b5954390ffa93b", + "source_manifest_sha256": "9beda5ab8d9f16ddacecfa1530e6679f7cb00a49eae01b671f318be04ce1bef6", "source_vintages": { "source": "MHCLG dwellings and ONS UK House Price Index December 2025 regional average prices.", "survey": "Public regional property reference" @@ -670,10 +744,57 @@ "stage": "regional_property_uprating", "status": "required_at_build" }, + "salary_sacrifice": { + "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", + "base_candidate_tier": "frs", + "effective_mass_requirements": {}, + "mass_change_semantics": "mass_conserving", + "output_weight_kind": "importance", + "outputs": [ + "pension_contributions_via_salary_sacrifice", + "employee_pension_contributions" + ], + "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", + "rewrites": [ + "pension_contributions_via_salary_sacrifice", + "employee_pension_contributions" + ], + "source_manifest": "source_stages.json", + "source_manifest_sha256": "9beda5ab8d9f16ddacecfa1530e6679f7cb00a49eae01b671f318be04ce1bef6", + "source_vintages": { + "source": "HMRC, Salary sacrifice reform for pension contributions effective from 6 April 2029", + "survey": "Family Resources Survey 2024-25 salary-sacrifice respondents and HMRC salary-sacrifice reform analysis" + }, + "stage": "salary_sacrifice", + "status": "required_at_build" + }, + "student_loans": { + "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", + "base_candidate_tier": "frs", + "effective_mass_requirements": {}, + "mass_change_semantics": "mass_conserving", + "output_weight_kind": "importance", + "outputs": [ + "student_loan_plan" + ], + "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", + "rewrites": [ + "student_loan_plan" + ], + "source_manifest": "source_stages.json", + "source_manifest_sha256": "9beda5ab8d9f16ddacecfa1530e6679f7cb00a49eae01b671f318be04ce1bef6", + "source_vintages": { + "source": "Explore Education Statistics Table 6a, Higher education total", + "survey": "Family Resources Survey 2024-25 and Student Loans Company borrower forecasts for England" + }, + "stage": "student_loans", + "status": "required_at_build" + }, "was_wealth": { "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", "base_candidate_tier": "frs", "effective_mass_requirements": {}, + "mass_change_semantics": "mass_conserving", "output_weight_kind": "importance", "outputs": [ "owned_land", @@ -693,7 +814,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "31818338ef62a19d9aebad7cdcf79af7cf05c2f0657ecc5e53b5954390ffa93b", + "source_manifest_sha256": "9beda5ab8d9f16ddacecfa1530e6679f7cb00a49eae01b671f318be04ce1bef6", "source_vintages": { "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed 2006-22 household tab.", "survey": "Wealth and Assets Survey round 8" diff --git a/packages/microcosm-build/src/microcosm/build/uk/salary_sacrifice_anchor.json b/packages/microcosm-build/src/microcosm/build/uk/salary_sacrifice_anchor.json new file mode 100644 index 000000000..9fe0255de --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk/salary_sacrifice_anchor.json @@ -0,0 +1,26 @@ +{ + "version": 1, + "country": "uk", + "source": { + "citation": "HMRC, Salary sacrifice reform for pension contributions effective from 6 April 2029.", + "url": "https://www.gov.uk/government/publications/salary-sacrifice-reform-for-pension-contributions-effective-from-6-april-2029/salary-sacrifice-reform-for-pension-contributions", + "published_year": 2025, + "raw_artifact_sha256": "0b966220ca665c4d08307080329e1f912bd5ebd78b60796d70a6ad52e3c68f7b", + "base_year": 2024, + "annual_growth": 0.024, + "growth_note": "The incumbent projects users at 2.4 percent annually from 2024; the committed 2024 anchor itself is HMRC's 7.7 million estimate." + }, + "hmrc_anchor": { + "total_users": 7700000, + "above_2000": 3300000, + "below_2000": 4300000 + }, + "derived": { + "staging_ratio": 0.7012987012987013, + "stage_target": 5400000 + }, + "notes": [ + "The 5.4 million stage target is a deliberate support-staging fraction, not a published target.", + "The incumbent obr/ naming is a misattribution: the 7.7 million total and 3.3/4.3 million split come from HMRC." + ] +} diff --git a/packages/microcosm-build/src/microcosm/build/uk/slc_liable_stocks.json b/packages/microcosm-build/src/microcosm/build/uk/slc_liable_stocks.json new file mode 100644 index 000000000..cd16c5cbe --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk/slc_liable_stocks.json @@ -0,0 +1,22 @@ +{ + "version": 1, + "country": "uk", + "source": { + "citation": "Student Loans Company, Student loan forecasts for England, Table 6a, Higher education total.", + "permalink": "https://explore-education-statistics.service.gov.uk/data-tables/permalink/6ff75517-7124-487c-cb4e-08de6eccf22d", + "raw_artifact_sha256": "b6de727ec0f8b1771c9a2d2c781b405830872634ed7283bf23e46a3898873922", + "chronicle_package_id": "slc-student-loan-borrower-forecasts-england-2025", + "chronicle_candidate": true, + "period_note": "Academic year 2024/25 maps to calendar year 2025 here; Chronicle stores the academic-year opening year 2024." + }, + "plans": { + "plan_2": { + "above_threshold": {"2025": 3985000, "2026": 4460000, "2027": 4825000, "2028": 5045000, "2029": 5160000, "2030": 5205000}, + "liable": {"2025": 8940000, "2026": 9710000, "2027": 10360000, "2028": 10615000, "2029": 10600000, "2030": 10525000} + }, + "plan_5": { + "above_threshold": {"2025": 0, "2026": 35000, "2027": 145000, "2028": 390000, "2029": 770000, "2030": 1235000}, + "liable": {"2025": 10000, "2026": 230000, "2027": 630000, "2028": 1380000, "2029": 2360000, "2030": 3400000} + } + } +} diff --git a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json index 9e052b4c0..f726f9fa0 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json @@ -2398,6 +2398,351 @@ ], "notes": "Runs the SPI-trained income QRFs on the raw-spine support channel, initializes FRS charity columns to zero, trains FRS-only stage 2 before redrawing base-channel dividends, and emits a sidecar-only 208-fact replay report for the spine path." }, + { + "stage": "cgt_incidence_clone", + "survey": "Family Resources Survey 2024-25, SPI synthetic support, and Advani-Summers capital-gains incidence", + "source": "Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465", + "grain": "household", + "artifacts": [ + { + "role": "capital_gains_incidence_and_quantiles", + "kind": "public_aggregate_reference", + "resource": "advani_summers_capital_gains_distribution.json", + "format": "json", + "runtime_sha256_required": true + } + ], + "operations": [ + { + "kind": "clone_records", + "entity": "household", + "copies": 2, + "flag_column": "household_is_capital_gains_clone", + "original_flag": false, + "clone_flag": true, + "mass_split": 0.5, + "weight_kind_out": "importance", + "conservation": "exact_total", + "id_remapping": "id_multiplier_for_values", + "declared_factor": 1.0, + "reason": "Capital-gains incidence clone splits every household's mass equally across original and clone records; total household mass is conserved." + }, + { + "kind": "draw_capital_gains_prior_from_banded_quantiles", + "resource": "advani_summers_capital_gains_distribution.json", + "income_proxy_components": [ + "employment_income", + "self_employment_income", + "state_pension_reported", + "private_pension_income", + "property_income", + "savings_interest_income", + "dividend_income", + "miscellaneous_income" + ], + "allowance_subtraction": false, + "carrier": "oldest adult; person_id ascending breaks age ties", + "adult_minimum_age": 16, + "quantile_points": [0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95], + "spline_degree": 1, + "extrapolation": "ext=0", + "keep_negative_draws": true, + "seed": 0, + "salt": "cgt_prior_amount" + } + ], + "outputs": [ + "household_is_capital_gains_clone", + "capital_gains" + ], + "rewrites": ["capital_gains"], + "notes": "Spine-only equal-mass incidence clone. The A&S prior fixes the gainer set and order for the following HMRC Table 3 redraw; negative extrapolated draws remain loss-makers." + }, + { + "stage": "cgt_band_donors", + "survey": "HMRC Capital Gains Tax statistics Table 2.1a and Advani-Summers capital-gains incidence", + "source": "HMRC Capital Gains Tax statistics, July 2025, Table 2.1a", + "grain": "household", + "artifacts": [ + { + "role": "hmrc_cgt_size_bands", + "kind": "public_aggregate_reference", + "resource": "hmrc_cgt_size_bands.json", + "format": "json", + "runtime_sha256_required": true + }, + { + "role": "capital_gains_incidence", + "kind": "public_aggregate_reference", + "resource": "advani_summers_capital_gains_distribution.json", + "format": "json", + "runtime_sha256_required": true + } + ], + "operations": [ + { + "kind": "stack_band_donor_households", + "size_band_resource": "hmrc_cgt_size_bands.json", + "incidence_resource": "advani_summers_capital_gains_distribution.json", + "minimum_band_lower": 12300, + "donors_per_band": 30, + "expected_band_count": 9, + "expected_donor_count": 270, + "candidate_order": "household_id ascending", + "draw": "weighted_without_replacement", + "propensity": "Advani-Summers percent_with_gains at oldest-adult component-sum income", + "seed": 1, + "flag_column": "household_is_cgt_band_donor", + "carrier": "oldest adult; person_id ascending breaks age ties", + "initial_weight": "published band taxpayers / donors_per_band", + "never_zero_weight": true, + "weight_kind_out": "importance", + "reason": "Stack 30 positive-weight HMRC Table 2.1a support households per retained gain band; published donor mass is added explicitly." + } + ], + "outputs": [ + "household_is_cgt_band_donor", + "capital_gains" + ], + "rewrites": ["capital_gains"], + "notes": "Adds 270 positive-weight band donors. Rows below GBP 12,300 are excluded because they mix annual-exempt-amount regimes and the spline body already supplies that support." + }, + { + "stage": "hmrc_cgt_gains_spine", + "survey": "HMRC Capital Gains Tax statistics table 3 (size of gain by taxable income), 2020-21 to 2023-24", + "source": "https://assets.publishing.service.gov.uk/media/6878ac62760bf6cedaf5bd93/Table_3_2025_Size_of_gain_by_income.ods", + "grain": "person", + "artifacts": [ + { + "role": "cgt_published_fact_surface", + "kind": "administrative_table", + "format": "ods", + "survey": "HMRC Capital Gains Tax statistics table 3", + "publication": "https://www.gov.uk/government/statistics/capital-gains-tax-statistics", + "vintage": "2023-24", + "tax_year_start": 2023, + "locator": "https://assets.publishing.service.gov.uk/media/6878ac62760bf6cedaf5bd93/Table_3_2025_Size_of_gain_by_income.ods", + "sha256": "8e75c00bab949348a7238fea6d995f626c85e5d02813b46606dd7fea85e9d0c3", + "size_bytes": 11996, + "mime_type": "application/vnd.oasis.opendocument.spreadsheet", + "sheets": ["3_1_2023-24", "3_2_2022-23", "3_3_2021-22", "3_4_2020-21"], + "mapped_build_period": 2024, + "period_mapping": "latest_published_tax_year", + "runtime_sha256_required": true + }, + { + "role": "policy_parameters", + "kind": "versioned_parameter_tree", + "dependency": "policyengine-uk>=2.88 via microcosm-build[uk]", + "parameters": [ + "gov.hmrc.income_tax.allowances.personal_allowance.amount", + "gov.hmrc.income_tax.allowances.personal_allowance.maximum_ANI", + "gov.hmrc.income_tax.allowances.personal_allowance.reduction_rate", + "gov.hmrc.cgt.annual_exempt_amount" + ], + "instant_rule": "raw dated parameter files evaluated at 1 June of the build period's tax year", + "runtime_sha256_required": false, + "dependency_discipline": "deferred inside uk_cgt_policy_parameters; the base package never imports policyengine-uk at import time" + } + ], + "operations": [ + { + "kind": "verify_pinned_cgt_ods", + "artifact_role": "cgt_published_fact_surface", + "require_before_source_read": true, + "runtime_sha256_required": true, + "fail_on_mismatch": true + }, + { + "kind": "taxable_income_proxy", + "components": [ + "employment_income", + "self_employment_income", + "state_pension_reported", + "private_pension_income", + "property_income", + "savings_interest_income", + "dividend_income", + "miscellaneous_income" + ], + "components_semantics": "Persisted leaves of the model's total_income concept (ITA 2007 s.23); state_pension_reported stands in for social_security_income, whose other taxable benefits are not persisted; reliefs such as pension contributions and Gift Aid are not deducted.", + "allowance": "tapered Personal Allowance from the policy_parameters artifact", + "fail_on_missing_component": true + }, + { + "kind": "rank_preserving_allocation", + "within": "income band", + "ordering": "existing gains descending, person_id ascending on ties", + "band_order": "highest gain band first", + "suppressed_cell_allocation": "count implied by the cell's published gains at the band-total mean", + "column_reconciliation": "every income column rescales onto its published All-row taxpayer total", + "shortfall_policy": "proportional scale-down when the population holds less gainer mass than published taxpayers", + "minimum_allocation_people": 1, + "weights": "household_weight mapped to persons; no person splits across bands" + }, + { + "kind": "within_band_draws", + "bounded_band_family": "truncated exponential matched to the cell's published mean", + "open_band_family": "Pareto with alpha = mean / (mean - lower bound)", + "mean_repair_margin": 0.02, + "mean_repair_reason": "Published counts round to the nearest thousand and amounts to the nearest million; four cells of the 2023-24 table imply a mean outside their own band, and repaired means clamp just inside the violated boundary.", + "bottom_band_floor": "annual exempt amount plus one pound", + "seed_base": 552, + "seed_mixing": "seed combined with the build period; draws ordered by allocation rank", + "deterministic": true + }, + { + "kind": "sub_aea_remainder", + "policy": "gainers beyond the published taxpayer mass keep their existing amounts capped at the annual exempt amount", + "rationale": "Table 3 covers only individuals with a CGT liability; remaining gainers are treated as sub-AEA gainers rather than invented into the liability distribution or deleted." + }, + { + "kind": "record_mass_conservation_receipt", + "entity": "household", + "reason": "Amounts-only capital gains redraw: household weights pass through unchanged and total household mass is conserved.", + "declared_factor": 1.0, + "gate_coupling": "The terminal family gate requires a valid mass-conserving MassChangeRecord carrying exactly this reason." + }, + { + "kind": "classify_cgt_band_facts_with_reviewed_fence", + "calibration_permitted": false, + "fact_fence_id": "cgt_band_facts_policy_endogenous_proxy_conditioned", + "fenced_fact_count": 76, + "fenced_fact_composition": "60 joint cells, 10 gain-band row totals, 6 income-column totals", + "classification_rationale": "The taxpayer count is endogenous to policy, the income conditioning is an arithmetic proxy, and the published surface needs rounding and suppression reconciliation before any per-band fact is exact.", + "calibrated_facts_unchanged": "The two aggregate facts in UK_CGT_TARGET_SPECS remain the only calibrated CGT facts.", + "promotion_path": "A separately reviewed target profile may lift specific band facts after the reconciliation and proxy adequacy are adjudicated.", + "adjudication": "https://github.com/PolicyEngine/microcosm/issues/552" + } + ], + "outputs": ["capital_gains"], + "rewrites": ["capital_gains"], + "notes": "Spine manifest projection of the merged HMRC Table 3 amounts stage. It deliberately omits base_candidate and verify_certified_candidate, which belong only to the certified-H5 path." + }, + { + "stage": "salary_sacrifice", + "survey": "Family Resources Survey 2024-25 salary-sacrifice respondents and HMRC salary-sacrifice reform analysis", + "source": "HMRC, Salary sacrifice reform for pension contributions effective from 6 April 2029", + "grain": "person", + "artifacts": [ + { + "role": "salary_sacrifice_anchor", + "kind": "public_aggregate_reference", + "resource": "salary_sacrifice_anchor.json", + "format": "json", + "runtime_sha256_required": true + } + ], + "operations": [ + { + "kind": "fit_weighted_qrf", + "training_population": "support_channel == frs and not capital-gains clone and not CGT band donor and salary_sacrifice_asked == 1", + "target_population": "salary_sacrifice_asked != 1 frame-wide", + "predictors": ["age", "employment_income"], + "targets": ["pension_contributions_via_salary_sacrifice"], + "weights": "household_weight", + "weight_mapping": "household_to_person", + "seed": 42, + "n_estimators": 100, + "clamp_minimum": 0, + "preserve_asked_rows": true, + "cache": false + }, + { + "kind": "convert_donors_to_target_stock", + "resource": "salary_sacrifice_anchor.json", + "target": 5400000, + "donor_pool": "employee_pension_contributions > 0 and pension_contributions_via_salary_sacrifice == 0 and employment_income > 0", + "rate_cap": 0.5, + "move": "full employee_pension_contributions to pension_contributions_via_salary_sacrifice; source zeroed", + "seed": 2024, + "salt": "salary_sacrifice_conversion", + "receipt": "weighted_headcount" + } + ], + "outputs": [ + "pension_contributions_via_salary_sacrifice", + "employee_pension_contributions" + ], + "nonnegative_outputs": [ + "pension_contributions_via_salary_sacrifice", + "employee_pension_contributions" + ], + "rewrites": [ + "pension_contributions_via_salary_sacrifice", + "employee_pension_contributions" + ], + "notes": "The QRF trains only on the 2024-25 FRS asked subset. The second arm creates support toward the reviewed 5.4m staging target by moving contributors' full pension amounts." + }, + { + "stage": "student_loans", + "survey": "Family Resources Survey 2024-25 and Student Loans Company borrower forecasts for England", + "source": "Explore Education Statistics Table 6a, Higher education total", + "grain": "person", + "artifacts": [ + { + "role": "frs_release", + "kind": "public_aggregate_reference", + "resource": "frs_release.json", + "format": "json", + "runtime_sha256_required": true + }, + { + "role": "slc_liable_stocks", + "kind": "public_aggregate_reference", + "resource": "slc_liable_stocks.json", + "format": "json", + "runtime_sha256_required": true + } + ], + "operations": [ + { + "kind": "assign_student_loan_plan_cohorts", + "year_rule": "calibration_year", + "start_year_formula": "year - age + 18", + "reported_repayment_test": "student_loan_repayments > 0", + "reported_country_gate": false, + "plan_1_before": 2012, + "plan_5_from": 2023, + "enum_domain": ["NONE", "PLAN_1", "PLAN_2", "PLAN_5"], + "plan_4_imputation": false + }, + { + "kind": "top_up_to_stock", + "plan": "PLAN_5", + "priority": 1, + "resource": "slc_liable_stocks.json", + "stock_series": "plan_5.liable", + "year_rule": "calibration_year", + "age_min": 18, + "age_max": 25, + "cohort_start_min": 2023, + "eligible_region_exclusions": ["SCOTLAND", "WALES", "NORTHERN_IRELAND"], + "highest_education": "TERTIARY", + "seed": 42, + "salt": "student_loan_plan_5" + }, + { + "kind": "top_up_to_stock", + "plan": "PLAN_2", + "priority": 2, + "resource": "slc_liable_stocks.json", + "stock_series": "plan_2.liable", + "year_rule": "calibration_year", + "age_min": 21, + "age_max": 55, + "cohort_start_min": 2012, + "cohort_start_max_exclusive": 2023, + "eligible_region_exclusions": ["SCOTLAND", "WALES", "NORTHERN_IRELAND"], + "highest_education": "TERTIARY", + "seed": 42, + "salt": "student_loan_plan_2" + } + ], + "outputs": ["student_loan_plan"], + "rewrites": ["student_loan_plan"], + "notes": "Reported PAYE repayers are classified without a country gate. England tertiary cohorts are then topped up PLAN_5 first and PLAN_2 second to the pinned liable stocks at the FRS release calibration year; PLAN_4 is never imputed." + }, { "stage": "frs_hmrc_retained_leaves", "survey": "Family Resources Survey 2024-25", diff --git a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml index e41947507..d377eebbb 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml @@ -1916,6 +1916,307 @@ stages: - hmrc_spi_unemployment_benefit_income - hmrc_spi_incapacity_benefit_income notes: Runs the SPI-trained income QRFs on the raw-spine support channel, initializes FRS charity columns to zero, trains FRS-only stage 2 before redrawing base-channel dividends, and emits a sidecar-only 208-fact replay report for the spine path. +- stage: cgt_incidence_clone + survey: Family Resources Survey 2024-25, SPI synthetic support, and Advani-Summers capital-gains incidence + source: Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465 + grain: household + artifacts: + - role: capital_gains_incidence_and_quantiles + kind: public_aggregate_reference + resource: advani_summers_capital_gains_distribution.json + format: json + runtime_sha256_required: true + operations: + - kind: clone_records + entity: household + copies: 2 + flag_column: household_is_capital_gains_clone + original_flag: false + clone_flag: true + mass_split: 0.5 + weight_kind_out: importance + conservation: exact_total + id_remapping: id_multiplier_for_values + declared_factor: 1.0 + reason: Capital-gains incidence clone splits every household's mass equally across original and clone records; total household mass is conserved. + - kind: draw_capital_gains_prior_from_banded_quantiles + resource: advani_summers_capital_gains_distribution.json + income_proxy_components: + - employment_income + - self_employment_income + - state_pension_reported + - private_pension_income + - property_income + - savings_interest_income + - dividend_income + - miscellaneous_income + allowance_subtraction: false + carrier: oldest adult; person_id ascending breaks age ties + adult_minimum_age: 16 + quantile_points: + - 0.05 + - 0.1 + - 0.25 + - 0.5 + - 0.75 + - 0.9 + - 0.95 + spline_degree: 1 + extrapolation: ext=0 + keep_negative_draws: true + seed: 0 + salt: cgt_prior_amount + outputs: + - household_is_capital_gains_clone + - capital_gains + rewrites: + - capital_gains + notes: Spine-only equal-mass incidence clone. The A&S prior fixes the gainer set and order for the following HMRC Table 3 redraw; negative extrapolated draws remain loss-makers. +- stage: cgt_band_donors + survey: HMRC Capital Gains Tax statistics Table 2.1a and Advani-Summers capital-gains incidence + source: HMRC Capital Gains Tax statistics, July 2025, Table 2.1a + grain: household + artifacts: + - role: hmrc_cgt_size_bands + kind: public_aggregate_reference + resource: hmrc_cgt_size_bands.json + format: json + runtime_sha256_required: true + - role: capital_gains_incidence + kind: public_aggregate_reference + resource: advani_summers_capital_gains_distribution.json + format: json + runtime_sha256_required: true + operations: + - kind: stack_band_donor_households + size_band_resource: hmrc_cgt_size_bands.json + incidence_resource: advani_summers_capital_gains_distribution.json + minimum_band_lower: 12300 + donors_per_band: 30 + expected_band_count: 9 + expected_donor_count: 270 + candidate_order: household_id ascending + draw: weighted_without_replacement + propensity: Advani-Summers percent_with_gains at oldest-adult component-sum income + seed: 1 + flag_column: household_is_cgt_band_donor + carrier: oldest adult; person_id ascending breaks age ties + initial_weight: published band taxpayers / donors_per_band + never_zero_weight: true + weight_kind_out: importance + reason: Stack 30 positive-weight HMRC Table 2.1a support households per retained gain band; published donor mass is added explicitly. + outputs: + - household_is_cgt_band_donor + - capital_gains + rewrites: + - capital_gains + notes: Adds 270 positive-weight band donors. Rows below GBP 12,300 are excluded because they mix annual-exempt-amount regimes and the spline body already supplies that support. +- stage: hmrc_cgt_gains_spine + survey: HMRC Capital Gains Tax statistics table 3 (size of gain by taxable income), 2020-21 to 2023-24 + source: https://assets.publishing.service.gov.uk/media/6878ac62760bf6cedaf5bd93/Table_3_2025_Size_of_gain_by_income.ods + grain: person + artifacts: + - role: cgt_published_fact_surface + kind: administrative_table + format: ods + survey: HMRC Capital Gains Tax statistics table 3 + publication: https://www.gov.uk/government/statistics/capital-gains-tax-statistics + vintage: 2023-24 + tax_year_start: 2023 + locator: https://assets.publishing.service.gov.uk/media/6878ac62760bf6cedaf5bd93/Table_3_2025_Size_of_gain_by_income.ods + sha256: 8e75c00bab949348a7238fea6d995f626c85e5d02813b46606dd7fea85e9d0c3 + size_bytes: 11996 + mime_type: application/vnd.oasis.opendocument.spreadsheet + sheets: + - 3_1_2023-24 + - 3_2_2022-23 + - 3_3_2021-22 + - 3_4_2020-21 + mapped_build_period: 2024 + period_mapping: latest_published_tax_year + runtime_sha256_required: true + - role: policy_parameters + kind: versioned_parameter_tree + dependency: policyengine-uk>=2.88 via microcosm-build[uk] + parameters: + - gov.hmrc.income_tax.allowances.personal_allowance.amount + - gov.hmrc.income_tax.allowances.personal_allowance.maximum_ANI + - gov.hmrc.income_tax.allowances.personal_allowance.reduction_rate + - gov.hmrc.cgt.annual_exempt_amount + instant_rule: raw dated parameter files evaluated at 1 June of the build period's tax year + runtime_sha256_required: false + dependency_discipline: deferred inside uk_cgt_policy_parameters; the base package never imports policyengine-uk at import time + operations: + - kind: verify_pinned_cgt_ods + artifact_role: cgt_published_fact_surface + require_before_source_read: true + runtime_sha256_required: true + fail_on_mismatch: true + - kind: taxable_income_proxy + components: + - employment_income + - self_employment_income + - state_pension_reported + - private_pension_income + - property_income + - savings_interest_income + - dividend_income + - miscellaneous_income + components_semantics: Persisted leaves of the model's total_income concept (ITA 2007 s.23); state_pension_reported stands in for social_security_income, whose other taxable benefits are not persisted; reliefs such as pension contributions and Gift Aid are not deducted. + allowance: tapered Personal Allowance from the policy_parameters artifact + fail_on_missing_component: true + - kind: rank_preserving_allocation + within: income band + ordering: existing gains descending, person_id ascending on ties + band_order: highest gain band first + suppressed_cell_allocation: count implied by the cell's published gains at the band-total mean + column_reconciliation: every income column rescales onto its published All-row taxpayer total + shortfall_policy: proportional scale-down when the population holds less gainer mass than published taxpayers + minimum_allocation_people: 1 + weights: household_weight mapped to persons; no person splits across bands + - kind: within_band_draws + bounded_band_family: truncated exponential matched to the cell's published mean + open_band_family: Pareto with alpha = mean / (mean - lower bound) + mean_repair_margin: 0.02 + mean_repair_reason: Published counts round to the nearest thousand and amounts to the nearest million; four cells of the 2023-24 table imply a mean outside their own band, and repaired means clamp just inside the violated boundary. + bottom_band_floor: annual exempt amount plus one pound + seed_base: 552 + seed_mixing: seed combined with the build period; draws ordered by allocation rank + deterministic: true + - kind: sub_aea_remainder + policy: gainers beyond the published taxpayer mass keep their existing amounts capped at the annual exempt amount + rationale: Table 3 covers only individuals with a CGT liability; remaining gainers are treated as sub-AEA gainers rather than invented into the liability distribution or deleted. + - kind: record_mass_conservation_receipt + entity: household + reason: 'Amounts-only capital gains redraw: household weights pass through unchanged and total household mass is conserved.' + declared_factor: 1.0 + gate_coupling: The terminal family gate requires a valid mass-conserving MassChangeRecord carrying exactly this reason. + - kind: classify_cgt_band_facts_with_reviewed_fence + calibration_permitted: false + fact_fence_id: cgt_band_facts_policy_endogenous_proxy_conditioned + fenced_fact_count: 76 + fenced_fact_composition: 60 joint cells, 10 gain-band row totals, 6 income-column totals + classification_rationale: The taxpayer count is endogenous to policy, the income conditioning is an arithmetic proxy, and the published surface needs rounding and suppression reconciliation before any per-band fact is exact. + calibrated_facts_unchanged: The two aggregate facts in UK_CGT_TARGET_SPECS remain the only calibrated CGT facts. + promotion_path: A separately reviewed target profile may lift specific band facts after the reconciliation and proxy adequacy are adjudicated. + adjudication: https://github.com/PolicyEngine/microcosm/issues/552 + outputs: + - capital_gains + rewrites: + - capital_gains + notes: Spine manifest projection of the merged HMRC Table 3 amounts stage. It deliberately omits base_candidate and verify_certified_candidate, which belong only to the certified-H5 path. +- stage: salary_sacrifice + survey: Family Resources Survey 2024-25 salary-sacrifice respondents and HMRC salary-sacrifice reform analysis + source: HMRC, Salary sacrifice reform for pension contributions effective from 6 April 2029 + grain: person + artifacts: + - role: salary_sacrifice_anchor + kind: public_aggregate_reference + resource: salary_sacrifice_anchor.json + format: json + runtime_sha256_required: true + operations: + - kind: fit_weighted_qrf + training_population: support_channel == frs and not capital-gains clone and not CGT band donor and salary_sacrifice_asked == 1 + target_population: salary_sacrifice_asked != 1 frame-wide + predictors: + - age + - employment_income + targets: + - pension_contributions_via_salary_sacrifice + weights: household_weight + weight_mapping: household_to_person + seed: 42 + n_estimators: 100 + clamp_minimum: 0 + preserve_asked_rows: true + cache: false + - kind: convert_donors_to_target_stock + resource: salary_sacrifice_anchor.json + target: 5400000 + donor_pool: employee_pension_contributions > 0 and pension_contributions_via_salary_sacrifice == 0 and employment_income > 0 + rate_cap: 0.5 + move: full employee_pension_contributions to pension_contributions_via_salary_sacrifice; source zeroed + seed: 2024 + salt: salary_sacrifice_conversion + receipt: weighted_headcount + outputs: + - pension_contributions_via_salary_sacrifice + - employee_pension_contributions + nonnegative_outputs: + - pension_contributions_via_salary_sacrifice + - employee_pension_contributions + rewrites: + - pension_contributions_via_salary_sacrifice + - employee_pension_contributions + notes: The QRF trains only on the 2024-25 FRS asked subset. The second arm creates support toward the reviewed 5.4m staging target by moving contributors' full pension amounts. +- stage: student_loans + survey: Family Resources Survey 2024-25 and Student Loans Company borrower forecasts for England + source: Explore Education Statistics Table 6a, Higher education total + grain: person + artifacts: + - role: frs_release + kind: public_aggregate_reference + resource: frs_release.json + format: json + runtime_sha256_required: true + - role: slc_liable_stocks + kind: public_aggregate_reference + resource: slc_liable_stocks.json + format: json + runtime_sha256_required: true + operations: + - kind: assign_student_loan_plan_cohorts + year_rule: calibration_year + start_year_formula: year - age + 18 + reported_repayment_test: student_loan_repayments > 0 + reported_country_gate: false + plan_1_before: 2012 + plan_5_from: 2023 + enum_domain: + - NONE + - PLAN_1 + - PLAN_2 + - PLAN_5 + plan_4_imputation: false + - kind: top_up_to_stock + plan: PLAN_5 + priority: 1 + resource: slc_liable_stocks.json + stock_series: plan_5.liable + year_rule: calibration_year + age_min: 18 + age_max: 25 + cohort_start_min: 2023 + eligible_region_exclusions: + - SCOTLAND + - WALES + - NORTHERN_IRELAND + highest_education: TERTIARY + seed: 42 + salt: student_loan_plan_5 + - kind: top_up_to_stock + plan: PLAN_2 + priority: 2 + resource: slc_liable_stocks.json + stock_series: plan_2.liable + year_rule: calibration_year + age_min: 21 + age_max: 55 + cohort_start_min: 2012 + cohort_start_max_exclusive: 2023 + eligible_region_exclusions: + - SCOTLAND + - WALES + - NORTHERN_IRELAND + highest_education: TERTIARY + seed: 42 + salt: student_loan_plan_2 + outputs: + - student_loan_plan + rewrites: + - student_loan_plan + notes: Reported PAYE repayers are classified without a country gate. England tertiary cohorts are then topped up PLAN_5 first and PLAN_2 second to the pinned liable stocks at the FRS release calibration year; PLAN_4 is never imputed. - stage: frs_hmrc_retained_leaves survey: Family Resources Survey 2024-25 source: Department for Work and Pensions Family Resources Survey 2024-25 raw adult.tab and benefits.tab, caller-supplied local input diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/battery_bindings.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/battery_bindings.py index 28cb33b40..5b97ddf2e 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/battery_bindings.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/battery_bindings.py @@ -272,23 +272,30 @@ def _evaluate_take_up_signal( return uk_take_up_signal_gate(context.frame, **dict(parameters)) -def _evaluate_brma_enum_domain( +def _evaluate_enum_domain( context: EvidenceContext, parameters: Mapping[str, Any] ) -> GateResult: columns = tuple(parameters.get("columns", ())) - if columns != ("brma",): - raise ValueError("uk_brma_enum_domain must declare columns ['brma'].") - domain = context.artifacts.get("brma_enum_domain") + if len(columns) != 1 or not isinstance(columns[0], str): + raise ValueError("UK enum_domain gates must declare exactly one column.") + column = columns[0] + domain = context.artifacts.get(f"{column}_enum_domain") if domain is None: engine = context.artifacts["rules_engine"] - variable = engine._variable("brma") + variable = engine._variable(column) domain = getattr(variable, "possible_values", None) if domain is None: - raise ValueError("brma enum domain could not be resolved from evidence.") - return enum_domain_gate( - {"brma": context.frame.table("household")["brma"]}, - {"brma": domain}, - ) + raise ValueError(f"{column} enum domain could not be resolved from evidence.") + matches = [ + context.frame.table(entity)[column] + for entity in context.frame.entities + if column in context.frame.table(entity).columns + ] + if len(matches) != 1: + raise ValueError( + f"{column} must occur on exactly one frame entity; found {len(matches)}." + ) + return enum_domain_gate({column: matches[0]}, {column: domain}) def _evaluate_support( @@ -830,7 +837,7 @@ def _ledger_compile_parity_registry( ), "enum_domain": UKGateBinding( name="enum_domain", - evaluator=_evaluate_brma_enum_domain, + evaluator=_evaluate_enum_domain, parameter_keys=frozenset({"columns"}), ), "support": UKGateBinding( diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_imputation.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_imputation.py index a26764d93..85caad735 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_imputation.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_imputation.py @@ -62,6 +62,7 @@ import numpy as np import pandas as pd +from microcosm.build.source_manifest import SourceStageSpec from microcosm.build.uk_runtime.hmrc_capital_gains import ( HMRC_CGT_GAIN_BAND_LOWER_BOUNDS, HMRC_CGT_INCOME_BAND_LOWER_BOUNDS, @@ -88,6 +89,7 @@ "impute_uk_capital_gains", "summarize_uk_cgt_imputation", "uk_capital_gains_imputation_stage", + "uk_cgt_spine_stage_transform", "uk_cgt_policy_parameters", "uk_cgt_taxable_income_proxy", ] @@ -634,3 +636,68 @@ def transform(frame: Frame) -> Frame: return impute_uk_capital_gains(frame, distribution, resolved, seed=seed) return UKNationalStage(name=UK_CGT_IMPUTATION_STAGE_NAME, transform=transform) + + +def uk_cgt_spine_stage_transform( + stage: SourceStageSpec, + ods_path: str | Path, +): + """Bind the spine manifest, then reuse the reviewed merged CGT runtime. + + The certified-H5 wrapper and its candidate verification remain untouched; + this source-plan seam deliberately delegates only the amounts transform. + """ + + _assert_cgt_spine_stage_parameters(stage) + return uk_capital_gains_imputation_stage(ods_path).transform + + +def _assert_cgt_spine_stage_parameters(stage: SourceStageSpec) -> None: + """Arm 1 of the #730/#684 two-arm rule for the spine projection.""" + + expected_kinds = ( + "verify_pinned_cgt_ods", + "taxable_income_proxy", + "rank_preserving_allocation", + "within_band_draws", + "sub_aea_remainder", + "record_mass_conservation_receipt", + "classify_cgt_band_facts_with_reviewed_fence", + ) + kinds = tuple(operation.kind for operation in stage.operations) + if kinds != expected_kinds: + raise ValueError( + f"CGT spine operation order drifted: expected {expected_kinds}, got {kinds}." + ) + operations = { + operation.kind: operation.parameters for operation in stage.operations + } + expected = { + ("verify_pinned_cgt_ods", "artifact_role"): "cgt_published_fact_surface", + ("verify_pinned_cgt_ods", "require_before_source_read"): True, + ("verify_pinned_cgt_ods", "runtime_sha256_required"): True, + ("verify_pinned_cgt_ods", "fail_on_mismatch"): True, + ("taxable_income_proxy", "components"): list( + UK_CGT_TAXABLE_INCOME_PROXY_COMPONENTS + ), + ("taxable_income_proxy", "fail_on_missing_component"): True, + ("rank_preserving_allocation", "minimum_allocation_people"): 1, + ("within_band_draws", "seed_base"): UK_CGT_IMPUTATION_SEED, + ("within_band_draws", "mean_repair_margin"): _MEAN_MARGIN, + ("within_band_draws", "deterministic"): True, + ("record_mass_conservation_receipt", "reason"): ( + UK_CGT_MASS_CONSERVATION_REASON + ), + ("record_mass_conservation_receipt", "declared_factor"): 1.0, + ("classify_cgt_band_facts_with_reviewed_fence", "calibration_permitted"): ( + False + ), + ("classify_cgt_band_facts_with_reviewed_fence", "fenced_fact_count"): 76, + } + for (kind, parameter), value in expected.items(): + actual = operations[kind].get(parameter) + if actual != value: + raise ValueError( + f"CGT spine {kind} parameter {parameter!r} drifted: expected " + f"{value!r}, got {actual!r}." + ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_structure.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_structure.py new file mode 100644 index 000000000..6021aa822 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_structure.py @@ -0,0 +1,633 @@ +"""UK capital-gains incidence cloning and HMRC size-band support.""" + +from __future__ import annotations + +import json +from collections.abc import Mapping +from dataclasses import dataclass, field +from importlib.resources import files +from typing import Any + +import numpy as np +import pandas as pd +from scipy.interpolate import UnivariateSpline + +from microcosm.build.source_manifest import SourceOperationSpec, SourceStageSpec +from microcosm.build.stochastic_assignment import stable_identity_uniforms +from microcosm.build.uk_runtime.cgt_imputation import ( + UK_CGT_TAXABLE_INCOME_PROXY_COMPONENTS, +) +from microcosm.build.uk_runtime.national_frame import ( + uk_national_frame, + uk_time_period, + validate_uk_national_frame, +) +from microcosm.build.uk_runtime.rowwise_geography import ( + clone_entity_frame, + id_multiplier_for_values, +) +from microcosm.build.uk_runtime.spi_support import ( + _importance_weights_with_exact_total, +) +from microcosm.frame import Frame, MassChangeRecord, WeightKind + +CGT_CLONE_MASS_SPLIT = 0.5 +CGT_PRIOR_SEED = 0 +CGT_PRIOR_SALT = "cgt_prior_amount" +CGT_QUANTILE_POINTS = (0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95) +CGT_PRIOR_PERCENTILE_COLUMNS = ("p05", "p10", "p25", "p50", "p75", "p90", "p95") +CGT_ADULT_MINIMUM_AGE = 16 +DONORS_PER_BAND = 30 +DONOR_BAND_COUNT = 9 +DONOR_TOTAL = 270 +MIN_DONOR_BAND_LOWER = 12_300 +DONOR_SEED = 1 +DONOR_NEVER_ZERO_WEIGHT = True +HOUSEHOLD_IS_CGT_CLONE = "household_is_capital_gains_clone" +HOUSEHOLD_IS_CGT_BAND_DONOR = "household_is_cgt_band_donor" +CGT_CLONE_MASS_CHANGE_REASON = ( + "Capital-gains incidence clone splits every household's mass equally across " + "original and clone records; total household mass is conserved." +) +CGT_DONOR_MASS_CHANGE_REASON = ( + "Stack 30 positive-weight HMRC Table 2.1a support households per retained " + "gain band; published donor mass is added explicitly." +) + + +def load_advani_summers_distribution() -> Mapping[str, Any]: + """Load the committed Advani-Summers incidence and quantile surface.""" + + return json.loads( + files("microcosm.build.uk") + .joinpath("advani_summers_capital_gains_distribution.json") + .read_text(encoding="utf-8") + ) + + +def load_hmrc_cgt_size_bands() -> Mapping[str, Any]: + """Load the committed HMRC Table 2.1a size-band surface.""" + + return json.loads( + files("microcosm.build.uk") + .joinpath("hmrc_cgt_size_bands.json") + .read_text(encoding="utf-8") + ) + + +@dataclass(frozen=True) +class UKCGTIncidenceCloneResult: + """Cloned frame and the executed-effect receipt for stage 19.""" + + frame: Frame + original_mass: float + clone_mass: float + carrier_count: int + negative_prior_count: int + + def evidence(self) -> dict[str, object]: + return { + "stage": "cgt_incidence_clone", + "mass_by_clone_flag": { + "false": self.original_mass, + "true": self.clone_mass, + }, + "carrier_count": self.carrier_count, + "negative_prior_count": self.negative_prior_count, + } + + +@dataclass(frozen=True) +class UKCGTBandDonorResult: + """Band-donor frame and the executed-effect receipt for stage 20.""" + + frame: Frame + band_rows: tuple[Mapping[str, object], ...] + frs_donors: int + spi_donors: int + + def evidence(self) -> dict[str, object]: + return { + "stage": "cgt_band_donors", + "bands": [dict(row) for row in self.band_rows], + "support_channel_split": { + "frs": self.frs_donors, + "spi": self.spi_donors, + }, + } + + +@dataclass(frozen=True) +class UKCGTIncidenceCloneStageTransform: + """Whole-stage transform for the incidence clone and prior draw.""" + + stage: SourceStageSpec + distribution: Mapping[str, Any] | None = None + last_result: UKCGTIncidenceCloneResult | None = field(default=None, init=False) + + def __call__(self, frame: Frame) -> Frame: + resource = self.distribution or load_advani_summers_distribution() + _assert_cgt_incidence_stage_parameters(self.stage) + result = clone_cgt_incidence(frame, distribution=resource) + object.__setattr__(self, "last_result", result) + return result.frame + + @staticmethod + def output_columns() -> tuple[str, ...]: + return (HOUSEHOLD_IS_CGT_CLONE, "capital_gains") + + def checkpoint_metadata(self) -> dict[str, object]: + if self.last_result is None: + raise RuntimeError("checkpoint metadata requires a completed stage run.") + return {"evidence": self.last_result.evidence()} + + +@dataclass(frozen=True) +class UKCGTBandDonorStageTransform: + """Whole-stage transform for positive-weight HMRC size-band donors.""" + + stage: SourceStageSpec + size_bands: Mapping[str, Any] | None = None + distribution: Mapping[str, Any] | None = None + last_result: UKCGTBandDonorResult | None = field(default=None, init=False) + + def __call__(self, frame: Frame) -> Frame: + bands = self.size_bands or load_hmrc_cgt_size_bands() + distribution = self.distribution or load_advani_summers_distribution() + _assert_cgt_donor_stage_parameters(self.stage, size_bands=bands) + result = stack_cgt_band_donors( + frame, + size_bands=bands, + distribution=distribution, + ) + object.__setattr__(self, "last_result", result) + return result.frame + + @staticmethod + def output_columns() -> tuple[str, ...]: + return (HOUSEHOLD_IS_CGT_BAND_DONOR, "capital_gains") + + def checkpoint_metadata(self) -> dict[str, object]: + if self.last_result is None: + raise RuntimeError("checkpoint metadata requires a completed stage run.") + return {"evidence": self.last_result.evidence()} + + +def clone_cgt_incidence( + frame: Frame, + *, + distribution: Mapping[str, Any], +) -> UKCGTIncidenceCloneResult: + """Clone every household at equal mass and assign A&S priors to clones.""" + + validate_uk_national_frame(frame) + person = frame.table("person").copy() + benunit = frame.table("benunit").copy() + household = frame.table("household").copy() + multiplier = id_multiplier_for_values( + person["person_id"], + person["person_household_id"], + person["person_benunit_id"], + benunit["benunit_id"], + household["household_id"], + ) + cloned_person = clone_entity_frame( + person, + id_columns=("person_id", "person_household_id", "person_benunit_id"), + n_clones=2, + id_multiplier=multiplier, + clone_index_column=None, + ).reset_index(drop=True) + cloned_benunit = clone_entity_frame( + benunit, + id_columns=("benunit_id",), + n_clones=2, + id_multiplier=multiplier, + clone_index_column=None, + ).reset_index(drop=True) + cloned_household = clone_entity_frame( + household, + id_columns=("household_id",), + n_clones=2, + id_multiplier=multiplier, + clone_index_column=None, + ).reset_index(drop=True) + n_households = len(household) + clone_flags = np.r_[ + np.zeros(n_households, dtype=bool), + np.ones(n_households, dtype=bool), + ] + cloned_household[HOUSEHOLD_IS_CGT_CLONE] = clone_flags + split = np.tile( + frame.weights_for("household").values * CGT_CLONE_MASS_SPLIT, + 2, + ) + exact_weights = _importance_weights_with_exact_total( + split, + frame.weights_for("household").total, + ) + cloned_person["capital_gains"] = 0.0 + carrier_indices = _oldest_adult_indices( + cloned_person, + household_ids=set(cloned_household.loc[clone_flags, "household_id"].to_numpy()), + ) + carrier_income = _component_sum_income(cloned_person.loc[carrier_indices]) + carrier_draws = stable_identity_uniforms( + cloned_person.loc[carrier_indices, "person_id"].to_numpy(), + seed=CGT_PRIOR_SEED, + salt=CGT_PRIOR_SALT, + ) + priors = _draw_banded_priors( + carrier_income, + carrier_draws, + distribution=distribution, + ) + cloned_person.loc[carrier_indices, "capital_gains"] = priors + receipt = MassChangeRecord( + entity="household", + old_total=frame.weights_for("household").total, + new_total=exact_weights.total, + declared_factor=1.0, + reason=CGT_CLONE_MASS_CHANGE_REASON, + ) + result = uk_national_frame( + person=cloned_person, + benunit=cloned_benunit, + household=cloned_household, + time_period=uk_time_period(frame), + weight_kind=WeightKind.IMPORTANCE, + household_weights=exact_weights.values, + mass_log=(*frame.mass_log, receipt), + ) + validate_uk_national_frame(result) + original_mass = float(exact_weights.values[~clone_flags].sum()) + clone_mass = float(exact_weights.values[clone_flags].sum()) + return UKCGTIncidenceCloneResult( + frame=result, + original_mass=original_mass, + clone_mass=clone_mass, + carrier_count=len(carrier_indices), + negative_prior_count=int((priors < 0.0).sum()), + ) + + +def stack_cgt_band_donors( + frame: Frame, + *, + size_bands: Mapping[str, Any], + distribution: Mapping[str, Any], +) -> UKCGTBandDonorResult: + """Add 30 households per retained HMRC size band at band-exact weights.""" + + validate_uk_national_frame(frame) + person = frame.table("person").copy() + benunit = frame.table("benunit").copy() + household = frame.table("household").copy() + household[HOUSEHOLD_IS_CGT_BAND_DONOR] = False + bands = _retained_size_bands(size_bands) + carriers = _oldest_adult_indices(person, household_ids=set(household.household_id)) + candidates = person.loc[carriers].copy() + candidates["_income"] = _component_sum_income(candidates) + candidates["_propensity"] = _incidence_propensity( + candidates["_income"].to_numpy(dtype=float), distribution=distribution + ) + candidates = candidates.sort_values("person_household_id", kind="stable") + if len(candidates) < DONOR_TOTAL: + raise ValueError( + f"CGT donor stage requires at least {DONOR_TOTAL} candidate households; " + f"found {len(candidates)}." + ) + propensities = candidates["_propensity"].to_numpy(dtype=float) + if not np.isfinite(propensities).all() or (propensities < 0).any(): + raise ValueError("CGT donor propensities must be finite and non-negative.") + if propensities.sum() <= 0: + raise ValueError("CGT donor propensities have no positive mass.") + rng = np.random.default_rng(DONOR_SEED) + selected = rng.choice( + candidates["person_household_id"].to_numpy(), + size=DONOR_TOTAL, + replace=False, + p=propensities / propensities.sum(), + ) + selected_set = set(selected.tolist()) + donor_person = person.loc[person.person_household_id.isin(selected_set)].copy() + donor_benunit_ids = set(donor_person.person_benunit_id) + donor_benunit = benunit.loc[benunit.benunit_id.isin(donor_benunit_ids)].copy() + donor_household = household.loc[household.household_id.isin(selected_set)].copy() + multiplier = id_multiplier_for_values( + person["person_id"], + person["person_household_id"], + person["person_benunit_id"], + benunit["benunit_id"], + household["household_id"], + ) + for column in ("person_id", "person_household_id", "person_benunit_id"): + donor_person[column] = donor_person[column].astype("int64") + multiplier + donor_benunit["benunit_id"] = ( + donor_benunit["benunit_id"].astype("int64") + multiplier + ) + donor_household["household_id"] = ( + donor_household["household_id"].astype("int64") + multiplier + ) + position = {household_id: index for index, household_id in enumerate(selected)} + donor_household["_band_position"] = ( + donor_household["household_id"].sub(multiplier).map(position) + ) + if donor_household["_band_position"].isna().any(): + raise ValueError("CGT donor selection failed to map every donor household.") + donor_household = donor_household.sort_values("_band_position", kind="stable") + band_index = ( + donor_household["_band_position"].to_numpy(dtype=int) // DONORS_PER_BAND + ) + taxpayers = np.asarray([row["taxpayers"] for row in bands], dtype=float) + means = np.asarray([row["mean_gain"] for row in bands], dtype=float) + donor_weights = taxpayers[band_index] / DONORS_PER_BAND + if DONOR_NEVER_ZERO_WEIGHT and not (donor_weights > 0.0).all(): + raise ValueError("CGT band donors must all carry positive initial weight.") + donor_household[HOUSEHOLD_IS_CGT_BAND_DONOR] = True + gain_by_household = dict( + zip(donor_household.household_id, means[band_index], strict=True) + ) + evidence_band_index = band_index.copy() + evidence_donor_weights = donor_weights.copy() + donor_household["_donor_weight"] = donor_weights + donor_household = donor_household.drop(columns=["_band_position"]).sort_values( + "household_id", kind="stable" + ) + donor_weights = donor_household.pop("_donor_weight").to_numpy(dtype=float) + donor_person["capital_gains"] = 0.0 + donor_carriers = _oldest_adult_indices( + donor_person, + household_ids=set(donor_household.household_id), + ) + donor_person.loc[donor_carriers, "capital_gains"] = donor_person.loc[ + donor_carriers, "person_household_id" + ].map(gain_by_household) + final_person = pd.concat([person, donor_person], ignore_index=True) + final_benunit = pd.concat([benunit, donor_benunit], ignore_index=True) + final_household = pd.concat([household, donor_household], ignore_index=True) + final_weights = np.r_[frame.weights_for("household").values, donor_weights] + old_total = frame.weights_for("household").total + new_total = float(final_weights.sum()) + receipt = MassChangeRecord( + entity="household", + old_total=old_total, + new_total=new_total, + declared_factor=None, + reason=CGT_DONOR_MASS_CHANGE_REASON, + ) + result = uk_national_frame( + person=final_person, + benunit=final_benunit, + household=final_household, + time_period=uk_time_period(frame), + weight_kind=WeightKind.IMPORTANCE, + household_weights=final_weights, + mass_log=(*frame.mass_log, receipt), + ) + validate_uk_national_frame(result) + channels = household.set_index("household_id").get("household_support_channel") + original_selected = donor_household.household_id.sub(multiplier) + selected_channels = ( + original_selected.map(channels).fillna("unknown") + if channels is not None + else pd.Series("unknown", index=original_selected.index) + ) + band_rows: list[Mapping[str, object]] = [] + for index, band in enumerate(bands): + mask = evidence_band_index == index + band_rows.append( + { + "lower_limit": band["lower_limit"], + "donor_count": int(mask.sum()), + "donor_weight": float(evidence_donor_weights[mask][0]), + "weighted_taxpayers": float(evidence_donor_weights[mask].sum()), + "mean_gain": band["mean_gain"], + } + ) + return UKCGTBandDonorResult( + frame=result, + band_rows=tuple(band_rows), + frs_donors=int(selected_channels.eq("frs").sum()), + spi_donors=int(selected_channels.eq("spi").sum()), + ) + + +def _oldest_adult_indices( + person: pd.DataFrame, + *, + household_ids: set[object], +) -> np.ndarray: + required = {"person_id", "person_household_id", "age"} + missing = sorted(required - set(person.columns)) + if missing: + raise ValueError(f"CGT carrier selection is missing person columns: {missing}.") + candidates = person.loc[ + person.person_household_id.isin(household_ids) + & (pd.to_numeric(person.age, errors="coerce") >= CGT_ADULT_MINIMUM_AGE) + ].copy() + missing_households = household_ids - set(candidates.person_household_id) + if missing_households: + raise ValueError( + "Every CGT household requires an adult carrier; missing household " + f"id(s): {sorted(missing_households)[:5]}." + ) + candidates["_row"] = candidates.index + candidates = candidates.sort_values( + ["person_household_id", "age", "person_id"], + ascending=[True, False, True], + kind="stable", + ) + return ( + candidates.groupby("person_household_id", sort=False)["_row"].first().to_numpy() + ) + + +def _component_sum_income(person: pd.DataFrame) -> np.ndarray: + missing = sorted(set(UK_CGT_TAXABLE_INCOME_PROXY_COMPONENTS) - set(person.columns)) + if missing: + raise ValueError(f"CGT income proxy components missing: {missing}.") + numeric = person.loc[:, UK_CGT_TAXABLE_INCOME_PROXY_COMPONENTS].apply( + pd.to_numeric, errors="coerce" + ) + if not np.isfinite(numeric.to_numpy(dtype=float)).all(): + raise ValueError("CGT component-sum income contains non-finite values.") + return numeric.sum(axis=1).to_numpy(dtype=float) + + +def _distribution_rows(resource: Mapping[str, Any]) -> list[Mapping[str, Any]]: + rows = resource.get("rows") + if not isinstance(rows, list) or not rows: + raise ValueError("Advani-Summers resource must contain a non-empty rows list.") + minimums = [float(row["minimum_total_income"]) for row in rows] + if minimums != sorted(minimums) or minimums[0] != 0.0: + raise ValueError("Advani-Summers income bands must be sorted and start at 0.") + return rows + + +def _draw_banded_priors( + income: np.ndarray, + draws: np.ndarray, + *, + distribution: Mapping[str, Any], +) -> np.ndarray: + rows = _distribution_rows(distribution) + minimums = np.asarray([row["minimum_total_income"] for row in rows], dtype=float) + indexes = np.clip( + np.searchsorted(minimums, income, side="right") - 1, 0, len(rows) - 1 + ) + values = np.zeros(len(income), dtype=float) + for index, row in enumerate(rows): + mask = indexes == index + if not mask.any(): + continue + knots = np.asarray( + [row[column] for column in CGT_PRIOR_PERCENTILE_COLUMNS], dtype=float + ) + spline = UnivariateSpline(CGT_QUANTILE_POINTS, knots, k=1, s=0, ext=0) + values[mask] = spline(draws[mask]) + return values + + +def _incidence_propensity( + income: np.ndarray, + *, + distribution: Mapping[str, Any], +) -> np.ndarray: + rows = _distribution_rows(distribution) + minimums = np.asarray([row["minimum_total_income"] for row in rows], dtype=float) + indexes = np.clip( + np.searchsorted(minimums, income, side="right") - 1, 0, len(rows) - 1 + ) + rates = np.asarray([row["percent_with_gains"] for row in rows], dtype=float) + return rates[indexes] + + +def _retained_size_bands(resource: Mapping[str, Any]) -> list[dict[str, float]]: + rows = resource.get("rows") + if not isinstance(rows, list): + raise ValueError("HMRC CGT size-band resource must contain a rows list.") + retained: list[dict[str, float]] = [] + for row in rows: + lower = float(row["lower_limit"]) + if lower < MIN_DONOR_BAND_LOWER: + continue + taxpayers = float(row["taxpayers_thousands"]) * 1_000.0 + gains = float(row["gains_gbp_millions"]) * 1_000_000.0 + if taxpayers <= 0.0: + raise ValueError( + "Retained CGT size bands may not produce a zero initial weight." + ) + retained.append( + { + "lower_limit": lower, + "taxpayers": taxpayers, + "gains": gains, + "mean_gain": gains / taxpayers, + } + ) + return retained + + +def _operation(stage: SourceStageSpec, kind: str) -> SourceOperationSpec: + matches = [operation for operation in stage.operations if operation.kind == kind] + if len(matches) != 1: + raise ValueError( + f"Stage {stage.stage!r} must declare exactly one {kind!r} operation." + ) + return matches[0] + + +def _assert_parameters( + operation: SourceOperationSpec, + expected: Mapping[str, object], +) -> None: + for name, value in expected.items(): + actual = operation.parameters.get(name) + if actual != value: + raise ValueError( + f"{operation.kind} manifest parameter {name!r} drifted: " + f"expected {value!r}, got {actual!r}." + ) + + +def _assert_cgt_incidence_stage_parameters(stage: SourceStageSpec) -> None: + """Bind every stage-19 manifest parameter to reviewed code constants. + + This is arm 1 of the #730/#684 two-arm rule documented in ``spi_spine``; + :class:`UKCGTIncidenceCloneResult` supplies the executed-effect receipt. + """ + + _assert_parameters( + _operation(stage, "clone_records"), + { + "entity": "household", + "copies": 2, + "flag_column": HOUSEHOLD_IS_CGT_CLONE, + "original_flag": False, + "clone_flag": True, + "mass_split": CGT_CLONE_MASS_SPLIT, + "weight_kind_out": WeightKind.IMPORTANCE.value, + "conservation": "exact_total", + "id_remapping": "id_multiplier_for_values", + "declared_factor": 1.0, + "reason": CGT_CLONE_MASS_CHANGE_REASON, + }, + ) + _assert_parameters( + _operation(stage, "draw_capital_gains_prior_from_banded_quantiles"), + { + "resource": "advani_summers_capital_gains_distribution.json", + "income_proxy_components": list(UK_CGT_TAXABLE_INCOME_PROXY_COMPONENTS), + "allowance_subtraction": False, + "carrier": "oldest adult; person_id ascending breaks age ties", + "adult_minimum_age": CGT_ADULT_MINIMUM_AGE, + "quantile_points": list(CGT_QUANTILE_POINTS), + "spline_degree": 1, + "extrapolation": "ext=0", + "keep_negative_draws": True, + "seed": CGT_PRIOR_SEED, + "salt": CGT_PRIOR_SALT, + }, + ) + + +def _assert_cgt_donor_stage_parameters( + stage: SourceStageSpec, + *, + size_bands: Mapping[str, Any], +) -> None: + """Bind stage-20 parameters and recompute the band/weight invariants.""" + + operation = _operation(stage, "stack_band_donor_households") + _assert_parameters( + operation, + { + "size_band_resource": "hmrc_cgt_size_bands.json", + "incidence_resource": "advani_summers_capital_gains_distribution.json", + "minimum_band_lower": MIN_DONOR_BAND_LOWER, + "donors_per_band": DONORS_PER_BAND, + "expected_band_count": DONOR_BAND_COUNT, + "expected_donor_count": DONOR_TOTAL, + "candidate_order": "household_id ascending", + "draw": "weighted_without_replacement", + "seed": DONOR_SEED, + "flag_column": HOUSEHOLD_IS_CGT_BAND_DONOR, + "carrier": "oldest adult; person_id ascending breaks age ties", + "initial_weight": "published band taxpayers / donors_per_band", + "never_zero_weight": DONOR_NEVER_ZERO_WEIGHT, + "weight_kind_out": WeightKind.IMPORTANCE.value, + "reason": CGT_DONOR_MASS_CHANGE_REASON, + }, + ) + bands = _retained_size_bands(size_bands) + if len(bands) != DONOR_BAND_COUNT: + raise ValueError( + f"HMRC retained donor-band count drifted: expected {DONOR_BAND_COUNT}, " + f"got {len(bands)}." + ) + if DONORS_PER_BAND * len(bands) != DONOR_TOTAL: + raise ValueError("CGT donor count no longer equals 30 times retained bands.") + weights = np.asarray([band["taxpayers"] / DONORS_PER_BAND for band in bands]) + if DONOR_NEVER_ZERO_WEIGHT and not (weights > 0.0).all(): + raise ValueError("HMRC retained donor bands imply a zero initial weight.") diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_build.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_build.py index d8a16a897..03744f793 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_build.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_build.py @@ -546,6 +546,9 @@ def build_uk_national_dataset( ) if brma_domain is not None: artifacts["brma_enum_domain"] = brma_domain + student_loan_plan_domain = _engine_enum_domain(engine, "student_loan_plan") + if student_loan_plan_domain is not None: + artifacts["student_loan_plan_enum_domain"] = student_loan_plan_domain fit_weight_records = _stage_fit_weight_records(materialized_stages) if fit_weight_records is not None: artifacts["fit_weight_records"] = fit_weight_records @@ -830,11 +833,15 @@ def _stage_calibration_evidence( def _brma_enum_domain(engine: object) -> tuple[str, ...] | None: + return _engine_enum_domain(engine, "brma") + + +def _engine_enum_domain(engine: object, variable_name: str) -> tuple[str, ...] | None: variable_getter = getattr(engine, "_variable", None) if not callable(variable_getter): return None try: - variable = variable_getter("brma") + variable = variable_getter(variable_name) except Exception: return None possible_values = getattr(variable, "possible_values", None) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/release_input_coverage.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/release_input_coverage.py index c9fc675d0..cbbcd41fc 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/release_input_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/release_input_coverage.py @@ -212,7 +212,7 @@ def _source_stage_base_candidate_tier( source_manifest: str, *, stage_name: str, -) -> str: +) -> str | None: payload = _resource_payload(source_manifest) stages = payload.get("stages") if not isinstance(stages, list): @@ -228,6 +228,8 @@ def _source_stage_base_candidate_tier( ) base_candidate = matching[0].get("base_candidate") if not isinstance(base_candidate, Mapping): + if source_manifest == "source_stages.json": + return None raise ValueError( f"{source_manifest}: stage {stage_name!r} needs base_candidate." ) @@ -308,6 +310,17 @@ def _parse_family_coverage( f"{resource}: family {name!r} needs a reviewed " "required_mass_change_reason." ) + mass_change_semantics = str( + raw_family.get("mass_change_semantics", "mass_conserving") + ).strip() + if mass_change_semantics not in { + "mass_conserving", + "mass_increasing_support", + }: + raise ValueError( + f"{resource}: family {name!r} has invalid " + f"mass_change_semantics {mass_change_semantics!r}." + ) raw_requirements = raw_family.get("effective_mass_requirements", {}) if not isinstance(raw_requirements, Mapping): @@ -371,6 +384,7 @@ def _parse_family_coverage( "base_candidate_tier": base_candidate_tier, "output_weight_kind": output_weight_kind, "required_mass_change_reason": required_mass_change_reason, + "mass_change_semantics": mass_change_semantics, "effective_mass_requirements": requirements, } return families @@ -931,20 +945,27 @@ def _family_build_state_diagnostics( required_reason = str(family.get("required_mass_change_reason", "")).strip() if required_reason: + semantics = str(family.get("mass_change_semantics", "mass_conserving")) records = tuple(getattr(frame, "mass_log", ())) matches = [ record for record in records if _mass_record_field(record, "reason") == required_reason ] - valid_matches = [record for record in matches if _valid_mass_record(record)] + valid_matches = [ + record + for record in matches + if _valid_mass_record(record, semantics=semantics) + ] details["required_mass_change_reason"] = required_reason + details["mass_change_semantics"] = semantics details["matching_mass_change_records"] = len(matches) details["valid_mass_change_records"] = len(valid_matches) if not valid_matches: failures.append( f"{family_name}: final dataset lacks the reviewed, " - "mass-conserving household MassChangeRecord carrying its " + f"{semantics.replace('_', '-')} household MassChangeRecord " + "carrying its " f"declared reason: {required_reason!r}." ) @@ -958,24 +979,30 @@ def _mass_record_field(record: object, name: str) -> object: return getattr(record, name, None) -def _valid_mass_record(record: object) -> bool: +def _valid_mass_record(record: object, *, semantics: str) -> bool: old_total = _mass_record_field(record, "old_total") new_total = _mass_record_field(record, "new_total") declared_factor = _mass_record_field(record, "declared_factor") try: old = float(old_total) new = float(new_total) - factor = float(declared_factor) except (TypeError, ValueError): return False - return bool( + common = bool( _mass_record_field(record, "entity") == "household" and np.isfinite(old) and old > 0.0 and np.isfinite(new) - and np.isclose(old, new, rtol=1e-9, atol=0.0) - and factor == 1.0 ) + if not common: + return False + if semantics == "mass_increasing_support": + return bool(new > old and declared_factor is None) + try: + factor = float(declared_factor) + except (TypeError, ValueError): + return False + return bool(np.isclose(old, new, rtol=1e-9, atol=0.0) and factor == 1.0) def uk_release_input_coverage_gate( diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/salary_sacrifice.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/salary_sacrifice.py new file mode 100644 index 000000000..1e927b4d8 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/salary_sacrifice.py @@ -0,0 +1,340 @@ +"""UK salary-sacrifice QRF and headcount support conversion.""" + +from __future__ import annotations + +import json +from collections.abc import Mapping +from dataclasses import dataclass, field +from importlib import import_module +from importlib.resources import files +from typing import Any + +import numpy as np +import pandas as pd + +from microcosm.build.source_manifest import SourceOperationSpec, SourceStageSpec +from microcosm.build.stochastic_assignment import stable_identity_uniforms +from microcosm.build.uk_runtime.cgt_structure import ( + HOUSEHOLD_IS_CGT_BAND_DONOR, + HOUSEHOLD_IS_CGT_CLONE, +) +from microcosm.build.uk_runtime.national_frame import ( + uk_household_weight_kind, + uk_national_frame, + uk_time_period, + validate_uk_national_frame, +) +from microcosm.build.uk_runtime.spi_support import support_channel_column +from microcosm.frame import Frame + +QRF: Any | None = None + +SALSAC_PREDICTORS = ("age", "employment_income") +SALSAC_OUTPUT = "pension_contributions_via_salary_sacrifice" +SALSAC_STAGE_TARGET = 5_400_000.0 +SALSAC_HMRC_ANCHOR = 7_700_000.0 +SALSAC_ABOVE_2000_ANCHOR = 3_300_000.0 +SALSAC_BELOW_2000_ANCHOR = 4_300_000.0 +SALSAC_STAGING_RATIO = SALSAC_STAGE_TARGET / SALSAC_HMRC_ANCHOR +SALSAC_RATE_CAP = 0.5 +SALSAC_QRF_SEED = 42 +SALSAC_QRF_ESTIMATORS = 100 +SALSAC_CONVERSION_SEED = 2024 +SALSAC_CONVERSION_SALT = "salary_sacrifice_conversion" + + +def load_salary_sacrifice_anchor() -> Mapping[str, Any]: + """Load the committed HMRC salary-sacrifice anchor.""" + + return json.loads( + files("microcosm.build.uk") + .joinpath("salary_sacrifice_anchor.json") + .read_text(encoding="utf-8") + ) + + +@dataclass(frozen=True) +class UKSalarySacrificeResult: + """Transformed frame and the full headcount executed-effect receipt.""" + + frame: Frame + training_rows: int + prediction_rows: int + pre_headcount: float + post_headcount: float + shortfall: float + donor_pool_mass: float + rate: float + cap_bound: bool + converted_rows: int + converted_mass: float + moved_amount: float + + def evidence(self) -> dict[str, object]: + return { + "stage": "salary_sacrifice", + "qrf": { + "training_rows": self.training_rows, + "prediction_rows": self.prediction_rows, + "seed": SALSAC_QRF_SEED, + }, + "headcount_receipt": { + "target": SALSAC_STAGE_TARGET, + "pre_headcount": self.pre_headcount, + "post_headcount": self.post_headcount, + "shortfall": self.shortfall, + "donor_pool_mass": self.donor_pool_mass, + "rate": self.rate, + "rate_cap": SALSAC_RATE_CAP, + "cap_bound": self.cap_bound, + "converted_rows": self.converted_rows, + "converted_mass": self.converted_mass, + "moved_amount": self.moved_amount, + }, + } + + +@dataclass(frozen=True) +class UKSalarySacrificeStageTransform: + """Whole-stage transform for salary-sacrifice support.""" + + stage: SourceStageSpec + anchor: Mapping[str, Any] | None = None + last_result: UKSalarySacrificeResult | None = field(default=None, init=False) + + def __call__(self, frame: Frame) -> Frame: + resource = self.anchor or load_salary_sacrifice_anchor() + _assert_salary_sacrifice_stage_parameters(self.stage, anchor=resource) + result = impute_salary_sacrifice(frame) + object.__setattr__(self, "last_result", result) + return result.frame + + @staticmethod + def output_columns() -> tuple[str, ...]: + return (SALSAC_OUTPUT, "employee_pension_contributions") + + def checkpoint_metadata(self) -> dict[str, object]: + if self.last_result is None: + raise RuntimeError("checkpoint metadata requires a completed stage run.") + return {"evidence": self.last_result.evidence()} + + +def impute_salary_sacrifice(frame: Frame) -> UKSalarySacrificeResult: + """Fit on the asked base-FRS subset, then create additional SS support.""" + + validate_uk_national_frame(frame) + person = frame.table("person").copy() + household = frame.table("household").copy() + required_person = { + "person_id", + "person_household_id", + "age", + "employment_income", + "salary_sacrifice_asked", + SALSAC_OUTPUT, + "employee_pension_contributions", + } + missing = sorted(required_person - set(person.columns)) + if missing: + raise ValueError(f"Salary-sacrifice person columns missing: {missing}.") + households = household.set_index("household_id") + person_households = person["person_household_id"] + if not person_households.isin(households.index).all(): + raise ValueError("Salary-sacrifice people must map to a household.") + channel_column = support_channel_column("household") + if channel_column not in household.columns: + raise ValueError( + f"Salary-sacrifice training requires household {channel_column!r}." + ) + channels = person_households.map(households[channel_column]) + clones = person_households.map( + households.get(HOUSEHOLD_IS_CGT_CLONE, pd.Series(False, index=households.index)) + ).fillna(False) + donors = person_households.map( + households.get( + HOUSEHOLD_IS_CGT_BAND_DONOR, + pd.Series(False, index=households.index), + ) + ).fillna(False) + asked = pd.to_numeric(person["salary_sacrifice_asked"], errors="coerce") + if asked.isna().any(): + raise ValueError("salary_sacrifice_asked contains non-numeric values.") + training_mask = ( + channels.eq("frs") & ~clones.astype(bool) & ~donors.astype(bool) & asked.eq(1) + ) + if not training_mask.any(): + raise ValueError("Salary-sacrifice QRF has no eligible asked FRS rows.") + predict_mask = ~asked.eq(1) + numeric = person.loc[:, [*SALSAC_PREDICTORS, SALSAC_OUTPUT]].apply( + pd.to_numeric, errors="coerce" + ) + if not np.isfinite(numeric.to_numpy(dtype=float)).all(): + raise ValueError("Salary-sacrifice QRF columns must be finite numeric values.") + household_weights = pd.Series( + frame.weights_for("household").values, + index=household["household_id"], + ) + person_weights = person_households.map(household_weights).to_numpy(dtype=float) + training = numeric.loc[training_mask, [*SALSAC_PREDICTORS, SALSAC_OUTPUT]].copy() + training["_fit_weight"] = person_weights[training_mask.to_numpy()] + model = _qrf_class()(n_estimators=SALSAC_QRF_ESTIMATORS, seed=SALSAC_QRF_SEED) + fitted = model.fit( + training, + list(SALSAC_PREDICTORS), + [SALSAC_OUTPUT], + weights="_fit_weight", + ) + if predict_mask.any(): + predictions = fitted.predict(numeric.loc[predict_mask, list(SALSAC_PREDICTORS)]) + predicted = pd.to_numeric(predictions[SALSAC_OUTPUT], errors="coerce").to_numpy( + dtype=float + ) + if not np.isfinite(predicted).all(): + raise ValueError("Salary-sacrifice QRF produced non-finite predictions.") + person.loc[predict_mask, SALSAC_OUTPUT] = np.maximum(0.0, predicted) + final_ss = pd.to_numeric(person[SALSAC_OUTPUT], errors="coerce").to_numpy( + dtype=float, copy=True + ) + employee = pd.to_numeric( + person["employee_pension_contributions"], errors="coerce" + ).to_numpy(dtype=float, copy=True) + employment_income = pd.to_numeric( + person["employment_income"], errors="coerce" + ).to_numpy(dtype=float) + if not np.isfinite(final_ss).all() or (final_ss < 0.0).any(): + raise ValueError("Salary-sacrifice amounts must be finite and non-negative.") + if not np.isfinite(employee).all() or (employee < 0.0).any(): + raise ValueError("Employee-pension amounts must be finite and non-negative.") + has_ss = final_ss > 0.0 + pre_headcount = float(person_weights[has_ss].sum()) + shortfall = max(0.0, SALSAC_STAGE_TARGET - pre_headcount) + donor_pool = (employee > 0.0) & ~has_ss & (employment_income > 0.0) + donor_pool_mass = float(person_weights[donor_pool].sum()) + uncapped_rate = shortfall / donor_pool_mass if donor_pool_mass > 0.0 else 0.0 + rate = min(SALSAC_RATE_CAP, uncapped_rate) + draws = stable_identity_uniforms( + person["person_id"].to_numpy(), + seed=SALSAC_CONVERSION_SEED, + salt=SALSAC_CONVERSION_SALT, + ) + converted = donor_pool & (draws < rate) + moved_amount = float(employee[converted].sum()) + final_ss[converted] = employee[converted] + employee[converted] = 0.0 + person[SALSAC_OUTPUT] = final_ss + person["employee_pension_contributions"] = employee + post_headcount = float(person_weights[final_ss > 0.0].sum()) + converted_mass = float(person_weights[converted].sum()) + result_frame = uk_national_frame( + person=person, + benunit=frame.table("benunit").copy(), + household=household, + time_period=uk_time_period(frame), + weight_kind=uk_household_weight_kind(frame), + household_weights=frame.weights_for("household").values, + mass_log=frame.mass_log, + ) + validate_uk_national_frame(result_frame) + return UKSalarySacrificeResult( + frame=result_frame, + training_rows=int(training_mask.sum()), + prediction_rows=int(predict_mask.sum()), + pre_headcount=pre_headcount, + post_headcount=post_headcount, + shortfall=shortfall, + donor_pool_mass=donor_pool_mass, + rate=rate, + cap_bound=uncapped_rate > SALSAC_RATE_CAP, + converted_rows=int(converted.sum()), + converted_mass=converted_mass, + moved_amount=moved_amount, + ) + + +def _qrf_class(): + if QRF is not None: + return QRF + return import_module("microcosm.fit").QRF + + +def _operation(stage: SourceStageSpec, kind: str) -> SourceOperationSpec: + matches = [operation for operation in stage.operations if operation.kind == kind] + if len(matches) != 1: + raise ValueError( + f"Stage {stage.stage!r} must declare exactly one {kind!r} operation." + ) + return matches[0] + + +def _assert_parameters( + operation: SourceOperationSpec, + expected: Mapping[str, object], +) -> None: + for name, value in expected.items(): + actual = operation.parameters.get(name) + if actual != value: + raise ValueError( + f"{operation.kind} manifest parameter {name!r} drifted: " + f"expected {value!r}, got {actual!r}." + ) + + +def _assert_salary_sacrifice_stage_parameters( + stage: SourceStageSpec, + *, + anchor: Mapping[str, Any], +) -> None: + """Bind all stage parameters; result evidence supplies arm 2.""" + + _assert_parameters( + _operation(stage, "fit_weighted_qrf"), + { + "training_population": "support_channel == frs and not capital-gains clone and not CGT band donor and salary_sacrifice_asked == 1", + "target_population": "salary_sacrifice_asked != 1 frame-wide", + "predictors": list(SALSAC_PREDICTORS), + "targets": [SALSAC_OUTPUT], + "weights": "household_weight", + "weight_mapping": "household_to_person", + "seed": SALSAC_QRF_SEED, + "n_estimators": SALSAC_QRF_ESTIMATORS, + "clamp_minimum": 0, + "preserve_asked_rows": True, + "cache": False, + }, + ) + _assert_parameters( + _operation(stage, "convert_donors_to_target_stock"), + { + "resource": "salary_sacrifice_anchor.json", + "target": int(SALSAC_STAGE_TARGET), + "donor_pool": "employee_pension_contributions > 0 and pension_contributions_via_salary_sacrifice == 0 and employment_income > 0", + "rate_cap": SALSAC_RATE_CAP, + "move": "full employee_pension_contributions to pension_contributions_via_salary_sacrifice; source zeroed", + "seed": SALSAC_CONVERSION_SEED, + "salt": SALSAC_CONVERSION_SALT, + "receipt": "weighted_headcount", + }, + ) + hmrc = anchor.get("hmrc_anchor", {}) + derived = anchor.get("derived", {}) + checks = { + "hmrc_anchor.total_users": (hmrc.get("total_users"), SALSAC_HMRC_ANCHOR), + "hmrc_anchor.above_2000": ( + hmrc.get("above_2000"), + SALSAC_ABOVE_2000_ANCHOR, + ), + "hmrc_anchor.below_2000": ( + hmrc.get("below_2000"), + SALSAC_BELOW_2000_ANCHOR, + ), + "derived.stage_target": (derived.get("stage_target"), SALSAC_STAGE_TARGET), + } + for label, (actual, expected) in checks.items(): + if actual != expected: + raise ValueError( + f"Salary-sacrifice resource {label} drifted: expected " + f"{expected!r}, got {actual!r}." + ) + ratio = float(derived.get("staging_ratio", np.nan)) + if not np.isclose(ratio, SALSAC_STAGING_RATIO, rtol=0.0, atol=1e-12): + raise ValueError("Salary-sacrifice resource staging ratio drifted.") diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/source_runtime.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/source_runtime.py index 573236568..a86e89fa3 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/source_runtime.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/source_runtime.py @@ -69,6 +69,11 @@ def uk_stage_implementations( frs_hmrc_spine_leaves_transform: Callable[[Frame], Frame] | None = None, spi_support_channel_transform: Callable[[Frame], Frame] | None = None, hmrc_spi_income_spine_transform: Callable[[Frame], Frame] | None = None, + cgt_incidence_clone_transform: Callable[[Frame], Frame] | None = None, + cgt_band_donors_transform: Callable[[Frame], Frame] | None = None, + hmrc_cgt_gains_spine_transform: Callable[[Frame], Frame] | None = None, + salary_sacrifice_transform: Callable[[Frame], Frame] | None = None, + student_loans_transform: Callable[[Frame], Frame] | None = None, ) -> dict[str, Callable[[Frame], Frame]]: """Return the whole-stage implementation map for the UK source plan.""" @@ -96,6 +101,11 @@ def uk_stage_implementations( "frs_hmrc_spine_leaves": frs_hmrc_spine_leaves_transform, "spi_support_channel": spi_support_channel_transform, "hmrc_spi_income_spine": hmrc_spi_income_spine_transform, + "cgt_incidence_clone": cgt_incidence_clone_transform, + "cgt_band_donors": cgt_band_donors_transform, + "hmrc_cgt_gains_spine": hmrc_cgt_gains_spine_transform, + "salary_sacrifice": salary_sacrifice_transform, + "student_loans": student_loans_transform, } implementations.update( { diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py index 6acb9a3bb..4b1043cb5 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py @@ -162,7 +162,10 @@ # Reviewed constants for every load-bearing manifest parameter the spine # transforms consume. The drift asserts below fail closed on any manifest-only # edit (adversarial-review finding on #717): a manifest change to these values -# requires a matching reviewed code change here. +# requires a matching reviewed code change here. The #730/#684 two-arm rule +# applies to every declared parameter: it needs (1) a drift assert and (2) an +# executed-effect receipt, or an explicit absence statement. Seeded draws use +# twin-build determinism as their executed-effect receipt. SPI_SPINE_STAGE1_PREDICTORS = ("age", "gender", "region") SPI_SPINE_STAGE2_PREDICTORS = ( "age", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/student_loans.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/student_loans.py new file mode 100644 index 000000000..594f874ef --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/student_loans.py @@ -0,0 +1,369 @@ +"""UK student-loan cohort assignment and SLC liable-stock support top-ups.""" + +from __future__ import annotations + +import json +from collections.abc import Mapping +from dataclasses import dataclass, field +from importlib.resources import files +from typing import Any + +import numpy as np +import pandas as pd + +from microcosm.build.source_manifest import SourceOperationSpec, SourceStageSpec +from microcosm.build.stochastic_assignment import stable_identity_uniforms +from microcosm.build.uk_runtime.frs_release import load_uk_frs_release +from microcosm.build.uk_runtime.national_frame import ( + uk_household_weight_kind, + uk_national_frame, + uk_time_period, + validate_uk_national_frame, +) +from microcosm.frame import Frame + +PLAN_1_BEFORE = 2012 +PLAN_5_FROM = 2023 +PLAN_2_MIN_AGE = 21 +PLAN_2_MAX_AGE = 55 +PLAN_5_MIN_AGE = 18 +PLAN_5_MAX_AGE = 25 +PLAN_PRIORITY = ("PLAN_5", "PLAN_2") +STUDENT_LOAN_SEED = 42 +YEAR_RULE = "calibration_year" +STUDENT_LOAN_ENUM_DOMAIN = ("NONE", "PLAN_1", "PLAN_2", "PLAN_5") +EXCLUDED_ENGLAND_REGIONS = ("SCOTLAND", "WALES", "NORTHERN_IRELAND") +PLAN_SALTS = { + "PLAN_5": "student_loan_plan_5", + "PLAN_2": "student_loan_plan_2", +} +PLAN_2025_STOCKS = {"PLAN_2": 8_940_000.0, "PLAN_5": 10_000.0} + + +def load_slc_liable_stocks() -> Mapping[str, Any]: + """Load the pinned SLC Table 6a liable-stock series.""" + + return json.loads( + files("microcosm.build.uk") + .joinpath("slc_liable_stocks.json") + .read_text(encoding="utf-8") + ) + + +@dataclass(frozen=True) +class UKStudentLoanPlanReceipt: + plan: str + stock: float + reported_count: float + reported_england_count: float + shortfall: float + eligible_mass: float + rate: float + topped_up_rows: int + topped_up_mass: float + final_england_count: float + + def evidence(self) -> dict[str, object]: + return { + "stock": self.stock, + "reported_count": self.reported_count, + "reported_england_count": self.reported_england_count, + "shortfall": self.shortfall, + "eligible_mass": self.eligible_mass, + "rate": self.rate, + "topped_up_rows": self.topped_up_rows, + "topped_up_mass": self.topped_up_mass, + "final_england_count": self.final_england_count, + } + + +@dataclass(frozen=True) +class UKStudentLoansResult: + """Transformed frame plus a per-plan executed-effect receipt.""" + + frame: Frame + calibration_year: int + plans: Mapping[str, UKStudentLoanPlanReceipt] + + def evidence(self) -> dict[str, object]: + return { + "stage": "student_loans", + "year_rule": YEAR_RULE, + "calibration_year": self.calibration_year, + "plans": {name: receipt.evidence() for name, receipt in self.plans.items()}, + } + + +@dataclass(frozen=True) +class UKStudentLoansStageTransform: + """Whole-stage transform for student-loan plan support.""" + + stage: SourceStageSpec + stocks: Mapping[str, Any] | None = None + calibration_year: int | None = None + last_result: UKStudentLoansResult | None = field(default=None, init=False) + + def __call__(self, frame: Frame) -> Frame: + resource = self.stocks or load_slc_liable_stocks() + year = ( + self.calibration_year + if self.calibration_year is not None + else load_uk_frs_release().calibration_year + ) + _assert_student_loans_stage_parameters(self.stage, stocks=resource, year=year) + result = assign_student_loan_plans(frame, stocks=resource, year=year) + object.__setattr__(self, "last_result", result) + return result.frame + + @staticmethod + def output_columns() -> tuple[str, ...]: + return ("student_loan_plan",) + + def checkpoint_metadata(self) -> dict[str, object]: + if self.last_result is None: + raise RuntimeError("checkpoint metadata requires a completed stage run.") + return {"evidence": self.last_result.evidence()} + + +def assign_student_loan_plans( + frame: Frame, + *, + stocks: Mapping[str, Any], + year: int, +) -> UKStudentLoansResult: + """Assign reported cohorts, then top up PLAN_5 before PLAN_2.""" + + validate_uk_national_frame(frame) + person = frame.table("person").copy() + household = frame.table("household").copy() + required = { + "person_id", + "person_household_id", + "age", + "student_loan_repayments", + "highest_education", + } + missing = sorted(required - set(person.columns)) + if missing: + raise ValueError(f"Student-loan person columns missing: {missing}.") + if "region" not in household.columns: + raise ValueError("Student-loan assignment requires household region.") + region_by_household = household.set_index("household_id")["region"] + region = person["person_household_id"].map(region_by_household) + if region.isna().any(): + raise ValueError("Student-loan people must all map to a household region.") + region_names = np.asarray([_enum_name(value) for value in region], dtype=object) + is_england = ~np.isin(region_names, EXCLUDED_ENGLAND_REGIONS) + education = np.asarray( + [_enum_name(value) for value in person["highest_education"]], dtype=object + ) + age = pd.to_numeric(person["age"], errors="coerce").to_numpy(dtype=float) + repayments = pd.to_numeric( + person["student_loan_repayments"], errors="coerce" + ).to_numpy(dtype=float) + if not np.isfinite(age).all() or not np.isfinite(repayments).all(): + raise ValueError("Student-loan age and repayment inputs must be finite.") + weights_by_household = pd.Series( + frame.weights_for("household").values, + index=household["household_id"], + ) + person_weights = ( + person["person_household_id"].map(weights_by_household).to_numpy(dtype=float) + ) + start_year = year - age + 18 + has_repayments = repayments > 0.0 + plan = np.full(len(person), "NONE", dtype=object) + plan[has_repayments & (start_year < PLAN_1_BEFORE)] = "PLAN_1" + plan[has_repayments & (start_year >= PLAN_5_FROM)] = "PLAN_5" + plan[has_repayments & (plan == "NONE")] = "PLAN_2" + reported_plan = plan.copy() + receipts: dict[str, UKStudentLoanPlanReceipt] = {} + for plan_name in PLAN_PRIORITY: + stock = _stock(stocks, plan_name, year) + current_england = float(person_weights[(plan == plan_name) & is_england].sum()) + shortfall = max(0.0, stock - current_england) + eligible = ( + (plan == "NONE") + & is_england + & (education == "TERTIARY") + & _plan_age_cohort_eligibility(plan_name, age=age, start_year=start_year) + ) + eligible_mass = float(person_weights[eligible].sum()) + rate = min(1.0, shortfall / eligible_mass) if eligible_mass > 0.0 else 0.0 + draws = stable_identity_uniforms( + person["person_id"].to_numpy(), + seed=STUDENT_LOAN_SEED, + salt=PLAN_SALTS[plan_name], + ) + topped_up = eligible & (draws < rate) + plan[topped_up] = plan_name + receipts[plan_name] = UKStudentLoanPlanReceipt( + plan=plan_name, + stock=stock, + reported_count=float(person_weights[reported_plan == plan_name].sum()), + reported_england_count=float( + person_weights[(reported_plan == plan_name) & is_england].sum() + ), + shortfall=shortfall, + eligible_mass=eligible_mass, + rate=rate, + topped_up_rows=int(topped_up.sum()), + topped_up_mass=float(person_weights[topped_up].sum()), + final_england_count=float( + person_weights[(plan == plan_name) & is_england].sum() + ), + ) + unknown = sorted(set(plan) - set(STUDENT_LOAN_ENUM_DOMAIN)) + if unknown: + raise ValueError(f"Student-loan assignment emitted unknown plan(s): {unknown}.") + person["student_loan_plan"] = plan + result_frame = uk_national_frame( + person=person, + benunit=frame.table("benunit").copy(), + household=household, + time_period=uk_time_period(frame), + weight_kind=uk_household_weight_kind(frame), + household_weights=frame.weights_for("household").values, + mass_log=frame.mass_log, + ) + validate_uk_national_frame(result_frame) + return UKStudentLoansResult( + frame=result_frame, + calibration_year=year, + plans=receipts, + ) + + +def _plan_age_cohort_eligibility( + plan: str, + *, + age: np.ndarray, + start_year: np.ndarray, +) -> np.ndarray: + if plan == "PLAN_5": + return ( + (age >= PLAN_5_MIN_AGE) + & (age <= PLAN_5_MAX_AGE) + & (start_year >= PLAN_5_FROM) + ) + if plan == "PLAN_2": + return ( + (age >= PLAN_2_MIN_AGE) + & (age <= PLAN_2_MAX_AGE) + & (start_year >= PLAN_1_BEFORE) + & (start_year < PLAN_5_FROM) + ) + raise ValueError(f"Unsupported student-loan top-up plan {plan!r}.") + + +def _stock(stocks: Mapping[str, Any], plan: str, year: int) -> float: + key = plan.lower() + try: + values = stocks["plans"][key]["liable"] + value = values[str(year)] + except (KeyError, TypeError) as error: + raise ValueError( + f"SLC liable-stock resource has no {key} value for {year}." + ) from error + result = float(value) + if result < 0 or not np.isfinite(result): + raise ValueError(f"SLC {key} liable stock for {year} is invalid: {value!r}.") + return result + + +def _enum_name(value: object) -> str: + if hasattr(value, "name"): + return str(value.name) + text = str(value) + return text.rsplit(".", 1)[-1] + + +def _operation(stage: SourceStageSpec, kind: str) -> SourceOperationSpec: + matches = [operation for operation in stage.operations if operation.kind == kind] + if len(matches) != 1: + raise ValueError( + f"Stage {stage.stage!r} must declare exactly one {kind!r} operation." + ) + return matches[0] + + +def _assert_parameters( + operation: SourceOperationSpec, + expected: Mapping[str, object], +) -> None: + for name, value in expected.items(): + actual = operation.parameters.get(name) + if actual != value: + raise ValueError( + f"{operation.kind} manifest parameter {name!r} drifted: " + f"expected {value!r}, got {actual!r}." + ) + + +def _assert_student_loans_stage_parameters( + stage: SourceStageSpec, + *, + stocks: Mapping[str, Any], + year: int, +) -> None: + """Bind every stage parameter; per-plan receipts supply arm 2.""" + + _assert_parameters( + _operation(stage, "assign_student_loan_plan_cohorts"), + { + "year_rule": YEAR_RULE, + "start_year_formula": "year - age + 18", + "reported_repayment_test": "student_loan_repayments > 0", + "reported_country_gate": False, + "plan_1_before": PLAN_1_BEFORE, + "plan_5_from": PLAN_5_FROM, + "enum_domain": list(STUDENT_LOAN_ENUM_DOMAIN), + "plan_4_imputation": False, + }, + ) + top_ups = [ + operation + for operation in stage.operations + if operation.kind == "top_up_to_stock" + ] + if [operation.parameters.get("plan") for operation in top_ups] != list( + PLAN_PRIORITY + ): + raise ValueError( + "Student-loan top-up priority drifted from PLAN_5 then PLAN_2." + ) + expected = { + "PLAN_5": { + "priority": 1, + "stock_series": "plan_5.liable", + "age_min": PLAN_5_MIN_AGE, + "age_max": PLAN_5_MAX_AGE, + "cohort_start_min": PLAN_5_FROM, + "salt": PLAN_SALTS["PLAN_5"], + }, + "PLAN_2": { + "priority": 2, + "stock_series": "plan_2.liable", + "age_min": PLAN_2_MIN_AGE, + "age_max": PLAN_2_MAX_AGE, + "cohort_start_min": PLAN_1_BEFORE, + "cohort_start_max_exclusive": PLAN_5_FROM, + "salt": PLAN_SALTS["PLAN_2"], + }, + } + for operation in top_ups: + plan = str(operation.parameters["plan"]) + shared = { + "resource": "slc_liable_stocks.json", + "year_rule": YEAR_RULE, + "eligible_region_exclusions": list(EXCLUDED_ENGLAND_REGIONS), + "highest_education": "TERTIARY", + "seed": STUDENT_LOAN_SEED, + } + _assert_parameters(operation, {**shared, **expected[plan]}) + if year == 2025: + for plan, expected_stock in PLAN_2025_STOCKS.items(): + if _stock(stocks, plan, year) != expected_stock: + raise ValueError( + f"SLC {plan} liable stock for 2025 drifted from {expected_stock}." + ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/terminal_gates.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/terminal_gates.py index cefeeb86e..3d39e4a8b 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/terminal_gates.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/terminal_gates.py @@ -168,6 +168,7 @@ def __post_init__(self) -> None: "household.gas_consumption", "household.has_fuel_consumption", "household.household_is_capital_gains_clone", + "household.household_is_cgt_band_donor", "household.household_is_spi_synthetic", "household.la_code_oa", "household.lsoa_code", diff --git a/packages/microcosm-build/tests/test_country_spec.py b/packages/microcosm-build/tests/test_country_spec.py index a7998c235..e3cf5778a 100644 --- a/packages/microcosm-build/tests/test_country_spec.py +++ b/packages/microcosm-build/tests/test_country_spec.py @@ -284,6 +284,11 @@ def test_spi_spine_adds_no_country_package_resources(self) -> None: "frs_release.json", "gates.json", "brma_rent_counts.json", + "hmrc_cgt_size_bands.json", + "advani_summers_capital_gains_distribution.json", + "salary_sacrifice_anchor.json", + "slc_liable_stocks.json", + "cgt_band_donor_support_bounds.json", "hmrc_income_release_gate_report.json", "hmrc_income_replay_report.json", "hmrc_income_source_stages.json", @@ -313,11 +318,11 @@ def test_spi_spine_adds_no_country_package_resources(self) -> None: "target_reference_membership.json", ) - def test_uk_source_manifest_loads_twenty_one_stages(self) -> None: + def test_uk_source_manifest_loads_twenty_six_stages(self) -> None: spec = load_country_spec("uk") assert spec.sources is not None - assert len(spec.sources.stages) == 21 + assert len(spec.sources.stages) == 26 class TestExistingPackagesGeneralize: @@ -350,6 +355,11 @@ def test_uk_package_loads(self) -> None: "frs_release.json", "gates.json", "brma_rent_counts.json", + "hmrc_cgt_size_bands.json", + "advani_summers_capital_gains_distribution.json", + "salary_sacrifice_anchor.json", + "slc_liable_stocks.json", + "cgt_band_donor_support_bounds.json", "hmrc_income_release_gate_report.json", "hmrc_income_replay_report.json", "hmrc_income_source_stages.json", @@ -620,6 +630,7 @@ def test_declares_the_full_june_battery(self, manifest) -> None: "uk_export_surface", "uk_take_up_signal", "uk_brma_enum_domain", + "uk_student_loan_plan_enum_domain", "uk_calibration_reference_coverage", "uk_target_surface", "uk_target_fit", diff --git a/packages/microcosm-build/tests/test_plan.py b/packages/microcosm-build/tests/test_plan.py index b7e7a30e0..d3cbcfda5 100644 --- a/packages/microcosm-build/tests/test_plan.py +++ b/packages/microcosm-build/tests/test_plan.py @@ -56,6 +56,25 @@ def test_two_producers_of_one_column_refused(self) -> None: ] ) + def test_explicit_rewrite_may_follow_canonical_producer(self) -> None: + plan = StagePlan( + [ + Stage( + name="produce", + transform=lambda frame: frame, + produces=("net_worth",), + ), + Stage( + name="rewrite", + transform=lambda frame: frame, + produces=("net_worth",), + rewrites=("net_worth",), + ), + ] + ) + + assert [stage.name for stage in plan.stages] == ["produce", "rewrite"] + def test_empty_donor_fields_refused(self) -> None: with pytest.raises(ValueError, match="source citation is required"): DonorSpec(survey="SCF", source="") diff --git a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py index 617eff559..62aaf7195 100644 --- a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py +++ b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py @@ -42,7 +42,7 @@ ), ( "uk", - "e12a2cb87c0e096af0173bd51fbede4b7df5e7c118ffe07ef616bbb30640e4ea", + "e70e4a9f7d43a1c7998f01e33b9d46b59f1c8b3ee9ca7158f5ea7235be58e4aa", { "benunit.benunit_id", "household.household_id", diff --git a/packages/microcosm-build/tests/test_uk_battery_bindings.py b/packages/microcosm-build/tests/test_uk_battery_bindings.py index 0444e691d..e33078296 100644 --- a/packages/microcosm-build/tests/test_uk_battery_bindings.py +++ b/packages/microcosm-build/tests/test_uk_battery_bindings.py @@ -451,9 +451,9 @@ def test_fully_armed_battery_evaluates_gate_for_gate(self) -> None: ] # 11 as on main (uk_nonnegative_columns passes with zero required # columns — the scheduled stages declare none), the two E4 stochastic - # gates, the E5 support gate, and the E6 aggregate-admin gate; their - # evaluators have direct tests. - assert len(passed) == 15 + # gates, the E5 support gate, the E6 aggregate-admin gate, and the E8 + # student-loan enum gate; their evaluators have direct tests. + assert len(passed) == 16 qrf = by_id["uk_qrf_tail_concentration"] assert qrf.status is GateStatus.FAILED assert "declared QRF output is absent" in qrf.result.failures[0] diff --git a/packages/microcosm-build/tests/test_uk_cgt_source_manifest.py b/packages/microcosm-build/tests/test_uk_cgt_source_manifest.py index d0d6f281c..0fadad94e 100644 --- a/packages/microcosm-build/tests/test_uk_cgt_source_manifest.py +++ b/packages/microcosm-build/tests/test_uk_cgt_source_manifest.py @@ -11,6 +11,10 @@ UK_CGT_MASS_CONSERVATION_REASON, UK_CGT_TAXABLE_INCOME_PROXY_COMPONENTS, ) +from microcosm.build.uk_runtime.cgt_structure import ( + CGT_CLONE_MASS_CHANGE_REASON, + CGT_DONOR_MASS_CHANGE_REASON, +) from microcosm.build.uk_runtime.hmrc_capital_gains import ( HMRC_CGT_JOINT_ODS_SHA256, HMRC_CGT_JOINT_ODS_SIZE_BYTES, @@ -130,6 +134,20 @@ def test_the_shipped_family_contracts_pass_the_terminal_gate_shape() -> None: declared_factor=1.0, reason=spi_reason, ), + MassChangeRecord( + entity="household", + old_total=100.0, + new_total=100.0, + declared_factor=1.0, + reason=CGT_CLONE_MASS_CHANGE_REASON, + ), + MassChangeRecord( + entity="household", + old_total=100.0, + new_total=110.0, + declared_factor=None, + reason=CGT_DONOR_MASS_CHANGE_REASON, + ), MassChangeRecord( entity="household", old_total=100.0, diff --git a/packages/microcosm-build/tests/test_uk_cgt_structure.py b/packages/microcosm-build/tests/test_uk_cgt_structure.py new file mode 100644 index 000000000..e27606764 --- /dev/null +++ b/packages/microcosm-build/tests/test_uk_cgt_structure.py @@ -0,0 +1,278 @@ +from __future__ import annotations + +import copy +from dataclasses import replace +from functools import lru_cache + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.country_spec import load_country_spec +from microcosm.build.source_manifest import SourceOperationSpec +from microcosm.build.uk_runtime.cgt_imputation import ( + UK_CGT_TAXABLE_INCOME_PROXY_COMPONENTS, +) +from microcosm.build.uk_runtime.cgt_structure import ( + CGT_CLONE_MASS_CHANGE_REASON, + DONOR_BAND_COUNT, + DONOR_TOTAL, + DONORS_PER_BAND, + HOUSEHOLD_IS_CGT_BAND_DONOR, + HOUSEHOLD_IS_CGT_CLONE, + MIN_DONOR_BAND_LOWER, + _assert_cgt_donor_stage_parameters, + _assert_cgt_incidence_stage_parameters, + _draw_banded_priors, + clone_cgt_incidence, + load_hmrc_cgt_size_bands, + stack_cgt_band_donors, +) +from microcosm.build.uk_runtime.national_frame import uk_national_frame +from microcosm.frame import WeightKind + + +def _distribution(*, negative: bool = False) -> dict[str, object]: + knots = [-70.0, -60.0, -40.0, -20.0, -10.0, -5.0, -1.0] + if not negative: + knots = [-10.0, 0.0, 25.0, 50.0, 75.0, 100.0, 125.0] + return { + "rows": [ + { + "minimum_total_income": 0, + "percent_with_gains": 1.0, + **dict( + zip( + ("p05", "p10", "p25", "p50", "p75", "p90", "p95"), + knots, + strict=True, + ) + ), + } + ] + } + + +def _one_person_households(n: int, *, reverse_people: bool = False): + ids = np.arange(1, n + 1, dtype="int64") + person = pd.DataFrame( + { + "person_id": ids, + "person_benunit_id": ids, + "person_household_id": ids, + "age": np.full(n, 40), + "capital_gains": np.zeros(n), + "employment_income": np.linspace(10_000.0, 100_000.0, n), + } + ) + for column in UK_CGT_TAXABLE_INCOME_PROXY_COMPONENTS: + if column not in person: + person[column] = 0.0 + if reverse_people: + person = person.iloc[::-1].reset_index(drop=True) + benunit = pd.DataFrame({"benunit_id": ids}) + household = pd.DataFrame( + { + "household_id": ids, + "household_support_channel": np.where(ids % 2, "frs", "spi"), + } + ) + return uk_national_frame( + person=person, + benunit=benunit, + household=household, + household_weights=np.linspace(1.0, 2.0, n), + time_period="2024", + ) + + +def _adult_frame(): + person = pd.DataFrame( + { + "person_id": [1, 2, 3, 4, 5], + "person_benunit_id": [1, 1, 1, 2, 2], + "person_household_id": [1, 1, 1, 2, 2], + "age": [45, 45, 12, 30, 50], + "capital_gains": [1.0, 2.0, 3.0, 4.0, 5.0], + "employment_income": [20_000.0] * 5, + } + ) + for column in UK_CGT_TAXABLE_INCOME_PROXY_COMPONENTS: + if column not in person: + person[column] = 0.0 + return uk_national_frame( + person=person, + benunit=pd.DataFrame({"benunit_id": [1, 2]}), + household=pd.DataFrame({"household_id": [1, 2]}), + household_weights=[3.0, 7.0], + time_period="2024", + ) + + +@lru_cache +def _stage(name: str): + return load_country_spec("uk").sources.stage_map()[name] + + +def _drift(stage, operation_index: int, parameter: str): + operations = list(stage.operations) + operation = operations[operation_index] + operations[operation_index] = SourceOperationSpec( + kind=operation.kind, + parameters={**operation.parameters, parameter: "__drift__"}, + ) + return replace(stage, operations=tuple(operations)) + + +def test_clone_splits_exact_mass_and_uses_oldest_adult_carriers() -> None: + result = clone_cgt_incidence( + _adult_frame(), distribution=_distribution(negative=True) + ) + household = result.frame.table("household") + person = result.frame.table("person") + + assert result.original_mass == pytest.approx(5.0) + assert result.clone_mass == pytest.approx(5.0) + assert result.frame.weights_for("household").kind is WeightKind.IMPORTANCE + assert result.frame.mass_log[-1].reason == CGT_CLONE_MASS_CHANGE_REASON + clone_households = set( + household.loc[household[HOUSEHOLD_IS_CGT_CLONE], "household_id"] + ) + gainers = person.loc[person.capital_gains != 0] + assert set(gainers.person_household_id) == clone_households + # Household 1 has tied oldest adults: lower person_id carries the draw. + assert ( + gainers.loc[ + gainers.person_household_id == min(clone_households) + ].person_id.nunique() + == 1 + ) + assert result.carrier_count == 2 + assert result.negative_prior_count == 1 + assert (gainers.capital_gains < 0.0).any() + + +def test_prior_spline_keeps_negative_values_and_extrapolates_linearly() -> None: + draws = _draw_banded_priors( + np.zeros(4), + np.asarray([0.0, 0.05, 0.95, 1.0]), + distribution=_distribution(), + ) + + assert draws[0] < 0.0 + assert draws[1] == pytest.approx(-10.0) + assert draws[2] == pytest.approx(125.0) + assert draws[3] > draws[2] + + +def test_band_donors_are_band_exact_positive_and_permutation_stable() -> None: + first = stack_cgt_band_donors( + _one_person_households(300), + size_bands=load_hmrc_cgt_size_bands(), + distribution=_distribution(), + ) + second = stack_cgt_band_donors( + _one_person_households(300, reverse_people=True), + size_bands=load_hmrc_cgt_size_bands(), + distribution=_distribution(), + ) + donor_households = first.frame.table("household").loc[ + lambda table: table[HOUSEHOLD_IS_CGT_BAND_DONOR] + ] + + assert len(donor_households) == DONOR_TOTAL == DONORS_PER_BAND * DONOR_BAND_COUNT + assert (first.frame.weights_for("household").values[-DONOR_TOTAL:] > 0).all() + assert [row["donor_count"] for row in first.band_rows] == [DONORS_PER_BAND] * 9 + assert [row["lower_limit"] for row in first.band_rows][0] == MIN_DONOR_BAND_LOWER + assert [row["weighted_taxpayers"] for row in first.band_rows] == pytest.approx( + [79_000, 74_000, 53_000, 37_000, 14_000, 8_000, 5_000, 3_000, 2_000] + ) + second_donors = second.frame.table("household").loc[ + lambda table: table[HOUSEHOLD_IS_CGT_BAND_DONOR] + ] + assert set(donor_households.household_id) == set(second_donors.household_id) + + +def test_never_zero_band_weight_assertion_fires() -> None: + resource = copy.deepcopy(load_hmrc_cgt_size_bands()) + retained = next(row for row in resource["rows"] if row["lower_limit"] == 12_300) + retained["taxpayers_thousands"] = 0 + + with pytest.raises(ValueError, match="zero initial weight"): + _assert_cgt_donor_stage_parameters( + _stage("cgt_band_donors"), size_bands=resource + ) + + +@pytest.mark.parametrize( + "operation_index,parameter", + [ + *[ + (0, name) + for name in ( + "entity", + "copies", + "flag_column", + "original_flag", + "clone_flag", + "mass_split", + "weight_kind_out", + "conservation", + "id_remapping", + "declared_factor", + "reason", + ) + ], + *[ + (1, name) + for name in ( + "resource", + "income_proxy_components", + "allowance_subtraction", + "carrier", + "adult_minimum_age", + "quantile_points", + "spline_degree", + "extrapolation", + "keep_negative_draws", + "seed", + "salt", + ) + ], + ], +) +def test_incidence_drift_assert_covers_every_reviewed_parameter( + operation_index: int, parameter: str +) -> None: + with pytest.raises(ValueError, match="drifted"): + _assert_cgt_incidence_stage_parameters( + _drift(_stage("cgt_incidence_clone"), operation_index, parameter) + ) + + +@pytest.mark.parametrize( + "parameter", + ( + "size_band_resource", + "incidence_resource", + "minimum_band_lower", + "donors_per_band", + "expected_band_count", + "expected_donor_count", + "candidate_order", + "draw", + "seed", + "flag_column", + "carrier", + "initial_weight", + "never_zero_weight", + "weight_kind_out", + "reason", + ), +) +def test_donor_drift_assert_covers_every_reviewed_parameter(parameter: str) -> None: + with pytest.raises(ValueError, match="drifted"): + _assert_cgt_donor_stage_parameters( + _drift(_stage("cgt_band_donors"), 0, parameter), + size_bands=load_hmrc_cgt_size_bands(), + ) diff --git a/packages/microcosm-build/tests/test_uk_frs_spine.py b/packages/microcosm-build/tests/test_uk_frs_spine.py index 822414392..d2d9ff82f 100644 --- a/packages/microcosm-build/tests/test_uk_frs_spine.py +++ b/packages/microcosm-build/tests/test_uk_frs_spine.py @@ -1577,6 +1577,7 @@ def test_input_artifact_pins_bind_spi_donor_and_ods() -> None: pins = tool._input_artifact_pins(stages) assert set(pins) == { + "cgt_published_fact_surface", "etb_household_tab", "lcfs_household_tab", "lcfs_person_tab", @@ -1597,8 +1598,32 @@ def test_input_artifact_pins_bind_spi_donor_and_ods() -> None: "etb_vat", "etb_services", "hmrc_spi_income_spine", + "hmrc_cgt_gains_spine", ) for artifact in stage_map[stage_name].artifacts - if "table" not in artifact and "resource" not in artifact + if "table" not in artifact + and "resource" not in artifact + and "sha256" in artifact } assert {role: pin["sha256"] for role, pin in pins.items()} == declared + + +def test_e8_manifest_seeds_all_reach_the_build_sidecar_harvester() -> None: + tool = _load_tool() + spec = load_country_spec("uk") + assert spec.sources is not None + stages = spec.sources.stage_map() + + declared = tool._declared_seeds([stages[name] for name in tool._STAGE_NAMES]) + + assert declared["cgt_incidence_clone"] == {"cgt_prior_amount": 0} + assert declared["cgt_band_donors"] == {"stack_band_donor_households": 1} + assert declared["hmrc_cgt_gains_spine"] == {"within_band_draws": 552} + assert declared["salary_sacrifice"] == { + "salary_sacrifice": 42, + "salary_sacrifice_conversion": 2024, + } + assert declared["student_loans"] == { + "student_loan_plan_5": 42, + "student_loan_plan_2": 42, + } diff --git a/packages/microcosm-build/tests/test_uk_national_build.py b/packages/microcosm-build/tests/test_uk_national_build.py index e8edf9ea7..1a9766e68 100644 --- a/packages/microcosm-build/tests/test_uk_national_build.py +++ b/packages/microcosm-build/tests/test_uk_national_build.py @@ -1104,6 +1104,7 @@ def test_national_build_real_terminal_batch_blocks_incomplete_qrf_before_staging "uk_aggregate_admin": "evidence_absent", "uk_take_up_signal": "passed", "uk_brma_enum_domain": "passed", + "uk_student_loan_plan_enum_domain": "failed", # The legacy report omitted unevidenced gates; the battery names # every gap — non-blocking off the release-candidate posture. "uk_export_surface": "evidence_absent", diff --git a/packages/microcosm-build/tests/test_uk_release_input_coverage.py b/packages/microcosm-build/tests/test_uk_release_input_coverage.py index 7467bf20b..3453fa6ad 100644 --- a/packages/microcosm-build/tests/test_uk_release_input_coverage.py +++ b/packages/microcosm-build/tests/test_uk_release_input_coverage.py @@ -605,6 +605,11 @@ def test_shipped_manifest_is_current(self) -> None: { "hmrc_spi_income", "hmrc_cgt_gains", + "cgt_incidence_clone", + "cgt_band_donors", + "hmrc_cgt_gains_spine", + "salary_sacrifice", + "student_loans", "was_wealth", "regional_property_uprating", "lcfs_consumption", diff --git a/packages/microcosm-build/tests/test_uk_salary_sacrifice.py b/packages/microcosm-build/tests/test_uk_salary_sacrifice.py new file mode 100644 index 000000000..a160a10f6 --- /dev/null +++ b/packages/microcosm-build/tests/test_uk_salary_sacrifice.py @@ -0,0 +1,224 @@ +from __future__ import annotations + +from dataclasses import replace +from functools import lru_cache + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.country_spec import load_country_spec +from microcosm.build.source_manifest import SourceOperationSpec +from microcosm.build.uk_runtime import salary_sacrifice +from microcosm.build.uk_runtime.cgt_structure import ( + HOUSEHOLD_IS_CGT_BAND_DONOR, + HOUSEHOLD_IS_CGT_CLONE, +) +from microcosm.build.uk_runtime.national_frame import uk_national_frame +from microcosm.build.uk_runtime.salary_sacrifice import ( + SALSAC_OUTPUT, + SALSAC_RATE_CAP, + SALSAC_STAGE_TARGET, + _assert_salary_sacrifice_stage_parameters, + impute_salary_sacrifice, + load_salary_sacrifice_anchor, +) + + +class _Fitted: + value = 0.0 + + def predict(self, predictors: pd.DataFrame) -> pd.DataFrame: + return pd.DataFrame({SALSAC_OUTPUT: self.value}, index=predictors.index) + + +class _FakeQRF: + training_frames: list[pd.DataFrame] = [] + + def __init__(self, *, n_estimators: int, seed: int) -> None: + assert n_estimators == 100 + assert seed == 42 + + def fit(self, frame, predictors, targets, *, weights): + assert predictors == ["age", "employment_income"] + assert targets == [SALSAC_OUTPUT] + assert weights == "_fit_weight" + self.training_frames.append(frame.copy()) + return _Fitted() + + +def _frame( + *, + asked, + salary_sacrifice_values, + employee_pension, + channels=None, + clones=None, + donors=None, + weights=None, +): + n = len(asked) + ids = np.arange(1, n + 1, dtype="int64") + channels = channels or ["frs"] * n + clones = clones or [False] * n + donors = donors or [False] * n + person = pd.DataFrame( + { + "person_id": ids, + "person_benunit_id": ids, + "person_household_id": ids, + "age": np.linspace(25, 55, n), + "employment_income": np.full(n, 30_000.0), + "salary_sacrifice_asked": asked, + SALSAC_OUTPUT: salary_sacrifice_values, + "employee_pension_contributions": employee_pension, + } + ) + household = pd.DataFrame( + { + "household_id": ids, + "household_support_channel": channels, + HOUSEHOLD_IS_CGT_CLONE: clones, + HOUSEHOLD_IS_CGT_BAND_DONOR: donors, + } + ) + return uk_national_frame( + person=person, + benunit=pd.DataFrame({"benunit_id": ids}), + household=household, + household_weights=np.ones(n) if weights is None else weights, + time_period="2024", + ) + + +@lru_cache +def _stage(): + return load_country_spec("uk").sources.stage_map()["salary_sacrifice"] + + +def _drift(operation_index: int, parameter: str): + stage = _stage() + operations = list(stage.operations) + operation = operations[operation_index] + operations[operation_index] = SourceOperationSpec( + operation.kind, + {**operation.parameters, parameter: "__drift__"}, + ) + return replace(stage, operations=tuple(operations)) + + +def test_qrf_preserves_asked_rows_and_excludes_nonbase_training_rows( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _FakeQRF.training_frames = [] + _Fitted.value = -25.0 + monkeypatch.setattr(salary_sacrifice, "QRF", _FakeQRF) + frame = _frame( + asked=[1, 1, 1, 1, 0], + salary_sacrifice_values=[10.0, 20.0, 30.0, 40.0, 0.0], + employee_pension=[0.0] * 5, + channels=["frs", "spi", "frs", "frs", "spi"], + clones=[False, False, True, False, False], + donors=[False, False, False, True, False], + ) + + result = impute_salary_sacrifice(frame) + + assert len(_FakeQRF.training_frames) == 1 + assert _FakeQRF.training_frames[0].index.tolist() == [0] + assert result.frame.table("person")[SALSAC_OUTPUT].tolist() == [ + 10.0, + 20.0, + 30.0, + 40.0, + 0.0, + ] + assert result.training_rows == 1 + assert result.prediction_rows == 1 + + +def test_conversion_moves_full_pension_zeros_source_and_records_cap( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _Fitted.value = 0.0 + monkeypatch.setattr(salary_sacrifice, "QRF", _FakeQRF) + n_donors = 100 + frame = _frame( + asked=[1, *([0] * n_donors)], + salary_sacrifice_values=[0.0] * (n_donors + 1), + employee_pension=[0.0, *np.linspace(100.0, 1_000.0, n_donors)], + weights=[1.0, *([100_000.0] * n_donors)], + ) + + result = impute_salary_sacrifice(frame) + person = result.frame.table("person") + converted = person[SALSAC_OUTPUT] > 0.0 + + assert result.rate == SALSAC_RATE_CAP + assert result.cap_bound is True + assert 0 < result.converted_rows < n_donors + assert (person.loc[converted, "employee_pension_contributions"] == 0.0).all() + assert result.moved_amount == pytest.approx( + person.loc[converted, SALSAC_OUTPUT].sum() + ) + evidence = result.evidence()["headcount_receipt"] + assert evidence["target"] == SALSAC_STAGE_TARGET + assert evidence["converted_rows"] == result.converted_rows + + +def test_anchor_is_self_consistent() -> None: + anchor = load_salary_sacrifice_anchor() + assert anchor["hmrc_anchor"]["total_users"] * anchor["derived"][ + "staging_ratio" + ] == pytest.approx(anchor["derived"]["stage_target"]) + + +@pytest.mark.parametrize( + "operation_index,parameter", + [ + *[ + (0, name) + for name in ( + "training_population", + "target_population", + "predictors", + "targets", + "weights", + "weight_mapping", + "seed", + "n_estimators", + "clamp_minimum", + "preserve_asked_rows", + "cache", + ) + ], + *[ + (1, name) + for name in ( + "resource", + "target", + "donor_pool", + "rate_cap", + "move", + "seed", + "salt", + "receipt", + ) + ], + ], +) +def test_manifest_drift_assert_covers_every_reviewed_parameter( + operation_index: int, parameter: str +) -> None: + with pytest.raises(ValueError, match="drifted"): + _assert_salary_sacrifice_stage_parameters( + _drift(operation_index, parameter), + anchor=load_salary_sacrifice_anchor(), + ) + + +def test_resource_drift_assert_rejects_anchor_change() -> None: + anchor = dict(load_salary_sacrifice_anchor()) + anchor["derived"] = {**anchor["derived"], "stage_target": 1} + with pytest.raises(ValueError, match="stage_target.*drifted"): + _assert_salary_sacrifice_stage_parameters(_stage(), anchor=anchor) diff --git a/packages/microcosm-build/tests/test_uk_source_runtime.py b/packages/microcosm-build/tests/test_uk_source_runtime.py index 0a288ab54..d9901c77a 100644 --- a/packages/microcosm-build/tests/test_uk_source_runtime.py +++ b/packages/microcosm-build/tests/test_uk_source_runtime.py @@ -119,6 +119,11 @@ def hmrc(frame: Frame) -> Frame: lcfs_consumption_transform=retained, etb_vat_transform=hmrc, etb_services_transform=retained, + cgt_incidence_clone_transform=retained, + cgt_band_donors_transform=hmrc, + hmrc_cgt_gains_spine_transform=retained, + salary_sacrifice_transform=hmrc, + student_loans_transform=retained, ) == { "frs_hmrc_retained_leaves": retained, "hmrc_spi_income": hmrc, @@ -127,6 +132,11 @@ def hmrc(frame: Frame) -> Frame: "lcfs_consumption": retained, "etb_vat": hmrc, "etb_services": retained, + "cgt_incidence_clone": retained, + "cgt_band_donors": hmrc, + "hmrc_cgt_gains_spine": retained, + "salary_sacrifice": hmrc, + "student_loans": retained, } diff --git a/packages/microcosm-build/tests/test_uk_source_stages.py b/packages/microcosm-build/tests/test_uk_source_stages.py index 8eef5ec37..c00ce2085 100644 --- a/packages/microcosm-build/tests/test_uk_source_stages.py +++ b/packages/microcosm-build/tests/test_uk_source_stages.py @@ -46,6 +46,13 @@ "spi_support_channel", "hmrc_spi_income_spine", ] +E8_STAGE_NAMES = [ + "cgt_incidence_clone", + "cgt_band_donors", + "hmrc_cgt_gains_spine", + "salary_sacrifice", + "student_loans", +] UK_SOURCE_STAGE_NAMES = [ "frs_spine", *E3_STAGE_NAMES, @@ -53,6 +60,7 @@ *E5_STAGE_NAMES, *E6_STAGE_NAMES, *E7_STAGE_NAMES, + *E8_STAGE_NAMES, "frs_hmrc_retained_leaves", "hmrc_spi_income", ] @@ -62,6 +70,7 @@ *E4_STAGE_NAMES, *E6_STAGE_NAMES, *E7_STAGE_NAMES, + *E8_STAGE_NAMES, ] FROZEN_SOURCE_STAGES_SHA256 = ( "c0341af7166ae3a85a3c1164e7d9e880c4b4aec122f1a8fa90c73b46c596e1ea" @@ -123,11 +132,20 @@ def test_e6_block_sits_between_e5_and_e7(self) -> None: == E6_STAGE_NAMES ) - def test_e7_block_is_contiguous_before_certified_pair(self) -> None: + def test_e7_block_sits_between_e6_and_e8(self) -> None: + canonical = _load_json(CANONICAL_SOURCE_STAGES) + names = [stage["stage"] for stage in canonical["stages"]] + + assert ( + names[names.index("etb_services") + 1 : names.index("cgt_incidence_clone")] + == E7_STAGE_NAMES + ) + + def test_e8_block_is_contiguous_before_certified_pair(self) -> None: canonical = _load_json(CANONICAL_SOURCE_STAGES) names = [stage["stage"] for stage in canonical["stages"]] - assert names[-5:-2] == E7_STAGE_NAMES + assert names[-7:-2] == E8_STAGE_NAMES assert names[-2:] == ["frs_hmrc_retained_leaves", "hmrc_spi_income"] def test_copy_is_lockstep_with_frozen_original_except_citation_rewrites( @@ -208,7 +226,7 @@ def test_country_stage_plan_assembles_two_certified_uk_national_stages( "hmrc_spi_income", ] - def test_country_stage_plan_assembles_fourteen_stage_spine_plan(self) -> None: + def test_country_stage_plan_assembles_spine_plan(self) -> None: spec = load_country_spec("uk") implementations = {name: _identity for name in UK_SOURCE_STAGE_NAMES} plan = country_stage_plan( @@ -246,6 +264,11 @@ def test_country_stage_plan_assembles_fourteen_stage_spine_plan(self) -> None: "frs_hmrc_spine_leaves": _identity, "spi_support_channel": _identity, "hmrc_spi_income_spine": _identity, + "cgt_incidence_clone": _identity, + "cgt_band_donors": _identity, + "hmrc_cgt_gains_spine": _identity, + "salary_sacrifice": _identity, + "student_loans": _identity, "frs_hmrc_retained_leaves": _identity, "hmrc_spi_income": _identity, "hmrc_spi_income_fallback": _identity, @@ -422,6 +445,33 @@ def test_e7_outputs_and_rewrites_are_backed_by_runtime_constants(self) -> None: assert income.rewrites == UK_SPI_INCOME_SPINE_REWRITE_COLUMNS assert not (set(income.outputs) & set(income.rewrites)) + def test_e8_outputs_and_rewrites_are_backed_by_runtime_constants(self) -> None: + from microcosm.build.uk_runtime.cgt_structure import ( + HOUSEHOLD_IS_CGT_BAND_DONOR, + HOUSEHOLD_IS_CGT_CLONE, + ) + from microcosm.build.uk_runtime.salary_sacrifice import SALSAC_OUTPUT + + stages = load_country_spec("uk").sources.stage_map() + + assert stages["cgt_incidence_clone"].outputs == ( + HOUSEHOLD_IS_CGT_CLONE, + "capital_gains", + ) + assert stages["cgt_incidence_clone"].rewrites == ("capital_gains",) + assert stages["cgt_band_donors"].outputs == ( + HOUSEHOLD_IS_CGT_BAND_DONOR, + "capital_gains", + ) + assert stages["cgt_band_donors"].rewrites == ("capital_gains",) + assert stages["hmrc_cgt_gains_spine"].outputs == ("capital_gains",) + assert stages["hmrc_cgt_gains_spine"].rewrites == ("capital_gains",) + assert stages["salary_sacrifice"].outputs == ( + SALSAC_OUTPUT, + "employee_pension_contributions", + ) + assert stages["student_loans"].outputs == ("student_loan_plan",) + class TestE3ManifestLockstep: def test_e3_raw_tab_pins_match_spine_artifacts(self) -> None: @@ -557,6 +607,31 @@ def test_e3_operation_kinds_are_declared_in_order(self) -> None: "classify_hmrc_income_facts_with_reviewed_fences", "gate_distributional_effective_mass", ] + assert [op.kind for op in stages["cgt_incidence_clone"].operations] == [ + "clone_records", + "draw_capital_gains_prior_from_banded_quantiles", + ] + assert [op.kind for op in stages["cgt_band_donors"].operations] == [ + "stack_band_donor_households" + ] + assert [op.kind for op in stages["hmrc_cgt_gains_spine"].operations] == [ + "verify_pinned_cgt_ods", + "taxable_income_proxy", + "rank_preserving_allocation", + "within_band_draws", + "sub_aea_remainder", + "record_mass_conservation_receipt", + "classify_cgt_band_facts_with_reviewed_fence", + ] + assert [op.kind for op in stages["salary_sacrifice"].operations] == [ + "fit_weighted_qrf", + "convert_donors_to_target_stock", + ] + assert [op.kind for op in stages["student_loans"].operations] == [ + "assign_student_loan_plan_cohorts", + "top_up_to_stock", + "top_up_to_stock", + ] def test_engine_predictor_and_rewrite_constants_match_manifest(self) -> None: from microcosm.build.uk_runtime.etb_services import ( @@ -735,6 +810,21 @@ def test_e7_declared_seed_lockstep(self) -> None: assert stages["hmrc_spi_income_spine"].operations[2].parameters["seed"] == 42 assert stages["hmrc_spi_income_spine"].operations[3].parameters["seed"] == 43 + def test_e8_declared_seed_lockstep(self) -> None: + stages = load_country_spec("uk").sources.stage_map() + + assert stages["cgt_incidence_clone"].operations[1].parameters["seed"] == 0 + assert stages["cgt_band_donors"].operations[0].parameters["seed"] == 1 + assert ( + stages["hmrc_cgt_gains_spine"].operations[3].parameters["seed_base"] == 552 + ) + assert stages["salary_sacrifice"].operations[0].parameters["seed"] == 42 + assert stages["salary_sacrifice"].operations[1].parameters["seed"] == 2024 + assert [ + operation.parameters["seed"] + for operation in stages["student_loans"].operations[1:] + ] == [42, 42] + def test_full_uk_source_stage_plan_compiles_with_e4_stages(self) -> None: spec = load_country_spec("uk") implementations = {name: _identity for name in UK_SOURCE_STAGE_NAMES} diff --git a/packages/microcosm-build/tests/test_uk_student_loans.py b/packages/microcosm-build/tests/test_uk_student_loans.py new file mode 100644 index 000000000..9c90db6b5 --- /dev/null +++ b/packages/microcosm-build/tests/test_uk_student_loans.py @@ -0,0 +1,238 @@ +from __future__ import annotations + +from dataclasses import replace +from functools import lru_cache + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.country_spec import load_country_spec +from microcosm.build.source_manifest import SourceOperationSpec +from microcosm.build.uk_runtime.national_frame import uk_national_frame +from microcosm.build.uk_runtime.student_loans import ( + PLAN_PRIORITY, + STUDENT_LOAN_ENUM_DOMAIN, + _assert_student_loans_stage_parameters, + assign_student_loan_plans, + load_slc_liable_stocks, +) + + +def _stocks(*, plan_2: float, plan_5: float, year: int = 2025): + return { + "plans": { + "plan_2": {"liable": {str(year): plan_2}}, + "plan_5": {"liable": {str(year): plan_5}}, + } + } + + +def _frame( + *, + ages, + repayments, + regions=None, + education=None, + weights=None, +): + n = len(ages) + ids = np.arange(1, n + 1, dtype="int64") + regions = regions or ["LONDON"] * n + education = education or ["TERTIARY"] * n + return uk_national_frame( + person=pd.DataFrame( + { + "person_id": ids, + "person_benunit_id": ids, + "person_household_id": ids, + "age": ages, + "student_loan_repayments": repayments, + "highest_education": education, + } + ), + benunit=pd.DataFrame({"benunit_id": ids}), + household=pd.DataFrame({"household_id": ids, "region": regions}), + household_weights=np.ones(n) if weights is None else weights, + time_period="2024", + ) + + +@lru_cache +def _stage(): + return load_country_spec("uk").sources.stage_map()["student_loans"] + + +def _drift(operation_index: int, parameter: str): + stage = _stage() + operations = list(stage.operations) + operation = operations[operation_index] + operations[operation_index] = SourceOperationSpec( + operation.kind, + {**operation.parameters, parameter: "__drift__"}, + ) + return replace(stage, operations=tuple(operations)) + + +def test_reported_cohort_boundaries_and_country_independence() -> None: + # At 2025: these ages imply start years 2011, 2012, 2022, and 2023. + result = assign_student_loan_plans( + _frame( + ages=[32, 31, 21, 20], + repayments=[100.0] * 4, + regions=["WALES", "SCOTLAND", "NORTHERN_IRELAND", "WALES"], + ), + stocks=_stocks(plan_2=0, plan_5=0), + year=2025, + ) + + assert result.frame.table("person")["student_loan_plan"].tolist() == [ + "PLAN_1", + "PLAN_2", + "PLAN_2", + "PLAN_5", + ] + + +def test_topups_apply_eligibility_gates_and_plan5_priority() -> None: + result = assign_student_loan_plans( + _frame( + ages=[20, 20, 20, 31, 31, 40], + repayments=[0.0] * 6, + regions=["LONDON", "WALES", "LONDON", "LONDON", "LONDON", "LONDON"], + education=["TERTIARY", "TERTIARY", "GCSE", "TERTIARY", "GCSE", "TERTIARY"], + ), + stocks=_stocks(plan_2=1, plan_5=1), + year=2025, + ) + plans = result.frame.table("person")["student_loan_plan"].tolist() + + assert plans == ["PLAN_5", "NONE", "NONE", "PLAN_2", "NONE", "NONE"] + assert tuple(result.plans) == PLAN_PRIORITY + assert "PLAN_4" not in plans + + +def test_rate_rule_receipt_uses_weighted_shortfall() -> None: + result = assign_student_loan_plans( + _frame( + ages=[31, 31], + repayments=[0.0, 0.0], + weights=[2.0, 2.0], + ), + stocks=_stocks(plan_2=2, plan_5=0), + year=2025, + ) + receipt = result.plans["PLAN_2"] + + assert receipt.shortfall == 2.0 + assert receipt.eligible_mass == 4.0 + assert receipt.rate == 0.5 + assert receipt.final_england_count == receipt.topped_up_mass + + +def test_calibration_year_changes_cohort_assignment() -> None: + frame = _frame(ages=[31], repayments=[100.0]) + + at_2025 = assign_student_loan_plans( + frame, stocks=_stocks(plan_2=0, plan_5=0), year=2025 + ) + at_2036 = assign_student_loan_plans( + frame, stocks=_stocks(plan_2=0, plan_5=0, year=2036), year=2036 + ) + + assert at_2025.frame.table("person").student_loan_plan.iloc[0] == "PLAN_2" + assert at_2036.frame.table("person").student_loan_plan.iloc[0] == "PLAN_5" + assert at_2036.calibration_year == 2036 + + +def test_committed_stocks_pin_full_2025_to_2030_series() -> None: + stocks = load_slc_liable_stocks()["plans"] + + assert stocks["plan_2"]["liable"] == { + "2025": 8_940_000, + "2026": 9_710_000, + "2027": 10_360_000, + "2028": 10_615_000, + "2029": 10_600_000, + "2030": 10_525_000, + } + assert stocks["plan_5"]["above_threshold"]["2030"] == 1_235_000 + + +def test_values_match_policyengine_enum_when_available() -> None: + module = pytest.importorskip( + "policyengine_uk.variables.gov.hmrc.student_loans.student_loan_plan" + ) + engine_names = set(module.StudentLoanPlan.__members__) + + assert set(STUDENT_LOAN_ENUM_DOMAIN) <= engine_names + assert "PLAN_4" in engine_names + assert "PLAN_4" not in STUDENT_LOAN_ENUM_DOMAIN + + +@pytest.mark.parametrize( + "operation_index,parameter", + [ + *[ + (0, name) + for name in ( + "year_rule", + "start_year_formula", + "reported_repayment_test", + "reported_country_gate", + "plan_1_before", + "plan_5_from", + "enum_domain", + "plan_4_imputation", + ) + ], + *[ + (1, name) + for name in ( + "priority", + "resource", + "stock_series", + "year_rule", + "age_min", + "age_max", + "cohort_start_min", + "eligible_region_exclusions", + "highest_education", + "seed", + "salt", + ) + ], + *[ + (2, name) + for name in ( + "priority", + "resource", + "stock_series", + "year_rule", + "age_min", + "age_max", + "cohort_start_min", + "cohort_start_max_exclusive", + "eligible_region_exclusions", + "highest_education", + "seed", + "salt", + ) + ], + ], +) +def test_manifest_drift_assert_covers_every_reviewed_parameter( + operation_index: int, parameter: str +) -> None: + with pytest.raises((ValueError, KeyError), match="drifted|priority"): + _assert_student_loans_stage_parameters( + _drift(operation_index, parameter), + stocks=load_slc_liable_stocks(), + year=2025, + ) + + +def test_stock_drift_assert_rejects_2025_change() -> None: + stocks = _stocks(plan_2=1, plan_5=10_000) + with pytest.raises(ValueError, match="PLAN_2.*drifted"): + _assert_student_loans_stage_parameters(_stage(), stocks=stocks, year=2025) diff --git a/packages/microcosm-build/tests/test_uk_take_up_gate.py b/packages/microcosm-build/tests/test_uk_take_up_gate.py index 50c6f329e..74f7dc8ae 100644 --- a/packages/microcosm-build/tests/test_uk_take_up_gate.py +++ b/packages/microcosm-build/tests/test_uk_take_up_gate.py @@ -115,6 +115,30 @@ def test_brma_enum_domain_binding_fails_off_domain() -> None: assert "OFF_DOMAIN" in result.failures[0] +def test_student_loan_enum_domain_binding_resolves_person_column() -> None: + frame = _frame() + frame.table("person")["student_loan_plan"] = ["NONE"] * 9 + ["PLAN_4"] + binding = UK_GATE_REGISTRY["enum_domain"] + + result = binding.evaluate( + EvidenceContext( + frame=frame, + artifacts={ + "student_loan_plan_enum_domain": ( + "NONE", + "PLAN_1", + "PLAN_2", + "PLAN_5", + ) + }, + ), + {"columns": ("student_loan_plan",)}, + ) + + assert result.passed is False + assert "PLAN_4" in result.failures[0] + + def test_gate_registry_vocabulary_round_trip() -> None: assert "take_up_signal" in UK_GATE_REGISTRY assert "enum_domain" in UK_GATE_REGISTRY diff --git a/packages/microcosm-data/src/microcosm/data/contract.py b/packages/microcosm-data/src/microcosm/data/contract.py index 3e5190c2c..c680b8727 100644 --- a/packages/microcosm-data/src/microcosm/data/contract.py +++ b/packages/microcosm-data/src/microcosm/data/contract.py @@ -368,6 +368,7 @@ "uk_export_surface": "export_surface", "uk_take_up_signal": "take_up_signal", "uk_brma_enum_domain": "enum_domain", + "uk_student_loan_plan_enum_domain": "enum_domain", "uk_target_surface": "target_surface", "uk_target_fit": "target_fit", "uk_input_mass_parity": "input_mass_parity", @@ -403,6 +404,7 @@ "uk_export_surface": ("export_surface", "terminal"), "uk_take_up_signal": ("take_up_signal", "terminal"), "uk_brma_enum_domain": ("enum_domain", "terminal"), + "uk_student_loan_plan_enum_domain": ("enum_domain", "terminal"), "uk_calibration_reference_coverage": ( "calibration_reference_coverage", "terminal", diff --git a/packages/microcosm-data/tests/test_contract.py b/packages/microcosm-data/tests/test_contract.py index 8daf0078f..fd9b49c4b 100644 --- a/packages/microcosm-data/tests/test_contract.py +++ b/packages/microcosm-data/tests/test_contract.py @@ -201,6 +201,11 @@ def _trusted_terminal_gate_signing_key(monkeypatch) -> None: "uk_export_surface": ("export_surface", "terminal", "export_surface"), "uk_take_up_signal": ("take_up_signal", "terminal", "take_up_signal"), "uk_brma_enum_domain": ("enum_domain", "terminal", "enum_domain"), + "uk_student_loan_plan_enum_domain": ( + "enum_domain", + "terminal", + "enum_domain", + ), "uk_calibration_reference_coverage": ( "calibration_reference_coverage", "terminal", diff --git a/tools/build_uk_frs_spine.py b/tools/build_uk_frs_spine.py index 5b4822a9b..8520969dd 100644 --- a/tools/build_uk_frs_spine.py +++ b/tools/build_uk_frs_spine.py @@ -32,6 +32,11 @@ sha256_argument, write_error_receipt, ) +from microcosm.build.uk_runtime.cgt_imputation import uk_cgt_spine_stage_transform +from microcosm.build.uk_runtime.cgt_structure import ( + UKCGTBandDonorStageTransform, + UKCGTIncidenceCloneStageTransform, +) from microcosm.build.uk_runtime.etb_services import UKETBServicesStageTransform from microcosm.build.uk_runtime.etb_vat import UKETBVATStageTransform from microcosm.build.uk_runtime.frs_brma import UKFRSBRMAStageTransform @@ -69,11 +74,13 @@ from microcosm.build.uk_runtime.regional_uprating import ( UKRegionalPropertyUpratingStageTransform, ) +from microcosm.build.uk_runtime.salary_sacrifice import UKSalarySacrificeStageTransform from microcosm.build.uk_runtime.spi_spine import ( UKFRSHMRCSpineLeavesStageTransform, UKSPIIncomeSpineStageTransform, UKSPISupportChannelStageTransform, ) +from microcosm.build.uk_runtime.student_loans import UKStudentLoansStageTransform from microcosm.build.uk_runtime.take_up_contract import load_uk_take_up_contract from microcosm.build.uk_runtime.was_wealth import UKWASWealthStageTransform from microcosm.frame.adapters.policyengine_uk import PolicyEngineUKEngine @@ -102,6 +109,11 @@ "frs_hmrc_spine_leaves", "spi_support_channel", "hmrc_spi_income_spine", + "cgt_incidence_clone", + "cgt_band_donors", + "hmrc_cgt_gains_spine", + "salary_sacrifice", + "student_loans", ) @@ -153,6 +165,11 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: required=True, help="Pinned local HMRC collated ODS path.", ) + parser.add_argument( + "--cgt-ods", + type=Path, + help="Pinned local HMRC Capital Gains Tax Table 3 ODS path.", + ) parser.add_argument( "--checkpoint-dir", type=Path, @@ -230,6 +247,11 @@ def _validate_args(args: argparse.Namespace) -> None: raise ValueError(f"--hmrc-ods must be an existing file: {args.hmrc_ods}") if args.hmrc_ods.suffix.lower() != ".ods": raise ValueError("--hmrc-ods must end with '.ods'.") + if args.cgt_ods is not None: + if not args.cgt_ods.is_file(): + raise ValueError(f"--cgt-ods must be an existing file: {args.cgt_ods}") + if args.cgt_ods.suffix.lower() != ".ods": + raise ValueError("--cgt-ods must end with '.ods'.") paths = { "spine_h5": args.spine_h5, "build_sidecar": args.spine_h5.with_suffix(".build.json"), @@ -382,6 +404,8 @@ def _declared_seeds(stages) -> dict[str, dict[str, int]]: for operation in stage.operations: output = operation.parameters.get("output") seed = operation.parameters.get("seed") + if seed is None: + seed = operation.parameters.get("seed_base") if isinstance(output, str) and isinstance(seed, int): stage_seeds[output] = seed elif isinstance(seed, int): @@ -405,6 +429,16 @@ def _declared_seeds(stages) -> dict[str, dict[str, int]]: stage_seeds[stage.stage] = seed elif operation.kind == "fit_weighted_qrf": stage_seeds[stage.stage] = seed + elif operation.kind == "draw_capital_gains_prior_from_banded_quantiles": + stage_seeds[str(operation.parameters["salt"])] = seed + elif operation.kind == "stack_band_donor_households": + stage_seeds["stack_band_donor_households"] = seed + elif operation.kind == "within_band_draws": + stage_seeds["within_band_draws"] = seed + elif operation.kind == "convert_donors_to_target_stock": + stage_seeds[str(operation.parameters["salt"])] = seed + elif operation.kind == "top_up_to_stock": + stage_seeds[str(operation.parameters["salt"])] = seed if stage_seeds: declared[stage.stage] = stage_seeds return declared @@ -633,6 +667,10 @@ def main(argv: list[str] | None = None) -> int: raise ValueError("UK country spec has no source stages.") stages_by_name = spec.sources.stage_map() stage_names = tuple(name for name in _STAGE_NAMES if name in stages_by_name) + if "hmrc_cgt_gains_spine" in stage_names and args.cgt_ods is None: + raise ValueError( + "--cgt-ods is required when hmrc_cgt_gains_spine is scheduled." + ) if "was_wealth" in stage_names and args.was_tab is None: raise ValueError( "--was-tab is required when the was_wealth stage is scheduled." @@ -780,6 +818,28 @@ def main(argv: list[str] | None = None) -> int: sample_fraction=args.sample_fraction, ) implementations["hmrc_spi_income_spine"] = hmrc_spine_transform + if "cgt_incidence_clone" in stage_names: + implementations["cgt_incidence_clone"] = UKCGTIncidenceCloneStageTransform( + stage=stages_by_name["cgt_incidence_clone"] + ) + if "cgt_band_donors" in stage_names: + implementations["cgt_band_donors"] = UKCGTBandDonorStageTransform( + stage=stages_by_name["cgt_band_donors"] + ) + if "hmrc_cgt_gains_spine" in stage_names: + implementations["hmrc_cgt_gains_spine"] = uk_cgt_spine_stage_transform( + stages_by_name["hmrc_cgt_gains_spine"], + args.cgt_ods, + ) + if "salary_sacrifice" in stage_names: + implementations["salary_sacrifice"] = UKSalarySacrificeStageTransform( + stage=stages_by_name["salary_sacrifice"] + ) + if "student_loans" in stage_names: + implementations["student_loans"] = UKStudentLoansStageTransform( + stage=stages_by_name["student_loans"], + calibration_year=frs_release.calibration_year, + ) plan = country_stage_plan( spec, implementations, diff --git a/tools/build_uk_release_input_coverage_manifest.py b/tools/build_uk_release_input_coverage_manifest.py index f0b17562e..cc901aa9a 100644 --- a/tools/build_uk_release_input_coverage_manifest.py +++ b/tools/build_uk_release_input_coverage_manifest.py @@ -790,6 +790,25 @@ def build_manifest( }, "effective_mass_coverage": EFFECTIVE_MASS_COVERAGE, "family_coverage": { + "cgt_incidence_clone": _source_stage_family_coverage_contract( + stage_name="cgt_incidence_clone", + candidate_source=candidate_source, + ), + "cgt_band_donors": _source_stage_family_coverage_contract( + stage_name="cgt_band_donors", + candidate_source=candidate_source, + ), + "hmrc_cgt_gains_spine": _cgt_spine_family_coverage_contract( + candidate_source=candidate_source, + ), + "salary_sacrifice": _source_stage_family_coverage_contract( + stage_name="salary_sacrifice", + candidate_source=candidate_source, + ), + "student_loans": _source_stage_family_coverage_contract( + stage_name="student_loans", + candidate_source=candidate_source, + ), "hmrc_cgt_gains": _cgt_family_coverage_contract( candidate_source=candidate_source, ), @@ -959,6 +978,32 @@ def _source_stage_family_coverage_contract( f"{SOURCE_STAGES_PATH}: expected exactly one {stage_name!r} stage." ) stage = matches[0] + operations = [ + operation + for operation in stage.get("operations", []) + if isinstance(operation, dict) + ] + declared_reasons = [ + str(operation["reason"]) + for operation in operations + if isinstance(operation.get("reason"), str) and operation.get("reason") + ] + required_mass_change_reason = ( + declared_reasons[-1] + if declared_reasons + else ( + "E5 source-stage transform preserves household rows and typed " + "household weights; total household mass is conserved." + ) + ) + mass_change_semantics = ( + "mass_increasing_support" + if any( + operation.get("kind") == "stack_band_donor_households" + for operation in operations + ) + else "mass_conserving" + ) return { "status": "required_at_build", "stage": stage_name, @@ -971,10 +1016,91 @@ def _source_stage_family_coverage_contract( "source": str(stage.get("source", "")), }, "output_weight_kind": "importance", - "required_mass_change_reason": ( - "E5 source-stage transform preserves household rows and typed " - "household weights; total household mass is conserved." + "required_mass_change_reason": required_mass_change_reason, + "mass_change_semantics": mass_change_semantics, + "outputs": list(stage.get("outputs", [])), + "rewrites": list(stage.get("rewrites", [])), + "effective_mass_requirements": {}, + } + + +def _cgt_spine_family_coverage_contract( + *, + candidate_source: dict[str, Any], +) -> dict[str, Any]: + """Emit the canonical spine-side CGT family without touching the frozen path.""" + + payload = _load(SOURCE_STAGES_PATH) + stages = payload.get("stages") + if not isinstance(stages, list): + raise ValueError(f"{SOURCE_STAGES_PATH}: expected source stages list.") + matches = [ + stage + for stage in stages + if isinstance(stage, dict) and stage.get("stage") == "hmrc_cgt_gains_spine" + ] + if len(matches) != 1: + raise ValueError( + f"{SOURCE_STAGES_PATH}: expected exactly one hmrc_cgt_gains_spine stage." + ) + stage = matches[0] + artifacts = { + artifact["role"]: artifact + for artifact in stage.get("artifacts", []) + if isinstance(artifact, dict) and isinstance(artifact.get("role"), str) + } + operations = { + operation["kind"]: operation + for operation in stage.get("operations", []) + if isinstance(operation, dict) and isinstance(operation.get("kind"), str) + } + required_artifacts = {"cgt_published_fact_surface", "policy_parameters"} + required_operations = { + "verify_pinned_cgt_ods", + "taxable_income_proxy", + "rank_preserving_allocation", + "within_band_draws", + "sub_aea_remainder", + "record_mass_conservation_receipt", + "classify_cgt_band_facts_with_reviewed_fence", + } + missing_artifacts = sorted(required_artifacts - set(artifacts)) + missing_operations = sorted(required_operations - set(operations)) + if missing_artifacts or missing_operations: + raise ValueError( + f"{SOURCE_STAGES_PATH}: incomplete spine CGT family contract; " + f"missing_artifacts={missing_artifacts}, " + f"missing_operations={missing_operations}." + ) + surface = artifacts["cgt_published_fact_surface"] + verify = operations["verify_pinned_cgt_ods"] + fence = operations["classify_cgt_band_facts_with_reviewed_fence"] + if verify.get("artifact_role") != "cgt_published_fact_surface": + raise ValueError("Spine CGT verification must bind its distinct ODS role.") + if not bool(verify.get("require_before_source_read")): + raise ValueError("Spine CGT ODS must be verified before source read.") + if bool(fence.get("calibration_permitted", True)): + raise ValueError("Spine CGT band facts must remain fenced from calibration.") + if str(surface.get("sha256", "")) == "" or int(surface.get("size_bytes", 0)) <= 0: + raise ValueError("Spine CGT surface must pin sha256 and size_bytes.") + return { + "status": "required_at_build", + "stage": "hmrc_cgt_gains_spine", + "source_manifest": SOURCE_STAGES_PATH.name, + "source_manifest_sha256": _sha256(SOURCE_STAGES_PATH), + "base_candidate_sha256": str(candidate_source["sha256"]), + "base_candidate_tier": validate_uk_release_tier(candidate_source["tier"]), + "source_vintages": { + "hmrc_surface": str(surface["vintage"]), + "mapped_build_period": str(surface["mapped_build_period"]), + }, + "output_weight_kind": "importance", + "required_mass_change_reason": str( + operations["record_mass_conservation_receipt"]["reason"] ), + "calibration_permitted": bool(fence["calibration_permitted"]), + "fact_fence_id": str(fence["fact_fence_id"]), + "fenced_fact_count": int(fence["fenced_fact_count"]), "outputs": list(stage.get("outputs", [])), "rewrites": list(stage.get("rewrites", [])), "effective_mass_requirements": {}, From 092bcd48fd9f9e2d43b467e51a7207ebe1c3f737 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Fri, 21 Aug 2026 20:12:47 +0200 Subject: [PATCH 2/5] Add the E8 licensed-acceptance instruments: identity receipt and sidecar evidence e8_identity_receipt in tools/verify_uk_identity_stability.py receipts the clone-pair structure, recomputes the band-donor selection from the committed resources in original and permuted row order, and recomputes student_loan_plan in full against the stored column; the A&S priors, the Table 3 redraw draws, and the salary-sacrifice QRF and conversion are scoped to twin-build determinism. The spine driver persists the four E8 transforms' executed-effect receipts into the build sidecar as stage_evidence (#730/#684 two-arm rule, arm 2). Co-Authored-By: Claude Fable 5 --- tools/build_uk_frs_spine.py | 17 ++ tools/verify_uk_identity_stability.py | 248 +++++++++++++++++++++++++- 2 files changed, 263 insertions(+), 2 deletions(-) diff --git a/tools/build_uk_frs_spine.py b/tools/build_uk_frs_spine.py index 8520969dd..59921589c 100644 --- a/tools/build_uk_frs_spine.py +++ b/tools/build_uk_frs_spine.py @@ -884,6 +884,23 @@ def main(argv: list[str] | None = None) -> int: frs_vintage=frs_release.vintage, sampling=sampling, ) + # E8 executed-effect receipts (#730/#684 two-arm rule, arm 2): the + # clone/donor/salsac/student-loan transforms record their receipts on + # last_result; persist them beside the declared seeds so the sidecar + # carries evidence that every declared parameter shaped the output. + e8_stage_evidence: dict[str, object] = {} + for e8_stage_name in ( + "cgt_incidence_clone", + "cgt_band_donors", + "salary_sacrifice", + "student_loans", + ): + e8_implementation = implementations.get(e8_stage_name) + e8_last_result = getattr(e8_implementation, "last_result", None) + if e8_last_result is not None: + e8_stage_evidence[e8_stage_name] = e8_last_result.evidence() + if e8_stage_evidence: + sidecar["stage_evidence"] = e8_stage_evidence atomic_write_json(sidecar_path, sidecar) append_phase(state, "build_sidecar_written") if args.emit_nonzero_shares is not None: diff --git a/tools/verify_uk_identity_stability.py b/tools/verify_uk_identity_stability.py index 68b428ade..3dfab7423 100644 --- a/tools/verify_uk_identity_stability.py +++ b/tools/verify_uk_identity_stability.py @@ -496,11 +496,247 @@ def recompute(person_t, benunit_t, household_t) -> dict[str, pd.DataFrame]: } +def e8_identity_receipt( + frame, + *, + permutation_seed: int, +) -> dict[str, object]: + """Receipt E8 deterministic layers under row permutation by entity id. + + Covered: (1) the clone-pair structure — the non-donor population splits + into equal-count original/clone halves whose paired household weights + agree to the exact-total correction tolerance and whose half-masses + match; (2) the CGT band-donor selection recomputed from the committed + resources over id-sorted candidates in original and permuted row order + (set equality with the flagged donors, 30 donors per band, band-exact + stored weights and carrier gains); (3) the student-loan plan column + recomputed in full (identity-keyed top-ups at the release calibration + year) in original and permuted row order against the stored column. + The A&S prior amounts (overwritten by the Table 3 redraw except the + sub-AEA remainder), the redraw's seeded within-band draws (covered by + the merged #560 embedded published-surface tests), and the + salary-sacrifice QRF and conversion (the pre-conversion state is + consumed by the stage) are covered by twin-build determinism. + """ + + from microcosm.build.uk_runtime.cgt_structure import ( + DONOR_SEED, + DONORS_PER_BAND, + HOUSEHOLD_IS_CGT_BAND_DONOR, + HOUSEHOLD_IS_CGT_CLONE, + _component_sum_income, + _incidence_propensity, + _oldest_adult_indices, + _retained_size_bands, + load_advani_summers_distribution, + load_hmrc_cgt_size_bands, + ) + from microcosm.build.uk_runtime.frs_release import load_uk_frs_release + from microcosm.build.uk_runtime.rowwise_geography import id_multiplier_for_values + from microcosm.build.uk_runtime.student_loans import ( + assign_student_loan_plans, + load_slc_liable_stocks, + ) + + problems: dict[str, object] = {} + person = frame.table("person") + benunit = frame.table("benunit") + household = frame.table("household").copy() + household["household_weight"] = frame.weights_for("household").values + + # (1) Clone-pair structure on the non-donor population. + donor_mask = household[HOUSEHOLD_IS_CGT_BAND_DONOR].astype(bool) + non_donor = household.loc[~donor_mask] + originals = non_donor.loc[ + ~non_donor[HOUSEHOLD_IS_CGT_CLONE].astype(bool) + ].sort_values("household_id") + clones = non_donor.loc[non_donor[HOUSEHOLD_IS_CGT_CLONE].astype(bool)].sort_values( + "household_id" + ) + if len(originals) != len(clones): + problems["clone_half_counts"] = [len(originals), len(clones)] + else: + left = originals["household_weight"].to_numpy(dtype=float) + right = clones["household_weight"].to_numpy(dtype=float) + if not np.allclose(left, right, rtol=1e-12, atol=1e-6): + problems["clone_pair_weights"] = int( + (~np.isclose(left, right, rtol=1e-12, atol=1e-6)).sum() + ) + if not np.isclose(left.sum(), right.sum(), rtol=1e-12, atol=1e-6): + problems["clone_half_masses"] = [float(left.sum()), float(right.sum())] + + # (2) Band-donor selection recomputed from the committed resources. + distribution = load_advani_summers_distribution() + bands = _retained_size_bands(load_hmrc_cgt_size_bands()) + non_donor_ids = set(non_donor["household_id"].tolist()) + nd_person = person.loc[ + person["person_household_id"].isin(non_donor_ids) + ].reset_index(drop=True) + nd_benunit = benunit.loc[ + benunit["benunit_id"].isin(set(nd_person["person_benunit_id"].tolist())) + ].reset_index(drop=True) + nd_household = non_donor.reset_index(drop=True) + multiplier = id_multiplier_for_values( + nd_person["person_id"], + nd_person["person_household_id"], + nd_person["person_benunit_id"], + nd_benunit["benunit_id"], + nd_household["household_id"], + ) + + def select_donors(person_t: pd.DataFrame) -> np.ndarray: + carriers = _oldest_adult_indices(person_t, household_ids=non_donor_ids) + candidates = person_t.loc[carriers].copy() + candidates["_income"] = _component_sum_income(candidates) + candidates["_propensity"] = _incidence_propensity( + candidates["_income"].to_numpy(dtype=float), distribution=distribution + ) + candidates = candidates.sort_values("person_household_id", kind="stable") + propensities = candidates["_propensity"].to_numpy(dtype=float) + rng = np.random.default_rng(DONOR_SEED) + return rng.choice( + candidates["person_household_id"].to_numpy(), + size=DONORS_PER_BAND * len(bands), + replace=False, + p=propensities / propensities.sum(), + ) + + selected = select_donors(nd_person) + permuted_rng = np.random.default_rng(permutation_seed) + selected_permuted = select_donors( + nd_person.iloc[permuted_rng.permutation(len(nd_person))].reset_index(drop=True) + ) + if selected.tolist() != selected_permuted.tolist(): + problems["donor_selection_permutation"] = True + stored_donors = household.loc[donor_mask] + stored_source_ids = set( + (stored_donors["household_id"].astype("int64") - multiplier).tolist() + ) + if stored_source_ids != set(int(value) for value in selected): + problems["donor_selection_stored"] = { + "missing": len(stored_source_ids - set(int(v) for v in selected)), + "extra": len(set(int(v) for v in selected) - stored_source_ids), + } + taxpayers = np.asarray([band["taxpayers"] for band in bands], dtype=float) + means = np.asarray([band["mean_gain"] for band in bands], dtype=float) + band_by_source = { + int(source_id): position // DONORS_PER_BAND + for position, source_id in enumerate(selected) + } + donor_band = ( + (stored_donors["household_id"].astype("int64") - multiplier) + .map(band_by_source) + .to_numpy() + ) + if pd.isna(donor_band).any(): + problems["donor_band_mapping"] = True + else: + donor_band = donor_band.astype(int) + counts = np.bincount(donor_band, minlength=len(bands)) + if not (counts == DONORS_PER_BAND).all(): + problems["donors_per_band"] = counts.tolist() + expected_weights = taxpayers[donor_band] / DONORS_PER_BAND + stored_weights = stored_donors["household_weight"].to_numpy(dtype=float) + if not np.allclose(stored_weights, expected_weights, rtol=1e-12, atol=0.0): + problems["donor_stored_weights"] = True + donor_person = person.loc[ + person["person_household_id"].isin(set(stored_donors["household_id"])) + ] + carrier_rows = _oldest_adult_indices( + donor_person, household_ids=set(stored_donors["household_id"]) + ) + carrier_gain = ( + donor_person.loc[carrier_rows] + .set_index("person_household_id")["capital_gains"] + .reindex(stored_donors["household_id"].to_numpy()) + .to_numpy(dtype=float) + ) + expected_gains = means[donor_band] + # The Table 3 redraw runs after the stack and moves carrier amounts + # within its own gain bands, so band means are not asserted against + # the stored carrier gains bitwise; presence and positivity are. + if not (np.isfinite(carrier_gain) & (carrier_gain > 0.0)).all(): + problems["donor_carrier_gains"] = True + del expected_gains + + # (3) Student-loan plan recomputed in full. + stocks = load_slc_liable_stocks() + year = load_uk_frs_release().calibration_year + recomputed = assign_student_loan_plans(frame, stocks=stocks, year=year) + stored_plan = person.set_index("person_id")["student_loan_plan"] + recomputed_plan = ( + recomputed.frame.table("person") + .set_index("person_id")["student_loan_plan"] + .reindex(stored_plan.index) + ) + plan_matches_store = bool(stored_plan.equals(recomputed_plan)) + permuted_result = assign_student_loan_plans( + _reverse_rows(frame), stocks=stocks, year=year + ) + permuted_plan = ( + permuted_result.frame.table("person") + .set_index("person_id")["student_loan_plan"] + .reindex(stored_plan.index) + ) + plan_permutation_stable = bool(recomputed_plan.equals(permuted_plan)) + if not plan_matches_store: + problems["student_loan_plan_stored"] = True + if not plan_permutation_stable: + problems["student_loan_plan_permutation"] = True + + structural_ok = not problems + return { + "check": "uk_e8_identity_stability", + "permutation_seed": permutation_seed, + "identical_under_permutation": bool( + "donor_selection_permutation" not in problems + and "student_loan_plan_permutation" not in problems + ), + "permutation_mismatches": { + key: value + for key, value in problems.items() + if key.endswith("_permutation") + }, + "matches_stored_columns": bool( + structural_ok + or not any(not key.endswith("_permutation") for key in problems) + ), + "stored_column_mismatches": { + key: value + for key, value in problems.items() + if not key.endswith("_permutation") + }, + "tolerance_policy": ( + "clone-pair weights and half-masses: rtol 1e-12 / atol 1e-6 " + "(the exact-total correction may move single weights by bit " + "corrections); donor stored weights: rtol 1e-12 bitwise-class " + "against published band taxpayers / 30; donor selection and " + "student_loan_plan: exact equality" + ), + "columns_by_entity": { + "household": [ + HOUSEHOLD_IS_CGT_CLONE, + HOUSEHOLD_IS_CGT_BAND_DONOR, + "household_weight", + ], + "person": ["student_loan_plan", "capital_gains"], + }, + "qrf_draw_columns_scope": ( + "excluded: the A&S prior amounts (overwritten by the Table 3 " + "redraw except the sub-AEA remainder), the redraw's seeded " + "within-band draws (the merged #560 embedded published-surface " + "tests cover the amounts logic), and the salary-sacrifice QRF " + "and conversion (the pre-conversion column state is consumed " + "by the stage) are covered by twin-build determinism" + ), + } + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--input-h5", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) - parser.add_argument("--check", choices=("e4", "e5", "e6"), default="e4") + parser.add_argument("--check", choices=("e4", "e5", "e6", "e8"), default="e4") parser.add_argument("--permutation-seed", type=int, default=123) args = parser.parse_args() @@ -536,7 +772,7 @@ def main() -> int: ok = bool( receipt["identical_under_permutation"] and receipt["matches_stored_columns"] ) - else: + elif args.check == "e6": receipt = e6_identity_receipt( frame, permutation_seed=args.permutation_seed, @@ -544,6 +780,14 @@ def main() -> int: ok = bool( receipt["identical_under_permutation"] and receipt["matches_stored_columns"] ) + else: + receipt = e8_identity_receipt( + frame, + permutation_seed=args.permutation_seed, + ) + ok = bool( + receipt["identical_under_permutation"] and receipt["matches_stored_columns"] + ) receipt["input_h5"] = str(args.input_h5) args.output.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n") print( From 45faff0e89679c6ee605e4c9f33c4d010576dee8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sat, 22 Aug 2026 12:13:28 +0200 Subject: [PATCH 3/5] Adversarial-review dispositions: source-verified A&S correction, closed-world CGT asserts, stage-bound mass receipts Five findings from the pre-push adversarial round, all dispositioned: 1. Gate-battery cross-shard triplet re-pinned in both mirrors from the live producer over the post-#733/#735 union (policy 5cb072a0, gates c5123517, fingerprint 23cf63b6) - the rebase had left main's pre-E8-gate values. 2. The Advani-Summers percentile-69 p95 corrected 15,190 -> 151,900, verified against CAGE WP 465 Table A1 (p. 39); all 61 rows checked value-for-value, single mismatch confirmed as the incumbent's dropped digit - fix-and-sign replaces replicate-and-document. The prior-draw loader now fails closed on any non-monotone quantile row. 3. Salary-sacrifice and student-loan receipts gain expected-vs-realized fields (expected mass = rate x pool, realization deviation), quantifying the Bernoulli realization the port-faithful mechanism accepts. 4. The CGT spine drift assert now binds every operation's full declared parameter mapping closed-world (extra keys rejected), not a subset. 5. Salary-sacrifice and student-loans declare stage-named mass-conservation reasons in both manifests (schema extended), emit matching MassChangeRecords, and the regenerated coverage manifest binds each family to its own receipt instead of the generic fallback (pre-E8 families keep the fallback - follow-up noted). Bundle spec sha re-pinned (17c820bc) for the manifest edits; gate triplet unchanged by them. Co-Authored-By: Claude Fable 5 --- .../spec_engine/schema/sources.schema.json | 2 + ...ni_summers_capital_gains_distribution.json | 874 ++++++++++++++++-- .../uk/release_input_coverage_manifest.json | 26 +- .../src/microcosm/build/uk/source_stages.json | 6 +- .../src/microcosm/build/uk/spec/sources.yaml | 4 + .../build/uk_runtime/cgt_imputation.py | 146 ++- .../build/uk_runtime/cgt_structure.py | 12 + .../build/uk_runtime/salary_sacrifice.py | 28 +- .../build/uk_runtime/student_loans.py | 29 +- .../tests/test_country_spec.py | 15 +- .../tests/test_spec_engine_country_bundles.py | 2 +- .../tests/test_uk_cgt_source_manifest.py | 20 + .../src/microcosm/data/contract.py | 6 +- .../microcosm-data/tests/test_contract.py | 6 +- 14 files changed, 1050 insertions(+), 126 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json index 3a9ae12d5..9d29c8090 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json @@ -6628,6 +6628,7 @@ "required": ["kind", "resource", "target", "donor_pool", "rate_cap", "move", "seed", "salt", "receipt"], "properties": { "kind": {"const": "convert_donors_to_target_stock"}, + "reason": {"type": "string"}, "resource": {"type": "string"}, "target": {"type": "number"}, "donor_pool": {"type": "string"}, @@ -6660,6 +6661,7 @@ "required": ["kind", "plan", "priority", "resource", "stock_series", "year_rule", "age_min", "age_max", "cohort_start_min", "eligible_region_exclusions", "highest_education", "seed", "salt"], "properties": { "kind": {"const": "top_up_to_stock"}, + "reason": {"type": "string"}, "plan": {"type": "string"}, "priority": {"type": "integer"}, "resource": {"type": "string"}, diff --git a/packages/microcosm-build/src/microcosm/build/uk/advani_summers_capital_gains_distribution.json b/packages/microcosm-build/src/microcosm/build/uk/advani_summers_capital_gains_distribution.json index 35dd075ff..3d86479bb 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/advani_summers_capital_gains_distribution.json +++ b/packages/microcosm-build/src/microcosm/build/uk/advani_summers_capital_gains_distribution.json @@ -5,72 +5,816 @@ "citation": "Advani, Arun and Andy Summers (May 2020), Capital Gains and UK Inequality, CAGE Working Paper 465, University of Warwick.", "url": "https://warwick.ac.uk/fac/soc/economics/research/centres/cage/manage/publications/wp465.2020.pdf", "incumbent_csv_sha256": "7cb73f8c0a35aeb07c7f13bb0476fb48e9d3bd9b31735f3229554289906297a0", - "columns": ["percentile", "minimum_total_income", "percent_with_gains", "mean_gains_given_gains", "p05", "p10", "p25", "p50", "p75", "p90", "p95"] + "columns": [ + "percentile", + "minimum_total_income", + "percent_with_gains", + "mean_gains_given_gains", + "p05", + "p10", + "p25", + "p50", + "p75", + "p90", + "p95" + ] }, - "known_anomalies": [ - "Percentile 69 has p95=15190 below p90=78200 in the incumbent vintage, likely a dropped digit. It is replicated unchanged because the vintage source table was not available to verify a correction; the later CGT redraw limits its footprint." - ], "rows": [ - {"percentile":"<40","minimum_total_income":0,"percent_with_gains":0.0031,"mean_gains_given_gains":45600,"p05":-16400,"p10":-4400,"p25":3800,"p50":14400,"p75":38400,"p90":92500,"p95":165000}, - {"percentile":40,"minimum_total_income":10000,"percent_with_gains":0.006,"mean_gains_given_gains":50300,"p05":-12600,"p10":-3100,"p25":4000,"p50":13500,"p75":37100,"p90":86900,"p95":156800}, - {"percentile":41,"minimum_total_income":10300,"percent_with_gains":0.0065,"mean_gains_given_gains":49000,"p05":-17900,"p10":-6600,"p25":3200,"p50":13300,"p75":36700,"p90":87700,"p95":142500}, - {"percentile":42,"minimum_total_income":10600,"percent_with_gains":0.0062,"mean_gains_given_gains":41100,"p05":-16200,"p10":-5500,"p25":3300,"p50":13000,"p75":31900,"p90":74000,"p95":125700}, - {"percentile":43,"minimum_total_income":11000,"percent_with_gains":0.0062,"mean_gains_given_gains":42600,"p05":-15100,"p10":-4700,"p25":3700,"p50":13200,"p75":34400,"p90":79500,"p95":143000}, - {"percentile":44,"minimum_total_income":11400,"percent_with_gains":0.0061,"mean_gains_given_gains":41900,"p05":-15900,"p10":-4100,"p25":2800,"p50":12800,"p75":34500,"p90":79400,"p95":144500}, - {"percentile":45,"minimum_total_income":11800,"percent_with_gains":0.0062,"mean_gains_given_gains":41800,"p05":-14500,"p10":-4000,"p25":3500,"p50":12500,"p75":34700,"p90":86900,"p95":147300}, - {"percentile":46,"minimum_total_income":12100,"percent_with_gains":0.0065,"mean_gains_given_gains":47700,"p05":-13800,"p10":-3800,"p25":3300,"p50":12500,"p75":35500,"p90":88400,"p95":158800}, - {"percentile":47,"minimum_total_income":12500,"percent_with_gains":0.0062,"mean_gains_given_gains":52400,"p05":-13500,"p10":-3800,"p25":3100,"p50":13200,"p75":35400,"p90":84100,"p95":158400}, - {"percentile":48,"minimum_total_income":12900,"percent_with_gains":0.0061,"mean_gains_given_gains":43600,"p05":-14200,"p10":-4000,"p25":3800,"p50":12800,"p75":33800,"p90":82200,"p95":137800}, - {"percentile":49,"minimum_total_income":13300,"percent_with_gains":0.0063,"mean_gains_given_gains":38600,"p05":-13700,"p10":-3700,"p25":3200,"p50":12800,"p75":34200,"p90":77000,"p95":129000}, - {"percentile":50,"minimum_total_income":13700,"percent_with_gains":0.0062,"mean_gains_given_gains":48200,"p05":-14100,"p10":-4300,"p25":2900,"p50":12300,"p75":33300,"p90":79400,"p95":150200}, - {"percentile":51,"minimum_total_income":14100,"percent_with_gains":0.006,"mean_gains_given_gains":40300,"p05":-14700,"p10":-4300,"p25":3100,"p50":12300,"p75":32900,"p90":79800,"p95":137700}, - {"percentile":52,"minimum_total_income":14500,"percent_with_gains":0.0063,"mean_gains_given_gains":46700,"p05":-10700,"p10":-3300,"p25":3600,"p50":12900,"p75":33500,"p90":74400,"p95":145500}, - {"percentile":53,"minimum_total_income":14900,"percent_with_gains":0.0062,"mean_gains_given_gains":49900,"p05":-15300,"p10":-5000,"p25":2400,"p50":11700,"p75":32800,"p90":78300,"p95":141700}, - {"percentile":54,"minimum_total_income":15300,"percent_with_gains":0.0062,"mean_gains_given_gains":38700,"p05":-15800,"p10":-4800,"p25":2800,"p50":11900,"p75":32400,"p90":80000,"p95":153800}, - {"percentile":55,"minimum_total_income":15700,"percent_with_gains":0.0063,"mean_gains_given_gains":40300,"p05":-12100,"p10":-4200,"p25":3000,"p50":12500,"p75":33000,"p90":79100,"p95":137000}, - {"percentile":56,"minimum_total_income":16100,"percent_with_gains":0.0063,"mean_gains_given_gains":42600,"p05":-13400,"p10":-3900,"p25":3400,"p50":12300,"p75":33800,"p90":82200,"p95":152800}, - {"percentile":57,"minimum_total_income":16600,"percent_with_gains":0.0062,"mean_gains_given_gains":50800,"p05":-12700,"p10":-3500,"p25":3700,"p50":13300,"p75":34900,"p90":88900,"p95":157600}, - {"percentile":58,"minimum_total_income":17000,"percent_with_gains":0.0065,"mean_gains_given_gains":49100,"p05":-16100,"p10":-4700,"p25":3100,"p50":11900,"p75":31900,"p90":80900,"p95":146800}, - {"percentile":59,"minimum_total_income":17400,"percent_with_gains":0.0064,"mean_gains_given_gains":42100,"p05":-15700,"p10":-4300,"p25":3100,"p50":11900,"p75":32500,"p90":81700,"p95":152100}, - {"percentile":60,"minimum_total_income":17900,"percent_with_gains":0.0065,"mean_gains_given_gains":50800,"p05":-13100,"p10":-4300,"p25":3100,"p50":12300,"p75":32600,"p90":79400,"p95":157000}, - {"percentile":61,"minimum_total_income":18300,"percent_with_gains":0.0062,"mean_gains_given_gains":42200,"p05":-13800,"p10":-4100,"p25":3300,"p50":12400,"p75":32300,"p90":78100,"p95":146700}, - {"percentile":62,"minimum_total_income":18800,"percent_with_gains":0.0065,"mean_gains_given_gains":39900,"p05":-13400,"p10":-4600,"p25":3200,"p50":12200,"p75":32500,"p90":80700,"p95":138500}, - {"percentile":63,"minimum_total_income":19200,"percent_with_gains":0.0069,"mean_gains_given_gains":46900,"p05":-12700,"p10":-3400,"p25":3500,"p50":12100,"p75":32100,"p90":84000,"p95":167000}, - {"percentile":64,"minimum_total_income":19700,"percent_with_gains":0.0069,"mean_gains_given_gains":43600,"p05":-13800,"p10":-4000,"p25":3200,"p50":12400,"p75":31000,"p90":85700,"p95":152000}, - {"percentile":65,"minimum_total_income":20200,"percent_with_gains":0.0069,"mean_gains_given_gains":52600,"p05":-14200,"p10":-4200,"p25":3600,"p50":12100,"p75":34800,"p90":88900,"p95":176400}, - {"percentile":66,"minimum_total_income":20700,"percent_with_gains":0.0069,"mean_gains_given_gains":39500,"p05":-11800,"p10":-3800,"p25":4000,"p50":12700,"p75":33300,"p90":78500,"p95":143300}, - {"percentile":67,"minimum_total_income":21200,"percent_with_gains":0.0072,"mean_gains_given_gains":47900,"p05":-11000,"p10":-3300,"p25":3900,"p50":12500,"p75":32700,"p90":79200,"p95":155400}, - {"percentile":68,"minimum_total_income":21800,"percent_with_gains":0.0074,"mean_gains_given_gains":40900,"p05":-12800,"p10":-4200,"p25":3300,"p50":11800,"p75":31200,"p90":72700,"p95":122300}, - {"percentile":69,"minimum_total_income":22300,"percent_with_gains":0.0077,"mean_gains_given_gains":46300,"p05":-13700,"p10":-4100,"p25":2900,"p50":11800,"p75":32100,"p90":78200,"p95":15190}, - {"percentile":70,"minimum_total_income":22900,"percent_with_gains":0.0077,"mean_gains_given_gains":50600,"p05":-10600,"p10":-3400,"p25":3400,"p50":12200,"p75":31700,"p90":75800,"p95":148000}, - {"percentile":71,"minimum_total_income":23500,"percent_with_gains":0.0082,"mean_gains_given_gains":41200,"p05":-11600,"p10":-3500,"p25":3500,"p50":12100,"p75":32600,"p90":83400,"p95":146800}, - {"percentile":72,"minimum_total_income":24100,"percent_with_gains":0.0084,"mean_gains_given_gains":42000,"p05":-12400,"p10":-4600,"p25":2900,"p50":11300,"p75":29900,"p90":77400,"p95":154000}, - {"percentile":73,"minimum_total_income":24800,"percent_with_gains":0.0086,"mean_gains_given_gains":49000,"p05":-15300,"p10":-5100,"p25":3100,"p50":11900,"p75":31700,"p90":83800,"p95":161800}, - {"percentile":74,"minimum_total_income":25400,"percent_with_gains":0.0084,"mean_gains_given_gains":41400,"p05":-14700,"p10":-4000,"p25":3300,"p50":11800,"p75":30000,"p90":82600,"p95":154200}, - {"percentile":75,"minimum_total_income":26100,"percent_with_gains":0.0087,"mean_gains_given_gains":42600,"p05":-16400,"p10":-5000,"p25":2900,"p50":11500,"p75":30900,"p90":83200,"p95":167800}, - {"percentile":76,"minimum_total_income":26800,"percent_with_gains":0.0092,"mean_gains_given_gains":42400,"p05":-13800,"p10":-3900,"p25":3300,"p50":11800,"p75":31500,"p90":84500,"p95":162700}, - {"percentile":77,"minimum_total_income":27500,"percent_with_gains":0.0094,"mean_gains_given_gains":42600,"p05":-13200,"p10":-3800,"p25":3800,"p50":12300,"p75":33200,"p90":82600,"p95":145000}, - {"percentile":78,"minimum_total_income":28300,"percent_with_gains":0.0097,"mean_gains_given_gains":51900,"p05":-14900,"p10":-3800,"p25":4100,"p50":12500,"p75":31900,"p90":82400,"p95":148500}, - {"percentile":79,"minimum_total_income":29100,"percent_with_gains":0.0102,"mean_gains_given_gains":44700,"p05":-12600,"p10":-4000,"p25":3000,"p50":11700,"p75":29200,"p90":75700,"p95":145400}, - {"percentile":80,"minimum_total_income":30000,"percent_with_gains":0.0103,"mean_gains_given_gains":49800,"p05":-15400,"p10":-4200,"p25":4000,"p50":12500,"p75":32700,"p90":93600,"p95":184700}, - {"percentile":81,"minimum_total_income":30900,"percent_with_gains":0.0112,"mean_gains_given_gains":50400,"p05":-11900,"p10":-3300,"p25":3200,"p50":11900,"p75":32300,"p90":87500,"p95":164400}, - {"percentile":82,"minimum_total_income":31800,"percent_with_gains":0.0113,"mean_gains_given_gains":46200,"p05":-13900,"p10":-4300,"p25":3100,"p50":11500,"p75":30100,"p90":86700,"p95":165700}, - {"percentile":83,"minimum_total_income":32800,"percent_with_gains":0.0118,"mean_gains_given_gains":47800,"p05":-12800,"p10":-3600,"p25":3300,"p50":11700,"p75":31700,"p90":91900,"p95":174100}, - {"percentile":84,"minimum_total_income":33900,"percent_with_gains":0.0121,"mean_gains_given_gains":46000,"p05":-13700,"p10":-4400,"p25":3600,"p50":11800,"p75":30700,"p90":93800,"p95":178300}, - {"percentile":85,"minimum_total_income":35000,"percent_with_gains":0.0136,"mean_gains_given_gains":46800,"p05":-14300,"p10":-4100,"p25":3700,"p50":12200,"p75":30800,"p90":90000,"p95":181200}, - {"percentile":86,"minimum_total_income":36200,"percent_with_gains":0.0147,"mean_gains_given_gains":55100,"p05":-13200,"p10":-4200,"p25":3500,"p50":12000,"p75":33300,"p90":100800,"p95":206800}, - {"percentile":87,"minimum_total_income":37400,"percent_with_gains":0.0172,"mean_gains_given_gains":59400,"p05":-14500,"p10":-4500,"p25":3600,"p50":11800,"p75":33800,"p90":108100,"p95":212600}, - {"percentile":88,"minimum_total_income":38700,"percent_with_gains":0.0191,"mean_gains_given_gains":52100,"p05":-15000,"p10":-5300,"p25":2300,"p50":11400,"p75":35100,"p90":110800,"p95":221600}, - {"percentile":89,"minimum_total_income":39900,"percent_with_gains":0.02,"mean_gains_given_gains":55600,"p05":-16700,"p10":-4900,"p25":2900,"p50":11500,"p75":34000,"p90":111700,"p95":226300}, - {"percentile":90,"minimum_total_income":41500,"percent_with_gains":0.0204,"mean_gains_given_gains":59000,"p05":-17200,"p10":-5500,"p25":2400,"p50":11400,"p75":32500,"p90":99800,"p95":209800}, - {"percentile":91,"minimum_total_income":43300,"percent_with_gains":0.021,"mean_gains_given_gains":53200,"p05":-16500,"p10":-5100,"p25":2800,"p50":11400,"p75":30700,"p90":99900,"p95":213000}, - {"percentile":92,"minimum_total_income":45500,"percent_with_gains":0.0226,"mean_gains_given_gains":58500,"p05":-16100,"p10":-5000,"p25":3200,"p50":11600,"p75":32700,"p90":100900,"p95":208600}, - {"percentile":93,"minimum_total_income":48300,"percent_with_gains":0.0247,"mean_gains_given_gains":68300,"p05":-14900,"p10":-4700,"p25":3800,"p50":11700,"p75":34600,"p90":106800,"p95":226200}, - {"percentile":94,"minimum_total_income":51900,"percent_with_gains":0.0281,"mean_gains_given_gains":50600,"p05":-15500,"p10":-4800,"p25":3400,"p50":11700,"p75":35500,"p90":113500,"p95":250000}, - {"percentile":95,"minimum_total_income":56700,"percent_with_gains":0.0326,"mean_gains_given_gains":73800,"p05":-17800,"p10":-5800,"p25":3300,"p50":11700,"p75":35000,"p90":120500,"p95":265700}, - {"percentile":96,"minimum_total_income":63400,"percent_with_gains":0.0391,"mean_gains_given_gains":83600,"p05":-18300,"p10":-5600,"p25":3600,"p50":12000,"p75":38100,"p90":134400,"p95":310900}, - {"percentile":97,"minimum_total_income":73400,"percent_with_gains":0.0503,"mean_gains_given_gains":96100,"p05":-20200,"p10":-5900,"p25":3600,"p50":12500,"p75":41900,"p90":161500,"p95":373900}, - {"percentile":98,"minimum_total_income":90100,"percent_with_gains":0.071,"mean_gains_given_gains":120400,"p05":-24100,"p10":-7500,"p25":2900,"p50":12700,"p75":48000,"p90":200000,"p95":470000}, - {"percentile":99,"minimum_total_income":128200,"percent_with_gains":0.1508,"mean_gains_given_gains":306800,"p05":-42800,"p10":-12400,"p25":200,"p50":13600,"p75":74900,"p90":431600,"p95":1162400} + { + "percentile": "<40", + "minimum_total_income": 0, + "percent_with_gains": 0.0031, + "mean_gains_given_gains": 45600, + "p05": -16400, + "p10": -4400, + "p25": 3800, + "p50": 14400, + "p75": 38400, + "p90": 92500, + "p95": 165000 + }, + { + "percentile": 40, + "minimum_total_income": 10000, + "percent_with_gains": 0.006, + "mean_gains_given_gains": 50300, + "p05": -12600, + "p10": -3100, + "p25": 4000, + "p50": 13500, + "p75": 37100, + "p90": 86900, + "p95": 156800 + }, + { + "percentile": 41, + "minimum_total_income": 10300, + "percent_with_gains": 0.0065, + "mean_gains_given_gains": 49000, + "p05": -17900, + "p10": -6600, + "p25": 3200, + "p50": 13300, + "p75": 36700, + "p90": 87700, + "p95": 142500 + }, + { + "percentile": 42, + "minimum_total_income": 10600, + "percent_with_gains": 0.0062, + "mean_gains_given_gains": 41100, + "p05": -16200, + "p10": -5500, + "p25": 3300, + "p50": 13000, + "p75": 31900, + "p90": 74000, + "p95": 125700 + }, + { + "percentile": 43, + "minimum_total_income": 11000, + "percent_with_gains": 0.0062, + "mean_gains_given_gains": 42600, + "p05": -15100, + "p10": -4700, + "p25": 3700, + "p50": 13200, + "p75": 34400, + "p90": 79500, + "p95": 143000 + }, + { + "percentile": 44, + "minimum_total_income": 11400, + "percent_with_gains": 0.0061, + "mean_gains_given_gains": 41900, + "p05": -15900, + "p10": -4100, + "p25": 2800, + "p50": 12800, + "p75": 34500, + "p90": 79400, + "p95": 144500 + }, + { + "percentile": 45, + "minimum_total_income": 11800, + "percent_with_gains": 0.0062, + "mean_gains_given_gains": 41800, + "p05": -14500, + "p10": -4000, + "p25": 3500, + "p50": 12500, + "p75": 34700, + "p90": 86900, + "p95": 147300 + }, + { + "percentile": 46, + "minimum_total_income": 12100, + "percent_with_gains": 0.0065, + "mean_gains_given_gains": 47700, + "p05": -13800, + "p10": -3800, + "p25": 3300, + "p50": 12500, + "p75": 35500, + "p90": 88400, + "p95": 158800 + }, + { + "percentile": 47, + "minimum_total_income": 12500, + "percent_with_gains": 0.0062, + "mean_gains_given_gains": 52400, + "p05": -13500, + "p10": -3800, + "p25": 3100, + "p50": 13200, + "p75": 35400, + "p90": 84100, + "p95": 158400 + }, + { + "percentile": 48, + "minimum_total_income": 12900, + "percent_with_gains": 0.0061, + "mean_gains_given_gains": 43600, + "p05": -14200, + "p10": -4000, + "p25": 3800, + "p50": 12800, + "p75": 33800, + "p90": 82200, + "p95": 137800 + }, + { + "percentile": 49, + "minimum_total_income": 13300, + "percent_with_gains": 0.0063, + "mean_gains_given_gains": 38600, + "p05": -13700, + "p10": -3700, + "p25": 3200, + "p50": 12800, + "p75": 34200, + "p90": 77000, + "p95": 129000 + }, + { + "percentile": 50, + "minimum_total_income": 13700, + "percent_with_gains": 0.0062, + "mean_gains_given_gains": 48200, + "p05": -14100, + "p10": -4300, + "p25": 2900, + "p50": 12300, + "p75": 33300, + "p90": 79400, + "p95": 150200 + }, + { + "percentile": 51, + "minimum_total_income": 14100, + "percent_with_gains": 0.006, + "mean_gains_given_gains": 40300, + "p05": -14700, + "p10": -4300, + "p25": 3100, + "p50": 12300, + "p75": 32900, + "p90": 79800, + "p95": 137700 + }, + { + "percentile": 52, + "minimum_total_income": 14500, + "percent_with_gains": 0.0063, + "mean_gains_given_gains": 46700, + "p05": -10700, + "p10": -3300, + "p25": 3600, + "p50": 12900, + "p75": 33500, + "p90": 74400, + "p95": 145500 + }, + { + "percentile": 53, + "minimum_total_income": 14900, + "percent_with_gains": 0.0062, + "mean_gains_given_gains": 49900, + "p05": -15300, + "p10": -5000, + "p25": 2400, + "p50": 11700, + "p75": 32800, + "p90": 78300, + "p95": 141700 + }, + { + "percentile": 54, + "minimum_total_income": 15300, + "percent_with_gains": 0.0062, + "mean_gains_given_gains": 38700, + "p05": -15800, + "p10": -4800, + "p25": 2800, + "p50": 11900, + "p75": 32400, + "p90": 80000, + "p95": 153800 + }, + { + "percentile": 55, + "minimum_total_income": 15700, + "percent_with_gains": 0.0063, + "mean_gains_given_gains": 40300, + "p05": -12100, + "p10": -4200, + "p25": 3000, + "p50": 12500, + "p75": 33000, + "p90": 79100, + "p95": 137000 + }, + { + "percentile": 56, + "minimum_total_income": 16100, + "percent_with_gains": 0.0063, + "mean_gains_given_gains": 42600, + "p05": -13400, + "p10": -3900, + "p25": 3400, + "p50": 12300, + "p75": 33800, + "p90": 82200, + "p95": 152800 + }, + { + "percentile": 57, + "minimum_total_income": 16600, + "percent_with_gains": 0.0062, + "mean_gains_given_gains": 50800, + "p05": -12700, + "p10": -3500, + "p25": 3700, + "p50": 13300, + "p75": 34900, + "p90": 88900, + "p95": 157600 + }, + { + "percentile": 58, + "minimum_total_income": 17000, + "percent_with_gains": 0.0065, + "mean_gains_given_gains": 49100, + "p05": -16100, + "p10": -4700, + "p25": 3100, + "p50": 11900, + "p75": 31900, + "p90": 80900, + "p95": 146800 + }, + { + "percentile": 59, + "minimum_total_income": 17400, + "percent_with_gains": 0.0064, + "mean_gains_given_gains": 42100, + "p05": -15700, + "p10": -4300, + "p25": 3100, + "p50": 11900, + "p75": 32500, + "p90": 81700, + "p95": 152100 + }, + { + "percentile": 60, + "minimum_total_income": 17900, + "percent_with_gains": 0.0065, + "mean_gains_given_gains": 50800, + "p05": -13100, + "p10": -4300, + "p25": 3100, + "p50": 12300, + "p75": 32600, + "p90": 79400, + "p95": 157000 + }, + { + "percentile": 61, + "minimum_total_income": 18300, + "percent_with_gains": 0.0062, + "mean_gains_given_gains": 42200, + "p05": -13800, + "p10": -4100, + "p25": 3300, + "p50": 12400, + "p75": 32300, + "p90": 78100, + "p95": 146700 + }, + { + "percentile": 62, + "minimum_total_income": 18800, + "percent_with_gains": 0.0065, + "mean_gains_given_gains": 39900, + "p05": -13400, + "p10": -4600, + "p25": 3200, + "p50": 12200, + "p75": 32500, + "p90": 80700, + "p95": 138500 + }, + { + "percentile": 63, + "minimum_total_income": 19200, + "percent_with_gains": 0.0069, + "mean_gains_given_gains": 46900, + "p05": -12700, + "p10": -3400, + "p25": 3500, + "p50": 12100, + "p75": 32100, + "p90": 84000, + "p95": 167000 + }, + { + "percentile": 64, + "minimum_total_income": 19700, + "percent_with_gains": 0.0069, + "mean_gains_given_gains": 43600, + "p05": -13800, + "p10": -4000, + "p25": 3200, + "p50": 12400, + "p75": 31000, + "p90": 85700, + "p95": 152000 + }, + { + "percentile": 65, + "minimum_total_income": 20200, + "percent_with_gains": 0.0069, + "mean_gains_given_gains": 52600, + "p05": -14200, + "p10": -4200, + "p25": 3600, + "p50": 12100, + "p75": 34800, + "p90": 88900, + "p95": 176400 + }, + { + "percentile": 66, + "minimum_total_income": 20700, + "percent_with_gains": 0.0069, + "mean_gains_given_gains": 39500, + "p05": -11800, + "p10": -3800, + "p25": 4000, + "p50": 12700, + "p75": 33300, + "p90": 78500, + "p95": 143300 + }, + { + "percentile": 67, + "minimum_total_income": 21200, + "percent_with_gains": 0.0072, + "mean_gains_given_gains": 47900, + "p05": -11000, + "p10": -3300, + "p25": 3900, + "p50": 12500, + "p75": 32700, + "p90": 79200, + "p95": 155400 + }, + { + "percentile": 68, + "minimum_total_income": 21800, + "percent_with_gains": 0.0074, + "mean_gains_given_gains": 40900, + "p05": -12800, + "p10": -4200, + "p25": 3300, + "p50": 11800, + "p75": 31200, + "p90": 72700, + "p95": 122300 + }, + { + "percentile": 69, + "minimum_total_income": 22300, + "percent_with_gains": 0.0077, + "mean_gains_given_gains": 46300, + "p05": -13700, + "p10": -4100, + "p25": 2900, + "p50": 11800, + "p75": 32100, + "p90": 78200, + "p95": 151900 + }, + { + "percentile": 70, + "minimum_total_income": 22900, + "percent_with_gains": 0.0077, + "mean_gains_given_gains": 50600, + "p05": -10600, + "p10": -3400, + "p25": 3400, + "p50": 12200, + "p75": 31700, + "p90": 75800, + "p95": 148000 + }, + { + "percentile": 71, + "minimum_total_income": 23500, + "percent_with_gains": 0.0082, + "mean_gains_given_gains": 41200, + "p05": -11600, + "p10": -3500, + "p25": 3500, + "p50": 12100, + "p75": 32600, + "p90": 83400, + "p95": 146800 + }, + { + "percentile": 72, + "minimum_total_income": 24100, + "percent_with_gains": 0.0084, + "mean_gains_given_gains": 42000, + "p05": -12400, + "p10": -4600, + "p25": 2900, + "p50": 11300, + "p75": 29900, + "p90": 77400, + "p95": 154000 + }, + { + "percentile": 73, + "minimum_total_income": 24800, + "percent_with_gains": 0.0086, + "mean_gains_given_gains": 49000, + "p05": -15300, + "p10": -5100, + "p25": 3100, + "p50": 11900, + "p75": 31700, + "p90": 83800, + "p95": 161800 + }, + { + "percentile": 74, + "minimum_total_income": 25400, + "percent_with_gains": 0.0084, + "mean_gains_given_gains": 41400, + "p05": -14700, + "p10": -4000, + "p25": 3300, + "p50": 11800, + "p75": 30000, + "p90": 82600, + "p95": 154200 + }, + { + "percentile": 75, + "minimum_total_income": 26100, + "percent_with_gains": 0.0087, + "mean_gains_given_gains": 42600, + "p05": -16400, + "p10": -5000, + "p25": 2900, + "p50": 11500, + "p75": 30900, + "p90": 83200, + "p95": 167800 + }, + { + "percentile": 76, + "minimum_total_income": 26800, + "percent_with_gains": 0.0092, + "mean_gains_given_gains": 42400, + "p05": -13800, + "p10": -3900, + "p25": 3300, + "p50": 11800, + "p75": 31500, + "p90": 84500, + "p95": 162700 + }, + { + "percentile": 77, + "minimum_total_income": 27500, + "percent_with_gains": 0.0094, + "mean_gains_given_gains": 42600, + "p05": -13200, + "p10": -3800, + "p25": 3800, + "p50": 12300, + "p75": 33200, + "p90": 82600, + "p95": 145000 + }, + { + "percentile": 78, + "minimum_total_income": 28300, + "percent_with_gains": 0.0097, + "mean_gains_given_gains": 51900, + "p05": -14900, + "p10": -3800, + "p25": 4100, + "p50": 12500, + "p75": 31900, + "p90": 82400, + "p95": 148500 + }, + { + "percentile": 79, + "minimum_total_income": 29100, + "percent_with_gains": 0.0102, + "mean_gains_given_gains": 44700, + "p05": -12600, + "p10": -4000, + "p25": 3000, + "p50": 11700, + "p75": 29200, + "p90": 75700, + "p95": 145400 + }, + { + "percentile": 80, + "minimum_total_income": 30000, + "percent_with_gains": 0.0103, + "mean_gains_given_gains": 49800, + "p05": -15400, + "p10": -4200, + "p25": 4000, + "p50": 12500, + "p75": 32700, + "p90": 93600, + "p95": 184700 + }, + { + "percentile": 81, + "minimum_total_income": 30900, + "percent_with_gains": 0.0112, + "mean_gains_given_gains": 50400, + "p05": -11900, + "p10": -3300, + "p25": 3200, + "p50": 11900, + "p75": 32300, + "p90": 87500, + "p95": 164400 + }, + { + "percentile": 82, + "minimum_total_income": 31800, + "percent_with_gains": 0.0113, + "mean_gains_given_gains": 46200, + "p05": -13900, + "p10": -4300, + "p25": 3100, + "p50": 11500, + "p75": 30100, + "p90": 86700, + "p95": 165700 + }, + { + "percentile": 83, + "minimum_total_income": 32800, + "percent_with_gains": 0.0118, + "mean_gains_given_gains": 47800, + "p05": -12800, + "p10": -3600, + "p25": 3300, + "p50": 11700, + "p75": 31700, + "p90": 91900, + "p95": 174100 + }, + { + "percentile": 84, + "minimum_total_income": 33900, + "percent_with_gains": 0.0121, + "mean_gains_given_gains": 46000, + "p05": -13700, + "p10": -4400, + "p25": 3600, + "p50": 11800, + "p75": 30700, + "p90": 93800, + "p95": 178300 + }, + { + "percentile": 85, + "minimum_total_income": 35000, + "percent_with_gains": 0.0136, + "mean_gains_given_gains": 46800, + "p05": -14300, + "p10": -4100, + "p25": 3700, + "p50": 12200, + "p75": 30800, + "p90": 90000, + "p95": 181200 + }, + { + "percentile": 86, + "minimum_total_income": 36200, + "percent_with_gains": 0.0147, + "mean_gains_given_gains": 55100, + "p05": -13200, + "p10": -4200, + "p25": 3500, + "p50": 12000, + "p75": 33300, + "p90": 100800, + "p95": 206800 + }, + { + "percentile": 87, + "minimum_total_income": 37400, + "percent_with_gains": 0.0172, + "mean_gains_given_gains": 59400, + "p05": -14500, + "p10": -4500, + "p25": 3600, + "p50": 11800, + "p75": 33800, + "p90": 108100, + "p95": 212600 + }, + { + "percentile": 88, + "minimum_total_income": 38700, + "percent_with_gains": 0.0191, + "mean_gains_given_gains": 52100, + "p05": -15000, + "p10": -5300, + "p25": 2300, + "p50": 11400, + "p75": 35100, + "p90": 110800, + "p95": 221600 + }, + { + "percentile": 89, + "minimum_total_income": 39900, + "percent_with_gains": 0.02, + "mean_gains_given_gains": 55600, + "p05": -16700, + "p10": -4900, + "p25": 2900, + "p50": 11500, + "p75": 34000, + "p90": 111700, + "p95": 226300 + }, + { + "percentile": 90, + "minimum_total_income": 41500, + "percent_with_gains": 0.0204, + "mean_gains_given_gains": 59000, + "p05": -17200, + "p10": -5500, + "p25": 2400, + "p50": 11400, + "p75": 32500, + "p90": 99800, + "p95": 209800 + }, + { + "percentile": 91, + "minimum_total_income": 43300, + "percent_with_gains": 0.021, + "mean_gains_given_gains": 53200, + "p05": -16500, + "p10": -5100, + "p25": 2800, + "p50": 11400, + "p75": 30700, + "p90": 99900, + "p95": 213000 + }, + { + "percentile": 92, + "minimum_total_income": 45500, + "percent_with_gains": 0.0226, + "mean_gains_given_gains": 58500, + "p05": -16100, + "p10": -5000, + "p25": 3200, + "p50": 11600, + "p75": 32700, + "p90": 100900, + "p95": 208600 + }, + { + "percentile": 93, + "minimum_total_income": 48300, + "percent_with_gains": 0.0247, + "mean_gains_given_gains": 68300, + "p05": -14900, + "p10": -4700, + "p25": 3800, + "p50": 11700, + "p75": 34600, + "p90": 106800, + "p95": 226200 + }, + { + "percentile": 94, + "minimum_total_income": 51900, + "percent_with_gains": 0.0281, + "mean_gains_given_gains": 50600, + "p05": -15500, + "p10": -4800, + "p25": 3400, + "p50": 11700, + "p75": 35500, + "p90": 113500, + "p95": 250000 + }, + { + "percentile": 95, + "minimum_total_income": 56700, + "percent_with_gains": 0.0326, + "mean_gains_given_gains": 73800, + "p05": -17800, + "p10": -5800, + "p25": 3300, + "p50": 11700, + "p75": 35000, + "p90": 120500, + "p95": 265700 + }, + { + "percentile": 96, + "minimum_total_income": 63400, + "percent_with_gains": 0.0391, + "mean_gains_given_gains": 83600, + "p05": -18300, + "p10": -5600, + "p25": 3600, + "p50": 12000, + "p75": 38100, + "p90": 134400, + "p95": 310900 + }, + { + "percentile": 97, + "minimum_total_income": 73400, + "percent_with_gains": 0.0503, + "mean_gains_given_gains": 96100, + "p05": -20200, + "p10": -5900, + "p25": 3600, + "p50": 12500, + "p75": 41900, + "p90": 161500, + "p95": 373900 + }, + { + "percentile": 98, + "minimum_total_income": 90100, + "percent_with_gains": 0.071, + "mean_gains_given_gains": 120400, + "p05": -24100, + "p10": -7500, + "p25": 2900, + "p50": 12700, + "p75": 48000, + "p90": 200000, + "p95": 470000 + }, + { + "percentile": 99, + "minimum_total_income": 128200, + "percent_with_gains": 0.1508, + "mean_gains_given_gains": 306800, + "p05": -42800, + "p10": -12400, + "p25": 200, + "p50": 13600, + "p75": 74900, + "p90": 431600, + "p95": 1162400 + } + ], + "corrections": [ + "Percentile 69 p95 corrected from the incumbent's 15,190 to 151,900: verified against Advani & Summers (2020), CAGE WP 465, Table A1 (p. 39) - '69 22,300 .0077 46,300 -13,700 -4,100 2,900 11,800 32,100 78,200 151,900'. The incumbent carries a dropped-digit transcription; all other 60 rows verified value-for-value against Table A1 (fix-and-sign, 2026-08-22). The signed effect vs the incumbent: band-69 prior draws above q~0.962 are no longer spurious loss-makers." ] } diff --git a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json index 0f9c19d58..c1924824c 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json @@ -473,7 +473,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "9beda5ab8d9f16ddacecfa1530e6679f7cb00a49eae01b671f318be04ce1bef6", + "source_manifest_sha256": "029545d43a254538baa05b6930e8a32019605a7543b9108c4d14b20e5b616881", "source_vintages": { "source": "HMRC Capital Gains Tax statistics, July 2025, Table 2.1a", "survey": "HMRC Capital Gains Tax statistics Table 2.1a and Advani-Summers capital-gains incidence" @@ -496,7 +496,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "9beda5ab8d9f16ddacecfa1530e6679f7cb00a49eae01b671f318be04ce1bef6", + "source_manifest_sha256": "029545d43a254538baa05b6930e8a32019605a7543b9108c4d14b20e5b616881", "source_vintages": { "source": "Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465", "survey": "Family Resources Survey 2024-25, SPI synthetic support, and Advani-Summers capital-gains incidence" @@ -525,7 +525,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "9beda5ab8d9f16ddacecfa1530e6679f7cb00a49eae01b671f318be04ce1bef6", + "source_manifest_sha256": "029545d43a254538baa05b6930e8a32019605a7543b9108c4d14b20e5b616881", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab, DfT rail fare index, and public NHS activity/cost table.", "survey": "Effects of Taxes and Benefits 1977-2024 and NHS age-gender public table" @@ -545,7 +545,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "9beda5ab8d9f16ddacecfa1530e6679f7cb00a49eae01b671f318be04ce1bef6", + "source_manifest_sha256": "029545d43a254538baa05b6930e8a32019605a7543b9108c4d14b20e5b616881", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab and cited VAT anchor resource.", "survey": "Effects of Taxes and Benefits 1977-2024" @@ -590,7 +590,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "9beda5ab8d9f16ddacecfa1530e6679f7cb00a49eae01b671f318be04ce1bef6", + "source_manifest_sha256": "029545d43a254538baa05b6930e8a32019605a7543b9108c4d14b20e5b616881", "source_vintages": { "hmrc_surface": "2023-24", "mapped_build_period": "2024" @@ -604,7 +604,7 @@ "base_candidate_tier": "frs", "calibration_permitted": false, "canonical_source_manifest": "source_stages.json", - "canonical_source_manifest_sha256": "9beda5ab8d9f16ddacecfa1530e6679f7cb00a49eae01b671f318be04ce1bef6", + "canonical_source_manifest_sha256": "029545d43a254538baa05b6930e8a32019605a7543b9108c4d14b20e5b616881", "effective_mass_requirements": { "charitable_investment_gifts": { "mass_share_denominator": "all_person_effective_mass", @@ -715,7 +715,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "9beda5ab8d9f16ddacecfa1530e6679f7cb00a49eae01b671f318be04ce1bef6", + "source_manifest_sha256": "029545d43a254538baa05b6930e8a32019605a7543b9108c4d14b20e5b616881", "source_vintages": { "source": "UK Data Service SN 9468 Living Costs and Food Survey 2023-24 household/person tabs, NEED 2023 headline energy tables, Ofgem Q2 2026 unit rates, and WAS round-8 bridge donor.", "survey": "Living Costs and Food Survey 2023-24" @@ -736,7 +736,7 @@ "property_wealth" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "9beda5ab8d9f16ddacecfa1530e6679f7cb00a49eae01b671f318be04ce1bef6", + "source_manifest_sha256": "029545d43a254538baa05b6930e8a32019605a7543b9108c4d14b20e5b616881", "source_vintages": { "source": "MHCLG dwellings and ONS UK House Price Index December 2025 regional average prices.", "survey": "Public regional property reference" @@ -754,13 +754,13 @@ "pension_contributions_via_salary_sacrifice", "employee_pension_contributions" ], - "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", + "required_mass_change_reason": "Salary-sacrifice support stage rewrites pension columns only; household rows and typed household weights pass through and total household mass is conserved.", "rewrites": [ "pension_contributions_via_salary_sacrifice", "employee_pension_contributions" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "9beda5ab8d9f16ddacecfa1530e6679f7cb00a49eae01b671f318be04ce1bef6", + "source_manifest_sha256": "029545d43a254538baa05b6930e8a32019605a7543b9108c4d14b20e5b616881", "source_vintages": { "source": "HMRC, Salary sacrifice reform for pension contributions effective from 6 April 2029", "survey": "Family Resources Survey 2024-25 salary-sacrifice respondents and HMRC salary-sacrifice reform analysis" @@ -777,12 +777,12 @@ "outputs": [ "student_loan_plan" ], - "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", + "required_mass_change_reason": "Student-loan plan assignment writes an enum column only; household rows and typed household weights pass through and total household mass is conserved.", "rewrites": [ "student_loan_plan" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "9beda5ab8d9f16ddacecfa1530e6679f7cb00a49eae01b671f318be04ce1bef6", + "source_manifest_sha256": "029545d43a254538baa05b6930e8a32019605a7543b9108c4d14b20e5b616881", "source_vintages": { "source": "Explore Education Statistics Table 6a, Higher education total", "survey": "Family Resources Survey 2024-25 and Student Loans Company borrower forecasts for England" @@ -814,7 +814,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "9beda5ab8d9f16ddacecfa1530e6679f7cb00a49eae01b671f318be04ce1bef6", + "source_manifest_sha256": "029545d43a254538baa05b6930e8a32019605a7543b9108c4d14b20e5b616881", "source_vintages": { "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed 2006-22 household tab.", "survey": "Wealth and Assets Survey round 8" diff --git a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json index f726f9fa0..aa03f18d9 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json @@ -2657,7 +2657,8 @@ "move": "full employee_pension_contributions to pension_contributions_via_salary_sacrifice; source zeroed", "seed": 2024, "salt": "salary_sacrifice_conversion", - "receipt": "weighted_headcount" + "receipt": "weighted_headcount", + "reason": "Salary-sacrifice support stage rewrites pension columns only; household rows and typed household weights pass through and total household mass is conserved." } ], "outputs": [ @@ -2736,7 +2737,8 @@ "eligible_region_exclusions": ["SCOTLAND", "WALES", "NORTHERN_IRELAND"], "highest_education": "TERTIARY", "seed": 42, - "salt": "student_loan_plan_2" + "salt": "student_loan_plan_2", + "reason": "Student-loan plan assignment writes an enum column only; household rows and typed household weights pass through and total household mass is conserved." } ], "outputs": ["student_loan_plan"], diff --git a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml index d377eebbb..0ba7e9725 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml @@ -2140,6 +2140,8 @@ stages: seed: 2024 salt: salary_sacrifice_conversion receipt: weighted_headcount + reason: Salary-sacrifice support stage rewrites pension columns only; household + rows and typed household weights pass through and total household mass is conserved. outputs: - pension_contributions_via_salary_sacrifice - employee_pension_contributions @@ -2212,6 +2214,8 @@ stages: highest_education: TERTIARY seed: 42 salt: student_loan_plan_2 + reason: Student-loan plan assignment writes an enum column only; household rows + and typed household weights pass through and total household mass is conserved. outputs: - student_loan_plan rewrites: diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_imputation.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_imputation.py index 85caad735..6649b8c1e 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_imputation.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_imputation.py @@ -670,34 +670,126 @@ def _assert_cgt_spine_stage_parameters(stage: SourceStageSpec) -> None: f"CGT spine operation order drifted: expected {expected_kinds}, got {kinds}." ) operations = { - operation.kind: operation.parameters for operation in stage.operations + operation.kind: dict(operation.parameters) for operation in stage.operations } - expected = { - ("verify_pinned_cgt_ods", "artifact_role"): "cgt_published_fact_surface", - ("verify_pinned_cgt_ods", "require_before_source_read"): True, - ("verify_pinned_cgt_ods", "runtime_sha256_required"): True, - ("verify_pinned_cgt_ods", "fail_on_mismatch"): True, - ("taxable_income_proxy", "components"): list( - UK_CGT_TAXABLE_INCOME_PROXY_COMPONENTS - ), - ("taxable_income_proxy", "fail_on_missing_component"): True, - ("rank_preserving_allocation", "minimum_allocation_people"): 1, - ("within_band_draws", "seed_base"): UK_CGT_IMPUTATION_SEED, - ("within_band_draws", "mean_repair_margin"): _MEAN_MARGIN, - ("within_band_draws", "deterministic"): True, - ("record_mass_conservation_receipt", "reason"): ( - UK_CGT_MASS_CONSERVATION_REASON - ), - ("record_mass_conservation_receipt", "declared_factor"): 1.0, - ("classify_cgt_band_facts_with_reviewed_fence", "calibration_permitted"): ( - False - ), - ("classify_cgt_band_facts_with_reviewed_fence", "fenced_fact_count"): 76, + # Closed-world reviewed mapping: every operation's FULL declared parameter + # payload must equal the reviewed constants below (adversarial-review + # finding on #740 — asserting a subset let lockstep manifest edits move + # behavioral declarations without a matching reviewed code change; whole- + # mapping equality also rejects extra keys). + expected_operations = { + "verify_pinned_cgt_ods": { + "artifact_role": "cgt_published_fact_surface", + "require_before_source_read": True, + "runtime_sha256_required": True, + "fail_on_mismatch": True, + }, + "taxable_income_proxy": { + "components": list(UK_CGT_TAXABLE_INCOME_PROXY_COMPONENTS), + "components_semantics": ( + "Persisted leaves of the model's total_income concept (ITA " + "2007 s.23); state_pension_reported stands in for " + "social_security_income, whose other taxable benefits are " + "not persisted; reliefs such as pension contributions and " + "Gift Aid are not deducted." + ), + "allowance": ( + "tapered Personal Allowance from the policy_parameters artifact" + ), + "fail_on_missing_component": True, + }, + "rank_preserving_allocation": { + "within": "income band", + "ordering": "existing gains descending, person_id ascending on ties", + "band_order": "highest gain band first", + "suppressed_cell_allocation": ( + "count implied by the cell's published gains at the band-total mean" + ), + "column_reconciliation": ( + "every income column rescales onto its published All-row taxpayer total" + ), + "shortfall_policy": ( + "proportional scale-down when the population holds less " + "gainer mass than published taxpayers" + ), + "minimum_allocation_people": int(_MINIMUM_ALLOCATION_PEOPLE), + "weights": ( + "household_weight mapped to persons; no person splits across bands" + ), + }, + "within_band_draws": { + "bounded_band_family": ( + "truncated exponential matched to the cell's published mean" + ), + "open_band_family": "Pareto with alpha = mean / (mean - lower bound)", + "mean_repair_margin": _MEAN_MARGIN, + "mean_repair_reason": ( + "Published counts round to the nearest thousand and amounts " + "to the nearest million; four cells of the 2023-24 table " + "imply a mean outside their own band, and repaired means " + "clamp just inside the violated boundary." + ), + "bottom_band_floor": "annual exempt amount plus one pound", + "seed_base": UK_CGT_IMPUTATION_SEED, + "seed_mixing": ( + "seed combined with the build period; draws ordered by allocation rank" + ), + "deterministic": True, + }, + "sub_aea_remainder": { + "policy": ( + "gainers beyond the published taxpayer mass keep their " + "existing amounts capped at the annual exempt amount" + ), + "rationale": ( + "Table 3 covers only individuals with a CGT liability; " + "remaining gainers are treated as sub-AEA gainers rather " + "than invented into the liability distribution or deleted." + ), + }, + "record_mass_conservation_receipt": { + "entity": "household", + "reason": UK_CGT_MASS_CONSERVATION_REASON, + "declared_factor": 1.0, + "gate_coupling": ( + "The terminal family gate requires a valid mass-conserving " + "MassChangeRecord carrying exactly this reason." + ), + }, + "classify_cgt_band_facts_with_reviewed_fence": { + "calibration_permitted": False, + "fact_fence_id": "cgt_band_facts_policy_endogenous_proxy_conditioned", + "fenced_fact_count": 76, + "fenced_fact_composition": ( + "60 joint cells, 10 gain-band row totals, 6 income-column totals" + ), + "classification_rationale": ( + "The taxpayer count is endogenous to policy, the income " + "conditioning is an arithmetic proxy, and the published " + "surface needs rounding and suppression reconciliation " + "before any per-band fact is exact." + ), + "calibrated_facts_unchanged": ( + "The two aggregate facts in UK_CGT_TARGET_SPECS remain the " + "only calibrated CGT facts." + ), + "promotion_path": ( + "A separately reviewed target profile may lift specific " + "band facts after the reconciliation and proxy adequacy " + "are adjudicated." + ), + "adjudication": "https://github.com/PolicyEngine/microcosm/issues/552", + }, } - for (kind, parameter), value in expected.items(): - actual = operations[kind].get(parameter) - if actual != value: + for kind, expected_parameters in expected_operations.items(): + actual = operations[kind] + if actual != expected_parameters: + drifted = sorted( + key + for key in {*actual, *expected_parameters} + if actual.get(key) != expected_parameters.get(key) + ) raise ValueError( - f"CGT spine {kind} parameter {parameter!r} drifted: expected " - f"{value!r}, got {actual!r}." + f"CGT spine {kind} declaration drifted from the reviewed " + f"mapping on parameter(s) {drifted}." ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_structure.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_structure.py index 6021aa822..64b22f3f5 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_structure.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_structure.py @@ -462,6 +462,18 @@ def _distribution_rows(resource: Mapping[str, Any]) -> list[Mapping[str, Any]]: minimums = [float(row["minimum_total_income"]) for row in rows] if minimums != sorted(minimums) or minimums[0] != 0.0: raise ValueError("Advani-Summers income bands must be sorted and start at 0.") + # Fail closed on non-monotone quantile rows: the prior draw interpolates + # and extrapolates these values as a quantile function, so a malformed + # row (the class the corrected percentile-69 dropped digit belonged to) + # would silently fabricate loss-makers instead of failing the build. + for row in rows: + knots = [float(row[column]) for column in CGT_PRIOR_PERCENTILE_COLUMNS] + if any(late < early for early, late in zip(knots, knots[1:], strict=False)): + raise ValueError( + "Advani-Summers quantile columns must be non-decreasing; " + f"row with minimum_total_income {row['minimum_total_income']!r} " + "is not a valid quantile function." + ) return rows diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/salary_sacrifice.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/salary_sacrifice.py index 1e927b4d8..240b62b82 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/salary_sacrifice.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/salary_sacrifice.py @@ -25,7 +25,7 @@ validate_uk_national_frame, ) from microcosm.build.uk_runtime.spi_support import support_channel_column -from microcosm.frame import Frame +from microcosm.frame import Frame, MassChangeRecord QRF: Any | None = None @@ -41,6 +41,11 @@ SALSAC_QRF_ESTIMATORS = 100 SALSAC_CONVERSION_SEED = 2024 SALSAC_CONVERSION_SALT = "salary_sacrifice_conversion" +SALSAC_MASS_CHANGE_REASON = ( + "Salary-sacrifice support stage rewrites pension columns only; household " + "rows and typed household weights pass through and total household mass " + "is conserved." +) def load_salary_sacrifice_anchor() -> Mapping[str, Any]: @@ -69,6 +74,8 @@ class UKSalarySacrificeResult: converted_rows: int converted_mass: float moved_amount: float + expected_converted_mass: float + realization_deviation: float def evidence(self) -> dict[str, object]: return { @@ -90,6 +97,8 @@ def evidence(self) -> dict[str, object]: "converted_rows": self.converted_rows, "converted_mass": self.converted_mass, "moved_amount": self.moved_amount, + "expected_converted_mass": self.expected_converted_mass, + "realization_deviation": self.realization_deviation, }, } @@ -225,6 +234,14 @@ def impute_salary_sacrifice(frame: Frame) -> UKSalarySacrificeResult: person["employee_pension_contributions"] = employee post_headcount = float(person_weights[final_ss > 0.0].sum()) converted_mass = float(person_weights[converted].sum()) + total = frame.weights_for("household").total + mass_receipt = MassChangeRecord( + entity="household", + old_total=total, + new_total=total, + declared_factor=1.0, + reason=SALSAC_MASS_CHANGE_REASON, + ) result_frame = uk_national_frame( person=person, benunit=frame.table("benunit").copy(), @@ -232,7 +249,7 @@ def impute_salary_sacrifice(frame: Frame) -> UKSalarySacrificeResult: time_period=uk_time_period(frame), weight_kind=uk_household_weight_kind(frame), household_weights=frame.weights_for("household").values, - mass_log=frame.mass_log, + mass_log=(*frame.mass_log, mass_receipt), ) validate_uk_national_frame(result_frame) return UKSalarySacrificeResult( @@ -248,6 +265,12 @@ def impute_salary_sacrifice(frame: Frame) -> UKSalarySacrificeResult: converted_rows=int(converted.sum()), converted_mass=converted_mass, moved_amount=moved_amount, + expected_converted_mass=rate * donor_pool_mass, + realization_deviation=( + (converted_mass - rate * donor_pool_mass) / (rate * donor_pool_mass) + if rate * donor_pool_mass > 0.0 + else 0.0 + ), ) @@ -313,6 +336,7 @@ def _assert_salary_sacrifice_stage_parameters( "seed": SALSAC_CONVERSION_SEED, "salt": SALSAC_CONVERSION_SALT, "receipt": "weighted_headcount", + "reason": SALSAC_MASS_CHANGE_REASON, }, ) hmrc = anchor.get("hmrc_anchor", {}) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/student_loans.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/student_loans.py index 594f874ef..dbc3d40a6 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/student_loans.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/student_loans.py @@ -20,7 +20,7 @@ uk_time_period, validate_uk_national_frame, ) -from microcosm.frame import Frame +from microcosm.frame import Frame, MassChangeRecord PLAN_1_BEFORE = 2012 PLAN_5_FROM = 2023 @@ -38,6 +38,11 @@ "PLAN_2": "student_loan_plan_2", } PLAN_2025_STOCKS = {"PLAN_2": 8_940_000.0, "PLAN_5": 10_000.0} +STUDENT_LOANS_MASS_CHANGE_REASON = ( + "Student-loan plan assignment writes an enum column only; household rows " + "and typed household weights pass through and total household mass is " + "conserved." +) def load_slc_liable_stocks() -> Mapping[str, Any]: @@ -61,6 +66,8 @@ class UKStudentLoanPlanReceipt: rate: float topped_up_rows: int topped_up_mass: float + expected_topped_up_mass: float + realization_deviation: float final_england_count: float def evidence(self) -> dict[str, object]: @@ -73,6 +80,8 @@ def evidence(self) -> dict[str, object]: "rate": self.rate, "topped_up_rows": self.topped_up_rows, "topped_up_mass": self.topped_up_mass, + "expected_topped_up_mass": self.expected_topped_up_mass, + "realization_deviation": self.realization_deviation, "final_england_count": self.final_england_count, } @@ -209,6 +218,13 @@ def assign_student_loan_plans( rate=rate, topped_up_rows=int(topped_up.sum()), topped_up_mass=float(person_weights[topped_up].sum()), + expected_topped_up_mass=rate * eligible_mass, + realization_deviation=( + (float(person_weights[topped_up].sum()) - rate * eligible_mass) + / (rate * eligible_mass) + if rate * eligible_mass > 0.0 + else 0.0 + ), final_england_count=float( person_weights[(plan == plan_name) & is_england].sum() ), @@ -217,6 +233,14 @@ def assign_student_loan_plans( if unknown: raise ValueError(f"Student-loan assignment emitted unknown plan(s): {unknown}.") person["student_loan_plan"] = plan + total = frame.weights_for("household").total + mass_receipt = MassChangeRecord( + entity="household", + old_total=total, + new_total=total, + declared_factor=1.0, + reason=STUDENT_LOANS_MASS_CHANGE_REASON, + ) result_frame = uk_national_frame( person=person, benunit=frame.table("benunit").copy(), @@ -224,7 +248,7 @@ def assign_student_loan_plans( time_period=uk_time_period(frame), weight_kind=uk_household_weight_kind(frame), household_weights=frame.weights_for("household").values, - mass_log=frame.mass_log, + mass_log=(*frame.mass_log, mass_receipt), ) validate_uk_national_frame(result_frame) return UKStudentLoansResult( @@ -349,6 +373,7 @@ def _assert_student_loans_stage_parameters( "cohort_start_min": PLAN_1_BEFORE, "cohort_start_max_exclusive": PLAN_5_FROM, "salt": PLAN_SALTS["PLAN_2"], + "reason": STUDENT_LOANS_MASS_CHANGE_REASON, }, } for operation in top_ups: diff --git a/packages/microcosm-build/tests/test_country_spec.py b/packages/microcosm-build/tests/test_country_spec.py index e3cf5778a..e4be86932 100644 --- a/packages/microcosm-build/tests/test_country_spec.py +++ b/packages/microcosm-build/tests/test_country_spec.py @@ -399,8 +399,7 @@ def test_uk_target_references_accept_regenerated_contract_fields(self) -> None: "calendar_year_average" ) assert ( - references["obr.income_tax"].assertion_policy - == "allow_source_projection" + references["obr.income_tax"].assertion_policy == "allow_source_projection" ) fanout = references["hmrc/employment_income_income_band_100_000_to_150_000"] @@ -646,12 +645,12 @@ def test_ledger_compile_parity_gates_pin_their_fixture_periods( ) -> None: params = {gate.id: gate.parameters for gate in manifest.gates} - assert params["uk_ledger_compile_parity_production_2023"][ - "target_period" - ] == 2023 - assert params["uk_ledger_compile_parity_incumbent_2025"][ - "target_period" - ] == 2025 + assert ( + params["uk_ledger_compile_parity_production_2023"]["target_period"] == 2023 + ) + assert ( + params["uk_ledger_compile_parity_incumbent_2025"]["target_period"] == 2025 + ) def test_only_the_weights_audit_blocks_on_absent_evidence(self, manifest) -> None: # "An absent audit is not a passing audit" — the retired schema-3 diff --git a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py index 62aaf7195..6f963e84e 100644 --- a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py +++ b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py @@ -42,7 +42,7 @@ ), ( "uk", - "e70e4a9f7d43a1c7998f01e33b9d46b59f1c8b3ee9ca7158f5ea7235be58e4aa", + "17c820bc63e16154056644f70c3947ccf8b288720ac5f1d7489d48cc5653d228", { "benunit.benunit_id", "household.household_id", diff --git a/packages/microcosm-build/tests/test_uk_cgt_source_manifest.py b/packages/microcosm-build/tests/test_uk_cgt_source_manifest.py index 0fadad94e..5453434a2 100644 --- a/packages/microcosm-build/tests/test_uk_cgt_source_manifest.py +++ b/packages/microcosm-build/tests/test_uk_cgt_source_manifest.py @@ -24,6 +24,12 @@ from microcosm.build.uk_runtime.release_input_coverage import ( load_uk_release_input_coverage_manifest, ) +from microcosm.build.uk_runtime.salary_sacrifice import ( + SALSAC_MASS_CHANGE_REASON, +) +from microcosm.build.uk_runtime.student_loans import ( + STUDENT_LOANS_MASS_CHANGE_REASON, +) _MANIFEST_PATH = ( Path(__file__).resolve().parents[1] @@ -155,6 +161,20 @@ def test_the_shipped_family_contracts_pass_the_terminal_gate_shape() -> None: declared_factor=1.0, reason=UK_CGT_MASS_CONSERVATION_REASON, ), + MassChangeRecord( + entity="household", + old_total=100.0, + new_total=100.0, + declared_factor=1.0, + reason=SALSAC_MASS_CHANGE_REASON, + ), + MassChangeRecord( + entity="household", + old_total=100.0, + new_total=100.0, + declared_factor=1.0, + reason=STUDENT_LOANS_MASS_CHANGE_REASON, + ), MassChangeRecord( entity="household", old_total=100.0, diff --git a/packages/microcosm-data/src/microcosm/data/contract.py b/packages/microcosm-data/src/microcosm/data/contract.py index c680b8727..66612dc57 100644 --- a/packages/microcosm-data/src/microcosm/data/contract.py +++ b/packages/microcosm-data/src/microcosm/data/contract.py @@ -344,13 +344,13 @@ # fingerprint derives from the manifest digest. Editing the spec moves all # three here in the same reviewed change. _UK_GATE_BATTERY_POLICY_SHA256 = ( - "404968fba9a626d4b534dfbef87721ff9d98c5af356758b2bab49dbaf004fdc3" + "5cb072a019617ba57e392fa19578e8c1b33fcb3af0144bcf33ff82b8874357d8" ) _UK_GATE_BATTERY_GATES_MANIFEST_SHA256 = ( - "59c7808d50a9ef84d37f524779a7518b4fb4f62dc7d17eb47e4a108d830c3798" + "c5123517586a8a4eed27606cb26c6e4ccfcbe45fd657e0d95162e15d49c83c85" ) _UK_GATE_BATTERY_SPEC_FINGERPRINT = ( - "bfb987361037e6475ea9906894cb16e5b3cd0ff515096bd55d4918dfd7331d2c" + "23cf63b64cdf06e186d12956043056ab8cc0f49cb44e984a5b0e25f1487cd731" ) #: Spec entry id -> the legacy gate name whose observable detail checks #: apply unchanged (the battery re-keys the report by entry id; the gate diff --git a/packages/microcosm-data/tests/test_contract.py b/packages/microcosm-data/tests/test_contract.py index fd9b49c4b..46b14344d 100644 --- a/packages/microcosm-data/tests/test_contract.py +++ b/packages/microcosm-data/tests/test_contract.py @@ -145,13 +145,13 @@ def _trusted_terminal_gate_signing_key(monkeypatch) -> None: UK_GATE_BATTERY_PRODUCER = "microcosm.build.gate_battery" UK_GATE_BATTERY_SIGNING_KEY_ENV = "MICROCOSM_UK_TERMINAL_GATE_SIGNING_KEY" UK_GATE_BATTERY_POLICY_SHA256 = ( - "404968fba9a626d4b534dfbef87721ff9d98c5af356758b2bab49dbaf004fdc3" + "5cb072a019617ba57e392fa19578e8c1b33fcb3af0144bcf33ff82b8874357d8" ) UK_GATE_BATTERY_GATES_MANIFEST_SHA256 = ( - "59c7808d50a9ef84d37f524779a7518b4fb4f62dc7d17eb47e4a108d830c3798" + "c5123517586a8a4eed27606cb26c6e4ccfcbe45fd657e0d95162e15d49c83c85" ) UK_GATE_BATTERY_SPEC_FINGERPRINT = ( - "bfb987361037e6475ea9906894cb16e5b3cd0ff515096bd55d4918dfd7331d2c" + "23cf63b64cdf06e186d12956043056ab8cc0f49cb44e984a5b0e25f1487cd731" ) UK_GATE_BATTERY_DEGENERATE_EVIDENCE_SHA256 = ( "d0d024043132fa07c378c393dbe2b24fe99bf19e876bcc39997d2c80cc9bd4f6" From e418f756b82d3831e4ef1bcc5a5858828062ecba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sat, 22 Aug 2026 16:02:43 +0200 Subject: [PATCH 4/5] Isolate the US fiscal-refresh module-leak canary in a fresh interpreter Fixes the nondeterministic main-CI red first seen on the FRS 2024-25 retarget merge run (test_reform_materialization_builds_one_engine_system _per_family asserting 0 > 1000): variable-module names are keyed by id(system), so in a warm suite process CPython can hand a fresh CountryTaxBenefitSystem a dead prior system's recycled address, re-registering its ~5,600 modules under already-existing names and zeroing the measured delta. No in-process counting survives id recycling, so the measurement now runs end-to-end in a subprocess whose import state is virgin by construction; the pytest test asserts on the child's outcome and surfaces its output on failure. Not an E8 surface - fixed-and-signed in the passing PR per the standing precedent (the WAS bridge-donor pin fix rode the retarget PR the same way). Co-Authored-By: Claude Fable 5 --- .../740-us-fiscal-memory-canary.fixed.md | 1 + .../tests/test_us_fiscal_refresh_memory.py | 49 +++++++++++++++---- 2 files changed, 41 insertions(+), 9 deletions(-) create mode 100644 changelog.d/740-us-fiscal-memory-canary.fixed.md diff --git a/changelog.d/740-us-fiscal-memory-canary.fixed.md b/changelog.d/740-us-fiscal-memory-canary.fixed.md new file mode 100644 index 000000000..f224356bf --- /dev/null +++ b/changelog.d/740-us-fiscal-memory-canary.fixed.md @@ -0,0 +1 @@ +Run the US fiscal-refresh per-family module-leak measurement in a fresh interpreter: variable-module names are keyed by id(system), so a warm suite process can recycle a dead system's address and re-register its module set under existing names, spuriously zeroing the leak canary (the nondeterministic main-CI red first seen on the FRS 2024-25 retarget merge run). diff --git a/packages/microcosm-build/tests/test_us_fiscal_refresh_memory.py b/packages/microcosm-build/tests/test_us_fiscal_refresh_memory.py index 5f4ff014d..dad887cdf 100644 --- a/packages/microcosm-build/tests/test_us_fiscal_refresh_memory.py +++ b/packages/microcosm-build/tests/test_us_fiscal_refresh_memory.py @@ -234,15 +234,17 @@ def _variable_module_count() -> int: ) -@requires_us -def test_reform_materialization_builds_one_engine_system_per_family() -> None: - """The pre-#456 builder rebuilt the full tax-benefit system every batch. - - Each build permanently registers one set of variable modules in - ``sys.modules`` (measured: ~5,600 entries, ~55-60 MB RSS floor, immune to - gc). Three batches per family must therefore add ~one set, not three: - against the old builder this assertion sees three sets per family and - fails. +def _isolated_family_measurement() -> None: + """Measure the per-family module registrations; asserts on failure. + + Runs the pre-#456 leak canary end-to-end. Must execute in a fresh + interpreter: variable-module names are keyed by ``id(system)``, and in a + warm suite process CPython can hand a fresh system a dead prior system's + recycled address, re-registering its module set under already-existing + names — the measured delta then reads 0 and the liveness assert fails + spuriously (the nondeterministic main-CI red first seen on the merge run + for the FRS 2024-25 retarget). A virgin process has no dead systems to + recycle, so the count deltas measure real registrations. """ builder = _load_builder_module() from policyengine_us import CountryTaxBenefitSystem, Microsimulation @@ -282,6 +284,31 @@ def test_reform_materialization_builds_one_engine_system_per_family() -> None: ) +@requires_us +def test_reform_materialization_builds_one_engine_system_per_family() -> None: + """The pre-#456 builder rebuilt the full tax-benefit system every batch. + + Each build permanently registers one set of variable modules in + ``sys.modules`` (measured: ~5,600 entries, ~55-60 MB RSS floor, immune to + gc). Three batches per family must therefore add ~one set, not three: + against the old builder this assertion sees three sets per family and + fails. The measurement runs in a fresh interpreter because the module + count is only meaningful there — see ``_isolated_family_measurement``. + """ + import subprocess + + result = subprocess.run( + [sys.executable, str(Path(__file__).resolve())], + capture_output=True, + text=True, + timeout=600, + ) + assert result.returncode == 0, ( + "isolated per-family measurement failed in the fresh interpreter:\n" + f"{result.stdout}\n{result.stderr}" + ) + + @requires_us def test_reform_materialization_batching_is_bit_identical() -> None: """Sharing one reform system across batches must not change results. @@ -345,3 +372,7 @@ def alive_microsimulations() -> int: f"{alive_after - alive_before} finished batch simulations survived " "the family boundary; release_engine_simulation has regressed" ) + + +if __name__ == "__main__": + _isolated_family_measurement() From 718c088e4b24a45e99c7f5e58ad4c586714666e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sat, 22 Aug 2026 20:19:26 +0200 Subject: [PATCH 5/5] Second adversarial round: closed-world asserts on every E8 stage, distinct CGT receipt identities Two findings from the pre-merge adversarial round, both dispositioned: 1. Closed-world coverage was only on the spine amounts stage. Every E8 stage assert now binds the full declared operation sequence and each operation's complete parameter payload (extra keys, missing keys, value drift, and extra or reordered operations all fail by name) via a shared ordered-sequence helper; the donor stage's previously-unbound propensity declaration is covered, with mutation tests for propensity, extra keys, and extra operations on all four stages. 2. The certified and spine CGT families shared one mass-conservation reason, so a single record satisfied both. The reused runtime gains an additive mass_change_reason parameter (certified default unchanged); the spine projection declares and emits its own spine-specific reason in both manifests, and the coverage generator now rejects duplicate stage-declared reasons outright. The five pre-E8 families sharing the generic fallback are a standing follow-up, unchanged here. Bundle spec sha re-pinned (1f163cbf) for the manifest edit; gate triplet unmoved. Co-Authored-By: Claude Fable 5 --- .../uk/release_input_coverage_manifest.json | 24 +-- .../src/microcosm/build/uk/source_stages.json | 4 +- .../src/microcosm/build/uk/spec/sources.yaml | 4 +- .../build/uk_runtime/cgt_imputation.py | 32 +++- .../build/uk_runtime/cgt_structure.py | 155 ++++++++++++------ .../build/uk_runtime/salary_sacrifice.py | 100 ++++++----- .../build/uk_runtime/student_loans.py | 138 +++++++--------- .../tests/test_spec_engine_country_bundles.py | 2 +- .../tests/test_uk_cgt_source_manifest.py | 19 +++ .../tests/test_uk_cgt_structure.py | 29 ++++ .../tests/test_uk_salary_sacrifice.py | 14 ++ .../tests/test_uk_student_loans.py | 15 ++ ...uild_uk_release_input_coverage_manifest.py | 19 +++ 13 files changed, 351 insertions(+), 204 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json index c1924824c..e6222dec2 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json @@ -473,7 +473,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "029545d43a254538baa05b6930e8a32019605a7543b9108c4d14b20e5b616881", + "source_manifest_sha256": "6f50637a95c340c07382d13af5a43da9d2a1a1167f713aba3605b3ca62efe449", "source_vintages": { "source": "HMRC Capital Gains Tax statistics, July 2025, Table 2.1a", "survey": "HMRC Capital Gains Tax statistics Table 2.1a and Advani-Summers capital-gains incidence" @@ -496,7 +496,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "029545d43a254538baa05b6930e8a32019605a7543b9108c4d14b20e5b616881", + "source_manifest_sha256": "6f50637a95c340c07382d13af5a43da9d2a1a1167f713aba3605b3ca62efe449", "source_vintages": { "source": "Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465", "survey": "Family Resources Survey 2024-25, SPI synthetic support, and Advani-Summers capital-gains incidence" @@ -525,7 +525,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "029545d43a254538baa05b6930e8a32019605a7543b9108c4d14b20e5b616881", + "source_manifest_sha256": "6f50637a95c340c07382d13af5a43da9d2a1a1167f713aba3605b3ca62efe449", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab, DfT rail fare index, and public NHS activity/cost table.", "survey": "Effects of Taxes and Benefits 1977-2024 and NHS age-gender public table" @@ -545,7 +545,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "029545d43a254538baa05b6930e8a32019605a7543b9108c4d14b20e5b616881", + "source_manifest_sha256": "6f50637a95c340c07382d13af5a43da9d2a1a1167f713aba3605b3ca62efe449", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab and cited VAT anchor resource.", "survey": "Effects of Taxes and Benefits 1977-2024" @@ -585,12 +585,12 @@ "outputs": [ "capital_gains" ], - "required_mass_change_reason": "Amounts-only capital gains redraw: household weights pass through unchanged and total household mass is conserved.", + "required_mass_change_reason": "Amounts-only capital gains redraw on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "029545d43a254538baa05b6930e8a32019605a7543b9108c4d14b20e5b616881", + "source_manifest_sha256": "6f50637a95c340c07382d13af5a43da9d2a1a1167f713aba3605b3ca62efe449", "source_vintages": { "hmrc_surface": "2023-24", "mapped_build_period": "2024" @@ -604,7 +604,7 @@ "base_candidate_tier": "frs", "calibration_permitted": false, "canonical_source_manifest": "source_stages.json", - "canonical_source_manifest_sha256": "029545d43a254538baa05b6930e8a32019605a7543b9108c4d14b20e5b616881", + "canonical_source_manifest_sha256": "6f50637a95c340c07382d13af5a43da9d2a1a1167f713aba3605b3ca62efe449", "effective_mass_requirements": { "charitable_investment_gifts": { "mass_share_denominator": "all_person_effective_mass", @@ -715,7 +715,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "029545d43a254538baa05b6930e8a32019605a7543b9108c4d14b20e5b616881", + "source_manifest_sha256": "6f50637a95c340c07382d13af5a43da9d2a1a1167f713aba3605b3ca62efe449", "source_vintages": { "source": "UK Data Service SN 9468 Living Costs and Food Survey 2023-24 household/person tabs, NEED 2023 headline energy tables, Ofgem Q2 2026 unit rates, and WAS round-8 bridge donor.", "survey": "Living Costs and Food Survey 2023-24" @@ -736,7 +736,7 @@ "property_wealth" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "029545d43a254538baa05b6930e8a32019605a7543b9108c4d14b20e5b616881", + "source_manifest_sha256": "6f50637a95c340c07382d13af5a43da9d2a1a1167f713aba3605b3ca62efe449", "source_vintages": { "source": "MHCLG dwellings and ONS UK House Price Index December 2025 regional average prices.", "survey": "Public regional property reference" @@ -760,7 +760,7 @@ "employee_pension_contributions" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "029545d43a254538baa05b6930e8a32019605a7543b9108c4d14b20e5b616881", + "source_manifest_sha256": "6f50637a95c340c07382d13af5a43da9d2a1a1167f713aba3605b3ca62efe449", "source_vintages": { "source": "HMRC, Salary sacrifice reform for pension contributions effective from 6 April 2029", "survey": "Family Resources Survey 2024-25 salary-sacrifice respondents and HMRC salary-sacrifice reform analysis" @@ -782,7 +782,7 @@ "student_loan_plan" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "029545d43a254538baa05b6930e8a32019605a7543b9108c4d14b20e5b616881", + "source_manifest_sha256": "6f50637a95c340c07382d13af5a43da9d2a1a1167f713aba3605b3ca62efe449", "source_vintages": { "source": "Explore Education Statistics Table 6a, Higher education total", "survey": "Family Resources Survey 2024-25 and Student Loans Company borrower forecasts for England" @@ -814,7 +814,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "029545d43a254538baa05b6930e8a32019605a7543b9108c4d14b20e5b616881", + "source_manifest_sha256": "6f50637a95c340c07382d13af5a43da9d2a1a1167f713aba3605b3ca62efe449", "source_vintages": { "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed 2006-22 household tab.", "survey": "Wealth and Assets Survey round 8" diff --git a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json index aa03f18d9..b777f420f 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json @@ -2599,9 +2599,9 @@ { "kind": "record_mass_conservation_receipt", "entity": "household", - "reason": "Amounts-only capital gains redraw: household weights pass through unchanged and total household mass is conserved.", + "reason": "Amounts-only capital gains redraw on the source spine: household weights pass through unchanged and total household mass is conserved.", "declared_factor": 1.0, - "gate_coupling": "The terminal family gate requires a valid mass-conserving MassChangeRecord carrying exactly this reason." + "gate_coupling": "The terminal family gate requires a valid mass-conserving MassChangeRecord carrying exactly this spine-specific reason." }, { "kind": "classify_cgt_band_facts_with_reviewed_fence", diff --git a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml index 0ba7e9725..3787b8306 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml @@ -2088,9 +2088,9 @@ stages: rationale: Table 3 covers only individuals with a CGT liability; remaining gainers are treated as sub-AEA gainers rather than invented into the liability distribution or deleted. - kind: record_mass_conservation_receipt entity: household - reason: 'Amounts-only capital gains redraw: household weights pass through unchanged and total household mass is conserved.' + reason: 'Amounts-only capital gains redraw on the source spine: household weights pass through unchanged and total household mass is conserved.' declared_factor: 1.0 - gate_coupling: The terminal family gate requires a valid mass-conserving MassChangeRecord carrying exactly this reason. + gate_coupling: The terminal family gate requires a valid mass-conserving MassChangeRecord carrying exactly this spine-specific reason. - kind: classify_cgt_band_facts_with_reviewed_fence calibration_permitted: false fact_fence_id: cgt_band_facts_policy_endogenous_proxy_conditioned diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_imputation.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_imputation.py index 6649b8c1e..219a63427 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_imputation.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_imputation.py @@ -82,6 +82,7 @@ __all__ = [ "UK_CGT_IMPUTATION_SEED", "UK_CGT_MASS_CONSERVATION_REASON", + "UK_CGT_SPINE_MASS_CONSERVATION_REASON", "UK_CGT_IMPUTATION_STAGE_NAME", "UK_CGT_TAXABLE_INCOME_PROXY_COMPONENTS", "UKCGTImputationSummary", @@ -109,6 +110,16 @@ #: periods draw differently while each build is reproducible. UK_CGT_IMPUTATION_SEED = 552 +#: The spine projection records the same conservation invariant under its +#: own reason so the terminal family validator can never satisfy the +#: certified and spine families with one shared record (adversarial-review +#: finding on the E8 PR: reason strings are the receipt identity). +UK_CGT_SPINE_MASS_CONSERVATION_REASON = ( + "Amounts-only capital gains redraw on the source spine: household " + "weights pass through unchanged and total household mass is conserved." +) + + #: Persisted components of the model's ``total_income`` concept (ITA 2007 #: s.23: taxable income after tax reliefs and before allowances). #: ``state_pension_reported`` stands in for ``social_security_income``; the @@ -437,6 +448,7 @@ def impute_uk_capital_gains( parameters: UKCGTPolicyParameters, *, seed: int = UK_CGT_IMPUTATION_SEED, + mass_change_reason: str = UK_CGT_MASS_CONSERVATION_REASON, ) -> Frame: """Redraw gainers' amounts from the published joint distribution.""" validate_uk_national_frame(frame) @@ -550,7 +562,7 @@ def impute_uk_capital_gains( old_total=household_mass, new_total=household_mass, declared_factor=1.0, - reason=UK_CGT_MASS_CONSERVATION_REASON, + reason=mass_change_reason, ) result_frame = uk_national_frame( person=new_person, @@ -619,6 +631,7 @@ def uk_capital_gains_imputation_stage( tax_year: str = HMRC_CGT_SOURCE_VINTAGE, parameters: UKCGTPolicyParameters | None = None, seed: int = UK_CGT_IMPUTATION_SEED, + mass_change_reason: str = UK_CGT_MASS_CONSERVATION_REASON, ) -> UKNationalStage: """Build the national stage that redraws capital gains amounts. @@ -633,7 +646,13 @@ def transform(frame: Frame) -> Frame: artifact_path, tax_year=tax_year ) resolved = parameters or uk_cgt_policy_parameters(uk_time_period(frame)) - return impute_uk_capital_gains(frame, distribution, resolved, seed=seed) + return impute_uk_capital_gains( + frame, + distribution, + resolved, + seed=seed, + mass_change_reason=mass_change_reason, + ) return UKNationalStage(name=UK_CGT_IMPUTATION_STAGE_NAME, transform=transform) @@ -649,7 +668,10 @@ def uk_cgt_spine_stage_transform( """ _assert_cgt_spine_stage_parameters(stage) - return uk_capital_gains_imputation_stage(ods_path).transform + return uk_capital_gains_imputation_stage( + ods_path, + mass_change_reason=UK_CGT_SPINE_MASS_CONSERVATION_REASON, + ).transform def _assert_cgt_spine_stage_parameters(stage: SourceStageSpec) -> None: @@ -749,11 +771,11 @@ def _assert_cgt_spine_stage_parameters(stage: SourceStageSpec) -> None: }, "record_mass_conservation_receipt": { "entity": "household", - "reason": UK_CGT_MASS_CONSERVATION_REASON, + "reason": UK_CGT_SPINE_MASS_CONSERVATION_REASON, "declared_factor": 1.0, "gate_coupling": ( "The terminal family gate requires a valid mass-conserving " - "MassChangeRecord carrying exactly this reason." + "MassChangeRecord carrying exactly this spine-specific reason." ), }, "classify_cgt_band_facts_with_reviewed_fence": { diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_structure.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_structure.py index 64b22f3f5..091cf65f2 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_structure.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_structure.py @@ -563,6 +563,42 @@ def _assert_parameters( ) +def _assert_closed_world_operations( + stage: SourceStageSpec, + expected_operations: tuple[tuple[str, dict[str, object]], ...], +) -> None: + """Exact operation order and full-mapping equality per operation. + + Whole-payload equality rejects value drift, missing keys, and extra keys + alike (adversarial-review finding on the E8 PR: asserting a named subset + let lockstep manifest edits move undeclared-but-load-bearing semantics). + The expected sequence is ordered so repeated kinds are supported and an + extra, missing, or reordered operation fails by position. + """ + + kinds = tuple(operation.kind for operation in stage.operations) + expected_kinds = tuple(kind for kind, _ in expected_operations) + if kinds != expected_kinds: + raise ValueError( + f"Stage {stage.stage!r} operation order drifted: expected " + f"{expected_kinds}, got {kinds}." + ) + for operation, (kind, expected) in zip( + stage.operations, expected_operations, strict=True + ): + actual = dict(operation.parameters) + if actual != expected: + drifted = sorted( + key + for key in {*actual, *expected} + if actual.get(key) != expected.get(key) + ) + raise ValueError( + f"Stage {stage.stage!r} {kind} declaration drifted " + f"from the reviewed mapping on parameter(s) {drifted}." + ) + + def _assert_cgt_incidence_stage_parameters(stage: SourceStageSpec) -> None: """Bind every stage-19 manifest parameter to reviewed code constants. @@ -570,37 +606,44 @@ def _assert_cgt_incidence_stage_parameters(stage: SourceStageSpec) -> None: :class:`UKCGTIncidenceCloneResult` supplies the executed-effect receipt. """ - _assert_parameters( - _operation(stage, "clone_records"), - { - "entity": "household", - "copies": 2, - "flag_column": HOUSEHOLD_IS_CGT_CLONE, - "original_flag": False, - "clone_flag": True, - "mass_split": CGT_CLONE_MASS_SPLIT, - "weight_kind_out": WeightKind.IMPORTANCE.value, - "conservation": "exact_total", - "id_remapping": "id_multiplier_for_values", - "declared_factor": 1.0, - "reason": CGT_CLONE_MASS_CHANGE_REASON, - }, - ) - _assert_parameters( - _operation(stage, "draw_capital_gains_prior_from_banded_quantiles"), - { - "resource": "advani_summers_capital_gains_distribution.json", - "income_proxy_components": list(UK_CGT_TAXABLE_INCOME_PROXY_COMPONENTS), - "allowance_subtraction": False, - "carrier": "oldest adult; person_id ascending breaks age ties", - "adult_minimum_age": CGT_ADULT_MINIMUM_AGE, - "quantile_points": list(CGT_QUANTILE_POINTS), - "spline_degree": 1, - "extrapolation": "ext=0", - "keep_negative_draws": True, - "seed": CGT_PRIOR_SEED, - "salt": CGT_PRIOR_SALT, - }, + _assert_closed_world_operations( + stage, + ( + ( + "clone_records", + { + "entity": "household", + "copies": 2, + "flag_column": HOUSEHOLD_IS_CGT_CLONE, + "original_flag": False, + "clone_flag": True, + "mass_split": CGT_CLONE_MASS_SPLIT, + "weight_kind_out": WeightKind.IMPORTANCE.value, + "conservation": "exact_total", + "id_remapping": "id_multiplier_for_values", + "declared_factor": 1.0, + "reason": CGT_CLONE_MASS_CHANGE_REASON, + }, + ), + ( + "draw_capital_gains_prior_from_banded_quantiles", + { + "resource": "advani_summers_capital_gains_distribution.json", + "income_proxy_components": list( + UK_CGT_TAXABLE_INCOME_PROXY_COMPONENTS + ), + "allowance_subtraction": False, + "carrier": "oldest adult; person_id ascending breaks age ties", + "adult_minimum_age": CGT_ADULT_MINIMUM_AGE, + "quantile_points": list(CGT_QUANTILE_POINTS), + "spline_degree": 1, + "extrapolation": "ext=0", + "keep_negative_draws": True, + "seed": CGT_PRIOR_SEED, + "salt": CGT_PRIOR_SALT, + }, + ), + ), ) @@ -611,26 +654,36 @@ def _assert_cgt_donor_stage_parameters( ) -> None: """Bind stage-20 parameters and recompute the band/weight invariants.""" - operation = _operation(stage, "stack_band_donor_households") - _assert_parameters( - operation, - { - "size_band_resource": "hmrc_cgt_size_bands.json", - "incidence_resource": "advani_summers_capital_gains_distribution.json", - "minimum_band_lower": MIN_DONOR_BAND_LOWER, - "donors_per_band": DONORS_PER_BAND, - "expected_band_count": DONOR_BAND_COUNT, - "expected_donor_count": DONOR_TOTAL, - "candidate_order": "household_id ascending", - "draw": "weighted_without_replacement", - "seed": DONOR_SEED, - "flag_column": HOUSEHOLD_IS_CGT_BAND_DONOR, - "carrier": "oldest adult; person_id ascending breaks age ties", - "initial_weight": "published band taxpayers / donors_per_band", - "never_zero_weight": DONOR_NEVER_ZERO_WEIGHT, - "weight_kind_out": WeightKind.IMPORTANCE.value, - "reason": CGT_DONOR_MASS_CHANGE_REASON, - }, + _assert_closed_world_operations( + stage, + ( + ( + "stack_band_donor_households", + { + "size_band_resource": "hmrc_cgt_size_bands.json", + "incidence_resource": ( + "advani_summers_capital_gains_distribution.json" + ), + "minimum_band_lower": MIN_DONOR_BAND_LOWER, + "donors_per_band": DONORS_PER_BAND, + "expected_band_count": DONOR_BAND_COUNT, + "expected_donor_count": DONOR_TOTAL, + "candidate_order": "household_id ascending", + "draw": "weighted_without_replacement", + "propensity": ( + "Advani-Summers percent_with_gains at oldest-adult " + "component-sum income" + ), + "seed": DONOR_SEED, + "flag_column": HOUSEHOLD_IS_CGT_BAND_DONOR, + "carrier": "oldest adult; person_id ascending breaks age ties", + "initial_weight": "published band taxpayers / donors_per_band", + "never_zero_weight": DONOR_NEVER_ZERO_WEIGHT, + "weight_kind_out": WeightKind.IMPORTANCE.value, + "reason": CGT_DONOR_MASS_CHANGE_REASON, + }, + ), + ), ) bands = _retained_size_bands(size_bands) if len(bands) != DONOR_BAND_COUNT: diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/salary_sacrifice.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/salary_sacrifice.py index 240b62b82..de9830cfc 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/salary_sacrifice.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/salary_sacrifice.py @@ -12,11 +12,12 @@ import numpy as np import pandas as pd -from microcosm.build.source_manifest import SourceOperationSpec, SourceStageSpec +from microcosm.build.source_manifest import SourceStageSpec from microcosm.build.stochastic_assignment import stable_identity_uniforms from microcosm.build.uk_runtime.cgt_structure import ( HOUSEHOLD_IS_CGT_BAND_DONOR, HOUSEHOLD_IS_CGT_CLONE, + _assert_closed_world_operations, ) from microcosm.build.uk_runtime.national_frame import ( uk_household_weight_kind, @@ -280,64 +281,57 @@ def _qrf_class(): return import_module("microcosm.fit").QRF -def _operation(stage: SourceStageSpec, kind: str) -> SourceOperationSpec: - matches = [operation for operation in stage.operations if operation.kind == kind] - if len(matches) != 1: - raise ValueError( - f"Stage {stage.stage!r} must declare exactly one {kind!r} operation." - ) - return matches[0] - - -def _assert_parameters( - operation: SourceOperationSpec, - expected: Mapping[str, object], -) -> None: - for name, value in expected.items(): - actual = operation.parameters.get(name) - if actual != value: - raise ValueError( - f"{operation.kind} manifest parameter {name!r} drifted: " - f"expected {value!r}, got {actual!r}." - ) - - def _assert_salary_sacrifice_stage_parameters( stage: SourceStageSpec, *, anchor: Mapping[str, Any], ) -> None: - """Bind all stage parameters; result evidence supplies arm 2.""" + """Bind all stage parameters closed-world; result evidence supplies arm 2.""" - _assert_parameters( - _operation(stage, "fit_weighted_qrf"), - { - "training_population": "support_channel == frs and not capital-gains clone and not CGT band donor and salary_sacrifice_asked == 1", - "target_population": "salary_sacrifice_asked != 1 frame-wide", - "predictors": list(SALSAC_PREDICTORS), - "targets": [SALSAC_OUTPUT], - "weights": "household_weight", - "weight_mapping": "household_to_person", - "seed": SALSAC_QRF_SEED, - "n_estimators": SALSAC_QRF_ESTIMATORS, - "clamp_minimum": 0, - "preserve_asked_rows": True, - "cache": False, - }, - ) - _assert_parameters( - _operation(stage, "convert_donors_to_target_stock"), - { - "resource": "salary_sacrifice_anchor.json", - "target": int(SALSAC_STAGE_TARGET), - "donor_pool": "employee_pension_contributions > 0 and pension_contributions_via_salary_sacrifice == 0 and employment_income > 0", - "rate_cap": SALSAC_RATE_CAP, - "move": "full employee_pension_contributions to pension_contributions_via_salary_sacrifice; source zeroed", - "seed": SALSAC_CONVERSION_SEED, - "salt": SALSAC_CONVERSION_SALT, - "receipt": "weighted_headcount", - "reason": SALSAC_MASS_CHANGE_REASON, - }, + _assert_closed_world_operations( + stage, + ( + ( + "fit_weighted_qrf", + { + "training_population": ( + "support_channel == frs and not capital-gains clone and " + "not CGT band donor and salary_sacrifice_asked == 1" + ), + "target_population": "salary_sacrifice_asked != 1 frame-wide", + "predictors": list(SALSAC_PREDICTORS), + "targets": [SALSAC_OUTPUT], + "weights": "household_weight", + "weight_mapping": "household_to_person", + "seed": SALSAC_QRF_SEED, + "n_estimators": SALSAC_QRF_ESTIMATORS, + "clamp_minimum": 0, + "preserve_asked_rows": True, + "cache": False, + }, + ), + ( + "convert_donors_to_target_stock", + { + "resource": "salary_sacrifice_anchor.json", + "target": int(SALSAC_STAGE_TARGET), + "donor_pool": ( + "employee_pension_contributions > 0 and " + "pension_contributions_via_salary_sacrifice == 0 and " + "employment_income > 0" + ), + "rate_cap": SALSAC_RATE_CAP, + "move": ( + "full employee_pension_contributions to " + "pension_contributions_via_salary_sacrifice; source zeroed" + ), + "seed": SALSAC_CONVERSION_SEED, + "salt": SALSAC_CONVERSION_SALT, + "receipt": "weighted_headcount", + "reason": SALSAC_MASS_CHANGE_REASON, + }, + ), + ), ) hmrc = anchor.get("hmrc_anchor", {}) derived = anchor.get("derived", {}) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/student_loans.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/student_loans.py index dbc3d40a6..6460c194e 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/student_loans.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/student_loans.py @@ -11,8 +11,11 @@ import numpy as np import pandas as pd -from microcosm.build.source_manifest import SourceOperationSpec, SourceStageSpec +from microcosm.build.source_manifest import SourceStageSpec from microcosm.build.stochastic_assignment import stable_identity_uniforms +from microcosm.build.uk_runtime.cgt_structure import ( + _assert_closed_world_operations, +) from microcosm.build.uk_runtime.frs_release import load_uk_frs_release from microcosm.build.uk_runtime.national_frame import ( uk_household_weight_kind, @@ -302,90 +305,69 @@ def _enum_name(value: object) -> str: return text.rsplit(".", 1)[-1] -def _operation(stage: SourceStageSpec, kind: str) -> SourceOperationSpec: - matches = [operation for operation in stage.operations if operation.kind == kind] - if len(matches) != 1: - raise ValueError( - f"Stage {stage.stage!r} must declare exactly one {kind!r} operation." - ) - return matches[0] - - -def _assert_parameters( - operation: SourceOperationSpec, - expected: Mapping[str, object], -) -> None: - for name, value in expected.items(): - actual = operation.parameters.get(name) - if actual != value: - raise ValueError( - f"{operation.kind} manifest parameter {name!r} drifted: " - f"expected {value!r}, got {actual!r}." - ) - - def _assert_student_loans_stage_parameters( stage: SourceStageSpec, *, stocks: Mapping[str, Any], year: int, ) -> None: - """Bind every stage parameter; per-plan receipts supply arm 2.""" - - _assert_parameters( - _operation(stage, "assign_student_loan_plan_cohorts"), - { - "year_rule": YEAR_RULE, - "start_year_formula": "year - age + 18", - "reported_repayment_test": "student_loan_repayments > 0", - "reported_country_gate": False, - "plan_1_before": PLAN_1_BEFORE, - "plan_5_from": PLAN_5_FROM, - "enum_domain": list(STUDENT_LOAN_ENUM_DOMAIN), - "plan_4_imputation": False, - }, + """Bind every stage parameter closed-world; per-plan receipts supply arm 2.""" + + region_exclusions = list(EXCLUDED_ENGLAND_REGIONS) + _assert_closed_world_operations( + stage, + ( + ( + "assign_student_loan_plan_cohorts", + { + "year_rule": YEAR_RULE, + "start_year_formula": "year - age + 18", + "reported_repayment_test": "student_loan_repayments > 0", + "reported_country_gate": False, + "plan_1_before": PLAN_1_BEFORE, + "plan_5_from": PLAN_5_FROM, + "enum_domain": list(STUDENT_LOAN_ENUM_DOMAIN), + "plan_4_imputation": False, + }, + ), + ( + "top_up_to_stock", + { + "plan": "PLAN_5", + "priority": 1, + "resource": "slc_liable_stocks.json", + "stock_series": "plan_5.liable", + "year_rule": YEAR_RULE, + "age_min": PLAN_5_MIN_AGE, + "age_max": PLAN_5_MAX_AGE, + "cohort_start_min": PLAN_5_FROM, + "eligible_region_exclusions": region_exclusions, + "highest_education": "TERTIARY", + "seed": STUDENT_LOAN_SEED, + "salt": PLAN_SALTS["PLAN_5"], + }, + ), + ( + "top_up_to_stock", + { + "plan": "PLAN_2", + "priority": 2, + "resource": "slc_liable_stocks.json", + "stock_series": "plan_2.liable", + "year_rule": YEAR_RULE, + "age_min": PLAN_2_MIN_AGE, + "age_max": PLAN_2_MAX_AGE, + "cohort_start_min": PLAN_1_BEFORE, + "cohort_start_max_exclusive": PLAN_5_FROM, + "eligible_region_exclusions": region_exclusions, + "highest_education": "TERTIARY", + "seed": STUDENT_LOAN_SEED, + "salt": PLAN_SALTS["PLAN_2"], + "reason": STUDENT_LOANS_MASS_CHANGE_REASON, + }, + ), + ), ) - top_ups = [ - operation - for operation in stage.operations - if operation.kind == "top_up_to_stock" - ] - if [operation.parameters.get("plan") for operation in top_ups] != list( - PLAN_PRIORITY - ): - raise ValueError( - "Student-loan top-up priority drifted from PLAN_5 then PLAN_2." - ) - expected = { - "PLAN_5": { - "priority": 1, - "stock_series": "plan_5.liable", - "age_min": PLAN_5_MIN_AGE, - "age_max": PLAN_5_MAX_AGE, - "cohort_start_min": PLAN_5_FROM, - "salt": PLAN_SALTS["PLAN_5"], - }, - "PLAN_2": { - "priority": 2, - "stock_series": "plan_2.liable", - "age_min": PLAN_2_MIN_AGE, - "age_max": PLAN_2_MAX_AGE, - "cohort_start_min": PLAN_1_BEFORE, - "cohort_start_max_exclusive": PLAN_5_FROM, - "salt": PLAN_SALTS["PLAN_2"], - "reason": STUDENT_LOANS_MASS_CHANGE_REASON, - }, - } - for operation in top_ups: - plan = str(operation.parameters["plan"]) - shared = { - "resource": "slc_liable_stocks.json", - "year_rule": YEAR_RULE, - "eligible_region_exclusions": list(EXCLUDED_ENGLAND_REGIONS), - "highest_education": "TERTIARY", - "seed": STUDENT_LOAN_SEED, - } - _assert_parameters(operation, {**shared, **expected[plan]}) if year == 2025: for plan, expected_stock in PLAN_2025_STOCKS.items(): if _stock(stocks, plan, year) != expected_stock: diff --git a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py index 6f963e84e..59f0531b3 100644 --- a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py +++ b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py @@ -42,7 +42,7 @@ ), ( "uk", - "17c820bc63e16154056644f70c3947ccf8b288720ac5f1d7489d48cc5653d228", + "1f163cbf7b35f07d49b6e2905d01a2519c0ba37e6d2d1470424272e18ac621db", { "benunit.benunit_id", "household.household_id", diff --git a/packages/microcosm-build/tests/test_uk_cgt_source_manifest.py b/packages/microcosm-build/tests/test_uk_cgt_source_manifest.py index 5453434a2..2348f23ea 100644 --- a/packages/microcosm-build/tests/test_uk_cgt_source_manifest.py +++ b/packages/microcosm-build/tests/test_uk_cgt_source_manifest.py @@ -9,6 +9,7 @@ UK_CGT_IMPUTATION_SEED, UK_CGT_IMPUTATION_STAGE_NAME, UK_CGT_MASS_CONSERVATION_REASON, + UK_CGT_SPINE_MASS_CONSERVATION_REASON, UK_CGT_TAXABLE_INCOME_PROXY_COMPONENTS, ) from microcosm.build.uk_runtime.cgt_structure import ( @@ -161,6 +162,13 @@ def test_the_shipped_family_contracts_pass_the_terminal_gate_shape() -> None: declared_factor=1.0, reason=UK_CGT_MASS_CONSERVATION_REASON, ), + MassChangeRecord( + entity="household", + old_total=100.0, + new_total=100.0, + declared_factor=1.0, + reason=UK_CGT_SPINE_MASS_CONSERVATION_REASON, + ), MassChangeRecord( entity="household", old_total=100.0, @@ -205,3 +213,14 @@ def test_the_shipped_family_contracts_pass_the_terminal_gate_shape() -> None: assert any( "hmrc_cgt_gains" in failure and "kind" in failure for failure in failures ) + + +def test_certified_and_spine_families_require_distinct_receipts() -> None: + """One record must never satisfy both CGT families (review finding).""" + manifest = load_uk_release_input_coverage_manifest() + families = manifest.family_coverage + certified = families["hmrc_cgt_gains"]["required_mass_change_reason"] + spine = families["hmrc_cgt_gains_spine"]["required_mass_change_reason"] + assert certified == UK_CGT_MASS_CONSERVATION_REASON + assert spine == UK_CGT_SPINE_MASS_CONSERVATION_REASON + assert certified != spine diff --git a/packages/microcosm-build/tests/test_uk_cgt_structure.py b/packages/microcosm-build/tests/test_uk_cgt_structure.py index e27606764..f675d5234 100644 --- a/packages/microcosm-build/tests/test_uk_cgt_structure.py +++ b/packages/microcosm-build/tests/test_uk_cgt_structure.py @@ -276,3 +276,32 @@ def test_donor_drift_assert_covers_every_reviewed_parameter(parameter: str) -> N _drift(_stage("cgt_band_donors"), 0, parameter), size_bands=load_hmrc_cgt_size_bands(), ) + + +def test_donor_drift_assert_rejects_propensity_and_extra_keys() -> None: + """Closed-world equality: undeclared and extra parameters both fail.""" + for parameter in ("propensity", "undeclared_extra_key"): + with pytest.raises(ValueError, match="drifted"): + _assert_cgt_donor_stage_parameters( + _drift(_stage("cgt_band_donors"), 0, parameter), + size_bands=load_hmrc_cgt_size_bands(), + ) + + +def test_drift_asserts_reject_extra_operations() -> None: + for name, check in ( + ("cgt_incidence_clone", _assert_cgt_incidence_stage_parameters), + ( + "cgt_band_donors", + lambda stage: _assert_cgt_donor_stage_parameters( + stage, size_bands=load_hmrc_cgt_size_bands() + ), + ), + ): + stage = _stage(name) + extra = replace( + stage, + operations=(*stage.operations, stage.operations[-1]), + ) + with pytest.raises(ValueError, match="operation order drifted"): + check(extra) diff --git a/packages/microcosm-build/tests/test_uk_salary_sacrifice.py b/packages/microcosm-build/tests/test_uk_salary_sacrifice.py index a160a10f6..f2a9d46d8 100644 --- a/packages/microcosm-build/tests/test_uk_salary_sacrifice.py +++ b/packages/microcosm-build/tests/test_uk_salary_sacrifice.py @@ -222,3 +222,17 @@ def test_resource_drift_assert_rejects_anchor_change() -> None: anchor["derived"] = {**anchor["derived"], "stage_target": 1} with pytest.raises(ValueError, match="stage_target.*drifted"): _assert_salary_sacrifice_stage_parameters(_stage(), anchor=anchor) + + +def test_drift_assert_rejects_extra_keys_and_operations() -> None: + with pytest.raises(ValueError, match="drifted"): + _assert_salary_sacrifice_stage_parameters( + _drift(1, "undeclared_extra_key"), + anchor=load_salary_sacrifice_anchor(), + ) + stage = _stage() + extra = replace(stage, operations=(*stage.operations, stage.operations[-1])) + with pytest.raises(ValueError, match="operation order drifted"): + _assert_salary_sacrifice_stage_parameters( + extra, anchor=load_salary_sacrifice_anchor() + ) diff --git a/packages/microcosm-build/tests/test_uk_student_loans.py b/packages/microcosm-build/tests/test_uk_student_loans.py index 9c90db6b5..b73c9d225 100644 --- a/packages/microcosm-build/tests/test_uk_student_loans.py +++ b/packages/microcosm-build/tests/test_uk_student_loans.py @@ -236,3 +236,18 @@ def test_stock_drift_assert_rejects_2025_change() -> None: stocks = _stocks(plan_2=1, plan_5=10_000) with pytest.raises(ValueError, match="PLAN_2.*drifted"): _assert_student_loans_stage_parameters(_stage(), stocks=stocks, year=2025) + + +def test_drift_assert_rejects_extra_keys_and_operations() -> None: + with pytest.raises(ValueError, match="drifted"): + _assert_student_loans_stage_parameters( + _drift(2, "undeclared_extra_key"), + stocks=load_slc_liable_stocks(), + year=2025, + ) + stage = _stage() + extra = replace(stage, operations=(*stage.operations, stage.operations[-1])) + with pytest.raises(ValueError, match="operation order drifted"): + _assert_student_loans_stage_parameters( + extra, stocks=load_slc_liable_stocks(), year=2025 + ) diff --git a/tools/build_uk_release_input_coverage_manifest.py b/tools/build_uk_release_input_coverage_manifest.py index cc901aa9a..2f9641727 100644 --- a/tools/build_uk_release_input_coverage_manifest.py +++ b/tools/build_uk_release_input_coverage_manifest.py @@ -1300,6 +1300,25 @@ def main() -> int: else: known_gaps = _load(KNOWN_GAPS_PATH) manifest = build_manifest(reference=reference, known_gaps_payload=known_gaps) + generic_fallback_reason = ( + "E5 source-stage transform preserves household rows and typed " + "household weights; total household mass is conserved." + ) + declared_reasons: dict[str, str] = {} + for family_name, family in manifest.get("family_coverage", {}).items(): + reason = str(family.get("required_mass_change_reason", "")).strip() + if not reason or reason == generic_fallback_reason: + # The pre-E8 families share the generic fallback (standing + # follow-up); every stage-declared reason must be unique so a + # receipt identifies exactly one family. + continue + if reason in declared_reasons: + raise ValueError( + f"family_coverage reasons must be unique receipt identities: " + f"{family_name!r} and {declared_reasons[reason]!r} share " + f"{reason!r}." + ) + declared_reasons[reason] = family_name _write_or_check(MANIFEST_PATH, manifest, check=args.check) action = "current" if args.check else "wrote" print(