diff --git a/changelog.d/684-uk-e8-cgt-salsac-student-loans.added.md b/changelog.d/684-uk-e8-cgt-salsac-student-loans.added.md new file mode 100644 index 00000000..e9b9c9e9 --- /dev/null +++ b/changelog.d/684-uk-e8-cgt-salsac-student-loans.added.md @@ -0,0 +1 @@ +Add UK spine stages for capital-gains structure and redraws, salary-sacrifice support, and student-loan plan cohorts. diff --git a/changelog.d/740-us-fiscal-memory-canary.fixed.md b/changelog.d/740-us-fiscal-memory-canary.fixed.md new file mode 100644 index 00000000..f224356b --- /dev/null +++ b/changelog.d/740-us-fiscal-memory-canary.fixed.md @@ -0,0 +1 @@ +Run the US fiscal-refresh per-family module-leak measurement in a fresh interpreter: variable-module names are keyed by id(system), so a warm suite process can recycle a dead system's address and re-register its module set under existing names, spuriously zeroing the leak canary (the nondeterministic main-CI red first seen on the FRS 2024-25 retarget merge run). diff --git a/packages/microcosm-build/src/microcosm/build/country_spec.py b/packages/microcosm-build/src/microcosm/build/country_spec.py index 495f4c74..e3fcde95 100644 --- a/packages/microcosm-build/src/microcosm/build/country_spec.py +++ b/packages/microcosm-build/src/microcosm/build/country_spec.py @@ -1644,11 +1644,12 @@ def country_stage_plan( f"Country {spec.country!r} declares no source_stages.json; there " "is no stage plan to assemble." ) - declared: list[tuple[str, DonorSpec | None, tuple[str, ...]]] = [ + declared: list[tuple[str, DonorSpec | None, tuple[str, ...], tuple[str, ...]]] = [ ( stage.stage, DonorSpec(survey=stage.survey, source=stage.source, notes=stage.notes), stage.outputs, + stage.rewrites, ) for stage in spec.sources.stages ] @@ -1663,9 +1664,10 @@ def country_stage_plan( notes=spine.assignment_source.notes, ), (spine.code_column,), + (), ) ) - declared_names = [name for name, _, _ in declared] + declared_names = [name for name, _, _, _ in declared] selected_names: tuple[str, ...] if stage_names is None: selected_names = tuple(declared_names) @@ -1702,8 +1704,8 @@ def country_stage_plan( f"{declared_names}." ) selected = [ - (name, donor, outputs) - for name, donor, outputs in declared + (name, donor, outputs, rewrites) + for name, donor, outputs, rewrites in declared if name in set(selected_names) ] return StagePlan( @@ -1711,7 +1713,8 @@ def country_stage_plan( name=name, transform=implementations[name], produces=outputs, + rewrites=rewrites, donor=donor, ) - for name, donor, outputs in selected + for name, donor, outputs, rewrites in selected ) diff --git a/packages/microcosm-build/src/microcosm/build/plan.py b/packages/microcosm-build/src/microcosm/build/plan.py index 8d8274e6..969ee3e8 100644 --- a/packages/microcosm-build/src/microcosm/build/plan.py +++ b/packages/microcosm-build/src/microcosm/build/plan.py @@ -71,6 +71,8 @@ class Stage: happens inside and any failure aborts the build. produces: Columns the stage must add (validated after the transform). Empty is allowed for assert/report-only stages. + rewrites: Declared outputs that may have an earlier canonical producer + and are intentionally replaced by this stage. consumes: Columns that must exist (on any entity) before the stage runs. donor: The donor survey, when the stage imputes. ``None`` for @@ -80,6 +82,7 @@ class Stage: name: str transform: Callable[[Frame], Frame] produces: tuple[str, ...] = () + rewrites: tuple[str, ...] = () consumes: tuple[str, ...] = () donor: DonorSpec | None = None @@ -143,12 +146,15 @@ def __init__(self, stages: Iterable[Stage]) -> None: names.add(stage.name) for column in stage.produces: if column in producers: - raise ValueError( - f"Column {column!r} is declared by two stages " - f"({producers[column]!r} and {stage.name!r}); every " - "column has one canonical producer." - ) - producers[column] = stage.name + if column not in stage.rewrites: + raise ValueError( + f"Column {column!r} is declared by two stages " + f"({producers[column]!r} and {stage.name!r}); every " + "column has one canonical producer unless a later " + "stage explicitly declares it as a rewrite." + ) + else: + producers[column] = stage.name self._stages = materialized @property diff --git a/packages/microcosm-build/src/microcosm/build/source_manifest.py b/packages/microcosm-build/src/microcosm/build/source_manifest.py index c4dafc79..fca25b48 100644 --- a/packages/microcosm-build/src/microcosm/build/source_manifest.py +++ b/packages/microcosm-build/src/microcosm/build/source_manifest.py @@ -47,6 +47,7 @@ "assign_binary_from_rate", "assign_binary_with_anchored_residual", "assign_clipped_normal", + "assign_student_loan_plan_cohorts", "assign_uniform_draw", "aggregate_person_to_benunit", "allocate_per_capita_from_cell_table", @@ -58,7 +59,10 @@ "bridge_donor_column_via_qrf", "calibrate_binary_assignment", "calibrate_binary_assignment_joint_targets", + "classify_cgt_band_facts_with_reviewed_fence", "classify_hmrc_income_facts_with_reviewed_fences", + "clone_records", + "convert_donors_to_target_stock", "convert_interest_to_structural_mortgage_inputs", "compute_ratio", "declare_income_reference_offset", @@ -88,6 +92,7 @@ "derive_weeks_unemployed", "derive_wic_claim", "disaggregate_aggregate_records", + "draw_capital_gains_prior_from_banded_quantiles", "fit_labor_market_models", "fit_tip_income_model", "fit_weighted_acs_rent_qrf", @@ -119,22 +124,30 @@ "map_coded_amounts", "materialize_hmrc_income_bands_fail_closed", "materialize_rules_engine_predictors", + "rank_preserving_allocation", "read_table", "read_tables", "read_acs_rent_donor", "redraw_columns_from_fitted_qrf", + "record_mass_conservation_receipt", "replace_zero_weight_spi_support", "retain_adjudicated_frs_hmrc_leaves", "sample_categorical_from_count_table", "replace_sentinels", "split_component_by_share", + "stack_band_donor_households", "stack_zero_weight_donors", "strict_read_private_table", "support_clip", + "sub_aea_remainder", + "taxable_income_proxy", + "top_up_to_stock", "uprate", "uprate_to_regional_reference", "verify_certified_candidate", + "verify_pinned_cgt_ods", "verify_pinned_hmrc_source_pair", + "within_band_draws", "zero_when_false", } ) diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json index 1c7d9d1d..9d29c809 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json @@ -5872,7 +5872,13 @@ "targets": {"type": "array", "items": {"type": "string"}}, "weights": {"type": "string"}, "n_estimators": {"type": "integer", "minimum": 1}, - "seed": {"type": "integer"} + "seed": {"type": "integer"}, + "training_population": {"type": "string"}, + "target_population": {"type": "string"}, + "weight_mapping": {"type": "string"}, + "clamp_minimum": {"type": "number"}, + "preserve_asked_rows": {"type": "boolean"}, + "cache": {"type": "boolean"} } }, { @@ -6554,6 +6560,123 @@ } } }, + { + "type": "object", + "additionalProperties": false, + "required": ["kind", "entity", "copies", "flag_column", "mass_split", "weight_kind_out", "conservation", "id_remapping", "declared_factor", "reason"], + "properties": { + "kind": {"const": "clone_records"}, + "entity": {"type": "string"}, + "copies": {"type": "integer", "minimum": 2}, + "flag_column": {"type": "string"}, + "original_flag": {"type": "boolean"}, + "clone_flag": {"type": "boolean"}, + "mass_split": {"type": "number"}, + "weight_kind_out": {"type": "string"}, + "conservation": {"type": "string"}, + "id_remapping": {"type": "string"}, + "declared_factor": {"type": "number"}, + "reason": {"type": "string"} + } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["kind", "resource", "income_proxy_components", "allowance_subtraction", "carrier", "adult_minimum_age", "quantile_points", "spline_degree", "extrapolation", "keep_negative_draws", "seed", "salt"], + "properties": { + "kind": {"const": "draw_capital_gains_prior_from_banded_quantiles"}, + "resource": {"type": "string"}, + "income_proxy_components": {"type": "array", "items": {"type": "string"}}, + "allowance_subtraction": {"type": "boolean"}, + "carrier": {"type": "string"}, + "adult_minimum_age": {"type": "integer"}, + "quantile_points": {"type": "array", "items": {"type": "number"}}, + "spline_degree": {"type": "integer"}, + "extrapolation": {"type": "string"}, + "keep_negative_draws": {"type": "boolean"}, + "seed": {"type": "integer"}, + "salt": {"type": "string"} + } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["kind", "size_band_resource", "incidence_resource", "minimum_band_lower", "donors_per_band", "expected_band_count", "expected_donor_count", "candidate_order", "draw", "propensity", "seed", "flag_column", "carrier", "initial_weight", "never_zero_weight", "weight_kind_out", "reason"], + "properties": { + "kind": {"const": "stack_band_donor_households"}, + "size_band_resource": {"type": "string"}, + "incidence_resource": {"type": "string"}, + "minimum_band_lower": {"type": "number"}, + "donors_per_band": {"type": "integer", "minimum": 1}, + "expected_band_count": {"type": "integer", "minimum": 1}, + "expected_donor_count": {"type": "integer", "minimum": 1}, + "candidate_order": {"type": "string"}, + "draw": {"type": "string"}, + "propensity": {"type": "string"}, + "seed": {"type": "integer"}, + "flag_column": {"type": "string"}, + "carrier": {"type": "string"}, + "initial_weight": {"type": "string"}, + "never_zero_weight": {"type": "boolean"}, + "weight_kind_out": {"type": "string"}, + "reason": {"type": "string"} + } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["kind", "resource", "target", "donor_pool", "rate_cap", "move", "seed", "salt", "receipt"], + "properties": { + "kind": {"const": "convert_donors_to_target_stock"}, + "reason": {"type": "string"}, + "resource": {"type": "string"}, + "target": {"type": "number"}, + "donor_pool": {"type": "string"}, + "rate_cap": {"type": "number"}, + "move": {"type": "string"}, + "seed": {"type": "integer"}, + "salt": {"type": "string"}, + "receipt": {"type": "string"} + } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["kind", "year_rule", "start_year_formula", "reported_repayment_test", "reported_country_gate", "plan_1_before", "plan_5_from", "enum_domain", "plan_4_imputation"], + "properties": { + "kind": {"const": "assign_student_loan_plan_cohorts"}, + "year_rule": {"type": "string"}, + "start_year_formula": {"type": "string"}, + "reported_repayment_test": {"type": "string"}, + "reported_country_gate": {"type": "boolean"}, + "plan_1_before": {"type": "integer"}, + "plan_5_from": {"type": "integer"}, + "enum_domain": {"type": "array", "items": {"type": "string"}}, + "plan_4_imputation": {"type": "boolean"} + } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["kind", "plan", "priority", "resource", "stock_series", "year_rule", "age_min", "age_max", "cohort_start_min", "eligible_region_exclusions", "highest_education", "seed", "salt"], + "properties": { + "kind": {"const": "top_up_to_stock"}, + "reason": {"type": "string"}, + "plan": {"type": "string"}, + "priority": {"type": "integer"}, + "resource": {"type": "string"}, + "stock_series": {"type": "string"}, + "year_rule": {"type": "string"}, + "age_min": {"type": "integer"}, + "age_max": {"type": "integer"}, + "cohort_start_min": {"type": "integer"}, + "cohort_start_max_exclusive": {"type": "integer"}, + "eligible_region_exclusions": {"type": "array", "items": {"type": "string"}}, + "highest_education": {"type": "string"}, + "seed": {"type": "integer"}, + "salt": {"type": "string"} + } + }, { "type": "object", "additionalProperties": false, @@ -6625,6 +6748,9 @@ }, "format": { "type": "string" + }, + "runtime_sha256_required": { + "type": "boolean" } } } diff --git a/packages/microcosm-build/src/microcosm/build/uk/advani_summers_capital_gains_distribution.json b/packages/microcosm-build/src/microcosm/build/uk/advani_summers_capital_gains_distribution.json new file mode 100644 index 00000000..3d86479b --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk/advani_summers_capital_gains_distribution.json @@ -0,0 +1,820 @@ +{ + "version": 1, + "country": "uk", + "source": { + "citation": "Advani, Arun and Andy Summers (May 2020), Capital Gains and UK Inequality, CAGE Working Paper 465, University of Warwick.", + "url": "https://warwick.ac.uk/fac/soc/economics/research/centres/cage/manage/publications/wp465.2020.pdf", + "incumbent_csv_sha256": "7cb73f8c0a35aeb07c7f13bb0476fb48e9d3bd9b31735f3229554289906297a0", + "columns": [ + "percentile", + "minimum_total_income", + "percent_with_gains", + "mean_gains_given_gains", + "p05", + "p10", + "p25", + "p50", + "p75", + "p90", + "p95" + ] + }, + "rows": [ + { + "percentile": "<40", + "minimum_total_income": 0, + "percent_with_gains": 0.0031, + "mean_gains_given_gains": 45600, + "p05": -16400, + "p10": -4400, + "p25": 3800, + "p50": 14400, + "p75": 38400, + "p90": 92500, + "p95": 165000 + }, + { + "percentile": 40, + "minimum_total_income": 10000, + "percent_with_gains": 0.006, + "mean_gains_given_gains": 50300, + "p05": -12600, + "p10": -3100, + "p25": 4000, + "p50": 13500, + "p75": 37100, + "p90": 86900, + "p95": 156800 + }, + { + "percentile": 41, + "minimum_total_income": 10300, + "percent_with_gains": 0.0065, + "mean_gains_given_gains": 49000, + "p05": -17900, + "p10": -6600, + "p25": 3200, + "p50": 13300, + "p75": 36700, + "p90": 87700, + "p95": 142500 + }, + { + "percentile": 42, + "minimum_total_income": 10600, + "percent_with_gains": 0.0062, + "mean_gains_given_gains": 41100, + "p05": -16200, + "p10": -5500, + "p25": 3300, + "p50": 13000, + "p75": 31900, + "p90": 74000, + "p95": 125700 + }, + { + "percentile": 43, + "minimum_total_income": 11000, + "percent_with_gains": 0.0062, + "mean_gains_given_gains": 42600, + "p05": -15100, + "p10": -4700, + "p25": 3700, + "p50": 13200, + "p75": 34400, + "p90": 79500, + "p95": 143000 + }, + { + "percentile": 44, + "minimum_total_income": 11400, + "percent_with_gains": 0.0061, + "mean_gains_given_gains": 41900, + "p05": -15900, + "p10": -4100, + "p25": 2800, + "p50": 12800, + "p75": 34500, + "p90": 79400, + "p95": 144500 + }, + { + "percentile": 45, + "minimum_total_income": 11800, + "percent_with_gains": 0.0062, + "mean_gains_given_gains": 41800, + "p05": -14500, + "p10": -4000, + "p25": 3500, + "p50": 12500, + "p75": 34700, + "p90": 86900, + "p95": 147300 + }, + { + "percentile": 46, + "minimum_total_income": 12100, + "percent_with_gains": 0.0065, + "mean_gains_given_gains": 47700, + "p05": -13800, + "p10": -3800, + "p25": 3300, + "p50": 12500, + "p75": 35500, + "p90": 88400, + "p95": 158800 + }, + { + "percentile": 47, + "minimum_total_income": 12500, + "percent_with_gains": 0.0062, + "mean_gains_given_gains": 52400, + "p05": -13500, + "p10": -3800, + "p25": 3100, + "p50": 13200, + "p75": 35400, + "p90": 84100, + "p95": 158400 + }, + { + "percentile": 48, + "minimum_total_income": 12900, + "percent_with_gains": 0.0061, + "mean_gains_given_gains": 43600, + "p05": -14200, + "p10": -4000, + "p25": 3800, + "p50": 12800, + "p75": 33800, + "p90": 82200, + "p95": 137800 + }, + { + "percentile": 49, + "minimum_total_income": 13300, + "percent_with_gains": 0.0063, + "mean_gains_given_gains": 38600, + "p05": -13700, + "p10": -3700, + "p25": 3200, + "p50": 12800, + "p75": 34200, + "p90": 77000, + "p95": 129000 + }, + { + "percentile": 50, + "minimum_total_income": 13700, + "percent_with_gains": 0.0062, + "mean_gains_given_gains": 48200, + "p05": -14100, + "p10": -4300, + "p25": 2900, + "p50": 12300, + "p75": 33300, + "p90": 79400, + "p95": 150200 + }, + { + "percentile": 51, + "minimum_total_income": 14100, + "percent_with_gains": 0.006, + "mean_gains_given_gains": 40300, + "p05": -14700, + "p10": -4300, + "p25": 3100, + "p50": 12300, + "p75": 32900, + "p90": 79800, + "p95": 137700 + }, + { + "percentile": 52, + "minimum_total_income": 14500, + "percent_with_gains": 0.0063, + "mean_gains_given_gains": 46700, + "p05": -10700, + "p10": -3300, + "p25": 3600, + "p50": 12900, + "p75": 33500, + "p90": 74400, + "p95": 145500 + }, + { + "percentile": 53, + "minimum_total_income": 14900, + "percent_with_gains": 0.0062, + "mean_gains_given_gains": 49900, + "p05": -15300, + "p10": -5000, + "p25": 2400, + "p50": 11700, + "p75": 32800, + "p90": 78300, + "p95": 141700 + }, + { + "percentile": 54, + "minimum_total_income": 15300, + "percent_with_gains": 0.0062, + "mean_gains_given_gains": 38700, + "p05": -15800, + "p10": -4800, + "p25": 2800, + "p50": 11900, + "p75": 32400, + "p90": 80000, + "p95": 153800 + }, + { + "percentile": 55, + "minimum_total_income": 15700, + "percent_with_gains": 0.0063, + "mean_gains_given_gains": 40300, + "p05": -12100, + "p10": -4200, + "p25": 3000, + "p50": 12500, + "p75": 33000, + "p90": 79100, + "p95": 137000 + }, + { + "percentile": 56, + "minimum_total_income": 16100, + "percent_with_gains": 0.0063, + "mean_gains_given_gains": 42600, + "p05": -13400, + "p10": -3900, + "p25": 3400, + "p50": 12300, + "p75": 33800, + "p90": 82200, + "p95": 152800 + }, + { + "percentile": 57, + "minimum_total_income": 16600, + "percent_with_gains": 0.0062, + "mean_gains_given_gains": 50800, + "p05": -12700, + "p10": -3500, + "p25": 3700, + "p50": 13300, + "p75": 34900, + "p90": 88900, + "p95": 157600 + }, + { + "percentile": 58, + "minimum_total_income": 17000, + "percent_with_gains": 0.0065, + "mean_gains_given_gains": 49100, + "p05": -16100, + "p10": -4700, + "p25": 3100, + "p50": 11900, + "p75": 31900, + "p90": 80900, + "p95": 146800 + }, + { + "percentile": 59, + "minimum_total_income": 17400, + "percent_with_gains": 0.0064, + "mean_gains_given_gains": 42100, + "p05": -15700, + "p10": -4300, + "p25": 3100, + "p50": 11900, + "p75": 32500, + "p90": 81700, + "p95": 152100 + }, + { + "percentile": 60, + "minimum_total_income": 17900, + "percent_with_gains": 0.0065, + "mean_gains_given_gains": 50800, + "p05": -13100, + "p10": -4300, + "p25": 3100, + "p50": 12300, + "p75": 32600, + "p90": 79400, + "p95": 157000 + }, + { + "percentile": 61, + "minimum_total_income": 18300, + "percent_with_gains": 0.0062, + "mean_gains_given_gains": 42200, + "p05": -13800, + "p10": -4100, + "p25": 3300, + "p50": 12400, + "p75": 32300, + "p90": 78100, + "p95": 146700 + }, + { + "percentile": 62, + "minimum_total_income": 18800, + "percent_with_gains": 0.0065, + "mean_gains_given_gains": 39900, + "p05": -13400, + "p10": -4600, + "p25": 3200, + "p50": 12200, + "p75": 32500, + "p90": 80700, + "p95": 138500 + }, + { + "percentile": 63, + "minimum_total_income": 19200, + "percent_with_gains": 0.0069, + "mean_gains_given_gains": 46900, + "p05": -12700, + "p10": -3400, + "p25": 3500, + "p50": 12100, + "p75": 32100, + "p90": 84000, + "p95": 167000 + }, + { + "percentile": 64, + "minimum_total_income": 19700, + "percent_with_gains": 0.0069, + "mean_gains_given_gains": 43600, + "p05": -13800, + "p10": -4000, + "p25": 3200, + "p50": 12400, + "p75": 31000, + "p90": 85700, + "p95": 152000 + }, + { + "percentile": 65, + "minimum_total_income": 20200, + "percent_with_gains": 0.0069, + "mean_gains_given_gains": 52600, + "p05": -14200, + "p10": -4200, + "p25": 3600, + "p50": 12100, + "p75": 34800, + "p90": 88900, + "p95": 176400 + }, + { + "percentile": 66, + "minimum_total_income": 20700, + "percent_with_gains": 0.0069, + "mean_gains_given_gains": 39500, + "p05": -11800, + "p10": -3800, + "p25": 4000, + "p50": 12700, + "p75": 33300, + "p90": 78500, + "p95": 143300 + }, + { + "percentile": 67, + "minimum_total_income": 21200, + "percent_with_gains": 0.0072, + "mean_gains_given_gains": 47900, + "p05": -11000, + "p10": -3300, + "p25": 3900, + "p50": 12500, + "p75": 32700, + "p90": 79200, + "p95": 155400 + }, + { + "percentile": 68, + "minimum_total_income": 21800, + "percent_with_gains": 0.0074, + "mean_gains_given_gains": 40900, + "p05": -12800, + "p10": -4200, + "p25": 3300, + "p50": 11800, + "p75": 31200, + "p90": 72700, + "p95": 122300 + }, + { + "percentile": 69, + "minimum_total_income": 22300, + "percent_with_gains": 0.0077, + "mean_gains_given_gains": 46300, + "p05": -13700, + "p10": -4100, + "p25": 2900, + "p50": 11800, + "p75": 32100, + "p90": 78200, + "p95": 151900 + }, + { + "percentile": 70, + "minimum_total_income": 22900, + "percent_with_gains": 0.0077, + "mean_gains_given_gains": 50600, + "p05": -10600, + "p10": -3400, + "p25": 3400, + "p50": 12200, + "p75": 31700, + "p90": 75800, + "p95": 148000 + }, + { + "percentile": 71, + "minimum_total_income": 23500, + "percent_with_gains": 0.0082, + "mean_gains_given_gains": 41200, + "p05": -11600, + "p10": -3500, + "p25": 3500, + "p50": 12100, + "p75": 32600, + "p90": 83400, + "p95": 146800 + }, + { + "percentile": 72, + "minimum_total_income": 24100, + "percent_with_gains": 0.0084, + "mean_gains_given_gains": 42000, + "p05": -12400, + "p10": -4600, + "p25": 2900, + "p50": 11300, + "p75": 29900, + "p90": 77400, + "p95": 154000 + }, + { + "percentile": 73, + "minimum_total_income": 24800, + "percent_with_gains": 0.0086, + "mean_gains_given_gains": 49000, + "p05": -15300, + "p10": -5100, + "p25": 3100, + "p50": 11900, + "p75": 31700, + "p90": 83800, + "p95": 161800 + }, + { + "percentile": 74, + "minimum_total_income": 25400, + "percent_with_gains": 0.0084, + "mean_gains_given_gains": 41400, + "p05": -14700, + "p10": -4000, + "p25": 3300, + "p50": 11800, + "p75": 30000, + "p90": 82600, + "p95": 154200 + }, + { + "percentile": 75, + "minimum_total_income": 26100, + "percent_with_gains": 0.0087, + "mean_gains_given_gains": 42600, + "p05": -16400, + "p10": -5000, + "p25": 2900, + "p50": 11500, + "p75": 30900, + "p90": 83200, + "p95": 167800 + }, + { + "percentile": 76, + "minimum_total_income": 26800, + "percent_with_gains": 0.0092, + "mean_gains_given_gains": 42400, + "p05": -13800, + "p10": -3900, + "p25": 3300, + "p50": 11800, + "p75": 31500, + "p90": 84500, + "p95": 162700 + }, + { + "percentile": 77, + "minimum_total_income": 27500, + "percent_with_gains": 0.0094, + "mean_gains_given_gains": 42600, + "p05": -13200, + "p10": -3800, + "p25": 3800, + "p50": 12300, + "p75": 33200, + "p90": 82600, + "p95": 145000 + }, + { + "percentile": 78, + "minimum_total_income": 28300, + "percent_with_gains": 0.0097, + "mean_gains_given_gains": 51900, + "p05": -14900, + "p10": -3800, + "p25": 4100, + "p50": 12500, + "p75": 31900, + "p90": 82400, + "p95": 148500 + }, + { + "percentile": 79, + "minimum_total_income": 29100, + "percent_with_gains": 0.0102, + "mean_gains_given_gains": 44700, + "p05": -12600, + "p10": -4000, + "p25": 3000, + "p50": 11700, + "p75": 29200, + "p90": 75700, + "p95": 145400 + }, + { + "percentile": 80, + "minimum_total_income": 30000, + "percent_with_gains": 0.0103, + "mean_gains_given_gains": 49800, + "p05": -15400, + "p10": -4200, + "p25": 4000, + "p50": 12500, + "p75": 32700, + "p90": 93600, + "p95": 184700 + }, + { + "percentile": 81, + "minimum_total_income": 30900, + "percent_with_gains": 0.0112, + "mean_gains_given_gains": 50400, + "p05": -11900, + "p10": -3300, + "p25": 3200, + "p50": 11900, + "p75": 32300, + "p90": 87500, + "p95": 164400 + }, + { + "percentile": 82, + "minimum_total_income": 31800, + "percent_with_gains": 0.0113, + "mean_gains_given_gains": 46200, + "p05": -13900, + "p10": -4300, + "p25": 3100, + "p50": 11500, + "p75": 30100, + "p90": 86700, + "p95": 165700 + }, + { + "percentile": 83, + "minimum_total_income": 32800, + "percent_with_gains": 0.0118, + "mean_gains_given_gains": 47800, + "p05": -12800, + "p10": -3600, + "p25": 3300, + "p50": 11700, + "p75": 31700, + "p90": 91900, + "p95": 174100 + }, + { + "percentile": 84, + "minimum_total_income": 33900, + "percent_with_gains": 0.0121, + "mean_gains_given_gains": 46000, + "p05": -13700, + "p10": -4400, + "p25": 3600, + "p50": 11800, + "p75": 30700, + "p90": 93800, + "p95": 178300 + }, + { + "percentile": 85, + "minimum_total_income": 35000, + "percent_with_gains": 0.0136, + "mean_gains_given_gains": 46800, + "p05": -14300, + "p10": -4100, + "p25": 3700, + "p50": 12200, + "p75": 30800, + "p90": 90000, + "p95": 181200 + }, + { + "percentile": 86, + "minimum_total_income": 36200, + "percent_with_gains": 0.0147, + "mean_gains_given_gains": 55100, + "p05": -13200, + "p10": -4200, + "p25": 3500, + "p50": 12000, + "p75": 33300, + "p90": 100800, + "p95": 206800 + }, + { + "percentile": 87, + "minimum_total_income": 37400, + "percent_with_gains": 0.0172, + "mean_gains_given_gains": 59400, + "p05": -14500, + "p10": -4500, + "p25": 3600, + "p50": 11800, + "p75": 33800, + "p90": 108100, + "p95": 212600 + }, + { + "percentile": 88, + "minimum_total_income": 38700, + "percent_with_gains": 0.0191, + "mean_gains_given_gains": 52100, + "p05": -15000, + "p10": -5300, + "p25": 2300, + "p50": 11400, + "p75": 35100, + "p90": 110800, + "p95": 221600 + }, + { + "percentile": 89, + "minimum_total_income": 39900, + "percent_with_gains": 0.02, + "mean_gains_given_gains": 55600, + "p05": -16700, + "p10": -4900, + "p25": 2900, + "p50": 11500, + "p75": 34000, + "p90": 111700, + "p95": 226300 + }, + { + "percentile": 90, + "minimum_total_income": 41500, + "percent_with_gains": 0.0204, + "mean_gains_given_gains": 59000, + "p05": -17200, + "p10": -5500, + "p25": 2400, + "p50": 11400, + "p75": 32500, + "p90": 99800, + "p95": 209800 + }, + { + "percentile": 91, + "minimum_total_income": 43300, + "percent_with_gains": 0.021, + "mean_gains_given_gains": 53200, + "p05": -16500, + "p10": -5100, + "p25": 2800, + "p50": 11400, + "p75": 30700, + "p90": 99900, + "p95": 213000 + }, + { + "percentile": 92, + "minimum_total_income": 45500, + "percent_with_gains": 0.0226, + "mean_gains_given_gains": 58500, + "p05": -16100, + "p10": -5000, + "p25": 3200, + "p50": 11600, + "p75": 32700, + "p90": 100900, + "p95": 208600 + }, + { + "percentile": 93, + "minimum_total_income": 48300, + "percent_with_gains": 0.0247, + "mean_gains_given_gains": 68300, + "p05": -14900, + "p10": -4700, + "p25": 3800, + "p50": 11700, + "p75": 34600, + "p90": 106800, + "p95": 226200 + }, + { + "percentile": 94, + "minimum_total_income": 51900, + "percent_with_gains": 0.0281, + "mean_gains_given_gains": 50600, + "p05": -15500, + "p10": -4800, + "p25": 3400, + "p50": 11700, + "p75": 35500, + "p90": 113500, + "p95": 250000 + }, + { + "percentile": 95, + "minimum_total_income": 56700, + "percent_with_gains": 0.0326, + "mean_gains_given_gains": 73800, + "p05": -17800, + "p10": -5800, + "p25": 3300, + "p50": 11700, + "p75": 35000, + "p90": 120500, + "p95": 265700 + }, + { + "percentile": 96, + "minimum_total_income": 63400, + "percent_with_gains": 0.0391, + "mean_gains_given_gains": 83600, + "p05": -18300, + "p10": -5600, + "p25": 3600, + "p50": 12000, + "p75": 38100, + "p90": 134400, + "p95": 310900 + }, + { + "percentile": 97, + "minimum_total_income": 73400, + "percent_with_gains": 0.0503, + "mean_gains_given_gains": 96100, + "p05": -20200, + "p10": -5900, + "p25": 3600, + "p50": 12500, + "p75": 41900, + "p90": 161500, + "p95": 373900 + }, + { + "percentile": 98, + "minimum_total_income": 90100, + "percent_with_gains": 0.071, + "mean_gains_given_gains": 120400, + "p05": -24100, + "p10": -7500, + "p25": 2900, + "p50": 12700, + "p75": 48000, + "p90": 200000, + "p95": 470000 + }, + { + "percentile": 99, + "minimum_total_income": 128200, + "percent_with_gains": 0.1508, + "mean_gains_given_gains": 306800, + "p05": -42800, + "p10": -12400, + "p25": 200, + "p50": 13600, + "p75": 74900, + "p90": 431600, + "p95": 1162400 + } + ], + "corrections": [ + "Percentile 69 p95 corrected from the incumbent's 15,190 to 151,900: verified against Advani & Summers (2020), CAGE WP 465, Table A1 (p. 39) - '69 22,300 .0077 46,300 -13,700 -4,100 2,900 11,800 32,100 78,200 151,900'. The incumbent carries a dropped-digit transcription; all other 60 rows verified value-for-value against Table A1 (fix-and-sign, 2026-08-22). The signed effect vs the incumbent: band-69 prior draws above q~0.962 are no longer spurious loss-makers." + ] +} diff --git a/packages/microcosm-build/src/microcosm/build/uk/cgt_band_donor_support_bounds.json b/packages/microcosm-build/src/microcosm/build/uk/cgt_band_donor_support_bounds.json new file mode 100644 index 00000000..3e6e4dda --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk/cgt_band_donor_support_bounds.json @@ -0,0 +1,25 @@ +{ + "version": 1, + "country": "uk", + "policy": "Disclosure-free support intervals derived from published HMRC Table 2.1a gain bands retained by the CGT donor stage. Intervals are lower-inclusive and upper-exclusive.", + "source": { + "resource": "hmrc_cgt_size_bands.json", + "table": "2.1a", + "minimum_lower_limit": 12300 + }, + "bounds": { + "capital_gains": [12300, null] + }, + "bands": [ + {"lower": 12300, "upper": 25000}, + {"lower": 25000, "upper": 50000}, + {"lower": 50000, "upper": 100000}, + {"lower": 100000, "upper": 250000}, + {"lower": 250000, "upper": 500000}, + {"lower": 500000, "upper": 1000000}, + {"lower": 1000000, "upper": 2000000}, + {"lower": 2000000, "upper": 5000000}, + {"lower": 5000000, "upper": null} + ], + "semantic_fit_note": "These intervals describe donor-stage initial support. The later Table 3 redraw can move amounts within a different band surface, so terminal support-gate applicability requires reviewer confirmation." +} diff --git a/packages/microcosm-build/src/microcosm/build/uk/country_package.json b/packages/microcosm-build/src/microcosm/build/uk/country_package.json index 9bc16e85..2c04f97d 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/country_package.json +++ b/packages/microcosm-build/src/microcosm/build/uk/country_package.json @@ -67,6 +67,31 @@ "kind": "legacy_json", "schema_id": "legacy_json" }, + { + "path": "hmrc_cgt_size_bands.json", + "kind": "legacy_json", + "schema_id": "legacy_json" + }, + { + "path": "advani_summers_capital_gains_distribution.json", + "kind": "legacy_json", + "schema_id": "legacy_json" + }, + { + "path": "salary_sacrifice_anchor.json", + "kind": "legacy_json", + "schema_id": "legacy_json" + }, + { + "path": "slc_liable_stocks.json", + "kind": "legacy_json", + "schema_id": "legacy_json" + }, + { + "path": "cgt_band_donor_support_bounds.json", + "kind": "legacy_json", + "schema_id": "legacy_json" + }, { "path": "hmrc_income_release_gate_report.json", "kind": "legacy_json", diff --git a/packages/microcosm-build/src/microcosm/build/uk/gates.json b/packages/microcosm-build/src/microcosm/build/uk/gates.json index 5b33c56c..39a384de 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/gates.json +++ b/packages/microcosm-build/src/microcosm/build/uk/gates.json @@ -208,6 +208,7 @@ "household.gas_consumption", "household.has_fuel_consumption", "household.household_is_capital_gains_clone", + "household.household_is_cgt_band_donor", "household.household_is_spi_synthetic", "household.la_code_oa", "household.lsoa_code", @@ -277,6 +278,18 @@ }, "notes": "The household BRMA assignment must remain inside the PolicyEngine-UK brma enum domain." }, + { + "id": "uk_student_loan_plan_enum_domain", + "gate": "enum_domain", + "phase": "terminal", + "criticality": "release_blocking", + "parameters": { + "columns": [ + "student_loan_plan" + ] + }, + "notes": "The person student-loan plan assignment must remain inside the PolicyEngine-UK StudentLoanPlan enum domain." + }, { "id": "uk_calibration_reference_coverage", "gate": "calibration_reference_coverage", diff --git a/packages/microcosm-build/src/microcosm/build/uk/hmrc_cgt_size_bands.json b/packages/microcosm-build/src/microcosm/build/uk/hmrc_cgt_size_bands.json new file mode 100644 index 00000000..f17ec7fa --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk/hmrc_cgt_size_bands.json @@ -0,0 +1,111 @@ +{ + "version": 1, + "country": "uk", + "source": { + "citation": "HMRC, Capital Gains Tax statistics, Table 2.1a: estimated number of taxpayers, amounts of gains and tax liabilities by size of gain, individuals, 2023 to 2024 (provisional).", + "collection_url": "https://www.gov.uk/government/statistics/capital-gains-tax-statistics", + "url": "https://assets.publishing.service.gov.uk/media/6878ac562bad77c3dae4dcef/Table_2_2025_Size_of_gain.ods", + "artifact": "Table_2_2025_Size_of_gain.ods", + "sheet": "2_1a_2023-24", + "published": "2025-07-24", + "ods_sha256": "696456114f408edd84c004fbe6175d918c5a511b6aaa7a53a95ca8936db22491", + "ods_size_bytes": 10658, + "ods_retrieved": "2026-08-21", + "ods_content_check": "Sheet 2_1a_2023-24 present; taxpayer (79/74/53 thousand) and gains (1418/2645/22714 GBP million) cell values match the committed extraction.", + "extracted_csv_sha256": "97e6bfc60d35eec230a62f4412e3adc034f58c46b3699570415a547f59628ecb", + "mapped_build_period": "2024", + "period_mapping": "latest_published_tax_year", + "units": { + "taxpayers_thousands": "thousand taxpayers", + "gains_gbp_millions": "GBP million", + "tax_gbp_millions": "GBP million" + }, + "definition_note": "Published figures include only taxpayers with a CGT liability; they are narrower than all people with positive gains or gains above the annual exempt amount." + }, + "rows": [ + { + "lower_limit": 0, + "taxpayers_thousands": 2, + "gains_gbp_millions": 1, + "tax_gbp_millions": 9 + }, + { + "lower_limit": 3000, + "taxpayers_thousands": 0, + "gains_gbp_millions": 1, + "tax_gbp_millions": 0 + }, + { + "lower_limit": 6000, + "taxpayers_thousands": 61, + "gains_gbp_millions": 461, + "tax_gbp_millions": 18 + }, + { + "lower_limit": 10000, + "taxpayers_thousands": 23, + "gains_gbp_millions": 251, + "tax_gbp_millions": 20 + }, + { + "lower_limit": 12300, + "taxpayers_thousands": 79, + "gains_gbp_millions": 1418, + "tax_gbp_millions": 169 + }, + { + "lower_limit": 25000, + "taxpayers_thousands": 74, + "gains_gbp_millions": 2645, + "tax_gbp_millions": 423 + }, + { + "lower_limit": 50000, + "taxpayers_thousands": 53, + "gains_gbp_millions": 3731, + "tax_gbp_millions": 706 + }, + { + "lower_limit": 100000, + "taxpayers_thousands": 37, + "gains_gbp_millions": 5649, + "tax_gbp_millions": 1077 + }, + { + "lower_limit": 250000, + "taxpayers_thousands": 14, + "gains_gbp_millions": 4766, + "tax_gbp_millions": 826 + }, + { + "lower_limit": 500000, + "taxpayers_thousands": 8, + "gains_gbp_millions": 5705, + "tax_gbp_millions": 895 + }, + { + "lower_limit": 1000000, + "taxpayers_thousands": 5, + "gains_gbp_millions": 6390, + "tax_gbp_millions": 1067 + }, + { + "lower_limit": 2000000, + "taxpayers_thousands": 3, + "gains_gbp_millions": 9189, + "tax_gbp_millions": 1718 + }, + { + "lower_limit": 5000000, + "taxpayers_thousands": 2, + "gains_gbp_millions": 22714, + "tax_gbp_millions": 4532 + } + ], + "retained_band_checksums": { + "minimum_lower_limit": 12300, + "band_count": 9, + "taxpayers": 275000, + "gains_gbp": 62207000000 + } +} diff --git a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json index 7c166b3c..e6222dec 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json @@ -458,10 +458,57 @@ "weight_source": "household_weight" }, "family_coverage": { + "cgt_band_donors": { + "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", + "base_candidate_tier": "frs", + "effective_mass_requirements": {}, + "mass_change_semantics": "mass_increasing_support", + "output_weight_kind": "importance", + "outputs": [ + "household_is_cgt_band_donor", + "capital_gains" + ], + "required_mass_change_reason": "Stack 30 positive-weight HMRC Table 2.1a support households per retained gain band; published donor mass is added explicitly.", + "rewrites": [ + "capital_gains" + ], + "source_manifest": "source_stages.json", + "source_manifest_sha256": "6f50637a95c340c07382d13af5a43da9d2a1a1167f713aba3605b3ca62efe449", + "source_vintages": { + "source": "HMRC Capital Gains Tax statistics, July 2025, Table 2.1a", + "survey": "HMRC Capital Gains Tax statistics Table 2.1a and Advani-Summers capital-gains incidence" + }, + "stage": "cgt_band_donors", + "status": "required_at_build" + }, + "cgt_incidence_clone": { + "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", + "base_candidate_tier": "frs", + "effective_mass_requirements": {}, + "mass_change_semantics": "mass_conserving", + "output_weight_kind": "importance", + "outputs": [ + "household_is_capital_gains_clone", + "capital_gains" + ], + "required_mass_change_reason": "Capital-gains incidence clone splits every household's mass equally across original and clone records; total household mass is conserved.", + "rewrites": [ + "capital_gains" + ], + "source_manifest": "source_stages.json", + "source_manifest_sha256": "6f50637a95c340c07382d13af5a43da9d2a1a1167f713aba3605b3ca62efe449", + "source_vintages": { + "source": "Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465", + "survey": "Family Resources Survey 2024-25, SPI synthetic support, and Advani-Summers capital-gains incidence" + }, + "stage": "cgt_incidence_clone", + "status": "required_at_build" + }, "etb_services": { "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", "base_candidate_tier": "frs", "effective_mass_requirements": {}, + "mass_change_semantics": "mass_conserving", "output_weight_kind": "importance", "outputs": [ "dfe_education_spending", @@ -478,7 +525,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "31818338ef62a19d9aebad7cdcf79af7cf05c2f0657ecc5e53b5954390ffa93b", + "source_manifest_sha256": "6f50637a95c340c07382d13af5a43da9d2a1a1167f713aba3605b3ca62efe449", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab, DfT rail fare index, and public NHS activity/cost table.", "survey": "Effects of Taxes and Benefits 1977-2024 and NHS age-gender public table" @@ -490,6 +537,7 @@ "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", "base_candidate_tier": "frs", "effective_mass_requirements": {}, + "mass_change_semantics": "mass_conserving", "output_weight_kind": "importance", "outputs": [ "full_rate_vat_expenditure_rate" @@ -497,7 +545,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "31818338ef62a19d9aebad7cdcf79af7cf05c2f0657ecc5e53b5954390ffa93b", + "source_manifest_sha256": "6f50637a95c340c07382d13af5a43da9d2a1a1167f713aba3605b3ca62efe449", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab and cited VAT anchor resource.", "survey": "Effects of Taxes and Benefits 1977-2024" @@ -526,13 +574,37 @@ "stage": "hmrc_cgt_gains", "status": "required_at_build" }, + "hmrc_cgt_gains_spine": { + "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", + "base_candidate_tier": "frs", + "calibration_permitted": false, + "effective_mass_requirements": {}, + "fact_fence_id": "cgt_band_facts_policy_endogenous_proxy_conditioned", + "fenced_fact_count": 76, + "output_weight_kind": "importance", + "outputs": [ + "capital_gains" + ], + "required_mass_change_reason": "Amounts-only capital gains redraw on the source spine: household weights pass through unchanged and total household mass is conserved.", + "rewrites": [ + "capital_gains" + ], + "source_manifest": "source_stages.json", + "source_manifest_sha256": "6f50637a95c340c07382d13af5a43da9d2a1a1167f713aba3605b3ca62efe449", + "source_vintages": { + "hmrc_surface": "2023-24", + "mapped_build_period": "2024" + }, + "stage": "hmrc_cgt_gains_spine", + "status": "required_at_build" + }, "hmrc_spi_income": { "band_measure": "hmrc_spi_assessable_income", "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", "base_candidate_tier": "frs", "calibration_permitted": false, "canonical_source_manifest": "source_stages.json", - "canonical_source_manifest_sha256": "31818338ef62a19d9aebad7cdcf79af7cf05c2f0657ecc5e53b5954390ffa93b", + "canonical_source_manifest_sha256": "6f50637a95c340c07382d13af5a43da9d2a1a1167f713aba3605b3ca62efe449", "effective_mass_requirements": { "charitable_investment_gifts": { "mass_share_denominator": "all_person_effective_mass", @@ -617,6 +689,7 @@ "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", "base_candidate_tier": "frs", "effective_mass_requirements": {}, + "mass_change_semantics": "mass_conserving", "output_weight_kind": "importance", "outputs": [ "food_and_non_alcoholic_beverages_consumption", @@ -642,7 +715,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "31818338ef62a19d9aebad7cdcf79af7cf05c2f0657ecc5e53b5954390ffa93b", + "source_manifest_sha256": "6f50637a95c340c07382d13af5a43da9d2a1a1167f713aba3605b3ca62efe449", "source_vintages": { "source": "UK Data Service SN 9468 Living Costs and Food Survey 2023-24 household/person tabs, NEED 2023 headline energy tables, Ofgem Q2 2026 unit rates, and WAS round-8 bridge donor.", "survey": "Living Costs and Food Survey 2023-24" @@ -654,6 +727,7 @@ "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", "base_candidate_tier": "frs", "effective_mass_requirements": {}, + "mass_change_semantics": "mass_conserving", "output_weight_kind": "importance", "outputs": [], "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", @@ -662,7 +736,7 @@ "property_wealth" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "31818338ef62a19d9aebad7cdcf79af7cf05c2f0657ecc5e53b5954390ffa93b", + "source_manifest_sha256": "6f50637a95c340c07382d13af5a43da9d2a1a1167f713aba3605b3ca62efe449", "source_vintages": { "source": "MHCLG dwellings and ONS UK House Price Index December 2025 regional average prices.", "survey": "Public regional property reference" @@ -670,10 +744,57 @@ "stage": "regional_property_uprating", "status": "required_at_build" }, + "salary_sacrifice": { + "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", + "base_candidate_tier": "frs", + "effective_mass_requirements": {}, + "mass_change_semantics": "mass_conserving", + "output_weight_kind": "importance", + "outputs": [ + "pension_contributions_via_salary_sacrifice", + "employee_pension_contributions" + ], + "required_mass_change_reason": "Salary-sacrifice support stage rewrites pension columns only; household rows and typed household weights pass through and total household mass is conserved.", + "rewrites": [ + "pension_contributions_via_salary_sacrifice", + "employee_pension_contributions" + ], + "source_manifest": "source_stages.json", + "source_manifest_sha256": "6f50637a95c340c07382d13af5a43da9d2a1a1167f713aba3605b3ca62efe449", + "source_vintages": { + "source": "HMRC, Salary sacrifice reform for pension contributions effective from 6 April 2029", + "survey": "Family Resources Survey 2024-25 salary-sacrifice respondents and HMRC salary-sacrifice reform analysis" + }, + "stage": "salary_sacrifice", + "status": "required_at_build" + }, + "student_loans": { + "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", + "base_candidate_tier": "frs", + "effective_mass_requirements": {}, + "mass_change_semantics": "mass_conserving", + "output_weight_kind": "importance", + "outputs": [ + "student_loan_plan" + ], + "required_mass_change_reason": "Student-loan plan assignment writes an enum column only; household rows and typed household weights pass through and total household mass is conserved.", + "rewrites": [ + "student_loan_plan" + ], + "source_manifest": "source_stages.json", + "source_manifest_sha256": "6f50637a95c340c07382d13af5a43da9d2a1a1167f713aba3605b3ca62efe449", + "source_vintages": { + "source": "Explore Education Statistics Table 6a, Higher education total", + "survey": "Family Resources Survey 2024-25 and Student Loans Company borrower forecasts for England" + }, + "stage": "student_loans", + "status": "required_at_build" + }, "was_wealth": { "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", "base_candidate_tier": "frs", "effective_mass_requirements": {}, + "mass_change_semantics": "mass_conserving", "output_weight_kind": "importance", "outputs": [ "owned_land", @@ -693,7 +814,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "31818338ef62a19d9aebad7cdcf79af7cf05c2f0657ecc5e53b5954390ffa93b", + "source_manifest_sha256": "6f50637a95c340c07382d13af5a43da9d2a1a1167f713aba3605b3ca62efe449", "source_vintages": { "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed 2006-22 household tab.", "survey": "Wealth and Assets Survey round 8" diff --git a/packages/microcosm-build/src/microcosm/build/uk/salary_sacrifice_anchor.json b/packages/microcosm-build/src/microcosm/build/uk/salary_sacrifice_anchor.json new file mode 100644 index 00000000..9fe0255d --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk/salary_sacrifice_anchor.json @@ -0,0 +1,26 @@ +{ + "version": 1, + "country": "uk", + "source": { + "citation": "HMRC, Salary sacrifice reform for pension contributions effective from 6 April 2029.", + "url": "https://www.gov.uk/government/publications/salary-sacrifice-reform-for-pension-contributions-effective-from-6-april-2029/salary-sacrifice-reform-for-pension-contributions", + "published_year": 2025, + "raw_artifact_sha256": "0b966220ca665c4d08307080329e1f912bd5ebd78b60796d70a6ad52e3c68f7b", + "base_year": 2024, + "annual_growth": 0.024, + "growth_note": "The incumbent projects users at 2.4 percent annually from 2024; the committed 2024 anchor itself is HMRC's 7.7 million estimate." + }, + "hmrc_anchor": { + "total_users": 7700000, + "above_2000": 3300000, + "below_2000": 4300000 + }, + "derived": { + "staging_ratio": 0.7012987012987013, + "stage_target": 5400000 + }, + "notes": [ + "The 5.4 million stage target is a deliberate support-staging fraction, not a published target.", + "The incumbent obr/ naming is a misattribution: the 7.7 million total and 3.3/4.3 million split come from HMRC." + ] +} diff --git a/packages/microcosm-build/src/microcosm/build/uk/slc_liable_stocks.json b/packages/microcosm-build/src/microcosm/build/uk/slc_liable_stocks.json new file mode 100644 index 00000000..cd16c5cb --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk/slc_liable_stocks.json @@ -0,0 +1,22 @@ +{ + "version": 1, + "country": "uk", + "source": { + "citation": "Student Loans Company, Student loan forecasts for England, Table 6a, Higher education total.", + "permalink": "https://explore-education-statistics.service.gov.uk/data-tables/permalink/6ff75517-7124-487c-cb4e-08de6eccf22d", + "raw_artifact_sha256": "b6de727ec0f8b1771c9a2d2c781b405830872634ed7283bf23e46a3898873922", + "chronicle_package_id": "slc-student-loan-borrower-forecasts-england-2025", + "chronicle_candidate": true, + "period_note": "Academic year 2024/25 maps to calendar year 2025 here; Chronicle stores the academic-year opening year 2024." + }, + "plans": { + "plan_2": { + "above_threshold": {"2025": 3985000, "2026": 4460000, "2027": 4825000, "2028": 5045000, "2029": 5160000, "2030": 5205000}, + "liable": {"2025": 8940000, "2026": 9710000, "2027": 10360000, "2028": 10615000, "2029": 10600000, "2030": 10525000} + }, + "plan_5": { + "above_threshold": {"2025": 0, "2026": 35000, "2027": 145000, "2028": 390000, "2029": 770000, "2030": 1235000}, + "liable": {"2025": 10000, "2026": 230000, "2027": 630000, "2028": 1380000, "2029": 2360000, "2030": 3400000} + } + } +} diff --git a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json index 9e052b4c..b777f420 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json @@ -2398,6 +2398,353 @@ ], "notes": "Runs the SPI-trained income QRFs on the raw-spine support channel, initializes FRS charity columns to zero, trains FRS-only stage 2 before redrawing base-channel dividends, and emits a sidecar-only 208-fact replay report for the spine path." }, + { + "stage": "cgt_incidence_clone", + "survey": "Family Resources Survey 2024-25, SPI synthetic support, and Advani-Summers capital-gains incidence", + "source": "Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465", + "grain": "household", + "artifacts": [ + { + "role": "capital_gains_incidence_and_quantiles", + "kind": "public_aggregate_reference", + "resource": "advani_summers_capital_gains_distribution.json", + "format": "json", + "runtime_sha256_required": true + } + ], + "operations": [ + { + "kind": "clone_records", + "entity": "household", + "copies": 2, + "flag_column": "household_is_capital_gains_clone", + "original_flag": false, + "clone_flag": true, + "mass_split": 0.5, + "weight_kind_out": "importance", + "conservation": "exact_total", + "id_remapping": "id_multiplier_for_values", + "declared_factor": 1.0, + "reason": "Capital-gains incidence clone splits every household's mass equally across original and clone records; total household mass is conserved." + }, + { + "kind": "draw_capital_gains_prior_from_banded_quantiles", + "resource": "advani_summers_capital_gains_distribution.json", + "income_proxy_components": [ + "employment_income", + "self_employment_income", + "state_pension_reported", + "private_pension_income", + "property_income", + "savings_interest_income", + "dividend_income", + "miscellaneous_income" + ], + "allowance_subtraction": false, + "carrier": "oldest adult; person_id ascending breaks age ties", + "adult_minimum_age": 16, + "quantile_points": [0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95], + "spline_degree": 1, + "extrapolation": "ext=0", + "keep_negative_draws": true, + "seed": 0, + "salt": "cgt_prior_amount" + } + ], + "outputs": [ + "household_is_capital_gains_clone", + "capital_gains" + ], + "rewrites": ["capital_gains"], + "notes": "Spine-only equal-mass incidence clone. The A&S prior fixes the gainer set and order for the following HMRC Table 3 redraw; negative extrapolated draws remain loss-makers." + }, + { + "stage": "cgt_band_donors", + "survey": "HMRC Capital Gains Tax statistics Table 2.1a and Advani-Summers capital-gains incidence", + "source": "HMRC Capital Gains Tax statistics, July 2025, Table 2.1a", + "grain": "household", + "artifacts": [ + { + "role": "hmrc_cgt_size_bands", + "kind": "public_aggregate_reference", + "resource": "hmrc_cgt_size_bands.json", + "format": "json", + "runtime_sha256_required": true + }, + { + "role": "capital_gains_incidence", + "kind": "public_aggregate_reference", + "resource": "advani_summers_capital_gains_distribution.json", + "format": "json", + "runtime_sha256_required": true + } + ], + "operations": [ + { + "kind": "stack_band_donor_households", + "size_band_resource": "hmrc_cgt_size_bands.json", + "incidence_resource": "advani_summers_capital_gains_distribution.json", + "minimum_band_lower": 12300, + "donors_per_band": 30, + "expected_band_count": 9, + "expected_donor_count": 270, + "candidate_order": "household_id ascending", + "draw": "weighted_without_replacement", + "propensity": "Advani-Summers percent_with_gains at oldest-adult component-sum income", + "seed": 1, + "flag_column": "household_is_cgt_band_donor", + "carrier": "oldest adult; person_id ascending breaks age ties", + "initial_weight": "published band taxpayers / donors_per_band", + "never_zero_weight": true, + "weight_kind_out": "importance", + "reason": "Stack 30 positive-weight HMRC Table 2.1a support households per retained gain band; published donor mass is added explicitly." + } + ], + "outputs": [ + "household_is_cgt_band_donor", + "capital_gains" + ], + "rewrites": ["capital_gains"], + "notes": "Adds 270 positive-weight band donors. Rows below GBP 12,300 are excluded because they mix annual-exempt-amount regimes and the spline body already supplies that support." + }, + { + "stage": "hmrc_cgt_gains_spine", + "survey": "HMRC Capital Gains Tax statistics table 3 (size of gain by taxable income), 2020-21 to 2023-24", + "source": "https://assets.publishing.service.gov.uk/media/6878ac62760bf6cedaf5bd93/Table_3_2025_Size_of_gain_by_income.ods", + "grain": "person", + "artifacts": [ + { + "role": "cgt_published_fact_surface", + "kind": "administrative_table", + "format": "ods", + "survey": "HMRC Capital Gains Tax statistics table 3", + "publication": "https://www.gov.uk/government/statistics/capital-gains-tax-statistics", + "vintage": "2023-24", + "tax_year_start": 2023, + "locator": "https://assets.publishing.service.gov.uk/media/6878ac62760bf6cedaf5bd93/Table_3_2025_Size_of_gain_by_income.ods", + "sha256": "8e75c00bab949348a7238fea6d995f626c85e5d02813b46606dd7fea85e9d0c3", + "size_bytes": 11996, + "mime_type": "application/vnd.oasis.opendocument.spreadsheet", + "sheets": ["3_1_2023-24", "3_2_2022-23", "3_3_2021-22", "3_4_2020-21"], + "mapped_build_period": 2024, + "period_mapping": "latest_published_tax_year", + "runtime_sha256_required": true + }, + { + "role": "policy_parameters", + "kind": "versioned_parameter_tree", + "dependency": "policyengine-uk>=2.88 via microcosm-build[uk]", + "parameters": [ + "gov.hmrc.income_tax.allowances.personal_allowance.amount", + "gov.hmrc.income_tax.allowances.personal_allowance.maximum_ANI", + "gov.hmrc.income_tax.allowances.personal_allowance.reduction_rate", + "gov.hmrc.cgt.annual_exempt_amount" + ], + "instant_rule": "raw dated parameter files evaluated at 1 June of the build period's tax year", + "runtime_sha256_required": false, + "dependency_discipline": "deferred inside uk_cgt_policy_parameters; the base package never imports policyengine-uk at import time" + } + ], + "operations": [ + { + "kind": "verify_pinned_cgt_ods", + "artifact_role": "cgt_published_fact_surface", + "require_before_source_read": true, + "runtime_sha256_required": true, + "fail_on_mismatch": true + }, + { + "kind": "taxable_income_proxy", + "components": [ + "employment_income", + "self_employment_income", + "state_pension_reported", + "private_pension_income", + "property_income", + "savings_interest_income", + "dividend_income", + "miscellaneous_income" + ], + "components_semantics": "Persisted leaves of the model's total_income concept (ITA 2007 s.23); state_pension_reported stands in for social_security_income, whose other taxable benefits are not persisted; reliefs such as pension contributions and Gift Aid are not deducted.", + "allowance": "tapered Personal Allowance from the policy_parameters artifact", + "fail_on_missing_component": true + }, + { + "kind": "rank_preserving_allocation", + "within": "income band", + "ordering": "existing gains descending, person_id ascending on ties", + "band_order": "highest gain band first", + "suppressed_cell_allocation": "count implied by the cell's published gains at the band-total mean", + "column_reconciliation": "every income column rescales onto its published All-row taxpayer total", + "shortfall_policy": "proportional scale-down when the population holds less gainer mass than published taxpayers", + "minimum_allocation_people": 1, + "weights": "household_weight mapped to persons; no person splits across bands" + }, + { + "kind": "within_band_draws", + "bounded_band_family": "truncated exponential matched to the cell's published mean", + "open_band_family": "Pareto with alpha = mean / (mean - lower bound)", + "mean_repair_margin": 0.02, + "mean_repair_reason": "Published counts round to the nearest thousand and amounts to the nearest million; four cells of the 2023-24 table imply a mean outside their own band, and repaired means clamp just inside the violated boundary.", + "bottom_band_floor": "annual exempt amount plus one pound", + "seed_base": 552, + "seed_mixing": "seed combined with the build period; draws ordered by allocation rank", + "deterministic": true + }, + { + "kind": "sub_aea_remainder", + "policy": "gainers beyond the published taxpayer mass keep their existing amounts capped at the annual exempt amount", + "rationale": "Table 3 covers only individuals with a CGT liability; remaining gainers are treated as sub-AEA gainers rather than invented into the liability distribution or deleted." + }, + { + "kind": "record_mass_conservation_receipt", + "entity": "household", + "reason": "Amounts-only capital gains redraw on the source spine: household weights pass through unchanged and total household mass is conserved.", + "declared_factor": 1.0, + "gate_coupling": "The terminal family gate requires a valid mass-conserving MassChangeRecord carrying exactly this spine-specific reason." + }, + { + "kind": "classify_cgt_band_facts_with_reviewed_fence", + "calibration_permitted": false, + "fact_fence_id": "cgt_band_facts_policy_endogenous_proxy_conditioned", + "fenced_fact_count": 76, + "fenced_fact_composition": "60 joint cells, 10 gain-band row totals, 6 income-column totals", + "classification_rationale": "The taxpayer count is endogenous to policy, the income conditioning is an arithmetic proxy, and the published surface needs rounding and suppression reconciliation before any per-band fact is exact.", + "calibrated_facts_unchanged": "The two aggregate facts in UK_CGT_TARGET_SPECS remain the only calibrated CGT facts.", + "promotion_path": "A separately reviewed target profile may lift specific band facts after the reconciliation and proxy adequacy are adjudicated.", + "adjudication": "https://github.com/PolicyEngine/microcosm/issues/552" + } + ], + "outputs": ["capital_gains"], + "rewrites": ["capital_gains"], + "notes": "Spine manifest projection of the merged HMRC Table 3 amounts stage. It deliberately omits base_candidate and verify_certified_candidate, which belong only to the certified-H5 path." + }, + { + "stage": "salary_sacrifice", + "survey": "Family Resources Survey 2024-25 salary-sacrifice respondents and HMRC salary-sacrifice reform analysis", + "source": "HMRC, Salary sacrifice reform for pension contributions effective from 6 April 2029", + "grain": "person", + "artifacts": [ + { + "role": "salary_sacrifice_anchor", + "kind": "public_aggregate_reference", + "resource": "salary_sacrifice_anchor.json", + "format": "json", + "runtime_sha256_required": true + } + ], + "operations": [ + { + "kind": "fit_weighted_qrf", + "training_population": "support_channel == frs and not capital-gains clone and not CGT band donor and salary_sacrifice_asked == 1", + "target_population": "salary_sacrifice_asked != 1 frame-wide", + "predictors": ["age", "employment_income"], + "targets": ["pension_contributions_via_salary_sacrifice"], + "weights": "household_weight", + "weight_mapping": "household_to_person", + "seed": 42, + "n_estimators": 100, + "clamp_minimum": 0, + "preserve_asked_rows": true, + "cache": false + }, + { + "kind": "convert_donors_to_target_stock", + "resource": "salary_sacrifice_anchor.json", + "target": 5400000, + "donor_pool": "employee_pension_contributions > 0 and pension_contributions_via_salary_sacrifice == 0 and employment_income > 0", + "rate_cap": 0.5, + "move": "full employee_pension_contributions to pension_contributions_via_salary_sacrifice; source zeroed", + "seed": 2024, + "salt": "salary_sacrifice_conversion", + "receipt": "weighted_headcount", + "reason": "Salary-sacrifice support stage rewrites pension columns only; household rows and typed household weights pass through and total household mass is conserved." + } + ], + "outputs": [ + "pension_contributions_via_salary_sacrifice", + "employee_pension_contributions" + ], + "nonnegative_outputs": [ + "pension_contributions_via_salary_sacrifice", + "employee_pension_contributions" + ], + "rewrites": [ + "pension_contributions_via_salary_sacrifice", + "employee_pension_contributions" + ], + "notes": "The QRF trains only on the 2024-25 FRS asked subset. The second arm creates support toward the reviewed 5.4m staging target by moving contributors' full pension amounts." + }, + { + "stage": "student_loans", + "survey": "Family Resources Survey 2024-25 and Student Loans Company borrower forecasts for England", + "source": "Explore Education Statistics Table 6a, Higher education total", + "grain": "person", + "artifacts": [ + { + "role": "frs_release", + "kind": "public_aggregate_reference", + "resource": "frs_release.json", + "format": "json", + "runtime_sha256_required": true + }, + { + "role": "slc_liable_stocks", + "kind": "public_aggregate_reference", + "resource": "slc_liable_stocks.json", + "format": "json", + "runtime_sha256_required": true + } + ], + "operations": [ + { + "kind": "assign_student_loan_plan_cohorts", + "year_rule": "calibration_year", + "start_year_formula": "year - age + 18", + "reported_repayment_test": "student_loan_repayments > 0", + "reported_country_gate": false, + "plan_1_before": 2012, + "plan_5_from": 2023, + "enum_domain": ["NONE", "PLAN_1", "PLAN_2", "PLAN_5"], + "plan_4_imputation": false + }, + { + "kind": "top_up_to_stock", + "plan": "PLAN_5", + "priority": 1, + "resource": "slc_liable_stocks.json", + "stock_series": "plan_5.liable", + "year_rule": "calibration_year", + "age_min": 18, + "age_max": 25, + "cohort_start_min": 2023, + "eligible_region_exclusions": ["SCOTLAND", "WALES", "NORTHERN_IRELAND"], + "highest_education": "TERTIARY", + "seed": 42, + "salt": "student_loan_plan_5" + }, + { + "kind": "top_up_to_stock", + "plan": "PLAN_2", + "priority": 2, + "resource": "slc_liable_stocks.json", + "stock_series": "plan_2.liable", + "year_rule": "calibration_year", + "age_min": 21, + "age_max": 55, + "cohort_start_min": 2012, + "cohort_start_max_exclusive": 2023, + "eligible_region_exclusions": ["SCOTLAND", "WALES", "NORTHERN_IRELAND"], + "highest_education": "TERTIARY", + "seed": 42, + "salt": "student_loan_plan_2", + "reason": "Student-loan plan assignment writes an enum column only; household rows and typed household weights pass through and total household mass is conserved." + } + ], + "outputs": ["student_loan_plan"], + "rewrites": ["student_loan_plan"], + "notes": "Reported PAYE repayers are classified without a country gate. England tertiary cohorts are then topped up PLAN_5 first and PLAN_2 second to the pinned liable stocks at the FRS release calibration year; PLAN_4 is never imputed." + }, { "stage": "frs_hmrc_retained_leaves", "survey": "Family Resources Survey 2024-25", diff --git a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml index e4194750..3787b830 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml @@ -1916,6 +1916,311 @@ stages: - hmrc_spi_unemployment_benefit_income - hmrc_spi_incapacity_benefit_income notes: Runs the SPI-trained income QRFs on the raw-spine support channel, initializes FRS charity columns to zero, trains FRS-only stage 2 before redrawing base-channel dividends, and emits a sidecar-only 208-fact replay report for the spine path. +- stage: cgt_incidence_clone + survey: Family Resources Survey 2024-25, SPI synthetic support, and Advani-Summers capital-gains incidence + source: Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465 + grain: household + artifacts: + - role: capital_gains_incidence_and_quantiles + kind: public_aggregate_reference + resource: advani_summers_capital_gains_distribution.json + format: json + runtime_sha256_required: true + operations: + - kind: clone_records + entity: household + copies: 2 + flag_column: household_is_capital_gains_clone + original_flag: false + clone_flag: true + mass_split: 0.5 + weight_kind_out: importance + conservation: exact_total + id_remapping: id_multiplier_for_values + declared_factor: 1.0 + reason: Capital-gains incidence clone splits every household's mass equally across original and clone records; total household mass is conserved. + - kind: draw_capital_gains_prior_from_banded_quantiles + resource: advani_summers_capital_gains_distribution.json + income_proxy_components: + - employment_income + - self_employment_income + - state_pension_reported + - private_pension_income + - property_income + - savings_interest_income + - dividend_income + - miscellaneous_income + allowance_subtraction: false + carrier: oldest adult; person_id ascending breaks age ties + adult_minimum_age: 16 + quantile_points: + - 0.05 + - 0.1 + - 0.25 + - 0.5 + - 0.75 + - 0.9 + - 0.95 + spline_degree: 1 + extrapolation: ext=0 + keep_negative_draws: true + seed: 0 + salt: cgt_prior_amount + outputs: + - household_is_capital_gains_clone + - capital_gains + rewrites: + - capital_gains + notes: Spine-only equal-mass incidence clone. The A&S prior fixes the gainer set and order for the following HMRC Table 3 redraw; negative extrapolated draws remain loss-makers. +- stage: cgt_band_donors + survey: HMRC Capital Gains Tax statistics Table 2.1a and Advani-Summers capital-gains incidence + source: HMRC Capital Gains Tax statistics, July 2025, Table 2.1a + grain: household + artifacts: + - role: hmrc_cgt_size_bands + kind: public_aggregate_reference + resource: hmrc_cgt_size_bands.json + format: json + runtime_sha256_required: true + - role: capital_gains_incidence + kind: public_aggregate_reference + resource: advani_summers_capital_gains_distribution.json + format: json + runtime_sha256_required: true + operations: + - kind: stack_band_donor_households + size_band_resource: hmrc_cgt_size_bands.json + incidence_resource: advani_summers_capital_gains_distribution.json + minimum_band_lower: 12300 + donors_per_band: 30 + expected_band_count: 9 + expected_donor_count: 270 + candidate_order: household_id ascending + draw: weighted_without_replacement + propensity: Advani-Summers percent_with_gains at oldest-adult component-sum income + seed: 1 + flag_column: household_is_cgt_band_donor + carrier: oldest adult; person_id ascending breaks age ties + initial_weight: published band taxpayers / donors_per_band + never_zero_weight: true + weight_kind_out: importance + reason: Stack 30 positive-weight HMRC Table 2.1a support households per retained gain band; published donor mass is added explicitly. + outputs: + - household_is_cgt_band_donor + - capital_gains + rewrites: + - capital_gains + notes: Adds 270 positive-weight band donors. Rows below GBP 12,300 are excluded because they mix annual-exempt-amount regimes and the spline body already supplies that support. +- stage: hmrc_cgt_gains_spine + survey: HMRC Capital Gains Tax statistics table 3 (size of gain by taxable income), 2020-21 to 2023-24 + source: https://assets.publishing.service.gov.uk/media/6878ac62760bf6cedaf5bd93/Table_3_2025_Size_of_gain_by_income.ods + grain: person + artifacts: + - role: cgt_published_fact_surface + kind: administrative_table + format: ods + survey: HMRC Capital Gains Tax statistics table 3 + publication: https://www.gov.uk/government/statistics/capital-gains-tax-statistics + vintage: 2023-24 + tax_year_start: 2023 + locator: https://assets.publishing.service.gov.uk/media/6878ac62760bf6cedaf5bd93/Table_3_2025_Size_of_gain_by_income.ods + sha256: 8e75c00bab949348a7238fea6d995f626c85e5d02813b46606dd7fea85e9d0c3 + size_bytes: 11996 + mime_type: application/vnd.oasis.opendocument.spreadsheet + sheets: + - 3_1_2023-24 + - 3_2_2022-23 + - 3_3_2021-22 + - 3_4_2020-21 + mapped_build_period: 2024 + period_mapping: latest_published_tax_year + runtime_sha256_required: true + - role: policy_parameters + kind: versioned_parameter_tree + dependency: policyengine-uk>=2.88 via microcosm-build[uk] + parameters: + - gov.hmrc.income_tax.allowances.personal_allowance.amount + - gov.hmrc.income_tax.allowances.personal_allowance.maximum_ANI + - gov.hmrc.income_tax.allowances.personal_allowance.reduction_rate + - gov.hmrc.cgt.annual_exempt_amount + instant_rule: raw dated parameter files evaluated at 1 June of the build period's tax year + runtime_sha256_required: false + dependency_discipline: deferred inside uk_cgt_policy_parameters; the base package never imports policyengine-uk at import time + operations: + - kind: verify_pinned_cgt_ods + artifact_role: cgt_published_fact_surface + require_before_source_read: true + runtime_sha256_required: true + fail_on_mismatch: true + - kind: taxable_income_proxy + components: + - employment_income + - self_employment_income + - state_pension_reported + - private_pension_income + - property_income + - savings_interest_income + - dividend_income + - miscellaneous_income + components_semantics: Persisted leaves of the model's total_income concept (ITA 2007 s.23); state_pension_reported stands in for social_security_income, whose other taxable benefits are not persisted; reliefs such as pension contributions and Gift Aid are not deducted. + allowance: tapered Personal Allowance from the policy_parameters artifact + fail_on_missing_component: true + - kind: rank_preserving_allocation + within: income band + ordering: existing gains descending, person_id ascending on ties + band_order: highest gain band first + suppressed_cell_allocation: count implied by the cell's published gains at the band-total mean + column_reconciliation: every income column rescales onto its published All-row taxpayer total + shortfall_policy: proportional scale-down when the population holds less gainer mass than published taxpayers + minimum_allocation_people: 1 + weights: household_weight mapped to persons; no person splits across bands + - kind: within_band_draws + bounded_band_family: truncated exponential matched to the cell's published mean + open_band_family: Pareto with alpha = mean / (mean - lower bound) + mean_repair_margin: 0.02 + mean_repair_reason: Published counts round to the nearest thousand and amounts to the nearest million; four cells of the 2023-24 table imply a mean outside their own band, and repaired means clamp just inside the violated boundary. + bottom_band_floor: annual exempt amount plus one pound + seed_base: 552 + seed_mixing: seed combined with the build period; draws ordered by allocation rank + deterministic: true + - kind: sub_aea_remainder + policy: gainers beyond the published taxpayer mass keep their existing amounts capped at the annual exempt amount + rationale: Table 3 covers only individuals with a CGT liability; remaining gainers are treated as sub-AEA gainers rather than invented into the liability distribution or deleted. + - kind: record_mass_conservation_receipt + entity: household + reason: 'Amounts-only capital gains redraw on the source spine: household weights pass through unchanged and total household mass is conserved.' + declared_factor: 1.0 + gate_coupling: The terminal family gate requires a valid mass-conserving MassChangeRecord carrying exactly this spine-specific reason. + - kind: classify_cgt_band_facts_with_reviewed_fence + calibration_permitted: false + fact_fence_id: cgt_band_facts_policy_endogenous_proxy_conditioned + fenced_fact_count: 76 + fenced_fact_composition: 60 joint cells, 10 gain-band row totals, 6 income-column totals + classification_rationale: The taxpayer count is endogenous to policy, the income conditioning is an arithmetic proxy, and the published surface needs rounding and suppression reconciliation before any per-band fact is exact. + calibrated_facts_unchanged: The two aggregate facts in UK_CGT_TARGET_SPECS remain the only calibrated CGT facts. + promotion_path: A separately reviewed target profile may lift specific band facts after the reconciliation and proxy adequacy are adjudicated. + adjudication: https://github.com/PolicyEngine/microcosm/issues/552 + outputs: + - capital_gains + rewrites: + - capital_gains + notes: Spine manifest projection of the merged HMRC Table 3 amounts stage. It deliberately omits base_candidate and verify_certified_candidate, which belong only to the certified-H5 path. +- stage: salary_sacrifice + survey: Family Resources Survey 2024-25 salary-sacrifice respondents and HMRC salary-sacrifice reform analysis + source: HMRC, Salary sacrifice reform for pension contributions effective from 6 April 2029 + grain: person + artifacts: + - role: salary_sacrifice_anchor + kind: public_aggregate_reference + resource: salary_sacrifice_anchor.json + format: json + runtime_sha256_required: true + operations: + - kind: fit_weighted_qrf + training_population: support_channel == frs and not capital-gains clone and not CGT band donor and salary_sacrifice_asked == 1 + target_population: salary_sacrifice_asked != 1 frame-wide + predictors: + - age + - employment_income + targets: + - pension_contributions_via_salary_sacrifice + weights: household_weight + weight_mapping: household_to_person + seed: 42 + n_estimators: 100 + clamp_minimum: 0 + preserve_asked_rows: true + cache: false + - kind: convert_donors_to_target_stock + resource: salary_sacrifice_anchor.json + target: 5400000 + donor_pool: employee_pension_contributions > 0 and pension_contributions_via_salary_sacrifice == 0 and employment_income > 0 + rate_cap: 0.5 + move: full employee_pension_contributions to pension_contributions_via_salary_sacrifice; source zeroed + seed: 2024 + salt: salary_sacrifice_conversion + receipt: weighted_headcount + reason: Salary-sacrifice support stage rewrites pension columns only; household + rows and typed household weights pass through and total household mass is conserved. + outputs: + - pension_contributions_via_salary_sacrifice + - employee_pension_contributions + nonnegative_outputs: + - pension_contributions_via_salary_sacrifice + - employee_pension_contributions + rewrites: + - pension_contributions_via_salary_sacrifice + - employee_pension_contributions + notes: The QRF trains only on the 2024-25 FRS asked subset. The second arm creates support toward the reviewed 5.4m staging target by moving contributors' full pension amounts. +- stage: student_loans + survey: Family Resources Survey 2024-25 and Student Loans Company borrower forecasts for England + source: Explore Education Statistics Table 6a, Higher education total + grain: person + artifacts: + - role: frs_release + kind: public_aggregate_reference + resource: frs_release.json + format: json + runtime_sha256_required: true + - role: slc_liable_stocks + kind: public_aggregate_reference + resource: slc_liable_stocks.json + format: json + runtime_sha256_required: true + operations: + - kind: assign_student_loan_plan_cohorts + year_rule: calibration_year + start_year_formula: year - age + 18 + reported_repayment_test: student_loan_repayments > 0 + reported_country_gate: false + plan_1_before: 2012 + plan_5_from: 2023 + enum_domain: + - NONE + - PLAN_1 + - PLAN_2 + - PLAN_5 + plan_4_imputation: false + - kind: top_up_to_stock + plan: PLAN_5 + priority: 1 + resource: slc_liable_stocks.json + stock_series: plan_5.liable + year_rule: calibration_year + age_min: 18 + age_max: 25 + cohort_start_min: 2023 + eligible_region_exclusions: + - SCOTLAND + - WALES + - NORTHERN_IRELAND + highest_education: TERTIARY + seed: 42 + salt: student_loan_plan_5 + - kind: top_up_to_stock + plan: PLAN_2 + priority: 2 + resource: slc_liable_stocks.json + stock_series: plan_2.liable + year_rule: calibration_year + age_min: 21 + age_max: 55 + cohort_start_min: 2012 + cohort_start_max_exclusive: 2023 + eligible_region_exclusions: + - SCOTLAND + - WALES + - NORTHERN_IRELAND + highest_education: TERTIARY + seed: 42 + salt: student_loan_plan_2 + reason: Student-loan plan assignment writes an enum column only; household rows + and typed household weights pass through and total household mass is conserved. + outputs: + - student_loan_plan + rewrites: + - student_loan_plan + notes: Reported PAYE repayers are classified without a country gate. England tertiary cohorts are then topped up PLAN_5 first and PLAN_2 second to the pinned liable stocks at the FRS release calibration year; PLAN_4 is never imputed. - stage: frs_hmrc_retained_leaves survey: Family Resources Survey 2024-25 source: Department for Work and Pensions Family Resources Survey 2024-25 raw adult.tab and benefits.tab, caller-supplied local input diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/battery_bindings.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/battery_bindings.py index 28cb33b4..5b97ddf2 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/battery_bindings.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/battery_bindings.py @@ -272,23 +272,30 @@ def _evaluate_take_up_signal( return uk_take_up_signal_gate(context.frame, **dict(parameters)) -def _evaluate_brma_enum_domain( +def _evaluate_enum_domain( context: EvidenceContext, parameters: Mapping[str, Any] ) -> GateResult: columns = tuple(parameters.get("columns", ())) - if columns != ("brma",): - raise ValueError("uk_brma_enum_domain must declare columns ['brma'].") - domain = context.artifacts.get("brma_enum_domain") + if len(columns) != 1 or not isinstance(columns[0], str): + raise ValueError("UK enum_domain gates must declare exactly one column.") + column = columns[0] + domain = context.artifacts.get(f"{column}_enum_domain") if domain is None: engine = context.artifacts["rules_engine"] - variable = engine._variable("brma") + variable = engine._variable(column) domain = getattr(variable, "possible_values", None) if domain is None: - raise ValueError("brma enum domain could not be resolved from evidence.") - return enum_domain_gate( - {"brma": context.frame.table("household")["brma"]}, - {"brma": domain}, - ) + raise ValueError(f"{column} enum domain could not be resolved from evidence.") + matches = [ + context.frame.table(entity)[column] + for entity in context.frame.entities + if column in context.frame.table(entity).columns + ] + if len(matches) != 1: + raise ValueError( + f"{column} must occur on exactly one frame entity; found {len(matches)}." + ) + return enum_domain_gate({column: matches[0]}, {column: domain}) def _evaluate_support( @@ -830,7 +837,7 @@ def _ledger_compile_parity_registry( ), "enum_domain": UKGateBinding( name="enum_domain", - evaluator=_evaluate_brma_enum_domain, + evaluator=_evaluate_enum_domain, parameter_keys=frozenset({"columns"}), ), "support": UKGateBinding( diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_imputation.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_imputation.py index a26764d9..219a6342 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_imputation.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_imputation.py @@ -62,6 +62,7 @@ import numpy as np import pandas as pd +from microcosm.build.source_manifest import SourceStageSpec from microcosm.build.uk_runtime.hmrc_capital_gains import ( HMRC_CGT_GAIN_BAND_LOWER_BOUNDS, HMRC_CGT_INCOME_BAND_LOWER_BOUNDS, @@ -81,6 +82,7 @@ __all__ = [ "UK_CGT_IMPUTATION_SEED", "UK_CGT_MASS_CONSERVATION_REASON", + "UK_CGT_SPINE_MASS_CONSERVATION_REASON", "UK_CGT_IMPUTATION_STAGE_NAME", "UK_CGT_TAXABLE_INCOME_PROXY_COMPONENTS", "UKCGTImputationSummary", @@ -88,6 +90,7 @@ "impute_uk_capital_gains", "summarize_uk_cgt_imputation", "uk_capital_gains_imputation_stage", + "uk_cgt_spine_stage_transform", "uk_cgt_policy_parameters", "uk_cgt_taxable_income_proxy", ] @@ -107,6 +110,16 @@ #: periods draw differently while each build is reproducible. UK_CGT_IMPUTATION_SEED = 552 +#: The spine projection records the same conservation invariant under its +#: own reason so the terminal family validator can never satisfy the +#: certified and spine families with one shared record (adversarial-review +#: finding on the E8 PR: reason strings are the receipt identity). +UK_CGT_SPINE_MASS_CONSERVATION_REASON = ( + "Amounts-only capital gains redraw on the source spine: household " + "weights pass through unchanged and total household mass is conserved." +) + + #: Persisted components of the model's ``total_income`` concept (ITA 2007 #: s.23: taxable income after tax reliefs and before allowances). #: ``state_pension_reported`` stands in for ``social_security_income``; the @@ -435,6 +448,7 @@ def impute_uk_capital_gains( parameters: UKCGTPolicyParameters, *, seed: int = UK_CGT_IMPUTATION_SEED, + mass_change_reason: str = UK_CGT_MASS_CONSERVATION_REASON, ) -> Frame: """Redraw gainers' amounts from the published joint distribution.""" validate_uk_national_frame(frame) @@ -548,7 +562,7 @@ def impute_uk_capital_gains( old_total=household_mass, new_total=household_mass, declared_factor=1.0, - reason=UK_CGT_MASS_CONSERVATION_REASON, + reason=mass_change_reason, ) result_frame = uk_national_frame( person=new_person, @@ -617,6 +631,7 @@ def uk_capital_gains_imputation_stage( tax_year: str = HMRC_CGT_SOURCE_VINTAGE, parameters: UKCGTPolicyParameters | None = None, seed: int = UK_CGT_IMPUTATION_SEED, + mass_change_reason: str = UK_CGT_MASS_CONSERVATION_REASON, ) -> UKNationalStage: """Build the national stage that redraws capital gains amounts. @@ -631,6 +646,172 @@ def transform(frame: Frame) -> Frame: artifact_path, tax_year=tax_year ) resolved = parameters or uk_cgt_policy_parameters(uk_time_period(frame)) - return impute_uk_capital_gains(frame, distribution, resolved, seed=seed) + return impute_uk_capital_gains( + frame, + distribution, + resolved, + seed=seed, + mass_change_reason=mass_change_reason, + ) return UKNationalStage(name=UK_CGT_IMPUTATION_STAGE_NAME, transform=transform) + + +def uk_cgt_spine_stage_transform( + stage: SourceStageSpec, + ods_path: str | Path, +): + """Bind the spine manifest, then reuse the reviewed merged CGT runtime. + + The certified-H5 wrapper and its candidate verification remain untouched; + this source-plan seam deliberately delegates only the amounts transform. + """ + + _assert_cgt_spine_stage_parameters(stage) + return uk_capital_gains_imputation_stage( + ods_path, + mass_change_reason=UK_CGT_SPINE_MASS_CONSERVATION_REASON, + ).transform + + +def _assert_cgt_spine_stage_parameters(stage: SourceStageSpec) -> None: + """Arm 1 of the #730/#684 two-arm rule for the spine projection.""" + + expected_kinds = ( + "verify_pinned_cgt_ods", + "taxable_income_proxy", + "rank_preserving_allocation", + "within_band_draws", + "sub_aea_remainder", + "record_mass_conservation_receipt", + "classify_cgt_band_facts_with_reviewed_fence", + ) + kinds = tuple(operation.kind for operation in stage.operations) + if kinds != expected_kinds: + raise ValueError( + f"CGT spine operation order drifted: expected {expected_kinds}, got {kinds}." + ) + operations = { + operation.kind: dict(operation.parameters) for operation in stage.operations + } + # Closed-world reviewed mapping: every operation's FULL declared parameter + # payload must equal the reviewed constants below (adversarial-review + # finding on #740 — asserting a subset let lockstep manifest edits move + # behavioral declarations without a matching reviewed code change; whole- + # mapping equality also rejects extra keys). + expected_operations = { + "verify_pinned_cgt_ods": { + "artifact_role": "cgt_published_fact_surface", + "require_before_source_read": True, + "runtime_sha256_required": True, + "fail_on_mismatch": True, + }, + "taxable_income_proxy": { + "components": list(UK_CGT_TAXABLE_INCOME_PROXY_COMPONENTS), + "components_semantics": ( + "Persisted leaves of the model's total_income concept (ITA " + "2007 s.23); state_pension_reported stands in for " + "social_security_income, whose other taxable benefits are " + "not persisted; reliefs such as pension contributions and " + "Gift Aid are not deducted." + ), + "allowance": ( + "tapered Personal Allowance from the policy_parameters artifact" + ), + "fail_on_missing_component": True, + }, + "rank_preserving_allocation": { + "within": "income band", + "ordering": "existing gains descending, person_id ascending on ties", + "band_order": "highest gain band first", + "suppressed_cell_allocation": ( + "count implied by the cell's published gains at the band-total mean" + ), + "column_reconciliation": ( + "every income column rescales onto its published All-row taxpayer total" + ), + "shortfall_policy": ( + "proportional scale-down when the population holds less " + "gainer mass than published taxpayers" + ), + "minimum_allocation_people": int(_MINIMUM_ALLOCATION_PEOPLE), + "weights": ( + "household_weight mapped to persons; no person splits across bands" + ), + }, + "within_band_draws": { + "bounded_band_family": ( + "truncated exponential matched to the cell's published mean" + ), + "open_band_family": "Pareto with alpha = mean / (mean - lower bound)", + "mean_repair_margin": _MEAN_MARGIN, + "mean_repair_reason": ( + "Published counts round to the nearest thousand and amounts " + "to the nearest million; four cells of the 2023-24 table " + "imply a mean outside their own band, and repaired means " + "clamp just inside the violated boundary." + ), + "bottom_band_floor": "annual exempt amount plus one pound", + "seed_base": UK_CGT_IMPUTATION_SEED, + "seed_mixing": ( + "seed combined with the build period; draws ordered by allocation rank" + ), + "deterministic": True, + }, + "sub_aea_remainder": { + "policy": ( + "gainers beyond the published taxpayer mass keep their " + "existing amounts capped at the annual exempt amount" + ), + "rationale": ( + "Table 3 covers only individuals with a CGT liability; " + "remaining gainers are treated as sub-AEA gainers rather " + "than invented into the liability distribution or deleted." + ), + }, + "record_mass_conservation_receipt": { + "entity": "household", + "reason": UK_CGT_SPINE_MASS_CONSERVATION_REASON, + "declared_factor": 1.0, + "gate_coupling": ( + "The terminal family gate requires a valid mass-conserving " + "MassChangeRecord carrying exactly this spine-specific reason." + ), + }, + "classify_cgt_band_facts_with_reviewed_fence": { + "calibration_permitted": False, + "fact_fence_id": "cgt_band_facts_policy_endogenous_proxy_conditioned", + "fenced_fact_count": 76, + "fenced_fact_composition": ( + "60 joint cells, 10 gain-band row totals, 6 income-column totals" + ), + "classification_rationale": ( + "The taxpayer count is endogenous to policy, the income " + "conditioning is an arithmetic proxy, and the published " + "surface needs rounding and suppression reconciliation " + "before any per-band fact is exact." + ), + "calibrated_facts_unchanged": ( + "The two aggregate facts in UK_CGT_TARGET_SPECS remain the " + "only calibrated CGT facts." + ), + "promotion_path": ( + "A separately reviewed target profile may lift specific " + "band facts after the reconciliation and proxy adequacy " + "are adjudicated." + ), + "adjudication": "https://github.com/PolicyEngine/microcosm/issues/552", + }, + } + for kind, expected_parameters in expected_operations.items(): + actual = operations[kind] + if actual != expected_parameters: + drifted = sorted( + key + for key in {*actual, *expected_parameters} + if actual.get(key) != expected_parameters.get(key) + ) + raise ValueError( + f"CGT spine {kind} declaration drifted from the reviewed " + f"mapping on parameter(s) {drifted}." + ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_structure.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_structure.py new file mode 100644 index 00000000..091cf65f --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_structure.py @@ -0,0 +1,698 @@ +"""UK capital-gains incidence cloning and HMRC size-band support.""" + +from __future__ import annotations + +import json +from collections.abc import Mapping +from dataclasses import dataclass, field +from importlib.resources import files +from typing import Any + +import numpy as np +import pandas as pd +from scipy.interpolate import UnivariateSpline + +from microcosm.build.source_manifest import SourceOperationSpec, SourceStageSpec +from microcosm.build.stochastic_assignment import stable_identity_uniforms +from microcosm.build.uk_runtime.cgt_imputation import ( + UK_CGT_TAXABLE_INCOME_PROXY_COMPONENTS, +) +from microcosm.build.uk_runtime.national_frame import ( + uk_national_frame, + uk_time_period, + validate_uk_national_frame, +) +from microcosm.build.uk_runtime.rowwise_geography import ( + clone_entity_frame, + id_multiplier_for_values, +) +from microcosm.build.uk_runtime.spi_support import ( + _importance_weights_with_exact_total, +) +from microcosm.frame import Frame, MassChangeRecord, WeightKind + +CGT_CLONE_MASS_SPLIT = 0.5 +CGT_PRIOR_SEED = 0 +CGT_PRIOR_SALT = "cgt_prior_amount" +CGT_QUANTILE_POINTS = (0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95) +CGT_PRIOR_PERCENTILE_COLUMNS = ("p05", "p10", "p25", "p50", "p75", "p90", "p95") +CGT_ADULT_MINIMUM_AGE = 16 +DONORS_PER_BAND = 30 +DONOR_BAND_COUNT = 9 +DONOR_TOTAL = 270 +MIN_DONOR_BAND_LOWER = 12_300 +DONOR_SEED = 1 +DONOR_NEVER_ZERO_WEIGHT = True +HOUSEHOLD_IS_CGT_CLONE = "household_is_capital_gains_clone" +HOUSEHOLD_IS_CGT_BAND_DONOR = "household_is_cgt_band_donor" +CGT_CLONE_MASS_CHANGE_REASON = ( + "Capital-gains incidence clone splits every household's mass equally across " + "original and clone records; total household mass is conserved." +) +CGT_DONOR_MASS_CHANGE_REASON = ( + "Stack 30 positive-weight HMRC Table 2.1a support households per retained " + "gain band; published donor mass is added explicitly." +) + + +def load_advani_summers_distribution() -> Mapping[str, Any]: + """Load the committed Advani-Summers incidence and quantile surface.""" + + return json.loads( + files("microcosm.build.uk") + .joinpath("advani_summers_capital_gains_distribution.json") + .read_text(encoding="utf-8") + ) + + +def load_hmrc_cgt_size_bands() -> Mapping[str, Any]: + """Load the committed HMRC Table 2.1a size-band surface.""" + + return json.loads( + files("microcosm.build.uk") + .joinpath("hmrc_cgt_size_bands.json") + .read_text(encoding="utf-8") + ) + + +@dataclass(frozen=True) +class UKCGTIncidenceCloneResult: + """Cloned frame and the executed-effect receipt for stage 19.""" + + frame: Frame + original_mass: float + clone_mass: float + carrier_count: int + negative_prior_count: int + + def evidence(self) -> dict[str, object]: + return { + "stage": "cgt_incidence_clone", + "mass_by_clone_flag": { + "false": self.original_mass, + "true": self.clone_mass, + }, + "carrier_count": self.carrier_count, + "negative_prior_count": self.negative_prior_count, + } + + +@dataclass(frozen=True) +class UKCGTBandDonorResult: + """Band-donor frame and the executed-effect receipt for stage 20.""" + + frame: Frame + band_rows: tuple[Mapping[str, object], ...] + frs_donors: int + spi_donors: int + + def evidence(self) -> dict[str, object]: + return { + "stage": "cgt_band_donors", + "bands": [dict(row) for row in self.band_rows], + "support_channel_split": { + "frs": self.frs_donors, + "spi": self.spi_donors, + }, + } + + +@dataclass(frozen=True) +class UKCGTIncidenceCloneStageTransform: + """Whole-stage transform for the incidence clone and prior draw.""" + + stage: SourceStageSpec + distribution: Mapping[str, Any] | None = None + last_result: UKCGTIncidenceCloneResult | None = field(default=None, init=False) + + def __call__(self, frame: Frame) -> Frame: + resource = self.distribution or load_advani_summers_distribution() + _assert_cgt_incidence_stage_parameters(self.stage) + result = clone_cgt_incidence(frame, distribution=resource) + object.__setattr__(self, "last_result", result) + return result.frame + + @staticmethod + def output_columns() -> tuple[str, ...]: + return (HOUSEHOLD_IS_CGT_CLONE, "capital_gains") + + def checkpoint_metadata(self) -> dict[str, object]: + if self.last_result is None: + raise RuntimeError("checkpoint metadata requires a completed stage run.") + return {"evidence": self.last_result.evidence()} + + +@dataclass(frozen=True) +class UKCGTBandDonorStageTransform: + """Whole-stage transform for positive-weight HMRC size-band donors.""" + + stage: SourceStageSpec + size_bands: Mapping[str, Any] | None = None + distribution: Mapping[str, Any] | None = None + last_result: UKCGTBandDonorResult | None = field(default=None, init=False) + + def __call__(self, frame: Frame) -> Frame: + bands = self.size_bands or load_hmrc_cgt_size_bands() + distribution = self.distribution or load_advani_summers_distribution() + _assert_cgt_donor_stage_parameters(self.stage, size_bands=bands) + result = stack_cgt_band_donors( + frame, + size_bands=bands, + distribution=distribution, + ) + object.__setattr__(self, "last_result", result) + return result.frame + + @staticmethod + def output_columns() -> tuple[str, ...]: + return (HOUSEHOLD_IS_CGT_BAND_DONOR, "capital_gains") + + def checkpoint_metadata(self) -> dict[str, object]: + if self.last_result is None: + raise RuntimeError("checkpoint metadata requires a completed stage run.") + return {"evidence": self.last_result.evidence()} + + +def clone_cgt_incidence( + frame: Frame, + *, + distribution: Mapping[str, Any], +) -> UKCGTIncidenceCloneResult: + """Clone every household at equal mass and assign A&S priors to clones.""" + + validate_uk_national_frame(frame) + person = frame.table("person").copy() + benunit = frame.table("benunit").copy() + household = frame.table("household").copy() + multiplier = id_multiplier_for_values( + person["person_id"], + person["person_household_id"], + person["person_benunit_id"], + benunit["benunit_id"], + household["household_id"], + ) + cloned_person = clone_entity_frame( + person, + id_columns=("person_id", "person_household_id", "person_benunit_id"), + n_clones=2, + id_multiplier=multiplier, + clone_index_column=None, + ).reset_index(drop=True) + cloned_benunit = clone_entity_frame( + benunit, + id_columns=("benunit_id",), + n_clones=2, + id_multiplier=multiplier, + clone_index_column=None, + ).reset_index(drop=True) + cloned_household = clone_entity_frame( + household, + id_columns=("household_id",), + n_clones=2, + id_multiplier=multiplier, + clone_index_column=None, + ).reset_index(drop=True) + n_households = len(household) + clone_flags = np.r_[ + np.zeros(n_households, dtype=bool), + np.ones(n_households, dtype=bool), + ] + cloned_household[HOUSEHOLD_IS_CGT_CLONE] = clone_flags + split = np.tile( + frame.weights_for("household").values * CGT_CLONE_MASS_SPLIT, + 2, + ) + exact_weights = _importance_weights_with_exact_total( + split, + frame.weights_for("household").total, + ) + cloned_person["capital_gains"] = 0.0 + carrier_indices = _oldest_adult_indices( + cloned_person, + household_ids=set(cloned_household.loc[clone_flags, "household_id"].to_numpy()), + ) + carrier_income = _component_sum_income(cloned_person.loc[carrier_indices]) + carrier_draws = stable_identity_uniforms( + cloned_person.loc[carrier_indices, "person_id"].to_numpy(), + seed=CGT_PRIOR_SEED, + salt=CGT_PRIOR_SALT, + ) + priors = _draw_banded_priors( + carrier_income, + carrier_draws, + distribution=distribution, + ) + cloned_person.loc[carrier_indices, "capital_gains"] = priors + receipt = MassChangeRecord( + entity="household", + old_total=frame.weights_for("household").total, + new_total=exact_weights.total, + declared_factor=1.0, + reason=CGT_CLONE_MASS_CHANGE_REASON, + ) + result = uk_national_frame( + person=cloned_person, + benunit=cloned_benunit, + household=cloned_household, + time_period=uk_time_period(frame), + weight_kind=WeightKind.IMPORTANCE, + household_weights=exact_weights.values, + mass_log=(*frame.mass_log, receipt), + ) + validate_uk_national_frame(result) + original_mass = float(exact_weights.values[~clone_flags].sum()) + clone_mass = float(exact_weights.values[clone_flags].sum()) + return UKCGTIncidenceCloneResult( + frame=result, + original_mass=original_mass, + clone_mass=clone_mass, + carrier_count=len(carrier_indices), + negative_prior_count=int((priors < 0.0).sum()), + ) + + +def stack_cgt_band_donors( + frame: Frame, + *, + size_bands: Mapping[str, Any], + distribution: Mapping[str, Any], +) -> UKCGTBandDonorResult: + """Add 30 households per retained HMRC size band at band-exact weights.""" + + validate_uk_national_frame(frame) + person = frame.table("person").copy() + benunit = frame.table("benunit").copy() + household = frame.table("household").copy() + household[HOUSEHOLD_IS_CGT_BAND_DONOR] = False + bands = _retained_size_bands(size_bands) + carriers = _oldest_adult_indices(person, household_ids=set(household.household_id)) + candidates = person.loc[carriers].copy() + candidates["_income"] = _component_sum_income(candidates) + candidates["_propensity"] = _incidence_propensity( + candidates["_income"].to_numpy(dtype=float), distribution=distribution + ) + candidates = candidates.sort_values("person_household_id", kind="stable") + if len(candidates) < DONOR_TOTAL: + raise ValueError( + f"CGT donor stage requires at least {DONOR_TOTAL} candidate households; " + f"found {len(candidates)}." + ) + propensities = candidates["_propensity"].to_numpy(dtype=float) + if not np.isfinite(propensities).all() or (propensities < 0).any(): + raise ValueError("CGT donor propensities must be finite and non-negative.") + if propensities.sum() <= 0: + raise ValueError("CGT donor propensities have no positive mass.") + rng = np.random.default_rng(DONOR_SEED) + selected = rng.choice( + candidates["person_household_id"].to_numpy(), + size=DONOR_TOTAL, + replace=False, + p=propensities / propensities.sum(), + ) + selected_set = set(selected.tolist()) + donor_person = person.loc[person.person_household_id.isin(selected_set)].copy() + donor_benunit_ids = set(donor_person.person_benunit_id) + donor_benunit = benunit.loc[benunit.benunit_id.isin(donor_benunit_ids)].copy() + donor_household = household.loc[household.household_id.isin(selected_set)].copy() + multiplier = id_multiplier_for_values( + person["person_id"], + person["person_household_id"], + person["person_benunit_id"], + benunit["benunit_id"], + household["household_id"], + ) + for column in ("person_id", "person_household_id", "person_benunit_id"): + donor_person[column] = donor_person[column].astype("int64") + multiplier + donor_benunit["benunit_id"] = ( + donor_benunit["benunit_id"].astype("int64") + multiplier + ) + donor_household["household_id"] = ( + donor_household["household_id"].astype("int64") + multiplier + ) + position = {household_id: index for index, household_id in enumerate(selected)} + donor_household["_band_position"] = ( + donor_household["household_id"].sub(multiplier).map(position) + ) + if donor_household["_band_position"].isna().any(): + raise ValueError("CGT donor selection failed to map every donor household.") + donor_household = donor_household.sort_values("_band_position", kind="stable") + band_index = ( + donor_household["_band_position"].to_numpy(dtype=int) // DONORS_PER_BAND + ) + taxpayers = np.asarray([row["taxpayers"] for row in bands], dtype=float) + means = np.asarray([row["mean_gain"] for row in bands], dtype=float) + donor_weights = taxpayers[band_index] / DONORS_PER_BAND + if DONOR_NEVER_ZERO_WEIGHT and not (donor_weights > 0.0).all(): + raise ValueError("CGT band donors must all carry positive initial weight.") + donor_household[HOUSEHOLD_IS_CGT_BAND_DONOR] = True + gain_by_household = dict( + zip(donor_household.household_id, means[band_index], strict=True) + ) + evidence_band_index = band_index.copy() + evidence_donor_weights = donor_weights.copy() + donor_household["_donor_weight"] = donor_weights + donor_household = donor_household.drop(columns=["_band_position"]).sort_values( + "household_id", kind="stable" + ) + donor_weights = donor_household.pop("_donor_weight").to_numpy(dtype=float) + donor_person["capital_gains"] = 0.0 + donor_carriers = _oldest_adult_indices( + donor_person, + household_ids=set(donor_household.household_id), + ) + donor_person.loc[donor_carriers, "capital_gains"] = donor_person.loc[ + donor_carriers, "person_household_id" + ].map(gain_by_household) + final_person = pd.concat([person, donor_person], ignore_index=True) + final_benunit = pd.concat([benunit, donor_benunit], ignore_index=True) + final_household = pd.concat([household, donor_household], ignore_index=True) + final_weights = np.r_[frame.weights_for("household").values, donor_weights] + old_total = frame.weights_for("household").total + new_total = float(final_weights.sum()) + receipt = MassChangeRecord( + entity="household", + old_total=old_total, + new_total=new_total, + declared_factor=None, + reason=CGT_DONOR_MASS_CHANGE_REASON, + ) + result = uk_national_frame( + person=final_person, + benunit=final_benunit, + household=final_household, + time_period=uk_time_period(frame), + weight_kind=WeightKind.IMPORTANCE, + household_weights=final_weights, + mass_log=(*frame.mass_log, receipt), + ) + validate_uk_national_frame(result) + channels = household.set_index("household_id").get("household_support_channel") + original_selected = donor_household.household_id.sub(multiplier) + selected_channels = ( + original_selected.map(channels).fillna("unknown") + if channels is not None + else pd.Series("unknown", index=original_selected.index) + ) + band_rows: list[Mapping[str, object]] = [] + for index, band in enumerate(bands): + mask = evidence_band_index == index + band_rows.append( + { + "lower_limit": band["lower_limit"], + "donor_count": int(mask.sum()), + "donor_weight": float(evidence_donor_weights[mask][0]), + "weighted_taxpayers": float(evidence_donor_weights[mask].sum()), + "mean_gain": band["mean_gain"], + } + ) + return UKCGTBandDonorResult( + frame=result, + band_rows=tuple(band_rows), + frs_donors=int(selected_channels.eq("frs").sum()), + spi_donors=int(selected_channels.eq("spi").sum()), + ) + + +def _oldest_adult_indices( + person: pd.DataFrame, + *, + household_ids: set[object], +) -> np.ndarray: + required = {"person_id", "person_household_id", "age"} + missing = sorted(required - set(person.columns)) + if missing: + raise ValueError(f"CGT carrier selection is missing person columns: {missing}.") + candidates = person.loc[ + person.person_household_id.isin(household_ids) + & (pd.to_numeric(person.age, errors="coerce") >= CGT_ADULT_MINIMUM_AGE) + ].copy() + missing_households = household_ids - set(candidates.person_household_id) + if missing_households: + raise ValueError( + "Every CGT household requires an adult carrier; missing household " + f"id(s): {sorted(missing_households)[:5]}." + ) + candidates["_row"] = candidates.index + candidates = candidates.sort_values( + ["person_household_id", "age", "person_id"], + ascending=[True, False, True], + kind="stable", + ) + return ( + candidates.groupby("person_household_id", sort=False)["_row"].first().to_numpy() + ) + + +def _component_sum_income(person: pd.DataFrame) -> np.ndarray: + missing = sorted(set(UK_CGT_TAXABLE_INCOME_PROXY_COMPONENTS) - set(person.columns)) + if missing: + raise ValueError(f"CGT income proxy components missing: {missing}.") + numeric = person.loc[:, UK_CGT_TAXABLE_INCOME_PROXY_COMPONENTS].apply( + pd.to_numeric, errors="coerce" + ) + if not np.isfinite(numeric.to_numpy(dtype=float)).all(): + raise ValueError("CGT component-sum income contains non-finite values.") + return numeric.sum(axis=1).to_numpy(dtype=float) + + +def _distribution_rows(resource: Mapping[str, Any]) -> list[Mapping[str, Any]]: + rows = resource.get("rows") + if not isinstance(rows, list) or not rows: + raise ValueError("Advani-Summers resource must contain a non-empty rows list.") + minimums = [float(row["minimum_total_income"]) for row in rows] + if minimums != sorted(minimums) or minimums[0] != 0.0: + raise ValueError("Advani-Summers income bands must be sorted and start at 0.") + # Fail closed on non-monotone quantile rows: the prior draw interpolates + # and extrapolates these values as a quantile function, so a malformed + # row (the class the corrected percentile-69 dropped digit belonged to) + # would silently fabricate loss-makers instead of failing the build. + for row in rows: + knots = [float(row[column]) for column in CGT_PRIOR_PERCENTILE_COLUMNS] + if any(late < early for early, late in zip(knots, knots[1:], strict=False)): + raise ValueError( + "Advani-Summers quantile columns must be non-decreasing; " + f"row with minimum_total_income {row['minimum_total_income']!r} " + "is not a valid quantile function." + ) + return rows + + +def _draw_banded_priors( + income: np.ndarray, + draws: np.ndarray, + *, + distribution: Mapping[str, Any], +) -> np.ndarray: + rows = _distribution_rows(distribution) + minimums = np.asarray([row["minimum_total_income"] for row in rows], dtype=float) + indexes = np.clip( + np.searchsorted(minimums, income, side="right") - 1, 0, len(rows) - 1 + ) + values = np.zeros(len(income), dtype=float) + for index, row in enumerate(rows): + mask = indexes == index + if not mask.any(): + continue + knots = np.asarray( + [row[column] for column in CGT_PRIOR_PERCENTILE_COLUMNS], dtype=float + ) + spline = UnivariateSpline(CGT_QUANTILE_POINTS, knots, k=1, s=0, ext=0) + values[mask] = spline(draws[mask]) + return values + + +def _incidence_propensity( + income: np.ndarray, + *, + distribution: Mapping[str, Any], +) -> np.ndarray: + rows = _distribution_rows(distribution) + minimums = np.asarray([row["minimum_total_income"] for row in rows], dtype=float) + indexes = np.clip( + np.searchsorted(minimums, income, side="right") - 1, 0, len(rows) - 1 + ) + rates = np.asarray([row["percent_with_gains"] for row in rows], dtype=float) + return rates[indexes] + + +def _retained_size_bands(resource: Mapping[str, Any]) -> list[dict[str, float]]: + rows = resource.get("rows") + if not isinstance(rows, list): + raise ValueError("HMRC CGT size-band resource must contain a rows list.") + retained: list[dict[str, float]] = [] + for row in rows: + lower = float(row["lower_limit"]) + if lower < MIN_DONOR_BAND_LOWER: + continue + taxpayers = float(row["taxpayers_thousands"]) * 1_000.0 + gains = float(row["gains_gbp_millions"]) * 1_000_000.0 + if taxpayers <= 0.0: + raise ValueError( + "Retained CGT size bands may not produce a zero initial weight." + ) + retained.append( + { + "lower_limit": lower, + "taxpayers": taxpayers, + "gains": gains, + "mean_gain": gains / taxpayers, + } + ) + return retained + + +def _operation(stage: SourceStageSpec, kind: str) -> SourceOperationSpec: + matches = [operation for operation in stage.operations if operation.kind == kind] + if len(matches) != 1: + raise ValueError( + f"Stage {stage.stage!r} must declare exactly one {kind!r} operation." + ) + return matches[0] + + +def _assert_parameters( + operation: SourceOperationSpec, + expected: Mapping[str, object], +) -> None: + for name, value in expected.items(): + actual = operation.parameters.get(name) + if actual != value: + raise ValueError( + f"{operation.kind} manifest parameter {name!r} drifted: " + f"expected {value!r}, got {actual!r}." + ) + + +def _assert_closed_world_operations( + stage: SourceStageSpec, + expected_operations: tuple[tuple[str, dict[str, object]], ...], +) -> None: + """Exact operation order and full-mapping equality per operation. + + Whole-payload equality rejects value drift, missing keys, and extra keys + alike (adversarial-review finding on the E8 PR: asserting a named subset + let lockstep manifest edits move undeclared-but-load-bearing semantics). + The expected sequence is ordered so repeated kinds are supported and an + extra, missing, or reordered operation fails by position. + """ + + kinds = tuple(operation.kind for operation in stage.operations) + expected_kinds = tuple(kind for kind, _ in expected_operations) + if kinds != expected_kinds: + raise ValueError( + f"Stage {stage.stage!r} operation order drifted: expected " + f"{expected_kinds}, got {kinds}." + ) + for operation, (kind, expected) in zip( + stage.operations, expected_operations, strict=True + ): + actual = dict(operation.parameters) + if actual != expected: + drifted = sorted( + key + for key in {*actual, *expected} + if actual.get(key) != expected.get(key) + ) + raise ValueError( + f"Stage {stage.stage!r} {kind} declaration drifted " + f"from the reviewed mapping on parameter(s) {drifted}." + ) + + +def _assert_cgt_incidence_stage_parameters(stage: SourceStageSpec) -> None: + """Bind every stage-19 manifest parameter to reviewed code constants. + + This is arm 1 of the #730/#684 two-arm rule documented in ``spi_spine``; + :class:`UKCGTIncidenceCloneResult` supplies the executed-effect receipt. + """ + + _assert_closed_world_operations( + stage, + ( + ( + "clone_records", + { + "entity": "household", + "copies": 2, + "flag_column": HOUSEHOLD_IS_CGT_CLONE, + "original_flag": False, + "clone_flag": True, + "mass_split": CGT_CLONE_MASS_SPLIT, + "weight_kind_out": WeightKind.IMPORTANCE.value, + "conservation": "exact_total", + "id_remapping": "id_multiplier_for_values", + "declared_factor": 1.0, + "reason": CGT_CLONE_MASS_CHANGE_REASON, + }, + ), + ( + "draw_capital_gains_prior_from_banded_quantiles", + { + "resource": "advani_summers_capital_gains_distribution.json", + "income_proxy_components": list( + UK_CGT_TAXABLE_INCOME_PROXY_COMPONENTS + ), + "allowance_subtraction": False, + "carrier": "oldest adult; person_id ascending breaks age ties", + "adult_minimum_age": CGT_ADULT_MINIMUM_AGE, + "quantile_points": list(CGT_QUANTILE_POINTS), + "spline_degree": 1, + "extrapolation": "ext=0", + "keep_negative_draws": True, + "seed": CGT_PRIOR_SEED, + "salt": CGT_PRIOR_SALT, + }, + ), + ), + ) + + +def _assert_cgt_donor_stage_parameters( + stage: SourceStageSpec, + *, + size_bands: Mapping[str, Any], +) -> None: + """Bind stage-20 parameters and recompute the band/weight invariants.""" + + _assert_closed_world_operations( + stage, + ( + ( + "stack_band_donor_households", + { + "size_band_resource": "hmrc_cgt_size_bands.json", + "incidence_resource": ( + "advani_summers_capital_gains_distribution.json" + ), + "minimum_band_lower": MIN_DONOR_BAND_LOWER, + "donors_per_band": DONORS_PER_BAND, + "expected_band_count": DONOR_BAND_COUNT, + "expected_donor_count": DONOR_TOTAL, + "candidate_order": "household_id ascending", + "draw": "weighted_without_replacement", + "propensity": ( + "Advani-Summers percent_with_gains at oldest-adult " + "component-sum income" + ), + "seed": DONOR_SEED, + "flag_column": HOUSEHOLD_IS_CGT_BAND_DONOR, + "carrier": "oldest adult; person_id ascending breaks age ties", + "initial_weight": "published band taxpayers / donors_per_band", + "never_zero_weight": DONOR_NEVER_ZERO_WEIGHT, + "weight_kind_out": WeightKind.IMPORTANCE.value, + "reason": CGT_DONOR_MASS_CHANGE_REASON, + }, + ), + ), + ) + bands = _retained_size_bands(size_bands) + if len(bands) != DONOR_BAND_COUNT: + raise ValueError( + f"HMRC retained donor-band count drifted: expected {DONOR_BAND_COUNT}, " + f"got {len(bands)}." + ) + if DONORS_PER_BAND * len(bands) != DONOR_TOTAL: + raise ValueError("CGT donor count no longer equals 30 times retained bands.") + weights = np.asarray([band["taxpayers"] / DONORS_PER_BAND for band in bands]) + if DONOR_NEVER_ZERO_WEIGHT and not (weights > 0.0).all(): + raise ValueError("HMRC retained donor bands imply a zero initial weight.") diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_build.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_build.py index d8a16a89..03744f79 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_build.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_build.py @@ -546,6 +546,9 @@ def build_uk_national_dataset( ) if brma_domain is not None: artifacts["brma_enum_domain"] = brma_domain + student_loan_plan_domain = _engine_enum_domain(engine, "student_loan_plan") + if student_loan_plan_domain is not None: + artifacts["student_loan_plan_enum_domain"] = student_loan_plan_domain fit_weight_records = _stage_fit_weight_records(materialized_stages) if fit_weight_records is not None: artifacts["fit_weight_records"] = fit_weight_records @@ -830,11 +833,15 @@ def _stage_calibration_evidence( def _brma_enum_domain(engine: object) -> tuple[str, ...] | None: + return _engine_enum_domain(engine, "brma") + + +def _engine_enum_domain(engine: object, variable_name: str) -> tuple[str, ...] | None: variable_getter = getattr(engine, "_variable", None) if not callable(variable_getter): return None try: - variable = variable_getter("brma") + variable = variable_getter(variable_name) except Exception: return None possible_values = getattr(variable, "possible_values", None) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/release_input_coverage.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/release_input_coverage.py index c9fc675d..cbbcd41f 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/release_input_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/release_input_coverage.py @@ -212,7 +212,7 @@ def _source_stage_base_candidate_tier( source_manifest: str, *, stage_name: str, -) -> str: +) -> str | None: payload = _resource_payload(source_manifest) stages = payload.get("stages") if not isinstance(stages, list): @@ -228,6 +228,8 @@ def _source_stage_base_candidate_tier( ) base_candidate = matching[0].get("base_candidate") if not isinstance(base_candidate, Mapping): + if source_manifest == "source_stages.json": + return None raise ValueError( f"{source_manifest}: stage {stage_name!r} needs base_candidate." ) @@ -308,6 +310,17 @@ def _parse_family_coverage( f"{resource}: family {name!r} needs a reviewed " "required_mass_change_reason." ) + mass_change_semantics = str( + raw_family.get("mass_change_semantics", "mass_conserving") + ).strip() + if mass_change_semantics not in { + "mass_conserving", + "mass_increasing_support", + }: + raise ValueError( + f"{resource}: family {name!r} has invalid " + f"mass_change_semantics {mass_change_semantics!r}." + ) raw_requirements = raw_family.get("effective_mass_requirements", {}) if not isinstance(raw_requirements, Mapping): @@ -371,6 +384,7 @@ def _parse_family_coverage( "base_candidate_tier": base_candidate_tier, "output_weight_kind": output_weight_kind, "required_mass_change_reason": required_mass_change_reason, + "mass_change_semantics": mass_change_semantics, "effective_mass_requirements": requirements, } return families @@ -931,20 +945,27 @@ def _family_build_state_diagnostics( required_reason = str(family.get("required_mass_change_reason", "")).strip() if required_reason: + semantics = str(family.get("mass_change_semantics", "mass_conserving")) records = tuple(getattr(frame, "mass_log", ())) matches = [ record for record in records if _mass_record_field(record, "reason") == required_reason ] - valid_matches = [record for record in matches if _valid_mass_record(record)] + valid_matches = [ + record + for record in matches + if _valid_mass_record(record, semantics=semantics) + ] details["required_mass_change_reason"] = required_reason + details["mass_change_semantics"] = semantics details["matching_mass_change_records"] = len(matches) details["valid_mass_change_records"] = len(valid_matches) if not valid_matches: failures.append( f"{family_name}: final dataset lacks the reviewed, " - "mass-conserving household MassChangeRecord carrying its " + f"{semantics.replace('_', '-')} household MassChangeRecord " + "carrying its " f"declared reason: {required_reason!r}." ) @@ -958,24 +979,30 @@ def _mass_record_field(record: object, name: str) -> object: return getattr(record, name, None) -def _valid_mass_record(record: object) -> bool: +def _valid_mass_record(record: object, *, semantics: str) -> bool: old_total = _mass_record_field(record, "old_total") new_total = _mass_record_field(record, "new_total") declared_factor = _mass_record_field(record, "declared_factor") try: old = float(old_total) new = float(new_total) - factor = float(declared_factor) except (TypeError, ValueError): return False - return bool( + common = bool( _mass_record_field(record, "entity") == "household" and np.isfinite(old) and old > 0.0 and np.isfinite(new) - and np.isclose(old, new, rtol=1e-9, atol=0.0) - and factor == 1.0 ) + if not common: + return False + if semantics == "mass_increasing_support": + return bool(new > old and declared_factor is None) + try: + factor = float(declared_factor) + except (TypeError, ValueError): + return False + return bool(np.isclose(old, new, rtol=1e-9, atol=0.0) and factor == 1.0) def uk_release_input_coverage_gate( diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/salary_sacrifice.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/salary_sacrifice.py new file mode 100644 index 00000000..de9830cf --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/salary_sacrifice.py @@ -0,0 +1,358 @@ +"""UK salary-sacrifice QRF and headcount support conversion.""" + +from __future__ import annotations + +import json +from collections.abc import Mapping +from dataclasses import dataclass, field +from importlib import import_module +from importlib.resources import files +from typing import Any + +import numpy as np +import pandas as pd + +from microcosm.build.source_manifest import SourceStageSpec +from microcosm.build.stochastic_assignment import stable_identity_uniforms +from microcosm.build.uk_runtime.cgt_structure import ( + HOUSEHOLD_IS_CGT_BAND_DONOR, + HOUSEHOLD_IS_CGT_CLONE, + _assert_closed_world_operations, +) +from microcosm.build.uk_runtime.national_frame import ( + uk_household_weight_kind, + uk_national_frame, + uk_time_period, + validate_uk_national_frame, +) +from microcosm.build.uk_runtime.spi_support import support_channel_column +from microcosm.frame import Frame, MassChangeRecord + +QRF: Any | None = None + +SALSAC_PREDICTORS = ("age", "employment_income") +SALSAC_OUTPUT = "pension_contributions_via_salary_sacrifice" +SALSAC_STAGE_TARGET = 5_400_000.0 +SALSAC_HMRC_ANCHOR = 7_700_000.0 +SALSAC_ABOVE_2000_ANCHOR = 3_300_000.0 +SALSAC_BELOW_2000_ANCHOR = 4_300_000.0 +SALSAC_STAGING_RATIO = SALSAC_STAGE_TARGET / SALSAC_HMRC_ANCHOR +SALSAC_RATE_CAP = 0.5 +SALSAC_QRF_SEED = 42 +SALSAC_QRF_ESTIMATORS = 100 +SALSAC_CONVERSION_SEED = 2024 +SALSAC_CONVERSION_SALT = "salary_sacrifice_conversion" +SALSAC_MASS_CHANGE_REASON = ( + "Salary-sacrifice support stage rewrites pension columns only; household " + "rows and typed household weights pass through and total household mass " + "is conserved." +) + + +def load_salary_sacrifice_anchor() -> Mapping[str, Any]: + """Load the committed HMRC salary-sacrifice anchor.""" + + return json.loads( + files("microcosm.build.uk") + .joinpath("salary_sacrifice_anchor.json") + .read_text(encoding="utf-8") + ) + + +@dataclass(frozen=True) +class UKSalarySacrificeResult: + """Transformed frame and the full headcount executed-effect receipt.""" + + frame: Frame + training_rows: int + prediction_rows: int + pre_headcount: float + post_headcount: float + shortfall: float + donor_pool_mass: float + rate: float + cap_bound: bool + converted_rows: int + converted_mass: float + moved_amount: float + expected_converted_mass: float + realization_deviation: float + + def evidence(self) -> dict[str, object]: + return { + "stage": "salary_sacrifice", + "qrf": { + "training_rows": self.training_rows, + "prediction_rows": self.prediction_rows, + "seed": SALSAC_QRF_SEED, + }, + "headcount_receipt": { + "target": SALSAC_STAGE_TARGET, + "pre_headcount": self.pre_headcount, + "post_headcount": self.post_headcount, + "shortfall": self.shortfall, + "donor_pool_mass": self.donor_pool_mass, + "rate": self.rate, + "rate_cap": SALSAC_RATE_CAP, + "cap_bound": self.cap_bound, + "converted_rows": self.converted_rows, + "converted_mass": self.converted_mass, + "moved_amount": self.moved_amount, + "expected_converted_mass": self.expected_converted_mass, + "realization_deviation": self.realization_deviation, + }, + } + + +@dataclass(frozen=True) +class UKSalarySacrificeStageTransform: + """Whole-stage transform for salary-sacrifice support.""" + + stage: SourceStageSpec + anchor: Mapping[str, Any] | None = None + last_result: UKSalarySacrificeResult | None = field(default=None, init=False) + + def __call__(self, frame: Frame) -> Frame: + resource = self.anchor or load_salary_sacrifice_anchor() + _assert_salary_sacrifice_stage_parameters(self.stage, anchor=resource) + result = impute_salary_sacrifice(frame) + object.__setattr__(self, "last_result", result) + return result.frame + + @staticmethod + def output_columns() -> tuple[str, ...]: + return (SALSAC_OUTPUT, "employee_pension_contributions") + + def checkpoint_metadata(self) -> dict[str, object]: + if self.last_result is None: + raise RuntimeError("checkpoint metadata requires a completed stage run.") + return {"evidence": self.last_result.evidence()} + + +def impute_salary_sacrifice(frame: Frame) -> UKSalarySacrificeResult: + """Fit on the asked base-FRS subset, then create additional SS support.""" + + validate_uk_national_frame(frame) + person = frame.table("person").copy() + household = frame.table("household").copy() + required_person = { + "person_id", + "person_household_id", + "age", + "employment_income", + "salary_sacrifice_asked", + SALSAC_OUTPUT, + "employee_pension_contributions", + } + missing = sorted(required_person - set(person.columns)) + if missing: + raise ValueError(f"Salary-sacrifice person columns missing: {missing}.") + households = household.set_index("household_id") + person_households = person["person_household_id"] + if not person_households.isin(households.index).all(): + raise ValueError("Salary-sacrifice people must map to a household.") + channel_column = support_channel_column("household") + if channel_column not in household.columns: + raise ValueError( + f"Salary-sacrifice training requires household {channel_column!r}." + ) + channels = person_households.map(households[channel_column]) + clones = person_households.map( + households.get(HOUSEHOLD_IS_CGT_CLONE, pd.Series(False, index=households.index)) + ).fillna(False) + donors = person_households.map( + households.get( + HOUSEHOLD_IS_CGT_BAND_DONOR, + pd.Series(False, index=households.index), + ) + ).fillna(False) + asked = pd.to_numeric(person["salary_sacrifice_asked"], errors="coerce") + if asked.isna().any(): + raise ValueError("salary_sacrifice_asked contains non-numeric values.") + training_mask = ( + channels.eq("frs") & ~clones.astype(bool) & ~donors.astype(bool) & asked.eq(1) + ) + if not training_mask.any(): + raise ValueError("Salary-sacrifice QRF has no eligible asked FRS rows.") + predict_mask = ~asked.eq(1) + numeric = person.loc[:, [*SALSAC_PREDICTORS, SALSAC_OUTPUT]].apply( + pd.to_numeric, errors="coerce" + ) + if not np.isfinite(numeric.to_numpy(dtype=float)).all(): + raise ValueError("Salary-sacrifice QRF columns must be finite numeric values.") + household_weights = pd.Series( + frame.weights_for("household").values, + index=household["household_id"], + ) + person_weights = person_households.map(household_weights).to_numpy(dtype=float) + training = numeric.loc[training_mask, [*SALSAC_PREDICTORS, SALSAC_OUTPUT]].copy() + training["_fit_weight"] = person_weights[training_mask.to_numpy()] + model = _qrf_class()(n_estimators=SALSAC_QRF_ESTIMATORS, seed=SALSAC_QRF_SEED) + fitted = model.fit( + training, + list(SALSAC_PREDICTORS), + [SALSAC_OUTPUT], + weights="_fit_weight", + ) + if predict_mask.any(): + predictions = fitted.predict(numeric.loc[predict_mask, list(SALSAC_PREDICTORS)]) + predicted = pd.to_numeric(predictions[SALSAC_OUTPUT], errors="coerce").to_numpy( + dtype=float + ) + if not np.isfinite(predicted).all(): + raise ValueError("Salary-sacrifice QRF produced non-finite predictions.") + person.loc[predict_mask, SALSAC_OUTPUT] = np.maximum(0.0, predicted) + final_ss = pd.to_numeric(person[SALSAC_OUTPUT], errors="coerce").to_numpy( + dtype=float, copy=True + ) + employee = pd.to_numeric( + person["employee_pension_contributions"], errors="coerce" + ).to_numpy(dtype=float, copy=True) + employment_income = pd.to_numeric( + person["employment_income"], errors="coerce" + ).to_numpy(dtype=float) + if not np.isfinite(final_ss).all() or (final_ss < 0.0).any(): + raise ValueError("Salary-sacrifice amounts must be finite and non-negative.") + if not np.isfinite(employee).all() or (employee < 0.0).any(): + raise ValueError("Employee-pension amounts must be finite and non-negative.") + has_ss = final_ss > 0.0 + pre_headcount = float(person_weights[has_ss].sum()) + shortfall = max(0.0, SALSAC_STAGE_TARGET - pre_headcount) + donor_pool = (employee > 0.0) & ~has_ss & (employment_income > 0.0) + donor_pool_mass = float(person_weights[donor_pool].sum()) + uncapped_rate = shortfall / donor_pool_mass if donor_pool_mass > 0.0 else 0.0 + rate = min(SALSAC_RATE_CAP, uncapped_rate) + draws = stable_identity_uniforms( + person["person_id"].to_numpy(), + seed=SALSAC_CONVERSION_SEED, + salt=SALSAC_CONVERSION_SALT, + ) + converted = donor_pool & (draws < rate) + moved_amount = float(employee[converted].sum()) + final_ss[converted] = employee[converted] + employee[converted] = 0.0 + person[SALSAC_OUTPUT] = final_ss + person["employee_pension_contributions"] = employee + post_headcount = float(person_weights[final_ss > 0.0].sum()) + converted_mass = float(person_weights[converted].sum()) + total = frame.weights_for("household").total + mass_receipt = MassChangeRecord( + entity="household", + old_total=total, + new_total=total, + declared_factor=1.0, + reason=SALSAC_MASS_CHANGE_REASON, + ) + result_frame = uk_national_frame( + person=person, + benunit=frame.table("benunit").copy(), + household=household, + time_period=uk_time_period(frame), + weight_kind=uk_household_weight_kind(frame), + household_weights=frame.weights_for("household").values, + mass_log=(*frame.mass_log, mass_receipt), + ) + validate_uk_national_frame(result_frame) + return UKSalarySacrificeResult( + frame=result_frame, + training_rows=int(training_mask.sum()), + prediction_rows=int(predict_mask.sum()), + pre_headcount=pre_headcount, + post_headcount=post_headcount, + shortfall=shortfall, + donor_pool_mass=donor_pool_mass, + rate=rate, + cap_bound=uncapped_rate > SALSAC_RATE_CAP, + converted_rows=int(converted.sum()), + converted_mass=converted_mass, + moved_amount=moved_amount, + expected_converted_mass=rate * donor_pool_mass, + realization_deviation=( + (converted_mass - rate * donor_pool_mass) / (rate * donor_pool_mass) + if rate * donor_pool_mass > 0.0 + else 0.0 + ), + ) + + +def _qrf_class(): + if QRF is not None: + return QRF + return import_module("microcosm.fit").QRF + + +def _assert_salary_sacrifice_stage_parameters( + stage: SourceStageSpec, + *, + anchor: Mapping[str, Any], +) -> None: + """Bind all stage parameters closed-world; result evidence supplies arm 2.""" + + _assert_closed_world_operations( + stage, + ( + ( + "fit_weighted_qrf", + { + "training_population": ( + "support_channel == frs and not capital-gains clone and " + "not CGT band donor and salary_sacrifice_asked == 1" + ), + "target_population": "salary_sacrifice_asked != 1 frame-wide", + "predictors": list(SALSAC_PREDICTORS), + "targets": [SALSAC_OUTPUT], + "weights": "household_weight", + "weight_mapping": "household_to_person", + "seed": SALSAC_QRF_SEED, + "n_estimators": SALSAC_QRF_ESTIMATORS, + "clamp_minimum": 0, + "preserve_asked_rows": True, + "cache": False, + }, + ), + ( + "convert_donors_to_target_stock", + { + "resource": "salary_sacrifice_anchor.json", + "target": int(SALSAC_STAGE_TARGET), + "donor_pool": ( + "employee_pension_contributions > 0 and " + "pension_contributions_via_salary_sacrifice == 0 and " + "employment_income > 0" + ), + "rate_cap": SALSAC_RATE_CAP, + "move": ( + "full employee_pension_contributions to " + "pension_contributions_via_salary_sacrifice; source zeroed" + ), + "seed": SALSAC_CONVERSION_SEED, + "salt": SALSAC_CONVERSION_SALT, + "receipt": "weighted_headcount", + "reason": SALSAC_MASS_CHANGE_REASON, + }, + ), + ), + ) + hmrc = anchor.get("hmrc_anchor", {}) + derived = anchor.get("derived", {}) + checks = { + "hmrc_anchor.total_users": (hmrc.get("total_users"), SALSAC_HMRC_ANCHOR), + "hmrc_anchor.above_2000": ( + hmrc.get("above_2000"), + SALSAC_ABOVE_2000_ANCHOR, + ), + "hmrc_anchor.below_2000": ( + hmrc.get("below_2000"), + SALSAC_BELOW_2000_ANCHOR, + ), + "derived.stage_target": (derived.get("stage_target"), SALSAC_STAGE_TARGET), + } + for label, (actual, expected) in checks.items(): + if actual != expected: + raise ValueError( + f"Salary-sacrifice resource {label} drifted: expected " + f"{expected!r}, got {actual!r}." + ) + ratio = float(derived.get("staging_ratio", np.nan)) + if not np.isclose(ratio, SALSAC_STAGING_RATIO, rtol=0.0, atol=1e-12): + raise ValueError("Salary-sacrifice resource staging ratio drifted.") diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/source_runtime.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/source_runtime.py index 57323656..a86e89fa 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/source_runtime.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/source_runtime.py @@ -69,6 +69,11 @@ def uk_stage_implementations( frs_hmrc_spine_leaves_transform: Callable[[Frame], Frame] | None = None, spi_support_channel_transform: Callable[[Frame], Frame] | None = None, hmrc_spi_income_spine_transform: Callable[[Frame], Frame] | None = None, + cgt_incidence_clone_transform: Callable[[Frame], Frame] | None = None, + cgt_band_donors_transform: Callable[[Frame], Frame] | None = None, + hmrc_cgt_gains_spine_transform: Callable[[Frame], Frame] | None = None, + salary_sacrifice_transform: Callable[[Frame], Frame] | None = None, + student_loans_transform: Callable[[Frame], Frame] | None = None, ) -> dict[str, Callable[[Frame], Frame]]: """Return the whole-stage implementation map for the UK source plan.""" @@ -96,6 +101,11 @@ def uk_stage_implementations( "frs_hmrc_spine_leaves": frs_hmrc_spine_leaves_transform, "spi_support_channel": spi_support_channel_transform, "hmrc_spi_income_spine": hmrc_spi_income_spine_transform, + "cgt_incidence_clone": cgt_incidence_clone_transform, + "cgt_band_donors": cgt_band_donors_transform, + "hmrc_cgt_gains_spine": hmrc_cgt_gains_spine_transform, + "salary_sacrifice": salary_sacrifice_transform, + "student_loans": student_loans_transform, } implementations.update( { diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py index 6acb9a3b..4b1043cb 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py @@ -162,7 +162,10 @@ # Reviewed constants for every load-bearing manifest parameter the spine # transforms consume. The drift asserts below fail closed on any manifest-only # edit (adversarial-review finding on #717): a manifest change to these values -# requires a matching reviewed code change here. +# requires a matching reviewed code change here. The #730/#684 two-arm rule +# applies to every declared parameter: it needs (1) a drift assert and (2) an +# executed-effect receipt, or an explicit absence statement. Seeded draws use +# twin-build determinism as their executed-effect receipt. SPI_SPINE_STAGE1_PREDICTORS = ("age", "gender", "region") SPI_SPINE_STAGE2_PREDICTORS = ( "age", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/student_loans.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/student_loans.py new file mode 100644 index 00000000..6460c194 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/student_loans.py @@ -0,0 +1,376 @@ +"""UK student-loan cohort assignment and SLC liable-stock support top-ups.""" + +from __future__ import annotations + +import json +from collections.abc import Mapping +from dataclasses import dataclass, field +from importlib.resources import files +from typing import Any + +import numpy as np +import pandas as pd + +from microcosm.build.source_manifest import SourceStageSpec +from microcosm.build.stochastic_assignment import stable_identity_uniforms +from microcosm.build.uk_runtime.cgt_structure import ( + _assert_closed_world_operations, +) +from microcosm.build.uk_runtime.frs_release import load_uk_frs_release +from microcosm.build.uk_runtime.national_frame import ( + uk_household_weight_kind, + uk_national_frame, + uk_time_period, + validate_uk_national_frame, +) +from microcosm.frame import Frame, MassChangeRecord + +PLAN_1_BEFORE = 2012 +PLAN_5_FROM = 2023 +PLAN_2_MIN_AGE = 21 +PLAN_2_MAX_AGE = 55 +PLAN_5_MIN_AGE = 18 +PLAN_5_MAX_AGE = 25 +PLAN_PRIORITY = ("PLAN_5", "PLAN_2") +STUDENT_LOAN_SEED = 42 +YEAR_RULE = "calibration_year" +STUDENT_LOAN_ENUM_DOMAIN = ("NONE", "PLAN_1", "PLAN_2", "PLAN_5") +EXCLUDED_ENGLAND_REGIONS = ("SCOTLAND", "WALES", "NORTHERN_IRELAND") +PLAN_SALTS = { + "PLAN_5": "student_loan_plan_5", + "PLAN_2": "student_loan_plan_2", +} +PLAN_2025_STOCKS = {"PLAN_2": 8_940_000.0, "PLAN_5": 10_000.0} +STUDENT_LOANS_MASS_CHANGE_REASON = ( + "Student-loan plan assignment writes an enum column only; household rows " + "and typed household weights pass through and total household mass is " + "conserved." +) + + +def load_slc_liable_stocks() -> Mapping[str, Any]: + """Load the pinned SLC Table 6a liable-stock series.""" + + return json.loads( + files("microcosm.build.uk") + .joinpath("slc_liable_stocks.json") + .read_text(encoding="utf-8") + ) + + +@dataclass(frozen=True) +class UKStudentLoanPlanReceipt: + plan: str + stock: float + reported_count: float + reported_england_count: float + shortfall: float + eligible_mass: float + rate: float + topped_up_rows: int + topped_up_mass: float + expected_topped_up_mass: float + realization_deviation: float + final_england_count: float + + def evidence(self) -> dict[str, object]: + return { + "stock": self.stock, + "reported_count": self.reported_count, + "reported_england_count": self.reported_england_count, + "shortfall": self.shortfall, + "eligible_mass": self.eligible_mass, + "rate": self.rate, + "topped_up_rows": self.topped_up_rows, + "topped_up_mass": self.topped_up_mass, + "expected_topped_up_mass": self.expected_topped_up_mass, + "realization_deviation": self.realization_deviation, + "final_england_count": self.final_england_count, + } + + +@dataclass(frozen=True) +class UKStudentLoansResult: + """Transformed frame plus a per-plan executed-effect receipt.""" + + frame: Frame + calibration_year: int + plans: Mapping[str, UKStudentLoanPlanReceipt] + + def evidence(self) -> dict[str, object]: + return { + "stage": "student_loans", + "year_rule": YEAR_RULE, + "calibration_year": self.calibration_year, + "plans": {name: receipt.evidence() for name, receipt in self.plans.items()}, + } + + +@dataclass(frozen=True) +class UKStudentLoansStageTransform: + """Whole-stage transform for student-loan plan support.""" + + stage: SourceStageSpec + stocks: Mapping[str, Any] | None = None + calibration_year: int | None = None + last_result: UKStudentLoansResult | None = field(default=None, init=False) + + def __call__(self, frame: Frame) -> Frame: + resource = self.stocks or load_slc_liable_stocks() + year = ( + self.calibration_year + if self.calibration_year is not None + else load_uk_frs_release().calibration_year + ) + _assert_student_loans_stage_parameters(self.stage, stocks=resource, year=year) + result = assign_student_loan_plans(frame, stocks=resource, year=year) + object.__setattr__(self, "last_result", result) + return result.frame + + @staticmethod + def output_columns() -> tuple[str, ...]: + return ("student_loan_plan",) + + def checkpoint_metadata(self) -> dict[str, object]: + if self.last_result is None: + raise RuntimeError("checkpoint metadata requires a completed stage run.") + return {"evidence": self.last_result.evidence()} + + +def assign_student_loan_plans( + frame: Frame, + *, + stocks: Mapping[str, Any], + year: int, +) -> UKStudentLoansResult: + """Assign reported cohorts, then top up PLAN_5 before PLAN_2.""" + + validate_uk_national_frame(frame) + person = frame.table("person").copy() + household = frame.table("household").copy() + required = { + "person_id", + "person_household_id", + "age", + "student_loan_repayments", + "highest_education", + } + missing = sorted(required - set(person.columns)) + if missing: + raise ValueError(f"Student-loan person columns missing: {missing}.") + if "region" not in household.columns: + raise ValueError("Student-loan assignment requires household region.") + region_by_household = household.set_index("household_id")["region"] + region = person["person_household_id"].map(region_by_household) + if region.isna().any(): + raise ValueError("Student-loan people must all map to a household region.") + region_names = np.asarray([_enum_name(value) for value in region], dtype=object) + is_england = ~np.isin(region_names, EXCLUDED_ENGLAND_REGIONS) + education = np.asarray( + [_enum_name(value) for value in person["highest_education"]], dtype=object + ) + age = pd.to_numeric(person["age"], errors="coerce").to_numpy(dtype=float) + repayments = pd.to_numeric( + person["student_loan_repayments"], errors="coerce" + ).to_numpy(dtype=float) + if not np.isfinite(age).all() or not np.isfinite(repayments).all(): + raise ValueError("Student-loan age and repayment inputs must be finite.") + weights_by_household = pd.Series( + frame.weights_for("household").values, + index=household["household_id"], + ) + person_weights = ( + person["person_household_id"].map(weights_by_household).to_numpy(dtype=float) + ) + start_year = year - age + 18 + has_repayments = repayments > 0.0 + plan = np.full(len(person), "NONE", dtype=object) + plan[has_repayments & (start_year < PLAN_1_BEFORE)] = "PLAN_1" + plan[has_repayments & (start_year >= PLAN_5_FROM)] = "PLAN_5" + plan[has_repayments & (plan == "NONE")] = "PLAN_2" + reported_plan = plan.copy() + receipts: dict[str, UKStudentLoanPlanReceipt] = {} + for plan_name in PLAN_PRIORITY: + stock = _stock(stocks, plan_name, year) + current_england = float(person_weights[(plan == plan_name) & is_england].sum()) + shortfall = max(0.0, stock - current_england) + eligible = ( + (plan == "NONE") + & is_england + & (education == "TERTIARY") + & _plan_age_cohort_eligibility(plan_name, age=age, start_year=start_year) + ) + eligible_mass = float(person_weights[eligible].sum()) + rate = min(1.0, shortfall / eligible_mass) if eligible_mass > 0.0 else 0.0 + draws = stable_identity_uniforms( + person["person_id"].to_numpy(), + seed=STUDENT_LOAN_SEED, + salt=PLAN_SALTS[plan_name], + ) + topped_up = eligible & (draws < rate) + plan[topped_up] = plan_name + receipts[plan_name] = UKStudentLoanPlanReceipt( + plan=plan_name, + stock=stock, + reported_count=float(person_weights[reported_plan == plan_name].sum()), + reported_england_count=float( + person_weights[(reported_plan == plan_name) & is_england].sum() + ), + shortfall=shortfall, + eligible_mass=eligible_mass, + rate=rate, + topped_up_rows=int(topped_up.sum()), + topped_up_mass=float(person_weights[topped_up].sum()), + expected_topped_up_mass=rate * eligible_mass, + realization_deviation=( + (float(person_weights[topped_up].sum()) - rate * eligible_mass) + / (rate * eligible_mass) + if rate * eligible_mass > 0.0 + else 0.0 + ), + final_england_count=float( + person_weights[(plan == plan_name) & is_england].sum() + ), + ) + unknown = sorted(set(plan) - set(STUDENT_LOAN_ENUM_DOMAIN)) + if unknown: + raise ValueError(f"Student-loan assignment emitted unknown plan(s): {unknown}.") + person["student_loan_plan"] = plan + total = frame.weights_for("household").total + mass_receipt = MassChangeRecord( + entity="household", + old_total=total, + new_total=total, + declared_factor=1.0, + reason=STUDENT_LOANS_MASS_CHANGE_REASON, + ) + result_frame = uk_national_frame( + person=person, + benunit=frame.table("benunit").copy(), + household=household, + time_period=uk_time_period(frame), + weight_kind=uk_household_weight_kind(frame), + household_weights=frame.weights_for("household").values, + mass_log=(*frame.mass_log, mass_receipt), + ) + validate_uk_national_frame(result_frame) + return UKStudentLoansResult( + frame=result_frame, + calibration_year=year, + plans=receipts, + ) + + +def _plan_age_cohort_eligibility( + plan: str, + *, + age: np.ndarray, + start_year: np.ndarray, +) -> np.ndarray: + if plan == "PLAN_5": + return ( + (age >= PLAN_5_MIN_AGE) + & (age <= PLAN_5_MAX_AGE) + & (start_year >= PLAN_5_FROM) + ) + if plan == "PLAN_2": + return ( + (age >= PLAN_2_MIN_AGE) + & (age <= PLAN_2_MAX_AGE) + & (start_year >= PLAN_1_BEFORE) + & (start_year < PLAN_5_FROM) + ) + raise ValueError(f"Unsupported student-loan top-up plan {plan!r}.") + + +def _stock(stocks: Mapping[str, Any], plan: str, year: int) -> float: + key = plan.lower() + try: + values = stocks["plans"][key]["liable"] + value = values[str(year)] + except (KeyError, TypeError) as error: + raise ValueError( + f"SLC liable-stock resource has no {key} value for {year}." + ) from error + result = float(value) + if result < 0 or not np.isfinite(result): + raise ValueError(f"SLC {key} liable stock for {year} is invalid: {value!r}.") + return result + + +def _enum_name(value: object) -> str: + if hasattr(value, "name"): + return str(value.name) + text = str(value) + return text.rsplit(".", 1)[-1] + + +def _assert_student_loans_stage_parameters( + stage: SourceStageSpec, + *, + stocks: Mapping[str, Any], + year: int, +) -> None: + """Bind every stage parameter closed-world; per-plan receipts supply arm 2.""" + + region_exclusions = list(EXCLUDED_ENGLAND_REGIONS) + _assert_closed_world_operations( + stage, + ( + ( + "assign_student_loan_plan_cohorts", + { + "year_rule": YEAR_RULE, + "start_year_formula": "year - age + 18", + "reported_repayment_test": "student_loan_repayments > 0", + "reported_country_gate": False, + "plan_1_before": PLAN_1_BEFORE, + "plan_5_from": PLAN_5_FROM, + "enum_domain": list(STUDENT_LOAN_ENUM_DOMAIN), + "plan_4_imputation": False, + }, + ), + ( + "top_up_to_stock", + { + "plan": "PLAN_5", + "priority": 1, + "resource": "slc_liable_stocks.json", + "stock_series": "plan_5.liable", + "year_rule": YEAR_RULE, + "age_min": PLAN_5_MIN_AGE, + "age_max": PLAN_5_MAX_AGE, + "cohort_start_min": PLAN_5_FROM, + "eligible_region_exclusions": region_exclusions, + "highest_education": "TERTIARY", + "seed": STUDENT_LOAN_SEED, + "salt": PLAN_SALTS["PLAN_5"], + }, + ), + ( + "top_up_to_stock", + { + "plan": "PLAN_2", + "priority": 2, + "resource": "slc_liable_stocks.json", + "stock_series": "plan_2.liable", + "year_rule": YEAR_RULE, + "age_min": PLAN_2_MIN_AGE, + "age_max": PLAN_2_MAX_AGE, + "cohort_start_min": PLAN_1_BEFORE, + "cohort_start_max_exclusive": PLAN_5_FROM, + "eligible_region_exclusions": region_exclusions, + "highest_education": "TERTIARY", + "seed": STUDENT_LOAN_SEED, + "salt": PLAN_SALTS["PLAN_2"], + "reason": STUDENT_LOANS_MASS_CHANGE_REASON, + }, + ), + ), + ) + if year == 2025: + for plan, expected_stock in PLAN_2025_STOCKS.items(): + if _stock(stocks, plan, year) != expected_stock: + raise ValueError( + f"SLC {plan} liable stock for 2025 drifted from {expected_stock}." + ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/terminal_gates.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/terminal_gates.py index cefeeb86..3d39e4a8 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/terminal_gates.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/terminal_gates.py @@ -168,6 +168,7 @@ def __post_init__(self) -> None: "household.gas_consumption", "household.has_fuel_consumption", "household.household_is_capital_gains_clone", + "household.household_is_cgt_band_donor", "household.household_is_spi_synthetic", "household.la_code_oa", "household.lsoa_code", diff --git a/packages/microcosm-build/tests/test_country_spec.py b/packages/microcosm-build/tests/test_country_spec.py index a7998c23..e4be8693 100644 --- a/packages/microcosm-build/tests/test_country_spec.py +++ b/packages/microcosm-build/tests/test_country_spec.py @@ -284,6 +284,11 @@ def test_spi_spine_adds_no_country_package_resources(self) -> None: "frs_release.json", "gates.json", "brma_rent_counts.json", + "hmrc_cgt_size_bands.json", + "advani_summers_capital_gains_distribution.json", + "salary_sacrifice_anchor.json", + "slc_liable_stocks.json", + "cgt_band_donor_support_bounds.json", "hmrc_income_release_gate_report.json", "hmrc_income_replay_report.json", "hmrc_income_source_stages.json", @@ -313,11 +318,11 @@ def test_spi_spine_adds_no_country_package_resources(self) -> None: "target_reference_membership.json", ) - def test_uk_source_manifest_loads_twenty_one_stages(self) -> None: + def test_uk_source_manifest_loads_twenty_six_stages(self) -> None: spec = load_country_spec("uk") assert spec.sources is not None - assert len(spec.sources.stages) == 21 + assert len(spec.sources.stages) == 26 class TestExistingPackagesGeneralize: @@ -350,6 +355,11 @@ def test_uk_package_loads(self) -> None: "frs_release.json", "gates.json", "brma_rent_counts.json", + "hmrc_cgt_size_bands.json", + "advani_summers_capital_gains_distribution.json", + "salary_sacrifice_anchor.json", + "slc_liable_stocks.json", + "cgt_band_donor_support_bounds.json", "hmrc_income_release_gate_report.json", "hmrc_income_replay_report.json", "hmrc_income_source_stages.json", @@ -389,8 +399,7 @@ def test_uk_target_references_accept_regenerated_contract_fields(self) -> None: "calendar_year_average" ) assert ( - references["obr.income_tax"].assertion_policy - == "allow_source_projection" + references["obr.income_tax"].assertion_policy == "allow_source_projection" ) fanout = references["hmrc/employment_income_income_band_100_000_to_150_000"] @@ -620,6 +629,7 @@ def test_declares_the_full_june_battery(self, manifest) -> None: "uk_export_surface", "uk_take_up_signal", "uk_brma_enum_domain", + "uk_student_loan_plan_enum_domain", "uk_calibration_reference_coverage", "uk_target_surface", "uk_target_fit", @@ -635,12 +645,12 @@ def test_ledger_compile_parity_gates_pin_their_fixture_periods( ) -> None: params = {gate.id: gate.parameters for gate in manifest.gates} - assert params["uk_ledger_compile_parity_production_2023"][ - "target_period" - ] == 2023 - assert params["uk_ledger_compile_parity_incumbent_2025"][ - "target_period" - ] == 2025 + assert ( + params["uk_ledger_compile_parity_production_2023"]["target_period"] == 2023 + ) + assert ( + params["uk_ledger_compile_parity_incumbent_2025"]["target_period"] == 2025 + ) def test_only_the_weights_audit_blocks_on_absent_evidence(self, manifest) -> None: # "An absent audit is not a passing audit" — the retired schema-3 diff --git a/packages/microcosm-build/tests/test_plan.py b/packages/microcosm-build/tests/test_plan.py index b7e7a30e..d3cbcfda 100644 --- a/packages/microcosm-build/tests/test_plan.py +++ b/packages/microcosm-build/tests/test_plan.py @@ -56,6 +56,25 @@ def test_two_producers_of_one_column_refused(self) -> None: ] ) + def test_explicit_rewrite_may_follow_canonical_producer(self) -> None: + plan = StagePlan( + [ + Stage( + name="produce", + transform=lambda frame: frame, + produces=("net_worth",), + ), + Stage( + name="rewrite", + transform=lambda frame: frame, + produces=("net_worth",), + rewrites=("net_worth",), + ), + ] + ) + + assert [stage.name for stage in plan.stages] == ["produce", "rewrite"] + def test_empty_donor_fields_refused(self) -> None: with pytest.raises(ValueError, match="source citation is required"): DonorSpec(survey="SCF", source="") diff --git a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py index 617eff55..59f0531b 100644 --- a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py +++ b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py @@ -42,7 +42,7 @@ ), ( "uk", - "e12a2cb87c0e096af0173bd51fbede4b7df5e7c118ffe07ef616bbb30640e4ea", + "1f163cbf7b35f07d49b6e2905d01a2519c0ba37e6d2d1470424272e18ac621db", { "benunit.benunit_id", "household.household_id", diff --git a/packages/microcosm-build/tests/test_uk_battery_bindings.py b/packages/microcosm-build/tests/test_uk_battery_bindings.py index 0444e691..e3307829 100644 --- a/packages/microcosm-build/tests/test_uk_battery_bindings.py +++ b/packages/microcosm-build/tests/test_uk_battery_bindings.py @@ -451,9 +451,9 @@ def test_fully_armed_battery_evaluates_gate_for_gate(self) -> None: ] # 11 as on main (uk_nonnegative_columns passes with zero required # columns — the scheduled stages declare none), the two E4 stochastic - # gates, the E5 support gate, and the E6 aggregate-admin gate; their - # evaluators have direct tests. - assert len(passed) == 15 + # gates, the E5 support gate, the E6 aggregate-admin gate, and the E8 + # student-loan enum gate; their evaluators have direct tests. + assert len(passed) == 16 qrf = by_id["uk_qrf_tail_concentration"] assert qrf.status is GateStatus.FAILED assert "declared QRF output is absent" in qrf.result.failures[0] diff --git a/packages/microcosm-build/tests/test_uk_cgt_source_manifest.py b/packages/microcosm-build/tests/test_uk_cgt_source_manifest.py index d0d6f281..2348f23e 100644 --- a/packages/microcosm-build/tests/test_uk_cgt_source_manifest.py +++ b/packages/microcosm-build/tests/test_uk_cgt_source_manifest.py @@ -9,8 +9,13 @@ UK_CGT_IMPUTATION_SEED, UK_CGT_IMPUTATION_STAGE_NAME, UK_CGT_MASS_CONSERVATION_REASON, + UK_CGT_SPINE_MASS_CONSERVATION_REASON, UK_CGT_TAXABLE_INCOME_PROXY_COMPONENTS, ) +from microcosm.build.uk_runtime.cgt_structure import ( + CGT_CLONE_MASS_CHANGE_REASON, + CGT_DONOR_MASS_CHANGE_REASON, +) from microcosm.build.uk_runtime.hmrc_capital_gains import ( HMRC_CGT_JOINT_ODS_SHA256, HMRC_CGT_JOINT_ODS_SIZE_BYTES, @@ -20,6 +25,12 @@ from microcosm.build.uk_runtime.release_input_coverage import ( load_uk_release_input_coverage_manifest, ) +from microcosm.build.uk_runtime.salary_sacrifice import ( + SALSAC_MASS_CHANGE_REASON, +) +from microcosm.build.uk_runtime.student_loans import ( + STUDENT_LOANS_MASS_CHANGE_REASON, +) _MANIFEST_PATH = ( Path(__file__).resolve().parents[1] @@ -130,6 +141,20 @@ def test_the_shipped_family_contracts_pass_the_terminal_gate_shape() -> None: declared_factor=1.0, reason=spi_reason, ), + MassChangeRecord( + entity="household", + old_total=100.0, + new_total=100.0, + declared_factor=1.0, + reason=CGT_CLONE_MASS_CHANGE_REASON, + ), + MassChangeRecord( + entity="household", + old_total=100.0, + new_total=110.0, + declared_factor=None, + reason=CGT_DONOR_MASS_CHANGE_REASON, + ), MassChangeRecord( entity="household", old_total=100.0, @@ -137,6 +162,27 @@ def test_the_shipped_family_contracts_pass_the_terminal_gate_shape() -> None: declared_factor=1.0, reason=UK_CGT_MASS_CONSERVATION_REASON, ), + MassChangeRecord( + entity="household", + old_total=100.0, + new_total=100.0, + declared_factor=1.0, + reason=UK_CGT_SPINE_MASS_CONSERVATION_REASON, + ), + MassChangeRecord( + entity="household", + old_total=100.0, + new_total=100.0, + declared_factor=1.0, + reason=SALSAC_MASS_CHANGE_REASON, + ), + MassChangeRecord( + entity="household", + old_total=100.0, + new_total=100.0, + declared_factor=1.0, + reason=STUDENT_LOANS_MASS_CHANGE_REASON, + ), MassChangeRecord( entity="household", old_total=100.0, @@ -167,3 +213,14 @@ def test_the_shipped_family_contracts_pass_the_terminal_gate_shape() -> None: assert any( "hmrc_cgt_gains" in failure and "kind" in failure for failure in failures ) + + +def test_certified_and_spine_families_require_distinct_receipts() -> None: + """One record must never satisfy both CGT families (review finding).""" + manifest = load_uk_release_input_coverage_manifest() + families = manifest.family_coverage + certified = families["hmrc_cgt_gains"]["required_mass_change_reason"] + spine = families["hmrc_cgt_gains_spine"]["required_mass_change_reason"] + assert certified == UK_CGT_MASS_CONSERVATION_REASON + assert spine == UK_CGT_SPINE_MASS_CONSERVATION_REASON + assert certified != spine diff --git a/packages/microcosm-build/tests/test_uk_cgt_structure.py b/packages/microcosm-build/tests/test_uk_cgt_structure.py new file mode 100644 index 00000000..f675d523 --- /dev/null +++ b/packages/microcosm-build/tests/test_uk_cgt_structure.py @@ -0,0 +1,307 @@ +from __future__ import annotations + +import copy +from dataclasses import replace +from functools import lru_cache + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.country_spec import load_country_spec +from microcosm.build.source_manifest import SourceOperationSpec +from microcosm.build.uk_runtime.cgt_imputation import ( + UK_CGT_TAXABLE_INCOME_PROXY_COMPONENTS, +) +from microcosm.build.uk_runtime.cgt_structure import ( + CGT_CLONE_MASS_CHANGE_REASON, + DONOR_BAND_COUNT, + DONOR_TOTAL, + DONORS_PER_BAND, + HOUSEHOLD_IS_CGT_BAND_DONOR, + HOUSEHOLD_IS_CGT_CLONE, + MIN_DONOR_BAND_LOWER, + _assert_cgt_donor_stage_parameters, + _assert_cgt_incidence_stage_parameters, + _draw_banded_priors, + clone_cgt_incidence, + load_hmrc_cgt_size_bands, + stack_cgt_band_donors, +) +from microcosm.build.uk_runtime.national_frame import uk_national_frame +from microcosm.frame import WeightKind + + +def _distribution(*, negative: bool = False) -> dict[str, object]: + knots = [-70.0, -60.0, -40.0, -20.0, -10.0, -5.0, -1.0] + if not negative: + knots = [-10.0, 0.0, 25.0, 50.0, 75.0, 100.0, 125.0] + return { + "rows": [ + { + "minimum_total_income": 0, + "percent_with_gains": 1.0, + **dict( + zip( + ("p05", "p10", "p25", "p50", "p75", "p90", "p95"), + knots, + strict=True, + ) + ), + } + ] + } + + +def _one_person_households(n: int, *, reverse_people: bool = False): + ids = np.arange(1, n + 1, dtype="int64") + person = pd.DataFrame( + { + "person_id": ids, + "person_benunit_id": ids, + "person_household_id": ids, + "age": np.full(n, 40), + "capital_gains": np.zeros(n), + "employment_income": np.linspace(10_000.0, 100_000.0, n), + } + ) + for column in UK_CGT_TAXABLE_INCOME_PROXY_COMPONENTS: + if column not in person: + person[column] = 0.0 + if reverse_people: + person = person.iloc[::-1].reset_index(drop=True) + benunit = pd.DataFrame({"benunit_id": ids}) + household = pd.DataFrame( + { + "household_id": ids, + "household_support_channel": np.where(ids % 2, "frs", "spi"), + } + ) + return uk_national_frame( + person=person, + benunit=benunit, + household=household, + household_weights=np.linspace(1.0, 2.0, n), + time_period="2024", + ) + + +def _adult_frame(): + person = pd.DataFrame( + { + "person_id": [1, 2, 3, 4, 5], + "person_benunit_id": [1, 1, 1, 2, 2], + "person_household_id": [1, 1, 1, 2, 2], + "age": [45, 45, 12, 30, 50], + "capital_gains": [1.0, 2.0, 3.0, 4.0, 5.0], + "employment_income": [20_000.0] * 5, + } + ) + for column in UK_CGT_TAXABLE_INCOME_PROXY_COMPONENTS: + if column not in person: + person[column] = 0.0 + return uk_national_frame( + person=person, + benunit=pd.DataFrame({"benunit_id": [1, 2]}), + household=pd.DataFrame({"household_id": [1, 2]}), + household_weights=[3.0, 7.0], + time_period="2024", + ) + + +@lru_cache +def _stage(name: str): + return load_country_spec("uk").sources.stage_map()[name] + + +def _drift(stage, operation_index: int, parameter: str): + operations = list(stage.operations) + operation = operations[operation_index] + operations[operation_index] = SourceOperationSpec( + kind=operation.kind, + parameters={**operation.parameters, parameter: "__drift__"}, + ) + return replace(stage, operations=tuple(operations)) + + +def test_clone_splits_exact_mass_and_uses_oldest_adult_carriers() -> None: + result = clone_cgt_incidence( + _adult_frame(), distribution=_distribution(negative=True) + ) + household = result.frame.table("household") + person = result.frame.table("person") + + assert result.original_mass == pytest.approx(5.0) + assert result.clone_mass == pytest.approx(5.0) + assert result.frame.weights_for("household").kind is WeightKind.IMPORTANCE + assert result.frame.mass_log[-1].reason == CGT_CLONE_MASS_CHANGE_REASON + clone_households = set( + household.loc[household[HOUSEHOLD_IS_CGT_CLONE], "household_id"] + ) + gainers = person.loc[person.capital_gains != 0] + assert set(gainers.person_household_id) == clone_households + # Household 1 has tied oldest adults: lower person_id carries the draw. + assert ( + gainers.loc[ + gainers.person_household_id == min(clone_households) + ].person_id.nunique() + == 1 + ) + assert result.carrier_count == 2 + assert result.negative_prior_count == 1 + assert (gainers.capital_gains < 0.0).any() + + +def test_prior_spline_keeps_negative_values_and_extrapolates_linearly() -> None: + draws = _draw_banded_priors( + np.zeros(4), + np.asarray([0.0, 0.05, 0.95, 1.0]), + distribution=_distribution(), + ) + + assert draws[0] < 0.0 + assert draws[1] == pytest.approx(-10.0) + assert draws[2] == pytest.approx(125.0) + assert draws[3] > draws[2] + + +def test_band_donors_are_band_exact_positive_and_permutation_stable() -> None: + first = stack_cgt_band_donors( + _one_person_households(300), + size_bands=load_hmrc_cgt_size_bands(), + distribution=_distribution(), + ) + second = stack_cgt_band_donors( + _one_person_households(300, reverse_people=True), + size_bands=load_hmrc_cgt_size_bands(), + distribution=_distribution(), + ) + donor_households = first.frame.table("household").loc[ + lambda table: table[HOUSEHOLD_IS_CGT_BAND_DONOR] + ] + + assert len(donor_households) == DONOR_TOTAL == DONORS_PER_BAND * DONOR_BAND_COUNT + assert (first.frame.weights_for("household").values[-DONOR_TOTAL:] > 0).all() + assert [row["donor_count"] for row in first.band_rows] == [DONORS_PER_BAND] * 9 + assert [row["lower_limit"] for row in first.band_rows][0] == MIN_DONOR_BAND_LOWER + assert [row["weighted_taxpayers"] for row in first.band_rows] == pytest.approx( + [79_000, 74_000, 53_000, 37_000, 14_000, 8_000, 5_000, 3_000, 2_000] + ) + second_donors = second.frame.table("household").loc[ + lambda table: table[HOUSEHOLD_IS_CGT_BAND_DONOR] + ] + assert set(donor_households.household_id) == set(second_donors.household_id) + + +def test_never_zero_band_weight_assertion_fires() -> None: + resource = copy.deepcopy(load_hmrc_cgt_size_bands()) + retained = next(row for row in resource["rows"] if row["lower_limit"] == 12_300) + retained["taxpayers_thousands"] = 0 + + with pytest.raises(ValueError, match="zero initial weight"): + _assert_cgt_donor_stage_parameters( + _stage("cgt_band_donors"), size_bands=resource + ) + + +@pytest.mark.parametrize( + "operation_index,parameter", + [ + *[ + (0, name) + for name in ( + "entity", + "copies", + "flag_column", + "original_flag", + "clone_flag", + "mass_split", + "weight_kind_out", + "conservation", + "id_remapping", + "declared_factor", + "reason", + ) + ], + *[ + (1, name) + for name in ( + "resource", + "income_proxy_components", + "allowance_subtraction", + "carrier", + "adult_minimum_age", + "quantile_points", + "spline_degree", + "extrapolation", + "keep_negative_draws", + "seed", + "salt", + ) + ], + ], +) +def test_incidence_drift_assert_covers_every_reviewed_parameter( + operation_index: int, parameter: str +) -> None: + with pytest.raises(ValueError, match="drifted"): + _assert_cgt_incidence_stage_parameters( + _drift(_stage("cgt_incidence_clone"), operation_index, parameter) + ) + + +@pytest.mark.parametrize( + "parameter", + ( + "size_band_resource", + "incidence_resource", + "minimum_band_lower", + "donors_per_band", + "expected_band_count", + "expected_donor_count", + "candidate_order", + "draw", + "seed", + "flag_column", + "carrier", + "initial_weight", + "never_zero_weight", + "weight_kind_out", + "reason", + ), +) +def test_donor_drift_assert_covers_every_reviewed_parameter(parameter: str) -> None: + with pytest.raises(ValueError, match="drifted"): + _assert_cgt_donor_stage_parameters( + _drift(_stage("cgt_band_donors"), 0, parameter), + size_bands=load_hmrc_cgt_size_bands(), + ) + + +def test_donor_drift_assert_rejects_propensity_and_extra_keys() -> None: + """Closed-world equality: undeclared and extra parameters both fail.""" + for parameter in ("propensity", "undeclared_extra_key"): + with pytest.raises(ValueError, match="drifted"): + _assert_cgt_donor_stage_parameters( + _drift(_stage("cgt_band_donors"), 0, parameter), + size_bands=load_hmrc_cgt_size_bands(), + ) + + +def test_drift_asserts_reject_extra_operations() -> None: + for name, check in ( + ("cgt_incidence_clone", _assert_cgt_incidence_stage_parameters), + ( + "cgt_band_donors", + lambda stage: _assert_cgt_donor_stage_parameters( + stage, size_bands=load_hmrc_cgt_size_bands() + ), + ), + ): + stage = _stage(name) + extra = replace( + stage, + operations=(*stage.operations, stage.operations[-1]), + ) + with pytest.raises(ValueError, match="operation order drifted"): + check(extra) diff --git a/packages/microcosm-build/tests/test_uk_frs_spine.py b/packages/microcosm-build/tests/test_uk_frs_spine.py index 82241439..d2d9ff82 100644 --- a/packages/microcosm-build/tests/test_uk_frs_spine.py +++ b/packages/microcosm-build/tests/test_uk_frs_spine.py @@ -1577,6 +1577,7 @@ def test_input_artifact_pins_bind_spi_donor_and_ods() -> None: pins = tool._input_artifact_pins(stages) assert set(pins) == { + "cgt_published_fact_surface", "etb_household_tab", "lcfs_household_tab", "lcfs_person_tab", @@ -1597,8 +1598,32 @@ def test_input_artifact_pins_bind_spi_donor_and_ods() -> None: "etb_vat", "etb_services", "hmrc_spi_income_spine", + "hmrc_cgt_gains_spine", ) for artifact in stage_map[stage_name].artifacts - if "table" not in artifact and "resource" not in artifact + if "table" not in artifact + and "resource" not in artifact + and "sha256" in artifact } assert {role: pin["sha256"] for role, pin in pins.items()} == declared + + +def test_e8_manifest_seeds_all_reach_the_build_sidecar_harvester() -> None: + tool = _load_tool() + spec = load_country_spec("uk") + assert spec.sources is not None + stages = spec.sources.stage_map() + + declared = tool._declared_seeds([stages[name] for name in tool._STAGE_NAMES]) + + assert declared["cgt_incidence_clone"] == {"cgt_prior_amount": 0} + assert declared["cgt_band_donors"] == {"stack_band_donor_households": 1} + assert declared["hmrc_cgt_gains_spine"] == {"within_band_draws": 552} + assert declared["salary_sacrifice"] == { + "salary_sacrifice": 42, + "salary_sacrifice_conversion": 2024, + } + assert declared["student_loans"] == { + "student_loan_plan_5": 42, + "student_loan_plan_2": 42, + } diff --git a/packages/microcosm-build/tests/test_uk_national_build.py b/packages/microcosm-build/tests/test_uk_national_build.py index e8edf9ea..1a9766e6 100644 --- a/packages/microcosm-build/tests/test_uk_national_build.py +++ b/packages/microcosm-build/tests/test_uk_national_build.py @@ -1104,6 +1104,7 @@ def test_national_build_real_terminal_batch_blocks_incomplete_qrf_before_staging "uk_aggregate_admin": "evidence_absent", "uk_take_up_signal": "passed", "uk_brma_enum_domain": "passed", + "uk_student_loan_plan_enum_domain": "failed", # The legacy report omitted unevidenced gates; the battery names # every gap — non-blocking off the release-candidate posture. "uk_export_surface": "evidence_absent", diff --git a/packages/microcosm-build/tests/test_uk_release_input_coverage.py b/packages/microcosm-build/tests/test_uk_release_input_coverage.py index 7467bf20..3453fa6a 100644 --- a/packages/microcosm-build/tests/test_uk_release_input_coverage.py +++ b/packages/microcosm-build/tests/test_uk_release_input_coverage.py @@ -605,6 +605,11 @@ def test_shipped_manifest_is_current(self) -> None: { "hmrc_spi_income", "hmrc_cgt_gains", + "cgt_incidence_clone", + "cgt_band_donors", + "hmrc_cgt_gains_spine", + "salary_sacrifice", + "student_loans", "was_wealth", "regional_property_uprating", "lcfs_consumption", diff --git a/packages/microcosm-build/tests/test_uk_salary_sacrifice.py b/packages/microcosm-build/tests/test_uk_salary_sacrifice.py new file mode 100644 index 00000000..f2a9d46d --- /dev/null +++ b/packages/microcosm-build/tests/test_uk_salary_sacrifice.py @@ -0,0 +1,238 @@ +from __future__ import annotations + +from dataclasses import replace +from functools import lru_cache + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.country_spec import load_country_spec +from microcosm.build.source_manifest import SourceOperationSpec +from microcosm.build.uk_runtime import salary_sacrifice +from microcosm.build.uk_runtime.cgt_structure import ( + HOUSEHOLD_IS_CGT_BAND_DONOR, + HOUSEHOLD_IS_CGT_CLONE, +) +from microcosm.build.uk_runtime.national_frame import uk_national_frame +from microcosm.build.uk_runtime.salary_sacrifice import ( + SALSAC_OUTPUT, + SALSAC_RATE_CAP, + SALSAC_STAGE_TARGET, + _assert_salary_sacrifice_stage_parameters, + impute_salary_sacrifice, + load_salary_sacrifice_anchor, +) + + +class _Fitted: + value = 0.0 + + def predict(self, predictors: pd.DataFrame) -> pd.DataFrame: + return pd.DataFrame({SALSAC_OUTPUT: self.value}, index=predictors.index) + + +class _FakeQRF: + training_frames: list[pd.DataFrame] = [] + + def __init__(self, *, n_estimators: int, seed: int) -> None: + assert n_estimators == 100 + assert seed == 42 + + def fit(self, frame, predictors, targets, *, weights): + assert predictors == ["age", "employment_income"] + assert targets == [SALSAC_OUTPUT] + assert weights == "_fit_weight" + self.training_frames.append(frame.copy()) + return _Fitted() + + +def _frame( + *, + asked, + salary_sacrifice_values, + employee_pension, + channels=None, + clones=None, + donors=None, + weights=None, +): + n = len(asked) + ids = np.arange(1, n + 1, dtype="int64") + channels = channels or ["frs"] * n + clones = clones or [False] * n + donors = donors or [False] * n + person = pd.DataFrame( + { + "person_id": ids, + "person_benunit_id": ids, + "person_household_id": ids, + "age": np.linspace(25, 55, n), + "employment_income": np.full(n, 30_000.0), + "salary_sacrifice_asked": asked, + SALSAC_OUTPUT: salary_sacrifice_values, + "employee_pension_contributions": employee_pension, + } + ) + household = pd.DataFrame( + { + "household_id": ids, + "household_support_channel": channels, + HOUSEHOLD_IS_CGT_CLONE: clones, + HOUSEHOLD_IS_CGT_BAND_DONOR: donors, + } + ) + return uk_national_frame( + person=person, + benunit=pd.DataFrame({"benunit_id": ids}), + household=household, + household_weights=np.ones(n) if weights is None else weights, + time_period="2024", + ) + + +@lru_cache +def _stage(): + return load_country_spec("uk").sources.stage_map()["salary_sacrifice"] + + +def _drift(operation_index: int, parameter: str): + stage = _stage() + operations = list(stage.operations) + operation = operations[operation_index] + operations[operation_index] = SourceOperationSpec( + operation.kind, + {**operation.parameters, parameter: "__drift__"}, + ) + return replace(stage, operations=tuple(operations)) + + +def test_qrf_preserves_asked_rows_and_excludes_nonbase_training_rows( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _FakeQRF.training_frames = [] + _Fitted.value = -25.0 + monkeypatch.setattr(salary_sacrifice, "QRF", _FakeQRF) + frame = _frame( + asked=[1, 1, 1, 1, 0], + salary_sacrifice_values=[10.0, 20.0, 30.0, 40.0, 0.0], + employee_pension=[0.0] * 5, + channels=["frs", "spi", "frs", "frs", "spi"], + clones=[False, False, True, False, False], + donors=[False, False, False, True, False], + ) + + result = impute_salary_sacrifice(frame) + + assert len(_FakeQRF.training_frames) == 1 + assert _FakeQRF.training_frames[0].index.tolist() == [0] + assert result.frame.table("person")[SALSAC_OUTPUT].tolist() == [ + 10.0, + 20.0, + 30.0, + 40.0, + 0.0, + ] + assert result.training_rows == 1 + assert result.prediction_rows == 1 + + +def test_conversion_moves_full_pension_zeros_source_and_records_cap( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _Fitted.value = 0.0 + monkeypatch.setattr(salary_sacrifice, "QRF", _FakeQRF) + n_donors = 100 + frame = _frame( + asked=[1, *([0] * n_donors)], + salary_sacrifice_values=[0.0] * (n_donors + 1), + employee_pension=[0.0, *np.linspace(100.0, 1_000.0, n_donors)], + weights=[1.0, *([100_000.0] * n_donors)], + ) + + result = impute_salary_sacrifice(frame) + person = result.frame.table("person") + converted = person[SALSAC_OUTPUT] > 0.0 + + assert result.rate == SALSAC_RATE_CAP + assert result.cap_bound is True + assert 0 < result.converted_rows < n_donors + assert (person.loc[converted, "employee_pension_contributions"] == 0.0).all() + assert result.moved_amount == pytest.approx( + person.loc[converted, SALSAC_OUTPUT].sum() + ) + evidence = result.evidence()["headcount_receipt"] + assert evidence["target"] == SALSAC_STAGE_TARGET + assert evidence["converted_rows"] == result.converted_rows + + +def test_anchor_is_self_consistent() -> None: + anchor = load_salary_sacrifice_anchor() + assert anchor["hmrc_anchor"]["total_users"] * anchor["derived"][ + "staging_ratio" + ] == pytest.approx(anchor["derived"]["stage_target"]) + + +@pytest.mark.parametrize( + "operation_index,parameter", + [ + *[ + (0, name) + for name in ( + "training_population", + "target_population", + "predictors", + "targets", + "weights", + "weight_mapping", + "seed", + "n_estimators", + "clamp_minimum", + "preserve_asked_rows", + "cache", + ) + ], + *[ + (1, name) + for name in ( + "resource", + "target", + "donor_pool", + "rate_cap", + "move", + "seed", + "salt", + "receipt", + ) + ], + ], +) +def test_manifest_drift_assert_covers_every_reviewed_parameter( + operation_index: int, parameter: str +) -> None: + with pytest.raises(ValueError, match="drifted"): + _assert_salary_sacrifice_stage_parameters( + _drift(operation_index, parameter), + anchor=load_salary_sacrifice_anchor(), + ) + + +def test_resource_drift_assert_rejects_anchor_change() -> None: + anchor = dict(load_salary_sacrifice_anchor()) + anchor["derived"] = {**anchor["derived"], "stage_target": 1} + with pytest.raises(ValueError, match="stage_target.*drifted"): + _assert_salary_sacrifice_stage_parameters(_stage(), anchor=anchor) + + +def test_drift_assert_rejects_extra_keys_and_operations() -> None: + with pytest.raises(ValueError, match="drifted"): + _assert_salary_sacrifice_stage_parameters( + _drift(1, "undeclared_extra_key"), + anchor=load_salary_sacrifice_anchor(), + ) + stage = _stage() + extra = replace(stage, operations=(*stage.operations, stage.operations[-1])) + with pytest.raises(ValueError, match="operation order drifted"): + _assert_salary_sacrifice_stage_parameters( + extra, anchor=load_salary_sacrifice_anchor() + ) diff --git a/packages/microcosm-build/tests/test_uk_source_runtime.py b/packages/microcosm-build/tests/test_uk_source_runtime.py index 0a288ab5..d9901c77 100644 --- a/packages/microcosm-build/tests/test_uk_source_runtime.py +++ b/packages/microcosm-build/tests/test_uk_source_runtime.py @@ -119,6 +119,11 @@ def hmrc(frame: Frame) -> Frame: lcfs_consumption_transform=retained, etb_vat_transform=hmrc, etb_services_transform=retained, + cgt_incidence_clone_transform=retained, + cgt_band_donors_transform=hmrc, + hmrc_cgt_gains_spine_transform=retained, + salary_sacrifice_transform=hmrc, + student_loans_transform=retained, ) == { "frs_hmrc_retained_leaves": retained, "hmrc_spi_income": hmrc, @@ -127,6 +132,11 @@ def hmrc(frame: Frame) -> Frame: "lcfs_consumption": retained, "etb_vat": hmrc, "etb_services": retained, + "cgt_incidence_clone": retained, + "cgt_band_donors": hmrc, + "hmrc_cgt_gains_spine": retained, + "salary_sacrifice": hmrc, + "student_loans": retained, } diff --git a/packages/microcosm-build/tests/test_uk_source_stages.py b/packages/microcosm-build/tests/test_uk_source_stages.py index 8eef5ec3..c00ce208 100644 --- a/packages/microcosm-build/tests/test_uk_source_stages.py +++ b/packages/microcosm-build/tests/test_uk_source_stages.py @@ -46,6 +46,13 @@ "spi_support_channel", "hmrc_spi_income_spine", ] +E8_STAGE_NAMES = [ + "cgt_incidence_clone", + "cgt_band_donors", + "hmrc_cgt_gains_spine", + "salary_sacrifice", + "student_loans", +] UK_SOURCE_STAGE_NAMES = [ "frs_spine", *E3_STAGE_NAMES, @@ -53,6 +60,7 @@ *E5_STAGE_NAMES, *E6_STAGE_NAMES, *E7_STAGE_NAMES, + *E8_STAGE_NAMES, "frs_hmrc_retained_leaves", "hmrc_spi_income", ] @@ -62,6 +70,7 @@ *E4_STAGE_NAMES, *E6_STAGE_NAMES, *E7_STAGE_NAMES, + *E8_STAGE_NAMES, ] FROZEN_SOURCE_STAGES_SHA256 = ( "c0341af7166ae3a85a3c1164e7d9e880c4b4aec122f1a8fa90c73b46c596e1ea" @@ -123,11 +132,20 @@ def test_e6_block_sits_between_e5_and_e7(self) -> None: == E6_STAGE_NAMES ) - def test_e7_block_is_contiguous_before_certified_pair(self) -> None: + def test_e7_block_sits_between_e6_and_e8(self) -> None: + canonical = _load_json(CANONICAL_SOURCE_STAGES) + names = [stage["stage"] for stage in canonical["stages"]] + + assert ( + names[names.index("etb_services") + 1 : names.index("cgt_incidence_clone")] + == E7_STAGE_NAMES + ) + + def test_e8_block_is_contiguous_before_certified_pair(self) -> None: canonical = _load_json(CANONICAL_SOURCE_STAGES) names = [stage["stage"] for stage in canonical["stages"]] - assert names[-5:-2] == E7_STAGE_NAMES + assert names[-7:-2] == E8_STAGE_NAMES assert names[-2:] == ["frs_hmrc_retained_leaves", "hmrc_spi_income"] def test_copy_is_lockstep_with_frozen_original_except_citation_rewrites( @@ -208,7 +226,7 @@ def test_country_stage_plan_assembles_two_certified_uk_national_stages( "hmrc_spi_income", ] - def test_country_stage_plan_assembles_fourteen_stage_spine_plan(self) -> None: + def test_country_stage_plan_assembles_spine_plan(self) -> None: spec = load_country_spec("uk") implementations = {name: _identity for name in UK_SOURCE_STAGE_NAMES} plan = country_stage_plan( @@ -246,6 +264,11 @@ def test_country_stage_plan_assembles_fourteen_stage_spine_plan(self) -> None: "frs_hmrc_spine_leaves": _identity, "spi_support_channel": _identity, "hmrc_spi_income_spine": _identity, + "cgt_incidence_clone": _identity, + "cgt_band_donors": _identity, + "hmrc_cgt_gains_spine": _identity, + "salary_sacrifice": _identity, + "student_loans": _identity, "frs_hmrc_retained_leaves": _identity, "hmrc_spi_income": _identity, "hmrc_spi_income_fallback": _identity, @@ -422,6 +445,33 @@ def test_e7_outputs_and_rewrites_are_backed_by_runtime_constants(self) -> None: assert income.rewrites == UK_SPI_INCOME_SPINE_REWRITE_COLUMNS assert not (set(income.outputs) & set(income.rewrites)) + def test_e8_outputs_and_rewrites_are_backed_by_runtime_constants(self) -> None: + from microcosm.build.uk_runtime.cgt_structure import ( + HOUSEHOLD_IS_CGT_BAND_DONOR, + HOUSEHOLD_IS_CGT_CLONE, + ) + from microcosm.build.uk_runtime.salary_sacrifice import SALSAC_OUTPUT + + stages = load_country_spec("uk").sources.stage_map() + + assert stages["cgt_incidence_clone"].outputs == ( + HOUSEHOLD_IS_CGT_CLONE, + "capital_gains", + ) + assert stages["cgt_incidence_clone"].rewrites == ("capital_gains",) + assert stages["cgt_band_donors"].outputs == ( + HOUSEHOLD_IS_CGT_BAND_DONOR, + "capital_gains", + ) + assert stages["cgt_band_donors"].rewrites == ("capital_gains",) + assert stages["hmrc_cgt_gains_spine"].outputs == ("capital_gains",) + assert stages["hmrc_cgt_gains_spine"].rewrites == ("capital_gains",) + assert stages["salary_sacrifice"].outputs == ( + SALSAC_OUTPUT, + "employee_pension_contributions", + ) + assert stages["student_loans"].outputs == ("student_loan_plan",) + class TestE3ManifestLockstep: def test_e3_raw_tab_pins_match_spine_artifacts(self) -> None: @@ -557,6 +607,31 @@ def test_e3_operation_kinds_are_declared_in_order(self) -> None: "classify_hmrc_income_facts_with_reviewed_fences", "gate_distributional_effective_mass", ] + assert [op.kind for op in stages["cgt_incidence_clone"].operations] == [ + "clone_records", + "draw_capital_gains_prior_from_banded_quantiles", + ] + assert [op.kind for op in stages["cgt_band_donors"].operations] == [ + "stack_band_donor_households" + ] + assert [op.kind for op in stages["hmrc_cgt_gains_spine"].operations] == [ + "verify_pinned_cgt_ods", + "taxable_income_proxy", + "rank_preserving_allocation", + "within_band_draws", + "sub_aea_remainder", + "record_mass_conservation_receipt", + "classify_cgt_band_facts_with_reviewed_fence", + ] + assert [op.kind for op in stages["salary_sacrifice"].operations] == [ + "fit_weighted_qrf", + "convert_donors_to_target_stock", + ] + assert [op.kind for op in stages["student_loans"].operations] == [ + "assign_student_loan_plan_cohorts", + "top_up_to_stock", + "top_up_to_stock", + ] def test_engine_predictor_and_rewrite_constants_match_manifest(self) -> None: from microcosm.build.uk_runtime.etb_services import ( @@ -735,6 +810,21 @@ def test_e7_declared_seed_lockstep(self) -> None: assert stages["hmrc_spi_income_spine"].operations[2].parameters["seed"] == 42 assert stages["hmrc_spi_income_spine"].operations[3].parameters["seed"] == 43 + def test_e8_declared_seed_lockstep(self) -> None: + stages = load_country_spec("uk").sources.stage_map() + + assert stages["cgt_incidence_clone"].operations[1].parameters["seed"] == 0 + assert stages["cgt_band_donors"].operations[0].parameters["seed"] == 1 + assert ( + stages["hmrc_cgt_gains_spine"].operations[3].parameters["seed_base"] == 552 + ) + assert stages["salary_sacrifice"].operations[0].parameters["seed"] == 42 + assert stages["salary_sacrifice"].operations[1].parameters["seed"] == 2024 + assert [ + operation.parameters["seed"] + for operation in stages["student_loans"].operations[1:] + ] == [42, 42] + def test_full_uk_source_stage_plan_compiles_with_e4_stages(self) -> None: spec = load_country_spec("uk") implementations = {name: _identity for name in UK_SOURCE_STAGE_NAMES} diff --git a/packages/microcosm-build/tests/test_uk_student_loans.py b/packages/microcosm-build/tests/test_uk_student_loans.py new file mode 100644 index 00000000..b73c9d22 --- /dev/null +++ b/packages/microcosm-build/tests/test_uk_student_loans.py @@ -0,0 +1,253 @@ +from __future__ import annotations + +from dataclasses import replace +from functools import lru_cache + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.country_spec import load_country_spec +from microcosm.build.source_manifest import SourceOperationSpec +from microcosm.build.uk_runtime.national_frame import uk_national_frame +from microcosm.build.uk_runtime.student_loans import ( + PLAN_PRIORITY, + STUDENT_LOAN_ENUM_DOMAIN, + _assert_student_loans_stage_parameters, + assign_student_loan_plans, + load_slc_liable_stocks, +) + + +def _stocks(*, plan_2: float, plan_5: float, year: int = 2025): + return { + "plans": { + "plan_2": {"liable": {str(year): plan_2}}, + "plan_5": {"liable": {str(year): plan_5}}, + } + } + + +def _frame( + *, + ages, + repayments, + regions=None, + education=None, + weights=None, +): + n = len(ages) + ids = np.arange(1, n + 1, dtype="int64") + regions = regions or ["LONDON"] * n + education = education or ["TERTIARY"] * n + return uk_national_frame( + person=pd.DataFrame( + { + "person_id": ids, + "person_benunit_id": ids, + "person_household_id": ids, + "age": ages, + "student_loan_repayments": repayments, + "highest_education": education, + } + ), + benunit=pd.DataFrame({"benunit_id": ids}), + household=pd.DataFrame({"household_id": ids, "region": regions}), + household_weights=np.ones(n) if weights is None else weights, + time_period="2024", + ) + + +@lru_cache +def _stage(): + return load_country_spec("uk").sources.stage_map()["student_loans"] + + +def _drift(operation_index: int, parameter: str): + stage = _stage() + operations = list(stage.operations) + operation = operations[operation_index] + operations[operation_index] = SourceOperationSpec( + operation.kind, + {**operation.parameters, parameter: "__drift__"}, + ) + return replace(stage, operations=tuple(operations)) + + +def test_reported_cohort_boundaries_and_country_independence() -> None: + # At 2025: these ages imply start years 2011, 2012, 2022, and 2023. + result = assign_student_loan_plans( + _frame( + ages=[32, 31, 21, 20], + repayments=[100.0] * 4, + regions=["WALES", "SCOTLAND", "NORTHERN_IRELAND", "WALES"], + ), + stocks=_stocks(plan_2=0, plan_5=0), + year=2025, + ) + + assert result.frame.table("person")["student_loan_plan"].tolist() == [ + "PLAN_1", + "PLAN_2", + "PLAN_2", + "PLAN_5", + ] + + +def test_topups_apply_eligibility_gates_and_plan5_priority() -> None: + result = assign_student_loan_plans( + _frame( + ages=[20, 20, 20, 31, 31, 40], + repayments=[0.0] * 6, + regions=["LONDON", "WALES", "LONDON", "LONDON", "LONDON", "LONDON"], + education=["TERTIARY", "TERTIARY", "GCSE", "TERTIARY", "GCSE", "TERTIARY"], + ), + stocks=_stocks(plan_2=1, plan_5=1), + year=2025, + ) + plans = result.frame.table("person")["student_loan_plan"].tolist() + + assert plans == ["PLAN_5", "NONE", "NONE", "PLAN_2", "NONE", "NONE"] + assert tuple(result.plans) == PLAN_PRIORITY + assert "PLAN_4" not in plans + + +def test_rate_rule_receipt_uses_weighted_shortfall() -> None: + result = assign_student_loan_plans( + _frame( + ages=[31, 31], + repayments=[0.0, 0.0], + weights=[2.0, 2.0], + ), + stocks=_stocks(plan_2=2, plan_5=0), + year=2025, + ) + receipt = result.plans["PLAN_2"] + + assert receipt.shortfall == 2.0 + assert receipt.eligible_mass == 4.0 + assert receipt.rate == 0.5 + assert receipt.final_england_count == receipt.topped_up_mass + + +def test_calibration_year_changes_cohort_assignment() -> None: + frame = _frame(ages=[31], repayments=[100.0]) + + at_2025 = assign_student_loan_plans( + frame, stocks=_stocks(plan_2=0, plan_5=0), year=2025 + ) + at_2036 = assign_student_loan_plans( + frame, stocks=_stocks(plan_2=0, plan_5=0, year=2036), year=2036 + ) + + assert at_2025.frame.table("person").student_loan_plan.iloc[0] == "PLAN_2" + assert at_2036.frame.table("person").student_loan_plan.iloc[0] == "PLAN_5" + assert at_2036.calibration_year == 2036 + + +def test_committed_stocks_pin_full_2025_to_2030_series() -> None: + stocks = load_slc_liable_stocks()["plans"] + + assert stocks["plan_2"]["liable"] == { + "2025": 8_940_000, + "2026": 9_710_000, + "2027": 10_360_000, + "2028": 10_615_000, + "2029": 10_600_000, + "2030": 10_525_000, + } + assert stocks["plan_5"]["above_threshold"]["2030"] == 1_235_000 + + +def test_values_match_policyengine_enum_when_available() -> None: + module = pytest.importorskip( + "policyengine_uk.variables.gov.hmrc.student_loans.student_loan_plan" + ) + engine_names = set(module.StudentLoanPlan.__members__) + + assert set(STUDENT_LOAN_ENUM_DOMAIN) <= engine_names + assert "PLAN_4" in engine_names + assert "PLAN_4" not in STUDENT_LOAN_ENUM_DOMAIN + + +@pytest.mark.parametrize( + "operation_index,parameter", + [ + *[ + (0, name) + for name in ( + "year_rule", + "start_year_formula", + "reported_repayment_test", + "reported_country_gate", + "plan_1_before", + "plan_5_from", + "enum_domain", + "plan_4_imputation", + ) + ], + *[ + (1, name) + for name in ( + "priority", + "resource", + "stock_series", + "year_rule", + "age_min", + "age_max", + "cohort_start_min", + "eligible_region_exclusions", + "highest_education", + "seed", + "salt", + ) + ], + *[ + (2, name) + for name in ( + "priority", + "resource", + "stock_series", + "year_rule", + "age_min", + "age_max", + "cohort_start_min", + "cohort_start_max_exclusive", + "eligible_region_exclusions", + "highest_education", + "seed", + "salt", + ) + ], + ], +) +def test_manifest_drift_assert_covers_every_reviewed_parameter( + operation_index: int, parameter: str +) -> None: + with pytest.raises((ValueError, KeyError), match="drifted|priority"): + _assert_student_loans_stage_parameters( + _drift(operation_index, parameter), + stocks=load_slc_liable_stocks(), + year=2025, + ) + + +def test_stock_drift_assert_rejects_2025_change() -> None: + stocks = _stocks(plan_2=1, plan_5=10_000) + with pytest.raises(ValueError, match="PLAN_2.*drifted"): + _assert_student_loans_stage_parameters(_stage(), stocks=stocks, year=2025) + + +def test_drift_assert_rejects_extra_keys_and_operations() -> None: + with pytest.raises(ValueError, match="drifted"): + _assert_student_loans_stage_parameters( + _drift(2, "undeclared_extra_key"), + stocks=load_slc_liable_stocks(), + year=2025, + ) + stage = _stage() + extra = replace(stage, operations=(*stage.operations, stage.operations[-1])) + with pytest.raises(ValueError, match="operation order drifted"): + _assert_student_loans_stage_parameters( + extra, stocks=load_slc_liable_stocks(), year=2025 + ) diff --git a/packages/microcosm-build/tests/test_uk_take_up_gate.py b/packages/microcosm-build/tests/test_uk_take_up_gate.py index 50c6f329..74f7dc8a 100644 --- a/packages/microcosm-build/tests/test_uk_take_up_gate.py +++ b/packages/microcosm-build/tests/test_uk_take_up_gate.py @@ -115,6 +115,30 @@ def test_brma_enum_domain_binding_fails_off_domain() -> None: assert "OFF_DOMAIN" in result.failures[0] +def test_student_loan_enum_domain_binding_resolves_person_column() -> None: + frame = _frame() + frame.table("person")["student_loan_plan"] = ["NONE"] * 9 + ["PLAN_4"] + binding = UK_GATE_REGISTRY["enum_domain"] + + result = binding.evaluate( + EvidenceContext( + frame=frame, + artifacts={ + "student_loan_plan_enum_domain": ( + "NONE", + "PLAN_1", + "PLAN_2", + "PLAN_5", + ) + }, + ), + {"columns": ("student_loan_plan",)}, + ) + + assert result.passed is False + assert "PLAN_4" in result.failures[0] + + def test_gate_registry_vocabulary_round_trip() -> None: assert "take_up_signal" in UK_GATE_REGISTRY assert "enum_domain" in UK_GATE_REGISTRY diff --git a/packages/microcosm-build/tests/test_us_fiscal_refresh_memory.py b/packages/microcosm-build/tests/test_us_fiscal_refresh_memory.py index 5f4ff014..dad887cd 100644 --- a/packages/microcosm-build/tests/test_us_fiscal_refresh_memory.py +++ b/packages/microcosm-build/tests/test_us_fiscal_refresh_memory.py @@ -234,15 +234,17 @@ def _variable_module_count() -> int: ) -@requires_us -def test_reform_materialization_builds_one_engine_system_per_family() -> None: - """The pre-#456 builder rebuilt the full tax-benefit system every batch. - - Each build permanently registers one set of variable modules in - ``sys.modules`` (measured: ~5,600 entries, ~55-60 MB RSS floor, immune to - gc). Three batches per family must therefore add ~one set, not three: - against the old builder this assertion sees three sets per family and - fails. +def _isolated_family_measurement() -> None: + """Measure the per-family module registrations; asserts on failure. + + Runs the pre-#456 leak canary end-to-end. Must execute in a fresh + interpreter: variable-module names are keyed by ``id(system)``, and in a + warm suite process CPython can hand a fresh system a dead prior system's + recycled address, re-registering its module set under already-existing + names — the measured delta then reads 0 and the liveness assert fails + spuriously (the nondeterministic main-CI red first seen on the merge run + for the FRS 2024-25 retarget). A virgin process has no dead systems to + recycle, so the count deltas measure real registrations. """ builder = _load_builder_module() from policyengine_us import CountryTaxBenefitSystem, Microsimulation @@ -282,6 +284,31 @@ def test_reform_materialization_builds_one_engine_system_per_family() -> None: ) +@requires_us +def test_reform_materialization_builds_one_engine_system_per_family() -> None: + """The pre-#456 builder rebuilt the full tax-benefit system every batch. + + Each build permanently registers one set of variable modules in + ``sys.modules`` (measured: ~5,600 entries, ~55-60 MB RSS floor, immune to + gc). Three batches per family must therefore add ~one set, not three: + against the old builder this assertion sees three sets per family and + fails. The measurement runs in a fresh interpreter because the module + count is only meaningful there — see ``_isolated_family_measurement``. + """ + import subprocess + + result = subprocess.run( + [sys.executable, str(Path(__file__).resolve())], + capture_output=True, + text=True, + timeout=600, + ) + assert result.returncode == 0, ( + "isolated per-family measurement failed in the fresh interpreter:\n" + f"{result.stdout}\n{result.stderr}" + ) + + @requires_us def test_reform_materialization_batching_is_bit_identical() -> None: """Sharing one reform system across batches must not change results. @@ -345,3 +372,7 @@ def alive_microsimulations() -> int: f"{alive_after - alive_before} finished batch simulations survived " "the family boundary; release_engine_simulation has regressed" ) + + +if __name__ == "__main__": + _isolated_family_measurement() diff --git a/packages/microcosm-data/src/microcosm/data/contract.py b/packages/microcosm-data/src/microcosm/data/contract.py index 3e5190c2..66612dc5 100644 --- a/packages/microcosm-data/src/microcosm/data/contract.py +++ b/packages/microcosm-data/src/microcosm/data/contract.py @@ -344,13 +344,13 @@ # fingerprint derives from the manifest digest. Editing the spec moves all # three here in the same reviewed change. _UK_GATE_BATTERY_POLICY_SHA256 = ( - "404968fba9a626d4b534dfbef87721ff9d98c5af356758b2bab49dbaf004fdc3" + "5cb072a019617ba57e392fa19578e8c1b33fcb3af0144bcf33ff82b8874357d8" ) _UK_GATE_BATTERY_GATES_MANIFEST_SHA256 = ( - "59c7808d50a9ef84d37f524779a7518b4fb4f62dc7d17eb47e4a108d830c3798" + "c5123517586a8a4eed27606cb26c6e4ccfcbe45fd657e0d95162e15d49c83c85" ) _UK_GATE_BATTERY_SPEC_FINGERPRINT = ( - "bfb987361037e6475ea9906894cb16e5b3cd0ff515096bd55d4918dfd7331d2c" + "23cf63b64cdf06e186d12956043056ab8cc0f49cb44e984a5b0e25f1487cd731" ) #: Spec entry id -> the legacy gate name whose observable detail checks #: apply unchanged (the battery re-keys the report by entry id; the gate @@ -368,6 +368,7 @@ "uk_export_surface": "export_surface", "uk_take_up_signal": "take_up_signal", "uk_brma_enum_domain": "enum_domain", + "uk_student_loan_plan_enum_domain": "enum_domain", "uk_target_surface": "target_surface", "uk_target_fit": "target_fit", "uk_input_mass_parity": "input_mass_parity", @@ -403,6 +404,7 @@ "uk_export_surface": ("export_surface", "terminal"), "uk_take_up_signal": ("take_up_signal", "terminal"), "uk_brma_enum_domain": ("enum_domain", "terminal"), + "uk_student_loan_plan_enum_domain": ("enum_domain", "terminal"), "uk_calibration_reference_coverage": ( "calibration_reference_coverage", "terminal", diff --git a/packages/microcosm-data/tests/test_contract.py b/packages/microcosm-data/tests/test_contract.py index 8daf0078..46b14344 100644 --- a/packages/microcosm-data/tests/test_contract.py +++ b/packages/microcosm-data/tests/test_contract.py @@ -145,13 +145,13 @@ def _trusted_terminal_gate_signing_key(monkeypatch) -> None: UK_GATE_BATTERY_PRODUCER = "microcosm.build.gate_battery" UK_GATE_BATTERY_SIGNING_KEY_ENV = "MICROCOSM_UK_TERMINAL_GATE_SIGNING_KEY" UK_GATE_BATTERY_POLICY_SHA256 = ( - "404968fba9a626d4b534dfbef87721ff9d98c5af356758b2bab49dbaf004fdc3" + "5cb072a019617ba57e392fa19578e8c1b33fcb3af0144bcf33ff82b8874357d8" ) UK_GATE_BATTERY_GATES_MANIFEST_SHA256 = ( - "59c7808d50a9ef84d37f524779a7518b4fb4f62dc7d17eb47e4a108d830c3798" + "c5123517586a8a4eed27606cb26c6e4ccfcbe45fd657e0d95162e15d49c83c85" ) UK_GATE_BATTERY_SPEC_FINGERPRINT = ( - "bfb987361037e6475ea9906894cb16e5b3cd0ff515096bd55d4918dfd7331d2c" + "23cf63b64cdf06e186d12956043056ab8cc0f49cb44e984a5b0e25f1487cd731" ) UK_GATE_BATTERY_DEGENERATE_EVIDENCE_SHA256 = ( "d0d024043132fa07c378c393dbe2b24fe99bf19e876bcc39997d2c80cc9bd4f6" @@ -201,6 +201,11 @@ def _trusted_terminal_gate_signing_key(monkeypatch) -> None: "uk_export_surface": ("export_surface", "terminal", "export_surface"), "uk_take_up_signal": ("take_up_signal", "terminal", "take_up_signal"), "uk_brma_enum_domain": ("enum_domain", "terminal", "enum_domain"), + "uk_student_loan_plan_enum_domain": ( + "enum_domain", + "terminal", + "enum_domain", + ), "uk_calibration_reference_coverage": ( "calibration_reference_coverage", "terminal", diff --git a/tools/build_uk_frs_spine.py b/tools/build_uk_frs_spine.py index 5b4822a9..59921589 100644 --- a/tools/build_uk_frs_spine.py +++ b/tools/build_uk_frs_spine.py @@ -32,6 +32,11 @@ sha256_argument, write_error_receipt, ) +from microcosm.build.uk_runtime.cgt_imputation import uk_cgt_spine_stage_transform +from microcosm.build.uk_runtime.cgt_structure import ( + UKCGTBandDonorStageTransform, + UKCGTIncidenceCloneStageTransform, +) from microcosm.build.uk_runtime.etb_services import UKETBServicesStageTransform from microcosm.build.uk_runtime.etb_vat import UKETBVATStageTransform from microcosm.build.uk_runtime.frs_brma import UKFRSBRMAStageTransform @@ -69,11 +74,13 @@ from microcosm.build.uk_runtime.regional_uprating import ( UKRegionalPropertyUpratingStageTransform, ) +from microcosm.build.uk_runtime.salary_sacrifice import UKSalarySacrificeStageTransform from microcosm.build.uk_runtime.spi_spine import ( UKFRSHMRCSpineLeavesStageTransform, UKSPIIncomeSpineStageTransform, UKSPISupportChannelStageTransform, ) +from microcosm.build.uk_runtime.student_loans import UKStudentLoansStageTransform from microcosm.build.uk_runtime.take_up_contract import load_uk_take_up_contract from microcosm.build.uk_runtime.was_wealth import UKWASWealthStageTransform from microcosm.frame.adapters.policyengine_uk import PolicyEngineUKEngine @@ -102,6 +109,11 @@ "frs_hmrc_spine_leaves", "spi_support_channel", "hmrc_spi_income_spine", + "cgt_incidence_clone", + "cgt_band_donors", + "hmrc_cgt_gains_spine", + "salary_sacrifice", + "student_loans", ) @@ -153,6 +165,11 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: required=True, help="Pinned local HMRC collated ODS path.", ) + parser.add_argument( + "--cgt-ods", + type=Path, + help="Pinned local HMRC Capital Gains Tax Table 3 ODS path.", + ) parser.add_argument( "--checkpoint-dir", type=Path, @@ -230,6 +247,11 @@ def _validate_args(args: argparse.Namespace) -> None: raise ValueError(f"--hmrc-ods must be an existing file: {args.hmrc_ods}") if args.hmrc_ods.suffix.lower() != ".ods": raise ValueError("--hmrc-ods must end with '.ods'.") + if args.cgt_ods is not None: + if not args.cgt_ods.is_file(): + raise ValueError(f"--cgt-ods must be an existing file: {args.cgt_ods}") + if args.cgt_ods.suffix.lower() != ".ods": + raise ValueError("--cgt-ods must end with '.ods'.") paths = { "spine_h5": args.spine_h5, "build_sidecar": args.spine_h5.with_suffix(".build.json"), @@ -382,6 +404,8 @@ def _declared_seeds(stages) -> dict[str, dict[str, int]]: for operation in stage.operations: output = operation.parameters.get("output") seed = operation.parameters.get("seed") + if seed is None: + seed = operation.parameters.get("seed_base") if isinstance(output, str) and isinstance(seed, int): stage_seeds[output] = seed elif isinstance(seed, int): @@ -405,6 +429,16 @@ def _declared_seeds(stages) -> dict[str, dict[str, int]]: stage_seeds[stage.stage] = seed elif operation.kind == "fit_weighted_qrf": stage_seeds[stage.stage] = seed + elif operation.kind == "draw_capital_gains_prior_from_banded_quantiles": + stage_seeds[str(operation.parameters["salt"])] = seed + elif operation.kind == "stack_band_donor_households": + stage_seeds["stack_band_donor_households"] = seed + elif operation.kind == "within_band_draws": + stage_seeds["within_band_draws"] = seed + elif operation.kind == "convert_donors_to_target_stock": + stage_seeds[str(operation.parameters["salt"])] = seed + elif operation.kind == "top_up_to_stock": + stage_seeds[str(operation.parameters["salt"])] = seed if stage_seeds: declared[stage.stage] = stage_seeds return declared @@ -633,6 +667,10 @@ def main(argv: list[str] | None = None) -> int: raise ValueError("UK country spec has no source stages.") stages_by_name = spec.sources.stage_map() stage_names = tuple(name for name in _STAGE_NAMES if name in stages_by_name) + if "hmrc_cgt_gains_spine" in stage_names and args.cgt_ods is None: + raise ValueError( + "--cgt-ods is required when hmrc_cgt_gains_spine is scheduled." + ) if "was_wealth" in stage_names and args.was_tab is None: raise ValueError( "--was-tab is required when the was_wealth stage is scheduled." @@ -780,6 +818,28 @@ def main(argv: list[str] | None = None) -> int: sample_fraction=args.sample_fraction, ) implementations["hmrc_spi_income_spine"] = hmrc_spine_transform + if "cgt_incidence_clone" in stage_names: + implementations["cgt_incidence_clone"] = UKCGTIncidenceCloneStageTransform( + stage=stages_by_name["cgt_incidence_clone"] + ) + if "cgt_band_donors" in stage_names: + implementations["cgt_band_donors"] = UKCGTBandDonorStageTransform( + stage=stages_by_name["cgt_band_donors"] + ) + if "hmrc_cgt_gains_spine" in stage_names: + implementations["hmrc_cgt_gains_spine"] = uk_cgt_spine_stage_transform( + stages_by_name["hmrc_cgt_gains_spine"], + args.cgt_ods, + ) + if "salary_sacrifice" in stage_names: + implementations["salary_sacrifice"] = UKSalarySacrificeStageTransform( + stage=stages_by_name["salary_sacrifice"] + ) + if "student_loans" in stage_names: + implementations["student_loans"] = UKStudentLoansStageTransform( + stage=stages_by_name["student_loans"], + calibration_year=frs_release.calibration_year, + ) plan = country_stage_plan( spec, implementations, @@ -824,6 +884,23 @@ def main(argv: list[str] | None = None) -> int: frs_vintage=frs_release.vintage, sampling=sampling, ) + # E8 executed-effect receipts (#730/#684 two-arm rule, arm 2): the + # clone/donor/salsac/student-loan transforms record their receipts on + # last_result; persist them beside the declared seeds so the sidecar + # carries evidence that every declared parameter shaped the output. + e8_stage_evidence: dict[str, object] = {} + for e8_stage_name in ( + "cgt_incidence_clone", + "cgt_band_donors", + "salary_sacrifice", + "student_loans", + ): + e8_implementation = implementations.get(e8_stage_name) + e8_last_result = getattr(e8_implementation, "last_result", None) + if e8_last_result is not None: + e8_stage_evidence[e8_stage_name] = e8_last_result.evidence() + if e8_stage_evidence: + sidecar["stage_evidence"] = e8_stage_evidence atomic_write_json(sidecar_path, sidecar) append_phase(state, "build_sidecar_written") if args.emit_nonzero_shares is not None: diff --git a/tools/build_uk_release_input_coverage_manifest.py b/tools/build_uk_release_input_coverage_manifest.py index f0b17562..2f964172 100644 --- a/tools/build_uk_release_input_coverage_manifest.py +++ b/tools/build_uk_release_input_coverage_manifest.py @@ -790,6 +790,25 @@ def build_manifest( }, "effective_mass_coverage": EFFECTIVE_MASS_COVERAGE, "family_coverage": { + "cgt_incidence_clone": _source_stage_family_coverage_contract( + stage_name="cgt_incidence_clone", + candidate_source=candidate_source, + ), + "cgt_band_donors": _source_stage_family_coverage_contract( + stage_name="cgt_band_donors", + candidate_source=candidate_source, + ), + "hmrc_cgt_gains_spine": _cgt_spine_family_coverage_contract( + candidate_source=candidate_source, + ), + "salary_sacrifice": _source_stage_family_coverage_contract( + stage_name="salary_sacrifice", + candidate_source=candidate_source, + ), + "student_loans": _source_stage_family_coverage_contract( + stage_name="student_loans", + candidate_source=candidate_source, + ), "hmrc_cgt_gains": _cgt_family_coverage_contract( candidate_source=candidate_source, ), @@ -959,6 +978,32 @@ def _source_stage_family_coverage_contract( f"{SOURCE_STAGES_PATH}: expected exactly one {stage_name!r} stage." ) stage = matches[0] + operations = [ + operation + for operation in stage.get("operations", []) + if isinstance(operation, dict) + ] + declared_reasons = [ + str(operation["reason"]) + for operation in operations + if isinstance(operation.get("reason"), str) and operation.get("reason") + ] + required_mass_change_reason = ( + declared_reasons[-1] + if declared_reasons + else ( + "E5 source-stage transform preserves household rows and typed " + "household weights; total household mass is conserved." + ) + ) + mass_change_semantics = ( + "mass_increasing_support" + if any( + operation.get("kind") == "stack_band_donor_households" + for operation in operations + ) + else "mass_conserving" + ) return { "status": "required_at_build", "stage": stage_name, @@ -971,10 +1016,91 @@ def _source_stage_family_coverage_contract( "source": str(stage.get("source", "")), }, "output_weight_kind": "importance", - "required_mass_change_reason": ( - "E5 source-stage transform preserves household rows and typed " - "household weights; total household mass is conserved." + "required_mass_change_reason": required_mass_change_reason, + "mass_change_semantics": mass_change_semantics, + "outputs": list(stage.get("outputs", [])), + "rewrites": list(stage.get("rewrites", [])), + "effective_mass_requirements": {}, + } + + +def _cgt_spine_family_coverage_contract( + *, + candidate_source: dict[str, Any], +) -> dict[str, Any]: + """Emit the canonical spine-side CGT family without touching the frozen path.""" + + payload = _load(SOURCE_STAGES_PATH) + stages = payload.get("stages") + if not isinstance(stages, list): + raise ValueError(f"{SOURCE_STAGES_PATH}: expected source stages list.") + matches = [ + stage + for stage in stages + if isinstance(stage, dict) and stage.get("stage") == "hmrc_cgt_gains_spine" + ] + if len(matches) != 1: + raise ValueError( + f"{SOURCE_STAGES_PATH}: expected exactly one hmrc_cgt_gains_spine stage." + ) + stage = matches[0] + artifacts = { + artifact["role"]: artifact + for artifact in stage.get("artifacts", []) + if isinstance(artifact, dict) and isinstance(artifact.get("role"), str) + } + operations = { + operation["kind"]: operation + for operation in stage.get("operations", []) + if isinstance(operation, dict) and isinstance(operation.get("kind"), str) + } + required_artifacts = {"cgt_published_fact_surface", "policy_parameters"} + required_operations = { + "verify_pinned_cgt_ods", + "taxable_income_proxy", + "rank_preserving_allocation", + "within_band_draws", + "sub_aea_remainder", + "record_mass_conservation_receipt", + "classify_cgt_band_facts_with_reviewed_fence", + } + missing_artifacts = sorted(required_artifacts - set(artifacts)) + missing_operations = sorted(required_operations - set(operations)) + if missing_artifacts or missing_operations: + raise ValueError( + f"{SOURCE_STAGES_PATH}: incomplete spine CGT family contract; " + f"missing_artifacts={missing_artifacts}, " + f"missing_operations={missing_operations}." + ) + surface = artifacts["cgt_published_fact_surface"] + verify = operations["verify_pinned_cgt_ods"] + fence = operations["classify_cgt_band_facts_with_reviewed_fence"] + if verify.get("artifact_role") != "cgt_published_fact_surface": + raise ValueError("Spine CGT verification must bind its distinct ODS role.") + if not bool(verify.get("require_before_source_read")): + raise ValueError("Spine CGT ODS must be verified before source read.") + if bool(fence.get("calibration_permitted", True)): + raise ValueError("Spine CGT band facts must remain fenced from calibration.") + if str(surface.get("sha256", "")) == "" or int(surface.get("size_bytes", 0)) <= 0: + raise ValueError("Spine CGT surface must pin sha256 and size_bytes.") + return { + "status": "required_at_build", + "stage": "hmrc_cgt_gains_spine", + "source_manifest": SOURCE_STAGES_PATH.name, + "source_manifest_sha256": _sha256(SOURCE_STAGES_PATH), + "base_candidate_sha256": str(candidate_source["sha256"]), + "base_candidate_tier": validate_uk_release_tier(candidate_source["tier"]), + "source_vintages": { + "hmrc_surface": str(surface["vintage"]), + "mapped_build_period": str(surface["mapped_build_period"]), + }, + "output_weight_kind": "importance", + "required_mass_change_reason": str( + operations["record_mass_conservation_receipt"]["reason"] ), + "calibration_permitted": bool(fence["calibration_permitted"]), + "fact_fence_id": str(fence["fact_fence_id"]), + "fenced_fact_count": int(fence["fenced_fact_count"]), "outputs": list(stage.get("outputs", [])), "rewrites": list(stage.get("rewrites", [])), "effective_mass_requirements": {}, @@ -1174,6 +1300,25 @@ def main() -> int: else: known_gaps = _load(KNOWN_GAPS_PATH) manifest = build_manifest(reference=reference, known_gaps_payload=known_gaps) + generic_fallback_reason = ( + "E5 source-stage transform preserves household rows and typed " + "household weights; total household mass is conserved." + ) + declared_reasons: dict[str, str] = {} + for family_name, family in manifest.get("family_coverage", {}).items(): + reason = str(family.get("required_mass_change_reason", "")).strip() + if not reason or reason == generic_fallback_reason: + # The pre-E8 families share the generic fallback (standing + # follow-up); every stage-declared reason must be unique so a + # receipt identifies exactly one family. + continue + if reason in declared_reasons: + raise ValueError( + f"family_coverage reasons must be unique receipt identities: " + f"{family_name!r} and {declared_reasons[reason]!r} share " + f"{reason!r}." + ) + declared_reasons[reason] = family_name _write_or_check(MANIFEST_PATH, manifest, check=args.check) action = "current" if args.check else "wrote" print( diff --git a/tools/verify_uk_identity_stability.py b/tools/verify_uk_identity_stability.py index 68b428ad..3dfab742 100644 --- a/tools/verify_uk_identity_stability.py +++ b/tools/verify_uk_identity_stability.py @@ -496,11 +496,247 @@ def recompute(person_t, benunit_t, household_t) -> dict[str, pd.DataFrame]: } +def e8_identity_receipt( + frame, + *, + permutation_seed: int, +) -> dict[str, object]: + """Receipt E8 deterministic layers under row permutation by entity id. + + Covered: (1) the clone-pair structure — the non-donor population splits + into equal-count original/clone halves whose paired household weights + agree to the exact-total correction tolerance and whose half-masses + match; (2) the CGT band-donor selection recomputed from the committed + resources over id-sorted candidates in original and permuted row order + (set equality with the flagged donors, 30 donors per band, band-exact + stored weights and carrier gains); (3) the student-loan plan column + recomputed in full (identity-keyed top-ups at the release calibration + year) in original and permuted row order against the stored column. + The A&S prior amounts (overwritten by the Table 3 redraw except the + sub-AEA remainder), the redraw's seeded within-band draws (covered by + the merged #560 embedded published-surface tests), and the + salary-sacrifice QRF and conversion (the pre-conversion state is + consumed by the stage) are covered by twin-build determinism. + """ + + from microcosm.build.uk_runtime.cgt_structure import ( + DONOR_SEED, + DONORS_PER_BAND, + HOUSEHOLD_IS_CGT_BAND_DONOR, + HOUSEHOLD_IS_CGT_CLONE, + _component_sum_income, + _incidence_propensity, + _oldest_adult_indices, + _retained_size_bands, + load_advani_summers_distribution, + load_hmrc_cgt_size_bands, + ) + from microcosm.build.uk_runtime.frs_release import load_uk_frs_release + from microcosm.build.uk_runtime.rowwise_geography import id_multiplier_for_values + from microcosm.build.uk_runtime.student_loans import ( + assign_student_loan_plans, + load_slc_liable_stocks, + ) + + problems: dict[str, object] = {} + person = frame.table("person") + benunit = frame.table("benunit") + household = frame.table("household").copy() + household["household_weight"] = frame.weights_for("household").values + + # (1) Clone-pair structure on the non-donor population. + donor_mask = household[HOUSEHOLD_IS_CGT_BAND_DONOR].astype(bool) + non_donor = household.loc[~donor_mask] + originals = non_donor.loc[ + ~non_donor[HOUSEHOLD_IS_CGT_CLONE].astype(bool) + ].sort_values("household_id") + clones = non_donor.loc[non_donor[HOUSEHOLD_IS_CGT_CLONE].astype(bool)].sort_values( + "household_id" + ) + if len(originals) != len(clones): + problems["clone_half_counts"] = [len(originals), len(clones)] + else: + left = originals["household_weight"].to_numpy(dtype=float) + right = clones["household_weight"].to_numpy(dtype=float) + if not np.allclose(left, right, rtol=1e-12, atol=1e-6): + problems["clone_pair_weights"] = int( + (~np.isclose(left, right, rtol=1e-12, atol=1e-6)).sum() + ) + if not np.isclose(left.sum(), right.sum(), rtol=1e-12, atol=1e-6): + problems["clone_half_masses"] = [float(left.sum()), float(right.sum())] + + # (2) Band-donor selection recomputed from the committed resources. + distribution = load_advani_summers_distribution() + bands = _retained_size_bands(load_hmrc_cgt_size_bands()) + non_donor_ids = set(non_donor["household_id"].tolist()) + nd_person = person.loc[ + person["person_household_id"].isin(non_donor_ids) + ].reset_index(drop=True) + nd_benunit = benunit.loc[ + benunit["benunit_id"].isin(set(nd_person["person_benunit_id"].tolist())) + ].reset_index(drop=True) + nd_household = non_donor.reset_index(drop=True) + multiplier = id_multiplier_for_values( + nd_person["person_id"], + nd_person["person_household_id"], + nd_person["person_benunit_id"], + nd_benunit["benunit_id"], + nd_household["household_id"], + ) + + def select_donors(person_t: pd.DataFrame) -> np.ndarray: + carriers = _oldest_adult_indices(person_t, household_ids=non_donor_ids) + candidates = person_t.loc[carriers].copy() + candidates["_income"] = _component_sum_income(candidates) + candidates["_propensity"] = _incidence_propensity( + candidates["_income"].to_numpy(dtype=float), distribution=distribution + ) + candidates = candidates.sort_values("person_household_id", kind="stable") + propensities = candidates["_propensity"].to_numpy(dtype=float) + rng = np.random.default_rng(DONOR_SEED) + return rng.choice( + candidates["person_household_id"].to_numpy(), + size=DONORS_PER_BAND * len(bands), + replace=False, + p=propensities / propensities.sum(), + ) + + selected = select_donors(nd_person) + permuted_rng = np.random.default_rng(permutation_seed) + selected_permuted = select_donors( + nd_person.iloc[permuted_rng.permutation(len(nd_person))].reset_index(drop=True) + ) + if selected.tolist() != selected_permuted.tolist(): + problems["donor_selection_permutation"] = True + stored_donors = household.loc[donor_mask] + stored_source_ids = set( + (stored_donors["household_id"].astype("int64") - multiplier).tolist() + ) + if stored_source_ids != set(int(value) for value in selected): + problems["donor_selection_stored"] = { + "missing": len(stored_source_ids - set(int(v) for v in selected)), + "extra": len(set(int(v) for v in selected) - stored_source_ids), + } + taxpayers = np.asarray([band["taxpayers"] for band in bands], dtype=float) + means = np.asarray([band["mean_gain"] for band in bands], dtype=float) + band_by_source = { + int(source_id): position // DONORS_PER_BAND + for position, source_id in enumerate(selected) + } + donor_band = ( + (stored_donors["household_id"].astype("int64") - multiplier) + .map(band_by_source) + .to_numpy() + ) + if pd.isna(donor_band).any(): + problems["donor_band_mapping"] = True + else: + donor_band = donor_band.astype(int) + counts = np.bincount(donor_band, minlength=len(bands)) + if not (counts == DONORS_PER_BAND).all(): + problems["donors_per_band"] = counts.tolist() + expected_weights = taxpayers[donor_band] / DONORS_PER_BAND + stored_weights = stored_donors["household_weight"].to_numpy(dtype=float) + if not np.allclose(stored_weights, expected_weights, rtol=1e-12, atol=0.0): + problems["donor_stored_weights"] = True + donor_person = person.loc[ + person["person_household_id"].isin(set(stored_donors["household_id"])) + ] + carrier_rows = _oldest_adult_indices( + donor_person, household_ids=set(stored_donors["household_id"]) + ) + carrier_gain = ( + donor_person.loc[carrier_rows] + .set_index("person_household_id")["capital_gains"] + .reindex(stored_donors["household_id"].to_numpy()) + .to_numpy(dtype=float) + ) + expected_gains = means[donor_band] + # The Table 3 redraw runs after the stack and moves carrier amounts + # within its own gain bands, so band means are not asserted against + # the stored carrier gains bitwise; presence and positivity are. + if not (np.isfinite(carrier_gain) & (carrier_gain > 0.0)).all(): + problems["donor_carrier_gains"] = True + del expected_gains + + # (3) Student-loan plan recomputed in full. + stocks = load_slc_liable_stocks() + year = load_uk_frs_release().calibration_year + recomputed = assign_student_loan_plans(frame, stocks=stocks, year=year) + stored_plan = person.set_index("person_id")["student_loan_plan"] + recomputed_plan = ( + recomputed.frame.table("person") + .set_index("person_id")["student_loan_plan"] + .reindex(stored_plan.index) + ) + plan_matches_store = bool(stored_plan.equals(recomputed_plan)) + permuted_result = assign_student_loan_plans( + _reverse_rows(frame), stocks=stocks, year=year + ) + permuted_plan = ( + permuted_result.frame.table("person") + .set_index("person_id")["student_loan_plan"] + .reindex(stored_plan.index) + ) + plan_permutation_stable = bool(recomputed_plan.equals(permuted_plan)) + if not plan_matches_store: + problems["student_loan_plan_stored"] = True + if not plan_permutation_stable: + problems["student_loan_plan_permutation"] = True + + structural_ok = not problems + return { + "check": "uk_e8_identity_stability", + "permutation_seed": permutation_seed, + "identical_under_permutation": bool( + "donor_selection_permutation" not in problems + and "student_loan_plan_permutation" not in problems + ), + "permutation_mismatches": { + key: value + for key, value in problems.items() + if key.endswith("_permutation") + }, + "matches_stored_columns": bool( + structural_ok + or not any(not key.endswith("_permutation") for key in problems) + ), + "stored_column_mismatches": { + key: value + for key, value in problems.items() + if not key.endswith("_permutation") + }, + "tolerance_policy": ( + "clone-pair weights and half-masses: rtol 1e-12 / atol 1e-6 " + "(the exact-total correction may move single weights by bit " + "corrections); donor stored weights: rtol 1e-12 bitwise-class " + "against published band taxpayers / 30; donor selection and " + "student_loan_plan: exact equality" + ), + "columns_by_entity": { + "household": [ + HOUSEHOLD_IS_CGT_CLONE, + HOUSEHOLD_IS_CGT_BAND_DONOR, + "household_weight", + ], + "person": ["student_loan_plan", "capital_gains"], + }, + "qrf_draw_columns_scope": ( + "excluded: the A&S prior amounts (overwritten by the Table 3 " + "redraw except the sub-AEA remainder), the redraw's seeded " + "within-band draws (the merged #560 embedded published-surface " + "tests cover the amounts logic), and the salary-sacrifice QRF " + "and conversion (the pre-conversion column state is consumed " + "by the stage) are covered by twin-build determinism" + ), + } + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--input-h5", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) - parser.add_argument("--check", choices=("e4", "e5", "e6"), default="e4") + parser.add_argument("--check", choices=("e4", "e5", "e6", "e8"), default="e4") parser.add_argument("--permutation-seed", type=int, default=123) args = parser.parse_args() @@ -536,7 +772,7 @@ def main() -> int: ok = bool( receipt["identical_under_permutation"] and receipt["matches_stored_columns"] ) - else: + elif args.check == "e6": receipt = e6_identity_receipt( frame, permutation_seed=args.permutation_seed, @@ -544,6 +780,14 @@ def main() -> int: ok = bool( receipt["identical_under_permutation"] and receipt["matches_stored_columns"] ) + else: + receipt = e8_identity_receipt( + frame, + permutation_seed=args.permutation_seed, + ) + ok = bool( + receipt["identical_under_permutation"] and receipt["matches_stored_columns"] + ) receipt["input_h5"] = str(args.input_h5) args.output.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n") print(