Port the CGT structure, salary sacrifice, and student loan stages onto the UK spine at FRS 2024-25

changelog.d/684-uk-e8-cgt-salsac-student-loans.added.md

@@ -0,0 +1 @@
+Add UK spine stages for capital-gains structure and redraws, salary-sacrifice support, and student-loan plan cohorts.

changelog.d/740-us-fiscal-memory-canary.fixed.md

@@ -0,0 +1 @@
+Run the US fiscal-refresh per-family module-leak measurement in a fresh interpreter: variable-module names are keyed by id(system), so a warm suite process can recycle a dead system's address and re-register its module set under existing names, spuriously zeroing the leak canary (the nondeterministic main-CI red first seen on the FRS 2024-25 retarget merge run).

packages/microcosm-build/src/microcosm/build/country_spec.py

@@ -1644,11 +1644,12 @@ def country_stage_plan(
             f"Country {spec.country!r} declares no source_stages.json; there "
             "is no stage plan to assemble."
         )
-    declared: list[tuple[str, DonorSpec | None, tuple[str, ...]]] = [
+    declared: list[tuple[str, DonorSpec | None, tuple[str, ...], tuple[str, ...]]] = [
         (
             stage.stage,
             DonorSpec(survey=stage.survey, source=stage.source, notes=stage.notes),
             stage.outputs,
+            stage.rewrites,
         )
         for stage in spec.sources.stages
     ]
@@ -1663,9 +1664,10 @@ def country_stage_plan(
                     notes=spine.assignment_source.notes,
                 ),
                 (spine.code_column,),
+                (),
             )
         )
-    declared_names = [name for name, _, _ in declared]
+    declared_names = [name for name, _, _, _ in declared]
     selected_names: tuple[str, ...]
     if stage_names is None:
         selected_names = tuple(declared_names)
@@ -1702,16 +1704,17 @@ def country_stage_plan(
             f"{declared_names}."
         )
     selected = [
-        (name, donor, outputs)
-        for name, donor, outputs in declared
+        (name, donor, outputs, rewrites)
+        for name, donor, outputs, rewrites in declared
         if name in set(selected_names)
     ]
     return StagePlan(
         Stage(
             name=name,
             transform=implementations[name],
             produces=outputs,
+            rewrites=rewrites,
             donor=donor,
         )
-        for name, donor, outputs in selected
+        for name, donor, outputs, rewrites in selected
     )

packages/microcosm-build/src/microcosm/build/plan.py

@@ -71,6 +71,8 @@ class Stage:
             happens inside and any failure aborts the build.
         produces: Columns the stage must add (validated after the
             transform). Empty is allowed for assert/report-only stages.
+        rewrites: Declared outputs that may have an earlier canonical producer
+            and are intentionally replaced by this stage.
         consumes: Columns that must exist (on any entity) before the stage
             runs.
         donor: The donor survey, when the stage imputes. ``None`` for
@@ -80,6 +82,7 @@ class Stage:
     name: str
     transform: Callable[[Frame], Frame]
     produces: tuple[str, ...] = ()
+    rewrites: tuple[str, ...] = ()
     consumes: tuple[str, ...] = ()
     donor: DonorSpec | None = None
 
@@ -143,12 +146,15 @@ def __init__(self, stages: Iterable[Stage]) -> None:
             names.add(stage.name)
             for column in stage.produces:
                 if column in producers:
-                    raise ValueError(
-                        f"Column {column!r} is declared by two stages "
-                        f"({producers[column]!r} and {stage.name!r}); every "
-                        "column has one canonical producer."
-                    )
-                producers[column] = stage.name
+                    if column not in stage.rewrites:
+                        raise ValueError(
+                            f"Column {column!r} is declared by two stages "
+                            f"({producers[column]!r} and {stage.name!r}); every "
+                            "column has one canonical producer unless a later "
+                            "stage explicitly declares it as a rewrite."
+                        )
+                else:
+                    producers[column] = stage.name
         self._stages = materialized
 
     @property

packages/microcosm-build/src/microcosm/build/source_manifest.py

@@ -47,6 +47,7 @@
         "assign_binary_from_rate",
         "assign_binary_with_anchored_residual",
         "assign_clipped_normal",
+        "assign_student_loan_plan_cohorts",
         "assign_uniform_draw",
         "aggregate_person_to_benunit",
         "allocate_per_capita_from_cell_table",
@@ -58,7 +59,10 @@
         "bridge_donor_column_via_qrf",
         "calibrate_binary_assignment",
         "calibrate_binary_assignment_joint_targets",
+        "classify_cgt_band_facts_with_reviewed_fence",
         "classify_hmrc_income_facts_with_reviewed_fences",
+        "clone_records",
+        "convert_donors_to_target_stock",
         "convert_interest_to_structural_mortgage_inputs",
         "compute_ratio",
         "declare_income_reference_offset",
@@ -88,6 +92,7 @@
         "derive_weeks_unemployed",
         "derive_wic_claim",
         "disaggregate_aggregate_records",
+        "draw_capital_gains_prior_from_banded_quantiles",
         "fit_labor_market_models",
         "fit_tip_income_model",
         "fit_weighted_acs_rent_qrf",
@@ -119,22 +124,30 @@
         "map_coded_amounts",
         "materialize_hmrc_income_bands_fail_closed",
         "materialize_rules_engine_predictors",
+        "rank_preserving_allocation",
         "read_table",
         "read_tables",
         "read_acs_rent_donor",
         "redraw_columns_from_fitted_qrf",
+        "record_mass_conservation_receipt",
         "replace_zero_weight_spi_support",
         "retain_adjudicated_frs_hmrc_leaves",
         "sample_categorical_from_count_table",
         "replace_sentinels",
         "split_component_by_share",
+        "stack_band_donor_households",
         "stack_zero_weight_donors",
         "strict_read_private_table",
         "support_clip",
+        "sub_aea_remainder",
+        "taxable_income_proxy",
+        "top_up_to_stock",
         "uprate",
         "uprate_to_regional_reference",
         "verify_certified_candidate",
+        "verify_pinned_cgt_ods",
         "verify_pinned_hmrc_source_pair",
+        "within_band_draws",
         "zero_when_false",
     }
 )

packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json

@@ -5872,7 +5872,13 @@
       "targets": {"type": "array", "items": {"type": "string"}},
       "weights": {"type": "string"},
       "n_estimators": {"type": "integer", "minimum": 1},
-      "seed": {"type": "integer"}
+      "seed": {"type": "integer"},
+      "training_population": {"type": "string"},
+      "target_population": {"type": "string"},
+      "weight_mapping": {"type": "string"},
+      "clamp_minimum": {"type": "number"},
+      "preserve_asked_rows": {"type": "boolean"},
+      "cache": {"type": "boolean"}
      }
     },
     {
@@ -6554,6 +6560,123 @@
       }
      }
     },
+    {
+     "type": "object",
+     "additionalProperties": false,
+     "required": ["kind", "entity", "copies", "flag_column", "mass_split", "weight_kind_out", "conservation", "id_remapping", "declared_factor", "reason"],
+     "properties": {
+      "kind": {"const": "clone_records"},
+      "entity": {"type": "string"},
+      "copies": {"type": "integer", "minimum": 2},
+      "flag_column": {"type": "string"},
+      "original_flag": {"type": "boolean"},
+      "clone_flag": {"type": "boolean"},
+      "mass_split": {"type": "number"},
+      "weight_kind_out": {"type": "string"},
+      "conservation": {"type": "string"},
+      "id_remapping": {"type": "string"},
+      "declared_factor": {"type": "number"},
+      "reason": {"type": "string"}
+     }
+    },
+    {
+     "type": "object",
+     "additionalProperties": false,
+     "required": ["kind", "resource", "income_proxy_components", "allowance_subtraction", "carrier", "adult_minimum_age", "quantile_points", "spline_degree", "extrapolation", "keep_negative_draws", "seed", "salt"],
+     "properties": {
+      "kind": {"const": "draw_capital_gains_prior_from_banded_quantiles"},
+      "resource": {"type": "string"},
+      "income_proxy_components": {"type": "array", "items": {"type": "string"}},
+      "allowance_subtraction": {"type": "boolean"},
+      "carrier": {"type": "string"},
+      "adult_minimum_age": {"type": "integer"},
+      "quantile_points": {"type": "array", "items": {"type": "number"}},
+      "spline_degree": {"type": "integer"},
+      "extrapolation": {"type": "string"},
+      "keep_negative_draws": {"type": "boolean"},
+      "seed": {"type": "integer"},
+      "salt": {"type": "string"}
+     }
+    },
+    {
+     "type": "object",
+     "additionalProperties": false,
+     "required": ["kind", "size_band_resource", "incidence_resource", "minimum_band_lower", "donors_per_band", "expected_band_count", "expected_donor_count", "candidate_order", "draw", "propensity", "seed", "flag_column", "carrier", "initial_weight", "never_zero_weight", "weight_kind_out", "reason"],
+     "properties": {
+      "kind": {"const": "stack_band_donor_households"},
+      "size_band_resource": {"type": "string"},
+      "incidence_resource": {"type": "string"},
+      "minimum_band_lower": {"type": "number"},
+      "donors_per_band": {"type": "integer", "minimum": 1},
+      "expected_band_count": {"type": "integer", "minimum": 1},
+      "expected_donor_count": {"type": "integer", "minimum": 1},
+      "candidate_order": {"type": "string"},
+      "draw": {"type": "string"},
+      "propensity": {"type": "string"},
+      "seed": {"type": "integer"},
+      "flag_column": {"type": "string"},
+      "carrier": {"type": "string"},
+      "initial_weight": {"type": "string"},
+      "never_zero_weight": {"type": "boolean"},
+      "weight_kind_out": {"type": "string"},
+      "reason": {"type": "string"}
+     }
+    },
+    {
+     "type": "object",
+     "additionalProperties": false,
+     "required": ["kind", "resource", "target", "donor_pool", "rate_cap", "move", "seed", "salt", "receipt"],
+     "properties": {
+      "kind": {"const": "convert_donors_to_target_stock"},
+      "reason": {"type": "string"},
+      "resource": {"type": "string"},
+      "target": {"type": "number"},
+      "donor_pool": {"type": "string"},
+      "rate_cap": {"type": "number"},
+      "move": {"type": "string"},
+      "seed": {"type": "integer"},
+      "salt": {"type": "string"},
+      "receipt": {"type": "string"}
+     }
+    },
+    {
+     "type": "object",
+     "additionalProperties": false,
+     "required": ["kind", "year_rule", "start_year_formula", "reported_repayment_test", "reported_country_gate", "plan_1_before", "plan_5_from", "enum_domain", "plan_4_imputation"],
+     "properties": {
+      "kind": {"const": "assign_student_loan_plan_cohorts"},
+      "year_rule": {"type": "string"},
+      "start_year_formula": {"type": "string"},
+      "reported_repayment_test": {"type": "string"},
+      "reported_country_gate": {"type": "boolean"},
+      "plan_1_before": {"type": "integer"},
+      "plan_5_from": {"type": "integer"},
+      "enum_domain": {"type": "array", "items": {"type": "string"}},
+      "plan_4_imputation": {"type": "boolean"}
+     }
+    },
+    {
+     "type": "object",
+     "additionalProperties": false,
+     "required": ["kind", "plan", "priority", "resource", "stock_series", "year_rule", "age_min", "age_max", "cohort_start_min", "eligible_region_exclusions", "highest_education", "seed", "salt"],
+     "properties": {
+      "kind": {"const": "top_up_to_stock"},
+      "reason": {"type": "string"},
+      "plan": {"type": "string"},
+      "priority": {"type": "integer"},
+      "resource": {"type": "string"},
+      "stock_series": {"type": "string"},
+      "year_rule": {"type": "string"},
+      "age_min": {"type": "integer"},
+      "age_max": {"type": "integer"},
+      "cohort_start_min": {"type": "integer"},
+      "cohort_start_max_exclusive": {"type": "integer"},
+      "eligible_region_exclusions": {"type": "array", "items": {"type": "string"}},
+      "highest_education": {"type": "string"},
+      "seed": {"type": "integer"},
+      "salt": {"type": "string"}
+     }
+    },
     {
      "type": "object",
      "additionalProperties": false,
@@ -6625,6 +6748,9 @@
       },
       "format": {
        "type": "string"
+      },
+      "runtime_sha256_required": {
+       "type": "boolean"
       }
      }
     }