diff --git a/data/populations.json b/data/populations.json index 931698b..44aaec1 100644 --- a/data/populations.json +++ b/data/populations.json @@ -1,8 +1,8 @@ { "built": "2026-08-19", - "note": "Non-Urban populations exported from scorecard.db: the populace reform-validation registry (issue #20) plus the compute campaign's attached comparisons (TPC/CPSP/PWBM/CBO/JCT). Statuses and calibration relationships are verbatim; nothing here is a pass/fail grade.", + "note": "Non-Urban populations exported from scorecard.db: the populace reform-validation registry (issue #20) plus the compute campaign's attached comparisons (US: TPC/CPSP/PWBM/CBO/JCT; UK: the HMRC ready-reckoner family). Statuses and calibration relationships are verbatim; nothing here is a pass/fail grade.", "summary": { - "claims": 270, + "claims": 284, "multi_release_claims": 127, "by_source": { "az_admin": 1, @@ -55,6 +55,7 @@ "sc_admin": 1, "tpc": 10, "treasury": 1, + "uk_hmrc": 14, "ut_admin": 1, "va_admin": 1, "vt_admin": 2, @@ -64,7 +65,7 @@ "by_latest_status": { "comparable": 37, "concept_mismatch": 2, - "constructed": 231 + "constructed": 245 } }, "rows": [ @@ -20213,6 +20214,972 @@ ], "diagnosis": null }, + { + "claim_id": "23cb5f489c32e17b4b49", + "source": "uk_hmrc", + "source_column": "reckoner", + "name": "Income Tax liabilities statistics 2023-24 to 2026-27 + June 2025 tax ready reckoner", + "window": "", + "publication_title": "Income Tax liabilities statistics 2023-24 to 2026-27 + June 2025 tax ready reckoner", + "url": "https://www.gov.uk/government/statistics/income-tax-liabilities-statistics-tax-year-2023-to-2024-to-tax-year-2026-to-2027", + "metric": "revenue_change", + "unit_concept": "gbp", + "value_kind": "gbp", + "period": 2027, + "time_basis": "fiscal_year", + "period_start": null, + "period_end": null, + "geography": "UK", + "program": "income_tax", + "conditions": { + "baseline_policy": "hmrc_indexed_baseline_spring_2025", + "change_direction": "unsigned", + "country": "UK", + "direction": "stated_change", + "fy": "2026-27", + "geography": "UK", + "option": "Change Savings allowance by \u00a3100 for BR and \u00a350 for HR taxpayers", + "program": "income_tax", + "sign_convention": "magnitude_with_direction_in_label" + }, + "reform_framework": "policy_ref", + "reform_key": "05b7ff3832f3f02d", + "external_value": 0.0, + "calibration_relationship": "held_out", + "claim_baseline": "hmrc_indexed_baseline_spring_2025", + "latest": { + "value": -69675676.99536133, + "status": "constructed", + "engine_version": "2.89.2", + "data_bundle": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", + "release": "8c73-4aa4b14", + "construction": "gov.hmrc.income_tax.allowances.personal_savings_allowance.basic: 1000 -> 1100; gov.hmrc.income_tax.allowances.personal_savings_allowance.higher: 500 -> 550 | heads income_tax", + "computed_at": "2026-08-06T18:42:04.081907+00:00", + "annotations": [ + "PE CY2026 static accrual vs HMRC projected FY direct effects (first-year cash vs fuller-year; TIE/behavioral in HMRC reckoner per their methodology notes); |magnitude| ratios", + "fuller-year FY2027-28 external value 30000000.0 (ratio 2.322522566512044)" + ], + "baseline": "current_law", + "status_effective": "constructed", + "ratio": null, + "delta": -69675676.99536133 + }, + "results": [ + { + "value": -69675676.99536133, + "status": "constructed", + "engine_version": "2.89.2", + "data_bundle": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", + "release": "8c73-4aa4b14", + "construction": "gov.hmrc.income_tax.allowances.personal_savings_allowance.basic: 1000 -> 1100; gov.hmrc.income_tax.allowances.personal_savings_allowance.higher: 500 -> 550 | heads income_tax", + "computed_at": "2026-08-06T18:42:04.081907+00:00", + "annotations": [ + "PE CY2026 static accrual vs HMRC projected FY direct effects (first-year cash vs fuller-year; TIE/behavioral in HMRC reckoner per their methodology notes); |magnitude| ratios", + "fuller-year FY2027-28 external value 30000000.0 (ratio 2.322522566512044)" + ], + "baseline": "current_law", + "status_effective": "constructed" + } + ], + "diagnosis": null + }, + { + "claim_id": "39bb15c3c6084fd9c437", + "source": "uk_hmrc", + "source_column": "reckoner", + "name": "Income Tax liabilities statistics 2023-24 to 2026-27 + June 2025 tax ready reckoner", + "window": "", + "publication_title": "Income Tax liabilities statistics 2023-24 to 2026-27 + June 2025 tax ready reckoner", + "url": "https://www.gov.uk/government/statistics/income-tax-liabilities-statistics-tax-year-2023-to-2024-to-tax-year-2026-to-2027", + "metric": "revenue_change", + "unit_concept": "gbp", + "value_kind": "gbp", + "period": 2027, + "time_basis": "fiscal_year", + "period_start": null, + "period_end": null, + "geography": "UK", + "program": "income_tax", + "conditions": { + "baseline_policy": "hmrc_indexed_baseline_spring_2025", + "change_direction": "increase", + "country": "UK", + "direction": "cost", + "fy": "2026-27", + "geography": "UK", + "option": "Increase basic rate limit by 10% (cost)", + "program": "income_tax", + "sign_convention": "magnitude_with_direction_in_label" + }, + "reform_framework": "policy_ref", + "reform_key": "61c2077173777dd4", + "external_value": 4600000000.0, + "calibration_relationship": "held_out", + "claim_baseline": "hmrc_indexed_baseline_spring_2025", + "latest": { + "value": -6935879147.342651, + "status": "constructed", + "engine_version": "2.89.2", + "data_bundle": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", + "release": "8c73-4aa4b14", + "construction": "gov.hmrc.income_tax.rates.uk[1].threshold: 37700 -> 41470.0 | heads income_tax", + "computed_at": "2026-08-06T18:42:04.081907+00:00", + "annotations": [ + "PE CY2026 static accrual vs HMRC projected FY direct effects (first-year cash vs fuller-year; TIE/behavioral in HMRC reckoner per their methodology notes); |magnitude| ratios", + "fuller-year FY2027-28 external value 6250000000.0 (ratio 1.1097406635748241)" + ], + "baseline": "current_law", + "status_effective": "constructed", + "ratio": -1.507799814639707, + "delta": -11535879147.342651 + }, + "results": [ + { + "value": -6935879147.342651, + "status": "constructed", + "engine_version": "2.89.2", + "data_bundle": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", + "release": "8c73-4aa4b14", + "construction": "gov.hmrc.income_tax.rates.uk[1].threshold: 37700 -> 41470.0 | heads income_tax", + "computed_at": "2026-08-06T18:42:04.081907+00:00", + "annotations": [ + "PE CY2026 static accrual vs HMRC projected FY direct effects (first-year cash vs fuller-year; TIE/behavioral in HMRC reckoner per their methodology notes); |magnitude| ratios", + "fuller-year FY2027-28 external value 6250000000.0 (ratio 1.1097406635748241)" + ], + "baseline": "current_law", + "status_effective": "constructed" + } + ], + "diagnosis": null + }, + { + "claim_id": "4170013e721c13e0ddc1", + "source": "uk_hmrc", + "source_column": "reckoner", + "name": "Income Tax liabilities statistics 2023-24 to 2026-27 + June 2025 tax ready reckoner", + "window": "", + "publication_title": "Income Tax liabilities statistics 2023-24 to 2026-27 + June 2025 tax ready reckoner", + "url": "https://www.gov.uk/government/statistics/income-tax-liabilities-statistics-tax-year-2023-to-2024-to-tax-year-2026-to-2027", + "metric": "revenue_change", + "unit_concept": "gbp", + "value_kind": "gbp", + "period": 2027, + "time_basis": "fiscal_year", + "period_start": null, + "period_end": null, + "geography": "UK", + "program": "income_tax", + "conditions": { + "baseline_policy": "hmrc_indexed_baseline_spring_2025", + "change_direction": "unsigned", + "country": "UK", + "direction": "stated_change", + "fy": "2026-27", + "geography": "UK", + "option": "Change personal allowance by 1%", + "program": "income_tax", + "sign_convention": "magnitude_with_direction_in_label" + }, + "reform_framework": "policy_ref", + "reform_key": "aae15369320e1efc", + "external_value": 1000000000.0, + "calibration_relationship": "held_out", + "claim_baseline": "hmrc_indexed_baseline_spring_2025", + "latest": { + "value": -1131417187.2644653, + "status": "constructed", + "engine_version": "2.89.2", + "data_bundle": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", + "release": "8c73-4aa4b14", + "construction": "gov.hmrc.income_tax.allowances.personal_allowance.amount: 12570 -> 12695.7 | heads income_tax", + "computed_at": "2026-08-06T18:42:04.081907+00:00", + "annotations": [ + "PE CY2026 static accrual vs HMRC projected FY direct effects (first-year cash vs fuller-year; TIE/behavioral in HMRC reckoner per their methodology notes); |magnitude| ratios", + "fuller-year FY2027-28 external value 1200000000.0 (ratio 0.9428476560537211)" + ], + "baseline": "current_law", + "status_effective": "constructed", + "ratio": -1.1314171872644654, + "delta": -2131417187.2644653 + }, + "results": [ + { + "value": -1131417187.2644653, + "status": "constructed", + "engine_version": "2.89.2", + "data_bundle": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", + "release": "8c73-4aa4b14", + "construction": "gov.hmrc.income_tax.allowances.personal_allowance.amount: 12570 -> 12695.7 | heads income_tax", + "computed_at": "2026-08-06T18:42:04.081907+00:00", + "annotations": [ + "PE CY2026 static accrual vs HMRC projected FY direct effects (first-year cash vs fuller-year; TIE/behavioral in HMRC reckoner per their methodology notes); |magnitude| ratios", + "fuller-year FY2027-28 external value 1200000000.0 (ratio 0.9428476560537211)" + ], + "baseline": "current_law", + "status_effective": "constructed" + } + ], + "diagnosis": null + }, + { + "claim_id": "5c6fc857d40327bb759c", + "source": "uk_hmrc", + "source_column": "reckoner", + "name": "Income Tax liabilities statistics 2023-24 to 2026-27 + June 2025 tax ready reckoner", + "window": "", + "publication_title": "Income Tax liabilities statistics 2023-24 to 2026-27 + June 2025 tax ready reckoner", + "url": "https://www.gov.uk/government/statistics/income-tax-liabilities-statistics-tax-year-2023-to-2024-to-tax-year-2026-to-2027", + "metric": "revenue_change", + "unit_concept": "gbp", + "value_kind": "gbp", + "period": 2027, + "time_basis": "fiscal_year", + "period_start": null, + "period_end": null, + "geography": "UK", + "program": "income_tax", + "conditions": { + "baseline_policy": "hmrc_indexed_baseline_spring_2025", + "change_direction": "unsigned", + "country": "UK", + "direction": "stated_change", + "fy": "2026-27", + "geography": "UK", + "option": "Change personal allowance by 10%", + "program": "income_tax", + "sign_convention": "magnitude_with_direction_in_label" + }, + "reform_framework": "policy_ref", + "reform_key": "b3c79dd8488bac94", + "external_value": 10000000000.0, + "calibration_relationship": "held_out", + "claim_baseline": "hmrc_indexed_baseline_spring_2025", + "latest": { + "value": -11148223306.159363, + "status": "constructed", + "engine_version": "2.89.2", + "data_bundle": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", + "release": "8c73-4aa4b14", + "construction": "gov.hmrc.income_tax.allowances.personal_allowance.amount: 12570 -> 13827.000000000002 | heads income_tax", + "computed_at": "2026-08-06T18:42:04.081907+00:00", + "annotations": [ + "PE CY2026 static accrual vs HMRC projected FY direct effects (first-year cash vs fuller-year; TIE/behavioral in HMRC reckoner per their methodology notes); |magnitude| ratios", + "fuller-year FY2027-28 external value 11650000000.0 (ratio 0.9569290391553101)" + ], + "baseline": "current_law", + "status_effective": "constructed", + "ratio": -1.1148223306159364, + "delta": -21148223306.159363 + }, + "results": [ + { + "value": -11148223306.159363, + "status": "constructed", + "engine_version": "2.89.2", + "data_bundle": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", + "release": "8c73-4aa4b14", + "construction": "gov.hmrc.income_tax.allowances.personal_allowance.amount: 12570 -> 13827.000000000002 | heads income_tax", + "computed_at": "2026-08-06T18:42:04.081907+00:00", + "annotations": [ + "PE CY2026 static accrual vs HMRC projected FY direct effects (first-year cash vs fuller-year; TIE/behavioral in HMRC reckoner per their methodology notes); |magnitude| ratios", + "fuller-year FY2027-28 external value 11650000000.0 (ratio 0.9569290391553101)" + ], + "baseline": "current_law", + "status_effective": "constructed" + } + ], + "diagnosis": null + }, + { + "claim_id": "6f708fb9feb19cb70eac", + "source": "uk_hmrc", + "source_column": "reckoner", + "name": "Income Tax liabilities statistics 2023-24 to 2026-27 + June 2025 tax ready reckoner", + "window": "", + "publication_title": "Income Tax liabilities statistics 2023-24 to 2026-27 + June 2025 tax ready reckoner", + "url": "https://www.gov.uk/government/statistics/income-tax-liabilities-statistics-tax-year-2023-to-2024-to-tax-year-2026-to-2027", + "metric": "revenue_change", + "unit_concept": "gbp", + "value_kind": "gbp", + "period": 2027, + "time_basis": "fiscal_year", + "period_start": null, + "period_end": null, + "geography": "UK", + "program": "child_benefit", + "conditions": { + "baseline_policy": "hmrc_indexed_baseline_spring_2025", + "change_direction": "increase", + "country": "UK", + "direction": "cost", + "fy": "2026-27", + "geography": "UK", + "option": "Increase subsequent child rate by \u00a31 per week (cost)", + "program": "child_benefit", + "sign_convention": "magnitude_with_direction_in_label" + }, + "reform_framework": "policy_ref", + "reform_key": "d674e5d7bd144e7f", + "external_value": 230000000.0, + "calibration_relationship": "held_out", + "claim_baseline": "hmrc_indexed_baseline_spring_2025", + "latest": { + "value": 307891828.10876465, + "status": "constructed", + "engine_version": "2.89.2", + "data_bundle": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", + "release": "8c73-4aa4b14", + "construction": "gov.hmrc.child_benefit.amount.additional: 17.9 -> 18.9 | heads child_benefit", + "computed_at": "2026-08-06T18:42:04.081907+00:00", + "annotations": [ + "PE CY2026 static accrual vs HMRC projected FY direct effects (first-year cash vs fuller-year; TIE/behavioral in HMRC reckoner per their methodology notes); |magnitude| ratios", + "fuller-year FY2027-28 external value 240000000.0 (ratio 1.2828826171198526)" + ], + "baseline": "current_law", + "status_effective": "constructed", + "ratio": 1.3386601222120202, + "delta": 77891828.10876465 + }, + "results": [ + { + "value": 307891828.10876465, + "status": "constructed", + "engine_version": "2.89.2", + "data_bundle": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", + "release": "8c73-4aa4b14", + "construction": "gov.hmrc.child_benefit.amount.additional: 17.9 -> 18.9 | heads child_benefit", + "computed_at": "2026-08-06T18:42:04.081907+00:00", + "annotations": [ + "PE CY2026 static accrual vs HMRC projected FY direct effects (first-year cash vs fuller-year; TIE/behavioral in HMRC reckoner per their methodology notes); |magnitude| ratios", + "fuller-year FY2027-28 external value 240000000.0 (ratio 1.2828826171198526)" + ], + "baseline": "current_law", + "status_effective": "constructed" + } + ], + "diagnosis": null + }, + { + "claim_id": "8e1c1f9b430e9d996f24", + "source": "uk_hmrc", + "source_column": "reckoner", + "name": "Income Tax liabilities statistics 2023-24 to 2026-27 + June 2025 tax ready reckoner", + "window": "", + "publication_title": "Income Tax liabilities statistics 2023-24 to 2026-27 + June 2025 tax ready reckoner", + "url": "https://www.gov.uk/government/statistics/income-tax-liabilities-statistics-tax-year-2023-to-2024-to-tax-year-2026-to-2027", + "metric": "revenue_change", + "unit_concept": "gbp", + "value_kind": "gbp", + "period": 2027, + "time_basis": "fiscal_year", + "period_start": null, + "period_end": null, + "geography": "UK", + "program": "national_insurance", + "conditions": { + "baseline_policy": "hmrc_indexed_baseline_spring_2025", + "change_direction": "unsigned", + "country": "UK", + "direction": "stated_change", + "fy": "2026-27", + "geography": "UK", + "option": "Change Class 1 employee additional rate by 1 percentage point", + "program": "national_insurance", + "sign_convention": "magnitude_with_direction_in_label" + }, + "reform_framework": "policy_ref", + "reform_key": "63ae5174945c7cb7", + "external_value": 2000000000.0, + "calibration_relationship": "held_out", + "claim_baseline": "hmrc_indexed_baseline_spring_2025", + "latest": { + "value": 4391879294.86554, + "status": "constructed", + "engine_version": "2.89.2", + "data_bundle": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", + "release": "8c73-4aa4b14", + "construction": "gov.hmrc.national_insurance.class_1.rates.employee.additional: 0.02 -> 0.03 | heads national_insurance", + "computed_at": "2026-08-06T18:42:04.081907+00:00", + "annotations": [ + "PE CY2026 static accrual vs HMRC projected FY direct effects (first-year cash vs fuller-year; TIE/behavioral in HMRC reckoner per their methodology notes); |magnitude| ratios", + "fuller-year FY2027-28 external value 2000000000.0 (ratio 2.1959396474327697)" + ], + "baseline": "current_law", + "status_effective": "constructed", + "ratio": 2.1959396474327697, + "delta": 2391879294.8655396 + }, + "results": [ + { + "value": 4391879294.86554, + "status": "constructed", + "engine_version": "2.89.2", + "data_bundle": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", + "release": "8c73-4aa4b14", + "construction": "gov.hmrc.national_insurance.class_1.rates.employee.additional: 0.02 -> 0.03 | heads national_insurance", + "computed_at": "2026-08-06T18:42:04.081907+00:00", + "annotations": [ + "PE CY2026 static accrual vs HMRC projected FY direct effects (first-year cash vs fuller-year; TIE/behavioral in HMRC reckoner per their methodology notes); |magnitude| ratios", + "fuller-year FY2027-28 external value 2000000000.0 (ratio 2.1959396474327697)" + ], + "baseline": "current_law", + "status_effective": "constructed" + } + ], + "diagnosis": null + }, + { + "claim_id": "b601650048d20e2606c9", + "source": "uk_hmrc", + "source_column": "reckoner", + "name": "Income Tax liabilities statistics 2023-24 to 2026-27 + June 2025 tax ready reckoner", + "window": "", + "publication_title": "Income Tax liabilities statistics 2023-24 to 2026-27 + June 2025 tax ready reckoner", + "url": "https://www.gov.uk/government/statistics/income-tax-liabilities-statistics-tax-year-2023-to-2024-to-tax-year-2026-to-2027", + "metric": "revenue_change", + "unit_concept": "gbp", + "value_kind": "gbp", + "period": 2027, + "time_basis": "fiscal_year", + "period_start": null, + "period_end": null, + "geography": "UK", + "program": "national_insurance", + "conditions": { + "baseline_policy": "hmrc_indexed_baseline_spring_2025", + "change_direction": "unsigned", + "country": "UK", + "direction": "stated_change", + "fy": "2026-27", + "geography": "UK", + "option": "Change upper profits limit by \u00a3520 per year", + "program": "national_insurance", + "sign_convention": "magnitude_with_direction_in_label" + }, + "reform_framework": "policy_ref", + "reform_key": "4637d737894b0e32", + "external_value": 10000000.0, + "calibration_relationship": "held_out", + "claim_baseline": "hmrc_indexed_baseline_spring_2025", + "latest": { + "value": 15204671.735198975, + "status": "constructed", + "engine_version": "2.89.2", + "data_bundle": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", + "release": "8c73-4aa4b14", + "construction": "gov.hmrc.national_insurance.class_4.thresholds.upper_profits_limit: 50270 -> 50790 | heads national_insurance", + "computed_at": "2026-08-06T18:42:04.081907+00:00", + "annotations": [ + "PE CY2026 static accrual vs HMRC projected FY direct effects (first-year cash vs fuller-year; TIE/behavioral in HMRC reckoner per their methodology notes); |magnitude| ratios", + "fuller-year FY2027-28 external value 10000000.0 (ratio 1.5204671735198974)" + ], + "baseline": "current_law", + "status_effective": "constructed", + "ratio": 1.5204671735198974, + "delta": 5204671.735198975 + }, + "results": [ + { + "value": 15204671.735198975, + "status": "constructed", + "engine_version": "2.89.2", + "data_bundle": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", + "release": "8c73-4aa4b14", + "construction": "gov.hmrc.national_insurance.class_4.thresholds.upper_profits_limit: 50270 -> 50790 | heads national_insurance", + "computed_at": "2026-08-06T18:42:04.081907+00:00", + "annotations": [ + "PE CY2026 static accrual vs HMRC projected FY direct effects (first-year cash vs fuller-year; TIE/behavioral in HMRC reckoner per their methodology notes); |magnitude| ratios", + "fuller-year FY2027-28 external value 10000000.0 (ratio 1.5204671735198974)" + ], + "baseline": "current_law", + "status_effective": "constructed" + } + ], + "diagnosis": null + }, + { + "claim_id": "b8be62add3033280162d", + "source": "uk_hmrc", + "source_column": "reckoner", + "name": "Income Tax liabilities statistics 2023-24 to 2026-27 + June 2025 tax ready reckoner", + "window": "", + "publication_title": "Income Tax liabilities statistics 2023-24 to 2026-27 + June 2025 tax ready reckoner", + "url": "https://www.gov.uk/government/statistics/income-tax-liabilities-statistics-tax-year-2023-to-2024-to-tax-year-2026-to-2027", + "metric": "revenue_change", + "unit_concept": "gbp", + "value_kind": "gbp", + "period": 2027, + "time_basis": "fiscal_year", + "period_start": null, + "period_end": null, + "geography": "UK", + "program": "national_insurance", + "conditions": { + "baseline_policy": "hmrc_indexed_baseline_spring_2025", + "change_direction": "unsigned", + "country": "UK", + "direction": "stated_change", + "fy": "2026-27", + "geography": "UK", + "option": "Change employee entry threshold by \u00a32 per week", + "program": "national_insurance", + "sign_convention": "magnitude_with_direction_in_label" + }, + "reform_framework": "policy_ref", + "reform_key": "b7bd756c288799e3", + "external_value": 210000000.0, + "calibration_relationship": "held_out", + "claim_baseline": "hmrc_indexed_baseline_spring_2025", + "latest": { + "value": -202893958.81115723, + "status": "constructed", + "engine_version": "2.89.2", + "data_bundle": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", + "release": "8c73-4aa4b14", + "construction": "gov.hmrc.national_insurance.class_1.thresholds.primary_threshold: 241.73 -> 243.73 | heads national_insurance", + "computed_at": "2026-08-06T18:42:04.081907+00:00", + "annotations": [ + "PE CY2026 static accrual vs HMRC projected FY direct effects (first-year cash vs fuller-year; TIE/behavioral in HMRC reckoner per their methodology notes); |magnitude| ratios", + "fuller-year FY2027-28 external value 230000000.0 (ratio 0.8821476470050315)" + ], + "baseline": "current_law", + "status_effective": "constructed", + "ratio": -0.9661617086245582, + "delta": -412893958.8111572 + }, + "results": [ + { + "value": -202893958.81115723, + "status": "constructed", + "engine_version": "2.89.2", + "data_bundle": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", + "release": "8c73-4aa4b14", + "construction": "gov.hmrc.national_insurance.class_1.thresholds.primary_threshold: 241.73 -> 243.73 | heads national_insurance", + "computed_at": "2026-08-06T18:42:04.081907+00:00", + "annotations": [ + "PE CY2026 static accrual vs HMRC projected FY direct effects (first-year cash vs fuller-year; TIE/behavioral in HMRC reckoner per their methodology notes); |magnitude| ratios", + "fuller-year FY2027-28 external value 230000000.0 (ratio 0.8821476470050315)" + ], + "baseline": "current_law", + "status_effective": "constructed" + } + ], + "diagnosis": null + }, + { + "claim_id": "d2fbcdc3204a5b6e27d5", + "source": "uk_hmrc", + "source_column": "reckoner", + "name": "Income Tax liabilities statistics 2023-24 to 2026-27 + June 2025 tax ready reckoner", + "window": "", + "publication_title": "Income Tax liabilities statistics 2023-24 to 2026-27 + June 2025 tax ready reckoner", + "url": "https://www.gov.uk/government/statistics/income-tax-liabilities-statistics-tax-year-2023-to-2024-to-tax-year-2026-to-2027", + "metric": "revenue_change", + "unit_concept": "gbp", + "value_kind": "gbp", + "period": 2027, + "time_basis": "fiscal_year", + "period_start": null, + "period_end": null, + "geography": "UK", + "program": "national_insurance", + "conditions": { + "baseline_policy": "hmrc_indexed_baseline_spring_2025", + "change_direction": "unsigned", + "country": "UK", + "direction": "stated_change", + "fy": "2026-27", + "geography": "UK", + "option": "Change upper earnings limit by \u00a310 per week", + "program": "national_insurance", + "sign_convention": "magnitude_with_direction_in_label" + }, + "reform_framework": "policy_ref", + "reform_key": "0bb1cd60307ae735", + "external_value": 220000000.0, + "calibration_relationship": "held_out", + "claim_baseline": "hmrc_indexed_baseline_spring_2025", + "latest": { + "value": 248033861.83112335, + "status": "constructed", + "engine_version": "2.89.2", + "data_bundle": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", + "release": "8c73-4aa4b14", + "construction": "gov.hmrc.national_insurance.class_1.thresholds.upper_earnings_limit: 966.73 -> 976.73 | heads national_insurance", + "computed_at": "2026-08-06T18:42:04.081907+00:00", + "annotations": [ + "PE CY2026 static accrual vs HMRC projected FY direct effects (first-year cash vs fuller-year; TIE/behavioral in HMRC reckoner per their methodology notes); |magnitude| ratios", + "fuller-year FY2027-28 external value 220000000.0 (ratio 1.1274266446869243)" + ], + "baseline": "current_law", + "status_effective": "constructed", + "ratio": 1.1274266446869243, + "delta": 28033861.831123352 + }, + "results": [ + { + "value": 248033861.83112335, + "status": "constructed", + "engine_version": "2.89.2", + "data_bundle": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", + "release": "8c73-4aa4b14", + "construction": "gov.hmrc.national_insurance.class_1.thresholds.upper_earnings_limit: 966.73 -> 976.73 | heads national_insurance", + "computed_at": "2026-08-06T18:42:04.081907+00:00", + "annotations": [ + "PE CY2026 static accrual vs HMRC projected FY direct effects (first-year cash vs fuller-year; TIE/behavioral in HMRC reckoner per their methodology notes); |magnitude| ratios", + "fuller-year FY2027-28 external value 220000000.0 (ratio 1.1274266446869243)" + ], + "baseline": "current_law", + "status_effective": "constructed" + } + ], + "diagnosis": null + }, + { + "claim_id": "e05ac73d999627cb18cb", + "source": "uk_hmrc", + "source_column": "reckoner", + "name": "Income Tax liabilities statistics 2023-24 to 2026-27 + June 2025 tax ready reckoner", + "window": "", + "publication_title": "Income Tax liabilities statistics 2023-24 to 2026-27 + June 2025 tax ready reckoner", + "url": "https://www.gov.uk/government/statistics/income-tax-liabilities-statistics-tax-year-2023-to-2024-to-tax-year-2026-to-2027", + "metric": "revenue_change", + "unit_concept": "gbp", + "value_kind": "gbp", + "period": 2027, + "time_basis": "fiscal_year", + "period_start": null, + "period_end": null, + "geography": "UK", + "program": "income_tax", + "conditions": { + "baseline_policy": "hmrc_indexed_baseline_spring_2025", + "change_direction": "unsigned", + "country": "UK", + "direction": "stated_change", + "fy": "2026-27", + "geography": "UK", + "option": "Change dividend allowance by \u00a3100", + "program": "income_tax", + "sign_convention": "magnitude_with_direction_in_label" + }, + "reform_framework": "policy_ref", + "reform_key": "ef2cbab3e901fc34", + "external_value": 0.0, + "calibration_relationship": "held_out", + "claim_baseline": "hmrc_indexed_baseline_spring_2025", + "latest": { + "value": -52777493.560424805, + "status": "constructed", + "engine_version": "2.89.2", + "data_bundle": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", + "release": "8c73-4aa4b14", + "construction": "gov.hmrc.income_tax.allowances.dividend_allowance: 500 -> 600 | heads income_tax", + "computed_at": "2026-08-06T18:42:04.081907+00:00", + "annotations": [ + "PE CY2026 static accrual vs HMRC projected FY direct effects (first-year cash vs fuller-year; TIE/behavioral in HMRC reckoner per their methodology notes); |magnitude| ratios", + "fuller-year FY2027-28 external value 70000000.0 (ratio 0.7539641937203544)" + ], + "baseline": "current_law", + "status_effective": "constructed", + "ratio": null, + "delta": -52777493.560424805 + }, + "results": [ + { + "value": -52777493.560424805, + "status": "constructed", + "engine_version": "2.89.2", + "data_bundle": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", + "release": "8c73-4aa4b14", + "construction": "gov.hmrc.income_tax.allowances.dividend_allowance: 500 -> 600 | heads income_tax", + "computed_at": "2026-08-06T18:42:04.081907+00:00", + "annotations": [ + "PE CY2026 static accrual vs HMRC projected FY direct effects (first-year cash vs fuller-year; TIE/behavioral in HMRC reckoner per their methodology notes); |magnitude| ratios", + "fuller-year FY2027-28 external value 70000000.0 (ratio 0.7539641937203544)" + ], + "baseline": "current_law", + "status_effective": "constructed" + } + ], + "diagnosis": null + }, + { + "claim_id": "e2aac04633b000d98d17", + "source": "uk_hmrc", + "source_column": "reckoner", + "name": "Income Tax liabilities statistics 2023-24 to 2026-27 + June 2025 tax ready reckoner", + "window": "", + "publication_title": "Income Tax liabilities statistics 2023-24 to 2026-27 + June 2025 tax ready reckoner", + "url": "https://www.gov.uk/government/statistics/income-tax-liabilities-statistics-tax-year-2023-to-2024-to-tax-year-2026-to-2027", + "metric": "revenue_change", + "unit_concept": "gbp", + "value_kind": "gbp", + "period": 2027, + "time_basis": "fiscal_year", + "period_start": null, + "period_end": null, + "geography": "UK", + "program": "income_tax", + "conditions": { + "baseline_policy": "hmrc_indexed_baseline_spring_2025", + "change_direction": "unsigned", + "country": "UK", + "direction": "stated_change", + "fy": "2026-27", + "geography": "UK", + "option": "Change starting rate limit for savings income by \u00a3100", + "program": "income_tax", + "sign_convention": "magnitude_with_direction_in_label" + }, + "reform_framework": "policy_ref", + "reform_key": "0d59821a6c4d3eb2", + "external_value": null, + "calibration_relationship": "held_out", + "claim_baseline": "hmrc_indexed_baseline_spring_2025", + "latest": { + "value": -33388707.427124023, + "status": "constructed", + "engine_version": "2.89.2", + "data_bundle": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", + "release": "8c73-4aa4b14", + "construction": "gov.hmrc.income_tax.rates.savings_starter_rate.allowance: 5000 -> 5100 | heads income_tax", + "computed_at": "2026-08-06T18:42:04.081907+00:00", + "annotations": [ + "PE CY2026 static accrual vs HMRC projected FY direct effects (first-year cash vs fuller-year; TIE/behavioral in HMRC reckoner per their methodology notes); |magnitude| ratios", + "fuller-year FY2027-28 external value 5000000.0 (ratio 6.6777414854248045)" + ], + "baseline": "current_law", + "status_effective": "constructed", + "ratio": null, + "delta": null + }, + "results": [ + { + "value": -33388707.427124023, + "status": "constructed", + "engine_version": "2.89.2", + "data_bundle": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", + "release": "8c73-4aa4b14", + "construction": "gov.hmrc.income_tax.rates.savings_starter_rate.allowance: 5000 -> 5100 | heads income_tax", + "computed_at": "2026-08-06T18:42:04.081907+00:00", + "annotations": [ + "PE CY2026 static accrual vs HMRC projected FY direct effects (first-year cash vs fuller-year; TIE/behavioral in HMRC reckoner per their methodology notes); |magnitude| ratios", + "fuller-year FY2027-28 external value 5000000.0 (ratio 6.6777414854248045)" + ], + "baseline": "current_law", + "status_effective": "constructed" + } + ], + "diagnosis": null + }, + { + "claim_id": "eea5432a24523ee02327", + "source": "uk_hmrc", + "source_column": "reckoner", + "name": "Income Tax liabilities statistics 2023-24 to 2026-27 + June 2025 tax ready reckoner", + "window": "", + "publication_title": "Income Tax liabilities statistics 2023-24 to 2026-27 + June 2025 tax ready reckoner", + "url": "https://www.gov.uk/government/statistics/income-tax-liabilities-statistics-tax-year-2023-to-2024-to-tax-year-2026-to-2027", + "metric": "revenue_change", + "unit_concept": "gbp", + "value_kind": "gbp", + "period": 2027, + "time_basis": "fiscal_year", + "period_start": null, + "period_end": null, + "geography": "UK", + "program": "national_insurance", + "conditions": { + "baseline_policy": "hmrc_indexed_baseline_spring_2025", + "change_direction": "unsigned", + "country": "UK", + "direction": "stated_change", + "fy": "2026-27", + "geography": "UK", + "option": "Change employer threshold by \u00a32 per week", + "program": "national_insurance", + "sign_convention": "magnitude_with_direction_in_label" + }, + "reform_framework": "policy_ref", + "reform_key": "1e02c43b824ed2f7", + "external_value": 420000000.0, + "calibration_relationship": "held_out", + "claim_baseline": "hmrc_indexed_baseline_spring_2025", + "latest": { + "value": -356685195.3192749, + "status": "constructed", + "engine_version": "2.89.2", + "data_bundle": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", + "release": "8c73-4aa4b14", + "construction": "gov.hmrc.national_insurance.class_1.thresholds.secondary_threshold: 96 -> 98 | heads ni_employer", + "computed_at": "2026-08-06T18:42:04.081907+00:00", + "annotations": [ + "PE CY2026 static accrual vs HMRC projected FY direct effects (first-year cash vs fuller-year; TIE/behavioral in HMRC reckoner per their methodology notes); |magnitude| ratios", + "fuller-year FY2027-28 external value 430000000.0 (ratio 0.8295004542308718)" + ], + "baseline": "current_law", + "status_effective": "constructed", + "ratio": -0.8492504650458926, + "delta": -776685195.3192749 + }, + "results": [ + { + "value": -356685195.3192749, + "status": "constructed", + "engine_version": "2.89.2", + "data_bundle": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", + "release": "8c73-4aa4b14", + "construction": "gov.hmrc.national_insurance.class_1.thresholds.secondary_threshold: 96 -> 98 | heads ni_employer", + "computed_at": "2026-08-06T18:42:04.081907+00:00", + "annotations": [ + "PE CY2026 static accrual vs HMRC projected FY direct effects (first-year cash vs fuller-year; TIE/behavioral in HMRC reckoner per their methodology notes); |magnitude| ratios", + "fuller-year FY2027-28 external value 430000000.0 (ratio 0.8295004542308718)" + ], + "baseline": "current_law", + "status_effective": "constructed" + } + ], + "diagnosis": null + }, + { + "claim_id": "fcefd20d8811df56657f", + "source": "uk_hmrc", + "source_column": "reckoner", + "name": "Income Tax liabilities statistics 2023-24 to 2026-27 + June 2025 tax ready reckoner", + "window": "", + "publication_title": "Income Tax liabilities statistics 2023-24 to 2026-27 + June 2025 tax ready reckoner", + "url": "https://www.gov.uk/government/statistics/income-tax-liabilities-statistics-tax-year-2023-to-2024-to-tax-year-2026-to-2027", + "metric": "revenue_change", + "unit_concept": "gbp", + "value_kind": "gbp", + "period": 2027, + "time_basis": "fiscal_year", + "period_start": null, + "period_end": null, + "geography": "UK", + "program": "income_tax", + "conditions": { + "baseline_policy": "hmrc_indexed_baseline_spring_2025", + "change_direction": "unsigned", + "country": "UK", + "direction": "stated_change", + "fy": "2026-27", + "geography": "UK", + "option": "Change basic rate limit by 1%", + "program": "income_tax", + "sign_convention": "magnitude_with_direction_in_label" + }, + "reform_framework": "policy_ref", + "reform_key": "6bda9aaade34d6da", + "external_value": 495000000.0, + "calibration_relationship": "held_out", + "claim_baseline": "hmrc_indexed_baseline_spring_2025", + "latest": { + "value": -725046338.0097046, + "status": "constructed", + "engine_version": "2.89.2", + "data_bundle": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", + "release": "8c73-4aa4b14", + "construction": "gov.hmrc.income_tax.rates.uk[1].threshold: 37700 -> 38077.0 | heads income_tax", + "computed_at": "2026-08-06T18:42:04.081907+00:00", + "annotations": [ + "PE CY2026 static accrual vs HMRC projected FY direct effects (first-year cash vs fuller-year; TIE/behavioral in HMRC reckoner per their methodology notes); |magnitude| ratios", + "fuller-year FY2027-28 external value 665000000.0 (ratio 1.0902952451273753)" + ], + "baseline": "current_law", + "status_effective": "constructed", + "ratio": -1.464740076787282, + "delta": -1220046338.0097046 + }, + "results": [ + { + "value": -725046338.0097046, + "status": "constructed", + "engine_version": "2.89.2", + "data_bundle": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", + "release": "8c73-4aa4b14", + "construction": "gov.hmrc.income_tax.rates.uk[1].threshold: 37700 -> 38077.0 | heads income_tax", + "computed_at": "2026-08-06T18:42:04.081907+00:00", + "annotations": [ + "PE CY2026 static accrual vs HMRC projected FY direct effects (first-year cash vs fuller-year; TIE/behavioral in HMRC reckoner per their methodology notes); |magnitude| ratios", + "fuller-year FY2027-28 external value 665000000.0 (ratio 1.0902952451273753)" + ], + "baseline": "current_law", + "status_effective": "constructed" + } + ], + "diagnosis": null + }, + { + "claim_id": "ff1a81248008fdad8c35", + "source": "uk_hmrc", + "source_column": "reckoner", + "name": "Income Tax liabilities statistics 2023-24 to 2026-27 + June 2025 tax ready reckoner", + "window": "", + "publication_title": "Income Tax liabilities statistics 2023-24 to 2026-27 + June 2025 tax ready reckoner", + "url": "https://www.gov.uk/government/statistics/income-tax-liabilities-statistics-tax-year-2023-to-2024-to-tax-year-2026-to-2027", + "metric": "revenue_change", + "unit_concept": "gbp", + "value_kind": "gbp", + "period": 2027, + "time_basis": "fiscal_year", + "period_start": null, + "period_end": null, + "geography": "UK", + "program": "national_insurance", + "conditions": { + "baseline_policy": "hmrc_indexed_baseline_spring_2025", + "change_direction": "unsigned", + "country": "UK", + "direction": "stated_change", + "fy": "2026-27", + "geography": "UK", + "option": "Change lower profits limit by \u00a3104 per year", + "program": "national_insurance", + "sign_convention": "magnitude_with_direction_in_label" + }, + "reform_framework": "policy_ref", + "reform_key": "7a0446aba36175e3", + "external_value": 15000000.0, + "calibration_relationship": "held_out", + "claim_baseline": "hmrc_indexed_baseline_spring_2025", + "latest": { + "value": -17503727.481391907, + "status": "constructed", + "engine_version": "2.89.2", + "data_bundle": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", + "release": "8c73-4aa4b14", + "construction": "gov.hmrc.national_insurance.class_4.thresholds.lower_profits_limit: 12570 -> 12674 | heads national_insurance", + "computed_at": "2026-08-06T18:42:04.081907+00:00", + "annotations": [ + "PE CY2026 static accrual vs HMRC projected FY direct effects (first-year cash vs fuller-year; TIE/behavioral in HMRC reckoner per their methodology notes); |magnitude| ratios", + "fuller-year FY2027-28 external value 15000000.0 (ratio 1.1669151654261272)" + ], + "baseline": "current_law", + "status_effective": "constructed", + "ratio": -1.1669151654261272, + "delta": -32503727.481391907 + }, + "results": [ + { + "value": -17503727.481391907, + "status": "constructed", + "engine_version": "2.89.2", + "data_bundle": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", + "release": "8c73-4aa4b14", + "construction": "gov.hmrc.national_insurance.class_4.thresholds.lower_profits_limit: 12570 -> 12674 | heads national_insurance", + "computed_at": "2026-08-06T18:42:04.081907+00:00", + "annotations": [ + "PE CY2026 static accrual vs HMRC projected FY direct effects (first-year cash vs fuller-year; TIE/behavioral in HMRC reckoner per their methodology notes); |magnitude| ratios", + "fuller-year FY2027-28 external value 15000000.0 (ratio 1.1669151654261272)" + ], + "baseline": "current_law", + "status_effective": "constructed" + } + ], + "diagnosis": null + }, { "claim_id": "93368a42000c01f07cc3", "source": "ut_admin", diff --git a/data/scorecard.db b/data/scorecard.db index 4c82772..dc72740 100644 Binary files a/data/scorecard.db and b/data/scorecard.db differ diff --git a/scorecard_db/export_populations.py b/scorecard_db/export_populations.py index 3f9a3ca..4dc0177 100644 --- a/scorecard_db/export_populations.py +++ b/scorecard_db/export_populations.py @@ -2,13 +2,15 @@ The Urban SotSN population reaches the app through the file-based pipeline/build_comparison.py export (data/comparison.json). Everything -else — today, exactly the reform-validation registry (issue #20): its 205 -minted claims plus the 36 harvested JCX-35-25 provision claims its OBBBA -results attach to — lives only in scorecard.db. This module exports every -non-Urban claim that has at least one pe_result, carrying the dimension -the Urban export doesn't have: the full per-release result history -(one row per certified release, engine pins and OBBBA scoring mode in the -construction), so cross-release drift is visible. +else lives only in scorecard.db: the reform-validation registry (issue +#20, its 205 minted claims plus the 36 harvested JCX-35-25 provision +claims its OBBBA results attach to), the US campaign attaches, and — +since the campaign-UK producer — the 14 uk_hmrc reckoner claims with +campaign results. This module exports every non-Urban claim that has at +least one pe_result, carrying the dimension the Urban export doesn't +have: the full per-release result history (one row per certified +release, engine pins and OBBBA scoring mode in the construction), so +cross-release drift is visible. Doctrine (issues #1/#9): descriptive only. Statuses and calibration relationships are exported verbatim; ratios are raw pe/external with no @@ -193,7 +195,8 @@ def export( "note": ( "Non-Urban populations exported from scorecard.db: the populace" " reform-validation registry (issue #20) plus the compute" - " campaign's attached comparisons (TPC/CPSP/PWBM/CBO/JCT)." + " campaign's attached comparisons (US: TPC/CPSP/PWBM/CBO/JCT;" + " UK: the HMRC ready-reckoner family)." " Statuses and calibration relationships are verbatim; nothing" " here is a pass/fail grade." ), diff --git a/scorecard_db/ingest_campaign.py b/scorecard_db/ingest_campaign.py index efbbc24..367219e 100644 --- a/scorecard_db/ingest_campaign.py +++ b/scorecard_db/ingest_campaign.py @@ -9,11 +9,15 @@ family-vocabulary descriptor (translated below) or, for claims already in the DB (the Urban subgroup joins), the claim_id directly. -US families attach here. The UK families (free_joins, hmrc_reckoner_t2, -obr_measures, uprating_april2026, two_child) are vendored alongside but -NOT ingested: their claims live in the UK harvest, which has no DB ingest -yet — they attach when it lands, and this module fails loudly if pointed -at them early. +US families attach from sources/campaign-20260802/us. UK families +attach from sources/campaign-20260802/uk_resolved — the DERIVED staging +produce_campaign_uk builds from the frozen uk/ archive by resolving +each row to a claim_id against the ingested UK claims (today: +hmrc_reckoner_t2's 14 rows; the other archived families stay blocked +with per-row reasons in that module until their target sources are +staged). Pointing this module at the frozen uk/ archive directly still +fails loudly (its descriptors under-specify by design — resolution is +the producer's job). Match contract: descriptors were verified by the campaign against the harvest STAGING files; the DB's per-source adapters normalized vocabulary @@ -263,4 +267,5 @@ def ingest(db_path: Path, staged_dir: Path | None = None) -> dict: import sys out = Path(sys.argv[1] if len(sys.argv) > 1 else "data/scorecard.db") - print(json.dumps(ingest(out), indent=1)) + staged = Path(sys.argv[2]) if len(sys.argv) > 2 else None + print(json.dumps(ingest(out, staged), indent=1)) diff --git a/scorecard_db/produce_campaign_uk.py b/scorecard_db/produce_campaign_uk.py new file mode 100644 index 0000000..baadd81 --- /dev/null +++ b/scorecard_db/produce_campaign_uk.py @@ -0,0 +1,204 @@ +"""Resolve the archived UK campaign staging against the ingested UK claims. + +The 2026-08-02 compute campaign staged five UK families before any UK +claims existed in the DB; the archive is frozen under +sources/campaign-20260802/uk (PR #70). Measured against the ingested +vocabulary (PR #48) the archived descriptors under-specify — the +adjudication's finding: two personal-allowance reckoner rows share +identical descriptor conditions, and the descriptor periods key the PE +calendar year, not the claim convention (FY END year). This producer +DERIVES a resolved staging deterministically instead of loosening the +match contract: + + archive row.pe_construction (its " | " prefix, an exact string) + -> uk_runs/_2026.json pe_construction (the executed run) + -> t2_collation.csv hmrc_hint (HMRC's verbatim change + label for that run) + -> exactly ONE ingested claim: source uk_hmrc, metric + revenue_change, conditions.option == hint, conditions.fy == + the archived row's fy [fail-loud] + -> {"claim_id": ...} (ingest_campaign's strict direct form) + +Every step is a closed lookup that raises on 0 or 2+ — a drifted +archive, collation, or claim re-ingest fails loudly, never mis-joins. + +Family disposition (this module's summary reports it; the per-row +compositions below are pinned in tests so a re-frozen archive cannot +silently outgrow its stated reason): + hmrc_reckoner_t2 RESOLVED (14 rows) -> uk_resolved/ + free_joins NOT RESOLVED (16 rows = 7 obr revenue_level + + 7 uk_dwp benefit_cost + 2 metaless exhibits): the OBR rows + target EFO receipts forecast lines and the uk_dwp rows DWP + benefit-expenditure forecast lines — NEITHER is staged as + claims, and pe-uk-data consumes both publication families as + calibration surfaces, so staging them is relationship-evidence + work (two source lanes), never a quick join + obr_measures NOT RESOLVED (10 rows = 9 obr revenue_change + fiscal-event costings + 1 metaless exhibit): targets the OBR + policy-measures costings database (long-tail source, held on + the DB-storage decision) + two_child NOT RESOLVED (4 rows = 2 resolution_foundation + + 1 ukmod poverty_count_change + 1 metaless exhibit): RF is a + long-tail source (held), and the ukmod row targets a REFORM + claim — the staged UKMOD family is baseline validation + statistics only, so a 2CL-reform claim needs its own staging + decision + uprating_april2026 NOT RESOLVED (4 rows = 3 resolution_foundation + benefit_uprating_pct + 1 metaless exhibit): RF long-tail, held + +The metaless exhibits (5 rows across the families — the free_joins +pair is one construction duplicated) carry exhibit_context but no +exhibit_meta; ingest_campaign would only ever defer them, so they +block with their families rather than shipping as noise. + +Usage: + PYTHONPATH=. python -m scorecard_db.produce_campaign_uk + PYTHONPATH=. python -m scorecard_db.ingest_campaign \ + data/scorecard.db sources/campaign-20260802/uk_resolved +""" + +from __future__ import annotations + +import csv +import json +from pathlib import Path + +from .db import ScorecardDB +from .harvest import REPO + +CAMPAIGN = REPO / "sources" / "campaign-20260802" +ARCHIVE = CAMPAIGN / "uk" +RUNS = CAMPAIGN / "uk_runs" +RESOLVED = CAMPAIGN / "uk_resolved" + +# Archived families this producer deliberately does NOT resolve, with +# the reason a future lane must clear first. Every archive family must +# appear either here or in the resolve/copy sets — a new family in the +# archive fails loudly rather than being silently skipped. +BLOCKED = { + "free_joins": ( + "16 rows = 7 obr revenue_level (EFO receipts forecast lines) + " + "7 uk_dwp benefit_cost (DWP benefit-expenditure forecast " + "lines) + 2 metaless exhibits; neither publication family is " + "staged as claims, and pe-uk-data consumes both as calibration " + "surfaces — staging needs the relationship evidence read at " + "the pin (two source lanes)" + ), + "obr_measures": ( + "10 rows = 9 obr revenue_change fiscal-event costings + 1 " + "metaless exhibit; targets the OBR policy-measures costings " + "database — long-tail source held on the DB-storage decision" + ), + "two_child": ( + "4 rows = 2 resolution_foundation + 1 ukmod " + "poverty_count_change + 1 metaless exhibit; RF is long-tail " + "(held), and the ukmod row targets a REFORM claim — the staged " + "UKMOD family is baseline statistics only, so a 2CL-reform " + "claim needs its own staging decision" + ), + "uprating_april2026": ( + "4 rows = 3 resolution_foundation benefit_uprating_pct + 1 " + "metaless exhibit; RF long-tail, held" + ), +} +RESOLVED_FAMILIES = {"hmrc_reckoner_t2"} + + +def _runs_by_construction() -> dict[str, str]: + """pe_construction -> reform_key over the archived executed runs.""" + out: dict[str, str] = {} + for f in sorted(RUNS.glob("*_2026.json")): + d = json.loads(f.read_text()) + if isinstance(d, dict) and "pe_construction" in d and "reform_key" in d: + if d["pe_construction"] in out: + raise ValueError( + f"two runs share a construction: {d['pe_construction']!r}" + ) + out[d["pe_construction"]] = d["reform_key"] + if not out: + raise FileNotFoundError(f"no run files under {RUNS}") + return out + + +def _hints_by_key() -> dict[str, str]: + """reform_key -> HMRC's verbatim change label (t2 collation).""" + with open(RUNS / "t2_collation.csv") as f: + rows = list(csv.DictReader(f)) + out = {r["key"]: r["hmrc_hint"] for r in rows} + if len(out) != len(rows): + raise ValueError("t2_collation.csv has duplicate keys") + return out + + +def _resolve_reckoner(db: ScorecardDB, rows: list[dict]) -> list[dict]: + constructions = _runs_by_construction() + hints = _hints_by_key() + resolved = [] + for row in rows: + prefix = row["pe_construction"].split(" | ")[0] + if prefix not in constructions: + raise ValueError( + f"hmrc_reckoner_t2: construction {prefix!r} matches no archived run" + ) + key = constructions[prefix] + if key not in hints: + raise ValueError(f"hmrc_reckoner_t2: run {key!r} not in t2 collation") + option = hints[key] + fy = row["external_claim_match"]["conditions"]["fy"] + hits = [ + r[0] + for r in db.conn.execute( + "SELECT claim_id FROM external_scores" + " WHERE source='uk_hmrc' AND metric='revenue_change'" + " AND json_extract(conditions, '$.option') = ?" + " AND json_extract(conditions, '$.fy') = ?", + (option, fy), + ) + ] + if len(hits) != 1: + raise ValueError( + f"hmrc_reckoner_t2: {len(hits)} claims for option " + f"{option!r} fy {fy!r} — need exactly one" + ) + out = dict(row) + out["external_claim_match"] = {"claim_id": hits[0]} + resolved.append(out) + if len({r["external_claim_match"]["claim_id"] for r in resolved}) != len(resolved): + raise ValueError("hmrc_reckoner_t2: two rows resolved to one claim") + return resolved + + +def produce(db_path: Path, out_dir: Path | None = None) -> dict: + """Write the resolved staging; returns the disposition summary.""" + out_dir = out_dir or RESOLVED + db = ScorecardDB(db_path) + families = {p.stem: p for p in sorted(ARCHIVE.glob("*.jsonl"))} + unaccounted = (set(families) - set(BLOCKED) - RESOLVED_FAMILIES) | ( + RESOLVED_FAMILIES - set(families) + ) + if unaccounted: + raise ValueError( + f"archive families without a disposition: {sorted(unaccounted)}" + ) + out_dir.mkdir(parents=True, exist_ok=True) + summary: dict = {"resolved": {}, "blocked": BLOCKED} + for name, path in families.items(): + rows = [ + json.loads(line) for line in path.read_text().splitlines() if line.strip() + ] + if name in BLOCKED: + continue + resolved = _resolve_reckoner(db, rows) + (out_dir / path.name).write_text( + "\n".join(json.dumps(r, sort_keys=True) for r in resolved) + "\n" + ) + summary["resolved"][name] = len(resolved) + db.close() + return summary + + +if __name__ == "__main__": + import sys + + db = Path(sys.argv[1] if len(sys.argv) > 1 else "data/scorecard.db") + print(json.dumps(produce(db), indent=1)) diff --git a/sources/campaign-20260802/uk_resolved/hmrc_reckoner_t2.jsonl b/sources/campaign-20260802/uk_resolved/hmrc_reckoner_t2.jsonl new file mode 100644 index 0000000..f18da2d --- /dev/null +++ b/sources/campaign-20260802/uk_resolved/hmrc_reckoner_t2.jsonl @@ -0,0 +1,14 @@ +{"annotations": ["PE CY2026 static accrual vs HMRC projected FY direct effects (first-year cash vs fuller-year; TIE/behavioral in HMRC reckoner per their methodology notes); |magnitude| ratios", "fuller-year FY2027-28 external value 1200000000.0 (ratio 0.9428476560537211)"], "computed_at": "2026-08-06T18:42:04.081907+00:00", "data_bundle": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", "engine_version": "2.89.2", "external_claim_match": {"claim_id": "4170013e721c13e0ddc1"}, "pe_construction": "gov.hmrc.income_tax.allowances.personal_allowance.amount: 12570 -> 12695.7 | heads income_tax", "pe_value": -1131417187.2644653, "run_id": "campaign-20260802-reckoner-t2", "status": "constructed"} +{"annotations": ["PE CY2026 static accrual vs HMRC projected FY direct effects (first-year cash vs fuller-year; TIE/behavioral in HMRC reckoner per their methodology notes); |magnitude| ratios", "fuller-year FY2027-28 external value 11650000000.0 (ratio 0.9569290391553101)"], "computed_at": "2026-08-06T18:42:04.081907+00:00", "data_bundle": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", "engine_version": "2.89.2", "external_claim_match": {"claim_id": "5c6fc857d40327bb759c"}, "pe_construction": "gov.hmrc.income_tax.allowances.personal_allowance.amount: 12570 -> 13827.000000000002 | heads income_tax", "pe_value": -11148223306.159363, "run_id": "campaign-20260802-reckoner-t2", "status": "constructed"} +{"annotations": ["PE CY2026 static accrual vs HMRC projected FY direct effects (first-year cash vs fuller-year; TIE/behavioral in HMRC reckoner per their methodology notes); |magnitude| ratios", "fuller-year FY2027-28 external value 665000000.0 (ratio 1.0902952451273753)"], "computed_at": "2026-08-06T18:42:04.081907+00:00", "data_bundle": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", "engine_version": "2.89.2", "external_claim_match": {"claim_id": "fcefd20d8811df56657f"}, "pe_construction": "gov.hmrc.income_tax.rates.uk[1].threshold: 37700 -> 38077.0 | heads income_tax", "pe_value": -725046338.0097046, "run_id": "campaign-20260802-reckoner-t2", "status": "constructed"} +{"annotations": ["PE CY2026 static accrual vs HMRC projected FY direct effects (first-year cash vs fuller-year; TIE/behavioral in HMRC reckoner per their methodology notes); |magnitude| ratios", "fuller-year FY2027-28 external value 6250000000.0 (ratio 1.1097406635748241)"], "computed_at": "2026-08-06T18:42:04.081907+00:00", "data_bundle": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", "engine_version": "2.89.2", "external_claim_match": {"claim_id": "39bb15c3c6084fd9c437"}, "pe_construction": "gov.hmrc.income_tax.rates.uk[1].threshold: 37700 -> 41470.0 | heads income_tax", "pe_value": -6935879147.342651, "run_id": "campaign-20260802-reckoner-t2", "status": "constructed"} +{"annotations": ["PE CY2026 static accrual vs HMRC projected FY direct effects (first-year cash vs fuller-year; TIE/behavioral in HMRC reckoner per their methodology notes); |magnitude| ratios", "fuller-year FY2027-28 external value 5000000.0 (ratio 6.6777414854248045)"], "computed_at": "2026-08-06T18:42:04.081907+00:00", "data_bundle": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", "engine_version": "2.89.2", "external_claim_match": {"claim_id": "e2aac04633b000d98d17"}, "pe_construction": "gov.hmrc.income_tax.rates.savings_starter_rate.allowance: 5000 -> 5100 | heads income_tax", "pe_value": -33388707.427124023, "run_id": "campaign-20260802-reckoner-t2", "status": "constructed"} +{"annotations": ["PE CY2026 static accrual vs HMRC projected FY direct effects (first-year cash vs fuller-year; TIE/behavioral in HMRC reckoner per their methodology notes); |magnitude| ratios", "fuller-year FY2027-28 external value 30000000.0 (ratio 2.322522566512044)"], "computed_at": "2026-08-06T18:42:04.081907+00:00", "data_bundle": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", "engine_version": "2.89.2", "external_claim_match": {"claim_id": "23cb5f489c32e17b4b49"}, "pe_construction": "gov.hmrc.income_tax.allowances.personal_savings_allowance.basic: 1000 -> 1100; gov.hmrc.income_tax.allowances.personal_savings_allowance.higher: 500 -> 550 | heads income_tax", "pe_value": -69675676.99536133, "run_id": "campaign-20260802-reckoner-t2", "status": "constructed"} +{"annotations": ["PE CY2026 static accrual vs HMRC projected FY direct effects (first-year cash vs fuller-year; TIE/behavioral in HMRC reckoner per their methodology notes); |magnitude| ratios", "fuller-year FY2027-28 external value 70000000.0 (ratio 0.7539641937203544)"], "computed_at": "2026-08-06T18:42:04.081907+00:00", "data_bundle": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", "engine_version": "2.89.2", "external_claim_match": {"claim_id": "e05ac73d999627cb18cb"}, "pe_construction": "gov.hmrc.income_tax.allowances.dividend_allowance: 500 -> 600 | heads income_tax", "pe_value": -52777493.560424805, "run_id": "campaign-20260802-reckoner-t2", "status": "constructed"} +{"annotations": ["PE CY2026 static accrual vs HMRC projected FY direct effects (first-year cash vs fuller-year; TIE/behavioral in HMRC reckoner per their methodology notes); |magnitude| ratios", "fuller-year FY2027-28 external value 230000000.0 (ratio 0.8821476470050315)"], "computed_at": "2026-08-06T18:42:04.081907+00:00", "data_bundle": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", "engine_version": "2.89.2", "external_claim_match": {"claim_id": "b8be62add3033280162d"}, "pe_construction": "gov.hmrc.national_insurance.class_1.thresholds.primary_threshold: 241.73 -> 243.73 | heads national_insurance", "pe_value": -202893958.81115723, "run_id": "campaign-20260802-reckoner-t2", "status": "constructed"} +{"annotations": ["PE CY2026 static accrual vs HMRC projected FY direct effects (first-year cash vs fuller-year; TIE/behavioral in HMRC reckoner per their methodology notes); |magnitude| ratios", "fuller-year FY2027-28 external value 430000000.0 (ratio 0.8295004542308718)"], "computed_at": "2026-08-06T18:42:04.081907+00:00", "data_bundle": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", "engine_version": "2.89.2", "external_claim_match": {"claim_id": "eea5432a24523ee02327"}, "pe_construction": "gov.hmrc.national_insurance.class_1.thresholds.secondary_threshold: 96 -> 98 | heads ni_employer", "pe_value": -356685195.3192749, "run_id": "campaign-20260802-reckoner-t2", "status": "constructed"} +{"annotations": ["PE CY2026 static accrual vs HMRC projected FY direct effects (first-year cash vs fuller-year; TIE/behavioral in HMRC reckoner per their methodology notes); |magnitude| ratios", "fuller-year FY2027-28 external value 220000000.0 (ratio 1.1274266446869243)"], "computed_at": "2026-08-06T18:42:04.081907+00:00", "data_bundle": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", "engine_version": "2.89.2", "external_claim_match": {"claim_id": "d2fbcdc3204a5b6e27d5"}, "pe_construction": "gov.hmrc.national_insurance.class_1.thresholds.upper_earnings_limit: 966.73 -> 976.73 | heads national_insurance", "pe_value": 248033861.83112335, "run_id": "campaign-20260802-reckoner-t2", "status": "constructed"} +{"annotations": ["PE CY2026 static accrual vs HMRC projected FY direct effects (first-year cash vs fuller-year; TIE/behavioral in HMRC reckoner per their methodology notes); |magnitude| ratios", "fuller-year FY2027-28 external value 15000000.0 (ratio 1.1669151654261272)"], "computed_at": "2026-08-06T18:42:04.081907+00:00", "data_bundle": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", "engine_version": "2.89.2", "external_claim_match": {"claim_id": "ff1a81248008fdad8c35"}, "pe_construction": "gov.hmrc.national_insurance.class_4.thresholds.lower_profits_limit: 12570 -> 12674 | heads national_insurance", "pe_value": -17503727.481391907, "run_id": "campaign-20260802-reckoner-t2", "status": "constructed"} +{"annotations": ["PE CY2026 static accrual vs HMRC projected FY direct effects (first-year cash vs fuller-year; TIE/behavioral in HMRC reckoner per their methodology notes); |magnitude| ratios", "fuller-year FY2027-28 external value 10000000.0 (ratio 1.5204671735198974)"], "computed_at": "2026-08-06T18:42:04.081907+00:00", "data_bundle": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", "engine_version": "2.89.2", "external_claim_match": {"claim_id": "b601650048d20e2606c9"}, "pe_construction": "gov.hmrc.national_insurance.class_4.thresholds.upper_profits_limit: 50270 -> 50790 | heads national_insurance", "pe_value": 15204671.735198975, "run_id": "campaign-20260802-reckoner-t2", "status": "constructed"} +{"annotations": ["PE CY2026 static accrual vs HMRC projected FY direct effects (first-year cash vs fuller-year; TIE/behavioral in HMRC reckoner per their methodology notes); |magnitude| ratios", "fuller-year FY2027-28 external value 240000000.0 (ratio 1.2828826171198526)"], "computed_at": "2026-08-06T18:42:04.081907+00:00", "data_bundle": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", "engine_version": "2.89.2", "external_claim_match": {"claim_id": "6f708fb9feb19cb70eac"}, "pe_construction": "gov.hmrc.child_benefit.amount.additional: 17.9 -> 18.9 | heads child_benefit", "pe_value": 307891828.10876465, "run_id": "campaign-20260802-reckoner-t2", "status": "constructed"} +{"annotations": ["PE CY2026 static accrual vs HMRC projected FY direct effects (first-year cash vs fuller-year; TIE/behavioral in HMRC reckoner per their methodology notes); |magnitude| ratios", "fuller-year FY2027-28 external value 2000000000.0 (ratio 2.1959396474327697)"], "computed_at": "2026-08-06T18:42:04.081907+00:00", "data_bundle": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", "engine_version": "2.89.2", "external_claim_match": {"claim_id": "8e1c1f9b430e9d996f24"}, "pe_construction": "gov.hmrc.national_insurance.class_1.rates.employee.additional: 0.02 -> 0.03 | heads national_insurance", "pe_value": 4391879294.86554, "run_id": "campaign-20260802-reckoner-t2", "status": "constructed"} diff --git a/tests/test_campaign_ingest.py b/tests/test_campaign_ingest.py index ba5b77c..ef60d6c 100644 --- a/tests/test_campaign_ingest.py +++ b/tests/test_campaign_ingest.py @@ -95,16 +95,22 @@ def test_full_attach_on_committed_db(db_copy): ("wic", "constructed"): 6, } assert all(r["urban"] == r["n"] for r in urb) - # Provenance verbatim from the staging: real engine + certified bundle. + # Provenance verbatim from the staging: real engine + certified + # bundle per country (the UK reckoner family rides the committed DB + # from the campaign-UK producer; the US ingest must not disturb it). rows = conn.execute( "SELECT DISTINCT engine_version, data_bundle FROM pe_results" - " WHERE run_id LIKE 'campaign-%'" + " WHERE run_id LIKE 'campaign-%' ORDER BY engine_version" ).fetchall() assert [tuple(r) for r in rows] == [ ( "1.764.6", "populace-us-2024-buildp-sparse-rmloss100-cae8640-20260728T011454Z", - ) + ), + ( + "2.89.2", + "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", + ), ] # The CPSP TCJA-world attach lands on the claim whose reform is the # scenario (value 13.3% — the brief's number). @@ -181,19 +187,28 @@ def test_full_attach_on_committed_db(db_copy): def test_reingest_idempotent(db_copy): + UK_RUN = "campaign-20260802-reckoner-t2" first = ingest(db_copy) again = ingest(db_copy) assert again == first conn = sqlite3.connect(db_copy) n = conn.execute( "SELECT COUNT(*) FROM pe_results WHERE run_id LIKE 'campaign-%'" + " AND run_id != ?", + (UK_RUN,), ).fetchone()[0] n_ex = conn.execute( "SELECT COUNT(*) FROM pe_exhibits WHERE run_id LIKE 'campaign-%'" ).fetchone()[0] + # Deletion is scoped to the run_ids being re-ingested: the US + # re-ingest must leave the committed UK reckoner family untouched. + uk = conn.execute( + "SELECT COUNT(*) FROM pe_results WHERE run_id = ?", (UK_RUN,) + ).fetchone()[0] conn.close() assert n == first["attached"] assert n_ex == first["exhibits"] + assert uk == 14 def test_metaless_exhibit_defers(db_copy, tmp_path): diff --git a/tests/test_campaign_uk_producer.py b/tests/test_campaign_uk_producer.py new file mode 100644 index 0000000..c49bbe7 --- /dev/null +++ b/tests/test_campaign_uk_producer.py @@ -0,0 +1,232 @@ +"""Tests for the UK campaign staging producer (archive -> resolved).""" + +import json +import shutil + +import pytest + +from scorecard_db import ScorecardDB +from scorecard_db.produce_campaign_uk import ARCHIVE, RUNS, produce + +DB = ARCHIVE.parent.parent.parent / "data" / "scorecard.db" + +pytestmark = pytest.mark.skipif( + not (ARCHIVE.exists() and DB.exists()), + reason="campaign archive or committed DB not present", +) + + +def test_produce_resolves_reckoner_and_blocks_the_rest(tmp_path): + summary = produce(DB, out_dir=tmp_path) + assert summary["resolved"] == {"hmrc_reckoner_t2": 14} + assert set(summary["blocked"]) == { + "free_joins", + "obr_measures", + "two_child", + "uprating_april2026", + } + # only the resolved family lands in the staging dir + assert [p.name for p in sorted(tmp_path.glob("*.jsonl"))] == [ + "hmrc_reckoner_t2.jsonl" + ] + + rows = [ + json.loads(line) + for line in (tmp_path / "hmrc_reckoner_t2.jsonl").read_text().splitlines() + ] + assert len(rows) == 14 + db = ScorecardDB(DB) + cids = set() + for r in rows: + match = r["external_claim_match"] + # the strict claim_id-direct form, nothing else + assert set(match) == {"claim_id"} + cids.add(match["claim_id"]) + hit = db.conn.execute( + "SELECT source, metric FROM external_scores WHERE claim_id = ?", + (match["claim_id"],), + ).fetchone() + assert hit is not None + assert (hit["source"], hit["metric"]) == ("uk_hmrc", "revenue_change") + # everything else preserved verbatim from the archive + assert r["run_id"] == "campaign-20260802-reckoner-t2" + assert r["status"] == "constructed" + db.close() + assert len(cids) == 14 # no two rows share a claim + + # the archive itself is untouched (frozen by #70) — descriptors there + # still carry the under-specified conditions form + archived = [ + json.loads(line) + for line in (ARCHIVE / "hmrc_reckoner_t2.jsonl").read_text().splitlines() + ] + assert all("claim_id" not in a["external_claim_match"] for a in archived) + + +def _composition(name): + rows = [ + json.loads(line) + for line in (ARCHIVE / f"{name}.jsonl").read_text().splitlines() + if line.strip() + ] + tally: dict[str, int] = {} + for r in rows: + if r.get("exhibit") and "external_claim_match" not in r: + # the DEFINING property is the missing exhibit_meta (that is + # what makes ingest defer it) — a refreeze that adds meta + # makes the row ingestible and must change this tally, not + # slip through as "still metaless" (round-2 gate) + key = "metaless_exhibit" if "exhibit_meta" not in r else "exhibit_with_meta" + else: + m = r["external_claim_match"] + key = f"{m['source']}:{m['metric']}" + tally[key] = tally.get(key, 0) + 1 + return tally + + +def test_blocked_dispositions_match_the_archive_exactly(): + """The stated reasons are per-row accurate (round-1 gate: the first + reasons generalized each family from its first row — free_joins is + NOT just OBR receipts). A re-frozen archive that changes any + family's composition fails here and forces a new disposition.""" + assert _composition("free_joins") == { + "obr:revenue_level": 7, + "uk_dwp:benefit_cost": 7, + "metaless_exhibit": 2, + } + assert _composition("obr_measures") == { + "obr:revenue_change": 9, + "metaless_exhibit": 1, + } + assert _composition("two_child") == { + "resolution_foundation:poverty_count_change": 1, + "resolution_foundation:reform_fiscal_cost": 1, + "ukmod:poverty_count_change": 1, + "metaless_exhibit": 1, + } + assert _composition("uprating_april2026") == { + "resolution_foundation:benefit_uprating_pct": 3, + "metaless_exhibit": 1, + } + + +def test_blocked_targets_have_no_claims_yet(): + """'Genuinely blocked' is machine-checked: zero DB claims exist for + every blocked target shape. When a future lane stages one of these + (OBR receipts, DWP expenditure forecasts, the OBR costings DB, RF, + a UKMOD 2CL-reform claim), this fails and forces the deliberate + unblock instead of a silent one.""" + db = ScorecardDB(DB) + + def n(sql, *args): + return db.conn.execute(sql, args).fetchone()[0] + + assert ( + n( + "SELECT COUNT(*) FROM external_scores WHERE source='obr'" + " AND metric IN ('revenue_level', 'revenue_change')" + ) + == 0 + ) + for source in ("uk_dwp", "resolution_foundation"): + assert n("SELECT COUNT(*) FROM external_scores WHERE source=?", source) == 0 + assert ( + n( + "SELECT COUNT(*) FROM external_scores WHERE source='ukmod'" + " AND metric='poverty_count_change'" + ) + == 0 + ) + db.close() + + +def test_resolution_chain_is_closed(tmp_path, monkeypatch): + """A construction the archived runs never executed must fail the + produce, never guess a claim.""" + import scorecard_db.produce_campaign_uk as mod + + bad_dir = tmp_path / "uk" + bad_dir.mkdir() + for p in ARCHIVE.glob("*.jsonl"): + shutil.copy(p, bad_dir / p.name) + target = bad_dir / "hmrc_reckoner_t2.jsonl" + rows = [json.loads(line) for line in target.read_text().splitlines()] + rows[0]["pe_construction"] = "gov.made.up.parameter: 1 -> 2 | heads income_tax" + target.write_text("\n".join(json.dumps(r) for r in rows) + "\n") + monkeypatch.setattr(mod, "ARCHIVE", bad_dir) + with pytest.raises(ValueError, match="matches no"): + produce(DB, out_dir=tmp_path / "out") + + +def test_new_archive_family_needs_a_disposition(tmp_path, monkeypatch): + """Every archived family must be resolved, copied, or blocked-with- + reason — a new family cannot be silently skipped.""" + import scorecard_db.produce_campaign_uk as mod + + new_dir = tmp_path / "uk" + new_dir.mkdir() + for p in ARCHIVE.glob("*.jsonl"): + shutil.copy(p, new_dir / p.name) + (new_dir / "brand_new_family.jsonl").write_text("{}\n") + monkeypatch.setattr(mod, "ARCHIVE", new_dir) + with pytest.raises(ValueError, match="without a disposition"): + produce(DB, out_dir=tmp_path / "out") + + +def test_attach_round_trip(tmp_path): + """Resolved staging attaches through ingest_campaign's strict + claim_id-direct form: 14 results land and re-ingest is idempotent.""" + from scorecard_db.ingest_campaign import ingest + + db_copy = tmp_path / "scorecard.db" + shutil.copy(DB, db_copy) + staged = tmp_path / "uk_resolved" + produce(db_copy, out_dir=staged) + + summary = ingest(db_copy, staged_dir=staged) + assert summary["attached"] == 14 + assert summary["exhibits"] == 0 + assert summary["exhibits_deferred"] == [] + + again = ingest(db_copy, staged_dir=staged) + assert again["attached"] == 14 + + db = ScorecardDB(db_copy) + n = db.conn.execute( + "SELECT COUNT(*) FROM pe_results WHERE run_id = ?", + ("campaign-20260802-reckoner-t2",), + ).fetchone()[0] + assert n == 14 + # every attached result rides a claim in the reckoner family and the + # result side executed a current-law PE baseline (stamped by the + # campaign ingest), distinct from the claim's indexed HMRC baseline — + # the cross-world guard is what keeps these 'constructed' + cross = db.conn.execute( + "SELECT COUNT(*) FROM pe_results r JOIN external_scores s USING" + " (claim_id) WHERE r.run_id = 'campaign-20260802-reckoner-t2'" + " AND s.baseline_key = r.baseline_key" + ).fetchone()[0] + assert cross == 0 + db.close() + + +def test_collation_hints_match_db_options_exactly(): + """The resolution key: HMRC's verbatim change label in the t2 + collation equals the ingested claim's option string, character for + character, for every resolved row.""" + import csv + + with open(RUNS / "t2_collation.csv") as f: + hints = {r["key"]: r["hmrc_hint"] for r in csv.DictReader(f)} + db = ScorecardDB(DB) + options = { + r[0] + for r in db.conn.execute( + "SELECT DISTINCT json_extract(conditions, '$.option')" + " FROM external_scores WHERE source='uk_hmrc'" + " AND metric='revenue_change'" + ) + } + db.close() + missing = {k: h for k, h in hints.items() if h not in options} + assert not missing, missing