Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -296,6 +296,17 @@ files are diagnostic reports for gating and review. Consumer-contract rows must
carry canonical constraints explicitly in `universe_constraints`; source-layout
`dimensions` are metadata and are not target constraints.

For the UK source-package feed, build the curated UK suite and then a facts-only
consumer artifact:

```bash
uv run chronicle build-bundle --suite uk --out /tmp/chronicle-uk --replace
uv run chronicle build-consumer-artifact --facts /tmp/chronicle-uk --out /tmp/chronicle-uk-artifact --replace
```

`--year` is inert for `--suite uk` because the UK packages are year-pinned.
The US off-year bundle behavior is unchanged and out of scope here.

Builds without an Axiom CLI still pass when the source package is otherwise
valid, but `agent_acceptance.json` warns with
`concept_alignment_validation_skipped`. For strict agent review, require every
Expand Down
107 changes: 107 additions & 0 deletions chronicle/bundle.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,113 @@
BUNDLE_COVERAGE_SCHEMA_VERSION = "ledger.bundle_coverage.v1"
BUNDLE_SOURCES_SCHEMA_VERSION = "ledger.bundle_sources.v1"
DEFAULT_BUNDLE_SOURCES = tuple(sorted(SOURCE_PACKAGE_ALIASES))
UK_BUNDLE_SOURCE_PREFIXES = (
"dft",
"dwp",
"hmrc",
"isc",
"mhclg",
"nisra",
"nrs",
"obr",
"ons",
"scotgov",
"slc",
"voa",
"welshgov",
)
UK_BUNDLE_SOURCES = (
"dft-nts-vehicle-ownership-2024",
"dwp-benefit-cap-november-2025",
"dwp-benefit-statistics-february-2026",
"dwp-pip-daily-living-foi-2025",
"dwp-uc-deductions-march-2025-february-2026",
"dwp-uc-households-by-constituency-children-may-2025",
"dwp-uc-households-by-constituency-may-2025",
"dwp-uc-households-by-local-authority-may-2025",
"dwp-uc-households-children-may-2025",
"dwp-uc-households-family-type-may-2025",
"dwp-uc-payment-distribution-may-2025",
"dwp-uc-scotland-youngest-child-may-2025",
"dwp-uc-two-child-limit-2025",
"hmrc-cgt-statistics-2025",
"hmrc-salary-sacrifice-reform-2029-headcounts",
"hmrc-salary-sacrifice-relief-2024-25",
"hmrc-spi-income-bands-2023-24",
"hmrc-spi-income-by-area-2023-24",
"hmrc-vat-firm-sector-targets-2024-25",
"hmrc-vat-firm-targets-2024-25",
"isc-annual-census-2023",
"isc-annual-census-2024",
"mhclg-council-tax-levels-england-2026-27",
"mhclg-ehs-weekly-housing-costs-2023-24",
"nisra-census2021-households-lgd",
"nisra-census2021-households-pcon24",
"nisra-census2021-tenure-lgd",
"nisra-pcon24-population-by-age-2024",
"nrs-census2022-households-ukpc24",
"nrs-census2022-uv404-tenure-council-area",
"nrs-pcon24-population-by-age-2024",
"obr-efo-aggregates-march-2026",
"obr-efo-economy-march-2026",
"obr-efo-expenditure-march-2026",
"obr-efo-receipts-march-2026",
"ons-census2021-ts041-households-lad",
"ons-census2021-ts041-households-pcon24",
"ons-census2021-ts054-tenure-lad",
"ons-families-households-2025",
"ons-lad-population-by-age-2024",
"ons-mye-2023-england-regions",
"ons-mye-2023-uk-countries",
"ons-mye-2024-uk",
"ons-national-balance-sheet-land-2025",
"ons-pcon24-population-by-age-2024",
"ons-pipr-private-rent-march-2026",
"ons-pipr-rents-by-area-june-2026",
"ons-public-sector-employment-2026",
"ons-savings-interest-income",
"ons-small-area-income-msoa-fye2023",
"ons-subnational-dwellings-by-tenure-2024",
"ons-uk-business-firm-sector-targets-2025",
"ons-uk-business-firm-targets-2025",
"ons-uk-population-projections-2024",
"scotgov-band-d-council-tax-rates-2026-27",
"scotgov-band-d-equivalents-2025",
"scotgov-council-tax-bands-2025",
"scotgov-scottish-budget-social-security-assistance-2026",
"slc-student-loan-borrower-forecasts-england-2025",
"slc-student-loan-repayments-england-2025",
"slc-student-loan-repayments-northern-ireland-2025",
"slc-student-loan-repayments-scotland-2025",
"slc-student-loan-repayments-wales-2025",
"slc-student-support-england-2025",
"voa-council-tax-bands-2025",
"voa-council-tax-stock-by-lad-2025",
"welshgov-council-tax-levels-2026-27",
)


def uk_bundle_sources_from_aliases() -> tuple[str, ...]:
"""Return UK-package aliases implied by the source-package directory prefixes."""
return tuple(
sorted(
alias
for alias, path in SOURCE_PACKAGE_ALIASES.items()
if path.parts and path.parts[0] in UK_BUNDLE_SOURCE_PREFIXES
)
)


def assert_uk_bundle_sources_match_aliases() -> None:
"""Fail loudly if the curated UK suite omits a UK-prefixed alias."""
expected = uk_bundle_sources_from_aliases()
if UK_BUNDLE_SOURCES != expected:
missing = sorted(set(expected) - set(UK_BUNDLE_SOURCES))
extra = sorted(set(UK_BUNDLE_SOURCES) - set(expected))
raise ValueError(
"UK_BUNDLE_SOURCES drifted from UK-prefixed SOURCE_PACKAGE_ALIASES: "
f"missing={missing}, extra={extra}"
)


@dataclass(frozen=True)
Expand Down
19 changes: 18 additions & 1 deletion chronicle/harness.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,12 @@
publish_derived_artifacts,
publish_source_artifacts,
)
from chronicle.bundle import BuildBundleReport, build_bundle
from chronicle.bundle import (
UK_BUNDLE_SOURCES,
BuildBundleReport,
assert_uk_bundle_sources_match_aliases,
build_bundle,
)
from chronicle.concepts import ConceptAlignmentReport, validate_concept_alignments
from chronicle.consumer_contract import (
ConsumerFactExportReport,
Expand Down Expand Up @@ -262,12 +267,18 @@ def build_bundle_dir(
*,
year: int,
sources: list[str | Path] | None = None,
suite: str | None = None,
axiom_command: list[str] | None = None,
axiom_roots: list[str | Path] | None = None,
require_axiom_validation: bool = False,
replace: bool = False,
) -> BuildBundleReport:
"""Build a merged Chronicle consumer bundle from source-package suites."""
if suite == "uk":
if sources:
raise ValueError("--suite uk cannot be combined with --source.")
assert_uk_bundle_sources_match_aliases()
sources = list(UK_BUNDLE_SOURCES)
return build_bundle(
output_dir,
year=year,
Expand Down Expand Up @@ -707,6 +718,11 @@ def main(argv: list[str] | None = None) -> int:
"path. May be repeated. Defaults to available packages for --year."
),
)
bundle_parser.add_argument(
"--suite",
choices=["uk"],
help="Curated source-package suite to build.",
)
bundle_parser.add_argument(
"--out",
type=Path,
Expand Down Expand Up @@ -1255,6 +1271,7 @@ def main(argv: list[str] | None = None) -> int:
args.out,
year=args.year,
sources=args.source,
suite=args.suite,
axiom_command=axiom_command,
axiom_roots=args.axiom_root,
require_axiom_validation=args.require_axiom_validation,
Expand Down
13 changes: 6 additions & 7 deletions chronicle/jurisdictions/uk/__init__.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,8 @@
"""United Kingdom Chronicle target loaders."""
"""United Kingdom Chronicle target loaders.

from chronicle.targets.loaders import (
load_hmrc_targets,
load_obr_targets,
load_ons_targets,
)
UK national measures are facts-only source packages. Target selection belongs
to the consumer contract, so this module intentionally exports no legacy DB
target loaders.
"""

__all__ = ["load_hmrc_targets", "load_obr_targets", "load_ons_targets"]
__all__: list[str] = []
2 changes: 2 additions & 0 deletions chronicle/source_package.py
Original file line number Diff line number Diff line change
Expand Up @@ -295,6 +295,8 @@
"voa-council-tax-stock-by-lad-2025": Path("voa/council_tax_stock_by_lad_2025"),
"obr-efo-receipts-march-2026": Path("obr/efo_receipts_march_2026"),
"obr-efo-expenditure-march-2026": Path("obr/efo_expenditure_march_2026"),
"obr-efo-economy-march-2026": Path("obr/efo_economy_march_2026"),
"obr-efo-aggregates-march-2026": Path("obr/efo_aggregates_march_2026"),
"ons-national-balance-sheet-land-2025": Path(
"ons/national_balance_sheet_land_2025"
),
Expand Down
6 changes: 0 additions & 6 deletions chronicle/targets/loaders.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,7 @@
from db.etl_cbo import load_cbo_targets
from db.etl_census import load_census_targets
from db.etl_cps import load_cps_targets
from db.etl_hmrc import load_hmrc_targets
from db.etl_medicaid import load_medicaid_targets
from db.etl_obr import load_obr_targets
from db.etl_ons import load_ons_targets
from db.etl_snap import load_snap_targets
from db.etl_soi import load_soi_targets
from db.etl_soi_credits import load_soi_credits_targets
Expand All @@ -24,10 +21,7 @@
"load_cbo_targets",
"load_census_targets",
"load_cps_targets",
"load_hmrc_targets",
"load_medicaid_targets",
"load_obr_targets",
"load_ons_targets",
"load_snap_targets",
"load_soi_targets",
"load_soi_credits_targets",
Expand Down
47 changes: 19 additions & 28 deletions db/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -94,13 +94,6 @@ def cmd_load(args):
f"Loaded ACA Marketplace targets for years: {years or 'all available'}"
)

if args.source == "hmrc" or args.source == "all":
from .etl_hmrc import load_hmrc_targets

years = [int(y) for y in args.years.split(",")] if args.years else None
load_hmrc_targets(session, years=years)
print(f"Loaded HMRC targets for years: {years or 'all available'}")

if args.source == "census" or args.source == "all":
from .etl_census import load_census_targets

Expand Down Expand Up @@ -143,20 +136,6 @@ def cmd_load(args):
load_cbo_targets(session, years=years)
print(f"Loaded CBO projections for years: {years or 'all available'}")

if args.source == "obr" or args.source == "all":
from .etl_obr import load_obr_targets

years = [int(y) for y in args.years.split(",")] if args.years else None
load_obr_targets(session, years=years)
print(f"Loaded OBR projections for years: {years or 'all available'}")

if args.source == "ons" or args.source == "all":
from .etl_ons import load_ons_targets

years = [int(y) for y in args.years.split(",")] if args.years else None
load_ons_targets(session, years=years)
print(f"Loaded ONS projections for years: {years or 'all available'}")


def cmd_stats(args):
"""Show database statistics."""
Expand Down Expand Up @@ -198,13 +177,17 @@ def cmd_load_source_files(args):
include_us = args.jurisdiction in {"all", "us"}
include_uk = args.jurisdiction in {"all", "uk"}
specs = pe_source_specs(
pe_us_root=Path(args.pe_us_root),
pe_uk_root=Path(args.pe_uk_root),
pe_us_root=Path(args.pe_us_root) if args.pe_us_root else None,
pe_uk_root=Path(args.pe_uk_root) if args.pe_uk_root else None,
include_us=include_us,
include_uk=include_uk,
)
if args.limit:
specs = specs[: args.limit]
if not specs and not args.limit:
raise ValueError(
"Refusing to prune source artifacts with an empty source-file inventory."
)

with Session(engine) as session:
results = ingest_source_artifacts(session, specs)
Expand Down Expand Up @@ -292,6 +275,17 @@ def cmd_query(args):
)


def _pe_source_root_env_default(jurisdiction: str) -> str | None:
from .pe_source_inventory import (
PE_UK_DATA_ROOT_ENV,
PE_US_DATA_ROOT_ENV,
_env_value,
)

env_var = PE_US_DATA_ROOT_ENV if jurisdiction == "us" else PE_UK_DATA_ROOT_ENV
return _env_value(env_var)


def main():
parser = argparse.ArgumentParser(description="Manage Chronicle target input data")
parser.add_argument("--db", help=f"Database path (default: {DEFAULT_DB_PATH})")
Expand All @@ -315,15 +309,12 @@ def main():
"snap",
"medicaid",
"aca",
"hmrc",
"census",
"ssa",
"ssi",
"bls",
"cps",
"cbo",
"obr",
"ons",
"all",
],
help="Data source to load",
Expand Down Expand Up @@ -355,12 +346,12 @@ def main():
)
source_parser.add_argument(
"--pe-us-root",
default="/Users/maxghenis/PolicyEngine/policyengine-us-data",
default=_pe_source_root_env_default("us"),
help="Path to the policyengine-us-data checkout",
)
source_parser.add_argument(
"--pe-uk-root",
default="/Users/maxghenis/PolicyEngine/policyengine-uk-data",
default=_pe_source_root_env_default("uk"),
help="Path to the policyengine-uk-data checkout",
)
source_parser.add_argument(
Expand Down
Binary file not shown.
18 changes: 18 additions & 0 deletions db/data/obr/efo_aggregates_march_2026/manifest.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
source_id: obr
package_id: obr-efo-aggregates-march-2026
dataset: obr_obr-efo-aggregates-march-2026
source_page: https://obr.uk/efo/economic-and-fiscal-outlook-march-2026/
table: 'EFO March 2026 fiscal supplementary tables: aggregates'
files:
2026:
filename: efo_aggregates.xlsx
source_url: https://obr.uk/download/march-2026-economic-and-fiscal-outlook-detailed-forecast-tables-aggregates/
sha256: f9eee880e9f39cc8c86d288a803a032b735fea5f9bbd1532f26bfca898a669d9
size_bytes: 231008
fetched_at: '2026-08-19T00:00:00+00:00'
storage:
r2:
provider: r2
bucket: ledger-raw
key: raw/obr/obr-efo-aggregates-march-2026/2026/f9eee880e9f39cc8c86d288a803a032b735fea5f9bbd1532f26bfca898a669d9/efo_aggregates.xlsx
uri: r2://ledger-raw/raw/obr/obr-efo-aggregates-march-2026/2026/f9eee880e9f39cc8c86d288a803a032b735fea5f9bbd1532f26bfca898a669d9/efo_aggregates.xlsx
Binary file not shown.
18 changes: 18 additions & 0 deletions db/data/obr/efo_economy_march_2026/manifest.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
source_id: obr
package_id: obr-efo-economy-march-2026
dataset: obr_obr-efo-economy-march-2026
source_page: https://obr.uk/efo/economic-and-fiscal-outlook-march-2026/
table: 'EFO March 2026 detailed forecast tables: economy'
files:
2026:
filename: efo_economy.xlsx
source_url: https://obr.uk/download/march-2026-economic-and-fiscal-outlook-detailed-forecast-tables-economy/
sha256: fde1de4bc5424dcf16c7b5c42eb75a35198e347f54db47690e95ede7541db510
size_bytes: 483251
fetched_at: '2026-08-19T00:00:00+00:00'
storage:
r2:
provider: r2
bucket: ledger-raw
key: raw/obr/obr-efo-economy-march-2026/2026/fde1de4bc5424dcf16c7b5c42eb75a35198e347f54db47690e95ede7541db510/efo_economy.xlsx
uri: r2://ledger-raw/raw/obr/obr-efo-economy-march-2026/2026/fde1de4bc5424dcf16c7b5c42eb75a35198e347f54db47690e95ede7541db510/efo_economy.xlsx
2 changes: 1 addition & 1 deletion db/data/ons/families_households_2025/manifest.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ source_id: ons
package_id: ons-families-households-2025
dataset: ons_ons-families-households-2025
source_page: https://www.ons.gov.uk/peoplepopulationandcommunity/birthsdeathsandmarriages/families/datasets/familiesandhouseholdsfamiliesandhouseholds
table: 'Families and households in the UK 2025, Table 7: households by type'
table: 'Families and households in the UK 2025, Tables 5 and 7: household size and households by type'
files:
2025:
filename: familiesandhouseholdsuk2025.xlsx
Expand Down
Loading
Loading