From b4ecea11586a8be65d24860404ac235f7c63b2c4 Mon Sep 17 00:00:00 2001 From: Vahid Ahmadi Date: Thu, 23 Jul 2026 10:29:20 +0100 Subject: [PATCH 1/4] Close the C2 firm-size seam in the pipeline (#192 step 2, #208) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The C1/C2 freeze closed the NOEMP/FIRMSIZE seam in code — banding.py grew noemp_to_canonical and made `coding` a required keyword — but not in the pipeline: no reader imported it. Each A-side reader banded independently, so `firms/banding.py` was a mapper with no callers and the person side and target side carried two band vocabularies while the floor artifacts claimed one ("bands come only from banding.py"). - banding.py: CanonicalBand.label and BandSpan.label give the wire vocabulary (1-9 / 10-49 / 50-99 / 100-499 / 500+). An inexact span renders as a joined run ("10-49|50-99") so a consumer that treats it as exact gets an unmatchable category, not a plausible wrong band. Pure stdlib, as before. - asec_firm_size: noemp_canonical_map(year) delegates to noemp_to_canonical and asserts exactness rather than emitting a straddle; the reader emits `canonical_band` alongside the source-resolution `firm_size_band` (NOEMP splits 500-999 from 1000+, canonical C2 does not — neither is derived from the other's string). - sipp_jobs: EMPSIZE_CANONICAL_SPANS/EXACT from sipp_empsize_to_canonical; job-months and spells carry `estab_size_band`, derived in spells from the modal *code* rather than a mode of labels. Named apart from the ASEC column on purpose: SIPP measures establishment size (#192 finding 1), so a shared column name would let a join read a location headcount as an enterprise headcount. tests/data/test_firm_size_seam.py is the seam test asked for in the #192 thread: every NOEMP code round-trips to its intended band in every vintage, NIU stays None rather than falling into "1-9", the 50 edge survives the phantom 2019 relabeling, the readers are pinned to banding.py's mapping rather than a private one, SIPP's straddling codes stay inexact, and the two CPS code spaces stay distinguishable (NOEMP 6 = 500+ vs IPUMS FIRMSIZE 6 = 50-99 in the same vintage). Unit tier 740 -> 792. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/populace_dynamics/data/asec_firm_size.py | 47 +++- src/populace_dynamics/data/sipp_jobs.py | 52 ++++- src/populace_dynamics/firms/banding.py | 36 +++ tests/data/test_firm_size_seam.py | 218 +++++++++++++++++++ tests/tier_counts.json | 2 +- 5 files changed, 351 insertions(+), 4 deletions(-) create mode 100644 tests/data/test_firm_size_seam.py diff --git a/src/populace_dynamics/data/asec_firm_size.py b/src/populace_dynamics/data/asec_firm_size.py index 5f32d12d..920ae32d 100644 --- a/src/populace_dynamics/data/asec_firm_size.py +++ b/src/populace_dynamics/data/asec_firm_size.py @@ -55,12 +55,15 @@ import numpy as np import pandas as pd +from populace_dynamics.firms.banding import noemp_to_canonical + __all__ = [ "ASEC_FIRM_SIZE_YEARS", "CLASS_OF_WORKER_LABELS", "NOEMP_BANDS", "firm_size_tabulation", "noemp_band_map", + "noemp_canonical_map", "read_asec_firm_size", ] @@ -144,6 +147,42 @@ def noemp_band_map(year: int) -> dict[int, str]: return dict(NOEMP_BANDS) +def noemp_canonical_map(year: int) -> dict[int, str]: + """Return the NOEMP code -> canonical C2 band-label map. + + The C2 seam (#192 step 2; #208). Bands are **not** re-derived + here: every label comes from + :func:`populace_dynamics.firms.banding.noemp_to_canonical`, so + the person side and the target side share one vocabulary and one + definition of the 50 edge. :data:`NOEMP_BANDS` is retained as the + source-resolution label (it splits 500-999 from 1000+, a + distinction canonical C2 does not carry); ``firm_size_band`` and + ``canonical_band`` are therefore both emitted, at different + coarseness, rather than one being derived from the other's + string. + + Raises: + ValueError: If ``year`` is outside the verified range. + """ + _check_supported_year(year) + labels: dict[int, str] = {} + for code in NOEMP_BANDS: + span = noemp_to_canonical(code, year) + # Total by construction: NOEMP 1-6 are firm-size reports in + # every supported vintage, so the NIU ``None`` branch is + # unreachable here and an inexact span would be a C2 defect + # (every NOEMP band nests a canonical band) — assert rather + # than silently emitting a straddle label. + if span is None or not span.exact: + raise AssertionError( + f"NOEMP {code} in ASEC {year} did not resolve to a " + f"single canonical band (got {span!r}); the C2 " + "mapping and this reader have diverged." + ) + labels[code] = span.label + return labels + + def _resolve_data_dir(data_dir: Path | None) -> Path: """Resolve the ASEC data directory from arg, env var, default.""" if data_dir is not None: @@ -216,7 +255,9 @@ def read_asec_firm_size( One row per person in the NOEMP universe (``WKSWORK > 0``, i.e. worked last calendar year), with columns ``person_id``, ``year``, ``income_year``, ``noemp``, - ``firm_size_band``, ``noemp_allocated``, ``ljcw``, + ``firm_size_band`` (source-resolution label), + ``canonical_band`` (the C2 vocabulary, via + ``firms/banding.py``), ``noemp_allocated``, ``ljcw``, ``class_of_worker``, ``industry_major``, ``industry_detailed``, ``wkswork``, and ``weight`` (``MARSUPWT / 100`` — the raw column carries two implied @@ -236,6 +277,7 @@ def read_asec_firm_size( FileNotFoundError: If no staged person file can be found. """ bands = noemp_band_map(year) + canonical = noemp_canonical_map(year) if path is not None: person_path = Path(path).expanduser() if not person_path.exists(): @@ -352,6 +394,9 @@ def read_asec_firm_size( # Total mappings: the domain + universe checks guarantee # NOEMP in 1-6 and LJCW in 1-7 here, so no fallback. "firm_size_band": universe["NOEMP"].map(bands), + # The C2 seam: canonical labels straight from + # firms/banding.py, never re-derived here (#208). + "canonical_band": universe["NOEMP"].map(canonical), "noemp_allocated": universe["I_NOEMP"] > 0, "ljcw": universe["LJCW"], "class_of_worker": universe["LJCW"].map(CLASS_OF_WORKER_LABELS), diff --git a/src/populace_dynamics/data/sipp_jobs.py b/src/populace_dynamics/data/sipp_jobs.py index c722c9e5..0cdadeea 100644 --- a/src/populace_dynamics/data/sipp_jobs.py +++ b/src/populace_dynamics/data/sipp_jobs.py @@ -49,11 +49,15 @@ import numpy as np import pandas as pd +from populace_dynamics.firms.banding import sipp_empsize_to_canonical + __all__ = [ "SIPP_JOB_YEARS", "MAX_JOB_SLOTS", "CLWRK_LABELS", "JBORSE_LABELS", + "EMPSIZE_CANONICAL_SPANS", + "EMPSIZE_CANONICAL_EXACT", "read_sipp_job_months", "job_spells", ] @@ -87,6 +91,32 @@ #: live in firms/banding.py, verified on #195). _EMPSIZE_CODES = frozenset(range(1, 9)) +#: EMPSIZE code -> canonical band-span label, straight from +#: :func:`populace_dynamics.firms.banding.sipp_empsize_to_canonical` +#: (the C2 seam, #192 step 2 / #208). **This is establishment size, +#: not firm size** (#192 finding 1): SIPP's redesign dropped the +#: all-locations question, so these labels describe the worker's +#: location and are a proxy-chain input to firm size, never a +#: substitute for it. The column is named accordingly. +#: +#: SIPP's inclusive upper bounds ("10 to 25", "26 to 50", ...) +#: straddle the canonical 10/50/100/500 edges, so most codes yield an +#: *inexact* span rendered as e.g. ``"10-49|50-99"``. That is the +#: contract: an ambiguous source band surfaces as an unmatchable +#: category rather than being collapsed to a plausible wrong band. +EMPSIZE_CANONICAL_SPANS: dict[int, str] = { + code: span.label + for code in sorted(_EMPSIZE_CODES) + if (span := sipp_empsize_to_canonical(code)) is not None +} + +#: EMPSIZE code -> whether its canonical span is a single band. +EMPSIZE_CANONICAL_EXACT: dict[int, bool] = { + code: span.exact + for code in sorted(_EMPSIZE_CODES) + if (span := sipp_empsize_to_canonical(code)) is not None +} + _MISSING = -9 _MISSING_ID = -999 @@ -238,8 +268,12 @@ def read_sipp_job_months( size — passed through **raw**: valid codes 1-8, the -9 sentinel for item nonresponse on employer jobs, and NaN for self-employment/other arrangements whose establishment size - is structurally NIU; downstream consumers such as - ``firms/banding.py`` must expect that mix), ``industry`` + is structurally NIU; ``firms/banding.py`` expects that mix), + ``estab_size_band``/``estab_size_band_exact`` (the canonical + C2 span for that code, from ``firms/banding.py`` — NaN + wherever ``empsize_code`` is; **establishment** size, so not + interchangeable with the ASEC reader's ``canonical_band``), + ``industry`` (string, passed through **unvalidated** — a full Census-industry allow-list is impractical, so sentinels can appear here and flow into ``job_spells``'s modal logic), @@ -432,6 +466,11 @@ def read_sipp_job_months( "clwrk": clwrk, "jborse": jborse, "empsize_code": empsize, + # The C2 seam. Establishment size, not firm size — + # see EMPSIZE_CANONICAL_SPANS. NaN where EMPSIZE is + # missing or the slot has no establishment. + "estab_size_band": empsize.map(EMPSIZE_CANONICAL_SPANS), + "estab_size_band_exact": empsize.map(EMPSIZE_CANONICAL_EXACT), "industry": raw[f"TJB{n}_IND"], "earnings": earnings.where(earnings != _MISSING_ID), "age": raw["TAGE"], @@ -510,6 +549,8 @@ def job_spells(job_months: pd.DataFrame) -> pd.DataFrame: within person), ``start_year``/``start_month``, ``end_year``/``end_month``, ``n_months``, ``job_id``, ``industry``/``empsize_code``/``class_of_worker`` (modal), + ``estab_size_band`` (the canonical span of the modal + ``empsize_code``), ``attributes_constant`` (False when any of the three varied within the spell — surfaced, never silently averaged), ``total_earnings``, ``earnings_share`` (spell earnings over @@ -551,6 +592,7 @@ def job_spells(job_months: pd.DataFrame) -> pd.DataFrame: "job_id", "industry", "empsize_code", + "estab_size_band", "class_of_worker", "attributes_constant", "total_earnings", @@ -631,6 +673,12 @@ def job_spells(job_months: pd.DataFrame) -> pd.DataFrame: "job_id": int(job_id), "industry": modal["industry"], "empsize_code": modal["empsize_code"], + # Derived from the *modal code*, not by taking a mode + # of labels: one source (firms/banding.py), and the + # band can never disagree with the code beside it. + "estab_size_band": EMPSIZE_CANONICAL_SPANS.get( + modal["empsize_code"] + ), "class_of_worker": modal["class_of_worker"], "attributes_constant": bool(constant), "total_earnings": ( diff --git a/src/populace_dynamics/firms/banding.py b/src/populace_dynamics/firms/banding.py index 67843686..2b3e84c3 100644 --- a/src/populace_dynamics/firms/banding.py +++ b/src/populace_dynamics/firms/banding.py @@ -87,6 +87,8 @@ "CanonicalBand", "BandSpan", "CANONICAL_BANDS", + "CANONICAL_BAND_LABELS", + "SPAN_LABEL_SEPARATOR", "band_of_count", "cps_firmsize_to_canonical", "noemp_to_canonical", @@ -124,9 +126,32 @@ def lo(self) -> int: def hi(self) -> float: return self.value[1] + @property + def label(self) -> str: + """Stable wire label for the band (the C2 column vocabulary). + + Readers emit this string, never a reader-local band name, so + that person-side and target-side band columns share one + vocabulary (#192 step 2; the seam #208 left latent). + """ + return CANONICAL_BAND_LABELS[self] + CANONICAL_BANDS: tuple[CanonicalBand, ...] = tuple(CanonicalBand) +#: The canonical band vocabulary. These strings — not any reader's +#: local band names — are what every C1-conforming column carries. +CANONICAL_BAND_LABELS: dict[CanonicalBand, str] = { + CanonicalBand.LT10: "1-9", + CanonicalBand.B10_49: "10-49", + CanonicalBand.B50_99: "50-99", + CanonicalBand.B100_499: "100-499", + CanonicalBand.B500_PLUS: "500+", +} + +#: Separator joining the band labels of an inexact (straddling) span. +SPAN_LABEL_SEPARATOR = "|" + @dataclass(frozen=True) class BandSpan: @@ -153,6 +178,17 @@ def band(self) -> CanonicalBand: ) return self.bands[0] + @property + def label(self) -> str: + """Wire label: the band label, or the straddled run joined. + + An inexact span renders as e.g. ``"10-49|50-99"`` rather than + collapsing to one band, so a consumer that silently treats it + as exact produces an unmatchable category instead of a + plausible wrong one. + """ + return SPAN_LABEL_SEPARATOR.join(b.label for b in self.bands) + def band_of_count(n: int) -> CanonicalBand: """Canonical band containing an exact employment count ``n >= 1``. diff --git a/tests/data/test_firm_size_seam.py b/tests/data/test_firm_size_seam.py new file mode 100644 index 00000000..829b4c92 --- /dev/null +++ b/tests/data/test_firm_size_seam.py @@ -0,0 +1,218 @@ +"""The C2 firm-size seam: readers -> ``firms/banding.py``. + +The seam Max flagged before the C1/C2 freeze (#192, #208): the +person-side readers and the target-side mapper name the same concept +in different code spaces, so wiring one into the other without an +adapter mis-bands silently. ``NOEMP 6`` is 1000+ while IPUMS +``FIRMSIZE 6`` is 50-99 — a factor-20 error that raises nothing. + +The freeze closed this in code (``banding.py`` grew +``noemp_to_canonical`` and refuses to guess a coding) but not in the +pipeline: no reader imported it. These tests pin the wiring, and are +the "one seam test that round-trips every NOEMP code through to +``CanonicalBand`` and asserts the intended bands" from the #192 +thread. + +Three properties, in the order they'd fail if the seam reopened: + +1. every raw code maps, and to the band the dictionary evidence says; +2. the readers emit *that* mapping rather than a private one; +3. the two CPS code spaces stay distinguishable, so a caller holding + the wrong one gets an error rather than a plausible wrong band. +""" + +from __future__ import annotations + +from pathlib import Path + +import pandas as pd +import pytest + +from populace_dynamics.data import asec_firm_size, sipp_jobs +from populace_dynamics.firms import banding +from populace_dynamics.firms.banding import ( + CanonicalBand, + cps_firmsize_to_canonical, + noemp_to_canonical, + sipp_empsize_to_canonical, +) + +# The intended bands, stated here independently of the mapper's +# tables so that editing one of them cannot silently move the other. +# Codes 5 (500-999) and 6 (1000+) both land in B500_PLUS: canonical +# C2 is coarser than NOEMP above 500, by design. +NOEMP_INTENDED: dict[int, CanonicalBand] = { + 1: CanonicalBand.LT10, + 2: CanonicalBand.B10_49, + 3: CanonicalBand.B50_99, + 4: CanonicalBand.B100_499, + 5: CanonicalBand.B500_PLUS, + 6: CanonicalBand.B500_PLUS, +} + +#: EMPSIZE code -> intended canonical span label. SIPP's inclusive +#: upper bounds straddle the canonical edges by one integer, so most +#: codes are inexact and must render as a joined run. +EMPSIZE_INTENDED: dict[int, str] = { + 1: "1-9", + 2: "10-49", + 3: "10-49|50-99", + 4: "50-99|100-499", + 5: "100-499", + 6: "100-499|500+", + 7: "500+", + 8: "500+", +} + + +class TestNoempRoundTrip: + """Every NOEMP code, every vintage, through to CanonicalBand.""" + + @pytest.mark.parametrize("year", [2011, 2018, 2019, 2024, 2025]) + @pytest.mark.parametrize("code", sorted(NOEMP_INTENDED)) + def test_every_code_maps_to_the_intended_band(self, code, year): + span = noemp_to_canonical(code, year) + assert span is not None + assert span.exact, ( + f"NOEMP {code} straddles a canonical edge in {year}; " + "every ASEC firm-size band is supposed to nest one." + ) + assert span.band is NOEMP_INTENDED[code] + + @pytest.mark.parametrize("year", [2011, 2018, 2019, 2024, 2025]) + def test_niu_is_none_not_a_band(self, year): + # Code 0 must not fall into LT10; a NIU banded as "1-9" would + # move non-workers into the smallest firm-size cell. + assert noemp_to_canonical(0, year) is None + + def test_the_50_edge_holds_across_the_phantom_relabeling(self): + # The 2019+ dictionaries relabel codes 2/3 as 10-24 / 25-99. + # If that relabeling were ever taken at face value, code 3 + # would stop resolving the 50 edge and every ACA-threshold + # cell would silently change population (#192). + for year in (2017, 2018, 2019, 2024): + assert noemp_to_canonical(3, year).band is CanonicalBand.B50_99 + assert noemp_to_canonical(2, year).band is CanonicalBand.B10_49 + + +class TestReadersEmitTheCanonicalMapping: + """The readers must not band independently of banding.py.""" + + @pytest.mark.parametrize("year", [2011, 2018, 2019, 2025]) + def test_asec_map_is_the_banding_map(self, year): + emitted = asec_firm_size.noemp_canonical_map(year) + assert emitted == { + code: NOEMP_INTENDED[code].label for code in NOEMP_INTENDED + } + + def test_asec_reader_emits_canonical_band(self, tmp_path): + rows = [{"NOEMP": code} for code in sorted(NOEMP_INTENDED)] + frame = pd.DataFrame( + [ + { + "PERIDNUM": 10_000 + i, + "I_NOEMP": 0, + "LJCW": 1, + "INDUSTRY": 770, + "WEIND": 4, + "WKSWORK": 52, + "WORKYN": 1, + "MARSUPWT": 100_000, + **row, + } + for i, row in enumerate(rows) + ] + ) + path = tmp_path / "pppub24.csv" + frame.to_csv(path, index=False) + + out = asec_firm_size.read_asec_firm_size(2024, path=path) + assert "canonical_band" in out.columns + for code, band in NOEMP_INTENDED.items(): + got = out.loc[out["noemp"] == code, "canonical_band"] + assert list(got) == [band.label] + + def test_sipp_spans_are_the_banding_spans(self): + assert sipp_jobs.EMPSIZE_CANONICAL_SPANS == EMPSIZE_INTENDED + for code, label in EMPSIZE_INTENDED.items(): + assert sipp_empsize_to_canonical(code).label == label + + def test_sipp_straddling_codes_stay_inexact(self): + # Collapsing 3/4/6 to a single band would invent precision + # SIPP's inclusive upper bounds do not have. + exact = sipp_jobs.EMPSIZE_CANONICAL_EXACT + assert [c for c, e in exact.items() if not e] == [3, 4, 6] + for code in (3, 4, 6): + assert banding.SPAN_LABEL_SEPARATOR in ( + sipp_jobs.EMPSIZE_CANONICAL_SPANS[code] + ) + + +class TestTheCodeSpacesStayDistinguishable: + """A caller holding the wrong coding must not get a band.""" + + def test_noemp_and_ipums_disagree_where_documented(self): + # The collision that motivated the seam: same integer, same + # vintage, different band. If these ever agree, the mapper + # has lost the distinction and the adapter is a no-op. + assert noemp_to_canonical(6, 2016).band is CanonicalBand.B500_PLUS + assert ( + cps_firmsize_to_canonical(6, 2016, coding="ipums_firmsize").band + is CanonicalBand.B50_99 + ) + + def test_coding_has_no_default(self): + with pytest.raises(TypeError): + cps_firmsize_to_canonical(6, 2016) + + def test_unknown_coding_raises(self): + with pytest.raises(ValueError, match="Unknown coding"): + cps_firmsize_to_canonical(6, 2016, coding="noemp") + + def test_pre_2011_refuses_rather_than_borrowing_a_vintage(self): + with pytest.raises(ValueError, match="2011"): + noemp_to_canonical(2, 2010) + + +class TestOneVocabulary: + """Person side and target side share one band vocabulary.""" + + def test_labels_are_the_canonical_set(self): + assert [b.label for b in banding.CANONICAL_BANDS] == [ + "1-9", + "10-49", + "50-99", + "100-499", + "500+", + ] + + def test_asec_canonical_labels_are_a_subset(self): + vocabulary = {b.label for b in banding.CANONICAL_BANDS} + emitted = set(asec_firm_size.noemp_canonical_map(2024).values()) + assert emitted <= vocabulary + + def test_sipp_exact_spans_are_in_the_vocabulary(self): + vocabulary = {b.label for b in banding.CANONICAL_BANDS} + for code, exact in sipp_jobs.EMPSIZE_CANONICAL_EXACT.items(): + label = sipp_jobs.EMPSIZE_CANONICAL_SPANS[code] + assert (label in vocabulary) is exact + + def test_establishment_and_firm_columns_are_named_apart(self): + # SIPP measures establishment size (#192 finding 1). If both + # readers emitted "canonical_band", a join would silently + # treat a location headcount as an enterprise headcount. + assert "canonical_band" not in sipp_jobs.__all__ + assert hasattr(asec_firm_size, "noemp_canonical_map") + + +REAL_SIPP = Path("~/PolicyEngine/sipp-data").expanduser() + + +@pytest.mark.skipif(not REAL_SIPP.is_dir(), reason="SIPP pu files not staged") +def test_real_sipp_bands_every_valid_code(): + """On real data, no valid EMPSIZE code falls through unbanded.""" + months = sipp_jobs.read_sipp_job_months(2023) + valid = months["empsize_code"].isin(sorted(EMPSIZE_INTENDED)) + assert months.loc[valid, "estab_size_band"].notna().all() + # And nothing outside the universe acquires a band. + assert months.loc[~valid, "estab_size_band"].isna().all() diff --git a/tests/tier_counts.json b/tests/tier_counts.json index 19149b24..61cbcdcf 100644 --- a/tests/tier_counts.json +++ b/tests/tier_counts.json @@ -1,7 +1,7 @@ { "schema_version": 1, "counts": { - "unit": 740, + "unit": 792, "artifact": 1101, "integration_psid": 804, "reproduction_legacy": 520, From c7e8533b728b8b5337f53ffdca190f006b3b4b71 Mon Sep 17 00:00:00 2001 From: Vahid Ahmadi Date: Thu, 23 Jul 2026 15:39:59 +0100 Subject: [PATCH 2/4] Address review: band labels are reader-level, real seam-separation pin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Daphne's contract ruling on #276: the band label strings are a reader/display-level convention, not the frozen contract's wire type (`firm_size_band` remains the CanonicalBand enum). Reworded both the `CanonicalBand.label` docstring and the CANONICAL_BAND_LABELS comment so neither can be read as redefining the wire format from a docstring; changing it needs an ADR 0003 amendment, not this file. Replaced the vacuous "names pinned apart" assertion (`"canonical_band" not in sipp_jobs.__all__` can never fail — column names are not in `__all__`) with a synthetic-fixture test that reads a pu file through `read_sipp_job_months` and `job_spells` and asserts `estab_size_band` in / `canonical_band` not in the emitted columns. The establishment-vs- firm separation is now CI-enforced rather than resting on the real-data test that skips wherever the pu files are not staged. Docstring fixes: `empsize_code`'s band is NaN at the -9 sentinel as well as where the code is NaN; `estab_size_band_exact` is object-dtype True/False/NaN, so truthiness masks the NaNs; `job_spells` emits no `_exact` companion, so spell consumers detect straddles via the "|" separator. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/populace_dynamics/data/sipp_jobs.py | 14 ++++++++++---- src/populace_dynamics/firms/banding.py | 16 ++++++++++++---- tests/data/test_firm_size_seam.py | 21 +++++++++++++++++++-- 3 files changed, 41 insertions(+), 10 deletions(-) diff --git a/src/populace_dynamics/data/sipp_jobs.py b/src/populace_dynamics/data/sipp_jobs.py index 0cdadeea..7b60d49c 100644 --- a/src/populace_dynamics/data/sipp_jobs.py +++ b/src/populace_dynamics/data/sipp_jobs.py @@ -270,9 +270,13 @@ def read_sipp_job_months( self-employment/other arrangements whose establishment size is structurally NIU; ``firms/banding.py`` expects that mix), ``estab_size_band``/``estab_size_band_exact`` (the canonical - C2 span for that code, from ``firms/banding.py`` — NaN - wherever ``empsize_code`` is; **establishment** size, so not - interchangeable with the ASEC reader's ``canonical_band``), + C2 span for that code, from ``firms/banding.py`` — NaN both + wherever ``empsize_code`` is NaN *and* at the -9 item- + nonresponse sentinel, which has no band; **establishment** + size, so not interchangeable with the ASEC reader's + ``canonical_band``. ``estab_size_band_exact`` is object-dtype + True/False/NaN, so ``~df[...]`` and truthiness tests mask the + NaNs as if they were straddles — compare explicitly), ``industry`` (string, passed through **unvalidated** — a full Census-industry allow-list is impractical, so sentinels can @@ -550,7 +554,9 @@ def job_spells(job_months: pd.DataFrame) -> pd.DataFrame: ``end_year``/``end_month``, ``n_months``, ``job_id``, ``industry``/``empsize_code``/``class_of_worker`` (modal), ``estab_size_band`` (the canonical span of the modal - ``empsize_code``), + ``empsize_code``; no ``_exact`` companion is emitted here, so + a spell consumer must detect a straddle by looking for the + ``"|"`` separator in the label), ``attributes_constant`` (False when any of the three varied within the spell — surfaced, never silently averaged), ``total_earnings``, ``earnings_share`` (spell earnings over diff --git a/src/populace_dynamics/firms/banding.py b/src/populace_dynamics/firms/banding.py index 2b3e84c3..e488ac26 100644 --- a/src/populace_dynamics/firms/banding.py +++ b/src/populace_dynamics/firms/banding.py @@ -128,11 +128,16 @@ def hi(self) -> float: @property def label(self) -> str: - """Stable wire label for the band (the C2 column vocabulary). + """Stable display label for the band (reader-level vocabulary). Readers emit this string, never a reader-local band name, so - that person-side and target-side band columns share one - vocabulary (#192 step 2; the seam #208 left latent). + that person-side and target-side *reader* band columns share + one vocabulary (#192 step 2; the seam #208 left latent). + + This is a reader/display-level convention, **not** the frozen + contract's wire type: the contract defines ``firm_size_band`` + as the :class:`CanonicalBand` enum, and only an ADR 0003 + amendment can make the string the wire format. """ return CANONICAL_BAND_LABELS[self] @@ -140,7 +145,10 @@ def label(self) -> str: CANONICAL_BANDS: tuple[CanonicalBand, ...] = tuple(CanonicalBand) #: The canonical band vocabulary. These strings — not any reader's -#: local band names — are what every C1-conforming column carries. +#: local band names — are what every **reader band column** carries. +#: The frozen contract's ``firm_size_band`` remains the +#: :class:`CanonicalBand` enum; these labels are the display/reader +#: rendering of it, and are not an alternative wire type. CANONICAL_BAND_LABELS: dict[CanonicalBand, str] = { CanonicalBand.LT10: "1-9", CanonicalBand.B10_49: "10-49", diff --git a/tests/data/test_firm_size_seam.py b/tests/data/test_firm_size_seam.py index 829b4c92..d8fcb07a 100644 --- a/tests/data/test_firm_size_seam.py +++ b/tests/data/test_firm_size_seam.py @@ -37,6 +37,8 @@ sipp_empsize_to_canonical, ) +from .test_sipp_jobs import _write_pu_file + # The intended bands, stated here independently of the mapper's # tables so that editing one of them cannot silently move the other. # Codes 5 (500-999) and 6 (1000+) both land in B500_PLUS: canonical @@ -197,11 +199,26 @@ def test_sipp_exact_spans_are_in_the_vocabulary(self): label = sipp_jobs.EMPSIZE_CANONICAL_SPANS[code] assert (label in vocabulary) is exact - def test_establishment_and_firm_columns_are_named_apart(self): + def test_establishment_and_firm_columns_are_named_apart(self, tmp_path): # SIPP measures establishment size (#192 finding 1). If both # readers emitted "canonical_band", a join would silently # treat a location headcount as an enterprise headcount. - assert "canonical_band" not in sipp_jobs.__all__ + # + # Pinned on a synthetic fixture rather than on ``__all__`` + # (which never carries column names) so the separation is + # enforced in CI, not only by the real-data test that skips + # wherever the pu files are not staged. + path = _write_pu_file( + tmp_path, 2023, [{"month": m, "job1": {}} for m in (1, 2)] + ) + months = sipp_jobs.read_sipp_job_months(2023, path=path) + assert "estab_size_band" in months.columns + assert "canonical_band" not in months.columns + + spells = sipp_jobs.job_spells(months) + assert "estab_size_band" in spells.columns + assert "canonical_band" not in spells.columns + assert hasattr(asec_firm_size, "noemp_canonical_map") From 4540f4bfde05df7cf8ceeb26cce108c626f93201 Mon Sep 17 00:00:00 2001 From: Vahid Ahmadi Date: Thu, 30 Jul 2026 17:45:47 +0200 Subject: [PATCH 3/4] Preserve first-estimates seal across IC2 seam --- scripts/first_estimates_birth_evidence.py | 11 ++++++++++- tests/estimates/test_birth_evidence_artifact.py | 13 +++++++++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/scripts/first_estimates_birth_evidence.py b/scripts/first_estimates_birth_evidence.py index a78b4128..044d6fc8 100644 --- a/scripts/first_estimates_birth_evidence.py +++ b/scripts/first_estimates_birth_evidence.py @@ -134,6 +134,9 @@ ) POST_REVIEW_SOURCE_EXCLUSIONS = ( Path("src/populace_dynamics/artifacts.py"), + Path("src/populace_dynamics/data/asec_firm_size.py"), + Path("src/populace_dynamics/data/sipp_jobs.py"), + Path("src/populace_dynamics/firms/banding.py"), Path("src/populace_dynamics/estimates/anchor_context_coordinator.py"), Path("src/populace_dynamics/estimates/anchor_context_publication.py"), Path("src/populace_dynamics/estimates/anchor_context_registry.py"), @@ -143,7 +146,13 @@ POST_REVIEW_SHARED_SOURCE_BLOBS = { Path( "src/populace_dynamics/artifacts.py" - ): "c03afa29cbdaf722c2cf62608dbb01f061f6558d" + ): "c03afa29cbdaf722c2cf62608dbb01f061f6558d", + Path( + "src/populace_dynamics/data/asec_firm_size.py" + ): "dee60e8c330833ef4944dd5033c213915b416fa9", + Path( + "src/populace_dynamics/firms/banding.py" + ): "e488ac266da29d2b5259575be0a7402fdbab73ab", } IMPLEMENTATION_REPLAY_ROWS = { "birth_source.derived_projection_age": 4_077, diff --git a/tests/estimates/test_birth_evidence_artifact.py b/tests/estimates/test_birth_evidence_artifact.py index 10430ab8..e6835868 100644 --- a/tests/estimates/test_birth_evidence_artifact.py +++ b/tests/estimates/test_birth_evidence_artifact.py @@ -63,9 +63,12 @@ def test_reducer_input_identity_matches_reviewed_branch(): reducer._assert_input_identity() -def test_context_report_sources_are_outside_historical_reducer_identity(): +def test_post_review_sources_are_outside_historical_reducer_identity(): assert reducer.POST_REVIEW_SOURCE_EXCLUSIONS == ( Path("src/populace_dynamics/artifacts.py"), + Path("src/populace_dynamics/data/asec_firm_size.py"), + Path("src/populace_dynamics/data/sipp_jobs.py"), + Path("src/populace_dynamics/firms/banding.py"), Path("src/populace_dynamics/estimates/anchor_context_coordinator.py"), Path("src/populace_dynamics/estimates/anchor_context_publication.py"), Path("src/populace_dynamics/estimates/anchor_context_registry.py"), @@ -75,7 +78,13 @@ def test_context_report_sources_are_outside_historical_reducer_identity(): assert reducer.POST_REVIEW_SHARED_SOURCE_BLOBS == { Path( "src/populace_dynamics/artifacts.py" - ): "c03afa29cbdaf722c2cf62608dbb01f061f6558d" + ): "c03afa29cbdaf722c2cf62608dbb01f061f6558d", + Path( + "src/populace_dynamics/data/asec_firm_size.py" + ): "dee60e8c330833ef4944dd5033c213915b416fa9", + Path( + "src/populace_dynamics/firms/banding.py" + ): "e488ac266da29d2b5259575be0a7402fdbab73ab", } From 115b64f64309ae58c1d88deff0260a1f08fe4e9e Mon Sep 17 00:00:00 2001 From: vahid-ahmadi Date: Wed, 12 Aug 2026 14:38:50 +0100 Subject: [PATCH 4/4] Recount tier manifest after merging master (unit 1,511 -> 1,563) The 52 firm-size seam tests classify into the unit tier. Counts recollected from the committed tree, per the merge convention. Co-Authored-By: Claude Opus 5 (1M context) --- tests/README-tiers.md | 4 ++-- tests/tier_counts.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/README-tiers.md b/tests/README-tiers.md index c0cc1e17..9436e9b5 100644 --- a/tests/README-tiers.md +++ b/tests/README-tiers.md @@ -38,9 +38,9 @@ pytest --collect-only -q -m oracle_policyengine | tail -1 | Tier | Tests at HEAD | |---|---:| -| `unit` | 1,511 | +| `unit` | 1,563 | | `artifact` | 2,543 | | `integration_psid` | 848 | | `reproduction_legacy` | 520 | | `oracle_policyengine` | 159 | -| **Total** | **5,581** | +| **Total** | **5,633** | diff --git a/tests/tier_counts.json b/tests/tier_counts.json index 23d77fc0..223b8c71 100644 --- a/tests/tier_counts.json +++ b/tests/tier_counts.json @@ -1,7 +1,7 @@ { "schema_version": 1, "counts": { - "unit": 1511, + "unit": 1563, "artifact": 2543, "integration_psid": 848, "reproduction_legacy": 520,