diff --git a/packages/microcosm-calibrate/src/microcosm/calibrate/__init__.py b/packages/microcosm-calibrate/src/microcosm/calibrate/__init__.py index b3f816c7f..238230371 100644 --- a/packages/microcosm-calibrate/src/microcosm/calibrate/__init__.py +++ b/packages/microcosm-calibrate/src/microcosm/calibrate/__init__.py @@ -79,6 +79,12 @@ def _assert_frame_compatible(version: str, required: tuple[int, int]) -> None: _assert_frame_compatible(_frame_version, _REQUIRED_FRAME_SERIES) +from microcosm.calibrate._target_loss_attribution import ( # noqa: E402 + TARGET_LOSS_ATTRIBUTION_ABS_TOLERANCE, + TARGET_LOSS_ATTRIBUTION_REL_TOLERANCE, + TARGET_LOSS_ATTRIBUTION_WARNING_CODES, + TARGET_LOSS_BASIS_HASH_ALGORITHM, +) from microcosm.calibrate.diagnostics import ( # noqa: E402 - after the compat gate CALIBRATION_DIAGNOSTICS_SCHEMA_VERSION, diagnostics_payload, @@ -126,6 +132,10 @@ def _assert_frame_compatible(version: str, required: tuple[int, int]) -> None: "CALIBRATION_DIAGNOSTICS_SCHEMA_VERSION", "CONSERVE_MASS", "FREE_MASS", + "TARGET_LOSS_ATTRIBUTION_ABS_TOLERANCE", + "TARGET_LOSS_ATTRIBUTION_REL_TOLERANCE", + "TARGET_LOSS_ATTRIBUTION_WARNING_CODES", + "TARGET_LOSS_BASIS_HASH_ALGORITHM", "CalibrationProblem", "CalibrationResult", "L0RefitResult", diff --git a/packages/microcosm-calibrate/src/microcosm/calibrate/_target_loss_attribution.py b/packages/microcosm-calibrate/src/microcosm/calibrate/_target_loss_attribution.py new file mode 100644 index 000000000..1bdeb3c3e --- /dev/null +++ b/packages/microcosm-calibrate/src/microcosm/calibrate/_target_loss_attribution.py @@ -0,0 +1,212 @@ +"""Build and validate schema-version-6 final target-loss attribution. + +This module owns the loss-specific contract: aligned basis validation, +per-target contribution calculation, and deterministic basis hashing. +``diagnostics`` remains responsible for embedding a complete result in the +artifact or translating an attribution-only failure into a structured warning. +""" + +from __future__ import annotations + +import hashlib +import math +import struct +from collections.abc import Mapping +from dataclasses import dataclass + +import numpy as np + +from microcosm.calibrate.solve import CalibrationResult + +__all__ = [ + "TARGET_LOSS_ATTRIBUTION_ABS_TOLERANCE", + "TARGET_LOSS_ATTRIBUTION_REL_TOLERANCE", + "TARGET_LOSS_ATTRIBUTION_WARNING_CODES", + "TARGET_LOSS_BASIS_HASH_ALGORITHM", + "TargetLossAttribution", + "TargetLossAttributionError", + "assemble_target_loss_attribution", + "target_loss_basis_hash", +] + +TARGET_LOSS_ATTRIBUTION_ABS_TOLERANCE = 1e-12 +TARGET_LOSS_ATTRIBUTION_REL_TOLERANCE = 1e-12 +TARGET_LOSS_BASIS_HASH_ALGORITHM = "sha256_utf8len32_f64be_v1" +TARGET_LOSS_ATTRIBUTION_WARNING_CODES = { + "alignment": "target_loss_attribution_alignment_error", + "invalid_basis": "target_loss_attribution_invalid_basis", + "contribution_mismatch": "target_loss_attribution_contribution_mismatch", + "assembly_error": "target_loss_attribution_assembly_error", +} + +_TARGET_LOSS_FORMULA = "weighted_mean(min(abs((estimate - target) / scale), cap))" + + +@dataclass(frozen=True) +class TargetLossAttribution: + """A complete attribution block ready to attach to diagnostics.""" + + basis: dict[str, object] + rows: tuple[dict[str, float], ...] + + +class TargetLossAttributionError(ValueError): + """A validation failure that degrades only supplementary attribution.""" + + def __init__(self, code: str, message: str) -> None: + super().__init__(message) + self.code = code + + +def _loss_basis_kind( + result: CalibrationResult, + option_name: str, + *, + default: str, +) -> str: + """Read a result's recorded loss-basis kind without rebuilding its values.""" + options = getattr(result, "options", None) + if not isinstance(options, Mapping): + return default + value = options.get(option_name) + if isinstance(value, Mapping): + kind = value.get("kind") + return str(kind) if isinstance(kind, str) and kind else default + if isinstance(value, str) and value: + return value + return default + + +def target_loss_basis_hash( + names: list[str], + weights: np.ndarray, + scales: np.ndarray, +) -> str: + """Hash ordered UTF-8 names and IEEE-754 values without JSON float ambiguity. + + Each name is encoded as a four-byte unsigned big-endian byte length followed + by its UTF-8 bytes, then its raw weight and scale as big-endian float64. The + versioned algorithm identifier travels with the digest so non-Python + consumers can reproduce it exactly. + """ + digest = hashlib.sha256() + digest.update(b"microcosm-target-loss-basis-v1\x00") + for name, weight, scale in zip(names, weights, scales, strict=True): + encoded_name = name.encode("utf-8") + digest.update(struct.pack(">I", len(encoded_name))) + digest.update(encoded_name) + digest.update(struct.pack(">d", float(weight))) + digest.update(struct.pack(">d", float(scale))) + return digest.hexdigest() + + +def assemble_target_loss_attribution( + result: CalibrationResult, +) -> TargetLossAttribution: + """Validate and assemble a complete final target-loss attribution block.""" + diagnostics = tuple(result.diagnostics) + target_count = len(diagnostics) + weights = np.asarray(result.target_loss_weights, dtype=np.float64) + scales = np.asarray(result.target_loss_scales, dtype=np.float64) + expected_shape = (target_count,) + if weights.shape != expected_shape or scales.shape != expected_shape: + raise TargetLossAttributionError( + TARGET_LOSS_ATTRIBUTION_WARNING_CODES["alignment"], + "Final target-loss weights and scales must each have one value per " + f"target diagnostic; got weights {weights.shape}, scales {scales.shape}, " + f"and {target_count} target rows.", + ) + if not np.isfinite(weights).all() or (weights < 0.0).any(): + raise TargetLossAttributionError( + TARGET_LOSS_ATTRIBUTION_WARNING_CODES["invalid_basis"], + "Final target-loss weights must be finite and non-negative.", + ) + if not np.isfinite(scales).all() or (scales <= 0.0).any(): + raise TargetLossAttributionError( + TARGET_LOSS_ATTRIBUTION_WARNING_CODES["invalid_basis"], + "Final target-loss scales must be finite and strictly positive.", + ) + total_weight = float(weights.sum()) + if not math.isfinite(total_weight) or total_weight <= 0.0: + raise TargetLossAttributionError( + TARGET_LOSS_ATTRIBUTION_WARNING_CODES["invalid_basis"], + "Final target-loss weights must have positive finite total weight.", + ) + cap = float(result.target_loss_cap) + if not math.isfinite(cap) or cap <= 0.0: + raise TargetLossAttributionError( + TARGET_LOSS_ATTRIBUTION_WARNING_CODES["invalid_basis"], + "The final target-loss cap must be finite and strictly positive.", + ) + + weight_shares = weights / total_weight + attribution_rows: list[dict[str, float]] = [] + names: list[str] = [] + for diagnostic, weight, weight_share, scale in zip( + diagnostics, + weights, + weight_shares, + scales, + strict=True, + ): + target = float(diagnostic.target) + estimate = float(diagnostic.final_estimate) + if not math.isfinite(target) or not math.isfinite(estimate): + raise TargetLossAttributionError( + TARGET_LOSS_ATTRIBUTION_WARNING_CODES["invalid_basis"], + f"Target {diagnostic.name!r} has a non-finite target or final estimate.", + ) + capped_error = min(abs(estimate - target) / float(scale), cap) + contribution = float(weight_share) * capped_error + if not math.isfinite(capped_error) or not math.isfinite(contribution): + raise TargetLossAttributionError( + TARGET_LOSS_ATTRIBUTION_WARNING_CODES["invalid_basis"], + f"Target {diagnostic.name!r} produced non-finite attribution values.", + ) + names.append(str(diagnostic.name)) + attribution_rows.append( + { + "target_loss_weight": float(weight), + "target_loss_weight_share": float(weight_share), + "target_loss_scale": float(scale), + "final_capped_scaled_error": float(capped_error), + "final_loss_contribution": float(contribution), + } + ) + + contribution_sum = float( + math.fsum(row["final_loss_contribution"] for row in attribution_rows) + ) + final_loss = float(result.final_loss) + if not math.isfinite(final_loss) or not math.isclose( + contribution_sum, + final_loss, + rel_tol=TARGET_LOSS_ATTRIBUTION_REL_TOLERANCE, + abs_tol=TARGET_LOSS_ATTRIBUTION_ABS_TOLERANCE, + ): + raise TargetLossAttributionError( + TARGET_LOSS_ATTRIBUTION_WARNING_CODES["contribution_mismatch"], + "Final target-loss contributions do not reproduce final_loss within " + "the schema-version-6 tolerance: " + f"contributions={contribution_sum!r}, final_loss={final_loss!r}.", + ) + + basis = { + "formula": _TARGET_LOSS_FORMULA, + "cap": cap, + "target_count": target_count, + "total_target_weight": total_weight, + "weight_kind": _loss_basis_kind( + result, + "target_loss_weights", + default="unknown", + ), + "scale_kind": _loss_basis_kind( + result, + "target_loss_scales", + default="unknown", + ), + "hash_algorithm": TARGET_LOSS_BASIS_HASH_ALGORITHM, + "sha256": target_loss_basis_hash(names, weights, scales), + } + return TargetLossAttribution(basis=basis, rows=tuple(attribution_rows)) diff --git a/packages/microcosm-calibrate/src/microcosm/calibrate/diagnostics.py b/packages/microcosm-calibrate/src/microcosm/calibrate/diagnostics.py index 2944f1f91..f63913d68 100644 --- a/packages/microcosm-calibrate/src/microcosm/calibrate/diagnostics.py +++ b/packages/microcosm-calibrate/src/microcosm/calibrate/diagnostics.py @@ -19,11 +19,17 @@ import hashlib import json +import logging import math from collections.abc import Mapping from pathlib import Path from typing import Any +from microcosm.calibrate._target_loss_attribution import ( + TARGET_LOSS_ATTRIBUTION_WARNING_CODES, + TargetLossAttributionError, + assemble_target_loss_attribution, +) from microcosm.calibrate.solve import CalibrationResult __all__ = [ @@ -40,7 +46,12 @@ #: v5 added the ``past_cap_census`` block (rows past the loss cap at #: initialization and at final, escaped/frozen/pushed-out counts, and the #: pushed-out row list). -CALIBRATION_DIAGNOSTICS_SCHEMA_VERSION = 5 +#: v6 added authoritative final per-target loss attribution and an explicit +#: warning-only degradation state when that supplementary attribution cannot +#: be validated. +CALIBRATION_DIAGNOSTICS_SCHEMA_VERSION = 6 + +_LOGGER = logging.getLogger(__name__) def _finite(value: float) -> float | None: @@ -234,12 +245,12 @@ def past_cap_census(result: CalibrationResult) -> dict[str, object] | None: its scaled misses, worst final miss first. ``init_rel`` / ``final_rel`` are scaled absolute misses on the default - scale rule ``max(abs(target), 1)`` — the units the cap applies to. A run - that supplied custom ``target_loss_scales`` is censused on the default - rule (the custom scales do not travel with the result); its options - record that the scales were provided. Rows with a non-finite target or - estimate are excluded from every count. Returns ``None`` when the - result's options record no cap — there is nothing to census against. + scale rule ``max(abs(target), 1)``. This block intentionally preserves its + schema-version-5 semantics: a run that supplied custom scales is still + censused on the default rule even though schema version 6 separately + retains and reports its actual aligned loss scales. Rows with a non-finite + target or estimate are excluded from every count. Returns ``None`` when + the result's options record no cap — there is nothing to census against. """ cap = _result_target_loss_cap(getattr(result, "options", None)) if cap is None: @@ -317,6 +328,17 @@ def diagnostics_payload( floats become ``null``). """ registry_specs = _registry_spec_lookup(target_registry) + target_rows = [ + _target_row( + diagnostic, + target, + compiled_target=result.problem.target_vector[index], + spec=registry_specs.get(diagnostic.name), + ) + for index, (diagnostic, target) in enumerate( + zip(result.diagnostics, result.problem.targets, strict=True) + ) + ] payload = { "schema_version": CALIBRATION_DIAGNOSTICS_SCHEMA_VERSION, "weight_entity": result.weight_entity, @@ -336,18 +358,45 @@ def diagnostics_payload( {"name": skip.target.name, "reason": skip.reason} for skip in result.skipped ], "past_cap_census": past_cap_census(result), - "targets": [ - _target_row( - diagnostic, - target, - compiled_target=result.problem.target_vector[index], - spec=registry_specs.get(diagnostic.name), - ) - for index, (diagnostic, target) in enumerate( - zip(result.diagnostics, result.problem.targets, strict=True) - ) - ], + "diagnostic_warnings": [], + "targets": target_rows, } + try: + attribution = assemble_target_loss_attribution(result) + except TargetLossAttributionError as error: + _LOGGER.warning( + "TARGET LOSS ATTRIBUTION UNAVAILABLE [%s]: %s", + error.code, + error, + ) + payload["diagnostic_warnings"].append( + { + "code": error.code, + "severity": "warning", + "message": str(error), + } + ) + except Exception as error: # pragma: no cover - defensive build protection + code = TARGET_LOSS_ATTRIBUTION_WARNING_CODES["assembly_error"] + _LOGGER.exception( + "TARGET LOSS ATTRIBUTION UNAVAILABLE [%s]: unexpected attribution " + "assembly error", + code, + ) + payload["diagnostic_warnings"].append( + { + "code": code, + "severity": "warning", + "message": ( + "Unexpected target-loss attribution assembly error: " + f"{type(error).__name__}: {error}" + ), + } + ) + else: + payload["target_loss_basis"] = attribution.basis + for row, attribution_row in zip(target_rows, attribution.rows, strict=True): + row.update(attribution_row) registry = _registry_payload(target_registry) if registry is not None: payload["target_registry"] = registry diff --git a/packages/microcosm-calibrate/src/microcosm/calibrate/score.py b/packages/microcosm-calibrate/src/microcosm/calibrate/score.py index 81750d7a2..00290a3a5 100644 --- a/packages/microcosm-calibrate/src/microcosm/calibrate/score.py +++ b/packages/microcosm-calibrate/src/microcosm/calibrate/score.py @@ -10,6 +10,11 @@ from microcosm.calibrate.solve import ( CalibrationResult, TargetDiagnostic, + _target_loss_scale_options, + _target_loss_weight_options, + _validate_target_loss_cap, + _validate_target_loss_scales, + _validate_target_loss_weights, default_target_loss_scales, relative_error_loss, ) @@ -77,16 +82,72 @@ def score_targets( f"{score_weights.shape} vs {initial_weights.shape}." ) - estimates = problem.estimates(score_weights) - scales = ( - default_target_loss_scales(problem.target_vector) + target_loss_cap = _validate_target_loss_cap(target_loss_cap) + target_loss_weights_input = _validate_target_loss_weights( + target_loss_weights, + (len(targets),), + ) + target_loss_scales_input = ( + None if target_loss_scales is None - else np.asarray(target_loss_scales, dtype=np.float64) + else _validate_target_loss_scales( + target_loss_scales, + (len(targets),), + targets=np.asarray([target.value for target in targets], dtype=np.float64), + ) ) + if target_loss_weights_input is None: + aligned_target_loss_weights = np.ones( + problem.target_vector.shape, + dtype=np.float64, + ) + loss_weights = None + else: + weights_by_key = { + target.key: weight + for target, weight in zip( + targets, + target_loss_weights_input, + strict=True, + ) + } + aligned_target_loss_weights = _validate_target_loss_weights( + np.asarray( + [weights_by_key[target.key] for target in problem.targets], + dtype=np.float64, + ), + problem.target_vector.shape, + ) + if aligned_target_loss_weights is None: # pragma: no cover - guarded above + raise RuntimeError( + "Provided target loss weights were lost during alignment." + ) + loss_weights = aligned_target_loss_weights + + estimates = problem.estimates(score_weights) + if target_loss_scales_input is None: + scales = default_target_loss_scales(problem.target_vector) + else: + scales_by_key = { + target.key: scale + for target, scale in zip( + targets, + target_loss_scales_input, + strict=True, + ) + } + scales = _validate_target_loss_scales( + np.asarray( + [scales_by_key[target.key] for target in problem.targets], + dtype=np.float64, + ), + problem.target_vector.shape, + targets=problem.target_vector, + ) loss = relative_error_loss( estimates, problem.target_vector, - target_loss_weights=target_loss_weights, + target_loss_weights=loss_weights, target_loss_scales=scales, target_loss_cap=target_loss_cap, ) @@ -103,12 +164,22 @@ def score_targets( l0_lambda=0.0, n_nonzero=int((score_weights > prune_atol).sum()), closing_loss=loss, + target_loss_weights=aligned_target_loss_weights.copy(), + target_loss_scales=scales.copy(), + target_loss_cap=target_loss_cap, options={ "method": "score_only", "target_loss_cap": float(target_loss_cap), - "target_loss_weights": "provided" - if target_loss_weights is not None - else "uniform", + "target_loss_weights": _target_loss_weight_options( + loss_weights, + ), + "target_loss_scales": _target_loss_scale_options( + scales, + kind="provided" + if target_loss_scales_input is not None + else "default_target", + target_loss_cap=target_loss_cap, + ), **dict(options or {}), }, gate_open_probabilities=None, diff --git a/packages/microcosm-calibrate/src/microcosm/calibrate/solve.py b/packages/microcosm-calibrate/src/microcosm/calibrate/solve.py index 5e52d0b77..746ed9e6a 100644 --- a/packages/microcosm-calibrate/src/microcosm/calibrate/solve.py +++ b/packages/microcosm-calibrate/src/microcosm/calibrate/solve.py @@ -185,6 +185,13 @@ class CalibrationResult: *returned* weights (after the closing mass/cap projections). Exposed as :attr:`final_loss`; recorded separately from the trajectory, whose tail is a pre-step/pre-projection value. + target_loss_weights: The effective non-negative target-importance vector, + aligned to :attr:`diagnostics`. Uniform defaults are materialized as + ones so diagnostics can publish the exact final loss basis. + target_loss_scales: The effective positive target-loss denominator vector, + aligned to :attr:`diagnostics` after skipped targets are removed. + target_loss_cap: The positive per-target scaled-error cap used by the final + loss evaluation. options: The solver configuration as passed (method, epochs, learning_rate, mass, max_weight_ratio, target_records, seed, l1_lambda, l2_lambda) plus the realized ``matrix_format`` @@ -211,6 +218,9 @@ class CalibrationResult: l0_lambda: float n_nonzero: int closing_loss: float + target_loss_weights: np.ndarray + target_loss_scales: np.ndarray + target_loss_cap: float options: Mapping[str, object] = field(default_factory=dict) gate_open_probabilities: np.ndarray | None = None @@ -374,6 +384,21 @@ def final_loss(self) -> float: """The post-L0 refit's final penalty-free target loss.""" return self.refit.final_loss + @property + def target_loss_weights(self) -> np.ndarray: + """The final refit's aligned target-importance vector.""" + return self.refit.target_loss_weights + + @property + def target_loss_scales(self) -> np.ndarray: + """The final refit's aligned target-loss scale vector.""" + return self.refit.target_loss_scales + + @property + def target_loss_cap(self) -> float: + """The final refit's per-target scaled-error cap.""" + return self.refit.target_loss_cap + @property def fraction_within_10pct(self) -> float: """Share of targets the post-L0 refit reproduces within 10%.""" @@ -1745,6 +1770,11 @@ def calibrate( target_loss_scales=target_loss_scales_np, target_loss_cap=target_loss_cap, ) + effective_target_loss_weights = ( + np.ones(problem.target_vector.shape, dtype=np.float64) + if target_loss_weights_np is None + else target_loss_weights_np.copy() + ) return CalibrationResult( frame=new_frame, @@ -1758,6 +1788,9 @@ def calibrate( l0_lambda=effective_l0, n_nonzero=n_nonzero, closing_loss=closing_loss, + target_loss_weights=effective_target_loss_weights, + target_loss_scales=target_loss_scales_np.copy(), + target_loss_cap=target_loss_cap, options={ "method": method, "epochs": epochs, diff --git a/packages/microcosm-calibrate/tests/fixtures/target_loss_attribution/unavailable_warning.json b/packages/microcosm-calibrate/tests/fixtures/target_loss_attribution/unavailable_warning.json new file mode 100644 index 000000000..20c5b1a9b --- /dev/null +++ b/packages/microcosm-calibrate/tests/fixtures/target_loss_attribution/unavailable_warning.json @@ -0,0 +1,23 @@ +{ + "schema_version": 6, + "final_loss": 0.15, + "diagnostic_warnings": [ + { + "code": "target_loss_attribution_alignment_error", + "severity": "warning", + "message": "Final target-loss weights and scales must each have one value per target diagnostic." + } + ], + "targets": [ + { + "name": "population@2024", + "target": 100.0, + "final_estimate": 110.0 + }, + { + "name": "income@2024", + "target": 200.0, + "final_estimate": 160.0 + } + ] +} diff --git a/packages/microcosm-calibrate/tests/fixtures/target_loss_attribution/valid_unequal_custom_scales.json b/packages/microcosm-calibrate/tests/fixtures/target_loss_attribution/valid_unequal_custom_scales.json new file mode 100644 index 000000000..bd1488349 --- /dev/null +++ b/packages/microcosm-calibrate/tests/fixtures/target_loss_attribution/valid_unequal_custom_scales.json @@ -0,0 +1,37 @@ +{ + "schema_version": 6, + "final_loss": 0.275, + "diagnostic_warnings": [], + "target_loss_basis": { + "formula": "weighted_mean(min(abs((estimate - target) / scale), cap))", + "cap": 0.5, + "target_count": 2, + "total_target_weight": 8.0, + "weight_kind": "provided", + "scale_kind": "provided", + "hash_algorithm": "sha256_utf8len32_f64be_v1", + "sha256": "80522115f344bf4726dad7a2c063f0d289428e247493b184bded04beb2ad61c7" + }, + "targets": [ + { + "name": "population@2024", + "target": 100.0, + "final_estimate": 125.0, + "target_loss_weight": 2.0, + "target_loss_weight_share": 0.25, + "target_loss_scale": 50.0, + "final_capped_scaled_error": 0.5, + "final_loss_contribution": 0.125 + }, + { + "name": "income@2024", + "target": 200.0, + "final_estimate": 180.0, + "target_loss_weight": 6.0, + "target_loss_weight_share": 0.75, + "target_loss_scale": 100.0, + "final_capped_scaled_error": 0.2, + "final_loss_contribution": 0.15 + } + ] +} diff --git a/packages/microcosm-calibrate/tests/fixtures/target_loss_attribution/valid_uniform.json b/packages/microcosm-calibrate/tests/fixtures/target_loss_attribution/valid_uniform.json new file mode 100644 index 000000000..f12541ce9 --- /dev/null +++ b/packages/microcosm-calibrate/tests/fixtures/target_loss_attribution/valid_uniform.json @@ -0,0 +1,37 @@ +{ + "schema_version": 6, + "final_loss": 0.15, + "diagnostic_warnings": [], + "target_loss_basis": { + "formula": "weighted_mean(min(abs((estimate - target) / scale), cap))", + "cap": 1.0, + "target_count": 2, + "total_target_weight": 2.0, + "weight_kind": "uniform", + "scale_kind": "default_target", + "hash_algorithm": "sha256_utf8len32_f64be_v1", + "sha256": "abaf0237d41295559f6ba0167b1075eca1bb93c544d272fcb60c6aeb07deb376" + }, + "targets": [ + { + "name": "population@2024", + "target": 100.0, + "final_estimate": 110.0, + "target_loss_weight": 1.0, + "target_loss_weight_share": 0.5, + "target_loss_scale": 100.0, + "final_capped_scaled_error": 0.1, + "final_loss_contribution": 0.05 + }, + { + "name": "income@2024", + "target": 200.0, + "final_estimate": 160.0, + "target_loss_weight": 1.0, + "target_loss_weight_share": 0.5, + "target_loss_scale": 200.0, + "final_capped_scaled_error": 0.2, + "final_loss_contribution": 0.1 + } + ] +} diff --git a/packages/microcosm-calibrate/tests/test_diagnostics.py b/packages/microcosm-calibrate/tests/test_diagnostics.py index e7e4c9329..e7e683285 100644 --- a/packages/microcosm-calibrate/tests/test_diagnostics.py +++ b/packages/microcosm-calibrate/tests/test_diagnostics.py @@ -9,11 +9,18 @@ from __future__ import annotations import json +import logging +from dataclasses import replace from pathlib import Path from types import SimpleNamespace +import numpy as np +import pytest + from microcosm.calibrate import ( CALIBRATION_DIAGNOSTICS_SCHEMA_VERSION, + TARGET_LOSS_ATTRIBUTION_WARNING_CODES, + TARGET_LOSS_BASIS_HASH_ALGORITHM, Target, TargetDiagnostic, TargetRegistry, @@ -26,6 +33,18 @@ score_targets, write_calibration_diagnostics, ) +from microcosm.calibrate._target_loss_attribution import target_loss_basis_hash + +_ATTRIBUTION_ROW_FIELDS = { + "target_loss_weight", + "target_loss_weight_share", + "target_loss_scale", + "final_capped_scaled_error", + "final_loss_contribution", +} +_ATTRIBUTION_FIXTURE_DIR = ( + Path(__file__).parent / "fixtures" / "target_loss_attribution" +) def _result(feasible_frame, *, with_skip: bool = False, epochs: int = 120): @@ -90,6 +109,216 @@ def test_payload_carries_full_evidence(feasible_frame) -> None: assert population["within_tolerance"] is None # no tolerance declared +def test_payload_reports_complete_uniform_final_loss_attribution( + feasible_frame, +) -> None: + result = _result(feasible_frame, epochs=1) + payload = diagnostics_payload(result) + + assert payload["schema_version"] == 6 + assert payload["diagnostic_warnings"] == [] + basis = payload["target_loss_basis"] + assert basis["formula"] == ( + "weighted_mean(min(abs((estimate - target) / scale), cap))" + ) + assert basis["cap"] == result.target_loss_cap + assert basis["target_count"] == len(result.diagnostics) + assert basis["total_target_weight"] == len(result.diagnostics) + assert basis["weight_kind"] == "uniform" + assert basis["scale_kind"] == "default_target" + assert basis["hash_algorithm"] == TARGET_LOSS_BASIS_HASH_ALGORITHM + assert len(basis["sha256"]) == 64 + + assert all(_ATTRIBUTION_ROW_FIELDS <= row.keys() for row in payload["targets"]) + assert sum(row["target_loss_weight_share"] for row in payload["targets"]) == ( + pytest.approx(1.0) + ) + assert sum(row["final_loss_contribution"] for row in payload["targets"]) == ( + pytest.approx(result.final_loss, rel=1e-12, abs=1e-12) + ) + + +def test_payload_reports_unequal_weights_custom_scales_and_zero_target( + feasible_frame, +) -> None: + frame, truths = feasible_frame() + targets = TargetSet( + ( + Target( + name="zero_population", + entity="household", + value=0.0, + measure="household_count", + ), + Target( + name="income", + entity="household", + value=truths["income"], + measure="income", + ), + ) + ) + result = score_targets( + frame, + targets, + target_loss_weights=np.asarray([2.0, 6.0]), + target_loss_scales=np.asarray([1.0, 50.0]), + target_loss_cap=0.75, + ) + payload = diagnostics_payload(result) + zero_row, income_row = payload["targets"] + + assert payload["target_loss_basis"]["weight_kind"] == "provided" + assert payload["target_loss_basis"]["scale_kind"] == "provided" + assert zero_row["target_loss_weight"] == 2.0 + assert zero_row["target_loss_weight_share"] == 0.25 + assert zero_row["target_loss_scale"] == 1.0 + assert zero_row["final_capped_scaled_error"] == 0.75 + assert income_row["target_loss_weight_share"] == 0.75 + assert sum(row["final_loss_contribution"] for row in payload["targets"]) == ( + pytest.approx(result.final_loss, rel=1e-12, abs=1e-12) + ) + + +def test_target_loss_basis_hash_is_stable_and_sensitive() -> None: + names = ["population@2024", "income@2024"] + weights = np.asarray([1.0, 2.0]) + scales = np.asarray([100.0, 200.0]) + expected = target_loss_basis_hash(names, weights, scales) + + assert target_loss_basis_hash(names, weights.copy(), scales.copy()) == expected + assert target_loss_basis_hash(list(reversed(names)), weights, scales) != expected + assert target_loss_basis_hash(names, np.asarray([1.0, 3.0]), scales) != expected + assert target_loss_basis_hash(names, weights, np.asarray([100.0, 201.0])) != ( + expected + ) + + +@pytest.mark.parametrize( + "fixture_name", + [ + "valid_uniform.json", + "valid_unequal_custom_scales.json", + "unavailable_warning.json", + ], +) +def test_schema_v6_target_loss_attribution_fixtures_are_valid( + fixture_name: str, +) -> None: + payload = json.loads((_ATTRIBUTION_FIXTURE_DIR / fixture_name).read_text()) + + assert payload["schema_version"] == 6 + json.dumps(payload, allow_nan=False) + if payload["diagnostic_warnings"]: + assert "target_loss_basis" not in payload + assert all( + _ATTRIBUTION_ROW_FIELDS.isdisjoint(row.keys()) for row in payload["targets"] + ) + else: + assert payload["target_loss_basis"]["hash_algorithm"] == ( + TARGET_LOSS_BASIS_HASH_ALGORITHM + ) + assert sum( + row["final_loss_contribution"] for row in payload["targets"] + ) == pytest.approx(payload["final_loss"], rel=1e-12, abs=1e-12) + + +def _assert_attribution_withheld(payload: dict, *, warning_code: str) -> None: + assert "target_loss_basis" not in payload + assert payload["diagnostic_warnings"] == [ + { + "code": warning_code, + "severity": "warning", + "message": payload["diagnostic_warnings"][0]["message"], + } + ] + assert all( + _ATTRIBUTION_ROW_FIELDS.isdisjoint(row.keys()) for row in payload["targets"] + ) + + +def test_attribution_alignment_failure_warns_and_preserves_core_diagnostics( + feasible_frame, + caplog, +) -> None: + result = _result(feasible_frame, epochs=1) + invalid = replace( + result, + target_loss_weights=np.ones(len(result.diagnostics) + 1), + ) + + with caplog.at_level(logging.WARNING): + payload = diagnostics_payload(invalid) + + code = TARGET_LOSS_ATTRIBUTION_WARNING_CODES["alignment"] + _assert_attribution_withheld(payload, warning_code=code) + assert code in caplog.text + assert payload["final_loss"] == result.final_loss + assert len(payload["targets"]) == len(result.diagnostics) + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("target_loss_weights", np.asarray([-1.0, 2.0])), + ("target_loss_weights", np.asarray([0.0, 0.0])), + ("target_loss_scales", np.asarray([1.0, np.nan])), + ("target_loss_scales", np.asarray([1.0, 0.0])), + ], +) +def test_invalid_attribution_values_warn_without_partial_rows( + feasible_frame, + field: str, + value: np.ndarray, +) -> None: + result = _result(feasible_frame, epochs=1) + payload = diagnostics_payload(replace(result, **{field: value})) + + _assert_attribution_withheld( + payload, + warning_code=TARGET_LOSS_ATTRIBUTION_WARNING_CODES["invalid_basis"], + ) + + +def test_contribution_mismatch_warns_instead_of_failing_diagnostics( + feasible_frame, +) -> None: + result = _result(feasible_frame, epochs=1) + invalid = replace(result, closing_loss=result.final_loss + 0.1) + + payload = diagnostics_payload(invalid) + + _assert_attribution_withheld( + payload, + warning_code=TARGET_LOSS_ATTRIBUTION_WARNING_CODES["contribution_mismatch"], + ) + assert payload["final_loss"] == invalid.final_loss + + +def test_unexpected_attribution_assembly_failure_warns_and_preserves_core_payload( + feasible_frame, + monkeypatch, +) -> None: + result = _result(feasible_frame, epochs=1) + + def fail_assembly(_result) -> None: + raise RuntimeError("synthetic assembly failure") + + monkeypatch.setattr( + "microcosm.calibrate.diagnostics.assemble_target_loss_attribution", + fail_assembly, + ) + payload = diagnostics_payload(result) + + _assert_attribution_withheld( + payload, + warning_code=TARGET_LOSS_ATTRIBUTION_WARNING_CODES["assembly_error"], + ) + warning_message = payload["diagnostic_warnings"][0]["message"] + assert "RuntimeError: synthetic assembly failure" in warning_message + assert payload["final_loss"] == result.final_loss + + def test_skipped_targets_ship_with_their_reason(feasible_frame) -> None: result = _result(feasible_frame, with_skip=True) payload = diagnostics_payload(result) @@ -101,6 +330,62 @@ def test_skipped_targets_ship_with_their_reason(feasible_frame) -> None: assert all(not row["name"].startswith("ghost") for row in payload["targets"]) +def test_result_retains_materialized_aligned_final_loss_basis(feasible_frame) -> None: + frame, truths = feasible_frame() + targets = TargetSet( + ( + Target( + name="population", + entity="household", + value=truths["population"] * 1.2, + measure="household_count", + ), + Target( + name="ghost", + entity="household", + value=1.0, + measure="no_such_column", + ), + Target( + name="income", + entity="household", + value=truths["income"] * 1.2, + measure="income", + ), + ) + ) + result = calibrate( + frame, + targets, + epochs=1, + seed=0, + target_loss_weights=np.asarray([2.0, 99.0, 6.0]), + target_loss_scales=np.asarray([10.0, 99.0, 50.0]), + target_loss_cap=0.75, + ) + + assert [diagnostic.name for diagnostic in result.diagnostics] == [ + "population@0", + "income@0", + ] + np.testing.assert_allclose(result.target_loss_weights, [2.0, 6.0]) + np.testing.assert_allclose(result.target_loss_scales, [10.0, 50.0]) + assert result.target_loss_cap == 0.75 + + default_result = _result(feasible_frame, epochs=1) + np.testing.assert_allclose( + default_result.target_loss_weights, + np.ones(len(default_result.diagnostics)), + ) + np.testing.assert_allclose( + default_result.target_loss_scales, + np.maximum( + np.abs([row.target for row in default_result.diagnostics]), + 1.0, + ), + ) + + def test_payload_is_strict_json(feasible_frame) -> None: result = _result(feasible_frame) payload = diagnostics_payload(result) @@ -118,6 +403,19 @@ def test_writer_round_trips(feasible_frame, tmp_path: Path) -> None: assert loaded["schema_version"] == CALIBRATION_DIAGNOSTICS_SCHEMA_VERSION +def test_writer_does_not_suppress_unrelated_output_failures( + feasible_frame, + tmp_path: Path, +) -> None: + result = _result(feasible_frame, epochs=1) + + with pytest.raises(FileNotFoundError): + write_calibration_diagnostics( + result, + tmp_path / "missing-parent" / "calibration_diagnostics.json", + ) + + def test_payload_can_carry_target_registry_identity(feasible_frame) -> None: frame, truths = feasible_frame() registry = TargetRegistry( @@ -199,6 +497,15 @@ def test_payload_accepts_l0_refit_result(feasible_frame) -> None: assert payload["options"]["post_l0_refit"] is True assert payload["fraction_within_10pct"] == result.refit.fraction_within_10pct assert payload["effective_sample_size"] == result.refit.effective_sample_size + np.testing.assert_allclose( + result.target_loss_weights, + result.refit.target_loss_weights, + ) + np.testing.assert_allclose( + result.target_loss_scales, + result.refit.target_loss_scales, + ) + assert result.target_loss_cap == result.refit.target_loss_cap # The census reads the cap through the L0RefitResult's merged options. assert payload["past_cap_census"]["cap"] == 10.0 json.dumps(payload, allow_nan=False) diff --git a/packages/microcosm-calibrate/tests/test_score.py b/packages/microcosm-calibrate/tests/test_score.py index 618dff300..152158e4c 100644 --- a/packages/microcosm-calibrate/tests/test_score.py +++ b/packages/microcosm-calibrate/tests/test_score.py @@ -58,6 +58,9 @@ def test_score_targets_evaluates_existing_weights_without_calibrating() -> None: assert result.diagnostics[0].initial_estimate == 800.0 assert result.diagnostics[0].final_estimate == 800.0 assert result.diagnostics[1].final_estimate == 5.0 + np.testing.assert_allclose(result.target_loss_weights, [1.0, 1.0]) + np.testing.assert_allclose(result.target_loss_scales, [1_000.0, 5.0]) + assert result.target_loss_cap == 1.0 expected = relative_error_loss( np.asarray([800.0, 5.0]), np.asarray([1_000.0, 5.0]), @@ -89,3 +92,40 @@ def test_score_targets_rejects_misaligned_weight_vector() -> None: with pytest.raises(ValueError, match="weights must align"): score_targets(frame, targets, weights=np.asarray([1.0])) + + +def test_score_targets_retains_loss_basis_after_skipped_target_alignment() -> None: + frame = _frame() + targets = TargetSet( + ( + Target(name="income", entity="household", value=500.0, measure="income"), + Target( + name="ghost", + entity="household", + value=1.0, + measure="no_such_column", + ), + Target( + name="households", + entity="household", + value=5.0, + measure="household_count", + ), + ) + ) + + result = score_targets( + frame, + targets, + target_loss_weights=np.asarray([2.0, 99.0, 6.0]), + target_loss_scales=np.asarray([500.0, 99.0, 5.0]), + target_loss_cap=0.75, + ) + + assert [diagnostic.name for diagnostic in result.diagnostics] == [ + "income@0", + "households@0", + ] + np.testing.assert_allclose(result.target_loss_weights, [2.0, 6.0]) + np.testing.assert_allclose(result.target_loss_scales, [500.0, 5.0]) + assert result.target_loss_cap == 0.75 diff --git a/packages/microcosm-data/src/microcosm/data/contract.py b/packages/microcosm-data/src/microcosm/data/contract.py index 614b5007f..5d285be67 100644 --- a/packages/microcosm-data/src/microcosm/data/contract.py +++ b/packages/microcosm-data/src/microcosm/data/contract.py @@ -96,10 +96,11 @@ ) # Lockstep with microcosm.calibrate.diagnostics.CALIBRATION_DIAGNOSTICS_SCHEMA_VERSION -# (schema 5 = the #492 past_cap_census block). microcosm-data cannot import +# (schema 6 = final per-target loss attribution plus warning-only degradation). +# microcosm-data cannot import # microcosm-calibrate (dependency direction), so the builder test suite pins the # two constants equal — see test_calibration_diagnostics_schema_lockstep. -CALIBRATION_DIAGNOSTICS_SCHEMA_VERSION = 5 +CALIBRATION_DIAGNOSTICS_SCHEMA_VERSION = 6 US_SOURCE_COVERAGE_DIAGNOSTICS_FILE = "us_source_coverage.json" SOURCE_COVERAGE_DIAGNOSTICS_SCHEMA_VERSION = 1 _SHA256_RE = re.compile(r"^[0-9a-f]{64}$") diff --git a/packages/microcosm-data/tests/test_contract.py b/packages/microcosm-data/tests/test_contract.py index 56df6e3ec..457507bb5 100644 --- a/packages/microcosm-data/tests/test_contract.py +++ b/packages/microcosm-data/tests/test_contract.py @@ -332,7 +332,7 @@ def _release_manifest( def _calibration_diagnostics() -> dict: return { - "schema_version": 5, + "schema_version": 6, "weight_entity": "household", "options": {"epochs": 120}, "target_surface": { @@ -1866,7 +1866,7 @@ def test_legacy_diagnostics_exemption_is_scoped_to_the_exact_june_id( payload=diagnostics, ) - with pytest.raises(ReleaseContractError, match="publishes version 5"): + with pytest.raises(ReleaseContractError, match="publishes version 6"): validate_release_dir(directory) diff --git a/packages/microcosm-data/tests/test_release.py b/packages/microcosm-data/tests/test_release.py index 74c773fb2..76f1f34c3 100644 --- a/packages/microcosm-data/tests/test_release.py +++ b/packages/microcosm-data/tests/test_release.py @@ -82,7 +82,7 @@ def _no_slack_webhook(monkeypatch): def _calibration_diagnostics() -> dict: return { - "schema_version": 5, + "schema_version": 6, "weight_entity": "household", "options": {"epochs": 120}, "target_surface": {