feat(retention): ASVS 14.2.7 — retention classification, automatic deletion, and the legacy outbox PHI leak

.github/workflows/ci.yml

@@ -426,7 +426,7 @@ jobs:
           # the ADR 0013/0016 RTE capture/re-ingress case); those suites are MEFOR_TEST_SQLSERVER-gated
           # and run ONLY on the server-DB legs, so a file not matched here edits with NO real SS/PG
           # coverage (it merely skips the gated leg). Keep this in sync with those steps
-          # (docs/throughput-build-plan.md).
+          # (docs/archive/throughput/throughput-build-plan.md).
           # The transports/ + config/wiring arms are the CAPTURE SURFACE: capture_response / reingress_to
           # are declared there (ADR 0013), and SQL Server / Postgres both declare supports_response_capture
           # + supports_pt_reingress True — so a regression in that surface is a SERVER-DB regression and must
@@ -676,7 +676,14 @@ jobs:
         # reliability-core invariant tests — crash-replay, poison-bound, and the per-lane-FIFO locked-head-
         # BLOCKS (#285) gate — that are MEFOR_TEST_SQLSERVER-gated but live in their OWN files, so the named
         # suites above never picked them up and they ran only on SQLite. Run them here against the real
-        # backend. APPEND each new lever's test file per docs/throughput-build-plan.md.
+        # backend. APPEND each new lever's test file per docs/archive/throughput/throughput-build-plan.md.
+        #
+        # NOT only throughput levers: this step is the catch-all for ANY MEFOR_TEST_SQLSERVER-gated file
+        # the named suites above do not collect. CI runs an explicit file LIST here, not the whole suite,
+        # so a new gated file that nobody appends never runs in CI at all — it just reports as skipped on
+        # the plain legs, which reads identical to passing. `tests/test_sqlserver_legacy_outbox_migration.py`
+        # (ASVS 14.2.7) is here for exactly that reason: it verifies a one-way schema migration that can
+        # only be observed against a real server.
         #
         # RETRY WRAPPER: pyodbc 5.3.0 intermittently segfaults in its C parameter-binding path under
         # py3.14 against the SQL Server 2025 container (upstream mkleehammer/pyodbc#1459, unfixed; 5.3.0
@@ -698,6 +705,7 @@ jobs:
           tests/test_per_lane_wake.py
           tests/test_stage_dispatcher.py
           tests/test_pooled_rider.py
+          tests/test_sqlserver_legacy_outbox_migration.py
 
       - name: Run the RTE capture / re-ingress loop on real SQL Server (ADR 0013/0016)
         env:
@@ -815,7 +823,7 @@ jobs:
           MEFOR_ALLOW_INSECURE_TLS: "1"
         # The throughput levers' MEFOR_TEST_POSTGRES-gated invariants (esp. the FOR-UPDATE no-SKIP-LOCKED
         # locked-head-BLOCKS twin of #285, and the batch FIFO/crash tests) live in their own files. APPEND
-        # each new lever's test file per docs/throughput-build-plan.md.
+        # each new lever's test file per docs/archive/throughput/throughput-build-plan.md.
         run: >-
           pytest -v
           tests/test_inline_fast_path.py

harness/load/connscale/runner.py

@@ -72,14 +72,14 @@
 # to). On a SERVER backend every (mode, count) step shares ONE database, so — mirroring the SQLite
 # fresh-file-per-step path — each step first EMPTIES these so the pooled arm never runs second against
 # the per_lane arm's residue (the carryover confound). Table lists verified against the store schemas
-# (store/sqlserver.py, store/postgres.py) and match the reset the store tests use: SQL Server carries
-# the legacy `outbox` table alongside the unified `queue`; the Postgres backend has no `outbox`.
+# (store/sqlserver.py, store/postgres.py) and match the reset the store tests use. Neither backend has
+# an `outbox` table: SQL Server's legacy one was folded into `queue` and DROPped (ASVS 14.2.7), and
+# Postgres never had one. Naming it here would fail the reset with *Invalid object name*.
 _SQLSERVER_PIPELINE_TABLES = (
     "message_events",
     "queue",
     "response",
     "delivered_keys",
-    "outbox",
     "messages",
 )
 

harness/load/shardcert.py

@@ -1180,7 +1180,6 @@ async def _reset_store(env: Mapping[str, str]) -> None:
             cur = await conn.cursor()
             for table in (
                 "queue",
-                "outbox",
                 "response",
                 "delivered_keys",
                 "state",

messagefoundry/__main__.py

@@ -980,7 +980,7 @@
 _DEFAULT_SERVICE_TOML = "messagefoundry.toml"
 
 
 def _serve(args: argparse.Namespace) -> int:
     import uvicorn
     from pydantic import ValidationError
 
@@ -2007,45 +2007,97 @@
         # default threads through to the RetentionRunner (no forbidden-file edit).
         # messages_days moved to [security].delete_message_bodies_after_days (ADR 0118);
         # dead_letter_days stays [retention] plumbing — label each window at its real home.
-        _RETENTION_WINDOW_LABEL = {
-            "messages_days": "[security].delete_message_bodies_after_days",
-            "dead_letter_days": "[retention].dead_letter_days",
-        }
-        if not enforcing and not settings.retention.allow_unbounded_phi:
-            auto_bounded = [
-                field
-                for field in ("messages_days", "dead_letter_days")
-                if field not in settings.retention.model_fields_set
+        # ASVS 14.2.7: the tier list is GENERATED from the classification in
+        # config/retention_classification.py, which a drift test holds equal — in both directions — to
+        # docs/PHI.md §2's Retention column. It used to be a two-element literal here, and the cell
+        # broke once because a new PHI tier landed and nobody widened it. A wider literal with no
+        # binding to the classification is the same defect with more characters.
+        from messagefoundry.config.retention_classification import (
+            MIN_PHI_RETENTION_WINDOWS,
+            PHI_RETENTION_WINDOWS,
+            auto_bounded_windows,
+        )
+        from messagefoundry.config.retention_classification import (
+            unbounded_windows as _unbounded_windows,
+        )
+
+        # A FLOOR, not an emptiness check. `if not PHI_RETENTION_WINDOWS` passes for a one-element
+        # tuple, so a bad merge dropping most entries would leave this gate checking one window while
+        # reporting success — the precise shape of failure this whole change set exists to remove.
+        if len(PHI_RETENTION_WINDOWS) < MIN_PHI_RETENTION_WINDOWS:
+            print(
+                f"error: the PHI retention classification has shrunk to "
+                f"{len(PHI_RETENTION_WINDOWS)} windows (floor {MIN_PHI_RETENTION_WINDOWS}); refusing "
+                "to start rather than gate on a partial classification. This is a build defect, not a "
+                "configuration one — see messagefoundry/config/retention_classification.py.",
+                file=sys.stderr,
+            )
+            return 2
+
+        # AUTO-BOUND. Owner ruling 2026-07-30: the three PHI-BODY windows default to 30 days when
+        # UNSET, on BOTH dials — previously this ran only when `not enforcing`, so on the shipped
+        # `enforce` posture an unset window took the refusal below instead of a default.
+        #
+        # THE SAFETY TRADE IS DELIBERATE AND WORTH STATING: a production PHI instance with an unset
+        # window used to REFUSE TO START, which forced an operator to choose a number. It now starts
+        # with 30. What survives is the fail-closed path for an EXPLICIT 0 — choosing keep-forever is
+        # still refused unless the audited opt-out is set. So "unbounded by accident" is still
+        # prevented; "unbounded by inattention" becomes "30 days by inattention".
+        #
+        # The warn-only windows are NOT auto-bounded, and that is also a ruling rather than an
+        # omission: `purge_state` and `purge_search_presets` key on timestamps that only move on a
+        # WRITE, so silently bounding them deletes live operational data a Handler is still reading.
+        if not settings.retention.allow_unbounded_phi:
+            defaulted = [
+                w
+                for w in auto_bounded_windows()
+                if w.field not in getattr(settings, w.reads_from.strip("[]")).model_fields_set
             ]
-            for field in auto_bounded:
-                setattr(settings.retention, field, 30)
-            if auto_bounded:
-                auto_desc = ", ".join(_RETENTION_WINDOW_LABEL[field] for field in auto_bounded)
+            for window in defaulted:
+                setattr(
+                    getattr(settings, window.reads_from.strip("[]")),
+                    window.field,
+                    window.auto_bound_days,
+                )
+            if defaulted:
                 print(
-                    f"info: {auto_desc} defaulted ON (30 days) for a PHI instance ({env_name!r}) — "
-                    "PHI message bodies are now bounded at rest (secure-by-default, ASVS 14.2.7). Set an "
-                    "explicit [retention] window to override, or [security].allow_keeping_phi_indefinitely=true to "
-                    "retain indefinitely.",
+                    f"info: {', '.join(w.setting for w in defaulted)} defaulted ON (30 days) for a PHI "
+                    f"instance ({env_name!r}) — these PHI tiers are now bounded at rest "
+                    "(secure-by-default, ASVS 14.2.7). Set an explicit window to override, or "
+                    "[security].allow_keeping_phi_indefinitely=true to retain indefinitely.",
                     file=sys.stderr,
                 )
-        unbounded_windows = [
-            field
-            for field, days in (
-                ("messages_days", settings.retention.messages_days),
-                ("dead_letter_days", settings.retention.dead_letter_days),
+
+        # REFUSE / WARN. `unbounded_windows` skips the tiers where 0 does not mean unbounded
+        # (`connection_event_retention_hours` INHERITS the body window; `uploads_retention_days` has a
+        # ge=1 floor so 0 is unrepresentable) and those whose `requires_setting` is unmet — with no
+        # [logging].log_dir there is nothing for the app-log sweep to sweep.
+        still_unbounded = _unbounded_windows(settings)
+        refusable = [w for w in still_unbounded if w.auto_bound_days is not None]
+        warn_only = [w for w in still_unbounded if w.auto_bound_days is None]
+
+        if warn_only:
+            # Classified and warned, never refused. Naming the tier AND its protection level is the
+            # point: an operator who sees "PL-1" knows a full body is involved.
+            print(
+                "warning: these classified PHI tiers have no retention window on a PHI instance "
+                f"({env_name!r}) and will accumulate without bound: "
+                + ", ".join(f"{w.setting} ({w.level})" for w in warn_only)
+                + ". They are deliberately NOT defaulted — each keys on a timestamp that only moves on "
+                "a write, so a silent default would delete data still in use (ASVS 14.2.7).",
+                file=sys.stderr,
             )
-            if days <= 0
-        ]
-        if unbounded_windows:
-            windows_desc = ", ".join(_RETENTION_WINDOW_LABEL[field] for field in unbounded_windows)
+
+        if refusable:
+            windows_desc = ", ".join(w.setting for w in refusable)
             if not settings.retention.allow_unbounded_phi:
                 if enforcing:
                     print(
-                        f"error: no data-retention window is configured for {windows_desc} on a "
-                        f"{'production ' if production else ''}PHI instance ({env_name!r}); refusing to "
-                        "start — PHI message bodies would be retained indefinitely (unbounded PHI at "
-                        "rest, ASVS 14.2.4). Set the window(s) to a positive number of days (e.g. 30) to "
-                        "bound PHI at rest; or, to deliberately retain forever, set "
+                        f"error: a data-retention window is explicitly disabled for {windows_desc} on "
+                        f"a {'production ' if production else ''}PHI instance ({env_name!r}); refusing "
+                        "to start — PHI message bodies would be retained indefinitely (unbounded PHI "
+                        "at rest, ASVS 14.2.4/14.2.7). Set the window(s) to a positive number of days "
+                        "(e.g. 30); or, to deliberately retain forever, set "
                         "[security].allow_keeping_phi_indefinitely=true (audited).",
                         file=sys.stderr,
                     )