From 0f5fde2610ae60ea9d02a93363573ecbdd0452ca Mon Sep 17 00:00:00 2001
From: Anthony Volk <14987227+anth-volk@users.noreply.github.com>
Date: Mon, 9 Nov 2026 21:49:41 +0400
Subject: [PATCH 1/5] Add failing tests that a preempted Modal operation is
 recorded at WARNING

They fail with policyengine-observability 3.0.4, which records an
operation that ends in KeyboardInterrupt at ERROR. They pass with the
change in PolicyEngine/policyengine-observability#44; the executor's
uv.lock moves to that release once it is published.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 ...record-preemptions-as-interrupted.fixed.md |  1 +
 .../tests/conftest.py                         | 49 +++++++++++++++
 .../tests/test_job_progress.py                | 60 +++++++++++++++++++
 .../tests/test_stage12_modal_app.py           | 36 +++++++++++
 4 files changed, 146 insertions(+)
 create mode 100644 changelog.d/record-preemptions-as-interrupted.fixed.md

diff --git a/changelog.d/record-preemptions-as-interrupted.fixed.md b/changelog.d/record-preemptions-as-interrupted.fixed.md
new file mode 100644
index 000000000..a9e2ea920
--- /dev/null
+++ b/changelog.d/record-preemptions-as-interrupted.fixed.md
@@ -0,0 +1 @@
+A Modal preemption of a simulation worker, segment worker or Stage 12 function is now recorded as an interrupted operation at WARNING instead of a failed one at ERROR, so it no longer opens telemetry error issues. Modal delivers a preemption as `KeyboardInterrupt` and runs the same input again; the simulation executor now uses a policyengine-observability release that records an operation ending in `KeyboardInterrupt` with the outcome `interrupted`.
diff --git a/projects/policyengine-simulation-executor/tests/conftest.py b/projects/policyengine-simulation-executor/tests/conftest.py
index a43259d61..84ef9668b 100644
--- a/projects/policyengine-simulation-executor/tests/conftest.py
+++ b/projects/policyengine-simulation-executor/tests/conftest.py
@@ -73,6 +73,55 @@ def real_observability_runtime(monkeypatch):
     runtime.shutdown()
 
 
+@pytest.fixture
+def recorded_observability_runtime():
+    """Build an observability runtime that keeps every record it writes.
+
+    Yields the runtime and the list its structured records are appended to,
+    in the order they were written. OpenTelemetry is disabled and nothing is
+    exported.
+    """
+    from policyengine_observability import (
+        CustomLogDestination,
+        DeploymentIdentity,
+        LoggingConfig,
+        ObservabilityConfig,
+        OTelConfig,
+        ServiceIdentity,
+        configure,
+    )
+
+    written_records: list[dict] = []
+
+    class ListRecordWriter:
+        def write(self, record: dict) -> None:
+            written_records.append(record)
+
+    runtime = configure(
+        ObservabilityConfig(
+            service=ServiceIdentity(
+                name="policyengine-simulation-executor-test",
+                namespace="policyengine.test",
+                version="test",
+                role="test",
+            ),
+            deployment=DeploymentIdentity(environment="test", platform="local"),
+            logging=LoggingConfig(
+                destinations=(
+                    CustomLogDestination(
+                        name="test-records",
+                        writer_factory=ListRecordWriter,
+                        delivery="inline",
+                    ),
+                ),
+            ),
+            otel=OTelConfig(enabled=False),
+        )
+    )
+    yield runtime, written_records
+    runtime.shutdown()
+
+
 @pytest.fixture(autouse=True)
 def _forget_spm_runtime_caches():
     """Keep the SPM runtime memoization out of every other test's way.
diff --git a/projects/policyengine-simulation-executor/tests/test_job_progress.py b/projects/policyengine-simulation-executor/tests/test_job_progress.py
index 4153bd85e..2e725fedc 100644
--- a/projects/policyengine-simulation-executor/tests/test_job_progress.py
+++ b/projects/policyengine-simulation-executor/tests/test_job_progress.py
@@ -919,6 +919,30 @@ def preempted(_runtime):
         assert writes[-1][1].stage is JobStage.REFORM
         assert writes[-1][1].attempts == 2
 
+    def test__preempted_job_is_recorded_at_warning(
+        self, modal_app, monkeypatch, recorded_observability_runtime
+    ):
+        # Modal preempts a container by raising KeyboardInterrupt in the
+        # running function, then runs the same input again.
+        runtime, written_records = recorded_observability_runtime
+
+        def dispatch(_params, *, app_name, runtime, progress):
+            progress.enter(JobStage.BASELINE)
+            raise KeyboardInterrupt
+
+        self._wire(modal_app, monkeypatch, dispatch)
+        monkeypatch.setattr(
+            modal_app,
+            "_configure_modal_observability",
+            lambda *, service_role: runtime,
+        )
+
+        with pytest.raises(KeyboardInterrupt):
+            modal_app.run_simulation({"country": "us"})
+
+        assert _operation_completions(written_records) == [("WARNING", "interrupted")]
+        assert _error_records(written_records) == []
+
     def test__store_failure_does_not_change_the_result(self, modal_app, monkeypatch):
         from policyengine_simulation_contract import job_progress_state
 
@@ -946,3 +970,39 @@ def missing_call_id():
         )
 
         assert modal_app._job_progress_reporter() is NULL_JOB_PROGRESS
+
+
+def test_preempted_segment_is_recorded_at_warning(
+    modal_app, monkeypatch, recorded_observability_runtime
+):
+    # A segment child of a national run is preempted the same way.
+    runtime, written_records = recorded_observability_runtime
+    monkeypatch.setattr(
+        modal_app,
+        "_configure_modal_observability",
+        lambda *, service_role: runtime,
+    )
+
+    def preempted(_params, *, runtime):
+        raise KeyboardInterrupt
+
+    monkeypatch.setattr(sr, "run_simulation_impl", preempted)
+
+    with pytest.raises(KeyboardInterrupt):
+        modal_app.run_simulation_segment({"country": "us"})
+
+    assert _operation_completions(written_records) == [("WARNING", "interrupted")]
+    assert _error_records(written_records) == []
+
+
+def _operation_completions(written_records: list[dict]) -> list[tuple[str, str]]:
+    """Return the severity and outcome of each operation completion record."""
+    return [
+        (record["severity"], record["outcome"])
+        for record in written_records
+        if record.get("event.name") == "operation.completed"
+    ]
+
+
+def _error_records(written_records: list[dict]) -> list[dict]:
+    return [record for record in written_records if record["severity"] == "ERROR"]
diff --git a/projects/policyengine-simulation-executor/tests/test_stage12_modal_app.py b/projects/policyengine-simulation-executor/tests/test_stage12_modal_app.py
index e511ee05e..7567a9d32 100644
--- a/projects/policyengine-simulation-executor/tests/test_stage12_modal_app.py
+++ b/projects/policyengine-simulation-executor/tests/test_stage12_modal_app.py
@@ -449,6 +449,42 @@ def fail(*_, **__):
         _invoke(monkeypatch, module, function_name, fail)
 
 
+@pytest.mark.parametrize(
+    "function_name",
+    [
+        "run_single_simulation_us",
+        "run_single_simulation_uk",
+        "run_single_simulation_segment_us",
+        "coordinate_report",
+    ],
+)
+def test_stage12_functions_record_a_preemption_at_warning(
+    monkeypatch,
+    recorded_observability_runtime,
+    function_name,
+) -> None:
+    # Modal preempts a container by raising KeyboardInterrupt in the running
+    # function, then runs the same input again in a new container.
+    runtime, written_records = recorded_observability_runtime
+    module = _load(monkeypatch)
+    monkeypatch.setattr(module, "init_process_observability", lambda **_: runtime)
+
+    def preempted(*_, **__):
+        raise KeyboardInterrupt
+
+    with pytest.raises(KeyboardInterrupt):
+        _invoke(monkeypatch, module, function_name, preempted)
+
+    [completion] = [
+        record
+        for record in written_records
+        if record.get("event.name") == "operation.completed"
+    ]
+    assert completion["severity"] == "WARNING"
+    assert completion["outcome"] == "interrupted"
+    assert [record for record in written_records if record["severity"] == "ERROR"] == []
+
+
 def test_external_package_values_are_assertions_only(monkeypatch) -> None:
     install_fake_modal(monkeypatch)
     monkeypatch.setenv("STAGE12_EXPECT_US_VERSION", "not-the-bundled-version")

From 193ff0a894c22d56efd7808343abefa3c8e7cab4 Mon Sep 17 00:00:00 2001
From: Anthony Volk <14987227+anth-volk@users.noreply.github.com>
Date: Tue, 10 Nov 2026 00:43:36 +0400
Subject: [PATCH 2/5] Require policyengine-observability 3.0.7 so preemptions
 are recorded at WARNING

3.0.7 records an operation, request or span that ends in KeyboardInterrupt
with the outcome "interrupted" at WARNING and leaves it out of the error
count metric. Modal delivers a preemption as KeyboardInterrupt, so a
preempted worker attempt no longer writes an ERROR record. The tests added
in the previous commit now pass.

Raises the floor to >=3.0.7,<4 in the observability library, the
entrypoint, the gateway and both executor dependency lists, and relocks the
six projects whose uv.lock holds the package with uv 0.13.6
(uv lock --upgrade-package policyengine-observability); only that package
moves in each lock. Updates the type-checking and observability skills.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .../record-preemptions-as-interrupted.fixed.md |  2 +-
 docs/engineering/skills/observability.md       | 18 +++++++++++++++---
 docs/engineering/skills/type-checking.md       |  6 +++---
 libs/policyengine-simulation-contract/uv.lock  |  8 ++++----
 .../pyproject.toml                             |  2 +-
 .../uv.lock                                    |  8 ++++----
 libs/policyengine-stage12-persistence/uv.lock  |  8 ++++----
 .../pyproject.toml                             |  2 +-
 projects/policyengine-simulation-entry/uv.lock | 10 +++++-----
 .../pyproject.toml                             |  4 ++--
 .../policyengine-simulation-executor/uv.lock   | 14 +++++++-------
 .../pyproject.toml                             |  2 +-
 .../policyengine-simulation-gateway/uv.lock    | 10 +++++-----
 13 files changed, 53 insertions(+), 41 deletions(-)

diff --git a/changelog.d/record-preemptions-as-interrupted.fixed.md b/changelog.d/record-preemptions-as-interrupted.fixed.md
index a9e2ea920..07bd1ae3f 100644
--- a/changelog.d/record-preemptions-as-interrupted.fixed.md
+++ b/changelog.d/record-preemptions-as-interrupted.fixed.md
@@ -1 +1 @@
-A Modal preemption of a simulation worker, segment worker or Stage 12 function is now recorded as an interrupted operation at WARNING instead of a failed one at ERROR, so it no longer opens telemetry error issues. Modal delivers a preemption as `KeyboardInterrupt` and runs the same input again; the simulation executor now uses a policyengine-observability release that records an operation ending in `KeyboardInterrupt` with the outcome `interrupted`.
+A Modal preemption of a simulation worker, segment worker or Stage 12 function is now recorded as an interrupted operation at WARNING instead of a failed one at ERROR, so it no longer opens telemetry error issues. Modal delivers a preemption as `KeyboardInterrupt` and runs the same input again; the executor, the gateway and the entrypoint now require policyengine-observability 3.0.7, which records an operation, request or span that ends in `KeyboardInterrupt` with the outcome `interrupted` and leaves it out of the error count metric. The move from 3.0.4 also brings 3.0.5's change: a record written while a FastAPI endpoint runs, such as `exception.recorded`, carries the matched route template as `http.route` instead of `<unmatched>`.
diff --git a/docs/engineering/skills/observability.md b/docs/engineering/skills/observability.md
index c7ad2ac63..0a80fe977 100644
--- a/docs/engineering/skills/observability.md
+++ b/docs/engineering/skills/observability.md
@@ -80,9 +80,9 @@ appropriate `StagePlan` and call `plan.name(Stage.VALUE)`. Do not add span name
 string literals in services or workers.
 
 Open a span with `runtime.span(plan.name(Stage.VALUE))`, and an operation with
-`runtime.operation(...)`. policyengine-observability 3.0.4 types both scopes
-correctly: their `__exit__` and `__aexit__` return `Literal[False]`, so Pyright
-knows an exception raised in the block propagates; see
+`runtime.operation(...)`. policyengine-observability 3.0.4 and later type both
+scopes correctly: their `__exit__` and `__aexit__` return `Literal[False]`, so
+Pyright knows an exception raised in the block propagates; see
 `docs/engineering/skills/type-checking.md`.
 
 When adding a run configuration or runtime stage:
@@ -122,6 +122,18 @@ Modal can run the same call again after a preemption, under the same call
 id. A later attempt continues the stored stage record, so the durations add
 up over attempts, and `job_attempts` counts them.
 
+### Preemptions
+
+Modal delivers a preemption to the running function as `KeyboardInterrupt`,
+then runs the same input again. policyengine-observability 3.0.7 and later
+record an operation, request or span that ends in `KeyboardInterrupt` with
+the outcome `interrupted` at WARNING, leave it out of the error count metric,
+and let the exception propagate unchanged, so a preempted attempt doesn't
+reach the daily telemetry error report. Let `KeyboardInterrupt` leave the
+operation block as it is: code that catches it and raises another exception
+instead reports the attempt as a failure at ERROR. `except Exception` doesn't
+catch it.
+
 ## Metric resource identity
 
 Every process that emits OpenTelemetry metrics must have a unique
diff --git a/docs/engineering/skills/type-checking.md b/docs/engineering/skills/type-checking.md
index b1b8abbd2..c666d8010 100644
--- a/docs/engineering/skills/type-checking.md
+++ b/docs/engineering/skills/type-checking.md
@@ -174,11 +174,11 @@ section is still in the registry fails the guard on every pull request.
 - If an error shows a real bug, first write a test that demonstrates it, then
   fix it.
 - Open observability spans and operations with `runtime.span(...)` and
-  `runtime.operation(...)`. policyengine-observability 3.0.4 types them
-  correctly: their `__exit__` and `__aexit__` return `Literal[False]`, so
+  `runtime.operation(...)`. policyengine-observability 3.0.4 and later type
+  them correctly: their `__exit__` and `__aexit__` return `Literal[False]`, so
   Pyright knows they never suppress an exception and doesn't report a variable
   the block assigns as possibly unbound after it. Every package that depends
-  on policyengine-observability requires 3.0.4 or later, and
+  on policyengine-observability requires 3.0.7 or later, and
   `libs/policyengine-simulation-observability/tests/test_scope_exit_typing.py`
   fails if an older release is installed.
 - Before reading a simulation's output frames, narrow
diff --git a/libs/policyengine-simulation-contract/uv.lock b/libs/policyengine-simulation-contract/uv.lock
index 3fa7e7d15..510557170 100644
--- a/libs/policyengine-simulation-contract/uv.lock
+++ b/libs/policyengine-simulation-contract/uv.lock
@@ -1069,11 +1069,11 @@ wheels = [
 
 [[package]]
 name = "policyengine-observability"
-version = "3.0.4"
+version = "3.0.7"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/c3/d0/09b91aca37fb3efd493c164082097e85705fcac2062d2b5ef8a8c4e2be04/policyengine_observability-3.0.4.tar.gz", hash = "sha256:7e43a5d0e9b9e5f683623643efca7c47e651baaad7b557a50ec8b344602dc61f", size = 127972, upload-time = "2026-10-27T17:23:24.343Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/8d/00/5c06e93c648c48cb3368f7916cd1b5d5e2d944b471a709534f52f6cd27fc/policyengine_observability-3.0.7.tar.gz", hash = "sha256:f26883cf89eae8dfe5e458c5050b8c433c8223d52d75202d9ea13eb6cb7825c3", size = 133163, upload-time = "2026-11-09T20:29:37.192Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/00/00/d9c4850a12b4d0bef72034316909aac97228de302384af8936a958b7dc8e/policyengine_observability-3.0.4-py3-none-any.whl", hash = "sha256:1b3ebbfd00c138abcc2ec9580b13aa55679b91745783c5567cc73f8b2bbaaaf7", size = 42204, upload-time = "2026-10-27T17:23:22.641Z" },
+    { url = "https://files.pythonhosted.org/packages/c2/d2/8cf771e779f2d6aaf08c1d96ed77220e0d358579104a3e43ada09584715c/policyengine_observability-3.0.7-py3-none-any.whl", hash = "sha256:3953201cc9ec618b46fcfed959355d64c715f8fc7df8c111996d7983588a40d0", size = 44004, upload-time = "2026-11-09T20:29:35.985Z" },
 ]
 
 [package.optional-dependencies]
@@ -1145,7 +1145,7 @@ requires-dist = [
     { name = "black", marker = "extra == 'build'", specifier = ">=25.1.0" },
     { name = "fastapi", specifier = ">=0.115.0" },
     { name = "httpx2", marker = "extra == 'test'" },
-    { name = "policyengine-observability", extras = ["fastapi", "google", "otlp-grpc"], specifier = ">=3.0.4,<4" },
+    { name = "policyengine-observability", extras = ["fastapi", "google", "otlp-grpc"], specifier = ">=3.0.7,<4" },
     { name = "pydantic", specifier = ">=2.0" },
     { name = "pyright", marker = "extra == 'build'", specifier = ">=1.1.401" },
     { name = "pytest", marker = "extra == 'test'", specifier = ">=8.3.4" },
diff --git a/libs/policyengine-simulation-observability/pyproject.toml b/libs/policyengine-simulation-observability/pyproject.toml
index d7cf62251..5d2318f05 100644
--- a/libs/policyengine-simulation-observability/pyproject.toml
+++ b/libs/policyengine-simulation-observability/pyproject.toml
@@ -10,7 +10,7 @@ requires-python = ">=3.13"
 dependencies = [
     "fastapi>=0.115.0",
     "pydantic>=2.0",
-    "policyengine-observability[fastapi,google,otlp-grpc]>=3.0.4,<4",
+    "policyengine-observability[fastapi,google,otlp-grpc]>=3.0.7,<4",
 ]
 
 [project.optional-dependencies]
diff --git a/libs/policyengine-simulation-observability/uv.lock b/libs/policyengine-simulation-observability/uv.lock
index 8e971eb08..53b73adab 100644
--- a/libs/policyengine-simulation-observability/uv.lock
+++ b/libs/policyengine-simulation-observability/uv.lock
@@ -720,11 +720,11 @@ wheels = [
 
 [[package]]
 name = "policyengine-observability"
-version = "3.0.4"
+version = "3.0.7"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/c3/d0/09b91aca37fb3efd493c164082097e85705fcac2062d2b5ef8a8c4e2be04/policyengine_observability-3.0.4.tar.gz", hash = "sha256:7e43a5d0e9b9e5f683623643efca7c47e651baaad7b557a50ec8b344602dc61f", size = 127972, upload-time = "2026-10-27T17:23:24.343Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/8d/00/5c06e93c648c48cb3368f7916cd1b5d5e2d944b471a709534f52f6cd27fc/policyengine_observability-3.0.7.tar.gz", hash = "sha256:f26883cf89eae8dfe5e458c5050b8c433c8223d52d75202d9ea13eb6cb7825c3", size = 133163, upload-time = "2026-11-09T20:29:37.192Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/00/00/d9c4850a12b4d0bef72034316909aac97228de302384af8936a958b7dc8e/policyengine_observability-3.0.4-py3-none-any.whl", hash = "sha256:1b3ebbfd00c138abcc2ec9580b13aa55679b91745783c5567cc73f8b2bbaaaf7", size = 42204, upload-time = "2026-10-27T17:23:22.641Z" },
+    { url = "https://files.pythonhosted.org/packages/c2/d2/8cf771e779f2d6aaf08c1d96ed77220e0d358579104a3e43ada09584715c/policyengine_observability-3.0.7-py3-none-any.whl", hash = "sha256:3953201cc9ec618b46fcfed959355d64c715f8fc7df8c111996d7983588a40d0", size = 44004, upload-time = "2026-11-09T20:29:35.985Z" },
 ]
 
 [package.optional-dependencies]
@@ -768,7 +768,7 @@ requires-dist = [
     { name = "black", marker = "extra == 'build'", specifier = ">=25.1.0" },
     { name = "fastapi", specifier = ">=0.115.0" },
     { name = "httpx2", marker = "extra == 'test'" },
-    { name = "policyengine-observability", extras = ["fastapi", "google", "otlp-grpc"], specifier = ">=3.0.4,<4" },
+    { name = "policyengine-observability", extras = ["fastapi", "google", "otlp-grpc"], specifier = ">=3.0.7,<4" },
     { name = "pydantic", specifier = ">=2.0" },
     { name = "pyright", marker = "extra == 'build'", specifier = ">=1.1.401" },
     { name = "pytest", marker = "extra == 'test'", specifier = ">=8.3.4" },
diff --git a/libs/policyengine-stage12-persistence/uv.lock b/libs/policyengine-stage12-persistence/uv.lock
index 20d14c19b..a1d9b1fab 100644
--- a/libs/policyengine-stage12-persistence/uv.lock
+++ b/libs/policyengine-stage12-persistence/uv.lock
@@ -566,11 +566,11 @@ wheels = [
 
 [[package]]
 name = "policyengine-observability"
-version = "3.0.4"
+version = "3.0.7"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/c3/d0/09b91aca37fb3efd493c164082097e85705fcac2062d2b5ef8a8c4e2be04/policyengine_observability-3.0.4.tar.gz", hash = "sha256:7e43a5d0e9b9e5f683623643efca7c47e651baaad7b557a50ec8b344602dc61f", size = 127972, upload-time = "2026-10-27T17:23:24.343Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/8d/00/5c06e93c648c48cb3368f7916cd1b5d5e2d944b471a709534f52f6cd27fc/policyengine_observability-3.0.7.tar.gz", hash = "sha256:f26883cf89eae8dfe5e458c5050b8c433c8223d52d75202d9ea13eb6cb7825c3", size = 133163, upload-time = "2026-11-09T20:29:37.192Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/00/00/d9c4850a12b4d0bef72034316909aac97228de302384af8936a958b7dc8e/policyengine_observability-3.0.4-py3-none-any.whl", hash = "sha256:1b3ebbfd00c138abcc2ec9580b13aa55679b91745783c5567cc73f8b2bbaaaf7", size = 42204, upload-time = "2026-10-27T17:23:22.641Z" },
+    { url = "https://files.pythonhosted.org/packages/c2/d2/8cf771e779f2d6aaf08c1d96ed77220e0d358579104a3e43ada09584715c/policyengine_observability-3.0.7-py3-none-any.whl", hash = "sha256:3953201cc9ec618b46fcfed959355d64c715f8fc7df8c111996d7983588a40d0", size = 44004, upload-time = "2026-11-09T20:29:35.985Z" },
 ]
 
 [package.optional-dependencies]
@@ -627,7 +627,7 @@ requires-dist = [
     { name = "black", marker = "extra == 'build'", specifier = ">=25.1.0" },
     { name = "fastapi", specifier = ">=0.115.0" },
     { name = "httpx2", marker = "extra == 'test'" },
-    { name = "policyengine-observability", extras = ["fastapi", "google", "otlp-grpc"], specifier = ">=3.0.4,<4" },
+    { name = "policyengine-observability", extras = ["fastapi", "google", "otlp-grpc"], specifier = ">=3.0.7,<4" },
     { name = "pydantic", specifier = ">=2.0" },
     { name = "pyright", marker = "extra == 'build'", specifier = ">=1.1.401" },
     { name = "pytest", marker = "extra == 'test'", specifier = ">=8.3.4" },
diff --git a/projects/policyengine-simulation-entry/pyproject.toml b/projects/policyengine-simulation-entry/pyproject.toml
index 11f569523..945a02fae 100644
--- a/projects/policyengine-simulation-entry/pyproject.toml
+++ b/projects/policyengine-simulation-entry/pyproject.toml
@@ -16,7 +16,7 @@ dependencies = [
     "fastapi>=0.115.0,<1",
     "httpx>=0.28.0,<1",
     "modal>=1.4,<2",
-    "policyengine-observability[fastapi,google,httpx,otlp-grpc]>=3.0.4,<4",
+    "policyengine-observability[fastapi,google,httpx,otlp-grpc]>=3.0.7,<4",
     "policyengine-simulation-contract",
     "policyengine-simulation-observability",
     "policyengine-simulation-persistence",
diff --git a/projects/policyengine-simulation-entry/uv.lock b/projects/policyengine-simulation-entry/uv.lock
index de87a4df5..7bef95438 100644
--- a/projects/policyengine-simulation-entry/uv.lock
+++ b/projects/policyengine-simulation-entry/uv.lock
@@ -840,11 +840,11 @@ wheels = [
 
 [[package]]
 name = "policyengine-observability"
-version = "3.0.4"
+version = "3.0.7"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/c3/d0/09b91aca37fb3efd493c164082097e85705fcac2062d2b5ef8a8c4e2be04/policyengine_observability-3.0.4.tar.gz", hash = "sha256:7e43a5d0e9b9e5f683623643efca7c47e651baaad7b557a50ec8b344602dc61f", size = 127972, upload-time = "2026-10-27T17:23:24.343Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/8d/00/5c06e93c648c48cb3368f7916cd1b5d5e2d944b471a709534f52f6cd27fc/policyengine_observability-3.0.7.tar.gz", hash = "sha256:f26883cf89eae8dfe5e458c5050b8c433c8223d52d75202d9ea13eb6cb7825c3", size = 133163, upload-time = "2026-11-09T20:29:37.192Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/00/00/d9c4850a12b4d0bef72034316909aac97228de302384af8936a958b7dc8e/policyengine_observability-3.0.4-py3-none-any.whl", hash = "sha256:1b3ebbfd00c138abcc2ec9580b13aa55679b91745783c5567cc73f8b2bbaaaf7", size = 42204, upload-time = "2026-10-27T17:23:22.641Z" },
+    { url = "https://files.pythonhosted.org/packages/c2/d2/8cf771e779f2d6aaf08c1d96ed77220e0d358579104a3e43ada09584715c/policyengine_observability-3.0.7-py3-none-any.whl", hash = "sha256:3953201cc9ec618b46fcfed959355d64c715f8fc7df8c111996d7983588a40d0", size = 44004, upload-time = "2026-11-09T20:29:35.985Z" },
 ]
 
 [package.optional-dependencies]
@@ -926,7 +926,7 @@ requires-dist = [
     { name = "httpx", specifier = ">=0.28.0,<1" },
     { name = "modal", specifier = ">=1.4,<2" },
     { name = "openapi-python-client", marker = "extra == 'build'", specifier = ">=0.21.6" },
-    { name = "policyengine-observability", extras = ["fastapi", "google", "httpx", "otlp-grpc"], specifier = ">=3.0.4,<4" },
+    { name = "policyengine-observability", extras = ["fastapi", "google", "httpx", "otlp-grpc"], specifier = ">=3.0.7,<4" },
     { name = "policyengine-simulation-contract", editable = "../../libs/policyengine-simulation-contract" },
     { name = "policyengine-simulation-observability", editable = "../../libs/policyengine-simulation-observability" },
     { name = "policyengine-simulation-persistence", editable = "../../libs/policyengine-simulation-persistence" },
@@ -956,7 +956,7 @@ requires-dist = [
     { name = "black", marker = "extra == 'build'", specifier = ">=25.1.0" },
     { name = "fastapi", specifier = ">=0.115.0" },
     { name = "httpx2", marker = "extra == 'test'" },
-    { name = "policyengine-observability", extras = ["fastapi", "google", "otlp-grpc"], specifier = ">=3.0.4,<4" },
+    { name = "policyengine-observability", extras = ["fastapi", "google", "otlp-grpc"], specifier = ">=3.0.7,<4" },
     { name = "pydantic", specifier = ">=2.0" },
     { name = "pyright", marker = "extra == 'build'", specifier = ">=1.1.401" },
     { name = "pytest", marker = "extra == 'test'", specifier = ">=8.3.4" },
diff --git a/projects/policyengine-simulation-executor/pyproject.toml b/projects/policyengine-simulation-executor/pyproject.toml
index 2e10b3192..dfc0e559c 100644
--- a/projects/policyengine-simulation-executor/pyproject.toml
+++ b/projects/policyengine-simulation-executor/pyproject.toml
@@ -24,7 +24,7 @@ dependencies = [
     "policyengine[models]==6.16.0",
     "tables>=3.10.2",
     "modal>=0.73.0",
-    "policyengine-observability[fastapi,google,otlp-grpc]>=3.0.4,<4",
+    "policyengine-observability[fastapi,google,otlp-grpc]>=3.0.7,<4",
     # Imported directly by the artifact store client; do not rely on the
     # transitive pin via policyengine.
     "google-cloud-storage>=2",
@@ -52,7 +52,7 @@ modal-simulation-image = [
     "policyengine[models]==6.16.0",
     "fastapi>=0.115.0",
     "tables>=3.10.2",
-    "policyengine-observability[fastapi,google,otlp-grpc]>=3.0.4,<4",
+    "policyengine-observability[fastapi,google,otlp-grpc]>=3.0.7,<4",
     # The artifact fetch layer and store client read GCS. The lib is also a
     # transitive dependency of policyengine, but the image must not depend
     # on an upstream package keeping it.
diff --git a/projects/policyengine-simulation-executor/uv.lock b/projects/policyengine-simulation-executor/uv.lock
index d5d9eaf83..28c6c5bd7 100644
--- a/projects/policyengine-simulation-executor/uv.lock
+++ b/projects/policyengine-simulation-executor/uv.lock
@@ -1840,11 +1840,11 @@ provides-extras = ["test", "build"]
 
 [[package]]
 name = "policyengine-observability"
-version = "3.0.4"
+version = "3.0.7"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/c3/d0/09b91aca37fb3efd493c164082097e85705fcac2062d2b5ef8a8c4e2be04/policyengine_observability-3.0.4.tar.gz", hash = "sha256:7e43a5d0e9b9e5f683623643efca7c47e651baaad7b557a50ec8b344602dc61f", size = 127972, upload-time = "2026-10-27T17:23:24.343Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/8d/00/5c06e93c648c48cb3368f7916cd1b5d5e2d944b471a709534f52f6cd27fc/policyengine_observability-3.0.7.tar.gz", hash = "sha256:f26883cf89eae8dfe5e458c5050b8c433c8223d52d75202d9ea13eb6cb7825c3", size = 133163, upload-time = "2026-11-09T20:29:37.192Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/00/00/d9c4850a12b4d0bef72034316909aac97228de302384af8936a958b7dc8e/policyengine_observability-3.0.4-py3-none-any.whl", hash = "sha256:1b3ebbfd00c138abcc2ec9580b13aa55679b91745783c5567cc73f8b2bbaaaf7", size = 42204, upload-time = "2026-10-27T17:23:22.641Z" },
+    { url = "https://files.pythonhosted.org/packages/c2/d2/8cf771e779f2d6aaf08c1d96ed77220e0d358579104a3e43ada09584715c/policyengine_observability-3.0.7-py3-none-any.whl", hash = "sha256:3953201cc9ec618b46fcfed959355d64c715f8fc7df8c111996d7983588a40d0", size = 44004, upload-time = "2026-11-09T20:29:35.985Z" },
 ]
 
 [package.optional-dependencies]
@@ -1955,7 +1955,7 @@ requires-dist = [
     { name = "opentelemetry-instrumentation-sqlalchemy", specifier = ">=0.65b0,<0.66" },
     { name = "policyengine", extras = ["models"], specifier = "==6.16.0" },
     { name = "policyengine-fastapi", editable = "../../libs/policyengine-fastapi" },
-    { name = "policyengine-observability", extras = ["fastapi", "google", "otlp-grpc"], specifier = ">=3.0.4,<4" },
+    { name = "policyengine-observability", extras = ["fastapi", "google", "otlp-grpc"], specifier = ">=3.0.7,<4" },
     { name = "policyengine-simulation-contract", extras = ["modal"], editable = "../../libs/policyengine-simulation-contract" },
     { name = "policyengine-simulation-observability", editable = "../../libs/policyengine-simulation-observability" },
     { name = "policyengine-stage12-persistence", editable = "../../libs/policyengine-stage12-persistence" },
@@ -1977,7 +1977,7 @@ modal-simulation-image = [
     { name = "google-cloud-storage", specifier = ">=2" },
     { name = "huggingface-hub", specifier = ">=1,<2" },
     { name = "policyengine", extras = ["models"], specifier = "==6.16.0" },
-    { name = "policyengine-observability", extras = ["fastapi", "google", "otlp-grpc"], specifier = ">=3.0.4,<4" },
+    { name = "policyengine-observability", extras = ["fastapi", "google", "otlp-grpc"], specifier = ">=3.0.7,<4" },
     { name = "psycopg", extras = ["binary"], specifier = ">=3.2,<4" },
     { name = "pyarrow", specifier = ">=20,<24" },
     { name = "requests", specifier = ">=2.32,<3" },
@@ -2008,7 +2008,7 @@ requires-dist = [
     { name = "modal", specifier = ">=1.4,<2" },
     { name = "openapi-python-client", marker = "extra == 'build'", specifier = ">=0.21.6" },
     { name = "packaging", specifier = ">=24" },
-    { name = "policyengine-observability", extras = ["fastapi", "google", "otlp-grpc"], specifier = ">=3.0.4,<4" },
+    { name = "policyengine-observability", extras = ["fastapi", "google", "otlp-grpc"], specifier = ">=3.0.7,<4" },
     { name = "pydantic", specifier = ">=2.0" },
     { name = "pyjwt", specifier = ">=2.10.1,<3.0.0" },
     { name = "pyright", marker = "extra == 'build'", specifier = ">=1.1.401" },
@@ -2040,7 +2040,7 @@ requires-dist = [
     { name = "black", marker = "extra == 'build'", specifier = ">=25.1.0" },
     { name = "fastapi", specifier = ">=0.115.0" },
     { name = "httpx2", marker = "extra == 'test'" },
-    { name = "policyengine-observability", extras = ["fastapi", "google", "otlp-grpc"], specifier = ">=3.0.4,<4" },
+    { name = "policyengine-observability", extras = ["fastapi", "google", "otlp-grpc"], specifier = ">=3.0.7,<4" },
     { name = "pydantic", specifier = ">=2.0" },
     { name = "pyright", marker = "extra == 'build'", specifier = ">=1.1.401" },
     { name = "pytest", marker = "extra == 'test'", specifier = ">=8.3.4" },
diff --git a/projects/policyengine-simulation-gateway/pyproject.toml b/projects/policyengine-simulation-gateway/pyproject.toml
index 4518faa1e..6f1b2d8a9 100644
--- a/projects/policyengine-simulation-gateway/pyproject.toml
+++ b/projects/policyengine-simulation-gateway/pyproject.toml
@@ -25,7 +25,7 @@ dependencies = [
     "pyjwt>=2.10.1,<3.0.0",
     # JWTDecoder (policyengine-fastapi lib) needs cryptography at runtime.
     "cryptography>=41.0.0",
-    "policyengine-observability[fastapi,google,otlp-grpc]>=3.0.4,<4",
+    "policyengine-observability[fastapi,google,otlp-grpc]>=3.0.7,<4",
     "modal>=1.4,<2",
 ]
 
diff --git a/projects/policyengine-simulation-gateway/uv.lock b/projects/policyengine-simulation-gateway/uv.lock
index 8e709799d..be91bef48 100644
--- a/projects/policyengine-simulation-gateway/uv.lock
+++ b/projects/policyengine-simulation-gateway/uv.lock
@@ -1261,11 +1261,11 @@ provides-extras = ["test", "build"]
 
 [[package]]
 name = "policyengine-observability"
-version = "3.0.4"
+version = "3.0.7"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/c3/d0/09b91aca37fb3efd493c164082097e85705fcac2062d2b5ef8a8c4e2be04/policyengine_observability-3.0.4.tar.gz", hash = "sha256:7e43a5d0e9b9e5f683623643efca7c47e651baaad7b557a50ec8b344602dc61f", size = 127972, upload-time = "2026-10-27T17:23:24.343Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/8d/00/5c06e93c648c48cb3368f7916cd1b5d5e2d944b471a709534f52f6cd27fc/policyengine_observability-3.0.7.tar.gz", hash = "sha256:f26883cf89eae8dfe5e458c5050b8c433c8223d52d75202d9ea13eb6cb7825c3", size = 133163, upload-time = "2026-11-09T20:29:37.192Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/00/00/d9c4850a12b4d0bef72034316909aac97228de302384af8936a958b7dc8e/policyengine_observability-3.0.4-py3-none-any.whl", hash = "sha256:1b3ebbfd00c138abcc2ec9580b13aa55679b91745783c5567cc73f8b2bbaaaf7", size = 42204, upload-time = "2026-10-27T17:23:22.641Z" },
+    { url = "https://files.pythonhosted.org/packages/c2/d2/8cf771e779f2d6aaf08c1d96ed77220e0d358579104a3e43ada09584715c/policyengine_observability-3.0.7-py3-none-any.whl", hash = "sha256:3953201cc9ec618b46fcfed959355d64c715f8fc7df8c111996d7983588a40d0", size = 44004, upload-time = "2026-11-09T20:29:35.985Z" },
 ]
 
 [package.optional-dependencies]
@@ -1355,7 +1355,7 @@ requires-dist = [
     { name = "modal", specifier = ">=1.4,<2" },
     { name = "openapi-python-client", marker = "extra == 'build'", specifier = ">=0.21.6" },
     { name = "packaging", specifier = ">=24" },
-    { name = "policyengine-observability", extras = ["fastapi", "google", "otlp-grpc"], specifier = ">=3.0.4,<4" },
+    { name = "policyengine-observability", extras = ["fastapi", "google", "otlp-grpc"], specifier = ">=3.0.7,<4" },
     { name = "pydantic", specifier = ">=2.0" },
     { name = "pyjwt", specifier = ">=2.10.1,<3.0.0" },
     { name = "pyright", marker = "extra == 'build'", specifier = ">=1.1.401" },
@@ -1387,7 +1387,7 @@ requires-dist = [
     { name = "black", marker = "extra == 'build'", specifier = ">=25.1.0" },
     { name = "fastapi", specifier = ">=0.115.0" },
     { name = "httpx2", marker = "extra == 'test'" },
-    { name = "policyengine-observability", extras = ["fastapi", "google", "otlp-grpc"], specifier = ">=3.0.4,<4" },
+    { name = "policyengine-observability", extras = ["fastapi", "google", "otlp-grpc"], specifier = ">=3.0.7,<4" },
     { name = "pydantic", specifier = ">=2.0" },
     { name = "pyright", marker = "extra == 'build'", specifier = ">=1.1.401" },
     { name = "pytest", marker = "extra == 'test'", specifier = ">=8.3.4" },

From 23aa9bc715a40f12faf2e8e2cf96bc0977f9fa20 Mon Sep 17 00:00:00 2001
From: Anthony Volk <14987227+anth-volk@users.noreply.github.com>
Date: Tue, 10 Nov 2026 01:02:26 +0400
Subject: [PATCH 3/5] Record standard-library logs in the preemption tests and
 correct the docs

The recorded_observability_runtime fixture now sets
capture_standard_library=True, as the deployed workers' runtime does, so
the tests' "no ERROR records" checks also see ERROR records written through
logging. runtime.shutdown() removes the handler it installs.

The observability skill no longer says a span is recorded at WARNING: a
span that ends in KeyboardInterrupt keeps an unset status and gets
policyengine.outcome=interrupted. The type-checking skill says the scope
typing test fails only below 3.0.4. The changelog fragment adds the other
3.0.7 changes: the new outcome value on the request and operation metrics,
the span status, and FastAPI requests that end in KeyboardInterrupt, which
3.0.4 recorded from their status code alone.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 changelog.d/record-preemptions-as-interrupted.fixed.md |  4 +++-
 docs/engineering/skills/observability.md               | 10 ++++++----
 docs/engineering/skills/type-checking.md               |  5 +++--
 .../policyengine-simulation-executor/tests/conftest.py |  7 +++++--
 4 files changed, 17 insertions(+), 9 deletions(-)

diff --git a/changelog.d/record-preemptions-as-interrupted.fixed.md b/changelog.d/record-preemptions-as-interrupted.fixed.md
index 07bd1ae3f..46dec58a3 100644
--- a/changelog.d/record-preemptions-as-interrupted.fixed.md
+++ b/changelog.d/record-preemptions-as-interrupted.fixed.md
@@ -1 +1,3 @@
-A Modal preemption of a simulation worker, segment worker or Stage 12 function is now recorded as an interrupted operation at WARNING instead of a failed one at ERROR, so it no longer opens telemetry error issues. Modal delivers a preemption as `KeyboardInterrupt` and runs the same input again; the executor, the gateway and the entrypoint now require policyengine-observability 3.0.7, which records an operation, request or span that ends in `KeyboardInterrupt` with the outcome `interrupted` and leaves it out of the error count metric. The move from 3.0.4 also brings 3.0.5's change: a record written while a FastAPI endpoint runs, such as `exception.recorded`, carries the matched route template as `http.route` instead of `<unmatched>`.
+A Modal preemption of a simulation worker, segment worker or Stage 12 function is now recorded as an interrupted operation at WARNING instead of a failed one at ERROR, so it no longer opens telemetry error issues. Modal delivers a preemption as `KeyboardInterrupt` and runs the same input again; the executor, the gateway and the entrypoint now require policyengine-observability 3.0.7, which records an operation or request that ends in `KeyboardInterrupt` with the outcome `interrupted` at WARNING and leaves it out of `policyengine.error.count`. The request and operation count and duration metrics get `interrupted` as a new value of their `outcome` attribute. A span that ends in `KeyboardInterrupt` keeps an unset status, where it was set to error, and gets the attribute `policyengine.outcome` set to `interrupted`. A gateway or entrypoint request that ends in `KeyboardInterrupt`, which 3.0.4 recorded from its status code alone (as `success` at INFO when no status had been sent), is now recorded as `interrupted` at WARNING too.
+
+The move from 3.0.4 also brings 3.0.5's change: a record written while a FastAPI endpoint runs, such as `exception.recorded`, carries the matched route template as `http.route` instead of `<unmatched>`.
diff --git a/docs/engineering/skills/observability.md b/docs/engineering/skills/observability.md
index 0a80fe977..396175744 100644
--- a/docs/engineering/skills/observability.md
+++ b/docs/engineering/skills/observability.md
@@ -126,10 +126,12 @@ up over attempts, and `job_attempts` counts them.
 
 Modal delivers a preemption to the running function as `KeyboardInterrupt`,
 then runs the same input again. policyengine-observability 3.0.7 and later
-record an operation, request or span that ends in `KeyboardInterrupt` with
-the outcome `interrupted` at WARNING, leave it out of the error count metric,
-and let the exception propagate unchanged, so a preempted attempt doesn't
-reach the daily telemetry error report. Let `KeyboardInterrupt` leave the
+record an operation or request that ends in `KeyboardInterrupt` with the
+outcome `interrupted` at WARNING and leave it out of the error count metric;
+a span that ends in it keeps an unset status and gets the attribute
+`policyengine.outcome` set to `interrupted`. The exception propagates
+unchanged, so a preempted attempt doesn't reach the daily telemetry error
+report. Let `KeyboardInterrupt` leave the
 operation block as it is: code that catches it and raises another exception
 instead reports the attempt as a failure at ERROR. `except Exception` doesn't
 catch it.
diff --git a/docs/engineering/skills/type-checking.md b/docs/engineering/skills/type-checking.md
index c666d8010..932661c75 100644
--- a/docs/engineering/skills/type-checking.md
+++ b/docs/engineering/skills/type-checking.md
@@ -178,9 +178,10 @@ section is still in the registry fails the guard on every pull request.
   them correctly: their `__exit__` and `__aexit__` return `Literal[False]`, so
   Pyright knows they never suppress an exception and doesn't report a variable
   the block assigns as possibly unbound after it. Every package that depends
-  on policyengine-observability requires 3.0.7 or later, and
+  on policyengine-observability requires 3.0.7 or later, which also records
+  a preemption at WARNING (see `observability.md`), and
   `libs/policyengine-simulation-observability/tests/test_scope_exit_typing.py`
-  fails if an older release is installed.
+  fails if a release older than 3.0.4 is installed.
 - Before reading a simulation's output frames, narrow
   `<simulation>.output_dataset.data` with the `EntityFrames` protocol in
   `policyengine_simulation_executor.output_frames`, and keep reading through
diff --git a/projects/policyengine-simulation-executor/tests/conftest.py b/projects/policyengine-simulation-executor/tests/conftest.py
index 84ef9668b..ee22529e4 100644
--- a/projects/policyengine-simulation-executor/tests/conftest.py
+++ b/projects/policyengine-simulation-executor/tests/conftest.py
@@ -78,8 +78,10 @@ def recorded_observability_runtime():
     """Build an observability runtime that keeps every record it writes.
 
     Yields the runtime and the list its structured records are appended to,
-    in the order they were written. OpenTelemetry is disabled and nothing is
-    exported.
+    in the order they were written. Like the deployed workers' runtime, it
+    also records standard-library log records, so a test sees an ERROR
+    record from ``logging`` too; ``runtime.shutdown()`` removes that handler.
+    OpenTelemetry is disabled and nothing is exported.
     """
     from policyengine_observability import (
         CustomLogDestination,
@@ -114,6 +116,7 @@ def write(self, record: dict) -> None:
                         delivery="inline",
                     ),
                 ),
+                capture_standard_library=True,
             ),
             otel=OTelConfig(enabled=False),
         )

From 7c5cadf23e6e869d02f1a12b09afcb36931e38e5 Mon Sep 17 00:00:00 2001
From: Anthony Volk <14987227+anth-volk@users.noreply.github.com>
Date: Tue, 10 Nov 2026 01:29:06 +0400
Subject: [PATCH 4/5] Say that a cancelled Stage 12 child call is still
 recorded at ERROR

Since #992, a preempted Stage 12 segment runner cancels the segment calls
it started, and a preempted coordinator with a report run cancels its
unfinished simulation calls. Modal ends a cancelled call with
InputCancellation, which is not a KeyboardInterrupt, so
policyengine-observability 3.0.7 still records it at ERROR. The
observability skill and the changelog fragment now say so, and the
fragment no longer says every span ending in KeyboardInterrupt was set to
error: 3.0.4 did that only for operation spans and runtime.span spans.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .../record-preemptions-as-interrupted.fixed.md   |  4 +++-
 docs/engineering/skills/observability.md         | 16 +++++++++++-----
 2 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/changelog.d/record-preemptions-as-interrupted.fixed.md b/changelog.d/record-preemptions-as-interrupted.fixed.md
index 46dec58a3..87c2b55ce 100644
--- a/changelog.d/record-preemptions-as-interrupted.fixed.md
+++ b/changelog.d/record-preemptions-as-interrupted.fixed.md
@@ -1,3 +1,5 @@
-A Modal preemption of a simulation worker, segment worker or Stage 12 function is now recorded as an interrupted operation at WARNING instead of a failed one at ERROR, so it no longer opens telemetry error issues. Modal delivers a preemption as `KeyboardInterrupt` and runs the same input again; the executor, the gateway and the entrypoint now require policyengine-observability 3.0.7, which records an operation or request that ends in `KeyboardInterrupt` with the outcome `interrupted` at WARNING and leaves it out of `policyengine.error.count`. The request and operation count and duration metrics get `interrupted` as a new value of their `outcome` attribute. A span that ends in `KeyboardInterrupt` keeps an unset status, where it was set to error, and gets the attribute `policyengine.outcome` set to `interrupted`. A gateway or entrypoint request that ends in `KeyboardInterrupt`, which 3.0.4 recorded from its status code alone (as `success` at INFO when no status had been sent), is now recorded as `interrupted` at WARNING too.
+A Modal preemption of a simulation worker, segment worker or Stage 12 function is now recorded as an interrupted operation at WARNING instead of a failed one at ERROR, so the preempted call no longer opens telemetry error issues. Modal delivers a preemption as `KeyboardInterrupt` and runs the same input again; the executor, the gateway and the entrypoint now require policyengine-observability 3.0.7, which records an operation or request that ends in `KeyboardInterrupt` with the outcome `interrupted` at WARNING and leaves it out of `policyengine.error.count`. The request and operation count and duration metrics get `interrupted` as a new value of their `outcome` attribute. An operation's span, or a span opened with `runtime.span`, that ends in `KeyboardInterrupt` is no longer set to error: its status stays unset and it gets the attribute `policyengine.outcome` set to `interrupted`. A gateway or entrypoint request that ends in `KeyboardInterrupt`, which 3.0.4 recorded from its status code alone (as `success` at INFO when no status had been sent), is now recorded as `interrupted` at WARNING too.
+
+A child call that a preempted Stage 12 segment runner or report coordinator cancels still writes an ERROR record: Modal ends a cancelled call with `InputCancellation`, which is not a `KeyboardInterrupt`.
 
 The move from 3.0.4 also brings 3.0.5's change: a record written while a FastAPI endpoint runs, such as `exception.recorded`, carries the matched route template as `http.route` instead of `<unmatched>`.
diff --git a/docs/engineering/skills/observability.md b/docs/engineering/skills/observability.md
index 396175744..3f94edc41 100644
--- a/docs/engineering/skills/observability.md
+++ b/docs/engineering/skills/observability.md
@@ -130,11 +130,17 @@ record an operation or request that ends in `KeyboardInterrupt` with the
 outcome `interrupted` at WARNING and leave it out of the error count metric;
 a span that ends in it keeps an unset status and gets the attribute
 `policyengine.outcome` set to `interrupted`. The exception propagates
-unchanged, so a preempted attempt doesn't reach the daily telemetry error
-report. Let `KeyboardInterrupt` leave the
-operation block as it is: code that catches it and raises another exception
-instead reports the attempt as a failure at ERROR. `except Exception` doesn't
-catch it.
+unchanged, so the preempted call's own records don't reach the daily
+telemetry error report. Let `KeyboardInterrupt` leave the operation block as
+it is: code that catches it and raises another exception instead reports the
+attempt as a failure at ERROR. `except Exception` doesn't catch it.
+
+Modal ends a call it cancels with `InputCancellation` instead, which is a
+`BaseException` but not a `KeyboardInterrupt`, and policyengine-observability
+records that operation with the outcome `error` at ERROR. A preempted Stage 12
+segment runner cancels the segment calls it started, and a preempted Stage 12
+coordinator with a report run cancels its unfinished simulation calls, so each
+of those child calls that was running writes an ERROR record.
 
 ## Metric resource identity
 

From fda7271aa0c275e52ceef2d23f155e90ce7ffb40 Mon Sep 17 00:00:00 2001
From: Anthony Volk <14987227+anth-volk@users.noreply.github.com>
Date: Tue, 10 Nov 2026 01:50:48 +0400
Subject: [PATCH 5/5] Describe the second level of cancelled Stage 12 calls and
 async apps

A segmented US simulation that a preempted coordinator cancels cancels its
own started segments, and each running segment writes an ERROR record too;
the observability skill and the changelog fragment now say so. The skill
says Modal delivers a preemption as KeyboardInterrupt only to synchronous
functions: in the gateway's ASGI app it cancels the running task, so a
request ends in asyncio.CancelledError. The fragment drops the sentence
about requests that end in KeyboardInterrupt, which no preemption in these
services reaches, names the observability library among the packages that
moved, and says older versioned executor apps keep the earlier behaviour
until they are redeployed.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .../record-preemptions-as-interrupted.fixed.md    |  4 ++--
 docs/engineering/skills/observability.md          | 15 +++++++++++----
 2 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/changelog.d/record-preemptions-as-interrupted.fixed.md b/changelog.d/record-preemptions-as-interrupted.fixed.md
index 87c2b55ce..ff4a5d5c0 100644
--- a/changelog.d/record-preemptions-as-interrupted.fixed.md
+++ b/changelog.d/record-preemptions-as-interrupted.fixed.md
@@ -1,5 +1,5 @@
-A Modal preemption of a simulation worker, segment worker or Stage 12 function is now recorded as an interrupted operation at WARNING instead of a failed one at ERROR, so the preempted call no longer opens telemetry error issues. Modal delivers a preemption as `KeyboardInterrupt` and runs the same input again; the executor, the gateway and the entrypoint now require policyengine-observability 3.0.7, which records an operation or request that ends in `KeyboardInterrupt` with the outcome `interrupted` at WARNING and leaves it out of `policyengine.error.count`. The request and operation count and duration metrics get `interrupted` as a new value of their `outcome` attribute. An operation's span, or a span opened with `runtime.span`, that ends in `KeyboardInterrupt` is no longer set to error: its status stays unset and it gets the attribute `policyengine.outcome` set to `interrupted`. A gateway or entrypoint request that ends in `KeyboardInterrupt`, which 3.0.4 recorded from its status code alone (as `success` at INFO when no status had been sent), is now recorded as `interrupted` at WARNING too.
+A Modal preemption of a simulation worker, segment worker or Stage 12 function is now recorded as an interrupted operation at WARNING instead of a failed one at ERROR, so the preempted call no longer opens telemetry error issues. Modal delivers a preemption to these synchronous functions as `KeyboardInterrupt` and runs the same input again; the observability library, the executor, the gateway and the entrypoint now require policyengine-observability 3.0.7, which records an operation or request that ends in `KeyboardInterrupt` with the outcome `interrupted` at WARNING and leaves it out of `policyengine.error.count`. The request and operation count and duration metrics get `interrupted` as a new value of their `outcome` attribute. An operation's span, or a span opened with `runtime.span`, that ends in `KeyboardInterrupt` is no longer set to error: its status stays unset and it gets the attribute `policyengine.outcome` set to `interrupted`. This applies to the executor app this release deploys; executor apps deployed for older policyengine.py versions keep the earlier behaviour until they are redeployed.
 
-A child call that a preempted Stage 12 segment runner or report coordinator cancels still writes an ERROR record: Modal ends a cancelled call with `InputCancellation`, which is not a `KeyboardInterrupt`.
+A child call that a preempted Stage 12 segment runner or report coordinator cancels still writes an ERROR record, and so does each running segment of a segmented US simulation cancelled that way: Modal ends a cancelled call with `InputCancellation`, which is not a `KeyboardInterrupt`.
 
 The move from 3.0.4 also brings 3.0.5's change: a record written while a FastAPI endpoint runs, such as `exception.recorded`, carries the matched route template as `http.route` instead of `<unmatched>`.
diff --git a/docs/engineering/skills/observability.md b/docs/engineering/skills/observability.md
index 3f94edc41..77fbd9423 100644
--- a/docs/engineering/skills/observability.md
+++ b/docs/engineering/skills/observability.md
@@ -124,8 +124,12 @@ up over attempts, and `job_attempts` counts them.
 
 ### Preemptions
 
-Modal delivers a preemption to the running function as `KeyboardInterrupt`,
-then runs the same input again. policyengine-observability 3.0.7 and later
+Modal delivers a preemption to a running synchronous function, which every
+executor and Stage 12 Modal function is, as `KeyboardInterrupt`, then runs
+the same input again. In an async app, such as the gateway's ASGI app, Modal
+cancels the running task instead, so a request there ends in
+`asyncio.CancelledError` and what follows doesn't apply to it.
+policyengine-observability 3.0.7 and later
 record an operation or request that ends in `KeyboardInterrupt` with the
 outcome `interrupted` at WARNING and leave it out of the error count metric;
 a span that ends in it keeps an unset status and gets the attribute
@@ -139,8 +143,11 @@ Modal ends a call it cancels with `InputCancellation` instead, which is a
 `BaseException` but not a `KeyboardInterrupt`, and policyengine-observability
 records that operation with the outcome `error` at ERROR. A preempted Stage 12
 segment runner cancels the segment calls it started, and a preempted Stage 12
-coordinator with a report run cancels its unfinished simulation calls, so each
-of those child calls that was running writes an ERROR record.
+coordinator with a report run cancels its unfinished simulation calls; a
+segmented US simulation cancelled that way cancels its own started segment
+calls in turn. Each of those calls that was running writes an ERROR record,
+so one coordinator preemption can write one for each simulation and each of
+its running segments.
 
 ## Metric resource identity