From 060b8e194dba6b3dcd23f78a9147117407af5c48 Mon Sep 17 00:00:00 2001
From: Bai Li <bai.li@uipath.com>
Date: Tue, 28 Jul 2026 15:40:19 -0700
Subject: [PATCH 01/20] fix(reports): count errors as misses in one canonical
 pass rate

The reported success rate divided by `tasks_run - tasks_error`, so a run was
rewarded for erroring: whichever harness errored most got the biggest bonus.
Measured on real nightlies, the inflation was +10.0 points for a codex run
(116 errors of 947) and +7.1 for a sonnet-5 run, and one LiteLLM run rendered
as 100.0% while passing 7 of its 861 rows.

`RunSummary.pass_rate` is now `tasks_succeeded / tasks_run` with errors in the
denominator, published as a computed field so consumers read it instead of
deriving their own. Four surfaces across two repos were each deriving a
denominator, which is why the dashboard and the markdown report disagreed by up
to 10 points on the same run.json. An error is still not a failure, so the
reason survives as diagnostics on the row (`error_message`, `error_category`)
and as `error_share` on the run: a bad infrastructure night now shows as a bad
night instead of being absorbed into a flattering denominator.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 src/coder_eval/models/experiment.py           |  23 +-
 src/coder_eval/models/results.py              | 121 ++++++++-
 src/coder_eval/reports.py                     |  58 ++++-
 src/coder_eval/reports_experiment.py          | 111 +++++++-
 src/coder_eval/reports_html.py                |  10 +-
 .../report_snapshots/experiment_2variant.md   |   2 +-
 .../report_snapshots/experiment_3variant.md   |   2 +-
 .../report_snapshots/experiment_replicates.md |   2 +-
 tests/_fixtures/report_snapshots/run_full.md  |   2 +-
 .../_fixtures/report_snapshots/run_minimal.md |   2 +-
 tests/test_experiment_reports.py              |   4 +-
 tests/test_reports.py                         |   7 +-
 tests/test_run_metrics.py                     | 241 ++++++++++++++++++
 13 files changed, 552 insertions(+), 33 deletions(-)
 create mode 100644 tests/test_run_metrics.py

diff --git a/src/coder_eval/models/experiment.py b/src/coder_eval/models/experiment.py
index eefe638f..95a21694 100644
--- a/src/coder_eval/models/experiment.py
+++ b/src/coder_eval/models/experiment.py
@@ -6,7 +6,7 @@
 from pathlib import Path
 from typing import Any, Literal, Self
 
-from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
+from pydantic import BaseModel, ConfigDict, Field, computed_field, field_validator, model_validator
 
 from coder_eval.models.enums import FinalStatus
 from coder_eval.models.limits import RunLimits
@@ -194,7 +194,14 @@ class VariantResult(BaseModel):  # noqa: CE009 -- persisted result model; round-
 
 
 class VariantAggregate(BaseModel):  # noqa: CE009 -- persisted result model; round-trip leniency like models/results.py
-    """Aggregated statistics for a single variant across all tasks."""
+    """Aggregated statistics for a single variant across all tasks.
+
+    ``pass_rate`` uses the same denominator as ``RunSummary.pass_rate``: every task
+    the variant ran, errors included as misses. The variant tables previously
+    divided by ``tasks_run - tasks_error``, so an A/B whose variants errored at
+    different rates compared two different denominators and read the noisier
+    variant as the better one.
+    """
 
     variant_id: str
     tasks_run: int
@@ -228,6 +235,18 @@ def _check_task_count_invariant(self) -> VariantAggregate:
             raise ValueError(f"Task count invariant violated: {total} != {self.tasks_run}")
         return self
 
+    @computed_field  # type: ignore[prop-decorator]
+    @property
+    def pass_rate(self) -> float | None:
+        """``tasks_succeeded / tasks_run`` as a 0-1 fraction. ``None`` on an empty variant."""
+        return self.tasks_succeeded / self.tasks_run if self.tasks_run else None
+
+    @computed_field  # type: ignore[prop-decorator]
+    @property
+    def error_share(self) -> float | None:
+        """``tasks_error / tasks_run`` as a 0-1 fraction. ``None`` on an empty variant."""
+        return self.tasks_error / self.tasks_run if self.tasks_run else None
+
 
 class TaskExperimentSummary(BaseModel):  # noqa: CE009 -- persisted result model; round-trip leniency like models/results.py
     """Cross-variant summary for a single task."""
diff --git a/src/coder_eval/models/results.py b/src/coder_eval/models/results.py
index ef3336e7..61d7fb7b 100644
--- a/src/coder_eval/models/results.py
+++ b/src/coder_eval/models/results.py
@@ -6,7 +6,16 @@
 from enum import StrEnum
 from typing import Annotated, Any, Literal
 
-from pydantic import AliasChoices, BaseModel, ConfigDict, Discriminator, Field, Tag, model_validator
+from pydantic import (
+    AliasChoices,
+    BaseModel,
+    ConfigDict,
+    Discriminator,
+    Field,
+    Tag,
+    computed_field,
+    model_validator,
+)
 
 from coder_eval.models.agent_config import ResolvedAgentConfig
 from coder_eval.models.criteria import SuccessCriterion
@@ -838,7 +847,24 @@ class SkippedTask(BaseModel):
 
 
 class RunSummary(BaseModel):
-    """Summary of an entire evaluation run across multiple tasks."""
+    """Summary of an entire evaluation run across multiple tasks.
+
+    ``pass_rate`` is ``tasks_succeeded / tasks_run``: **every dispatched task is in
+    the denominator, errors included as misses.** An error is not a free pass. The
+    previous formula excluded errors, which paid a bonus for erroring — up to 10
+    points on a real nightly, and one run that rendered as 100% while passing 0.8%
+    of its rows. A denominator that shrinks when a run goes wrong is not a metric
+    you can set beside another run's. ``error_share`` says how much of the rate is
+    errors, so a bad infrastructure night *shows* instead of being absorbed.
+
+    This is the single denominator for the whole framework. Every reporting
+    surface reads ``pass_rate`` rather than re-deriving one: four surfaces across
+    two repos each computing their own is what made the dashboard and the markdown
+    report disagree by up to 10 points on the same run.json.
+
+    Every derived metric here is computed, never stored, so it cannot drift from
+    the counts it comes from.
+    """
 
     run_id: str = Field(description="Run identifier (timestamp like '2025-10-09_15-30-45')")
     start_time: datetime = Field(description="Run start time")
@@ -901,3 +927,94 @@ def _check_task_count_invariant(self) -> RunSummary:
             total = f"{self.tasks_succeeded} + {self.tasks_failed} + {self.tasks_error}"
             raise ValueError(f"Task count invariant violated: {total} != {self.tasks_run}")
         return self
+
+    # ------------------------------------------------------------------
+    # Derived run metrics.
+    #
+    # Every one is a computed_field over the stored counts and ``task_results``,
+    # so it serializes into run.json for downstream consumers (evalboard, the
+    # external eval-runner) while staying impossible to set to something the
+    # rows disagree with. Consumers should READ these rather than re-deriving
+    # them: four surfaces re-deriving two different denominators is what made
+    # the dashboard and the markdown report disagree by up to 10 points.
+    # ------------------------------------------------------------------
+
+    @computed_field  # type: ignore[prop-decorator]
+    @property
+    def pass_rate(self) -> float | None:
+        """``tasks_succeeded / tasks_run`` as a 0-1 fraction. ``None`` on an empty run.
+
+        The run's one pass rate. Errors are in the denominator as misses — see the
+        class docstring for why there is no second, error-excluding figure.
+        """
+        return self.tasks_succeeded / self.tasks_run if self.tasks_run else None
+
+    @computed_field  # type: ignore[prop-decorator]
+    @property
+    def error_share(self) -> float | None:
+        """``tasks_error / tasks_run`` as a 0-1 fraction. ``None`` on an empty run.
+
+        How much of ``pass_rate`` is being held down by rows that never produced a
+        gradeable attempt. Diagnostic: read it with the framework/harness split to
+        tell an infrastructure night from a genuinely weak model.
+        """
+        return self.tasks_error / self.tasks_run if self.tasks_run else None
+
+    @computed_field  # type: ignore[prop-decorator]
+    @property
+    def tasks_unpriced(self) -> int:
+        """Rows whose recorded spend is incomplete.
+
+        Each one is real money missing from ``agent_cost_usd``. Two ways in: the
+        row burned tokens and got no cost at all, or its per-row ``cost_complete``
+        says some turn within it went unpriced. Both come down to a model the rate
+        card cannot price — the card is a static table baked into the installed
+        version, so a model released after it prices as ``null`` on every turn,
+        with only a log line to say so.
+
+        A row that burned nothing does not count: an error before the agent ran
+        genuinely cost nothing.
+        """
+        return sum(
+            1
+            for row in self.task_results
+            if ((row.get("total_tokens") or 0) > 0 and row.get("total_cost_usd") is None)
+            or row.get("cost_complete") is False
+        )
+
+    @computed_field  # type: ignore[prop-decorator]
+    @property
+    def cost_complete(self) -> bool:
+        """False when any row's spend is incomplete, so every cost total here is a floor."""
+        return self.tasks_unpriced == 0
+
+    @computed_field  # type: ignore[prop-decorator]
+    @property
+    def agent_cost_usd(self) -> float | None:
+        """Subject-agent spend across the run. ``None`` when no row reported cost.
+
+        Excludes evaluation machinery — see ``eval_overhead_cost_usd``. Read
+        alongside ``cost_complete``: with unpriced rows present this is a floor,
+        not the bill.
+        """
+        costs = [c for row in self.task_results if (c := row.get("total_cost_usd")) is not None]
+        return sum(costs) if costs else None
+
+    @computed_field  # type: ignore[prop-decorator]
+    @property
+    def eval_overhead_cost_usd(self) -> float | None:
+        """Judge + simulator spend across the run. ``None`` when neither reported cost.
+
+        Deliberately NOT folded into ``agent_cost_usd``: comparing harnesses
+        means comparing what the agents cost, and the judge bill is a property of
+        the suite's criteria, identical across harnesses. But it is real money
+        and was previously reported nowhere, so a run's true bill is the two
+        added together.
+        """
+        costs = [
+            c
+            for row in self.task_results
+            for key in ("judge_cost_usd", "simulator_cost_usd")
+            if (c := row.get(key)) is not None
+        ]
+        return sum(costs) if costs else None
diff --git a/src/coder_eval/reports.py b/src/coder_eval/reports.py
index d5cce11f..d60e5be7 100644
--- a/src/coder_eval/reports.py
+++ b/src/coder_eval/reports.py
@@ -158,6 +158,30 @@ def _iteration_of(t: dict[str, Any]) -> int | None:
     return total, recovered, terminal
 
 
+def _fmt_rate(rate: float | None) -> str:
+    """Render a 0-1 rate as a percentage, or ``n/a`` when it is unknown."""
+    return f"{rate * 100:.1f}%" if rate is not None else "n/a"
+
+
+def _pass_rate_lines(summary: RunSummary) -> list[str]:
+    """The pass rate, plus how much of it is being held down by errors.
+
+    One rate over every dispatched task. The old line divided by
+    ``tasks_run - tasks_error``, which read like a pass rate but paid a bonus for
+    erroring: up to 10 points on a real nightly, and one run that rendered as
+    100.0% while passing 0.8% of its rows. When errors are material the error
+    share is printed beside the rate so a bad infrastructure night is visible
+    rather than absorbed.
+    """
+    lines = [f"- **Pass Rate**: {_fmt_rate(summary.pass_rate)} ({summary.tasks_succeeded}/{summary.tasks_run})"]
+    if summary.tasks_error:
+        lines.append(
+            f"- **Error Share**: {_fmt_rate(summary.error_share)} of tasks never produced a "
+            + "gradeable attempt and count as misses"
+        )
+    return lines
+
+
 class ReportGenerator:
     """Generates reports from evaluation results."""
 
@@ -286,9 +310,6 @@ def _summary_section_lines(summary: RunSummary) -> list[str]:
 
         Returns a ``## Header``-led block with no leading blank (caller prepends it).
         """
-        evaluable = summary.tasks_run - summary.tasks_error
-        success_rate = (summary.tasks_succeeded / evaluable * 100) if evaluable > 0 else 0
-
         failed_line = f"- **Failed**: {summary.tasks_failed}"
         if summary.tasks_token_budget_exceeded or summary.tasks_cost_budget_exceeded:
             failed_line += (
@@ -303,7 +324,7 @@ def _summary_section_lines(summary: RunSummary) -> list[str]:
             f"- **Succeeded**: {summary.tasks_succeeded}",
             failed_line,
             f"- **Errors**: {summary.tasks_error}",
-            f"- **Success Rate**: {success_rate:.1f}%",
+            *_pass_rate_lines(summary),
         ]
 
         # Aggregate P0 metrics
@@ -530,11 +551,38 @@ def _generate_token_usage_section(task_results: list[dict[str, Any]]) -> list[st
         costs = [t["total_cost_usd"] for t in tasks_with_tokens if t.get("total_cost_usd") is not None]
         total_cost = sum(costs) if costs else None
 
+        # Rows whose spend is only partly priced. Reported explicitly because the
+        # alternative is what used to happen: a run understating its bill by 19%
+        # with nothing on the report to say the number was incomplete.
+        unpriced = [
+            t
+            for t in task_results
+            if ((t.get("total_tokens") or 0) > 0 and t.get("total_cost_usd") is None) or t.get("cost_complete") is False
+        ]
+        overhead = [
+            c for t in task_results for key in ("judge_cost_usd", "simulator_cost_usd") if (c := t.get(key)) is not None
+        ]
+
         lines.append(f"**Total Tokens**: {total_tokens:,} (input: {total_input:,}, output: {total_output:,})")
         if total_cache_write > 0 or total_cache_read > 0:
             lines.append(f"**Cache Tokens**: write: {total_cache_write:,}, read: {total_cache_read:,}")
+        # Judge/simulator spend is broken out only when there is some: it is a
+        # property of the suite's criteria and identical across harnesses, so folding
+        # it into the agent bill would make two harnesses look closer than they are.
+        # **Total Cost** always means the whole bill.
+        if overhead and total_cost is not None:
+            lines.append(f"**Agent Cost**: ${total_cost:.4f}")
+            lines.append(f"**Eval Overhead (judge + simulator)**: ${sum(overhead):.4f}")
         if total_cost is not None:
-            lines.append(f"**Total Cost**: ${total_cost:.4f}")
+            cost_line = f"**Total Cost**: ${total_cost + sum(overhead):.4f}"
+            if unpriced:
+                cost_line += f" (floor — {len(unpriced)} task(s) burned tokens the rate card could not price)"
+            lines.append(cost_line)
+        elif unpriced:
+            lines.append(
+                f"**Total Cost**: unavailable — {len(unpriced)} task(s) burned tokens "
+                + "the rate card could not price"
+            )
         lines.append(f"**Avg Tokens/Task**: {total_tokens // len(tasks_with_tokens):,}")
         lines.append("")
 
diff --git a/src/coder_eval/reports_experiment.py b/src/coder_eval/reports_experiment.py
index cafe9e4c..6ff0f394 100644
--- a/src/coder_eval/reports_experiment.py
+++ b/src/coder_eval/reports_experiment.py
@@ -6,6 +6,7 @@
 from pathlib import Path
 from typing import Any
 
+from coder_eval.errors import truncate_crash_message
 from coder_eval.models import (
     EvaluationResult,
     ExperimentDefinition,
@@ -13,6 +14,7 @@
     TaskExperimentSummary,
 )
 from coder_eval.path_utils import replicate_subdir_name
+from coder_eval.pricing import calculate_cost
 from coder_eval.reports import resolve_agent_settings
 from coder_eval.reports_stats import (
     VariantSeries,
@@ -34,6 +36,78 @@
 # Default pass_threshold from BaseSuccessCriterion — used for Wilson pass-rate in replicate stats.
 _REPLICATE_PASS_THRESHOLD = 0.9
 
+# Cap on the ``error_message`` carried into each run.json row. Long enough to
+# identify a failure without a per-task artifact fetch (which is the whole point
+# — a rollup of 900 rows whose errors are only diagnosable one task.json at a
+# time is not diagnosable), short enough that a wholly-errored run doesn't bloat
+# run.json. The untruncated message stays on task.json.
+_ROW_ERROR_MESSAGE_MAX_CHARS = 400
+
+
+def _cost_complete(result: EvaluationResult) -> bool:
+    """Whether every turn that burned tokens on this row also carries a cost.
+
+    False means ``total_cost_usd`` is a floor, not the bill: some turn's spend is
+    missing from it. After the orchestrator's rate-card backfill the only way to
+    land here is a model the rate card cannot price at all, so this is the
+    per-row signal that feeds ``RunSummary.tasks_unpriced``.
+
+    True for a row that burned nothing (an error before the agent ran genuinely
+    cost nothing, and must not be reported as missing cost).
+    """
+    return all(
+        usage.total_cost_usd is not None
+        for t in result.iterations
+        if (usage := t.token_usage) is not None and not usage.is_empty()
+    )
+
+
+def _judge_cost_usd(result: EvaluationResult) -> float | None:
+    """Sum the priced judge spend across this row's criterion results.
+
+    Covers both judge flavors: ``llm_judge`` prices its own one-shot call from
+    the criterion's model, ``agent_judge`` inherits the SDK's cost on the
+    sub-agent's turn. ``None`` when no criterion reported cost, so "no judge ran"
+    stays distinguishable from "a judge ran free".
+    """
+    costs = [
+        usage.total_cost_usd
+        for cr in result.success_criteria_results
+        if (usage := getattr(cr, "token_usage", None)) is not None and usage.total_cost_usd is not None
+    ]
+    return sum(costs) if costs else None
+
+
+def _simulator_cost_usd(result: EvaluationResult) -> float | None:
+    """Price this row's simulator turns. ``None`` outside simulation mode.
+
+    Priced at the ROUTE's model, not the subject agent's. ``UserSimulator`` builds
+    its agent config with ``model=None`` on purpose, so the model resolves to the
+    route default (``BEDROCK_MODEL``) — which is a different model from the subject
+    whenever a task pins ``agent.model``, as the skills suite does throughout.
+    Pricing the simulator at the subject's model would mis-bill every such row.
+
+    Simulator telemetry buckets everything as prompt/completion tokens with no
+    cache split, so the whole prompt prices at the uncached input rate: an upper
+    bound on a dialog whose prefix was cached.
+    """
+    sim = result.simulation
+    if sim is None:
+        return None
+    if not (sim.simulator_input_tokens or sim.simulator_output_tokens):
+        return None
+    route_model = (result.environment_info or {}).get("bedrock_model")
+    # Falls back to the subject's model on a non-Bedrock route, where the SDK
+    # picks its own default and nothing on the record names it.
+    model = route_model if isinstance(route_model, str) and route_model else result.model_used
+    if not model:
+        return None
+    return calculate_cost(
+        model,
+        uncached_input_tokens=sim.simulator_input_tokens,
+        output_tokens=sim.simulator_output_tokens,
+    )
+
 
 # ---------------------------------------------------------------------------
 # Helper: build task_result dict from EvaluationResult (for variant reports)
@@ -124,6 +198,27 @@ def eval_result_to_task_dict(
         ),
         "total_tokens": (result.total_token_usage.total_tokens if result.total_token_usage else None),
         "total_cost_usd": (result.total_token_usage.total_cost_usd if result.total_token_usage else None),
+        # False when some turn's tokens are missing a price, so total_cost_usd above
+        # is a floor. Rolled up as RunSummary.tasks_unpriced / cost_complete.
+        "cost_complete": _cost_complete(result),
+        # Evaluation-machinery spend, kept strictly beside the agent bill rather
+        # than inside it: the judge cost is a property of the suite's criteria and
+        # is identical across harnesses, so folding it into total_cost_usd would
+        # make two harnesses look closer than they are. Rolled up run-level as
+        # RunSummary.eval_overhead_cost_usd.
+        "judge_cost_usd": _judge_cost_usd(result),
+        "simulator_cost_usd": _simulator_cost_usd(result),
+        # Why this row errored, on the row. Errors now count as misses, so the
+        # rollup needs to say *why* it lost those points: without these, an errored
+        # run is untriageable without fetching per-task artifacts one at a time
+        # (the 109 zero-iteration codex errors of 2026-07-22 could not be
+        # characterised from run.json at all).
+        "error_message": (
+            truncate_crash_message(result.error_message, limit=_ROW_ERROR_MESSAGE_MAX_CHARS)
+            if result.error_message
+            else None
+        ),
+        "error_category": (result.error_details or {}).get("error_category"),
         "expected_commands": result.expected_commands,
         "actual_commands": result.actual_commands,
         "commands_efficiency": result.commands_efficiency,
@@ -276,13 +371,12 @@ def _aggregate_count_rows(result: ExperimentResult, show_p_values: bool) -> list
             row += " | —"
         lines.append(row + " |")
 
-        # Row: Success Rate (errors excluded from denominator — they're infrastructure failures, not task failures)
-        row = "| Success Rate"
+        # Row: Pass Rate. Every task the variant ran is in the denominator, errors
+        # included — see VariantAggregate.pass_rate.
+        row = "| Pass Rate"
         for vid in result.variant_ids:
-            agg = result.variant_aggregates[vid]
-            evaluable = agg.tasks_run - agg.tasks_error
-            rate = (agg.tasks_succeeded / evaluable * 100) if evaluable > 0 else 0
-            row += f" | {rate:.1f}%"
+            rate = result.variant_aggregates[vid].pass_rate
+            row += f" | {rate * 100:.1f}%" if rate is not None else " | n/a"
         if show_p_values:
             row += " | —"
         lines.append(row + " |")
@@ -539,8 +633,7 @@ def generate_variant_report(variant_id: str, result: ExperimentResult, run_dir:
         from coder_eval.reports import ReportGenerator
 
         agg = result.variant_aggregates[variant_id]
-        evaluable = agg.tasks_run - agg.tasks_error
-        success_rate = (agg.tasks_succeeded / evaluable * 100) if evaluable > 0 else 0
+        pass_rate_str = f"{agg.pass_rate * 100:.1f}%" if agg.pass_rate is not None else "n/a"
         tokens_str = f"{agg.total_tokens:,}" if agg.total_tokens is not None else "N/A"
 
         failed_line = f"- **Failed**: {agg.tasks_failed}"
@@ -562,7 +655,7 @@ def generate_variant_report(variant_id: str, result: ExperimentResult, run_dir:
             f"- **Succeeded**: {agg.tasks_succeeded}",
             failed_line,
             f"- **Errors**: {agg.tasks_error}",
-            f"- **Success Rate**: {success_rate:.1f}%",
+            f"- **Pass Rate**: {pass_rate_str} ({agg.tasks_succeeded}/{agg.tasks_run})",
             f"- **Average Score**: {agg.average_score:.3f}",
             f"- **Average Duration**: {agg.average_duration:.1f}s",
             f"- **Total Tokens**: {tokens_str}",
diff --git a/src/coder_eval/reports_html.py b/src/coder_eval/reports_html.py
index a6f74991..d9102dc5 100644
--- a/src/coder_eval/reports_html.py
+++ b/src/coder_eval/reports_html.py
@@ -1320,13 +1320,11 @@ def _row(label: str, values: list[str], p: str | None) -> str:
     rows.append(_row("Failed", [str(result.variant_aggregates[vid].tasks_failed) for vid in result.variant_ids], None))
     rows.append(_row("Errors", [str(result.variant_aggregates[vid].tasks_error) for vid in result.variant_ids], None))
 
-    def _success_rate(vid: str) -> str:
-        agg = result.variant_aggregates[vid]
-        evaluable = agg.tasks_run - agg.tasks_error
-        rate = (agg.tasks_succeeded / evaluable * 100) if evaluable > 0 else 0.0
-        return f"{rate:.1f}%"
+    def _pass_rate(vid: str) -> str:
+        rate = result.variant_aggregates[vid].pass_rate
+        return f"{rate * 100:.1f}%" if rate is not None else "n/a"
 
-    rows.append(_row("Success Rate", [_success_rate(vid) for vid in result.variant_ids], None))
+    rows.append(_row("Pass Rate", [_pass_rate(vid) for vid in result.variant_ids], None))
 
     rows.append(
         _row(
diff --git a/tests/_fixtures/report_snapshots/experiment_2variant.md b/tests/_fixtures/report_snapshots/experiment_2variant.md
index e19649fd..2cf11a74 100644
--- a/tests/_fixtures/report_snapshots/experiment_2variant.md
+++ b/tests/_fixtures/report_snapshots/experiment_2variant.md
@@ -19,7 +19,7 @@
 | - Token budget | 1 | 0 | — |
 | - Cost budget | 0 | 1 | — |
 | Errors | 0 | 0 | — |
-| Success Rate | 50.0% | 50.0% | — |
+| Pass Rate | 50.0% | 50.0% | — |
 | Score | 0.700 ± 0.283 | 0.825 ± 0.177 | 0.658 |
 | Avg Duration (s) | 35.0 ± 7.1 | 55.0 ± 7.1 | 0.106 |
 | Assistant Turns | 5.5 ± 0.7 | 7.5 ± 0.7 | 0.106 |
diff --git a/tests/_fixtures/report_snapshots/experiment_3variant.md b/tests/_fixtures/report_snapshots/experiment_3variant.md
index ff93b009..40ede665 100644
--- a/tests/_fixtures/report_snapshots/experiment_3variant.md
+++ b/tests/_fixtures/report_snapshots/experiment_3variant.md
@@ -12,7 +12,7 @@
 | Succeeded | 1 | 1 | 1 |
 | Failed | 1 | 1 | 1 |
 | Errors | 0 | 0 | 0 |
-| Success Rate | 50.0% | 50.0% | 50.0% |
+| Pass Rate | 50.0% | 50.0% | 50.0% |
 | Score | 0.650 ± 0.354 | 0.775 ± 0.247 | 0.650 ± 0.212 |
 | Avg Duration (s) | 21.0 ± 1.4 | 26.5 ± 2.1 | 31.5 ± 2.1 |
 | Assistant Turns | 3.5 ± 0.7 | 6.0 ± 1.4 | 5.5 ± 0.7 |
diff --git a/tests/_fixtures/report_snapshots/experiment_replicates.md b/tests/_fixtures/report_snapshots/experiment_replicates.md
index dfcd8988..73333d07 100644
--- a/tests/_fixtures/report_snapshots/experiment_replicates.md
+++ b/tests/_fixtures/report_snapshots/experiment_replicates.md
@@ -12,7 +12,7 @@
 | Succeeded | 1 | 1 | — |
 | Failed | 0 | 0 | — |
 | Errors | 0 | 0 | — |
-| Success Rate | 100.0% | 100.0% | — |
+| Pass Rate | 100.0% | 100.0% | — |
 | Score | 0.900 | 0.650 | — |
 | Avg Duration (s) | 3.3 | 3.3 | — |
 | Tokens | 1,000 | 1,000 | — |
diff --git a/tests/_fixtures/report_snapshots/run_full.md b/tests/_fixtures/report_snapshots/run_full.md
index 1b9dc828..3e14b5ec 100644
--- a/tests/_fixtures/report_snapshots/run_full.md
+++ b/tests/_fixtures/report_snapshots/run_full.md
@@ -11,7 +11,7 @@
 - **Succeeded**: 1
 - **Failed**: 1 (incl. 1 token budget, 0 cost budget exceeded)
 - **Errors**: 0
-- **Success Rate**: 50.0%
+- **Pass Rate**: 50.0% (1/2)
 - **Avg Reliability Score**: 0.625
 - **Avg Generation Latency**: 10.2s
 - **Total Assistant Turns**: 4
diff --git a/tests/_fixtures/report_snapshots/run_minimal.md b/tests/_fixtures/report_snapshots/run_minimal.md
index 87969535..be0e8308 100644
--- a/tests/_fixtures/report_snapshots/run_minimal.md
+++ b/tests/_fixtures/report_snapshots/run_minimal.md
@@ -10,7 +10,7 @@
 - **Succeeded**: 0
 - **Failed**: 0
 - **Errors**: 0
-- **Success Rate**: 0.0%
+- **Pass Rate**: n/a (0/0)
 
 ## Task Details
 
diff --git a/tests/test_experiment_reports.py b/tests/test_experiment_reports.py
index de432278..3e212da5 100644
--- a/tests/test_experiment_reports.py
+++ b/tests/test_experiment_reports.py
@@ -529,7 +529,7 @@ def test_variant_report_content(self, tmp_path, sample_result):
         # New: variant report should include experiment context and summary
         assert "**Experiment**: model-comparison" in content
         assert "## Summary" in content
-        assert "Success Rate" in content
+        assert "Pass Rate" in content
 
     def test_variant_html_task_links_include_replicate_segment(self, tmp_path, sample_result):
         """variant.html task links must include the /00/ replicate segment so they
@@ -641,7 +641,7 @@ def test_experiment_report_has_aggregate_metrics(self):
         assert "| Avg Duration (s) |" in md
         assert "| Tokens |" in md
         assert "| Tasks Run |" in md
-        assert "| Success Rate |" in md
+        assert "| Pass Rate |" in md
 
         # Two-variant comparison should show p-value column
         assert "p-value" in md
diff --git a/tests/test_reports.py b/tests/test_reports.py
index 8b741ddb..756a345f 100644
--- a/tests/test_reports.py
+++ b/tests/test_reports.py
@@ -212,7 +212,9 @@ def test_generate_markdown_basic():
     assert "Succeeded**: 2" in report_md
     assert "Failed**: 1" in report_md
     assert "Errors**: 0" in report_md
-    assert "Success Rate**: 66.7%" in report_md
+    assert "Pass Rate**: 66.7% (2/3)" in report_md
+    # No errors in this run, so no error-share line to explain the denominator.
+    assert "Error Share" not in report_md
 
     # Check P0 aggregate metrics
     assert "Avg Reliability Score**:" in report_md
@@ -259,7 +261,8 @@ def test_generate_markdown_empty_tasks():
     report_md = ReportGenerator.generate_markdown(summary)
 
     assert "Total Tasks**: 0" in report_md
-    assert "Success Rate**: 0.0%" in report_md
+    # A run with no tasks has no pass rate; 0.0% would read as "everything failed".
+    assert "Pass Rate**: n/a (0/0)" in report_md
     # No aggregate metrics when there are no tasks
     assert "Avg Reliability Score" not in report_md
     assert "Generation Metrics" not in report_md
diff --git a/tests/test_run_metrics.py b/tests/test_run_metrics.py
new file mode 100644
index 00000000..b7d00d92
--- /dev/null
+++ b/tests/test_run_metrics.py
@@ -0,0 +1,241 @@
+"""Run-level derived metrics: one pass-rate denominator, and honest cost totals.
+
+Two bugs are pinned here.
+
+**The denominator.** ``pass_rate`` used to be ``succeeded / (run - error)``, which
+paid a bonus for erroring. Measured on real nightlies: +10.0 points for a codex run
+with 116 errors of 947, +7.1 for a sonnet-5 run, and a LiteLLM run that rendered as
+**100.0%** while passing 7 of 861 rows. Every surface now divides by ``tasks_run``.
+
+**The bill.** Cost was summed over whatever rows happened to carry one, so a run
+whose model was missing from the rate card, or whose turns were killed before the
+SDK reported a cost, understated its spend silently. One nightly hid $209.81
+(18.9%) that way. Unpriced spend is now counted and the total is labelled a floor.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+import pytest
+import yaml
+
+from coder_eval.models import FinalStatus, RunSummary
+from coder_eval.pricing import is_priced, unpriced_models
+
+
+def _row(status: FinalStatus | str, **extra: Any) -> dict[str, Any]:
+    return {"task_id": f"t{id(extra)}", "status": status, **extra}
+
+
+def _summary(rows: list[dict[str, Any]]) -> RunSummary:
+    """Build a RunSummary whose stored counts are derived from ``rows``."""
+    statuses = [FinalStatus(r["status"]) for r in rows]
+    return RunSummary(
+        run_id="2026-07-28_00-00-00",
+        start_time=datetime(2026, 7, 28, 0, 0, 0),
+        end_time=datetime(2026, 7, 28, 1, 0, 0),
+        total_duration_seconds=3600.0,
+        tasks_run=len(rows),
+        tasks_succeeded=sum(1 for s in statuses if s.category == "succeeded"),
+        tasks_failed=sum(1 for s in statuses if s.category == "failed"),
+        tasks_error=sum(1 for s in statuses if s.category == "error"),
+        task_results=rows,
+        framework_version="0.0.0-test",
+    )
+
+
+class TestPassRateDenominator:
+    def test_errors_count_as_misses(self):
+        """The whole point: an error is in the denominator, not excluded from it."""
+        summary = _summary(
+            [
+                _row(FinalStatus.SUCCESS),
+                _row(FinalStatus.FAILURE),
+                _row(FinalStatus.ERROR),
+                _row(FinalStatus.ERROR),
+            ]
+        )
+        assert summary.pass_rate == pytest.approx(0.25)
+        # The old formula would have divided by 2 and published 50%.
+        assert summary.error_share == pytest.approx(0.5)
+
+    def test_build_failed_is_an_error_row(self):
+        """BUILD_FAILED buckets to 'error' and so still lands in the denominator."""
+        summary = _summary([_row(FinalStatus.SUCCESS), _row(FinalStatus.BUILD_FAILED)])
+        assert summary.tasks_error == 1
+        assert summary.pass_rate == pytest.approx(0.5)
+
+    def test_timeout_and_budget_statuses_are_failures_not_errors(self):
+        """These are task outcomes; they were always in the denominator and stay there."""
+        summary = _summary(
+            [
+                _row(FinalStatus.SUCCESS),
+                _row(FinalStatus.TIMEOUT),
+                _row(FinalStatus.MAX_TURNS_EXHAUSTED),
+                _row(FinalStatus.TOKEN_BUDGET_EXCEEDED),
+                _row(FinalStatus.COST_BUDGET_EXCEEDED),
+            ]
+        )
+        assert summary.tasks_error == 0
+        assert summary.error_share == pytest.approx(0.0)
+        assert summary.pass_rate == pytest.approx(0.2)
+
+    def test_mostly_errored_run_cannot_render_as_perfect(self):
+        """The reductio from run adhoc-2026-07-16_18-12-48: 854 errors of 861 rows.
+
+        Reported 100.0% under the old formula (7 evaluable, 7 passed). Must now
+        read as what it was.
+        """
+        rows = [_row(FinalStatus.SUCCESS) for _ in range(7)] + [_row(FinalStatus.ERROR) for _ in range(854)]
+        summary = _summary(rows)
+        assert summary.pass_rate == pytest.approx(7 / 861)
+        assert summary.pass_rate < 0.01
+
+    def test_empty_run_has_no_rate(self):
+        """0/0 is unknown, not 0% — which would read as 'everything failed'."""
+        summary = _summary([])
+        assert summary.pass_rate is None
+        assert summary.error_share is None
+
+    def test_variant_aggregate_shares_the_denominator(self):
+        """An A/B whose variants error at different rates must compare like with like."""
+        from coder_eval.models import VariantAggregate
+
+        agg = VariantAggregate(
+            variant_id="v",
+            tasks_run=4,
+            tasks_succeeded=1,
+            tasks_failed=1,
+            tasks_error=2,
+            average_score=0.5,
+            average_duration=1.0,
+        )
+        assert agg.pass_rate == pytest.approx(0.25)
+        assert agg.error_share == pytest.approx(0.5)
+
+    def test_rates_serialize_into_run_json(self):
+        """Downstream consumers must be able to READ the rate instead of re-deriving it.
+
+        Four surfaces across two repos each deriving their own denominator is what
+        made the dashboard and the markdown report disagree by up to 10 points.
+        """
+        summary = _summary([_row(FinalStatus.SUCCESS), _row(FinalStatus.ERROR)])
+        dumped = summary.model_dump()
+        assert dumped["pass_rate"] == pytest.approx(0.5)
+        assert dumped["error_share"] == pytest.approx(0.5)
+        # And the round-trip tolerates its own output (computed fields aren't inputs).
+        assert RunSummary.model_validate_json(summary.model_dump_json()).pass_rate == pytest.approx(0.5)
+
+
+class TestCostCompleteness:
+    def test_unpriced_row_is_counted_and_flagged(self):
+        summary = _summary(
+            [
+                _row(FinalStatus.SUCCESS, total_tokens=1000, total_cost_usd=0.5, cost_complete=True),
+                _row(FinalStatus.TIMEOUT, total_tokens=8_000_000, total_cost_usd=None, cost_complete=False),
+            ]
+        )
+        assert summary.tasks_unpriced == 1
+        assert summary.cost_complete is False
+        # The priced row's cost still totals — a floor beats no number at all.
+        assert summary.agent_cost_usd == pytest.approx(0.5)
+
+    def test_partially_priced_row_is_counted(self):
+        """A row whose cost is a partial sum of its own turns is incomplete.
+
+        This is the timeout shape: turn 1 priced by the SDK, turn 2 killed
+        mid-flight with real tokens and no cost. Summing turn 1 alone and calling
+        it the row's cost is the silent 19% understatement.
+        """
+        summary = _summary(
+            [_row(FinalStatus.TIMEOUT, total_tokens=5_000_000, total_cost_usd=1.25, cost_complete=False)]
+        )
+        assert summary.tasks_unpriced == 1
+        assert summary.cost_complete is False
+
+    def test_zero_token_error_is_not_unpriced(self):
+        """A row that died before the agent ran genuinely cost nothing."""
+        summary = _summary([_row(FinalStatus.ERROR, total_tokens=0, total_cost_usd=None, cost_complete=True)])
+        assert summary.tasks_unpriced == 0
+        assert summary.cost_complete is True
+        assert summary.agent_cost_usd is None
+
+    def test_eval_overhead_is_separate_from_the_agent_bill(self):
+        """Judge/simulator spend is real money but must not inflate the agent's cost.
+
+        It is a property of the suite's criteria and identical across harnesses, so
+        folding it in would make two harnesses look closer than they are.
+        """
+        summary = _summary(
+            [
+                _row(
+                    FinalStatus.SUCCESS,
+                    total_tokens=1000,
+                    total_cost_usd=1.0,
+                    cost_complete=True,
+                    judge_cost_usd=0.2,
+                    simulator_cost_usd=0.05,
+                ),
+            ]
+        )
+        assert summary.agent_cost_usd == pytest.approx(1.0)
+        assert summary.eval_overhead_cost_usd == pytest.approx(0.25)
+
+    def test_no_overhead_reports_none(self):
+        summary = _summary([_row(FinalStatus.SUCCESS, total_tokens=1, total_cost_usd=0.1, cost_complete=True)])
+        assert summary.eval_overhead_cost_usd is None
+
+
+class TestPricingCoverage:
+    def test_is_priced_normalizes_bedrock_prefixes(self):
+        assert is_priced("claude-sonnet-5")
+        assert is_priced("eu.anthropic.claude-sonnet-5")
+        assert not is_priced("claude-sonnet-99")
+
+    def test_unpriced_models_dedupes_and_drops_empty(self):
+        assert unpriced_models(["claude-sonnet-5", None, "", "made-up-1", "made-up-1", "made-up-0"]) == [
+            "made-up-0",
+            "made-up-1",
+        ]
+
+
+class TestRateCardCoversCheckedInExperiments:
+    """CI guard: a model referenced by a committed experiment must be priced.
+
+    The 2026-07-21 nightly recorded $209.81 as null because Sonnet 5's rates landed
+    one release after the run. The rate card is a static table, so this class is what
+    makes the next new model fail CI instead of a nightly's cost column.
+    """
+
+    @staticmethod
+    def _models_in(doc: dict[str, Any]) -> set[str]:
+        found: set[str] = set()
+        for block in (doc.get("defaults") or {}, *(doc.get("variants") or [])):
+            if not isinstance(block, dict):
+                continue
+            agent = block.get("agent")
+            if isinstance(agent, dict) and agent.get("model"):
+                found.add(str(agent["model"]))
+        return found
+
+    def test_every_experiment_model_is_priced(self):
+        experiments_dir = Path(__file__).resolve().parents[1] / "experiments"
+        referenced: dict[str, str] = {}
+        for path in sorted(experiments_dir.glob("*.yaml")):
+            doc = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
+            for model in self._models_in(doc):
+                referenced.setdefault(model, path.name)
+
+        # Guard the guard: an empty scan would pass vacuously.
+        assert referenced, f"No agent.model found under {experiments_dir}; the discovery filter is broken."
+
+        missing = {m: src for m, src in referenced.items() if not is_priced(m)}
+        assert not missing, (
+            "Models referenced by committed experiments have no pricing rate: "
+            + ", ".join(f"{m!r} (from {src})" for m, src in sorted(missing.items()))
+            + ". Add the rate to coder_eval.pricing._PRICING — an unpriced model records "
+            + "null cost for every task and understates the run's bill silently."
+        )

From e45e6de4617e2f502ac05ed91cb728401184f1cc Mon Sep 17 00:00:00 2001
From: Bai Li <bai.li@uipath.com>
Date: Tue, 28 Jul 2026 15:40:32 -0700
Subject: [PATCH 02/20] fix(cost): book spend on the error and timeout paths,
 flag what is unpriced

Cost went missing three ways, each silently.

A turn killed mid-flight carries real billed tokens and no SDK-reported cost,
and those partials were summed as free. The orchestrator now prices any turn
that burned tokens without a cost from the rate card, leaving an SDK-reported
cost untouched as the authoritative figure.

The rate card is a static table baked into the installed version, so a model
released after it prices every turn as null. That understated one nightly by
$209.81 across 62 rows, 18.9% of its true bill, with one log line to show for
it. Runs now pre-flight their models against the card and warn (or refuse under
--strict-pricing), report `tasks_unpriced` / `cost_complete`, and label a total
built from partly-priced rows as the floor it is. A committed experiment whose
model has no rate now fails CI rather than a nightly's cost column.

Judge and simulator spend was captured per criterion and rolled up nowhere.
Both are now priced and reported as `eval_overhead_cost_usd`, deliberately
beside the agent bill rather than inside it: judge cost is a property of the
suite's criteria and identical across harnesses, so folding it in would make
two harnesses look closer than they are.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 src/coder_eval/cli/run_command.py        |   8 +
 src/coder_eval/criteria/llm_judge.py     |   4 +-
 src/coder_eval/evaluation/judge_usage.py |  21 +-
 src/coder_eval/orchestration/batch.py    |  42 ++++
 src/coder_eval/orchestration/config.py   |  10 +
 src/coder_eval/orchestrator.py           |  46 +++-
 src/coder_eval/pricing.py                |  21 ++
 tests/test_cost_accounting_paths.py      | 295 +++++++++++++++++++++++
 tests/test_route_seam_exhaustiveness.py  |   2 +-
 tests/test_token_usage.py                |  54 ++++-
 10 files changed, 494 insertions(+), 9 deletions(-)
 create mode 100644 tests/test_cost_accounting_paths.py

diff --git a/src/coder_eval/cli/run_command.py b/src/coder_eval/cli/run_command.py
index 9b22929c..19e36c24 100644
--- a/src/coder_eval/cli/run_command.py
+++ b/src/coder_eval/cli/run_command.py
@@ -211,6 +211,11 @@ def run_command(
         "-v",
         help="Enable verbose (DEBUG level) logging",
     ),
+    strict_pricing: bool = typer.Option(
+        False,
+        "--strict-pricing",
+        help="Refuse to start if a task's model has no pricing rate (default: warn and run)",
+    ),
     log_file: Path | None = typer.Option(  # noqa: B008
         None,
         "--log-file",
@@ -414,6 +419,7 @@ def run_command(
                 resume=resume,
                 include_skipped=include_skipped,
                 junit_xml=junit_xml,
+                strict_pricing=strict_pricing,
             )
         )
     except KeyboardInterrupt:
@@ -439,6 +445,7 @@ async def _run_all_tasks(
     resume: bool = False,
     include_skipped: bool = False,
     junit_xml: Path | None = None,
+    strict_pricing: bool = False,
 ) -> None:
     """Async entry point for running all tasks (optionally in parallel).
 
@@ -484,6 +491,7 @@ async def _run_all_tasks(
         repeats=repeats,
         verbose=verbose,
         include_skipped=include_skipped,
+        strict_pricing=strict_pricing,
     )
 
     from ..telemetry import flush_telemetry, track_event
diff --git a/src/coder_eval/criteria/llm_judge.py b/src/coder_eval/criteria/llm_judge.py
index 64d0abfa..13b660d3 100644
--- a/src/coder_eval/criteria/llm_judge.py
+++ b/src/coder_eval/criteria/llm_judge.py
@@ -192,7 +192,7 @@ def _invoke_tool_channel(
                 tool_spec=SUBMIT_VERDICT_ANTHROPIC_TOOL,
             )
             verdict, err = extract_verdict_from_anthropic_response(response)
-            response_usage = token_usage_from_anthropic_dict(response)
+            response_usage = token_usage_from_anthropic_dict(response, model=criterion.model)
         case DirectRoute():
             anthropic_response = invoke_anthropic_judge(
                 model=criterion.model,
@@ -203,7 +203,7 @@ def _invoke_tool_channel(
                 tool_spec=SUBMIT_VERDICT_ANTHROPIC_TOOL,
             )
             verdict, err = extract_verdict_from_anthropic_response(anthropic_response)
-            response_usage = token_usage_from_anthropic_dict(anthropic_response)
+            response_usage = token_usage_from_anthropic_dict(anthropic_response, model=criterion.model)
         case LiteLLMRoute():
             # Defensive: the evaluation route is pinned to Bedrock/Direct by
             # resolve_evaluation_route, so a LiteLLM route should never reach the
diff --git a/src/coder_eval/evaluation/judge_usage.py b/src/coder_eval/evaluation/judge_usage.py
index 312a889a..284d38ec 100644
--- a/src/coder_eval/evaluation/judge_usage.py
+++ b/src/coder_eval/evaluation/judge_usage.py
@@ -15,6 +15,7 @@
 from typing import Any
 
 from coder_eval.models import TokenUsage
+from coder_eval.pricing import calculate_cost
 
 
 def _coerce_int(value: Any) -> int:
@@ -31,12 +32,18 @@ def _coerce_int(value: Any) -> int:
         return 0
 
 
-def token_usage_from_anthropic_dict(resp: dict[str, Any]) -> TokenUsage | None:
+def token_usage_from_anthropic_dict(resp: dict[str, Any], *, model: str | None = None) -> TokenUsage | None:
     """Extract usage from an Anthropic / Bedrock-invoke Messages response dict.
 
     Both ``invoke_anthropic_judge`` (``response.model_dump()``) and
     ``invoke_bedrock_judge`` (parsed ``/invoke`` JSON) carry an Anthropic-shaped
     ``usage`` block. Returns ``None`` when usage is missing or carries no tokens.
+
+    ``model`` prices the call from the rate card. Neither judge backend returns a
+    cost, so without it the judge's spend is invisible in every rollup — a suite
+    with ~100 ``llm_judge`` rows books real Bedrock tokens against no dollar
+    figure anywhere. Left unpriced (``total_cost_usd=None``) when the model is
+    absent from the rate card, which the run-level unpriced count then surfaces.
     """
     u = resp.get("usage")
     if not isinstance(u, dict):
@@ -47,4 +54,14 @@ def token_usage_from_anthropic_dict(resp: dict[str, Any]) -> TokenUsage | None:
         cache_creation_input_tokens=_coerce_int(u.get("cache_creation_input_tokens")),
         cache_read_input_tokens=_coerce_int(u.get("cache_read_input_tokens")),
     )
-    return None if tu.is_empty() else tu
+    if tu.is_empty():
+        return None
+    if model:
+        tu.total_cost_usd = calculate_cost(
+            model,
+            uncached_input_tokens=tu.uncached_input_tokens,
+            output_tokens=tu.output_tokens,
+            cache_creation_tokens=tu.cache_creation_input_tokens,
+            cache_read_tokens=tu.cache_read_input_tokens,
+        )
+    return tu
diff --git a/src/coder_eval/orchestration/batch.py b/src/coder_eval/orchestration/batch.py
index d4c218f5..639a4972 100644
--- a/src/coder_eval/orchestration/batch.py
+++ b/src/coder_eval/orchestration/batch.py
@@ -29,6 +29,7 @@
     TaskResult,
 )
 from ..path_utils import format_task_log_id
+from ..pricing import unpriced_models
 from ..reports_experiment import eval_result_to_task_dict
 from ..streaming.callbacks import StreamCallback
 from ..utils import get_version_info, looks_like_version
@@ -38,6 +39,45 @@
 logger = logging.getLogger(__name__)
 
 
+def check_pricing_coverage(resolved_tasks: list[ResolvedTask], *, strict: bool = False) -> list[str]:
+    """Pre-flight the rate card against the models this run will use.
+
+    The rate card is a static table baked into the installed framework version,
+    so a model released after that version prices as ``None`` on every turn — the
+    run completes, records full token counts, and reports a cost that is silently
+    low. (One nightly under-reported $209.81, 18.9% of its bill, because the
+    model's rates landed in the next release.) This turns that into a warning at
+    dispatch, or a refusal under ``strict``.
+
+    Only pinned ``agent.model`` values are visible here; a task that defers its
+    model to the route resolves it inside the agent and can't be pre-flighted.
+    Those still get counted after the fact by ``RunSummary.tasks_unpriced``.
+
+    Args:
+        resolved_tasks: The fully-resolved tasks about to run.
+        strict: Raise instead of warning when any model is unpriced.
+
+    Returns:
+        The sorted, de-duplicated unpriced model ids (empty when all are priced).
+
+    Raises:
+        ValueError: Under ``strict`` when at least one model is unpriced.
+    """
+    missing = unpriced_models(rt.task.agent.model if rt.task.agent else None for rt in resolved_tasks)
+    if not missing:
+        return []
+    detail = (
+        f"No pricing rate for {', '.join(repr(m) for m in missing)}. Cost for these tasks "
+        + "will be recorded as null and every run-level total will understate the bill "
+        + "(RunSummary.cost_complete reports false). Add the rate to coder_eval.pricing "
+        + "or register it from a plugin via register_pricing()."
+    )
+    if strict:
+        raise ValueError(f"strict_pricing: refusing to start. {detail}")
+    logger.warning(detail)
+    return missing
+
+
 async def run_batch(
     resolved_tasks: list[ResolvedTask],
     config: BatchRunConfig,
@@ -75,6 +115,8 @@ async def run_batch(
 
     start_time = datetime.now()
 
+    check_pricing_coverage(resolved_tasks, strict=config.strict_pricing)
+
     if on_batch_start is not None:
         on_batch_start(len(resolved_tasks))
 
diff --git a/src/coder_eval/orchestration/config.py b/src/coder_eval/orchestration/config.py
index 695b5594..19c0e378 100644
--- a/src/coder_eval/orchestration/config.py
+++ b/src/coder_eval/orchestration/config.py
@@ -97,6 +97,16 @@ class BatchRunConfig(BaseModel):
     # Logging
     verbose: bool = Field(default=False, description="Enable verbose (DEBUG level) logging for Docker output")
 
+    strict_pricing: bool = Field(
+        default=False,
+        description=(
+            "Refuse to start when a task's model is absent from the pricing rate card, "
+            "instead of warning and running with that model's spend recorded as null. "
+            "Off by default so a brand-new model is still evaluable; on for cost-bearing "
+            "runs (the nightly), where an unpriced model silently understates the bill."
+        ),
+    )
+
     # TODO(container-death-diagnostics): consider a run-level default resource
     # cap. Containers run uncapped today (sandbox.limits.{max_memory_mb,
     # max_cpus,max_pids} default to None -> _build_argv emits no --memory/
diff --git a/src/coder_eval/orchestrator.py b/src/coder_eval/orchestrator.py
index f5a5206e..de71b3a0 100644
--- a/src/coder_eval/orchestrator.py
+++ b/src/coder_eval/orchestrator.py
@@ -56,6 +56,7 @@
 from .orchestration.early_stop import EarlyStopWatcher, validate_early_stop
 from .orchestration.evaluation import load_reference
 from .path_utils import format_task_log_id, task_log_path
+from .pricing import calculate_cost
 from .sandbox import Sandbox
 from .simulation import DialogStopReason, SimulatorResult, UserSimulator, evaluate_stop
 from .streaming.callbacks import CompositeStreamCallback, StreamCallback, TaskScopedCallback, safe_emit
@@ -660,17 +661,20 @@ def _finalize_result(self, start_time: float) -> None:
         if not self.result.model_used and self.task.agent is not None and self.task.agent.model:
             self.result.model_used = self.task.agent.model
 
-        # Aggregate token usage
-        self._aggregate_token_usage()
-
         # Record whether per-turn cost data was available when a cost budget was set.
         # Lets users audit whether a configured max_usd budget was actually enforceable.
+        # Computed BEFORE the aggregate, because _aggregate_token_usage backfills
+        # missing per-turn cost from the rate card: enforceability depends on the cost
+        # the SDK reported live, during the run, not on what we could price afterwards.
         if self.task.run_limits is not None and self.task.run_limits.max_usd is not None:
             any_cost_reported = any(
                 t.token_usage is not None and t.token_usage.total_cost_usd is not None for t in self.result.iterations
             )
             self.result.environment_info["cost_data_available"] = any_cost_reported
 
+        # Aggregate token usage
+        self._aggregate_token_usage()
+
         if self.result.iterations:
             self.result.total_assistant_turns = sum(t.assistant_turn_count for t in self.result.iterations)
 
@@ -852,6 +856,37 @@ def _check_expected_turns(self, *, iteration: int) -> None:
             )
             self._expected_turns_warning_emitted = True
 
+    def _backfill_turn_costs(self) -> None:
+        """Price any turn that recorded tokens but no cost, from the rate card.
+
+        The agent-agnostic cost seam. A turn arrives unpriced whenever the SDK had
+        no chance to report one — which is precisely the error and timeout paths:
+        the agent is killed mid-turn, so the partial ``TurnRecord`` carries real
+        billed tokens and ``total_cost_usd=None``. Those turns were being summed as
+        free. On one nightly that hid $209.81 (18.9% of the run) across 62
+        timed-out tasks, each holding 3.5M-8M tokens.
+
+        Only fills in what is missing: a turn the SDK priced keeps the SDK's
+        number, which is the authoritative billed figure. A model absent from the
+        rate card stays unpriced and is counted by ``RunSummary.tasks_unpriced``
+        rather than silently dropped.
+        """
+        assert self.result is not None
+        for turn in self.result.iterations:
+            usage = turn.token_usage
+            if usage is None or usage.total_cost_usd is not None or usage.is_empty():
+                continue
+            model = turn.model_used or self.result.model_used
+            if not model:
+                continue
+            usage.total_cost_usd = calculate_cost(
+                model,
+                uncached_input_tokens=usage.uncached_input_tokens,
+                output_tokens=usage.output_tokens,
+                cache_creation_tokens=usage.cache_creation_input_tokens,
+                cache_read_tokens=usage.cache_read_input_tokens,
+            )
+
     def _aggregate_token_usage(self) -> None:
         """Aggregate token usage from turns, storing on self.result.
 
@@ -863,9 +898,14 @@ def _aggregate_token_usage(self) -> None:
         the corresponding ``JudgeCriterionResult.token_usage`` and is
         intentionally NOT included in this aggregate — it represents the
         main agent's bill, not the eval-machinery overhead.
+
+        Runs ``_backfill_turn_costs`` first so an error/timeout partial
+        contributes its cost instead of being summed as free.
         """
         assert self.result is not None
 
+        self._backfill_turn_costs()
+
         # Include crashed=True partials: each API call is billed independently.
         if self.result.iterations:
             usages = [t.token_usage for t in self.result.iterations if t.token_usage is not None]
diff --git a/src/coder_eval/pricing.py b/src/coder_eval/pricing.py
index 28784360..02d713a4 100644
--- a/src/coder_eval/pricing.py
+++ b/src/coder_eval/pricing.py
@@ -5,6 +5,7 @@
 Source: https://www.anthropic.com/pricing
 """
 
+from collections.abc import Iterable
 from dataclasses import dataclass
 
 
@@ -174,6 +175,26 @@ def _normalize_model(model: str) -> str:
     return model
 
 
+def is_priced(model: str) -> bool:
+    """Whether the rate card can price this model (after prefix normalization).
+
+    The rate card is a static table, so a model that shipped after the installed
+    framework version prices as ``None`` — silently, per turn, for a whole run.
+    Call this at run start to make that a warning (or a refusal) instead of a
+    number that reads 19% low with nothing on the report to say so.
+    """
+    return _lookup_rate(_normalize_model(model)) is not None
+
+
+def unpriced_models(models: Iterable[str | None]) -> list[str]:
+    """Sorted, de-duplicated models from ``models`` that the rate card can't price.
+
+    Falsy entries are dropped: a task with no pinned model resolves its model at
+    the route level, which this pre-flight check cannot see.
+    """
+    return sorted({m for m in models if m and not is_priced(m)})
+
+
 def calculate_cost(
     model: str,
     uncached_input_tokens: int,
diff --git a/tests/test_cost_accounting_paths.py b/tests/test_cost_accounting_paths.py
new file mode 100644
index 00000000..3b5ee4fd
--- /dev/null
+++ b/tests/test_cost_accounting_paths.py
@@ -0,0 +1,295 @@
+"""Cost accounting on the error and timeout paths, where it used to go missing.
+
+Three seams, each of which independently lost real spend:
+
+1. **The pre-flight** (``check_pricing_coverage``) — the rate card is a static table
+   baked into the installed version, so a model released after it prices every turn
+   as ``null``. Sonnet 5's rates landed one release after the 2026-07-21 nightly,
+   which recorded $209.81 (18.9% of the run) as no cost at all, with one log line to
+   show for it.
+2. **The per-turn backfill** (``Orchestrator._backfill_turn_costs``) — a turn killed
+   mid-flight by a timeout or crash carries real billed tokens and no SDK cost.
+   Those partials were summed as free.
+3. **The row projection** (``eval_result_to_task_dict``) — judge spend was captured
+   per criterion and rolled up nowhere, and a row whose turns were only partly
+   priced reported its partial sum as the whole.
+"""
+
+from __future__ import annotations
+
+import logging
+from datetime import datetime
+from pathlib import Path
+
+import pytest
+
+from coder_eval.models import (
+    ClaudeCodeAgentConfig,
+    EvaluationResult,
+    FinalStatus,
+    JudgeCriterionResult,
+    ResolvedTask,
+    SimulationTelemetry,
+    TaskDefinition,
+    TokenUsage,
+    TurnRecord,
+)
+from coder_eval.orchestration.batch import check_pricing_coverage
+from coder_eval.reports_experiment import eval_result_to_task_dict
+
+
+def _resolved(model: str | None, tmp_path: Path) -> ResolvedTask:
+    task = TaskDefinition(
+        task_id="t1",
+        description="d",
+        initial_prompt="p",
+        agent=ClaudeCodeAgentConfig(type="claude-code", model=model),
+        sandbox={"driver": "tempdir"},
+        success_criteria=[{"type": "file_exists", "path": "f.py", "description": "d"}],
+    )
+    return ResolvedTask(
+        task=task,
+        task_file=tmp_path / "t1.yaml",
+        run_dir=tmp_path / "runs" / "t1",
+        variant_id="default",
+    )
+
+
+def _turn(iteration: int, usage: TokenUsage | None, model: str | None = None, crashed: bool = False) -> TurnRecord:
+    return TurnRecord(
+        iteration=iteration,
+        user_input="p",
+        agent_output="o",
+        token_usage=usage,
+        model_used=model,
+        crashed=crashed,
+    )
+
+
+def _result(turns: list[TurnRecord], *, model: str | None = "claude-sonnet-5", **extra) -> EvaluationResult:
+    return EvaluationResult(
+        task_id="t1",
+        task_description="d",
+        agent_type="claude-code",
+        model_used=model,
+        started_at=datetime(2026, 7, 28, 0, 0, 0),
+        final_status=FinalStatus.SUCCESS,
+        iteration_count=len(turns),
+        iterations=turns,
+        **extra,
+    )
+
+
+class TestPricingPreflight:
+    def test_priced_model_is_silent(self, tmp_path, caplog):
+        with caplog.at_level(logging.WARNING):
+            assert check_pricing_coverage([_resolved("claude-sonnet-5", tmp_path)]) == []
+        assert "No pricing rate" not in caplog.text
+
+    def test_unpriced_model_warns_but_runs(self, tmp_path, caplog):
+        """Default is a loud warning, not a refusal: a brand-new model stays evaluable."""
+        with caplog.at_level(logging.WARNING):
+            missing = check_pricing_coverage([_resolved("claude-sonnet-99", tmp_path)])
+        assert missing == ["claude-sonnet-99"]
+        assert "No pricing rate" in caplog.text
+        assert "understate the bill" in caplog.text
+
+    def test_strict_refuses_to_start(self, tmp_path):
+        """The cost-bearing-run setting: fail at dispatch, not in the cost column."""
+        with pytest.raises(ValueError, match="strict_pricing"):
+            check_pricing_coverage([_resolved("claude-sonnet-99", tmp_path)], strict=True)
+
+    def test_unpinned_model_is_not_flagged(self, tmp_path):
+        """A task deferring its model to the route can't be pre-flighted from here."""
+        assert check_pricing_coverage([_resolved(None, tmp_path)], strict=True) == []
+
+
+class TestTurnCostBackfill:
+    """``_backfill_turn_costs`` is exercised through a bare Orchestrator instance.
+
+    Built with ``__new__`` rather than a full construction because the method reads
+    only ``self.result`` — a real Orchestrator needs a sandbox, an agent, and a
+    route, none of which participate in pricing.
+    """
+
+    @staticmethod
+    def _backfill(result: EvaluationResult) -> None:
+        from coder_eval.orchestrator import Orchestrator
+
+        orch = Orchestrator.__new__(Orchestrator)
+        orch.result = result
+        orch._backfill_turn_costs()
+
+    def test_prices_a_killed_turn_from_the_rate_card(self):
+        """The timeout shape: real tokens on the wire, no SDK cost, previously free."""
+        result = self._result_with_partial()
+        self._backfill(result)
+
+        priced = result.iterations[1].token_usage
+        assert priced is not None and priced.total_cost_usd is not None
+        # 1M uncached input + 100k output on sonnet-5 ($3/$15 per MTok).
+        assert priced.total_cost_usd == pytest.approx(3.0 + 1.5)
+
+    def test_leaves_an_sdk_reported_cost_alone(self):
+        """The SDK's number is the authoritative billed figure; never overwrite it."""
+        result = self._result_with_partial()
+        self._backfill(result)
+        assert result.iterations[0].token_usage.total_cost_usd == pytest.approx(0.99)
+
+    def test_falls_back_to_the_row_model(self):
+        """A partial that never learned its model still prices off the resolved one."""
+        result = _result(
+            [_turn(1, TokenUsage(uncached_input_tokens=1_000_000, output_tokens=0), model=None, crashed=True)],
+            model="claude-sonnet-5",
+        )
+        self._backfill(result)
+        assert result.iterations[0].token_usage.total_cost_usd == pytest.approx(3.0)
+
+    def test_unpriceable_model_stays_unpriced(self):
+        """Not a guess: leave it null so tasks_unpriced can count it."""
+        result = _result(
+            [_turn(1, TokenUsage(uncached_input_tokens=1_000_000, output_tokens=0), model="made-up-model")],
+            model="made-up-model",
+        )
+        self._backfill(result)
+        assert result.iterations[0].token_usage.total_cost_usd is None
+
+    def test_empty_usage_is_not_priced(self):
+        """A turn that burned nothing must not acquire a $0.00 cost it never had."""
+        result = _result([_turn(1, TokenUsage())])
+        self._backfill(result)
+        assert result.iterations[0].token_usage.total_cost_usd is None
+
+    @staticmethod
+    def _result_with_partial() -> EvaluationResult:
+        return _result(
+            [
+                _turn(
+                    1,
+                    TokenUsage(uncached_input_tokens=1000, output_tokens=100, total_cost_usd=0.99),
+                    model="claude-sonnet-5",
+                ),
+                _turn(
+                    2,
+                    TokenUsage(uncached_input_tokens=1_000_000, output_tokens=100_000),
+                    model="claude-sonnet-5",
+                    crashed=True,
+                ),
+            ]
+        )
+
+
+class TestRowCostProjection:
+    def test_cost_complete_false_when_a_turn_is_unpriced(self):
+        result = _result(
+            [
+                _turn(1, TokenUsage(uncached_input_tokens=10, output_tokens=1, total_cost_usd=0.1)),
+                _turn(2, TokenUsage(uncached_input_tokens=10, output_tokens=1)),
+            ]
+        )
+        assert eval_result_to_task_dict(result)["cost_complete"] is False
+
+    def test_cost_complete_true_when_every_burning_turn_is_priced(self):
+        result = _result([_turn(1, TokenUsage(uncached_input_tokens=10, output_tokens=1, total_cost_usd=0.1))])
+        assert eval_result_to_task_dict(result)["cost_complete"] is True
+
+    def test_cost_complete_true_when_nothing_burned(self):
+        """An error before the agent ran genuinely cost nothing — not 'missing cost'."""
+        assert eval_result_to_task_dict(_result([]))["cost_complete"] is True
+        assert eval_result_to_task_dict(_result([_turn(1, TokenUsage())]))["cost_complete"] is True
+
+    def test_judge_cost_rolls_up_onto_the_row(self):
+        """Judge spend was captured per criterion and totalled nowhere."""
+        result = _result([_turn(1, TokenUsage(uncached_input_tokens=10, output_tokens=1, total_cost_usd=0.1))])
+        result.total_token_usage = TokenUsage(uncached_input_tokens=10, output_tokens=1, total_cost_usd=0.1)
+        result.success_criteria_results = [
+            JudgeCriterionResult(
+                criterion_type="llm_judge",
+                description="d",
+                score=1.0,
+                token_usage=TokenUsage(uncached_input_tokens=5000, output_tokens=500, total_cost_usd=0.02),
+            ),
+            JudgeCriterionResult(
+                criterion_type="llm_judge",
+                description="d2",
+                score=1.0,
+                token_usage=TokenUsage(uncached_input_tokens=5000, output_tokens=500, total_cost_usd=0.03),
+            ),
+        ]
+        row = eval_result_to_task_dict(result)
+        assert row["judge_cost_usd"] == pytest.approx(0.05)
+        # Kept out of the agent's bill.
+        assert row["total_cost_usd"] == pytest.approx(0.1)
+
+    def test_no_judge_means_no_judge_cost(self):
+        """None, not 0.0 — 'no judge ran' must stay distinct from 'a judge ran free'."""
+        result = _result([_turn(1, TokenUsage(uncached_input_tokens=10, output_tokens=1, total_cost_usd=0.1))])
+        assert eval_result_to_task_dict(result)["judge_cost_usd"] is None
+
+    @staticmethod
+    def _simulated(**env) -> EvaluationResult:
+        return _result(
+            [_turn(1, TokenUsage(uncached_input_tokens=10, output_tokens=1, total_cost_usd=0.1))],
+            simulation=SimulationTelemetry(
+                n_trials=1,
+                replicate_index=0,
+                stop_reason="stop_token",
+                simulator_input_tokens=1_000_000,
+                simulator_output_tokens=100_000,
+                total_turns=3,
+            ),
+            environment_info=env,
+        )
+
+    def test_simulator_prices_at_the_route_model_not_the_subject(self):
+        """UserSimulator pins model=None, so it bills at BEDROCK_MODEL.
+
+        Every skills task pins ``agent.model``, so pricing the simulator at the
+        subject's model would mis-bill the whole suite. Here the subject is
+        sonnet-5 ($3/$15) while the route is haiku-4.5 ($0.80/$4) — the simulator
+        must cost the haiku rate.
+        """
+        result = self._simulated(bedrock_model="claude-haiku-4-5-20251001")
+        assert eval_result_to_task_dict(result)["simulator_cost_usd"] == pytest.approx(0.80 + 0.40)
+
+    def test_simulator_falls_back_to_the_subject_model_off_bedrock(self):
+        """A non-Bedrock route names no model on the record; the subject's is the best available."""
+        result = self._simulated()
+        # sonnet-5 at $3/$15 per MTok.
+        assert eval_result_to_task_dict(result)["simulator_cost_usd"] == pytest.approx(3.0 + 1.5)
+
+    def test_single_shot_row_has_no_simulator_cost(self):
+        result = _result([_turn(1, TokenUsage(uncached_input_tokens=10, output_tokens=1, total_cost_usd=0.1))])
+        assert eval_result_to_task_dict(result)["simulator_cost_usd"] is None
+
+
+class TestErrorDiagnosticsOnTheRow:
+    """Errors count as misses, so the rollup has to say why it lost those points.
+
+    The 2026-07-22 codex nightly had 109 zero-iteration errors that could not be
+    characterised from run.json at all — every one needed its own task.json fetch.
+    """
+
+    def test_error_message_and_category_land_on_the_row(self):
+        result = _result([], model=None)
+        result.final_status = FinalStatus.ERROR
+        result.error_message = "sandbox setup failed: no space left on device"
+        result.error_details = {"error_category": "disk_full", "component": "orchestrator.setup"}
+
+        row = eval_result_to_task_dict(result)
+        assert row["error_message"] == "sandbox setup failed: no space left on device"
+        assert row["error_category"] == "disk_full"
+
+    def test_error_message_is_truncated(self):
+        result = _result([], model=None)
+        result.final_status = FinalStatus.ERROR
+        result.error_message = "x" * 5000
+
+        row = eval_result_to_task_dict(result)
+        assert len(row["error_message"]) < 500
+        assert row["error_message"].endswith("…")
+
+    def test_clean_row_carries_no_error_fields(self):
+        row = eval_result_to_task_dict(_result([]))
+        assert row["error_message"] is None
+        assert row["error_category"] is None
diff --git a/tests/test_route_seam_exhaustiveness.py b/tests/test_route_seam_exhaustiveness.py
index e1b41f19..58180b2f 100644
--- a/tests/test_route_seam_exhaustiveness.py
+++ b/tests/test_route_seam_exhaustiveness.py
@@ -65,7 +65,7 @@ def test_invoke_tool_channel_handles_every_route(monkeypatch):
     monkeypatch.setattr(llm_judge, "invoke_bedrock_judge", lambda **_: {})
     monkeypatch.setattr(llm_judge, "invoke_anthropic_judge", lambda **_: {})
     monkeypatch.setattr(llm_judge, "extract_verdict_from_anthropic_response", lambda _resp: (None, "stub"))
-    monkeypatch.setattr(llm_judge, "token_usage_from_anthropic_dict", lambda _resp: None)
+    monkeypatch.setattr(llm_judge, "token_usage_from_anthropic_dict", lambda _resp, **_kwargs: None)
     criterion = MagicMock()
     for r in _INSTANCES:
         result = _invoke_tool_channel(criterion=criterion, route=r, system_msg="s", user_msg="u")  # type: ignore[arg-type]
diff --git a/tests/test_token_usage.py b/tests/test_token_usage.py
index 7eb70b72..ccb15fae 100644
--- a/tests/test_token_usage.py
+++ b/tests/test_token_usage.py
@@ -882,9 +882,61 @@ def test_token_section_handles_missing_cost(self):
 
         assert "## Token Usage" in joined
         assert "**Total Tokens**: 5,000 (input: 3,000, output: 2,000)" in joined
-        assert "**Total Cost**" not in joined  # No cost line when all costs are None
+        # Tokens were burned, so there IS a bill — the report must say the number is
+        # missing rather than omit the cost line, which reads as "this run was free".
+        assert "**Total Cost**: unavailable" in joined
+        assert "1 task(s) burned tokens the rate card could not price" in joined
         assert "| task1 | 3,000 | 2,000 | 0 | 0 | 5,000 | N/A |" in joined
 
+    def test_token_section_omits_cost_when_no_tokens_burned(self):
+        """A row that burned nothing is genuinely free — no unpriced warning."""
+        task_results = [
+            {
+                "task_id": "task1",
+                "input_tokens": 0,
+                "output_tokens": 0,
+                "total_tokens": 0,
+                "total_cost_usd": None,
+            },
+        ]
+        joined = "\n".join(ReportGenerator._generate_token_usage_section(task_results))
+
+        assert "## Token Usage" in joined
+        assert "**Total Cost**" not in joined
+
+    def test_token_section_marks_partial_cost_as_floor(self):
+        """A priced row beside an unpriced one reports the total as a floor.
+
+        The failure this guards: summing only the rows that carry a cost and
+        presenting it as the run's bill. One nightly under-reported $209.81 (18.9%)
+        exactly this way.
+        """
+        task_results = [
+            {"task_id": "priced", "total_tokens": 1000, "total_cost_usd": 0.25, "cost_complete": True},
+            {"task_id": "unpriced", "total_tokens": 4_000_000, "total_cost_usd": None, "cost_complete": False},
+        ]
+        joined = "\n".join(ReportGenerator._generate_token_usage_section(task_results))
+
+        assert "**Total Cost**: $0.2500 (floor" in joined
+        assert "1 task(s) burned tokens the rate card could not price" in joined
+
+    def test_token_section_breaks_out_eval_overhead(self):
+        """Judge spend is reported beside the agent bill, and folded into the total."""
+        task_results = [
+            {
+                "task_id": "task1",
+                "total_tokens": 1000,
+                "total_cost_usd": 1.0,
+                "cost_complete": True,
+                "judge_cost_usd": 0.25,
+            },
+        ]
+        joined = "\n".join(ReportGenerator._generate_token_usage_section(task_results))
+
+        assert "**Agent Cost**: $1.0000" in joined
+        assert "**Eval Overhead (judge + simulator)**: $0.2500" in joined
+        assert "**Total Cost**: $1.2500" in joined
+
     def test_token_section_in_full_report(self):
         """Test that token section appears in generate_markdown when data is available."""
         summary = RunSummary(

From 871a5280ae94a595de32152239be520c198006a0 Mon Sep 17 00:00:00 2001
From: Bai Li <bai.li@uipath.com>
Date: Tue, 28 Jul 2026 15:40:32 -0700
Subject: [PATCH 03/20] feat(evalboard): read the canonical pass rate and
 surface incomplete cost

The dashboard derived its own pass rate and summed per-task cost treating a
missing cost as zero, so an unpriced row was indistinguishable from a free one
and a run's bill read low with nothing to say so. It now prefers run.json's
`pass_rate`, falling back to the identical formula so historical runs render
unchanged, and counts unpriced rows into the existing cost-partial caveat.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 evalboard/lib/__tests__/overview.test.ts | 21 ++++++
 evalboard/lib/__tests__/runs.test.ts     | 38 ++++++++++
 evalboard/lib/overview.ts                | 17 ++++-
 evalboard/lib/runs.ts                    | 91 +++++++++++++++++++++++-
 4 files changed, 163 insertions(+), 4 deletions(-)

diff --git a/evalboard/lib/__tests__/overview.test.ts b/evalboard/lib/__tests__/overview.test.ts
index eaa935d1..0dcd2476 100644
--- a/evalboard/lib/__tests__/overview.test.ts
+++ b/evalboard/lib/__tests__/overview.test.ts
@@ -97,6 +97,27 @@ describe("summarizeListing", () => {
         expect(t.durationSeconds).toBeNull();
         expect(t.durationPartial).toBe(false);
     });
+
+    test("flags partial when a run's own cost is incomplete", () => {
+        // Every run reported a cost, so the old costRuns check is satisfied — but
+        // one of them under-counted because some task's tokens went unpriced. The
+        // 2026-07-21 nightly was exactly this: $902.81 reported, $209.81 missing
+        // (18.9%), and nothing anywhere said the total was a floor.
+        const t = summarizeListing([
+            row({ totalCostUsd: 902.81, costComplete: false }),
+            row({ totalCostUsd: 100, costComplete: true }),
+        ]);
+        expect(t.costUsd).toBeCloseTo(1002.81);
+        expect(t.costPartial).toBe(true);
+    });
+
+    test("complete costs across every run are not flagged", () => {
+        const t = summarizeListing([
+            row({ totalCostUsd: 1, costComplete: true }),
+            row({ totalCostUsd: 2, costComplete: true }),
+        ]);
+        expect(t.costPartial).toBe(false);
+    });
 });
 
 describe("turnBudgetRateForTasks", () => {
diff --git a/evalboard/lib/__tests__/runs.test.ts b/evalboard/lib/__tests__/runs.test.ts
index 5fd0c719..f27c4887 100644
--- a/evalboard/lib/__tests__/runs.test.ts
+++ b/evalboard/lib/__tests__/runs.test.ts
@@ -46,6 +46,44 @@ describe("toTaskRow", () => {
         const row = toTaskRow({ task_id: "x", expected_turns: null });
         expect(row.expectedTurns).toBeNull();
     });
+
+    test("reads cost_complete and the eval-overhead costs", () => {
+        const row = toTaskRow({
+            task_id: "x",
+            cost_complete: false,
+            judge_cost_usd: 0.02,
+            simulator_cost_usd: 0.01,
+        });
+        expect(row.costComplete).toBe(false);
+        expect(row.judgeCostUsd).toBeCloseTo(0.02);
+        expect(row.simulatorCostUsd).toBeCloseTo(0.01);
+    });
+
+    test("infers incomplete cost on a run predating the field", () => {
+        // Tokens on the wire with no cost is the unpriced shape, whether or not
+        // the row carries the explicit flag.
+        const row = toTaskRow({ task_id: "x", total_tokens: 5_000_000 });
+        expect(row.costComplete).toBe(false);
+    });
+
+    test("a row that burned nothing is complete, not unpriced", () => {
+        // An error before the agent ran genuinely cost nothing.
+        const row = toTaskRow({ task_id: "x", total_tokens: 0 });
+        expect(row.costComplete).toBe(true);
+    });
+
+    test("carries the error reason for an errored row", () => {
+        // Errors count as misses in the pass rate, so the dashboard has to be
+        // able to say where the points went.
+        const row = toTaskRow({
+            task_id: "x",
+            status: "ERROR",
+            error_message: "no space left on device",
+            error_category: "disk_full",
+        });
+        expect(row.errorMessage).toBe("no space left on device");
+        expect(row.errorCategory).toBe("disk_full");
+    });
 });
 
 describe("aggregateSubAgentUsage", () => {
diff --git a/evalboard/lib/overview.ts b/evalboard/lib/overview.ts
index 4e6cfa6b..c3f3b63c 100644
--- a/evalboard/lib/overview.ts
+++ b/evalboard/lib/overview.ts
@@ -95,6 +95,10 @@ export interface RunListingRow {
     tasksSucceeded: number;
     tasksRun: number;
     totalCostUsd: number | null;
+    // False when a task in scope burned tokens whose cost is missing, so
+    // totalCostUsd is a floor. Optional so existing test factories stay valid;
+    // absent reads as complete.
+    costComplete?: boolean;
     taskDurationSeconds: number | null;
     // Run-level harness (coder-eval AgentKind) for the Harness column; null on
     // legacy runs that predate the RunConfig stamp / carry no agent_config.type.
@@ -109,7 +113,10 @@ export interface RunListingRow {
 // sum instead of silently understating it.
 export interface RunListingTotals {
     costUsd: number | null; // null when no matched run recorded a cost
-    costPartial: boolean; // some matched runs had no cost (sum understates)
+    // The sum understates: either a matched run recorded no cost at all, or one
+    // recorded a partial cost because some task's tokens went unpriced. Both make
+    // costUsd a floor, and both used to be invisible.
+    costPartial: boolean;
     tasksSucceeded: number;
     tasksRun: number;
     durationSeconds: number | null; // null when no matched run recorded a duration
@@ -126,6 +133,7 @@ export function summarizeListing(rows: RunListingRow[]): RunListingTotals {
     let tasksRun = 0;
     let durationSeconds = 0;
     let durationRuns = 0;
+    let anyCostIncomplete = false;
     for (const r of rows) {
         tasksSucceeded += r.tasksSucceeded;
         tasksRun += r.tasksRun;
@@ -133,6 +141,7 @@ export function summarizeListing(rows: RunListingRow[]): RunListingTotals {
             costUsd += r.totalCostUsd;
             costRuns += 1;
         }
+        if (r.costComplete === false) anyCostIncomplete = true;
         if (r.taskDurationSeconds != null) {
             durationSeconds += r.taskDurationSeconds;
             durationRuns += 1;
@@ -140,7 +149,8 @@ export function summarizeListing(rows: RunListingRow[]): RunListingTotals {
     }
     return {
         costUsd: costRuns > 0 ? costUsd : null,
-        costPartial: costRuns > 0 && costRuns < rows.length,
+        costPartial:
+            (costRuns > 0 && costRuns < rows.length) || anyCostIncomplete,
         tasksSucceeded,
         tasksRun,
         durationSeconds: durationRuns > 0 ? durationSeconds : null,
@@ -754,6 +764,9 @@ export async function getRunListing(
                 .length,
             tasksRun: scopedTasks.length,
             totalCostUsd: scopedCost,
+            // Scoped like cost: with a filter active, only the matching tasks'
+            // completeness bears on the cost shown for this row.
+            costComplete: scopedTasks.every((t) => t.costComplete !== false),
             taskDurationSeconds: scopedDur,
             harness: overview.harness ?? null,
         });
diff --git a/evalboard/lib/runs.ts b/evalboard/lib/runs.ts
index 45de6f98..fef14e0c 100644
--- a/evalboard/lib/runs.ts
+++ b/evalboard/lib/runs.ts
@@ -54,7 +54,16 @@ export interface RunSummary {
     tasksSucceeded: number;
     tasksFailed: number;
     tasksError: number;
+    // Pass rate as a 0-1 fraction, errors counted as misses. Read from run.json's
+    // canonical `pass_rate` when present so this never diverges from the report.
+    passRate: number | null;
     totalCostUsd: number | null;
+    // False when some task's spend went unpriced, so totalCostUsd is a floor.
+    costComplete: boolean;
+    tasksUnpriced: number;
+    // Judge + simulator spend. Kept out of totalCostUsd (a property of the suite's
+    // criteria, identical across harnesses) but real money all the same.
+    evalOverheadCostUsd: number | null;
     componentShas: ComponentSha[];
 }
 
@@ -95,6 +104,19 @@ export interface TaskResultSummary {
     // carried forward as a pass (run.json `mature_skipped`). The task wasn't
     // executed, so it has no per-task detail to link to. False on normal rows.
     matureSkipped: boolean;
+    // False when this row burned tokens whose cost is missing, so totalCostUsd is
+    // a floor. Every cost sum treats a null cost as $0, so without this an
+    // unpriced row is indistinguishable from a free one and the run's bill reads
+    // low with nothing to say so. All five fields below are optional so test
+    // factories predating them stay valid (undefined === complete / absent).
+    costComplete?: boolean;
+    // Judge + simulator spend for this row, outside totalCostUsd.
+    judgeCostUsd?: number | null;
+    simulatorCostUsd?: number | null;
+    // Why this row errored (null otherwise). Errors count as misses in the pass
+    // rate, so the dashboard has to be able to say what the points went to.
+    errorMessage?: string | null;
+    errorCategory?: string | null;
 }
 
 export interface CriterionResult {
@@ -326,6 +348,9 @@ interface RawTaskResult {
     weighted_score?: number;
     duration?: number;
     total_cost_usd?: number;
+    // Row-level token total. Present alongside the disjoint buckets below; used to
+    // tell "burned tokens but has no cost" (unpriced) from "burned nothing" (free).
+    total_tokens?: number | null;
     input_tokens?: number | null;
     output_tokens?: number | null;
     cache_creation_input_tokens?: number | null;
@@ -367,6 +392,16 @@ interface RawTaskResult {
     // Set by the nightly runner (eval_runner) on a carried-forward row for a
     // "mature" task it skipped this run. Absent on normal rows.
     mature_skipped?: boolean;
+    // False when a turn on this row burned tokens the rate card could not price,
+    // so total_cost_usd is a floor. Absent on runs predating the field.
+    cost_complete?: boolean | null;
+    // Evaluation-machinery spend for this row, kept out of total_cost_usd.
+    judge_cost_usd?: number | null;
+    simulator_cost_usd?: number | null;
+    // Why an errored row errored. Errors count as misses, so the rollup needs to
+    // say what it lost the points to.
+    error_message?: string | null;
+    error_category?: string | null;
 }
 
 interface RawRunJson {
@@ -378,6 +413,19 @@ interface RawRunJson {
     tasks_succeeded?: number;
     tasks_failed?: number;
     tasks_error?: number;
+    // Canonical pass rate (0-1) computed by coder_eval: tasks_succeeded /
+    // tasks_run, errors included as misses. READ THIS rather than re-deriving a
+    // denominator here — four surfaces each deriving their own is what made this
+    // dashboard and the markdown report disagree by up to 10 points on the same
+    // file. Absent on runs predating the field; the fallback below reproduces the
+    // identical formula, so historical runs render unchanged.
+    pass_rate?: number | null;
+    // False when some task burned tokens the rate card could not price, making
+    // every cost total on this run a floor rather than the bill.
+    cost_complete?: boolean | null;
+    tasks_unpriced?: number | null;
+    // Judge + simulator spend, deliberately outside the agent bill.
+    eval_overhead_cost_usd?: number | null;
     task_results?: RawTaskResult[];
     // Values are scalars except `tool_plugins`, a {plugin: version} map of
     // the installed @uipath/*-tool packages (recorded since coder_eval #366).
@@ -640,6 +688,15 @@ export function toTaskRow(t: RawTaskResult): TaskResultSummary {
         tags,
         skill: deriveSkill(t.task_path, tags),
         matureSkipped: t.mature_skipped ?? false,
+        // A row predating the field is assumed complete unless it visibly burned
+        // tokens with no cost — the same test the Python side applies.
+        costComplete:
+            t.cost_complete ??
+            !((t.total_tokens ?? 0) > 0 && t.total_cost_usd == null),
+        judgeCostUsd: t.judge_cost_usd ?? null,
+        simulatorCostUsd: t.simulator_cost_usd ?? null,
+        errorMessage: t.error_message ?? null,
+        errorCategory: t.error_category ?? null,
     };
 }
 
@@ -665,16 +722,38 @@ export async function readRunSummary(id: string): Promise<RunSummary | null> {
     const taskDurationSeconds = allHaveDuration
         ? taskDurationSum
         : (data.total_duration_seconds ?? null);
+    const tasksRun = data.tasks_run ?? taskResults.length;
+    const tasksSucceeded = data.tasks_succeeded ?? 0;
+    // A row whose tokens went unpriced contributes 0 to totalCost above, so the
+    // sum silently understates the bill. Count those rows and say so rather than
+    // presenting a floor as the total.
+    const tasksUnpriced = taskResults.filter(
+        (t) =>
+            ((t.total_tokens ?? 0) > 0 && t.total_cost_usd == null) ||
+            t.cost_complete === false,
+    ).length;
+    const overhead = taskResults.reduce(
+        (a, t) => a + (t.judge_cost_usd ?? 0) + (t.simulator_cost_usd ?? 0),
+        0,
+    );
     return {
         id,
         startTime: data.start_time ?? null,
         endTime: data.end_time ?? null,
         taskDurationSeconds,
-        tasksRun: data.tasks_run ?? taskResults.length,
-        tasksSucceeded: data.tasks_succeeded ?? 0,
+        tasksRun,
+        tasksSucceeded,
         tasksFailed: data.tasks_failed ?? 0,
         tasksError: data.tasks_error ?? 0,
+        // Prefer the canonical field; the fallback is the identical formula, so
+        // historical runs without it render exactly as they did before.
+        passRate:
+            data.pass_rate ?? (tasksRun > 0 ? tasksSucceeded / tasksRun : null),
         totalCostUsd: taskResults.length ? totalCost : null,
+        costComplete: data.cost_complete ?? tasksUnpriced === 0,
+        tasksUnpriced: data.tasks_unpriced ?? tasksUnpriced,
+        evalOverheadCostUsd:
+            data.eval_overhead_cost_usd ?? (overhead > 0 ? overhead : null),
         componentShas: extractComponentShas(data.environment_info),
     };
 }
@@ -790,6 +869,11 @@ export interface RunOverviewTask {
     // Optional so test factories that predate the field stay valid (undefined
     // === a normal executed row).
     matureSkipped?: boolean;
+    // False when this row burned tokens whose cost is missing, so totalCostUsd is
+    // a floor. Every cost sum counts a null cost as $0, so without this an
+    // unpriced row is indistinguishable from a free one. Optional so test
+    // factories predating it stay valid (undefined === complete).
+    costComplete?: boolean;
 }
 
 export interface RunOverview {
@@ -905,6 +989,9 @@ export async function readRunOverview(
                 visibleTurns: visibleTurnsFromRaw(t),
                 hasFinalReply: t.has_final_reply ?? false,
                 matureSkipped: t.mature_skipped ?? false,
+                costComplete:
+                    t.cost_complete ??
+                    !((t.total_tokens ?? 0) > 0 && t.total_cost_usd == null),
             };
         });
     const totalCost = taskResults.reduce(

From 5afc9359cecb643c7e2b1a57ee1cc7d372871f0e Mon Sep 17 00:00:00 2001
From: Bai Li <bai.li@uipath.com>
Date: Tue, 28 Jul 2026 15:47:48 -0700
Subject: [PATCH 04/20] docs(cost): describe the per-turn backfill as the net
 it is
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

An A/B on a real timed-out Bedrock run booked its spend with the backfill
disabled, so the earlier claim that killed partials "were summed as free" does
not hold: every in-tree agent already prices its own partials
(ClaudeCodeAgent._backfill_cost; codex and antigravity compute from buckets and
never depend on an SDK cost). The measured $209.81 loss was the rate-card miss
alone.

The backfill stays, described accurately: it makes "tokens on the record imply a
cost on the record" an invariant at one agent-agnostic seam, which the plugin
SPI needs — an out-of-tree agent that registers pricing but never applies it
would otherwise lose all of its spend silently.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 src/coder_eval/orchestrator.py      | 19 +++++++++++------
 tests/test_cost_accounting_paths.py | 33 +++++++++++++++--------------
 2 files changed, 29 insertions(+), 23 deletions(-)

diff --git a/src/coder_eval/orchestrator.py b/src/coder_eval/orchestrator.py
index de71b3a0..1b97673b 100644
--- a/src/coder_eval/orchestrator.py
+++ b/src/coder_eval/orchestrator.py
@@ -859,17 +859,22 @@ def _check_expected_turns(self, *, iteration: int) -> None:
     def _backfill_turn_costs(self) -> None:
         """Price any turn that recorded tokens but no cost, from the rate card.
 
-        The agent-agnostic cost seam. A turn arrives unpriced whenever the SDK had
-        no chance to report one — which is precisely the error and timeout paths:
-        the agent is killed mid-turn, so the partial ``TurnRecord`` carries real
-        billed tokens and ``total_cost_usd=None``. Those turns were being summed as
-        free. On one nightly that hid $209.81 (18.9% of the run) across 62
-        timed-out tasks, each holding 3.5M-8M tokens.
+        Makes "tokens on the record imply a cost on the record" an invariant at one
+        agent-agnostic seam, instead of a convention each agent has to remember.
+
+        Every in-tree agent already prices its own turns, including the killed
+        partials an error or timeout produces (``ClaudeCodeAgent._backfill_cost``;
+        codex and antigravity compute from buckets and never depend on an SDK
+        cost), so this is a net rather than a fix for a live gap — verified by
+        A/B on a real timed-out Bedrock run, which booked its spend with this
+        disabled. It earns its keep for the plugin SPI: an out-of-tree agent that
+        registers pricing but doesn't apply it would otherwise lose 100% of its
+        cost silently, with only its token counts to show anything happened.
 
         Only fills in what is missing: a turn the SDK priced keeps the SDK's
         number, which is the authoritative billed figure. A model absent from the
         rate card stays unpriced and is counted by ``RunSummary.tasks_unpriced``
-        rather than silently dropped.
+        rather than being passed off as free.
         """
         assert self.result is not None
         for turn in self.result.iterations:
diff --git a/tests/test_cost_accounting_paths.py b/tests/test_cost_accounting_paths.py
index 3b5ee4fd..3df2a1e9 100644
--- a/tests/test_cost_accounting_paths.py
+++ b/tests/test_cost_accounting_paths.py
@@ -1,18 +1,19 @@
-"""Cost accounting on the error and timeout paths, where it used to go missing.
-
-Three seams, each of which independently lost real spend:
-
-1. **The pre-flight** (``check_pricing_coverage``) — the rate card is a static table
-   baked into the installed version, so a model released after it prices every turn
-   as ``null``. Sonnet 5's rates landed one release after the 2026-07-21 nightly,
-   which recorded $209.81 (18.9% of the run) as no cost at all, with one log line to
-   show for it.
-2. **The per-turn backfill** (``Orchestrator._backfill_turn_costs``) — a turn killed
-   mid-flight by a timeout or crash carries real billed tokens and no SDK cost.
-   Those partials were summed as free.
-3. **The row projection** (``eval_result_to_task_dict``) — judge spend was captured
-   per criterion and rolled up nowhere, and a row whose turns were only partly
-   priced reported its partial sum as the whole.
+"""Cost accounting on the error and timeout paths, where spend went missing.
+
+Three seams:
+
+1. **The pre-flight** (``check_pricing_coverage``) — the measured loss. The rate
+   card is a static table baked into the installed version, so a model released
+   after it prices every turn as ``null``. Sonnet 5's rates landed one release
+   after the 2026-07-21 nightly, which recorded $209.81 (18.9% of its true bill)
+   across 62 errored rows as no cost at all, with one log line to show for it.
+2. **The per-turn backfill** (``Orchestrator._backfill_turn_costs``) — a net, not a
+   fix for a live gap: every in-tree agent already prices its own killed partials.
+   It makes "tokens on the record imply a cost on the record" an invariant at one
+   seam, so a plugin agent that doesn't price can't silently lose all of its spend.
+3. **The row projection** (``eval_result_to_task_dict``) — judge and simulator
+   spend was captured and rolled up nowhere, and a row whose turns were only
+   partly priced reported its partial sum as the whole.
 """
 
 from __future__ import annotations
@@ -121,7 +122,7 @@ def _backfill(result: EvaluationResult) -> None:
         orch._backfill_turn_costs()
 
     def test_prices_a_killed_turn_from_the_rate_card(self):
-        """The timeout shape: real tokens on the wire, no SDK cost, previously free."""
+        """The timeout shape: real tokens on the wire, no cost attached yet."""
         result = self._result_with_partial()
         self._backfill(result)
 

From 5b49a24770d79fcb4bf4b4b3cb64821435342a4a Mon Sep 17 00:00:00 2001
From: Bai Li <bai.li@uipath.com>
Date: Tue, 28 Jul 2026 18:27:45 -0700
Subject: [PATCH 05/20] revert(cost): drop the speculative turn-cost backfill
 and --strict-pricing
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Both were justified by anticipation rather than by measurement, and this PR's
other claims were all measured. Removing them leaves the orchestrator, the batch
config, and the CLI byte-identical to main, so the PR no longer touches the run
path at all.

The per-turn backfill (Orchestrator._backfill_turn_costs) protected against an
out-of-tree plugin agent that registers pricing but never applies it. No such
agent exists, and the A/B established that every in-tree agent already prices
its own killed partials — so it guarded a hypothetical while adding a mutation
to the aggregate path and forcing cost_data_available to be computed earlier to
stay honest about budget enforceability. Reverting restores main's ordering.

--strict-pricing had no caller: nothing in CI or the nightly passed it, and the
pre-flight warning already surfaces the rate-card miss that cost $209.81. The
warning and the is_priced/unpriced_models seam stay, as does the CI guard that
fails on a committed experiment referencing an unpriced model.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 src/coder_eval/cli/run_command.py      |  8 ---
 src/coder_eval/orchestration/batch.py  | 28 ++++----
 src/coder_eval/orchestration/config.py | 10 ---
 src/coder_eval/orchestrator.py         | 51 +--------------
 tests/test_cost_accounting_paths.py    | 90 +-------------------------
 5 files changed, 18 insertions(+), 169 deletions(-)

diff --git a/src/coder_eval/cli/run_command.py b/src/coder_eval/cli/run_command.py
index 19e36c24..9b22929c 100644
--- a/src/coder_eval/cli/run_command.py
+++ b/src/coder_eval/cli/run_command.py
@@ -211,11 +211,6 @@ def run_command(
         "-v",
         help="Enable verbose (DEBUG level) logging",
     ),
-    strict_pricing: bool = typer.Option(
-        False,
-        "--strict-pricing",
-        help="Refuse to start if a task's model has no pricing rate (default: warn and run)",
-    ),
     log_file: Path | None = typer.Option(  # noqa: B008
         None,
         "--log-file",
@@ -419,7 +414,6 @@ def run_command(
                 resume=resume,
                 include_skipped=include_skipped,
                 junit_xml=junit_xml,
-                strict_pricing=strict_pricing,
             )
         )
     except KeyboardInterrupt:
@@ -445,7 +439,6 @@ async def _run_all_tasks(
     resume: bool = False,
     include_skipped: bool = False,
     junit_xml: Path | None = None,
-    strict_pricing: bool = False,
 ) -> None:
     """Async entry point for running all tasks (optionally in parallel).
 
@@ -491,7 +484,6 @@ async def _run_all_tasks(
         repeats=repeats,
         verbose=verbose,
         include_skipped=include_skipped,
-        strict_pricing=strict_pricing,
     )
 
     from ..telemetry import flush_telemetry, track_event
diff --git a/src/coder_eval/orchestration/batch.py b/src/coder_eval/orchestration/batch.py
index 639a4972..6e7bee93 100644
--- a/src/coder_eval/orchestration/batch.py
+++ b/src/coder_eval/orchestration/batch.py
@@ -39,7 +39,7 @@
 logger = logging.getLogger(__name__)
 
 
-def check_pricing_coverage(resolved_tasks: list[ResolvedTask], *, strict: bool = False) -> list[str]:
+def check_pricing_coverage(resolved_tasks: list[ResolvedTask]) -> list[str]:
     """Pre-flight the rate card against the models this run will use.
 
     The rate card is a static table baked into the installed framework version,
@@ -47,34 +47,30 @@ def check_pricing_coverage(resolved_tasks: list[ResolvedTask], *, strict: bool =
     run completes, records full token counts, and reports a cost that is silently
     low. (One nightly under-reported $209.81, 18.9% of its bill, because the
     model's rates landed in the next release.) This turns that into a warning at
-    dispatch, or a refusal under ``strict``.
+    dispatch instead of a discovery weeks later.
+
+    A warning rather than a refusal, so a brand-new model stays evaluable on the
+    day it ships: the cost is what degrades, not the evaluation. What the run
+    actually lost is then counted after the fact by ``RunSummary.tasks_unpriced``.
 
     Only pinned ``agent.model`` values are visible here; a task that defers its
     model to the route resolves it inside the agent and can't be pre-flighted.
-    Those still get counted after the fact by ``RunSummary.tasks_unpriced``.
 
     Args:
         resolved_tasks: The fully-resolved tasks about to run.
-        strict: Raise instead of warning when any model is unpriced.
 
     Returns:
         The sorted, de-duplicated unpriced model ids (empty when all are priced).
-
-    Raises:
-        ValueError: Under ``strict`` when at least one model is unpriced.
     """
     missing = unpriced_models(rt.task.agent.model if rt.task.agent else None for rt in resolved_tasks)
     if not missing:
         return []
-    detail = (
-        f"No pricing rate for {', '.join(repr(m) for m in missing)}. Cost for these tasks "
-        + "will be recorded as null and every run-level total will understate the bill "
-        + "(RunSummary.cost_complete reports false). Add the rate to coder_eval.pricing "
-        + "or register it from a plugin via register_pricing()."
+    logger.warning(
+        "No pricing rate for %s. Cost for these tasks will be recorded as null and every "
+        + "run-level total will understate the bill (RunSummary.cost_complete reports false). "
+        + "Add the rate to coder_eval.pricing or register it from a plugin via register_pricing().",
+        ", ".join(repr(m) for m in missing),
     )
-    if strict:
-        raise ValueError(f"strict_pricing: refusing to start. {detail}")
-    logger.warning(detail)
     return missing
 
 
@@ -115,7 +111,7 @@ async def run_batch(
 
     start_time = datetime.now()
 
-    check_pricing_coverage(resolved_tasks, strict=config.strict_pricing)
+    check_pricing_coverage(resolved_tasks)
 
     if on_batch_start is not None:
         on_batch_start(len(resolved_tasks))
diff --git a/src/coder_eval/orchestration/config.py b/src/coder_eval/orchestration/config.py
index 19c0e378..695b5594 100644
--- a/src/coder_eval/orchestration/config.py
+++ b/src/coder_eval/orchestration/config.py
@@ -97,16 +97,6 @@ class BatchRunConfig(BaseModel):
     # Logging
     verbose: bool = Field(default=False, description="Enable verbose (DEBUG level) logging for Docker output")
 
-    strict_pricing: bool = Field(
-        default=False,
-        description=(
-            "Refuse to start when a task's model is absent from the pricing rate card, "
-            "instead of warning and running with that model's spend recorded as null. "
-            "Off by default so a brand-new model is still evaluable; on for cost-bearing "
-            "runs (the nightly), where an unpriced model silently understates the bill."
-        ),
-    )
-
     # TODO(container-death-diagnostics): consider a run-level default resource
     # cap. Containers run uncapped today (sandbox.limits.{max_memory_mb,
     # max_cpus,max_pids} default to None -> _build_argv emits no --memory/
diff --git a/src/coder_eval/orchestrator.py b/src/coder_eval/orchestrator.py
index 1b97673b..f5a5206e 100644
--- a/src/coder_eval/orchestrator.py
+++ b/src/coder_eval/orchestrator.py
@@ -56,7 +56,6 @@
 from .orchestration.early_stop import EarlyStopWatcher, validate_early_stop
 from .orchestration.evaluation import load_reference
 from .path_utils import format_task_log_id, task_log_path
-from .pricing import calculate_cost
 from .sandbox import Sandbox
 from .simulation import DialogStopReason, SimulatorResult, UserSimulator, evaluate_stop
 from .streaming.callbacks import CompositeStreamCallback, StreamCallback, TaskScopedCallback, safe_emit
@@ -661,20 +660,17 @@ def _finalize_result(self, start_time: float) -> None:
         if not self.result.model_used and self.task.agent is not None and self.task.agent.model:
             self.result.model_used = self.task.agent.model
 
+        # Aggregate token usage
+        self._aggregate_token_usage()
+
         # Record whether per-turn cost data was available when a cost budget was set.
         # Lets users audit whether a configured max_usd budget was actually enforceable.
-        # Computed BEFORE the aggregate, because _aggregate_token_usage backfills
-        # missing per-turn cost from the rate card: enforceability depends on the cost
-        # the SDK reported live, during the run, not on what we could price afterwards.
         if self.task.run_limits is not None and self.task.run_limits.max_usd is not None:
             any_cost_reported = any(
                 t.token_usage is not None and t.token_usage.total_cost_usd is not None for t in self.result.iterations
             )
             self.result.environment_info["cost_data_available"] = any_cost_reported
 
-        # Aggregate token usage
-        self._aggregate_token_usage()
-
         if self.result.iterations:
             self.result.total_assistant_turns = sum(t.assistant_turn_count for t in self.result.iterations)
 
@@ -856,42 +852,6 @@ def _check_expected_turns(self, *, iteration: int) -> None:
             )
             self._expected_turns_warning_emitted = True
 
-    def _backfill_turn_costs(self) -> None:
-        """Price any turn that recorded tokens but no cost, from the rate card.
-
-        Makes "tokens on the record imply a cost on the record" an invariant at one
-        agent-agnostic seam, instead of a convention each agent has to remember.
-
-        Every in-tree agent already prices its own turns, including the killed
-        partials an error or timeout produces (``ClaudeCodeAgent._backfill_cost``;
-        codex and antigravity compute from buckets and never depend on an SDK
-        cost), so this is a net rather than a fix for a live gap — verified by
-        A/B on a real timed-out Bedrock run, which booked its spend with this
-        disabled. It earns its keep for the plugin SPI: an out-of-tree agent that
-        registers pricing but doesn't apply it would otherwise lose 100% of its
-        cost silently, with only its token counts to show anything happened.
-
-        Only fills in what is missing: a turn the SDK priced keeps the SDK's
-        number, which is the authoritative billed figure. A model absent from the
-        rate card stays unpriced and is counted by ``RunSummary.tasks_unpriced``
-        rather than being passed off as free.
-        """
-        assert self.result is not None
-        for turn in self.result.iterations:
-            usage = turn.token_usage
-            if usage is None or usage.total_cost_usd is not None or usage.is_empty():
-                continue
-            model = turn.model_used or self.result.model_used
-            if not model:
-                continue
-            usage.total_cost_usd = calculate_cost(
-                model,
-                uncached_input_tokens=usage.uncached_input_tokens,
-                output_tokens=usage.output_tokens,
-                cache_creation_tokens=usage.cache_creation_input_tokens,
-                cache_read_tokens=usage.cache_read_input_tokens,
-            )
-
     def _aggregate_token_usage(self) -> None:
         """Aggregate token usage from turns, storing on self.result.
 
@@ -903,14 +863,9 @@ def _aggregate_token_usage(self) -> None:
         the corresponding ``JudgeCriterionResult.token_usage`` and is
         intentionally NOT included in this aggregate — it represents the
         main agent's bill, not the eval-machinery overhead.
-
-        Runs ``_backfill_turn_costs`` first so an error/timeout partial
-        contributes its cost instead of being summed as free.
         """
         assert self.result is not None
 
-        self._backfill_turn_costs()
-
         # Include crashed=True partials: each API call is billed independently.
         if self.result.iterations:
             usages = [t.token_usage for t in self.result.iterations if t.token_usage is not None]
diff --git a/tests/test_cost_accounting_paths.py b/tests/test_cost_accounting_paths.py
index 3df2a1e9..f4456880 100644
--- a/tests/test_cost_accounting_paths.py
+++ b/tests/test_cost_accounting_paths.py
@@ -1,17 +1,13 @@
 """Cost accounting on the error and timeout paths, where spend went missing.
 
-Three seams:
+Two seams:
 
 1. **The pre-flight** (``check_pricing_coverage``) — the measured loss. The rate
    card is a static table baked into the installed version, so a model released
    after it prices every turn as ``null``. Sonnet 5's rates landed one release
    after the 2026-07-21 nightly, which recorded $209.81 (18.9% of its true bill)
    across 62 errored rows as no cost at all, with one log line to show for it.
-2. **The per-turn backfill** (``Orchestrator._backfill_turn_costs``) — a net, not a
-   fix for a live gap: every in-tree agent already prices its own killed partials.
-   It makes "tokens on the record imply a cost on the record" an invariant at one
-   seam, so a plugin agent that doesn't price can't silently lose all of its spend.
-3. **The row projection** (``eval_result_to_task_dict``) — judge and simulator
+2. **The row projection** (``eval_result_to_task_dict``) — judge and simulator
    spend was captured and rolled up nowhere, and a row whose turns were only
    partly priced reported its partial sum as the whole.
 """
@@ -95,89 +91,9 @@ def test_unpriced_model_warns_but_runs(self, tmp_path, caplog):
         assert "No pricing rate" in caplog.text
         assert "understate the bill" in caplog.text
 
-    def test_strict_refuses_to_start(self, tmp_path):
-        """The cost-bearing-run setting: fail at dispatch, not in the cost column."""
-        with pytest.raises(ValueError, match="strict_pricing"):
-            check_pricing_coverage([_resolved("claude-sonnet-99", tmp_path)], strict=True)
-
     def test_unpinned_model_is_not_flagged(self, tmp_path):
         """A task deferring its model to the route can't be pre-flighted from here."""
-        assert check_pricing_coverage([_resolved(None, tmp_path)], strict=True) == []
-
-
-class TestTurnCostBackfill:
-    """``_backfill_turn_costs`` is exercised through a bare Orchestrator instance.
-
-    Built with ``__new__`` rather than a full construction because the method reads
-    only ``self.result`` — a real Orchestrator needs a sandbox, an agent, and a
-    route, none of which participate in pricing.
-    """
-
-    @staticmethod
-    def _backfill(result: EvaluationResult) -> None:
-        from coder_eval.orchestrator import Orchestrator
-
-        orch = Orchestrator.__new__(Orchestrator)
-        orch.result = result
-        orch._backfill_turn_costs()
-
-    def test_prices_a_killed_turn_from_the_rate_card(self):
-        """The timeout shape: real tokens on the wire, no cost attached yet."""
-        result = self._result_with_partial()
-        self._backfill(result)
-
-        priced = result.iterations[1].token_usage
-        assert priced is not None and priced.total_cost_usd is not None
-        # 1M uncached input + 100k output on sonnet-5 ($3/$15 per MTok).
-        assert priced.total_cost_usd == pytest.approx(3.0 + 1.5)
-
-    def test_leaves_an_sdk_reported_cost_alone(self):
-        """The SDK's number is the authoritative billed figure; never overwrite it."""
-        result = self._result_with_partial()
-        self._backfill(result)
-        assert result.iterations[0].token_usage.total_cost_usd == pytest.approx(0.99)
-
-    def test_falls_back_to_the_row_model(self):
-        """A partial that never learned its model still prices off the resolved one."""
-        result = _result(
-            [_turn(1, TokenUsage(uncached_input_tokens=1_000_000, output_tokens=0), model=None, crashed=True)],
-            model="claude-sonnet-5",
-        )
-        self._backfill(result)
-        assert result.iterations[0].token_usage.total_cost_usd == pytest.approx(3.0)
-
-    def test_unpriceable_model_stays_unpriced(self):
-        """Not a guess: leave it null so tasks_unpriced can count it."""
-        result = _result(
-            [_turn(1, TokenUsage(uncached_input_tokens=1_000_000, output_tokens=0), model="made-up-model")],
-            model="made-up-model",
-        )
-        self._backfill(result)
-        assert result.iterations[0].token_usage.total_cost_usd is None
-
-    def test_empty_usage_is_not_priced(self):
-        """A turn that burned nothing must not acquire a $0.00 cost it never had."""
-        result = _result([_turn(1, TokenUsage())])
-        self._backfill(result)
-        assert result.iterations[0].token_usage.total_cost_usd is None
-
-    @staticmethod
-    def _result_with_partial() -> EvaluationResult:
-        return _result(
-            [
-                _turn(
-                    1,
-                    TokenUsage(uncached_input_tokens=1000, output_tokens=100, total_cost_usd=0.99),
-                    model="claude-sonnet-5",
-                ),
-                _turn(
-                    2,
-                    TokenUsage(uncached_input_tokens=1_000_000, output_tokens=100_000),
-                    model="claude-sonnet-5",
-                    crashed=True,
-                ),
-            ]
-        )
+        assert check_pricing_coverage([_resolved(None, tmp_path)]) == []
 
 
 class TestRowCostProjection:

From b0421e649b48b8e8e89013bfeb6452a0b412c6c8 Mon Sep 17 00:00:00 2001
From: Bai Li <bai.li@uipath.com>
Date: Tue, 28 Jul 2026 19:04:53 -0700
Subject: [PATCH 06/20] refactor(cost): define the unpriced-row test once, and
 only for new runs

Two definitions of "which rows lost money" had appeared: RunSummary.tasks_unpriced
computed one, and the report's token section re-derived the same predicate inline.
That is the duplication this PR exists to remove, reintroduced one layer down. Both
now call row_cost_incomplete / eval_overhead_costs, defined on the row schema where
the reports (which have task dicts, not a RunSummary) can reach them.

The predicate also gets simpler: read the row's cost_complete flag, and treat its
absence as complete. It previously fell back to inferring unpriced-ness from
"burned tokens but carries no cost", which only ever mattered for runs written
before the field existed. Every new run sets the flag, so the fallback bought a
caveat on historical runs at the cost of a second definition living in TypeScript
and drifting from the Python one. Old runs now render exactly as they did before.

Unpriced stays reachable, so the field is not going away: any model released after
the installed framework version prices as null, and the dispatch pre-flight only
sees pinned agent.model values, so a run pointed at a new model through the route
gets no warning at all. tasks_unpriced is the only thing that catches that.

Also drops a stale reference to the reverted turn-cost backfill, and one to the
deleted framework/harness error split.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 evalboard/lib/__tests__/runs.test.ts | 14 ++----
 evalboard/lib/runs.ts                | 24 ++++------
 src/coder_eval/models/__init__.py    |  6 +++
 src/coder_eval/models/results.py     | 67 ++++++++++++++++++----------
 src/coder_eval/reports.py            | 17 ++++---
 src/coder_eval/reports_experiment.py |  4 +-
 tests/test_token_usage.py            | 17 +++++++
 7 files changed, 88 insertions(+), 61 deletions(-)

diff --git a/evalboard/lib/__tests__/runs.test.ts b/evalboard/lib/__tests__/runs.test.ts
index f27c4887..755a9ed7 100644
--- a/evalboard/lib/__tests__/runs.test.ts
+++ b/evalboard/lib/__tests__/runs.test.ts
@@ -59,16 +59,10 @@ describe("toTaskRow", () => {
         expect(row.simulatorCostUsd).toBeCloseTo(0.01);
     });
 
-    test("infers incomplete cost on a run predating the field", () => {
-        // Tokens on the wire with no cost is the unpriced shape, whether or not
-        // the row carries the explicit flag.
-        const row = toTaskRow({ task_id: "x", total_tokens: 5_000_000 });
-        expect(row.costComplete).toBe(false);
-    });
-
-    test("a row that burned nothing is complete, not unpriced", () => {
-        // An error before the agent ran genuinely cost nothing.
-        const row = toTaskRow({ task_id: "x", total_tokens: 0 });
+    test("a row predating the field reads as complete", () => {
+        // Deliberately not inferred from tokens: an old run renders exactly as it
+        // did before, and "unpriced" keeps one definition instead of two.
+        const row = toTaskRow({ task_id: "x" });
         expect(row.costComplete).toBe(true);
     });
 
diff --git a/evalboard/lib/runs.ts b/evalboard/lib/runs.ts
index fef14e0c..54ec4d6d 100644
--- a/evalboard/lib/runs.ts
+++ b/evalboard/lib/runs.ts
@@ -348,9 +348,6 @@ interface RawTaskResult {
     weighted_score?: number;
     duration?: number;
     total_cost_usd?: number;
-    // Row-level token total. Present alongside the disjoint buckets below; used to
-    // tell "burned tokens but has no cost" (unpriced) from "burned nothing" (free).
-    total_tokens?: number | null;
     input_tokens?: number | null;
     output_tokens?: number | null;
     cache_creation_input_tokens?: number | null;
@@ -688,11 +685,11 @@ export function toTaskRow(t: RawTaskResult): TaskResultSummary {
         tags,
         skill: deriveSkill(t.task_path, tags),
         matureSkipped: t.mature_skipped ?? false,
-        // A row predating the field is assumed complete unless it visibly burned
-        // tokens with no cost — the same test the Python side applies.
-        costComplete:
-            t.cost_complete ??
-            !((t.total_tokens ?? 0) > 0 && t.total_cost_usd == null),
+        // Absent means complete: a row written before the field existed is read as
+        // priced rather than inferred from its tokens. Guessing at the past would
+        // buy a caveat on old runs at the cost of a second definition of
+        // "unpriced" living here, out of step with the Python one.
+        costComplete: t.cost_complete ?? true,
         judgeCostUsd: t.judge_cost_usd ?? null,
         simulatorCostUsd: t.simulator_cost_usd ?? null,
         errorMessage: t.error_message ?? null,
@@ -727,11 +724,8 @@ export async function readRunSummary(id: string): Promise<RunSummary | null> {
     // A row whose tokens went unpriced contributes 0 to totalCost above, so the
     // sum silently understates the bill. Count those rows and say so rather than
     // presenting a floor as the total.
-    const tasksUnpriced = taskResults.filter(
-        (t) =>
-            ((t.total_tokens ?? 0) > 0 && t.total_cost_usd == null) ||
-            t.cost_complete === false,
-    ).length;
+    const tasksUnpriced = taskResults.filter((t) => t.cost_complete === false)
+        .length;
     const overhead = taskResults.reduce(
         (a, t) => a + (t.judge_cost_usd ?? 0) + (t.simulator_cost_usd ?? 0),
         0,
@@ -989,9 +983,7 @@ export async function readRunOverview(
                 visibleTurns: visibleTurnsFromRaw(t),
                 hasFinalReply: t.has_final_reply ?? false,
                 matureSkipped: t.mature_skipped ?? false,
-                costComplete:
-                    t.cost_complete ??
-                    !((t.total_tokens ?? 0) > 0 && t.total_cost_usd == null),
+                costComplete: t.cost_complete ?? true,
             };
         });
     const totalCost = taskResults.reduce(
diff --git a/src/coder_eval/models/__init__.py b/src/coder_eval/models/__init__.py
index 3a9b3796..7e6157c0 100644
--- a/src/coder_eval/models/__init__.py
+++ b/src/coder_eval/models/__init__.py
@@ -127,6 +127,8 @@
     TaskConfigRecord,
     ThresholdCheck,
     TurnRecord,
+    eval_overhead_costs,
+    row_cost_incomplete,
 )
 
 # Routing
@@ -291,6 +293,10 @@
     "TaskConfigRecord",
     "RunSummary",
     "SkippedTask",
+    # Row-level cost predicates, shared by RunSummary's computed fields and the
+    # reports so every surface agrees on which rows lost money.
+    "row_cost_incomplete",
+    "eval_overhead_costs",
     # Judge defaults
     "DEFAULT_JUDGE_MODEL",
     # Judge
diff --git a/src/coder_eval/models/results.py b/src/coder_eval/models/results.py
index 61d7fb7b..3384830c 100644
--- a/src/coder_eval/models/results.py
+++ b/src/coder_eval/models/results.py
@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+from collections.abc import Iterable, Mapping
 from datetime import datetime
 from enum import StrEnum
 from typing import Annotated, Any, Literal
@@ -846,6 +847,35 @@ class SkippedTask(BaseModel):
     )
 
 
+def row_cost_incomplete(row: Mapping[str, Any]) -> bool:
+    """True when a task row's recorded spend is missing money.
+
+    Reads the row's own ``cost_complete``, which the row projection sets by asking
+    whether every turn that burned tokens also carries a cost. A row that burned
+    nothing is complete, not unpriced: an error before the agent ran genuinely cost
+    nothing.
+
+    Absent means complete. Rows written before the field existed are read as
+    priced rather than inferred from their token counts: guessing at the past would
+    buy a caveat on old runs at the cost of a second definition of "unpriced",
+    which is the kind of drift this helper exists to prevent.
+
+    Defined once here, on the row schema, because every surface that reports cost
+    has to apply the same test. Reports and ``RunSummary`` disagreeing about which
+    rows lost money would put the framework back where it started.
+    """
+    return row.get("cost_complete") is False
+
+
+def eval_overhead_costs(rows: Iterable[Mapping[str, Any]]) -> list[float]:
+    """Every judge / simulator cost present across ``rows``, unsummed.
+
+    Returned as a list rather than a total because callers need to distinguish
+    "no overhead was recorded" from "overhead was recorded and came to $0".
+    """
+    return [c for row in rows for key in ("judge_cost_usd", "simulator_cost_usd") if (c := row.get(key)) is not None]
+
+
 class RunSummary(BaseModel):
     """Summary of an entire evaluation run across multiple tasks.
 
@@ -955,8 +985,10 @@ def error_share(self) -> float | None:
         """``tasks_error / tasks_run`` as a 0-1 fraction. ``None`` on an empty run.
 
         How much of ``pass_rate`` is being held down by rows that never produced a
-        gradeable attempt. Diagnostic: read it with the framework/harness split to
-        tell an infrastructure night from a genuinely weak model.
+        gradeable attempt. Diagnostic only: it does not adjust the rate. A run whose
+        rate dropped with a high ``error_share`` had an infrastructure night; the same
+        drop at a normal share is the model. Per-row ``error_message`` /
+        ``error_category`` say which.
         """
         return self.tasks_error / self.tasks_run if self.tasks_run else None
 
@@ -965,22 +997,13 @@ def error_share(self) -> float | None:
     def tasks_unpriced(self) -> int:
         """Rows whose recorded spend is incomplete.
 
-        Each one is real money missing from ``agent_cost_usd``. Two ways in: the
-        row burned tokens and got no cost at all, or its per-row ``cost_complete``
-        says some turn within it went unpriced. Both come down to a model the rate
-        card cannot price — the card is a static table baked into the installed
-        version, so a model released after it prices as ``null`` on every turn,
-        with only a log line to say so.
-
-        A row that burned nothing does not count: an error before the agent ran
-        genuinely cost nothing.
+        Each one is real money missing from ``agent_cost_usd``, because the rate
+        card is a static table baked into the installed version: a model released
+        after it prices as ``null`` on every turn, with only a log line to say so.
+        ``row_cost_incomplete`` defines what counts, and the report applies the
+        same helper so the two can never disagree.
         """
-        return sum(
-            1
-            for row in self.task_results
-            if ((row.get("total_tokens") or 0) > 0 and row.get("total_cost_usd") is None)
-            or row.get("cost_complete") is False
-        )
+        return sum(1 for row in self.task_results if row_cost_incomplete(row))
 
     @computed_field  # type: ignore[prop-decorator]
     @property
@@ -993,7 +1016,8 @@ def cost_complete(self) -> bool:
     def agent_cost_usd(self) -> float | None:
         """Subject-agent spend across the run. ``None`` when no row reported cost.
 
-        Excludes evaluation machinery — see ``eval_overhead_cost_usd``. Read
+        Excludes evaluation machinery: that is ``eval_overhead_cost_usd``, and the
+        two are published as a pair because a run's true bill is their sum. Read
         alongside ``cost_complete``: with unpriced rows present this is a floor,
         not the bill.
         """
@@ -1011,10 +1035,5 @@ def eval_overhead_cost_usd(self) -> float | None:
         and was previously reported nowhere, so a run's true bill is the two
         added together.
         """
-        costs = [
-            c
-            for row in self.task_results
-            for key in ("judge_cost_usd", "simulator_cost_usd")
-            if (c := row.get(key)) is not None
-        ]
+        costs = eval_overhead_costs(self.task_results)
         return sum(costs) if costs else None
diff --git a/src/coder_eval/reports.py b/src/coder_eval/reports.py
index d60e5be7..11ed12fb 100644
--- a/src/coder_eval/reports.py
+++ b/src/coder_eval/reports.py
@@ -15,6 +15,8 @@
     SuiteRollup,
     TaskResult,
     ThresholdCheck,
+    eval_overhead_costs,
+    row_cost_incomplete,
 )
 from .path_utils import build_task_run_dir
 
@@ -553,15 +555,12 @@ def _generate_token_usage_section(task_results: list[dict[str, Any]]) -> list[st
 
         # Rows whose spend is only partly priced. Reported explicitly because the
         # alternative is what used to happen: a run understating its bill by 19%
-        # with nothing on the report to say the number was incomplete.
-        unpriced = [
-            t
-            for t in task_results
-            if ((t.get("total_tokens") or 0) > 0 and t.get("total_cost_usd") is None) or t.get("cost_complete") is False
-        ]
-        overhead = [
-            c for t in task_results for key in ("judge_cost_usd", "simulator_cost_usd") if (c := t.get(key)) is not None
-        ]
+        # with nothing on the report to say the number was incomplete. Both of
+        # these come from the same helpers RunSummary.tasks_unpriced /
+        # eval_overhead_cost_usd use, so the report and run.json cannot disagree
+        # about which rows lost money.
+        unpriced = [t for t in task_results if row_cost_incomplete(t)]
+        overhead = eval_overhead_costs(task_results)
 
         lines.append(f"**Total Tokens**: {total_tokens:,} (input: {total_input:,}, output: {total_output:,})")
         if total_cache_write > 0 or total_cache_read > 0:
diff --git a/src/coder_eval/reports_experiment.py b/src/coder_eval/reports_experiment.py
index 6ff0f394..6dc61fa1 100644
--- a/src/coder_eval/reports_experiment.py
+++ b/src/coder_eval/reports_experiment.py
@@ -48,8 +48,8 @@ def _cost_complete(result: EvaluationResult) -> bool:
     """Whether every turn that burned tokens on this row also carries a cost.
 
     False means ``total_cost_usd`` is a floor, not the bill: some turn's spend is
-    missing from it. After the orchestrator's rate-card backfill the only way to
-    land here is a model the rate card cannot price at all, so this is the
+    missing from it. Every in-tree agent prices its own turns, so in practice the
+    way to land here is a model the rate card cannot price at all. This is the
     per-row signal that feeds ``RunSummary.tasks_unpriced``.
 
     True for a row that burned nothing (an error before the agent ran genuinely
diff --git a/tests/test_token_usage.py b/tests/test_token_usage.py
index ccb15fae..eee2c00c 100644
--- a/tests/test_token_usage.py
+++ b/tests/test_token_usage.py
@@ -875,6 +875,7 @@ def test_token_section_handles_missing_cost(self):
                 "output_tokens": 2000,
                 "total_tokens": 5000,
                 "total_cost_usd": None,
+                "cost_complete": False,
             },
         ]
         lines = ReportGenerator._generate_token_usage_section(task_results)
@@ -888,6 +889,22 @@ def test_token_section_handles_missing_cost(self):
         assert "1 task(s) burned tokens the rate card could not price" in joined
         assert "| task1 | 3,000 | 2,000 | 0 | 0 | 5,000 | N/A |" in joined
 
+    def test_token_section_leaves_a_legacy_row_uncaveated(self):
+        """A row predating ``cost_complete`` is read as priced, not inferred.
+
+        The same shape as above minus the flag, which is what runs written before
+        the field look like. Deliberately silent: inferring unpriced-ness from
+        tokens would give old runs a caveat at the price of a second definition of
+        "unpriced", and every new run carries the flag.
+        """
+        task_results = [
+            {"task_id": "task1", "total_tokens": 5000, "total_cost_usd": None},
+        ]
+        joined = "\n".join(ReportGenerator._generate_token_usage_section(task_results))
+
+        assert "## Token Usage" in joined
+        assert "could not price" not in joined
+
     def test_token_section_omits_cost_when_no_tokens_burned(self):
         """A row that burned nothing is genuinely free — no unpriced warning."""
         task_results = [

From a89e673cd219426d9cac31e35f3fdc739fe08403 Mon Sep 17 00:00:00 2001
From: Bai Li <bai.li@uipath.com>
Date: Tue, 28 Jul 2026 19:05:03 -0700
Subject: [PATCH 07/20] feat(evalboard): mark a partly-priced run total as a
 floor, not the bill
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The run page's Total cost tile summed the priced tasks and rendered the result with
a p50/p90 subtitle under it. An unpriced task contributes $0 to that sum, so the
2026-07-21 nightly displayed $902.81 as though measured, with $209.81 missing and
percentiles describing only the tasks that happened to be priced.

It now reads ≥$902.81 with "floor · 62 tasks unpriced" in place of the percentiles,
rather than beside them: a number presented as exact is worse than one presented as
a bound. The count is filter-aware and skips mature-skipped tasks, matching how the
tile already scopes cost itself.

This was the surface that mattered. The overview tile already flagged the run, so
the flow a person actually takes — see something off, click the run to find out
what — landed on a page that denied it.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 evalboard/app/runs/[id]/run-view.tsx | 25 +++++++++++++++++++++----
 1 file changed, 21 insertions(+), 4 deletions(-)

diff --git a/evalboard/app/runs/[id]/run-view.tsx b/evalboard/app/runs/[id]/run-view.tsx
index 160e1303..90dcad33 100644
--- a/evalboard/app/runs/[id]/run-view.tsx
+++ b/evalboard/app/runs/[id]/run-view.tsx
@@ -44,6 +44,10 @@ export interface RunMetrics {
     // repeated runs.
     taskFailed: number;
     cost: number | null;
+    // Executed tasks whose spend is missing money, so `cost` is a floor. A null
+    // cost sums as $0, so without this an unpriced task is indistinguishable from
+    // a free one and the tile presents a floor as the bill.
+    costUnpriced: number;
     costP50: number | null;
     costP90: number | null;
     duration: number | null;
@@ -66,6 +70,7 @@ export function computeRunMetrics(tasks: TaskResultSummary[]): RunMetrics {
     let failed = 0;
     let errored = 0;
     let cost = 0;
+    let costUnpriced = 0;
     let durationSum = 0;
     const costSamples: number[] = [];
     const durSamples: number[] = [];
@@ -75,6 +80,9 @@ export function computeRunMetrics(tasks: TaskResultSummary[]): RunMetrics {
         else if (cat === "error") errored++;
         else failed++;
         if (t.matureSkipped) continue;
+        // Counted after the mature-skip guard, alongside cost itself: a task that
+        // never executed has no spend to be missing.
+        if (t.costComplete === false) costUnpriced++;
         if (t.totalCostUsd != null) {
             cost += t.totalCostUsd;
             costSamples.push(t.totalCostUsd);
@@ -103,6 +111,7 @@ export function computeRunMetrics(tasks: TaskResultSummary[]): RunMetrics {
             };
         })(),
         cost: costSamples.length ? cost : null,
+        costUnpriced,
         costP50: percentile(costSamples, 0.5),
         costP90: percentile(costSamples, 0.9),
         duration: durSamples.length ? durationSum : null,
@@ -459,17 +468,25 @@ export function RunView({
                 {activation && (
                     <ActivationCard runId={runId} activation={activation} />
                 )}
+                {/* An unpriced task sums as $0, so the total is a floor and the
+                    percentiles describe only the priced tasks. Say so in place of
+                    the percentiles rather than beside them: a number presented as
+                    exact is worse than one presented as a bound. */}
                 <Metric
                     label="Total cost"
                     value={
                         metrics.cost != null
-                            ? `$${metrics.cost.toFixed(2)}`
+                            ? `${metrics.costUnpriced ? "≥" : ""}$${metrics.cost.toFixed(2)}`
                             : "—"
                     }
                     sub={
-                        metrics.costP50 != null && metrics.costP90 != null
-                            ? `p50 $${metrics.costP50.toFixed(2)} · p90 $${metrics.costP90.toFixed(2)}`
-                            : undefined
+                        metrics.costUnpriced
+                            ? `floor · ${metrics.costUnpriced} task${
+                                  metrics.costUnpriced === 1 ? "" : "s"
+                              } unpriced`
+                            : metrics.costP50 != null && metrics.costP90 != null
+                              ? `p50 $${metrics.costP50.toFixed(2)} · p90 $${metrics.costP90.toFixed(2)}`
+                              : undefined
                     }
                 />
                 <Metric

From 955a524bec93254ab32d81d3db0c191ff832778c Mon Sep 17 00:00:00 2001
From: Bai Li <bai.li@uipath.com>
Date: Tue, 28 Jul 2026 19:10:41 -0700
Subject: [PATCH 08/20] revert(evalboard): drop the unpriced-cost surface
 entirely
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The dashboard now reads exactly as it did before this PR: `git diff main -- evalboard/`
is empty.

Marking a total as `≥$902.81 · floor · 62 tasks unpriced` is hedging in a place that
should just show a number. A dashboard that qualifies its own figures trains people
to distrust the unqualified ones, and it is worse to read than a number that is
quietly a little low. The run.json fields stay, so anyone who wants the caveat can
compute it; the markdown report still labels a partly-priced total as a floor, which
is a written artifact where a caveat belongs.

This also retires eight declared-but-unread fields, one of which duplicated an
existing `errorMessage` already rendered on the task-detail page from task.json.

The pass rate needed nothing here: lib/trends.ts already counted every row in its
denominator, so the dashboard was correct before this PR and is correct after. Only
the markdown report and the downstream runner were inflating.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 evalboard/app/runs/[id]/run-view.tsx     | 25 ++-----
 evalboard/lib/__tests__/overview.test.ts | 21 ------
 evalboard/lib/__tests__/runs.test.ts     | 32 ---------
 evalboard/lib/overview.ts                | 17 +----
 evalboard/lib/runs.ts                    | 83 +-----------------------
 5 files changed, 8 insertions(+), 170 deletions(-)

diff --git a/evalboard/app/runs/[id]/run-view.tsx b/evalboard/app/runs/[id]/run-view.tsx
index 90dcad33..160e1303 100644
--- a/evalboard/app/runs/[id]/run-view.tsx
+++ b/evalboard/app/runs/[id]/run-view.tsx
@@ -44,10 +44,6 @@ export interface RunMetrics {
     // repeated runs.
     taskFailed: number;
     cost: number | null;
-    // Executed tasks whose spend is missing money, so `cost` is a floor. A null
-    // cost sums as $0, so without this an unpriced task is indistinguishable from
-    // a free one and the tile presents a floor as the bill.
-    costUnpriced: number;
     costP50: number | null;
     costP90: number | null;
     duration: number | null;
@@ -70,7 +66,6 @@ export function computeRunMetrics(tasks: TaskResultSummary[]): RunMetrics {
     let failed = 0;
     let errored = 0;
     let cost = 0;
-    let costUnpriced = 0;
     let durationSum = 0;
     const costSamples: number[] = [];
     const durSamples: number[] = [];
@@ -80,9 +75,6 @@ export function computeRunMetrics(tasks: TaskResultSummary[]): RunMetrics {
         else if (cat === "error") errored++;
         else failed++;
         if (t.matureSkipped) continue;
-        // Counted after the mature-skip guard, alongside cost itself: a task that
-        // never executed has no spend to be missing.
-        if (t.costComplete === false) costUnpriced++;
         if (t.totalCostUsd != null) {
             cost += t.totalCostUsd;
             costSamples.push(t.totalCostUsd);
@@ -111,7 +103,6 @@ export function computeRunMetrics(tasks: TaskResultSummary[]): RunMetrics {
             };
         })(),
         cost: costSamples.length ? cost : null,
-        costUnpriced,
         costP50: percentile(costSamples, 0.5),
         costP90: percentile(costSamples, 0.9),
         duration: durSamples.length ? durationSum : null,
@@ -468,25 +459,17 @@ export function RunView({
                 {activation && (
                     <ActivationCard runId={runId} activation={activation} />
                 )}
-                {/* An unpriced task sums as $0, so the total is a floor and the
-                    percentiles describe only the priced tasks. Say so in place of
-                    the percentiles rather than beside them: a number presented as
-                    exact is worse than one presented as a bound. */}
                 <Metric
                     label="Total cost"
                     value={
                         metrics.cost != null
-                            ? `${metrics.costUnpriced ? "≥" : ""}$${metrics.cost.toFixed(2)}`
+                            ? `$${metrics.cost.toFixed(2)}`
                             : "—"
                     }
                     sub={
-                        metrics.costUnpriced
-                            ? `floor · ${metrics.costUnpriced} task${
-                                  metrics.costUnpriced === 1 ? "" : "s"
-                              } unpriced`
-                            : metrics.costP50 != null && metrics.costP90 != null
-                              ? `p50 $${metrics.costP50.toFixed(2)} · p90 $${metrics.costP90.toFixed(2)}`
-                              : undefined
+                        metrics.costP50 != null && metrics.costP90 != null
+                            ? `p50 $${metrics.costP50.toFixed(2)} · p90 $${metrics.costP90.toFixed(2)}`
+                            : undefined
                     }
                 />
                 <Metric
diff --git a/evalboard/lib/__tests__/overview.test.ts b/evalboard/lib/__tests__/overview.test.ts
index 0dcd2476..eaa935d1 100644
--- a/evalboard/lib/__tests__/overview.test.ts
+++ b/evalboard/lib/__tests__/overview.test.ts
@@ -97,27 +97,6 @@ describe("summarizeListing", () => {
         expect(t.durationSeconds).toBeNull();
         expect(t.durationPartial).toBe(false);
     });
-
-    test("flags partial when a run's own cost is incomplete", () => {
-        // Every run reported a cost, so the old costRuns check is satisfied — but
-        // one of them under-counted because some task's tokens went unpriced. The
-        // 2026-07-21 nightly was exactly this: $902.81 reported, $209.81 missing
-        // (18.9%), and nothing anywhere said the total was a floor.
-        const t = summarizeListing([
-            row({ totalCostUsd: 902.81, costComplete: false }),
-            row({ totalCostUsd: 100, costComplete: true }),
-        ]);
-        expect(t.costUsd).toBeCloseTo(1002.81);
-        expect(t.costPartial).toBe(true);
-    });
-
-    test("complete costs across every run are not flagged", () => {
-        const t = summarizeListing([
-            row({ totalCostUsd: 1, costComplete: true }),
-            row({ totalCostUsd: 2, costComplete: true }),
-        ]);
-        expect(t.costPartial).toBe(false);
-    });
 });
 
 describe("turnBudgetRateForTasks", () => {
diff --git a/evalboard/lib/__tests__/runs.test.ts b/evalboard/lib/__tests__/runs.test.ts
index 755a9ed7..5fd0c719 100644
--- a/evalboard/lib/__tests__/runs.test.ts
+++ b/evalboard/lib/__tests__/runs.test.ts
@@ -46,38 +46,6 @@ describe("toTaskRow", () => {
         const row = toTaskRow({ task_id: "x", expected_turns: null });
         expect(row.expectedTurns).toBeNull();
     });
-
-    test("reads cost_complete and the eval-overhead costs", () => {
-        const row = toTaskRow({
-            task_id: "x",
-            cost_complete: false,
-            judge_cost_usd: 0.02,
-            simulator_cost_usd: 0.01,
-        });
-        expect(row.costComplete).toBe(false);
-        expect(row.judgeCostUsd).toBeCloseTo(0.02);
-        expect(row.simulatorCostUsd).toBeCloseTo(0.01);
-    });
-
-    test("a row predating the field reads as complete", () => {
-        // Deliberately not inferred from tokens: an old run renders exactly as it
-        // did before, and "unpriced" keeps one definition instead of two.
-        const row = toTaskRow({ task_id: "x" });
-        expect(row.costComplete).toBe(true);
-    });
-
-    test("carries the error reason for an errored row", () => {
-        // Errors count as misses in the pass rate, so the dashboard has to be
-        // able to say where the points went.
-        const row = toTaskRow({
-            task_id: "x",
-            status: "ERROR",
-            error_message: "no space left on device",
-            error_category: "disk_full",
-        });
-        expect(row.errorMessage).toBe("no space left on device");
-        expect(row.errorCategory).toBe("disk_full");
-    });
 });
 
 describe("aggregateSubAgentUsage", () => {
diff --git a/evalboard/lib/overview.ts b/evalboard/lib/overview.ts
index c3f3b63c..4e6cfa6b 100644
--- a/evalboard/lib/overview.ts
+++ b/evalboard/lib/overview.ts
@@ -95,10 +95,6 @@ export interface RunListingRow {
     tasksSucceeded: number;
     tasksRun: number;
     totalCostUsd: number | null;
-    // False when a task in scope burned tokens whose cost is missing, so
-    // totalCostUsd is a floor. Optional so existing test factories stay valid;
-    // absent reads as complete.
-    costComplete?: boolean;
     taskDurationSeconds: number | null;
     // Run-level harness (coder-eval AgentKind) for the Harness column; null on
     // legacy runs that predate the RunConfig stamp / carry no agent_config.type.
@@ -113,10 +109,7 @@ export interface RunListingRow {
 // sum instead of silently understating it.
 export interface RunListingTotals {
     costUsd: number | null; // null when no matched run recorded a cost
-    // The sum understates: either a matched run recorded no cost at all, or one
-    // recorded a partial cost because some task's tokens went unpriced. Both make
-    // costUsd a floor, and both used to be invisible.
-    costPartial: boolean;
+    costPartial: boolean; // some matched runs had no cost (sum understates)
     tasksSucceeded: number;
     tasksRun: number;
     durationSeconds: number | null; // null when no matched run recorded a duration
@@ -133,7 +126,6 @@ export function summarizeListing(rows: RunListingRow[]): RunListingTotals {
     let tasksRun = 0;
     let durationSeconds = 0;
     let durationRuns = 0;
-    let anyCostIncomplete = false;
     for (const r of rows) {
         tasksSucceeded += r.tasksSucceeded;
         tasksRun += r.tasksRun;
@@ -141,7 +133,6 @@ export function summarizeListing(rows: RunListingRow[]): RunListingTotals {
             costUsd += r.totalCostUsd;
             costRuns += 1;
         }
-        if (r.costComplete === false) anyCostIncomplete = true;
         if (r.taskDurationSeconds != null) {
             durationSeconds += r.taskDurationSeconds;
             durationRuns += 1;
@@ -149,8 +140,7 @@ export function summarizeListing(rows: RunListingRow[]): RunListingTotals {
     }
     return {
         costUsd: costRuns > 0 ? costUsd : null,
-        costPartial:
-            (costRuns > 0 && costRuns < rows.length) || anyCostIncomplete,
+        costPartial: costRuns > 0 && costRuns < rows.length,
         tasksSucceeded,
         tasksRun,
         durationSeconds: durationRuns > 0 ? durationSeconds : null,
@@ -764,9 +754,6 @@ export async function getRunListing(
                 .length,
             tasksRun: scopedTasks.length,
             totalCostUsd: scopedCost,
-            // Scoped like cost: with a filter active, only the matching tasks'
-            // completeness bears on the cost shown for this row.
-            costComplete: scopedTasks.every((t) => t.costComplete !== false),
             taskDurationSeconds: scopedDur,
             harness: overview.harness ?? null,
         });
diff --git a/evalboard/lib/runs.ts b/evalboard/lib/runs.ts
index 54ec4d6d..45de6f98 100644
--- a/evalboard/lib/runs.ts
+++ b/evalboard/lib/runs.ts
@@ -54,16 +54,7 @@ export interface RunSummary {
     tasksSucceeded: number;
     tasksFailed: number;
     tasksError: number;
-    // Pass rate as a 0-1 fraction, errors counted as misses. Read from run.json's
-    // canonical `pass_rate` when present so this never diverges from the report.
-    passRate: number | null;
     totalCostUsd: number | null;
-    // False when some task's spend went unpriced, so totalCostUsd is a floor.
-    costComplete: boolean;
-    tasksUnpriced: number;
-    // Judge + simulator spend. Kept out of totalCostUsd (a property of the suite's
-    // criteria, identical across harnesses) but real money all the same.
-    evalOverheadCostUsd: number | null;
     componentShas: ComponentSha[];
 }
 
@@ -104,19 +95,6 @@ export interface TaskResultSummary {
     // carried forward as a pass (run.json `mature_skipped`). The task wasn't
     // executed, so it has no per-task detail to link to. False on normal rows.
     matureSkipped: boolean;
-    // False when this row burned tokens whose cost is missing, so totalCostUsd is
-    // a floor. Every cost sum treats a null cost as $0, so without this an
-    // unpriced row is indistinguishable from a free one and the run's bill reads
-    // low with nothing to say so. All five fields below are optional so test
-    // factories predating them stay valid (undefined === complete / absent).
-    costComplete?: boolean;
-    // Judge + simulator spend for this row, outside totalCostUsd.
-    judgeCostUsd?: number | null;
-    simulatorCostUsd?: number | null;
-    // Why this row errored (null otherwise). Errors count as misses in the pass
-    // rate, so the dashboard has to be able to say what the points went to.
-    errorMessage?: string | null;
-    errorCategory?: string | null;
 }
 
 export interface CriterionResult {
@@ -389,16 +367,6 @@ interface RawTaskResult {
     // Set by the nightly runner (eval_runner) on a carried-forward row for a
     // "mature" task it skipped this run. Absent on normal rows.
     mature_skipped?: boolean;
-    // False when a turn on this row burned tokens the rate card could not price,
-    // so total_cost_usd is a floor. Absent on runs predating the field.
-    cost_complete?: boolean | null;
-    // Evaluation-machinery spend for this row, kept out of total_cost_usd.
-    judge_cost_usd?: number | null;
-    simulator_cost_usd?: number | null;
-    // Why an errored row errored. Errors count as misses, so the rollup needs to
-    // say what it lost the points to.
-    error_message?: string | null;
-    error_category?: string | null;
 }
 
 interface RawRunJson {
@@ -410,19 +378,6 @@ interface RawRunJson {
     tasks_succeeded?: number;
     tasks_failed?: number;
     tasks_error?: number;
-    // Canonical pass rate (0-1) computed by coder_eval: tasks_succeeded /
-    // tasks_run, errors included as misses. READ THIS rather than re-deriving a
-    // denominator here — four surfaces each deriving their own is what made this
-    // dashboard and the markdown report disagree by up to 10 points on the same
-    // file. Absent on runs predating the field; the fallback below reproduces the
-    // identical formula, so historical runs render unchanged.
-    pass_rate?: number | null;
-    // False when some task burned tokens the rate card could not price, making
-    // every cost total on this run a floor rather than the bill.
-    cost_complete?: boolean | null;
-    tasks_unpriced?: number | null;
-    // Judge + simulator spend, deliberately outside the agent bill.
-    eval_overhead_cost_usd?: number | null;
     task_results?: RawTaskResult[];
     // Values are scalars except `tool_plugins`, a {plugin: version} map of
     // the installed @uipath/*-tool packages (recorded since coder_eval #366).
@@ -685,15 +640,6 @@ export function toTaskRow(t: RawTaskResult): TaskResultSummary {
         tags,
         skill: deriveSkill(t.task_path, tags),
         matureSkipped: t.mature_skipped ?? false,
-        // Absent means complete: a row written before the field existed is read as
-        // priced rather than inferred from its tokens. Guessing at the past would
-        // buy a caveat on old runs at the cost of a second definition of
-        // "unpriced" living here, out of step with the Python one.
-        costComplete: t.cost_complete ?? true,
-        judgeCostUsd: t.judge_cost_usd ?? null,
-        simulatorCostUsd: t.simulator_cost_usd ?? null,
-        errorMessage: t.error_message ?? null,
-        errorCategory: t.error_category ?? null,
     };
 }
 
@@ -719,35 +665,16 @@ export async function readRunSummary(id: string): Promise<RunSummary | null> {
     const taskDurationSeconds = allHaveDuration
         ? taskDurationSum
         : (data.total_duration_seconds ?? null);
-    const tasksRun = data.tasks_run ?? taskResults.length;
-    const tasksSucceeded = data.tasks_succeeded ?? 0;
-    // A row whose tokens went unpriced contributes 0 to totalCost above, so the
-    // sum silently understates the bill. Count those rows and say so rather than
-    // presenting a floor as the total.
-    const tasksUnpriced = taskResults.filter((t) => t.cost_complete === false)
-        .length;
-    const overhead = taskResults.reduce(
-        (a, t) => a + (t.judge_cost_usd ?? 0) + (t.simulator_cost_usd ?? 0),
-        0,
-    );
     return {
         id,
         startTime: data.start_time ?? null,
         endTime: data.end_time ?? null,
         taskDurationSeconds,
-        tasksRun,
-        tasksSucceeded,
+        tasksRun: data.tasks_run ?? taskResults.length,
+        tasksSucceeded: data.tasks_succeeded ?? 0,
         tasksFailed: data.tasks_failed ?? 0,
         tasksError: data.tasks_error ?? 0,
-        // Prefer the canonical field; the fallback is the identical formula, so
-        // historical runs without it render exactly as they did before.
-        passRate:
-            data.pass_rate ?? (tasksRun > 0 ? tasksSucceeded / tasksRun : null),
         totalCostUsd: taskResults.length ? totalCost : null,
-        costComplete: data.cost_complete ?? tasksUnpriced === 0,
-        tasksUnpriced: data.tasks_unpriced ?? tasksUnpriced,
-        evalOverheadCostUsd:
-            data.eval_overhead_cost_usd ?? (overhead > 0 ? overhead : null),
         componentShas: extractComponentShas(data.environment_info),
     };
 }
@@ -863,11 +790,6 @@ export interface RunOverviewTask {
     // Optional so test factories that predate the field stay valid (undefined
     // === a normal executed row).
     matureSkipped?: boolean;
-    // False when this row burned tokens whose cost is missing, so totalCostUsd is
-    // a floor. Every cost sum counts a null cost as $0, so without this an
-    // unpriced row is indistinguishable from a free one. Optional so test
-    // factories predating it stay valid (undefined === complete).
-    costComplete?: boolean;
 }
 
 export interface RunOverview {
@@ -983,7 +905,6 @@ export async function readRunOverview(
                 visibleTurns: visibleTurnsFromRaw(t),
                 hasFinalReply: t.has_final_reply ?? false,
                 matureSkipped: t.mature_skipped ?? false,
-                costComplete: t.cost_complete ?? true,
             };
         });
     const totalCost = taskResults.reduce(

From 09e034417db7c16e0faaa70e1c218bdc76798526 Mon Sep 17 00:00:00 2001
From: Bai Li <bai.li@uipath.com>
Date: Tue, 28 Jul 2026 19:29:44 -0700
Subject: [PATCH 09/20] refactor(cost): correct the simulator-cost bound and
 drop the unread variant error share

The simulator cost is a floor, not an upper bound. UserSimulator keeps
uncached_input_tokens and drops both cache buckets, so a cached prompt prefix is
absent from the count: a live run's full persona-and-goal prompt recorded 6 input
tokens. The docstring claimed the opposite, which would tell a reader the figure
is conservatively high and stop them looking.

Also drops is_priced's reference to a refusal path that was cut, and removes
VariantAggregate.error_share: no surface rendered it, the variant tables already
print Errors beside Pass Rate (n/m), and the experiment JSON has no downstream
reader, so it would have shipped published and unread. pass_rate stays.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 src/coder_eval/models/experiment.py  | 13 ++++++-------
 src/coder_eval/pricing.py            |  4 ++--
 src/coder_eval/reports_experiment.py |  8 +++++---
 tests/test_run_metrics.py            |  1 -
 4 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/src/coder_eval/models/experiment.py b/src/coder_eval/models/experiment.py
index 95a21694..161be48b 100644
--- a/src/coder_eval/models/experiment.py
+++ b/src/coder_eval/models/experiment.py
@@ -238,14 +238,13 @@ def _check_task_count_invariant(self) -> VariantAggregate:
     @computed_field  # type: ignore[prop-decorator]
     @property
     def pass_rate(self) -> float | None:
-        """``tasks_succeeded / tasks_run`` as a 0-1 fraction. ``None`` on an empty variant."""
-        return self.tasks_succeeded / self.tasks_run if self.tasks_run else None
+        """``tasks_succeeded / tasks_run`` as a 0-1 fraction. ``None`` on an empty variant.
 
-    @computed_field  # type: ignore[prop-decorator]
-    @property
-    def error_share(self) -> float | None:
-        """``tasks_error / tasks_run`` as a 0-1 fraction. ``None`` on an empty variant."""
-        return self.tasks_error / self.tasks_run if self.tasks_run else None
+        No variant-level ``error_share`` counterpart: the variant tables already
+        print ``Errors`` beside ``Pass Rate (n/m)``, and nothing else consumes the
+        experiment JSON, so a second computed field would be published unread.
+        """
+        return self.tasks_succeeded / self.tasks_run if self.tasks_run else None
 
 
 class TaskExperimentSummary(BaseModel):  # noqa: CE009 -- persisted result model; round-trip leniency like models/results.py
diff --git a/src/coder_eval/pricing.py b/src/coder_eval/pricing.py
index 02d713a4..bff434b9 100644
--- a/src/coder_eval/pricing.py
+++ b/src/coder_eval/pricing.py
@@ -180,8 +180,8 @@ def is_priced(model: str) -> bool:
 
     The rate card is a static table, so a model that shipped after the installed
     framework version prices as ``None`` — silently, per turn, for a whole run.
-    Call this at run start to make that a warning (or a refusal) instead of a
-    number that reads 19% low with nothing on the report to say so.
+    Call this at run start to make that a warning instead of a number that reads
+    19% low with nothing on the report to say so.
     """
     return _lookup_rate(_normalize_model(model)) is not None
 
diff --git a/src/coder_eval/reports_experiment.py b/src/coder_eval/reports_experiment.py
index 6dc61fa1..10e65973 100644
--- a/src/coder_eval/reports_experiment.py
+++ b/src/coder_eval/reports_experiment.py
@@ -87,9 +87,11 @@ def _simulator_cost_usd(result: EvaluationResult) -> float | None:
     whenever a task pins ``agent.model``, as the skills suite does throughout.
     Pricing the simulator at the subject's model would mis-bill every such row.
 
-    Simulator telemetry buckets everything as prompt/completion tokens with no
-    cache split, so the whole prompt prices at the uncached input rate: an upper
-    bound on a dialog whose prefix was cached.
+    A floor, not the exact figure. ``UserSimulator`` records only
+    ``uncached_input_tokens`` and drops both cache buckets, so a prompt whose
+    prefix was cached is largely absent from the count: a live run's full
+    persona-and-goal prompt recorded 6 input tokens. Widening that telemetry is a
+    change to what the run captures, not to what this reports.
     """
     sim = result.simulation
     if sim is None:
diff --git a/tests/test_run_metrics.py b/tests/test_run_metrics.py
index b7d00d92..2d51a077 100644
--- a/tests/test_run_metrics.py
+++ b/tests/test_run_metrics.py
@@ -114,7 +114,6 @@ def test_variant_aggregate_shares_the_denominator(self):
             average_duration=1.0,
         )
         assert agg.pass_rate == pytest.approx(0.25)
-        assert agg.error_share == pytest.approx(0.5)
 
     def test_rates_serialize_into_run_json(self):
         """Downstream consumers must be able to READ the rate instead of re-deriving it.

From c6f5e119af576d54d8611506ecce79dbdd6b36d1 Mon Sep 17 00:00:00 2001
From: Bai Li <bai.li@uipath.com>
Date: Tue, 28 Jul 2026 19:50:15 -0700
Subject: [PATCH 10/20] docs(cost): describe the unpriced-crash mechanism
 accurately and keep comments framework-general

The pre-flight warning claimed cost "will be recorded as null" for an unpriced
model. That is wrong for any agent whose backend reports its own cost: those turns
price fine. The rate card is the FALLBACK, and the only source for a turn the
backend never priced, so what an unpriced model actually costs you is the killed
and timed-out partials, which arrive with full token counts and no cost. Traced
against a real timed-out row carrying 1.2M tokens and no dollars.

Also strips run ids, dates, dollar amounts and suite names from comments and test
docstrings. The behavioural claim each one made is kept; the incident it came from
is not something a general framework should narrate.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 src/coder_eval/models/results.py      | 23 +++++++++++------------
 src/coder_eval/orchestration/batch.py | 23 +++++++++++++++--------
 src/coder_eval/pricing.py             |  8 +++++---
 src/coder_eval/reports.py             | 11 +++++------
 src/coder_eval/reports_experiment.py  | 21 +++++++++------------
 tests/test_cost_accounting_paths.py   | 14 +++++++-------
 tests/test_run_metrics.py             | 25 +++++++++++++------------
 tests/test_token_usage.py             |  4 ++--
 8 files changed, 67 insertions(+), 62 deletions(-)

diff --git a/src/coder_eval/models/results.py b/src/coder_eval/models/results.py
index 3384830c..9f01e0ad 100644
--- a/src/coder_eval/models/results.py
+++ b/src/coder_eval/models/results.py
@@ -881,16 +881,16 @@ class RunSummary(BaseModel):
 
     ``pass_rate`` is ``tasks_succeeded / tasks_run``: **every dispatched task is in
     the denominator, errors included as misses.** An error is not a free pass. The
-    previous formula excluded errors, which paid a bonus for erroring — up to 10
-    points on a real nightly, and one run that rendered as 100% while passing 0.8%
-    of its rows. A denominator that shrinks when a run goes wrong is not a metric
-    you can set beside another run's. ``error_share`` says how much of the rate is
+    previous formula excluded errors, which paid a bonus for erroring: a harness
+    that fell over enough could render as a perfect score while passing almost
+    nothing. A denominator that shrinks when a run goes wrong is not a metric you
+    can set beside another run's. ``error_share`` says how much of the rate is
     errors, so a bad infrastructure night *shows* instead of being absorbed.
 
     This is the single denominator for the whole framework. Every reporting
-    surface reads ``pass_rate`` rather than re-deriving one: four surfaces across
-    two repos each computing their own is what made the dashboard and the markdown
-    report disagree by up to 10 points on the same run.json.
+    surface reads ``pass_rate`` rather than re-deriving one: independent
+    re-derivations drift, and consumers reading the same run.json then publish
+    different rates for it.
 
     Every derived metric here is computed, never stored, so it cannot drift from
     the counts it comes from.
@@ -962,11 +962,10 @@ def _check_task_count_invariant(self) -> RunSummary:
     # Derived run metrics.
     #
     # Every one is a computed_field over the stored counts and ``task_results``,
-    # so it serializes into run.json for downstream consumers (evalboard, the
-    # external eval-runner) while staying impossible to set to something the
-    # rows disagree with. Consumers should READ these rather than re-deriving
-    # them: four surfaces re-deriving two different denominators is what made
-    # the dashboard and the markdown report disagree by up to 10 points.
+    # so it serializes into run.json for downstream consumers (dashboards,
+    # external runners) while staying impossible to set to something the rows
+    # disagree with. Consumers should READ these rather than re-deriving them:
+    # every independent re-derivation is another denominator that can drift.
     # ------------------------------------------------------------------
 
     @computed_field  # type: ignore[prop-decorator]
diff --git a/src/coder_eval/orchestration/batch.py b/src/coder_eval/orchestration/batch.py
index 6e7bee93..8952427b 100644
--- a/src/coder_eval/orchestration/batch.py
+++ b/src/coder_eval/orchestration/batch.py
@@ -42,12 +42,18 @@
 def check_pricing_coverage(resolved_tasks: list[ResolvedTask]) -> list[str]:
     """Pre-flight the rate card against the models this run will use.
 
-    The rate card is a static table baked into the installed framework version,
-    so a model released after that version prices as ``None`` on every turn — the
-    run completes, records full token counts, and reports a cost that is silently
-    low. (One nightly under-reported $209.81, 18.9% of its bill, because the
-    model's rates landed in the next release.) This turns that into a warning at
-    dispatch instead of a discovery weeks later.
+    The rate card is a static table baked into the installed framework version, so
+    a model released after that version has no rate here. Whether that costs you
+    anything depends on the turn: an agent whose backend reports its own cost (the
+    Claude Code SDK does) still prices a clean turn, so most rows look fine. The
+    rate card is the FALLBACK, and it is the only source for a turn the backend
+    never priced — a timed-out or killed partial, which arrives with full token
+    counts and no cost. With no rate, that fallback is a no-op and the tokens book
+    no money at all.
+
+    That is not a rare corner: on a large suite the killed tail is where the
+    biggest token counts live, since a task that ran to the wall spent the most
+    getting there.
 
     A warning rather than a refusal, so a brand-new model stays evaluable on the
     day it ships: the cost is what degrades, not the evaluation. What the run
@@ -66,8 +72,9 @@ def check_pricing_coverage(resolved_tasks: list[ResolvedTask]) -> list[str]:
     if not missing:
         return []
     logger.warning(
-        "No pricing rate for %s. Cost for these tasks will be recorded as null and every "
-        + "run-level total will understate the bill (RunSummary.cost_complete reports false). "
+        "No pricing rate for %s. Turns the agent's own backend prices are unaffected, but any "
+        + "timed-out or killed partial will book its tokens with no cost, so run-level totals will "
+        + "understate the bill (RunSummary.cost_complete reports false). "
         + "Add the rate to coder_eval.pricing or register it from a plugin via register_pricing().",
         ", ".join(repr(m) for m in missing),
     )
diff --git a/src/coder_eval/pricing.py b/src/coder_eval/pricing.py
index bff434b9..0fb4c4dd 100644
--- a/src/coder_eval/pricing.py
+++ b/src/coder_eval/pricing.py
@@ -179,9 +179,11 @@ def is_priced(model: str) -> bool:
     """Whether the rate card can price this model (after prefix normalization).
 
     The rate card is a static table, so a model that shipped after the installed
-    framework version prices as ``None`` — silently, per turn, for a whole run.
-    Call this at run start to make that a warning instead of a number that reads
-    19% low with nothing on the report to say so.
+    framework version has no rate. It is the fallback for turns the agent's own
+    backend never priced (timed-out and killed partials), so an unpriced model
+    means those turns book their tokens against no money. Call this at run start
+    to make that a warning instead of a total that reads low with nothing on the
+    report to say so.
     """
     return _lookup_rate(_normalize_model(model)) is not None
 
diff --git a/src/coder_eval/reports.py b/src/coder_eval/reports.py
index 11ed12fb..42f8be6a 100644
--- a/src/coder_eval/reports.py
+++ b/src/coder_eval/reports.py
@@ -170,10 +170,9 @@ def _pass_rate_lines(summary: RunSummary) -> list[str]:
 
     One rate over every dispatched task. The old line divided by
     ``tasks_run - tasks_error``, which read like a pass rate but paid a bonus for
-    erroring: up to 10 points on a real nightly, and one run that rendered as
-    100.0% while passing 0.8% of its rows. When errors are material the error
-    share is printed beside the rate so a bad infrastructure night is visible
-    rather than absorbed.
+    erroring: a run that fell over enough could print a near-perfect score while
+    passing almost nothing. When errors are material the error share is printed
+    beside the rate so a bad infrastructure night is visible rather than absorbed.
     """
     lines = [f"- **Pass Rate**: {_fmt_rate(summary.pass_rate)} ({summary.tasks_succeeded}/{summary.tasks_run})"]
     if summary.tasks_error:
@@ -554,8 +553,8 @@ def _generate_token_usage_section(task_results: list[dict[str, Any]]) -> list[st
         total_cost = sum(costs) if costs else None
 
         # Rows whose spend is only partly priced. Reported explicitly because the
-        # alternative is what used to happen: a run understating its bill by 19%
-        # with nothing on the report to say the number was incomplete. Both of
+        # alternative is a total that reads materially low with nothing on the
+        # report to say the number was incomplete. Both of
         # these come from the same helpers RunSummary.tasks_unpriced /
         # eval_overhead_cost_usd use, so the report and run.json cannot disagree
         # about which rows lost money.
diff --git a/src/coder_eval/reports_experiment.py b/src/coder_eval/reports_experiment.py
index 10e65973..d754ee6a 100644
--- a/src/coder_eval/reports_experiment.py
+++ b/src/coder_eval/reports_experiment.py
@@ -37,10 +37,10 @@
 _REPLICATE_PASS_THRESHOLD = 0.9
 
 # Cap on the ``error_message`` carried into each run.json row. Long enough to
-# identify a failure without a per-task artifact fetch (which is the whole point
-# — a rollup of 900 rows whose errors are only diagnosable one task.json at a
-# time is not diagnosable), short enough that a wholly-errored run doesn't bloat
-# run.json. The untruncated message stays on task.json.
+# identify a failure without a per-task artifact fetch (a large rollup whose
+# errors are only diagnosable one task.json at a time is not diagnosable),
+# short enough that a wholly-errored run doesn't bloat run.json. The untruncated
+# message stays on task.json.
 _ROW_ERROR_MESSAGE_MAX_CHARS = 400
 
 
@@ -84,14 +84,13 @@ def _simulator_cost_usd(result: EvaluationResult) -> float | None:
     Priced at the ROUTE's model, not the subject agent's. ``UserSimulator`` builds
     its agent config with ``model=None`` on purpose, so the model resolves to the
     route default (``BEDROCK_MODEL``) — which is a different model from the subject
-    whenever a task pins ``agent.model``, as the skills suite does throughout.
-    Pricing the simulator at the subject's model would mis-bill every such row.
+    whenever a task pins ``agent.model``, which suites routinely do. Pricing the
+    simulator at the subject's model would mis-bill every such row.
 
     A floor, not the exact figure. ``UserSimulator`` records only
     ``uncached_input_tokens`` and drops both cache buckets, so a prompt whose
-    prefix was cached is largely absent from the count: a live run's full
-    persona-and-goal prompt recorded 6 input tokens. Widening that telemetry is a
-    change to what the run captures, not to what this reports.
+    prefix was cached is largely absent from the count. Widening that telemetry is
+    a change to what the run captures, not to what this reports.
     """
     sim = result.simulation
     if sim is None:
@@ -212,9 +211,7 @@ def eval_result_to_task_dict(
         "simulator_cost_usd": _simulator_cost_usd(result),
         # Why this row errored, on the row. Errors now count as misses, so the
         # rollup needs to say *why* it lost those points: without these, an errored
-        # run is untriageable without fetching per-task artifacts one at a time
-        # (the 109 zero-iteration codex errors of 2026-07-22 could not be
-        # characterised from run.json at all).
+        # run is untriageable without fetching per-task artifacts one at a time.
         "error_message": (
             truncate_crash_message(result.error_message, limit=_ROW_ERROR_MESSAGE_MAX_CHARS)
             if result.error_message
diff --git a/tests/test_cost_accounting_paths.py b/tests/test_cost_accounting_paths.py
index f4456880..6f8aa07a 100644
--- a/tests/test_cost_accounting_paths.py
+++ b/tests/test_cost_accounting_paths.py
@@ -2,11 +2,11 @@
 
 Two seams:
 
-1. **The pre-flight** (``check_pricing_coverage``) — the measured loss. The rate
-   card is a static table baked into the installed version, so a model released
-   after it prices every turn as ``null``. Sonnet 5's rates landed one release
-   after the 2026-07-21 nightly, which recorded $209.81 (18.9% of its true bill)
-   across 62 errored rows as no cost at all, with one log line to show for it.
+1. **The pre-flight** (``check_pricing_coverage``). The rate card is a static
+   table baked into the installed version, so a model released after it has no
+   rate. It is the fallback for turns the agent's backend never priced, which
+   makes killed and timed-out partials the ones that book their tokens against no
+   money at all, with one log line to show for it.
 2. **The row projection** (``eval_result_to_task_dict``) — judge and simulator
    spend was captured and rolled up nowhere, and a row whose turns were only
    partly priced reported its partial sum as the whole.
@@ -183,8 +183,8 @@ def test_single_shot_row_has_no_simulator_cost(self):
 class TestErrorDiagnosticsOnTheRow:
     """Errors count as misses, so the rollup has to say why it lost those points.
 
-    The 2026-07-22 codex nightly had 109 zero-iteration errors that could not be
-    characterised from run.json at all — every one needed its own task.json fetch.
+    Without these, a run's zero-iteration errors cannot be characterised from
+    run.json at all: every one needs its own task.json fetch.
     """
 
     def test_error_message_and_category_land_on_the_row(self):
diff --git a/tests/test_run_metrics.py b/tests/test_run_metrics.py
index 2d51a077..e0aed2bc 100644
--- a/tests/test_run_metrics.py
+++ b/tests/test_run_metrics.py
@@ -3,14 +3,14 @@
 Two bugs are pinned here.
 
 **The denominator.** ``pass_rate`` used to be ``succeeded / (run - error)``, which
-paid a bonus for erroring. Measured on real nightlies: +10.0 points for a codex run
-with 116 errors of 947, +7.1 for a sonnet-5 run, and a LiteLLM run that rendered as
-**100.0%** while passing 7 of 861 rows. Every surface now divides by ``tasks_run``.
+paid a bonus for erroring: the more a run fell over, the smaller its denominator
+got, up to the degenerate case of a run rendering as a perfect score while passing
+a handful of rows. Every surface now divides by ``tasks_run``.
 
 **The bill.** Cost was summed over whatever rows happened to carry one, so a run
 whose model was missing from the rate card, or whose turns were killed before the
-SDK reported a cost, understated its spend silently. One nightly hid $209.81
-(18.9%) that way. Unpriced spend is now counted and the total is labelled a floor.
+backend reported a cost, understated its spend silently. Unpriced spend is now
+counted and the total is labelled a floor.
 """
 
 from __future__ import annotations
@@ -84,9 +84,9 @@ def test_timeout_and_budget_statuses_are_failures_not_errors(self):
         assert summary.pass_rate == pytest.approx(0.2)
 
     def test_mostly_errored_run_cannot_render_as_perfect(self):
-        """The reductio from run adhoc-2026-07-16_18-12-48: 854 errors of 861 rows.
+        """The reductio: 854 errors out of 861 rows.
 
-        Reported 100.0% under the old formula (7 evaluable, 7 passed). Must now
+        Under the old formula this reported 100.0% (7 evaluable, 7 passed). It must
         read as what it was.
         """
         rows = [_row(FinalStatus.SUCCESS) for _ in range(7)] + [_row(FinalStatus.ERROR) for _ in range(854)]
@@ -118,8 +118,8 @@ def test_variant_aggregate_shares_the_denominator(self):
     def test_rates_serialize_into_run_json(self):
         """Downstream consumers must be able to READ the rate instead of re-deriving it.
 
-        Four surfaces across two repos each deriving their own denominator is what
-        made the dashboard and the markdown report disagree by up to 10 points.
+        Independent re-derivations drift, and then two consumers publish different
+        rates for the same run.
         """
         summary = _summary([_row(FinalStatus.SUCCESS), _row(FinalStatus.ERROR)])
         dumped = summary.model_dump()
@@ -204,9 +204,10 @@ def test_unpriced_models_dedupes_and_drops_empty(self):
 class TestRateCardCoversCheckedInExperiments:
     """CI guard: a model referenced by a committed experiment must be priced.
 
-    The 2026-07-21 nightly recorded $209.81 as null because Sonnet 5's rates landed
-    one release after the run. The rate card is a static table, so this class is what
-    makes the next new model fail CI instead of a nightly's cost column.
+    The rate card is a static table baked into the installed version, so a model
+    whose rates land in a later release prices as null on any turn the agent's own
+    backend did not price. This class is what makes the next new model fail CI
+    instead of a run's cost column.
     """
 
     @staticmethod
diff --git a/tests/test_token_usage.py b/tests/test_token_usage.py
index eee2c00c..38bb0593 100644
--- a/tests/test_token_usage.py
+++ b/tests/test_token_usage.py
@@ -925,8 +925,8 @@ def test_token_section_marks_partial_cost_as_floor(self):
         """A priced row beside an unpriced one reports the total as a floor.
 
         The failure this guards: summing only the rows that carry a cost and
-        presenting it as the run's bill. One nightly under-reported $209.81 (18.9%)
-        exactly this way.
+        presenting it as the run's bill. The unpriced rows are typically the
+        killed ones, which are also the ones that burned the most getting there.
         """
         task_results = [
             {"task_id": "priced", "total_tokens": 1000, "total_cost_usd": 0.25, "cost_complete": True},

From 117559c26504f6b0dda1e88ccaa9e7945aa17eb9 Mon Sep 17 00:00:00 2001
From: Bai Li <bai.li@uipath.com>
Date: Tue, 28 Jul 2026 19:55:23 -0700
Subject: [PATCH 11/20] fix(pricing): add the claude-opus-5 rate so killed
 turns stop booking zero

The rate card had no entry for claude-opus-5. On a clean turn that costs nothing,
since the Claude Code SDK reports its own cost, but the card is the only fallback
for a turn the backend never priced. So every timed-out partial booked its full
token counts against no money.

Verified against a real timed-out row: 32 uncached / 18,519 out / 99,959 cache
write / 1,083,097 cache read priced as None before, $1.6294 now. Across that run's
nine killed rows, $40.37 that previously read as zero.

Rates from the published table: $5/$25 per MTok, with the standard 1.25x cache
write and 0.1x cache read multipliers.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 src/coder_eval/pricing.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/coder_eval/pricing.py b/src/coder_eval/pricing.py
index 0fb4c4dd..9783d3fd 100644
--- a/src/coder_eval/pricing.py
+++ b/src/coder_eval/pricing.py
@@ -22,6 +22,8 @@ class ModelPricing:
 # Official Anthropic pricing as of 2025
 # Key: CLI model name (before gateway mapping)
 _PRICING: dict[str, ModelPricing] = {
+    # Claude 5 Opus: $5/$25, cache write 1.25x input, cache read 0.1x input.
+    "claude-opus-5": ModelPricing(5.0, 25.0, 6.25, 0.50),
     # Claude 4.8 / 4.7 / 4.6 / 4.5 / 4 Opus
     "claude-opus-4-8": ModelPricing(15.0, 75.0, 18.75, 1.50),
     "claude-opus-4-7": ModelPricing(15.0, 75.0, 18.75, 1.50),

From f915afa5cffab32f69cd8a3d04b2b7a4e4dd39a4 Mon Sep 17 00:00:00 2001
From: Bai Li <bai.li@uipath.com>
Date: Tue, 28 Jul 2026 20:03:31 -0700
Subject: [PATCH 12/20] fix(pricing): correct every wrong rate-card entry and
 close the alias gaps

Audited the whole card against the published tables.

Opus 4.5 and later dropped to $5/$25 while Opus 4.1 and Opus 4 stayed at $15/$75,
so the version boundary is the price boundary. The card had opus-4-5 through 4-8 at
the old $15/$75, a 3x overcharge on every turn the rate card actually prices.

haiku-4-5 carried Haiku 3.5's rates ($0.80/$4 instead of $1/$5). The gpt-5.6 family
had an Anthropic-style 1.25x cache-write rate, where OpenAI bills no cache-write fee
and every other OpenAI entry sets cache_write == input.

Also keys the bare aliases that previously went unpriced because only the dated id
was present (opus-4-5, opus-4-1, opus-4, sonnet-4-5, haiku-4-5, haiku-3-5), and adds
fable-5 / mythos-5.

sonnet-5 deliberately stays at the standard $3/$15 rather than the $2/$10
introductory rate: a static table cannot express a promo window, and this error
overstates cost for a few weeks instead of understating it indefinitely after.

Two pre-existing tests hardcoded the old Opus and Haiku rates; their expected values
are recomputed from the corrected ones.

Verified correct and unchanged: the Sonnet 4.x and Claude 3.x entries, the nine other
OpenAI entries, and the Gemini entries.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 src/coder_eval/pricing.py           | 45 ++++++++++++++++++++---------
 tests/test_cost_accounting_paths.py | 10 +++----
 tests/test_token_usage.py           |  4 +--
 3 files changed, 38 insertions(+), 21 deletions(-)

diff --git a/src/coder_eval/pricing.py b/src/coder_eval/pricing.py
index 9783d3fd..f4b572d7 100644
--- a/src/coder_eval/pricing.py
+++ b/src/coder_eval/pricing.py
@@ -22,24 +22,39 @@ class ModelPricing:
 # Official Anthropic pricing as of 2025
 # Key: CLI model name (before gateway mapping)
 _PRICING: dict[str, ModelPricing] = {
-    # Claude 5 Opus: $5/$25, cache write 1.25x input, cache read 0.1x input.
+    # Claude 5 Fable / Mythos: $10/$50.
+    "claude-fable-5": ModelPricing(10.0, 50.0, 12.50, 1.0),
+    "claude-mythos-5": ModelPricing(10.0, 50.0, 12.50, 1.0),
+    # Opus 4.5 and later dropped to $5/$25. Opus 4.1 and Opus 4 keep the old
+    # $15/$75 — the version boundary is the price boundary, so do NOT assume a
+    # newer Opus costs more than an older one.
     "claude-opus-5": ModelPricing(5.0, 25.0, 6.25, 0.50),
-    # Claude 4.8 / 4.7 / 4.6 / 4.5 / 4 Opus
-    "claude-opus-4-8": ModelPricing(15.0, 75.0, 18.75, 1.50),
-    "claude-opus-4-7": ModelPricing(15.0, 75.0, 18.75, 1.50),
-    "claude-opus-4-6": ModelPricing(15.0, 75.0, 18.75, 1.50),
-    "claude-opus-4-6-20250514": ModelPricing(15.0, 75.0, 18.75, 1.50),
-    "claude-opus-4-5-20251101": ModelPricing(15.0, 75.0, 18.75, 1.50),
+    "claude-opus-4-8": ModelPricing(5.0, 25.0, 6.25, 0.50),
+    "claude-opus-4-7": ModelPricing(5.0, 25.0, 6.25, 0.50),
+    "claude-opus-4-6": ModelPricing(5.0, 25.0, 6.25, 0.50),
+    "claude-opus-4-6-20250514": ModelPricing(5.0, 25.0, 6.25, 0.50),
+    "claude-opus-4-5": ModelPricing(5.0, 25.0, 6.25, 0.50),
+    "claude-opus-4-5-20251101": ModelPricing(5.0, 25.0, 6.25, 0.50),
+    # Claude 4.1 / 4 Opus (deprecated / retired) — still $15/$75.
+    "claude-opus-4-1": ModelPricing(15.0, 75.0, 18.75, 1.50),
+    "claude-opus-4": ModelPricing(15.0, 75.0, 18.75, 1.50),
     "claude-opus-4-20250514": ModelPricing(15.0, 75.0, 18.75, 1.50),
-    # Claude 5 Sonnet (2026-06-30 on Bedrock): standard $3/$15 (promo $2/$10 thru 2026-08-31).
+    # Claude 5 Sonnet. Deliberately the STANDARD $3/$15 rather than the $2/$10
+    # introductory rate in effect through 2026-08-31: a static table cannot
+    # express a promo window, and of the two errors available this one overstates
+    # cost for a few weeks instead of understating it indefinitely afterwards.
     "claude-sonnet-5": ModelPricing(3.0, 15.0, 3.75, 0.30),
     # Claude 4.6 / 4.5 / 4 Sonnet
     "claude-sonnet-4-6": ModelPricing(3.0, 15.0, 3.75, 0.30),
     "claude-sonnet-4-6-20250514": ModelPricing(3.0, 15.0, 3.75, 0.30),
+    "claude-sonnet-4-5": ModelPricing(3.0, 15.0, 3.75, 0.30),
     "claude-sonnet-4-5-20250929": ModelPricing(3.0, 15.0, 3.75, 0.30),
     "claude-sonnet-4-20250514": ModelPricing(3.0, 15.0, 3.75, 0.30),
-    # Claude 4.5 Haiku
-    "claude-haiku-4-5-20251001": ModelPricing(0.80, 4.0, 1.0, 0.08),
+    # Claude 4.5 Haiku: $1/$5. (Not $0.80/$4 — those are Haiku 3.5's rates.)
+    "claude-haiku-4-5": ModelPricing(1.0, 5.0, 1.25, 0.10),
+    "claude-haiku-4-5-20251001": ModelPricing(1.0, 5.0, 1.25, 0.10),
+    # Claude 3.5 Haiku (retired)
+    "claude-haiku-3-5": ModelPricing(0.80, 4.0, 1.0, 0.08),
     # Claude 3.7 Sonnet
     "claude-3-7-sonnet-20250219": ModelPricing(3.0, 15.0, 3.75, 0.30),
     # Claude 3.5 Sonnet
@@ -75,10 +90,12 @@ class ModelPricing:
     "gpt-5.4-mini": ModelPricing(0.75, 4.5, 0.75, 0.075),
     "gpt-5.4-nano": ModelPricing(0.20, 1.25, 0.20, 0.02),
     # GPT-5.6 family (2026-07-09): sol flagship / terra balanced (Codex default) / luna
-    # economy. Terra matches gpt-5.4's rate; sol matches gpt-5.5.
-    "gpt-5.6-sol": ModelPricing(5.0, 30.0, 6.25, 0.50),
-    "gpt-5.6-terra": ModelPricing(2.5, 15.0, 3.125, 0.25),
-    "gpt-5.6-luna": ModelPricing(1.0, 6.0, 1.25, 0.10),
+    # economy. Terra matches gpt-5.4's rate; sol matches gpt-5.5. cache_write ==
+    # input, as for every other OpenAI entry above: OpenAI bills no separate
+    # cache-write fee, so a 1.25x Anthropic-style write rate would overcharge.
+    "gpt-5.6-sol": ModelPricing(5.0, 30.0, 5.0, 0.50),
+    "gpt-5.6-terra": ModelPricing(2.5, 15.0, 2.5, 0.25),
+    "gpt-5.6-luna": ModelPricing(1.0, 6.0, 1.0, 0.10),
     # Google Gemini (AntigravityAgent, via the Gemini Developer API). Per-MTok
     # rates from ai.google.dev/gemini-api/docs/pricing (2026). Gemini bills no
     # separate cache-WRITE fee, so cache_write == input (the agent maps
diff --git a/tests/test_cost_accounting_paths.py b/tests/test_cost_accounting_paths.py
index 6f8aa07a..8797a5d5 100644
--- a/tests/test_cost_accounting_paths.py
+++ b/tests/test_cost_accounting_paths.py
@@ -161,13 +161,13 @@ def _simulated(**env) -> EvaluationResult:
     def test_simulator_prices_at_the_route_model_not_the_subject(self):
         """UserSimulator pins model=None, so it bills at BEDROCK_MODEL.
 
-        Every skills task pins ``agent.model``, so pricing the simulator at the
-        subject's model would mis-bill the whole suite. Here the subject is
-        sonnet-5 ($3/$15) while the route is haiku-4.5 ($0.80/$4) — the simulator
-        must cost the haiku rate.
+        A task that pins ``agent.model`` would otherwise mis-bill every simulated
+        row. Here the subject is sonnet-5 ($3/$15) while the route is haiku-4.5
+        ($1/$5), so the simulator must cost the haiku rate.
         """
         result = self._simulated(bedrock_model="claude-haiku-4-5-20251001")
-        assert eval_result_to_task_dict(result)["simulator_cost_usd"] == pytest.approx(0.80 + 0.40)
+        # 1M uncached input at $1/MTok + 100K output at $5/MTok.
+        assert eval_result_to_task_dict(result)["simulator_cost_usd"] == pytest.approx(1.00 + 0.50)
 
     def test_simulator_falls_back_to_the_subject_model_off_bedrock(self):
         """A non-Bedrock route names no model on the record; the subject's is the best available."""
diff --git a/tests/test_token_usage.py b/tests/test_token_usage.py
index 38bb0593..0f60af96 100644
--- a/tests/test_token_usage.py
+++ b/tests/test_token_usage.py
@@ -436,8 +436,8 @@ def test_timeout_turn_backfills_cost_from_buckets(self):
         ]
         usage = ClaudeCodeAgent._build_token_usage(messages, None, None, None, "claude-opus-4-8")
         assert usage is not None
-        # opus-4-8: 15/M in, 75/M out, 18.75/M cache-write, 1.50/M cache-read.
-        expected = (1_000_000 * 15.0 + 500_000 * 75.0 + 200_000 * 18.75 + 4_000_000 * 1.50) / 1_000_000
+        # opus-4-8: 5/M in, 25/M out, 6.25/M cache-write, 0.50/M cache-read.
+        expected = (1_000_000 * 5.0 + 500_000 * 25.0 + 200_000 * 6.25 + 4_000_000 * 0.50) / 1_000_000
         assert usage.total_cost_usd == pytest.approx(expected)
 
     def test_backfill_via_resultmessage_snapshot_path(self):

From b3a0457541efb76adedd2a284d263cd4c7a8b5c0 Mon Sep 17 00:00:00 2001
From: Bai Li <bai.li@uipath.com>
Date: Tue, 28 Jul 2026 20:13:35 -0700
Subject: [PATCH 13/20] fix(cost): a task timeout with no preserved turn is
 unrecorded spend, not free

_cost_complete returned True for a row whose iterations list was empty, on the
reasoning that a row which burned nothing is not missing cost. That is right for a
setup error and wrong for a task-level timeout.

TaskTimeoutError comes from the ThreadedWatchdog, which SIGKILLs the agent by PID
from a non-asyncio thread. Unlike a turn-level timeout it never reaches
_on_attempt_failure, so no partial turn is drained and the row lands with zero
turns, zero tokens and no cost. The evaluation loop was still running, so that
spend is real; reporting the row as fully priced is a false claim.

Keyed on TIMEOUT rather than elapsed time, so a slow setup failure stays free while
a task timeout never does.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 src/coder_eval/reports_experiment.py | 41 ++++++++++++++++++----------
 tests/test_cost_accounting_paths.py  | 24 ++++++++++++++++
 2 files changed, 51 insertions(+), 14 deletions(-)

diff --git a/src/coder_eval/reports_experiment.py b/src/coder_eval/reports_experiment.py
index d754ee6a..fa8497d4 100644
--- a/src/coder_eval/reports_experiment.py
+++ b/src/coder_eval/reports_experiment.py
@@ -11,6 +11,7 @@
     EvaluationResult,
     ExperimentDefinition,
     ExperimentResult,
+    FinalStatus,
     TaskExperimentSummary,
 )
 from coder_eval.path_utils import replicate_subdir_name
@@ -45,21 +46,33 @@
 
 
 def _cost_complete(result: EvaluationResult) -> bool:
-    """Whether every turn that burned tokens on this row also carries a cost.
-
-    False means ``total_cost_usd`` is a floor, not the bill: some turn's spend is
-    missing from it. Every in-tree agent prices its own turns, so in practice the
-    way to land here is a model the rate card cannot price at all. This is the
-    per-row signal that feeds ``RunSummary.tasks_unpriced``.
-
-    True for a row that burned nothing (an error before the agent ran genuinely
-    cost nothing, and must not be reported as missing cost).
+    """Whether this row's recorded spend accounts for everything it spent.
+
+    False means ``total_cost_usd`` is a floor, not the bill. Two ways to get there,
+    and they are different failures:
+
+    1. **A turn burned tokens the rate card could not price.** The agent's own
+       backend prices a clean turn, so the rate card only matters for a turn the
+       backend never priced (a killed partial). With no rate, that turn books
+       tokens against no money.
+    2. **A task-level timeout preserved no turn at all.** ``TaskTimeoutError``
+       comes from the watchdog, which SIGKILLs the agent by PID; unlike a
+       turn-level timeout it never reaches ``_on_attempt_failure``, so no partial
+       turn is drained and the row lands with zero turns and zero tokens. A task
+       timeout means the evaluation loop was still running, so that spend is real
+       and simply unrecorded. Reporting such a row as fully priced would be a
+       false claim: it is the one case where cost is missing with no tokens to
+       point at.
+
+    True for a row that burned nothing. An error before the agent ran genuinely
+    cost zero and must not be reported as missing cost, which is why case 2 is
+    keyed on ``TIMEOUT`` rather than on elapsed time: a fast setup failure and a
+    slow one are both free, while a task timeout is never free.
     """
-    return all(
-        usage.total_cost_usd is not None
-        for t in result.iterations
-        if (usage := t.token_usage) is not None and not usage.is_empty()
-    )
+    priced = [usage for t in result.iterations if (usage := t.token_usage) is not None and not usage.is_empty()]
+    if not priced:
+        return result.final_status is not FinalStatus.TIMEOUT
+    return all(usage.total_cost_usd is not None for usage in priced)
 
 
 def _judge_cost_usd(result: EvaluationResult) -> float | None:
diff --git a/tests/test_cost_accounting_paths.py b/tests/test_cost_accounting_paths.py
index 8797a5d5..0d825a76 100644
--- a/tests/test_cost_accounting_paths.py
+++ b/tests/test_cost_accounting_paths.py
@@ -115,6 +115,30 @@ def test_cost_complete_true_when_nothing_burned(self):
         assert eval_result_to_task_dict(_result([]))["cost_complete"] is True
         assert eval_result_to_task_dict(_result([_turn(1, TokenUsage())]))["cost_complete"] is True
 
+    def test_cost_complete_false_when_a_task_timeout_preserved_no_turn(self):
+        """A task timeout with zero turns is unrecorded spend, not free.
+
+        The watchdog SIGKILLs the agent by PID, so unlike a turn-level timeout this
+        never reaches ``_on_attempt_failure`` and no partial turn is drained. The
+        row lands with no turns, no tokens and no cost. Since a task timeout means
+        the evaluation loop was still running, calling that row fully priced would
+        be a false claim — the one case where cost is missing with no tokens to
+        point at.
+        """
+        result = _result([])
+        result.final_status = FinalStatus.TIMEOUT
+        assert eval_result_to_task_dict(result)["cost_complete"] is False
+
+    def test_cost_complete_true_for_a_fast_error_with_no_turn(self):
+        """The companion: a setup failure has no turns either, and really is free.
+
+        Keyed on status rather than elapsed time so a slow setup failure stays free
+        while a task timeout never does.
+        """
+        result = _result([])
+        result.final_status = FinalStatus.ERROR
+        assert eval_result_to_task_dict(result)["cost_complete"] is True
+
     def test_judge_cost_rolls_up_onto_the_row(self):
         """Judge spend was captured per criterion and totalled nowhere."""
         result = _result([_turn(1, TokenUsage(uncached_input_tokens=10, output_tokens=1, total_cost_usd=0.1))])

From 75fb76659592e799bd74761ef3b7d6b57b6f5073 Mon Sep 17 00:00:00 2001
From: Bai Li <bai.li@uipath.com>
Date: Tue, 28 Jul 2026 20:26:35 -0700
Subject: [PATCH 14/20] fix(cost): flag every hard-killed task as a cost floor,
 not just the empty ones

Live testing found the previous rule under-flagged. Keying on "TIMEOUT with no
preserved turn" caught a task killed during its first turn, but a task killed
mid-dialog after two turns had completed still reported cost_complete: true,
because those two turns carry costs. The turn that was in flight when the wall hit
is lost either way.

The watchdog fires while the evaluation loop is running, so a TIMEOUT row always has
an in-flight turn whose spend was never recorded. Keyed on the status alone.

The report wording drops its rate-card explanation, since the two causes (a turn the
rate card could not price, and a hard kill that recorded nothing) reach the same
conclusion and the report cannot always tell which applied.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 src/coder_eval/reports.py            | 18 +++++++-------
 src/coder_eval/reports_experiment.py | 36 ++++++++++++++++------------
 tests/test_token_usage.py            |  6 ++---
 3 files changed, 33 insertions(+), 27 deletions(-)

diff --git a/src/coder_eval/reports.py b/src/coder_eval/reports.py
index 42f8be6a..4fc8d8d7 100644
--- a/src/coder_eval/reports.py
+++ b/src/coder_eval/reports.py
@@ -552,10 +552,13 @@ def _generate_token_usage_section(task_results: list[dict[str, Any]]) -> list[st
         costs = [t["total_cost_usd"] for t in tasks_with_tokens if t.get("total_cost_usd") is not None]
         total_cost = sum(costs) if costs else None
 
-        # Rows whose spend is only partly priced. Reported explicitly because the
-        # alternative is a total that reads materially low with nothing on the
-        # report to say the number was incomplete. Both of
-        # these come from the same helpers RunSummary.tasks_unpriced /
+        # Rows whose recorded spend is missing money, either because a turn's tokens
+        # could not be priced or because a hard kill lost the in-flight turn
+        # entirely. Worded cause-agnostically ("spend missing") because those two
+        # causes reach the same conclusion and the report cannot always tell which
+        # applied. Reported explicitly because the alternative is a total that reads
+        # materially low with nothing to say the number is incomplete. Both of these
+        # come from the same helpers RunSummary.tasks_unpriced /
         # eval_overhead_cost_usd use, so the report and run.json cannot disagree
         # about which rows lost money.
         unpriced = [t for t in task_results if row_cost_incomplete(t)]
@@ -574,13 +577,10 @@ def _generate_token_usage_section(task_results: list[dict[str, Any]]) -> list[st
         if total_cost is not None:
             cost_line = f"**Total Cost**: ${total_cost + sum(overhead):.4f}"
             if unpriced:
-                cost_line += f" (floor — {len(unpriced)} task(s) burned tokens the rate card could not price)"
+                cost_line += f" (floor — {len(unpriced)} task(s) have spend missing from this total)"
             lines.append(cost_line)
         elif unpriced:
-            lines.append(
-                f"**Total Cost**: unavailable — {len(unpriced)} task(s) burned tokens "
-                + "the rate card could not price"
-            )
+            lines.append(f"**Total Cost**: unavailable — {len(unpriced)} task(s) have spend missing from this total")
         lines.append(f"**Avg Tokens/Task**: {total_tokens // len(tasks_with_tokens):,}")
         lines.append("")
 
diff --git a/src/coder_eval/reports_experiment.py b/src/coder_eval/reports_experiment.py
index fa8497d4..bdaa794e 100644
--- a/src/coder_eval/reports_experiment.py
+++ b/src/coder_eval/reports_experiment.py
@@ -55,24 +55,30 @@ def _cost_complete(result: EvaluationResult) -> bool:
        backend prices a clean turn, so the rate card only matters for a turn the
        backend never priced (a killed partial). With no rate, that turn books
        tokens against no money.
-    2. **A task-level timeout preserved no turn at all.** ``TaskTimeoutError``
-       comes from the watchdog, which SIGKILLs the agent by PID; unlike a
-       turn-level timeout it never reaches ``_on_attempt_failure``, so no partial
-       turn is drained and the row lands with zero turns and zero tokens. A task
-       timeout means the evaluation loop was still running, so that spend is real
-       and simply unrecorded. Reporting such a row as fully priced would be a
-       false claim: it is the one case where cost is missing with no tokens to
-       point at.
+    2. **The task was hard-killed by the task-level timeout.** ``TaskTimeoutError``
+       comes from the watchdog, which SIGKILLs the agent by PID. Unlike a
+       turn-level timeout it never reaches ``_on_attempt_failure``, so the turn
+       that was in flight is never drained and its spend is not merely unpriced
+       but unrecorded: no tokens, no cost, nothing to point at.
+
+       Every ``TIMEOUT`` row is affected, not just the ones that look empty. The
+       watchdog fires while the evaluation loop is running, so there is always an
+       in-flight turn. A row that completed two dialog turns before the wall hit
+       still lost the third, and reporting it as fully priced because the first two
+       carry costs would be the same false claim in a less obvious costume.
 
     True for a row that burned nothing. An error before the agent ran genuinely
-    cost zero and must not be reported as missing cost, which is why case 2 is
-    keyed on ``TIMEOUT`` rather than on elapsed time: a fast setup failure and a
-    slow one are both free, while a task timeout is never free.
+    cost zero and must not be reported as missing cost, which is why case 2 keys on
+    the status rather than on elapsed time: a fast setup failure and a slow one are
+    both free, while a hard-killed task never is.
     """
-    priced = [usage for t in result.iterations if (usage := t.token_usage) is not None and not usage.is_empty()]
-    if not priced:
-        return result.final_status is not FinalStatus.TIMEOUT
-    return all(usage.total_cost_usd is not None for usage in priced)
+    if result.final_status is FinalStatus.TIMEOUT:
+        return False
+    return all(
+        usage.total_cost_usd is not None
+        for t in result.iterations
+        if (usage := t.token_usage) is not None and not usage.is_empty()
+    )
 
 
 def _judge_cost_usd(result: EvaluationResult) -> float | None:
diff --git a/tests/test_token_usage.py b/tests/test_token_usage.py
index 0f60af96..2b715583 100644
--- a/tests/test_token_usage.py
+++ b/tests/test_token_usage.py
@@ -886,7 +886,7 @@ def test_token_section_handles_missing_cost(self):
         # Tokens were burned, so there IS a bill — the report must say the number is
         # missing rather than omit the cost line, which reads as "this run was free".
         assert "**Total Cost**: unavailable" in joined
-        assert "1 task(s) burned tokens the rate card could not price" in joined
+        assert "1 task(s) have spend missing from this total" in joined
         assert "| task1 | 3,000 | 2,000 | 0 | 0 | 5,000 | N/A |" in joined
 
     def test_token_section_leaves_a_legacy_row_uncaveated(self):
@@ -903,7 +903,7 @@ def test_token_section_leaves_a_legacy_row_uncaveated(self):
         joined = "\n".join(ReportGenerator._generate_token_usage_section(task_results))
 
         assert "## Token Usage" in joined
-        assert "could not price" not in joined
+        assert "spend missing" not in joined
 
     def test_token_section_omits_cost_when_no_tokens_burned(self):
         """A row that burned nothing is genuinely free — no unpriced warning."""
@@ -935,7 +935,7 @@ def test_token_section_marks_partial_cost_as_floor(self):
         joined = "\n".join(ReportGenerator._generate_token_usage_section(task_results))
 
         assert "**Total Cost**: $0.2500 (floor" in joined
-        assert "1 task(s) burned tokens the rate card could not price" in joined
+        assert "1 task(s) have spend missing from this total" in joined
 
     def test_token_section_breaks_out_eval_overhead(self):
         """Judge spend is reported beside the agent bill, and folded into the total."""

From 56f3d4b75f891f276a65c61e077b9b0ea4150412 Mon Sep 17 00:00:00 2001
From: Bai Li <bai.li@uipath.com>
Date: Tue, 28 Jul 2026 20:46:30 -0700
Subject: [PATCH 15/20] fix(orchestrator): recover the in-flight turn's spend
 on a hard kill

A task-level timeout kills the agent and cancels the task awaiting
communicate(), so the turn in flight never returns a record. The Claude
agent finalized that cancel as COMPLETED, which keeps no record, so the
tokens it had already spent were dropped: the row landed with no turns, no
tokens and no cost for work that was billed.

The telemetry is intact at cancel time, so finalize as a crash instead,
which parks it on pending_turn under the existing contract. Codex and
Antigravity already did this; their fragment moves to a shared kernel on
the base class.

Nothing read that slot on this path either: the cancel is a BaseException,
so it never reaches the retry executor's per-attempt hook that drains it on
a turn-level timeout. The task-timeout handler now drains it, before
teardown clears the slot and before finalization, so the recovered turn
feeds token aggregation and command stats like any other.

Rows stay flagged cost_complete=false. Recovery captures everything the
event stream delivered, but the generation the agent was waiting on when it
died was never delivered by anyone, so the total is still a floor.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 src/coder_eval/agent.py                    | 18 +++++
 src/coder_eval/agents/antigravity_agent.py |  3 +-
 src/coder_eval/agents/claude_code_agent.py |  6 ++
 src/coder_eval/agents/codex_agent.py       |  3 +-
 src/coder_eval/orchestrator.py             | 48 +++++++++++++
 src/coder_eval/reports_experiment.py       | 16 +++--
 tests/test_agent_timeout.py                | 59 ++++++++++++++++
 tests/test_cost_accounting_paths.py        | 17 +++++
 tests/test_timeout_orchestrator.py         | 82 ++++++++++++++++++++++
 9 files changed, 241 insertions(+), 11 deletions(-)

diff --git a/src/coder_eval/agent.py b/src/coder_eval/agent.py
index 2b244ce2..aab36efd 100644
--- a/src/coder_eval/agent.py
+++ b/src/coder_eval/agent.py
@@ -147,6 +147,24 @@ def _finalize_and_raise_crash(
             raise AgentCrashError(message) from cause
         raise AgentCrashError(message)
 
+    def _finalize_external_cancel(self, finalize: _FinalizeFn) -> None:
+        """Mark ERROR and finalize an externally-cancelled turn as a crash. Does NOT raise.
+
+        A cancel that is not this turn's own timeout was delivered from outside the
+        turn — the task-level watchdog kills the agent and cancels the task running
+        it. Everything the event stream delivered is still intact at this point and
+        ``finalize`` reduces it into a complete record, but only the ``crashed``
+        branch parks that record on ``pending_turn``; finalizing as ``COMPLETED``
+        drops it, and the frame is unwinding so the return value goes nowhere
+        either. So a turn killed from outside must finalize as a crash for its
+        telemetry to survive at all.
+
+        The caller re-raises the ``CancelledError`` afterwards — this helper only
+        preserves the record, it does not alter cancellation semantics.
+        """
+        self._state = AgentState.ERROR
+        finalize(AgentEndStatus.CRASHED, crashed=True, crash_reason="turn cancelled")
+
     def _capture_partial_turn(self, collector: EventCollector) -> None:
         """Build the crashed partial ``TurnRecord`` into ``pending_turn`` (best-effort).
 
diff --git a/src/coder_eval/agents/antigravity_agent.py b/src/coder_eval/agents/antigravity_agent.py
index 8506154b..582eaf89 100644
--- a/src/coder_eval/agents/antigravity_agent.py
+++ b/src/coder_eval/agents/antigravity_agent.py
@@ -489,8 +489,7 @@ def _on_turn_timeout() -> None:
             raise
         except asyncio.CancelledError:
             if not state.finalized:
-                self._state = AgentState.ERROR
-                state.finalize(AgentEndStatus.CRASHED, crashed=True, crash_reason="turn cancelled")
+                self._finalize_external_cancel(state.finalize)
             raise
         except Exception as e:
             self._finalize_and_raise_crash(
diff --git a/src/coder_eval/agents/claude_code_agent.py b/src/coder_eval/agents/claude_code_agent.py
index 9c2e2887..606f809d 100644
--- a/src/coder_eval/agents/claude_code_agent.py
+++ b/src/coder_eval/agents/claude_code_agent.py
@@ -980,6 +980,12 @@ def _on_turn_timeout() -> None:
             if self._timed_out(state.timeout_hit, deadline):
                 assert timeout is not None
                 self._finalize_and_raise_timeout(state.finalize, timeout)
+            # Cancelled from outside this turn. Finalize as a crash so the turn's
+            # telemetry is parked on `pending_turn` for the caller to drain: without
+            # this the `finally` below finalizes as COMPLETED, which keeps no record,
+            # and the unwinding frame takes the only other copy with it.
+            if not state.finalized:
+                self._finalize_external_cancel(state.finalize)
             raise
         except ProcessError as e:
             # When the watchdog SIGKILLs the subprocess, the SDK surfaces it as a
diff --git a/src/coder_eval/agents/codex_agent.py b/src/coder_eval/agents/codex_agent.py
index 9be3274c..8b2f4be8 100644
--- a/src/coder_eval/agents/codex_agent.py
+++ b/src/coder_eval/agents/codex_agent.py
@@ -852,8 +852,7 @@ def _on_turn_timeout() -> None:
             # stays balanced and the pending-turn contract holds. finalize is
             # idempotent, so the timeout case is a no-op here.
             if not state.finalized:
-                self._state = AgentState.ERROR
-                state.finalize(AgentEndStatus.CRASHED, crashed=True, crash_reason="turn cancelled")
+                self._finalize_external_cancel(state.finalize)
             raise
         except Exception as e:
             # Catches failures OUTSIDE the inner turn block — notably thread_start
diff --git a/src/coder_eval/orchestrator.py b/src/coder_eval/orchestrator.py
index f5a5206e..2dcf8cf9 100644
--- a/src/coder_eval/orchestrator.py
+++ b/src/coder_eval/orchestrator.py
@@ -529,6 +529,13 @@ def _kill_agent_subprocess_sync() -> None:
                 )
 
                 logger.error(f"Task timed out: {e}")
+
+                # Recover the turn that was in flight when the watchdog killed the
+                # agent. Nothing else on this path does: the cancel is delivered as
+                # a BaseException, so it never reaches the retry executor's
+                # per-attempt failure hook that drains the slot on a turn-level
+                # timeout. Runs here, before teardown clears the slot.
+                await self._drain_killed_turn()
             except BudgetExceededError as e:
                 # Map token-budget breaches and cost-budget breaches to distinct
                 # statuses so per-task records preserve the failure mode.
@@ -612,6 +619,47 @@ def _kill_agent_subprocess_sync() -> None:
 
         return self.result
 
+    async def _drain_killed_turn(self) -> None:
+        """Move a hard-killed turn's partial record from the agent onto the result.
+
+        The task-level watchdog kills the agent and cancels the task running it, so
+        the in-flight turn never returns a record. What it managed to spend is real
+        and already recorded by the agent, which parks the partial on
+        ``pending_turn`` when it is cancelled from outside; this is the only reader
+        of that slot on the task-timeout path.
+
+        Ordering matters in both directions. It runs before ``_cleanup``, whose
+        ``agent.stop()`` clears the slot, and before ``_finalize_result``, so the
+        recovered turn is picked up by token aggregation, command statistics and
+        model resolution exactly like a turn that ended on its own.
+
+        Best-effort by design: a task killed before its first turn has nothing
+        parked, and this runs on the way to a saved row, so it must not raise.
+        """
+        if self.agent is None or self.result is None:
+            return
+        try:
+            partial = self.agent.pending_turn
+            # Type-checked because `pending_turn` is a slot any agent implementation
+            # fills, and a non-record here would fail validation during teardown and
+            # take the whole row down with it.
+            if not isinstance(partial, TurnRecord):
+                logger.debug("[%s] Hard-killed task preserved no partial turn", self.task.task_id)
+                return
+            self.result.iterations.append(partial)
+            await self.agent.discard_pending_turn()
+            usage = partial.token_usage
+            logger.info(
+                "[%s] Recovered the hard-killed turn: %d tokens, %s",
+                self.task.task_id,
+                usage.total_tokens if usage is not None else 0,
+                f"${usage.total_cost_usd:.4f}"
+                if usage is not None and usage.total_cost_usd is not None
+                else "unpriced",
+            )
+        except Exception:
+            logger.warning("[%s] Could not recover the hard-killed turn", self.task.task_id, exc_info=True)
+
     def _finalize_result(self, start_time: float) -> None:
         """Finalize the evaluation result: scores, telemetry, and persistence."""
         if not self.result:
diff --git a/src/coder_eval/reports_experiment.py b/src/coder_eval/reports_experiment.py
index bdaa794e..f75805ab 100644
--- a/src/coder_eval/reports_experiment.py
+++ b/src/coder_eval/reports_experiment.py
@@ -55,17 +55,19 @@ def _cost_complete(result: EvaluationResult) -> bool:
        backend prices a clean turn, so the rate card only matters for a turn the
        backend never priced (a killed partial). With no rate, that turn books
        tokens against no money.
-    2. **The task was hard-killed by the task-level timeout.** ``TaskTimeoutError``
-       comes from the watchdog, which SIGKILLs the agent by PID. Unlike a
-       turn-level timeout it never reaches ``_on_attempt_failure``, so the turn
-       that was in flight is never drained and its spend is not merely unpriced
-       but unrecorded: no tokens, no cost, nothing to point at.
+    2. **The task was hard-killed by the task-level timeout.** The agent is killed
+       mid-turn, so the generation it was waiting on is never delivered and its
+       tokens are never reported by anyone. ``Orchestrator._drain_killed_turn``
+       recovers everything the event stream had delivered up to the kill, which is
+       most of it, but the tail is not observable from inside the harness and no
+       amount of bookkeeping makes it so.
 
        Every ``TIMEOUT`` row is affected, not just the ones that look empty. The
        watchdog fires while the evaluation loop is running, so there is always an
        in-flight turn. A row that completed two dialog turns before the wall hit
-       still lost the third, and reporting it as fully priced because the first two
-       carry costs would be the same false claim in a less obvious costume.
+       still lost part of the third, and reporting it as fully priced because the
+       recovered turns carry costs would be the same false claim in a less obvious
+       costume.
 
     True for a row that burned nothing. An error before the agent ran genuinely
     cost zero and must not be reported as missing cost, which is why case 2 keys on
diff --git a/tests/test_agent_timeout.py b/tests/test_agent_timeout.py
index bd9fff7b..6c7e29ed 100644
--- a/tests/test_agent_timeout.py
+++ b/tests/test_agent_timeout.py
@@ -258,3 +258,62 @@ async def mock_query(prompt, options):
         ):
             await agent.communicate("prompt")  # no timeout
             mock_transport_cls.assert_not_called()
+
+
+class _MockTextBlock:
+    def __init__(self, text: str) -> None:
+        self.text = text
+
+
+class _MockAssistantMessage:
+    """Duck-typed SDK AssistantMessage carrying one text block and a usage dict."""
+
+    def __init__(self, text: str, usage: dict[str, int]) -> None:
+        self.content = [_MockTextBlock(text)]
+        self.model = "claude-sonnet-5"
+        self.usage = usage
+        self.stop_reason = "end_turn"
+        self.message_id = "msg_partial_1"
+
+
+@pytest.mark.asyncio
+async def test_external_cancel_parks_the_turn_it_interrupted():
+    """A turn cancelled from outside must leave its telemetry on ``pending_turn``.
+
+    This is the task-level timeout's kill path: the watchdog kills the agent and
+    cancels the task awaiting ``communicate()``, so the turn never returns a record
+    and the frame that held one unwinds. Whatever it spent before the kill is real
+    and billed, so the pending slot is the only place it can survive.
+    """
+    config = parse_agent_config(type=AgentKind.CLAUDE_CODE, permission_mode="acceptEdits", model="claude-sonnet-5")
+    agent = ClaudeCodeAgent(config)
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        await agent.start(tmpdir)
+
+        streaming = asyncio.Event()
+
+        async def mock_query(prompt, options):
+            yield _MockAssistantMessage("working on it", {"input_tokens": 40_000, "output_tokens": 2_000})
+            streaming.set()
+            # Still mid-turn when the cancel lands, as on a real hard kill: no
+            # terminal ResultMessage is ever delivered.
+            await asyncio.sleep(30)
+
+        with patch("coder_eval.agents.claude_code_agent.query", mock_query):
+            turn = asyncio.create_task(agent.communicate("prompt"))
+            await streaming.wait()
+            turn.cancel()
+            with pytest.raises(asyncio.CancelledError):
+                await turn
+
+    partial = agent.pending_turn
+    assert partial is not None, "the interrupted turn's record was discarded"
+    assert partial.crashed is True
+    usage = partial.token_usage
+    assert usage is not None
+    assert usage.output_tokens == 2_000
+    # No ResultMessage means the backend never priced this turn, so the cost is
+    # backfilled from the buckets at the model's rate.
+    assert usage.total_cost_usd is not None
+    assert usage.total_cost_usd > 0
diff --git a/tests/test_cost_accounting_paths.py b/tests/test_cost_accounting_paths.py
index 0d825a76..1393b634 100644
--- a/tests/test_cost_accounting_paths.py
+++ b/tests/test_cost_accounting_paths.py
@@ -129,6 +129,23 @@ def test_cost_complete_false_when_a_task_timeout_preserved_no_turn(self):
         result.final_status = FinalStatus.TIMEOUT
         assert eval_result_to_task_dict(result)["cost_complete"] is False
 
+    def test_recovered_timeout_row_reports_its_cost_and_still_flags_incomplete(self):
+        """Recovering the killed turn narrows the gap; it does not close it.
+
+        The interrupted turn is drained onto the row and priced, so the row carries
+        a real number instead of nothing. The generation the agent was waiting on
+        when it died was never delivered by anyone, so the number is still a floor.
+        """
+        killed = _turn(1, TokenUsage(uncached_input_tokens=40_000, output_tokens=2_000, total_cost_usd=0.15))
+        killed.crashed = True
+        result = _result([killed])
+        result.final_status = FinalStatus.TIMEOUT
+        result.total_token_usage = TokenUsage(uncached_input_tokens=40_000, output_tokens=2_000, total_cost_usd=0.15)
+
+        row = eval_result_to_task_dict(result)
+        assert row["total_cost_usd"] == pytest.approx(0.15)
+        assert row["cost_complete"] is False
+
     def test_cost_complete_true_for_a_fast_error_with_no_turn(self):
         """The companion: a setup failure has no turns either, and really is free.
 
diff --git a/tests/test_timeout_orchestrator.py b/tests/test_timeout_orchestrator.py
index cac28b1b..b0f6696a 100644
--- a/tests/test_timeout_orchestrator.py
+++ b/tests/test_timeout_orchestrator.py
@@ -16,6 +16,7 @@
     FileExistsCriterion,
     SandboxConfig,
     TaskDefinition,
+    TokenUsage,
     TurnRecord,
 )
 from coder_eval.orchestrator import Orchestrator
@@ -227,6 +228,87 @@ async def slow_loop():
     mock_agent.kill_sync.assert_called_once()
 
 
+@pytest.mark.asyncio
+async def test_task_timeout_recovers_the_killed_turn(tmp_path) -> None:
+    """A hard-killed task's spend lands on the result instead of vanishing.
+
+    The agent parks the interrupted turn on ``pending_turn`` when it is cancelled,
+    and the task-timeout handler is the only reader of that slot: the cancel is a
+    BaseException, so it never reaches the retry executor's per-attempt hook that
+    drains it on a turn-level timeout. Without the drain the row reports no turns
+    and no cost for a task that spent real money.
+    """
+    task = _make_task(task_timeout=0.1)
+    run_dir = tmp_path / "run" / "timeout_test"
+    run_dir.mkdir(parents=True)
+
+    orchestrator = Orchestrator(task=task, run_dir=run_dir, variant_id="test-variant")
+    orchestrator._setup = AsyncMock()  # type: ignore[method-assign]
+    orchestrator._cleanup = AsyncMock()  # type: ignore[method-assign]
+
+    partial = TurnRecord(
+        iteration=1,
+        user_input="test prompt",
+        agent_output="<partial record>",
+        crashed=True,
+        token_usage=TokenUsage(uncached_input_tokens=40_000, output_tokens=2_000, total_cost_usd=0.15),
+    )
+
+    mock_agent = MagicMock()
+    mock_agent.pending_turn = partial
+    mock_agent.discard_pending_turn = AsyncMock()
+    mock_agent.get_sdk_options = MagicMock(return_value=None)
+    orchestrator.agent = mock_agent
+
+    async def slow_loop():
+        await asyncio.sleep(10)
+        return False
+
+    orchestrator._evaluation_loop = slow_loop  # type: ignore[method-assign]
+
+    result = await orchestrator.run()
+
+    assert result.final_status == "TIMEOUT"
+    assert result.iterations == [partial]
+    assert result.total_token_usage is not None
+    assert result.total_token_usage.output_tokens == 2_000
+    assert result.total_token_usage.total_cost_usd == pytest.approx(0.15)
+    # Drained through the documented contract, so the slot is left clean.
+    mock_agent.discard_pending_turn.assert_awaited_once()
+
+
+@pytest.mark.asyncio
+async def test_task_timeout_with_nothing_to_recover_still_lands(tmp_path) -> None:
+    """A task killed before its first turn has nothing parked, and that is not an error.
+
+    The recovery is best-effort and runs on the way to a saved row, so an empty slot
+    must leave the TIMEOUT row intact rather than raising through teardown.
+    """
+    task = _make_task(task_timeout=0.1)
+    run_dir = tmp_path / "run" / "timeout_test"
+    run_dir.mkdir(parents=True)
+
+    orchestrator = Orchestrator(task=task, run_dir=run_dir, variant_id="test-variant")
+    orchestrator._setup = AsyncMock()  # type: ignore[method-assign]
+    orchestrator._cleanup = AsyncMock()  # type: ignore[method-assign]
+
+    mock_agent = MagicMock()
+    mock_agent.pending_turn = None
+    mock_agent.get_sdk_options = MagicMock(return_value=None)
+    orchestrator.agent = mock_agent
+
+    async def slow_loop():
+        await asyncio.sleep(10)
+        return False
+
+    orchestrator._evaluation_loop = slow_loop  # type: ignore[method-assign]
+
+    result = await orchestrator.run()
+
+    assert result.final_status == "TIMEOUT"
+    assert result.iterations == []
+
+
 @pytest.mark.asyncio
 async def test_turn_timeout_not_rewrapped_as_task_timeout(tmp_path) -> None:
     """A per-turn TurnTimeoutError propagates unchanged even when a larger

From bcdf0a82f5c5c27a7b4855a558ec82096c583385 Mon Sep 17 00:00:00 2001
From: Bai Li <bai.li@uipath.com>
Date: Wed, 29 Jul 2026 09:57:00 -0700
Subject: [PATCH 16/20] fix(pricing): refresh the rate card and correct
 gemini-3-flash-preview

gemini-3-flash-preview carried gemini-3.5-flash's rates ($1.50/$9.00), so
every run on it was costed roughly 3x too high; its real rate is
$0.50/$3.00. Adds gemini-3.6-flash plus the 3.5/3.1 Flash-Lite tiers, and
picks up an OpenRouter price drop on z-ai/glm-5.2.

Anthropic, OpenAI, and the Bedrock open-weight entries were re-checked
against their vendor rate cards and needed no value changes; the Bedrock
rates are eu-north-1 and now say so, so they are not "corrected" against
the US column later.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 src/coder_eval/pricing.py       | 45 +++++++++++++++++++++------------
 tests/test_antigravity_agent.py |  4 +++
 2 files changed, 33 insertions(+), 16 deletions(-)

diff --git a/src/coder_eval/pricing.py b/src/coder_eval/pricing.py
index f4b572d7..86a89eab 100644
--- a/src/coder_eval/pricing.py
+++ b/src/coder_eval/pricing.py
@@ -1,8 +1,9 @@
 """Model pricing for cost calculation.
 
-Anthropic/OpenAI built-in rates; plugins contribute additional rates via
+Anthropic/OpenAI/Google built-in rates; plugins contribute additional rates via
 ``register_pricing()``. Prices are per million tokens (MTok).
-Source: https://www.anthropic.com/pricing
+Sources: https://claude.com/pricing#api, https://developers.openai.com/api/docs/pricing,
+https://ai.google.dev/gemini-api/docs/pricing (all verified 2026-07-29).
 """
 
 from collections.abc import Iterable
@@ -19,7 +20,7 @@ class ModelPricing:
     cache_read_per_mtok: float  # prompt caching read
 
 
-# Official Anthropic pricing as of 2025
+# Official vendor rate cards, verified 2026-07-29.
 # Key: CLI model name (before gateway mapping)
 _PRICING: dict[str, ModelPricing] = {
     # Claude 5 Fable / Mythos: $10/$50.
@@ -67,8 +68,8 @@ class ModelPricing:
     # Claude 3 Haiku
     "claude-3-haiku-20240307": ModelPricing(0.25, 1.25, 0.30, 0.03),
     # OpenAI GPT-5 / Codex (direct or via Azure OpenAI). Released 2025-09-23.
-    # Source: https://openai.com/api/pricing (gpt-5-codex: $1.25/M in,
-    # $0.125/M cached in, $10/M out). OpenAI does not bill cache writes
+    # Source: https://developers.openai.com/api/docs/pricing (gpt-5-codex: $1.25/M
+    # in, $0.125/M cached in, $10/M out). OpenAI does not bill cache writes
     # separately, so cache_write == input rate.
     "gpt-5-codex": ModelPricing(1.25, 10.0, 1.25, 0.125),
     "gpt-5": ModelPricing(1.25, 10.0, 1.25, 0.125),
@@ -97,21 +98,33 @@ class ModelPricing:
     "gpt-5.6-terra": ModelPricing(2.5, 15.0, 2.5, 0.25),
     "gpt-5.6-luna": ModelPricing(1.0, 6.0, 1.0, 0.10),
     # Google Gemini (AntigravityAgent, via the Gemini Developer API). Per-MTok
-    # rates from ai.google.dev/gemini-api/docs/pricing (2026). Gemini bills no
-    # separate cache-WRITE fee, so cache_write == input (the agent maps
-    # cache_creation_tokens to 0, so this value is effectively unused); cache_read
-    # is the cached-input rate (~10% of input). CAVEAT: Pro's >200K-token tier is
-    # higher ($4/$18); this flat rate reads low for very-large-context runs — fine
-    # for typical eval tasks. Keyed on the bare model id in agent.model.
-    "gemini-3-pro-preview": ModelPricing(2.0, 12.0, 2.0, 0.20),
+    # rates from ai.google.dev/gemini-api/docs/pricing. Gemini bills no separate
+    # cache-WRITE fee, so cache_write == input (the agent maps cache_creation_tokens
+    # to 0, so this value is effectively unused); cache_read is the cached-input
+    # rate (~10% of input). Keyed on the bare model id in agent.model — the ids
+    # below are the literal codes returned by the ListModels endpoint.
+    # Two CAVEATS, both of which read LOW rather than high:
+    #  - Pro's >200K-token tier costs more ($4/$18 in/out, $0.40 cached). Fine for
+    #    typical eval tasks; revisit if benchmarking very-large-context runs.
+    #  - Audio input is billed above the text/image/video rate on the Flash-Lite and
+    #    3-Flash-Preview tiers. Coding agents send no audio, so the text rate applies.
+    "gemini-3.6-flash": ModelPricing(1.5, 7.5, 1.5, 0.15),
+    "gemini-3.5-flash": ModelPricing(1.5, 9.0, 1.5, 0.15),
+    "gemini-3.5-flash-lite": ModelPricing(0.30, 2.5, 0.30, 0.03),
     "gemini-3.1-pro-preview": ModelPricing(2.0, 12.0, 2.0, 0.20),
     "gemini-3.1-pro-preview-customtools": ModelPricing(2.0, 12.0, 2.0, 0.20),
-    "gemini-3.5-flash": ModelPricing(1.5, 9.0, 1.5, 0.15),
-    "gemini-3-flash-preview": ModelPricing(1.5, 9.0, 1.5, 0.15),
+    "gemini-3.1-flash-lite": ModelPricing(0.25, 1.5, 0.25, 0.025),
+    "gemini-3.1-flash-lite-preview": ModelPricing(0.25, 1.5, 0.25, 0.025),
+    "gemini-3-flash-preview": ModelPricing(0.50, 3.0, 0.50, 0.05),
+    # Gemini 3 Pro Preview has been dropped from the public rate card (superseded by
+    # 3.1 Pro). Its last published rate is kept so historical runs still price.
+    "gemini-3-pro-preview": ModelPricing(2.0, 12.0, 2.0, 0.20),
     # Open-weight models on Bedrock (eu-north-1), driven via the LiteLLM backend.
+    # These are the Stockholm rates, a ~20% premium over the us-east-1 rate card —
+    # do NOT "correct" them against the US column of the AWS pricing page.
     # Bedrock lists no prompt-cache read/write rate for these, so cache-creation
     # is priced at the input rate and cache-read at 0 (conservative — see the
-    # per-provider cost-accounting caveat; revisit against the AWS model cards).
+    # per-provider cost-accounting caveat).
     "deepseek.v3.2": ModelPricing(0.74, 2.22, 0.74, 0.0),
     "zai.glm-5": ModelPricing(1.2, 3.84, 1.2, 0.0),
     "moonshotai.kimi-k2.5": ModelPricing(0.72, 3.6, 0.72, 0.0),
@@ -121,7 +134,7 @@ class ModelPricing:
     # at OpenRouter's published input_cache_read rate. Rates per OpenRouter's
     # /models endpoint (per-token x 1e6).
     "moonshotai/kimi-k3": ModelPricing(3.0, 15.0, 3.0, 0.30),
-    "z-ai/glm-5.2": ModelPricing(0.826, 2.596, 0.826, 0.1534),
+    "z-ai/glm-5.2": ModelPricing(0.7168, 2.2528, 0.7168, 0.13312),
     "deepseek/deepseek-v4-pro": ModelPricing(0.435, 0.87, 0.435, 0.003625),
 }
 
diff --git a/tests/test_antigravity_agent.py b/tests/test_antigravity_agent.py
index 4ac68408..99747950 100644
--- a/tests/test_antigravity_agent.py
+++ b/tests/test_antigravity_agent.py
@@ -202,7 +202,11 @@ def test_tool_name_map_covers_core_builtins():
         "gemini-3.1-pro-preview",
         "gemini-3.1-pro-preview-customtools",
         "gemini-3-pro-preview",
+        "gemini-3.6-flash",
         "gemini-3.5-flash",
+        "gemini-3.5-flash-lite",
+        "gemini-3.1-flash-lite",
+        "gemini-3.1-flash-lite-preview",
         "gemini-3-flash-preview",
     ],
 )

From 30bb508a76a3c269934056b71b674da3b2da5d80 Mon Sep 17 00:00:00 2001
From: Bai Li <bai.li@uipath.com>
Date: Wed, 29 Jul 2026 09:59:11 -0700
Subject: [PATCH 17/20] fix(pricing): add the five unpriced codex tiers still
 on OpenAI's rate card

A coverage sweep over every model id the harness can emit found gpt-5.2-codex,
gpt-5.1-codex-max, gpt-5.1-codex, gpt-5.1-codex-mini, and codex-mini-latest
absent from the table. agent.model is a free-form string, so pinning any of
them booked the run's tokens against no money.

codex-mini-latest is the one OpenAI entry whose cached rate is 25% of input
rather than 10%, so it is called out in a comment.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 src/coder_eval/pricing.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/src/coder_eval/pricing.py b/src/coder_eval/pricing.py
index 86a89eab..63d770a3 100644
--- a/src/coder_eval/pricing.py
+++ b/src/coder_eval/pricing.py
@@ -73,8 +73,17 @@ class ModelPricing:
     # separately, so cache_write == input rate.
     "gpt-5-codex": ModelPricing(1.25, 10.0, 1.25, 0.125),
     "gpt-5": ModelPricing(1.25, 10.0, 1.25, 0.125),
-    # gpt-5.3-codex (2026-02-24): $1.75/M input, $0.175/M cached, $14/M output.
+    # Older codex tiers still on OpenAI's rate card — a task may pin any of them,
+    # and an unpriced model books its tokens against no money.
+    "gpt-5.1-codex-max": ModelPricing(1.25, 10.0, 1.25, 0.125),
+    "gpt-5.1-codex": ModelPricing(1.25, 10.0, 1.25, 0.125),
+    "gpt-5.1-codex-mini": ModelPricing(0.25, 2.0, 0.25, 0.025),
+    # codex-mini-latest is the one OpenAI entry whose cached rate is NOT 10% of
+    # input — it is 25% ($0.375 against $1.50). Do not "normalize" it.
+    "codex-mini-latest": ModelPricing(1.50, 6.0, 1.50, 0.375),
+    # gpt-5.3-codex / gpt-5.2-codex: $1.75/M input, $0.175/M cached, $14/M output.
     "gpt-5.3-codex": ModelPricing(1.75, 14.0, 1.75, 0.175),
+    "gpt-5.2-codex": ModelPricing(1.75, 14.0, 1.75, 0.175),
     # gpt-5.4 (2026-03-05): $2.50/M input, $0.25/M cached, $15/M output.
     "gpt-5.4": ModelPricing(2.5, 15.0, 2.5, 0.25),
     # gpt-5.5: $5/M input, $0.50/M cached, $30/M output.

From 5e8b89931d44c75f096b69f6debe526447a6a730 Mon Sep 17 00:00:00 2001
From: Bai Li <bai.li@uipath.com>
Date: Wed, 29 Jul 2026 11:15:00 -0700
Subject: [PATCH 18/20] feat(cost): publish one accurate total on every
 reporting surface

A run's bill was agent-only everywhere except the markdown report's Total Cost
line, which added the judge and simulator itself. Nothing published the sum, so
run.json carried no run-level total at all and the per-task Cost column did not
add up to the total printed above it.

full_cost() is now the one way cost components combine: it skips an unpriced
component instead of counting it as zero, so a partial total is a floor and
never an exception. It backs full_cost_usd on both the task row and RunSummary,
and feeds the markdown report, its per-task Cost column, and the task and
variant HTML. total_cost_usd keeps meaning agent-only at both levels: it is what
stays comparable across harnesses, and redefining it would move every historical
figure. The dashboard is untouched.

The pre-flight also covers criterion.model now. No judge backend reports a cost,
so the rate card is the only source for a judge call, unlike an agent turn the
SDK prices itself; and a criterion model is always pinned in the task YAML, so
unlike agent.model it can never hide behind the route. An unpriced judge or
simulator lowers the total and is warned about at dispatch, rather than failing
the run.

The EvaluationResult cost derivations move to models/results.py beside the model
they derive from, which is also what lets reports_html read them without an
import cycle through reports_experiment.

tasks_unpriced is renamed tasks_cost_incomplete: it counts hard-killed rows too,
which are frequently priced. eval_overhead_costs returns a total rather than a
list both callers immediately summed.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 src/coder_eval/models/__init__.py     |  14 +-
 src/coder_eval/models/results.py      | 181 +++++++++++++++-----------
 src/coder_eval/orchestration/batch.py |  47 ++++---
 src/coder_eval/reports.py             |  64 ++++-----
 src/coder_eval/reports_experiment.py  | 139 ++++++--------------
 src/coder_eval/reports_html.py        |  18 +--
 tests/test_cost_accounting_paths.py   | 126 ++++++++++++++++--
 tests/test_run_metrics.py             |   6 +-
 8 files changed, 338 insertions(+), 257 deletions(-)

diff --git a/src/coder_eval/models/__init__.py b/src/coder_eval/models/__init__.py
index 7e6157c0..c50e5af1 100644
--- a/src/coder_eval/models/__init__.py
+++ b/src/coder_eval/models/__init__.py
@@ -127,8 +127,12 @@
     TaskConfigRecord,
     ThresholdCheck,
     TurnRecord,
-    eval_overhead_costs,
+    eval_overhead_cost,
+    eval_result_full_cost,
+    full_cost,
+    judge_cost_usd,
     row_cost_incomplete,
+    simulator_cost_usd,
 )
 
 # Routing
@@ -293,10 +297,14 @@
     "TaskConfigRecord",
     "RunSummary",
     "SkippedTask",
-    # Row-level cost predicates, shared by RunSummary's computed fields and the
+    # Row-level cost helpers, shared by RunSummary's computed fields and the
     # reports so every surface agrees on which rows lost money.
     "row_cost_incomplete",
-    "eval_overhead_costs",
+    "eval_overhead_cost",
+    "full_cost",
+    "eval_result_full_cost",
+    "judge_cost_usd",
+    "simulator_cost_usd",
     # Judge defaults
     "DEFAULT_JUDGE_MODEL",
     # Judge
diff --git a/src/coder_eval/models/results.py b/src/coder_eval/models/results.py
index 9f01e0ad..02409cf9 100644
--- a/src/coder_eval/models/results.py
+++ b/src/coder_eval/models/results.py
@@ -850,50 +850,96 @@ class SkippedTask(BaseModel):
 def row_cost_incomplete(row: Mapping[str, Any]) -> bool:
     """True when a task row's recorded spend is missing money.
 
-    Reads the row's own ``cost_complete``, which the row projection sets by asking
-    whether every turn that burned tokens also carries a cost. A row that burned
-    nothing is complete, not unpriced: an error before the agent ran genuinely cost
-    nothing.
-
-    Absent means complete. Rows written before the field existed are read as
-    priced rather than inferred from their token counts: guessing at the past would
-    buy a caveat on old runs at the cost of a second definition of "unpriced",
-    which is the kind of drift this helper exists to prevent.
-
-    Defined once here, on the row schema, because every surface that reports cost
-    has to apply the same test. Reports and ``RunSummary`` disagreeing about which
-    rows lost money would put the framework back where it started.
+    Reads the row's own ``cost_complete`` flag. Absent means complete: rows written
+    before the field existed are read as priced rather than inferred from their
+    token counts, which would be a second definition of the same predicate.
+
+    Defined here, on the row schema, so the reports and ``RunSummary`` cannot
+    disagree about which rows lost money.
     """
     return row.get("cost_complete") is False
 
 
-def eval_overhead_costs(rows: Iterable[Mapping[str, Any]]) -> list[float]:
-    """Every judge / simulator cost present across ``rows``, unsummed.
+def full_cost(*components: float | None) -> float | None:
+    """Total of whichever cost components were priced. ``None`` when none were.
+
+    The one way cost components are combined, so every surface adds them up the
+    same way. Never raises and never invents a zero: an unpriced component is
+    skipped, which makes the result a floor rather than a failure.
+    """
+    priced = [c for c in components if c is not None]
+    return sum(priced) if priced else None
+
+
+def eval_overhead_cost(rows: Iterable[Mapping[str, Any]]) -> float | None:
+    """Total judge + simulator spend across ``rows``, or ``None`` if none was recorded.
+
+    ``None`` rather than ``0.0`` so "no judge ran" stays distinct from "a judge ran free".
+    """
+    return full_cost(*(row.get(key) for row in rows for key in ("judge_cost_usd", "simulator_cost_usd")))
+
+
+def judge_cost_usd(result: EvaluationResult) -> float | None:
+    """Sum the priced judge spend across an evaluation's criterion results.
+
+    Covers both flavors: ``llm_judge`` prices its own one-shot call from the
+    criterion's model, ``agent_judge`` inherits the SDK's cost on the sub-agent's
+    turn. ``None`` when no criterion reported cost.
+    """
+    usages = [u for cr in result.success_criteria_results if (u := getattr(cr, "token_usage", None)) is not None]
+    return full_cost(*(u.total_cost_usd for u in usages))
+
+
+def simulator_cost_usd(result: EvaluationResult) -> float | None:
+    """Price an evaluation's simulator turns. ``None`` outside simulation mode.
+
+    Priced at the ROUTE's model, not the subject's: ``UserSimulator`` pins
+    ``model=None`` so it resolves to ``BEDROCK_MODEL``, which differs from the
+    subject on any task that pins ``agent.model``.
 
-    Returned as a list rather than a total because callers need to distinguish
-    "no overhead was recorded" from "overhead was recorded and came to $0".
+    A floor. ``UserSimulator`` records only ``uncached_input_tokens`` and drops
+    both cache buckets, so a cached prefix is largely absent from the count.
+    """
+    from coder_eval.pricing import calculate_cost
+
+    sim = result.simulation
+    if sim is None or not (sim.simulator_input_tokens or sim.simulator_output_tokens):
+        return None
+    route_model = (result.environment_info or {}).get("bedrock_model")
+    # Falls back to the subject's model on a non-Bedrock route, where the SDK picks
+    # its own default and nothing on the record names it.
+    model = route_model if isinstance(route_model, str) and route_model else result.model_used
+    if not model:
+        return None
+    return calculate_cost(
+        model,
+        uncached_input_tokens=sim.simulator_input_tokens,
+        output_tokens=sim.simulator_output_tokens,
+    )
+
+
+def eval_result_full_cost(result: EvaluationResult) -> float | None:
+    """Everything one evaluation cost: agent + judge + simulator.
+
+    ``None`` when nothing could be priced, a floor when only part of it could. The
+    same figure the row projection publishes as ``full_cost_usd``, for the surfaces
+    that hold an ``EvaluationResult`` rather than a row dict.
     """
-    return [c for row in rows for key in ("judge_cost_usd", "simulator_cost_usd") if (c := row.get(key)) is not None]
+    agent = result.total_token_usage.total_cost_usd if result.total_token_usage else None
+    return full_cost(agent, judge_cost_usd(result), simulator_cost_usd(result))
 
 
 class RunSummary(BaseModel):
     """Summary of an entire evaluation run across multiple tasks.
 
-    ``pass_rate`` is ``tasks_succeeded / tasks_run``: **every dispatched task is in
-    the denominator, errors included as misses.** An error is not a free pass. The
-    previous formula excluded errors, which paid a bonus for erroring: a harness
-    that fell over enough could render as a perfect score while passing almost
-    nothing. A denominator that shrinks when a run goes wrong is not a metric you
-    can set beside another run's. ``error_share`` says how much of the rate is
-    errors, so a bad infrastructure night *shows* instead of being absorbed.
-
-    This is the single denominator for the whole framework. Every reporting
-    surface reads ``pass_rate`` rather than re-deriving one: independent
-    re-derivations drift, and consumers reading the same run.json then publish
-    different rates for it.
-
-    Every derived metric here is computed, never stored, so it cannot drift from
-    the counts it comes from.
+    ``pass_rate`` is ``tasks_succeeded / tasks_run``: every dispatched task is in the
+    denominator, errors included as misses. The previous formula excluded errors,
+    which paid a bonus for erroring. ``error_share`` reports how much of the rate is
+    errors, so a bad infrastructure night shows instead of being absorbed.
+
+    This is the framework's single denominator: every reporting surface reads
+    ``pass_rate`` rather than re-deriving one. Derived metrics here are computed,
+    never stored, so they cannot drift from the counts they come from.
     """
 
     run_id: str = Field(description="Run identifier (timestamp like '2025-10-09_15-30-45')")
@@ -958,24 +1004,15 @@ def _check_task_count_invariant(self) -> RunSummary:
             raise ValueError(f"Task count invariant violated: {total} != {self.tasks_run}")
         return self
 
-    # ------------------------------------------------------------------
-    # Derived run metrics.
-    #
-    # Every one is a computed_field over the stored counts and ``task_results``,
-    # so it serializes into run.json for downstream consumers (dashboards,
-    # external runners) while staying impossible to set to something the rows
-    # disagree with. Consumers should READ these rather than re-deriving them:
-    # every independent re-derivation is another denominator that can drift.
-    # ------------------------------------------------------------------
+    # Derived run metrics: computed_fields over the stored counts and
+    # ``task_results``, so they serialize into run.json while staying impossible to
+    # set to something the rows disagree with. Consumers should read these rather
+    # than re-derive them.
 
     @computed_field  # type: ignore[prop-decorator]
     @property
     def pass_rate(self) -> float | None:
-        """``tasks_succeeded / tasks_run`` as a 0-1 fraction. ``None`` on an empty run.
-
-        The run's one pass rate. Errors are in the denominator as misses — see the
-        class docstring for why there is no second, error-excluding figure.
-        """
+        """``tasks_succeeded / tasks_run`` as a 0-1 fraction. ``None`` on an empty run."""
         return self.tasks_succeeded / self.tasks_run if self.tasks_run else None
 
     @computed_field  # type: ignore[prop-decorator]
@@ -983,56 +1020,50 @@ class docstring for why there is no second, error-excluding figure.
     def error_share(self) -> float | None:
         """``tasks_error / tasks_run`` as a 0-1 fraction. ``None`` on an empty run.
 
-        How much of ``pass_rate`` is being held down by rows that never produced a
-        gradeable attempt. Diagnostic only: it does not adjust the rate. A run whose
-        rate dropped with a high ``error_share`` had an infrastructure night; the same
-        drop at a normal share is the model. Per-row ``error_message`` /
-        ``error_category`` say which.
+        Diagnostic only, never adjusts the rate: a drop at a high error share is an
+        infrastructure night, the same drop at a normal share is the model.
         """
         return self.tasks_error / self.tasks_run if self.tasks_run else None
 
     @computed_field  # type: ignore[prop-decorator]
     @property
-    def tasks_unpriced(self) -> int:
-        """Rows whose recorded spend is incomplete.
-
-        Each one is real money missing from ``agent_cost_usd``, because the rate
-        card is a static table baked into the installed version: a model released
-        after it prices as ``null`` on every turn, with only a log line to say so.
-        ``row_cost_incomplete`` defines what counts, and the report applies the
-        same helper so the two can never disagree.
-        """
+    def tasks_cost_incomplete(self) -> int:
+        """Rows with money missing from their recorded spend (see ``row_cost_incomplete``)."""
         return sum(1 for row in self.task_results if row_cost_incomplete(row))
 
     @computed_field  # type: ignore[prop-decorator]
     @property
     def cost_complete(self) -> bool:
         """False when any row's spend is incomplete, so every cost total here is a floor."""
-        return self.tasks_unpriced == 0
+        return self.tasks_cost_incomplete == 0
 
     @computed_field  # type: ignore[prop-decorator]
     @property
     def agent_cost_usd(self) -> float | None:
         """Subject-agent spend across the run. ``None`` when no row reported cost.
 
-        Excludes evaluation machinery: that is ``eval_overhead_cost_usd``, and the
-        two are published as a pair because a run's true bill is their sum. Read
-        alongside ``cost_complete``: with unpriced rows present this is a floor,
-        not the bill.
+        Excludes evaluation machinery (``eval_overhead_cost_usd``); ``full_cost_usd``
+        is the two added together. A floor when ``cost_complete`` is False.
         """
-        costs = [c for row in self.task_results if (c := row.get("total_cost_usd")) is not None]
-        return sum(costs) if costs else None
+        return full_cost(*(row.get("total_cost_usd") for row in self.task_results))
 
     @computed_field  # type: ignore[prop-decorator]
     @property
     def eval_overhead_cost_usd(self) -> float | None:
         """Judge + simulator spend across the run. ``None`` when neither reported cost.
 
-        Deliberately NOT folded into ``agent_cost_usd``: comparing harnesses
-        means comparing what the agents cost, and the judge bill is a property of
-        the suite's criteria, identical across harnesses. But it is real money
-        and was previously reported nowhere, so a run's true bill is the two
-        added together.
+        Kept out of ``agent_cost_usd`` on purpose: the judge bill is a property of the
+        suite's criteria and identical across harnesses, so folding it in would make
+        two harnesses look closer than they are.
+        """
+        return eval_overhead_cost(self.task_results)
+
+    @computed_field  # type: ignore[prop-decorator]
+    @property
+    def full_cost_usd(self) -> float | None:
+        """What the run actually cost: agent + judge + simulator.
+
+        The number to quote for a run's bill. ``None`` when nothing could be priced,
+        and a floor rather than an error when only some of it could.
         """
-        costs = eval_overhead_costs(self.task_results)
-        return sum(costs) if costs else None
+        return full_cost(self.agent_cost_usd, self.eval_overhead_cost_usd)
diff --git a/src/coder_eval/orchestration/batch.py b/src/coder_eval/orchestration/batch.py
index 8952427b..8576cdfc 100644
--- a/src/coder_eval/orchestration/batch.py
+++ b/src/coder_eval/orchestration/batch.py
@@ -12,7 +12,7 @@
 import json
 import logging
 import shutil
-from collections.abc import Callable
+from collections.abc import Callable, Iterator
 from datetime import datetime
 from pathlib import Path
 from typing import Any
@@ -39,42 +39,41 @@
 logger = logging.getLogger(__name__)
 
 
-def check_pricing_coverage(resolved_tasks: list[ResolvedTask]) -> list[str]:
-    """Pre-flight the rate card against the models this run will use.
+def _run_models(resolved_tasks: list[ResolvedTask]) -> Iterator[str | None]:
+    """Every model id this run pins up front: subject agents and judge criteria.
 
-    The rate card is a static table baked into the installed framework version, so
-    a model released after that version has no rate here. Whether that costs you
-    anything depends on the turn: an agent whose backend reports its own cost (the
-    Claude Code SDK does) still prices a clean turn, so most rows look fine. The
-    rate card is the FALLBACK, and it is the only source for a turn the backend
-    never priced — a timed-out or killed partial, which arrives with full token
-    counts and no cost. With no rate, that fallback is a no-op and the tokens book
-    no money at all.
+    A criterion's model is always priced from the rate card (no judge backend
+    reports a cost), so it matters more here than the agent's, which only falls
+    back to the card on a turn the backend never priced.
+    """
+    for rt in resolved_tasks:
+        yield rt.task.agent.model if rt.task.agent else None
+        for criterion in rt.task.success_criteria:
+            yield getattr(criterion, "model", None)
 
-    That is not a rare corner: on a large suite the killed tail is where the
-    biggest token counts live, since a task that ran to the wall spent the most
-    getting there.
 
-    A warning rather than a refusal, so a brand-new model stays evaluable on the
-    day it ships: the cost is what degrades, not the evaluation. What the run
-    actually lost is then counted after the fact by ``RunSummary.tasks_unpriced``.
+def check_pricing_coverage(resolved_tasks: list[ResolvedTask]) -> list[str]:
+    """Pre-flight the rate card against the models this run will use.
 
-    Only pinned ``agent.model`` values are visible here; a task that defers its
-    model to the route resolves it inside the agent and can't be pre-flighted.
+    The rate card is a static table baked into the installed version, so a model
+    released after it has no rate and its tokens book no money. A warning rather
+    than a refusal, so a brand-new model stays evaluable the day it ships: cost is
+    what degrades, not the evaluation. What the run actually lost is counted after
+    the fact by ``RunSummary.tasks_cost_incomplete``.
 
-    Args:
-        resolved_tasks: The fully-resolved tasks about to run.
+    Only models pinned in the task YAML are visible here; one deferred to the route
+    resolves inside the agent and can't be pre-flighted.
 
     Returns:
         The sorted, de-duplicated unpriced model ids (empty when all are priced).
     """
-    missing = unpriced_models(rt.task.agent.model if rt.task.agent else None for rt in resolved_tasks)
+    missing = unpriced_models(_run_models(resolved_tasks))
     if not missing:
         return []
     logger.warning(
         "No pricing rate for %s. Turns the agent's own backend prices are unaffected, but any "
-        + "timed-out or killed partial will book its tokens with no cost, so run-level totals will "
-        + "understate the bill (RunSummary.cost_complete reports false). "
+        + "judge call, timed-out turn or killed partial will book its tokens with no cost, so "
+        + "run-level totals will understate the bill (RunSummary.cost_complete reports false). "
         + "Add the rate to coder_eval.pricing or register it from a plugin via register_pricing().",
         ", ".join(repr(m) for m in missing),
     )
diff --git a/src/coder_eval/reports.py b/src/coder_eval/reports.py
index f1b54125..dce380e1 100644
--- a/src/coder_eval/reports.py
+++ b/src/coder_eval/reports.py
@@ -15,7 +15,8 @@
     SuiteRollup,
     TaskResult,
     ThresholdCheck,
-    eval_overhead_costs,
+    eval_overhead_cost,
+    full_cost,
     row_cost_incomplete,
 )
 from .path_utils import build_task_run_dir
@@ -166,14 +167,7 @@ def _fmt_rate(rate: float | None) -> str:
 
 
 def _pass_rate_lines(summary: RunSummary) -> list[str]:
-    """The pass rate, plus how much of it is being held down by errors.
-
-    One rate over every dispatched task. The old line divided by
-    ``tasks_run - tasks_error``, which read like a pass rate but paid a bonus for
-    erroring: a run that fell over enough could print a near-perfect score while
-    passing almost nothing. When errors are material the error share is printed
-    beside the rate so a bad infrastructure night is visible rather than absorbed.
-    """
+    """The pass rate over every dispatched task, plus the error share when non-zero."""
     lines = [f"- **Pass Rate**: {_fmt_rate(summary.pass_rate)} ({summary.tasks_succeeded}/{summary.tasks_run})"]
     if summary.tasks_error:
         lines.append(
@@ -549,38 +543,34 @@ def _generate_token_usage_section(task_results: list[dict[str, Any]]) -> list[st
         total_cache_write = sum(t.get("cache_creation_input_tokens") or 0 for t in tasks_with_tokens)
         total_cache_read = sum(t.get("cache_read_input_tokens") or 0 for t in tasks_with_tokens)
         total_tokens = sum(t["total_tokens"] for t in tasks_with_tokens)
-        costs = [t["total_cost_usd"] for t in tasks_with_tokens if t.get("total_cost_usd") is not None]
-        total_cost = sum(costs) if costs else None
-
-        # Rows whose recorded spend is missing money, either because a turn's tokens
-        # could not be priced or because a hard kill lost the in-flight turn
-        # entirely. Worded cause-agnostically ("spend missing") because those two
-        # causes reach the same conclusion and the report cannot always tell which
-        # applied. Reported explicitly because the alternative is a total that reads
-        # materially low with nothing to say the number is incomplete. Both of these
-        # come from the same helpers RunSummary.tasks_unpriced /
-        # eval_overhead_cost_usd use, so the report and run.json cannot disagree
-        # about which rows lost money.
-        unpriced = [t for t in task_results if row_cost_incomplete(t)]
-        overhead = eval_overhead_costs(task_results)
+        agent_cost = full_cost(*(t.get("total_cost_usd") for t in tasks_with_tokens))
+
+        # Same helpers RunSummary uses, so the report and run.json cannot disagree
+        # about the bill. Worded cause-agnostically ("spend missing") because an
+        # unpriced turn and a hard kill reach the same conclusion and the report
+        # cannot always tell which applied.
+        incomplete = [t for t in task_results if row_cost_incomplete(t)]
+        overhead = eval_overhead_cost(task_results)
+        total_cost = full_cost(agent_cost, overhead)
 
         lines.append(f"**Total Tokens**: {total_tokens:,} (input: {total_input:,}, output: {total_output:,})")
         if total_cache_write > 0 or total_cache_read > 0:
             lines.append(f"**Cache Tokens**: write: {total_cache_write:,}, read: {total_cache_read:,}")
-        # Judge/simulator spend is broken out only when there is some: it is a
-        # property of the suite's criteria and identical across harnesses, so folding
-        # it into the agent bill would make two harnesses look closer than they are.
-        # **Total Cost** always means the whole bill.
-        if overhead and total_cost is not None:
-            lines.append(f"**Agent Cost**: ${total_cost:.4f}")
-            lines.append(f"**Eval Overhead (judge + simulator)**: ${sum(overhead):.4f}")
+        # The agent bill is broken out separately only when there is overhead to
+        # distinguish it from: judge spend is a property of the suite's criteria and
+        # identical across harnesses, so comparing harnesses means comparing the
+        # agent line. **Total Cost** always means the whole bill.
+        if overhead is not None:
+            if agent_cost is not None:
+                lines.append(f"**Agent Cost**: ${agent_cost:.4f}")
+            lines.append(f"**Eval Overhead (judge + simulator)**: ${overhead:.4f}")
         if total_cost is not None:
-            cost_line = f"**Total Cost**: ${total_cost + sum(overhead):.4f}"
-            if unpriced:
-                cost_line += f" (floor — {len(unpriced)} task(s) have spend missing from this total)"
+            cost_line = f"**Total Cost**: ${total_cost:.4f}"
+            if incomplete:
+                cost_line += f" (floor — {len(incomplete)} task(s) have spend missing from this total)"
             lines.append(cost_line)
-        elif unpriced:
-            lines.append(f"**Total Cost**: unavailable — {len(unpriced)} task(s) have spend missing from this total")
+        elif incomplete:
+            lines.append(f"**Total Cost**: unavailable — {len(incomplete)} task(s) have spend missing from this total")
         lines.append(f"**Avg Tokens/Task**: {total_tokens // len(tasks_with_tokens):,}")
         lines.append("")
 
@@ -597,7 +587,9 @@ def _generate_token_usage_section(task_results: list[dict[str, Any]]) -> list[st
             cache_write = t.get("cache_creation_input_tokens") or 0
             cache_read = t.get("cache_read_input_tokens") or 0
             tokens = t.get("total_tokens", 0)
-            cost = t.get("total_cost_usd")
+            # The row's whole bill, so the column sums to **Total Cost** above.
+            # Legacy rows predate full_cost_usd and carry only the agent figure.
+            cost = t.get("full_cost_usd", t.get("total_cost_usd"))
             cost_str = f"${cost:.4f}" if cost is not None else "N/A"
             row = (
                 f"| {t['task_id']} | {input_tok:,} | {output_tok:,} "
diff --git a/src/coder_eval/reports_experiment.py b/src/coder_eval/reports_experiment.py
index f75805ab..88ec52e5 100644
--- a/src/coder_eval/reports_experiment.py
+++ b/src/coder_eval/reports_experiment.py
@@ -13,9 +13,11 @@
     ExperimentResult,
     FinalStatus,
     TaskExperimentSummary,
+    full_cost,
+    judge_cost_usd,
+    simulator_cost_usd,
 )
 from coder_eval.path_utils import replicate_subdir_name
-from coder_eval.pricing import calculate_cost
 from coder_eval.reports import resolve_agent_settings
 from coder_eval.reports_stats import (
     VariantSeries,
@@ -37,42 +39,27 @@
 # Default pass_threshold from BaseSuccessCriterion — used for Wilson pass-rate in replicate stats.
 _REPLICATE_PASS_THRESHOLD = 0.9
 
-# Cap on the ``error_message`` carried into each run.json row. Long enough to
-# identify a failure without a per-task artifact fetch (a large rollup whose
-# errors are only diagnosable one task.json at a time is not diagnosable),
-# short enough that a wholly-errored run doesn't bloat run.json. The untruncated
-# message stays on task.json.
+# Cap on the ``error_message`` carried into each run.json row: enough to identify
+# a failure without fetching the task artifact, short enough that a wholly-errored
+# run doesn't bloat run.json. The untruncated message stays on task.json.
 _ROW_ERROR_MESSAGE_MAX_CHARS = 400
 
 
 def _cost_complete(result: EvaluationResult) -> bool:
-    """Whether this row's recorded spend accounts for everything it spent.
-
-    False means ``total_cost_usd`` is a floor, not the bill. Two ways to get there,
-    and they are different failures:
-
-    1. **A turn burned tokens the rate card could not price.** The agent's own
-       backend prices a clean turn, so the rate card only matters for a turn the
-       backend never priced (a killed partial). With no rate, that turn books
-       tokens against no money.
-    2. **The task was hard-killed by the task-level timeout.** The agent is killed
-       mid-turn, so the generation it was waiting on is never delivered and its
-       tokens are never reported by anyone. ``Orchestrator._drain_killed_turn``
-       recovers everything the event stream had delivered up to the kill, which is
-       most of it, but the tail is not observable from inside the harness and no
-       amount of bookkeeping makes it so.
-
-       Every ``TIMEOUT`` row is affected, not just the ones that look empty. The
-       watchdog fires while the evaluation loop is running, so there is always an
-       in-flight turn. A row that completed two dialog turns before the wall hit
-       still lost part of the third, and reporting it as fully priced because the
-       recovered turns carry costs would be the same false claim in a less obvious
-       costume.
-
-    True for a row that burned nothing. An error before the agent ran genuinely
-    cost zero and must not be reported as missing cost, which is why case 2 keys on
-    the status rather than on elapsed time: a fast setup failure and a slow one are
-    both free, while a hard-killed task never is.
+    """Whether this row's recorded agent spend accounts for everything it spent.
+
+    False means the costs on the row are a floor, not the bill. Two ways in:
+
+    1. A turn burned tokens the rate card could not price. The card is the fallback
+       for anything the backend did not price itself, so with no rate those tokens
+       book no money.
+    2. The task was hard-killed by the task-level timeout. Keyed on the status
+       rather than on emptiness: the watchdog fires while the evaluation loop is
+       running, so a TIMEOUT row always lost an in-flight turn, even one that
+       completed earlier turns that do carry costs.
+
+    True for a row that burned nothing: an error before the agent ran genuinely
+    cost zero, and a slow setup failure is as free as a fast one.
     """
     if result.final_status is FinalStatus.TIMEOUT:
         return False
@@ -83,54 +70,6 @@ def _cost_complete(result: EvaluationResult) -> bool:
     )
 
 
-def _judge_cost_usd(result: EvaluationResult) -> float | None:
-    """Sum the priced judge spend across this row's criterion results.
-
-    Covers both judge flavors: ``llm_judge`` prices its own one-shot call from
-    the criterion's model, ``agent_judge`` inherits the SDK's cost on the
-    sub-agent's turn. ``None`` when no criterion reported cost, so "no judge ran"
-    stays distinguishable from "a judge ran free".
-    """
-    costs = [
-        usage.total_cost_usd
-        for cr in result.success_criteria_results
-        if (usage := getattr(cr, "token_usage", None)) is not None and usage.total_cost_usd is not None
-    ]
-    return sum(costs) if costs else None
-
-
-def _simulator_cost_usd(result: EvaluationResult) -> float | None:
-    """Price this row's simulator turns. ``None`` outside simulation mode.
-
-    Priced at the ROUTE's model, not the subject agent's. ``UserSimulator`` builds
-    its agent config with ``model=None`` on purpose, so the model resolves to the
-    route default (``BEDROCK_MODEL``) — which is a different model from the subject
-    whenever a task pins ``agent.model``, which suites routinely do. Pricing the
-    simulator at the subject's model would mis-bill every such row.
-
-    A floor, not the exact figure. ``UserSimulator`` records only
-    ``uncached_input_tokens`` and drops both cache buckets, so a prompt whose
-    prefix was cached is largely absent from the count. Widening that telemetry is
-    a change to what the run captures, not to what this reports.
-    """
-    sim = result.simulation
-    if sim is None:
-        return None
-    if not (sim.simulator_input_tokens or sim.simulator_output_tokens):
-        return None
-    route_model = (result.environment_info or {}).get("bedrock_model")
-    # Falls back to the subject's model on a non-Bedrock route, where the SDK
-    # picks its own default and nothing on the record names it.
-    model = route_model if isinstance(route_model, str) and route_model else result.model_used
-    if not model:
-        return None
-    return calculate_cost(
-        model,
-        uncached_input_tokens=sim.simulator_input_tokens,
-        output_tokens=sim.simulator_output_tokens,
-    )
-
-
 # ---------------------------------------------------------------------------
 # Helper: build task_result dict from EvaluationResult (for variant reports)
 # ---------------------------------------------------------------------------
@@ -180,6 +119,11 @@ def eval_result_to_task_dict(
     # per-task content.
     has_reply = _has_final_reply(result)
 
+    agent_cost = result.total_token_usage.total_cost_usd if result.total_token_usage else None
+    judge_cost = judge_cost_usd(result)
+    simulator_cost = simulator_cost_usd(result)
+    row_full_cost = full_cost(agent_cost, judge_cost, simulator_cost)
+
     expected_turns_value: int | None = None
     if result.task_config is not None:
         rl = (result.task_config.resolved or {}).get("run_limits") or {}
@@ -219,20 +163,24 @@ def eval_result_to_task_dict(
             result.total_token_usage.cache_read_input_tokens if result.total_token_usage else None
         ),
         "total_tokens": (result.total_token_usage.total_tokens if result.total_token_usage else None),
-        "total_cost_usd": (result.total_token_usage.total_cost_usd if result.total_token_usage else None),
-        # False when some turn's tokens are missing a price, so total_cost_usd above
-        # is a floor. Rolled up as RunSummary.tasks_unpriced / cost_complete.
+        # Subject-agent spend only. `total_cost_usd` predates the judge/simulator
+        # fields and the dashboard already reads it as the agent bill, so its
+        # meaning is unchanged and `full_cost_usd` below carries the whole figure.
+        "total_cost_usd": agent_cost,
+        # False when the agent spend above is missing money, so it is a floor.
+        # Rolled up as RunSummary.tasks_cost_incomplete / cost_complete.
         "cost_complete": _cost_complete(result),
-        # Evaluation-machinery spend, kept strictly beside the agent bill rather
-        # than inside it: the judge cost is a property of the suite's criteria and
-        # is identical across harnesses, so folding it into total_cost_usd would
-        # make two harnesses look closer than they are. Rolled up run-level as
-        # RunSummary.eval_overhead_cost_usd.
-        "judge_cost_usd": _judge_cost_usd(result),
-        "simulator_cost_usd": _simulator_cost_usd(result),
-        # Why this row errored, on the row. Errors now count as misses, so the
-        # rollup needs to say *why* it lost those points: without these, an errored
-        # run is untriageable without fetching per-task artifacts one at a time.
+        # Evaluation-machinery spend, kept beside the agent bill rather than inside
+        # it: judge cost is a property of the suite's criteria and identical across
+        # harnesses, so folding it in would make two harnesses look closer than they
+        # are. Rolled up as RunSummary.eval_overhead_cost_usd.
+        "judge_cost_usd": judge_cost,
+        "simulator_cost_usd": simulator_cost,
+        # What the task actually cost: agent + judge + simulator. None when nothing
+        # was priced at all. Published so no consumer has to add the three itself.
+        "full_cost_usd": row_full_cost,
+        # Errors count as misses, so the rollup has to say why it lost those points.
+        # Without these, triaging an errored run needs one task.json fetch per row.
         "error_message": (
             truncate_crash_message(result.error_message, limit=_ROW_ERROR_MESSAGE_MAX_CHARS)
             if result.error_message
@@ -391,8 +339,7 @@ def _aggregate_count_rows(result: ExperimentResult, show_p_values: bool) -> list
             row += " | —"
         lines.append(row + " |")
 
-        # Row: Pass Rate. Every task the variant ran is in the denominator, errors
-        # included — see VariantAggregate.pass_rate.
+        # Every task the variant ran is in the denominator, errors included.
         row = "| Pass Rate"
         for vid in result.variant_ids:
             rate = result.variant_aggregates[vid].pass_rate
diff --git a/src/coder_eval/reports_html.py b/src/coder_eval/reports_html.py
index d9102dc5..c86c6bdb 100644
--- a/src/coder_eval/reports_html.py
+++ b/src/coder_eval/reports_html.py
@@ -17,7 +17,7 @@
 from pathlib import Path
 from typing import TYPE_CHECKING, Any
 
-from coder_eval.models import FinalStatus
+from coder_eval.models import FinalStatus, eval_result_full_cost, full_cost
 
 
 if TYPE_CHECKING:
@@ -335,8 +335,9 @@ def _render_header(result: EvaluationResult) -> str:
     subtitle = _esc(result.task_description.strip().splitlines()[0] if result.task_description else "")
     turns_count = result.total_assistant_turns or 0
     cost_badge = ""
-    if result.total_token_usage and result.total_token_usage.total_cost_usd is not None:
-        cost_badge = f'<span class="badge neutral">${result.total_token_usage.total_cost_usd:.4f}</span>'
+    task_cost = eval_result_full_cost(result)
+    if task_cost is not None:
+        cost_badge = f'<span class="badge neutral">${task_cost:.4f}</span>'
     expected_turns_badge = ""
     overage = expected_turns_overage(result)
     if overage is not None:
@@ -870,12 +871,13 @@ def _render_error_details(result: EvaluationResult) -> str:
 
 
 def _render_token_usage(result: EvaluationResult) -> str:
-    """Render Token Usage section — totals + cost."""
+    """Render Token Usage section — totals + cost (agent + judge + simulator)."""
     tu = result.total_token_usage
     if tu is None:
         return ""
     total = tu.total_tokens
-    cost_str = f"${tu.total_cost_usd:.4f}" if tu.total_cost_usd is not None else "N/A"
+    task_cost = eval_result_full_cost(result)
+    cost_str = f"${task_cost:.4f}" if task_cost is not None else "N/A"
     uncached_fmt = f"{tu.uncached_input_tokens:,}"
     cache_write_fmt = f"{tu.cache_creation_input_tokens:,}"
     cache_read_fmt = f"{tu.cache_read_input_tokens:,}"
@@ -1167,7 +1169,7 @@ def _render_variant_generation_metrics(eval_results: list[EvaluationResult]) ->
 
 
 def _render_variant_token_usage(eval_results: list[EvaluationResult]) -> str:
-    """Aggregate token usage across all tasks in a variant."""
+    """Aggregate token usage across all tasks in a variant. Cost is the whole bill."""
     usages = [r.total_token_usage for r in eval_results if r.total_token_usage is not None]
     if not usages:
         return ""
@@ -1176,8 +1178,8 @@ def _render_variant_token_usage(eval_results: list[EvaluationResult]) -> str:
     cache_write = sum(u.cache_creation_input_tokens for u in usages)
     cache_read = sum(u.cache_read_input_tokens for u in usages)
     total = input_tok + output_tok + cache_write + cache_read
-    costs = [u.total_cost_usd for u in usages if u.total_cost_usd is not None]
-    cost_str = f"${sum(costs):.4f}" if costs else "N/A"
+    variant_cost = full_cost(*(eval_result_full_cost(r) for r in eval_results))
+    cost_str = f"${variant_cost:.4f}" if variant_cost is not None else "N/A"
     return f"""
 <h2>Token Usage</h2>
 <div class="card">
diff --git a/tests/test_cost_accounting_paths.py b/tests/test_cost_accounting_paths.py
index 1393b634..eafeee42 100644
--- a/tests/test_cost_accounting_paths.py
+++ b/tests/test_cost_accounting_paths.py
@@ -1,15 +1,8 @@
 """Cost accounting on the error and timeout paths, where spend went missing.
 
-Two seams:
-
-1. **The pre-flight** (``check_pricing_coverage``). The rate card is a static
-   table baked into the installed version, so a model released after it has no
-   rate. It is the fallback for turns the agent's backend never priced, which
-   makes killed and timed-out partials the ones that book their tokens against no
-   money at all, with one log line to show for it.
-2. **The row projection** (``eval_result_to_task_dict``) — judge and simulator
-   spend was captured and rolled up nowhere, and a row whose turns were only
-   partly priced reported its partial sum as the whole.
+Two seams: the ``check_pricing_coverage`` pre-flight, which warns when a model the
+run will use has no rate, and the ``eval_result_to_task_dict`` row projection,
+which reports judge/simulator spend and flags a row whose costs are only partial.
 """
 
 from __future__ import annotations
@@ -17,6 +10,7 @@
 import logging
 from datetime import datetime
 from pathlib import Path
+from typing import Any
 
 import pytest
 
@@ -35,14 +29,14 @@
 from coder_eval.reports_experiment import eval_result_to_task_dict
 
 
-def _resolved(model: str | None, tmp_path: Path) -> ResolvedTask:
+def _resolved(model: str | None, tmp_path: Path, criteria: list[dict[str, Any]] | None = None) -> ResolvedTask:
     task = TaskDefinition(
         task_id="t1",
         description="d",
         initial_prompt="p",
         agent=ClaudeCodeAgentConfig(type="claude-code", model=model),
         sandbox={"driver": "tempdir"},
-        success_criteria=[{"type": "file_exists", "path": "f.py", "description": "d"}],
+        success_criteria=criteria or [{"type": "file_exists", "path": "f.py", "description": "d"}],
     )
     return ResolvedTask(
         task=task,
@@ -95,6 +89,20 @@ def test_unpinned_model_is_not_flagged(self, tmp_path):
         """A task deferring its model to the route can't be pre-flighted from here."""
         assert check_pricing_coverage([_resolved(None, tmp_path)]) == []
 
+    def test_judge_criterion_model_is_pre_flighted(self, tmp_path):
+        """A judge call is ALWAYS priced from the rate card, so its model matters most.
+
+        No judge backend reports a cost, unlike an agent whose SDK prices a clean
+        turn, so an unpriced ``criterion.model`` loses money on every graded row.
+        """
+        judge = {"type": "llm_judge", "description": "d", "prompt": "grade it", "model": "claude-sonnet-99"}
+        assert check_pricing_coverage([_resolved("claude-sonnet-5", tmp_path, [judge])]) == ["claude-sonnet-99"]
+
+    def test_default_judge_model_is_priced(self, tmp_path):
+        """The out-of-the-box judge must not warn on a stock task."""
+        judge = {"type": "llm_judge", "description": "d", "prompt": "grade it"}
+        assert check_pricing_coverage([_resolved("claude-sonnet-5", tmp_path, [judge])]) == []
+
 
 class TestRowCostProjection:
     def test_cost_complete_false_when_a_turn_is_unpriced(self):
@@ -184,6 +192,33 @@ def test_no_judge_means_no_judge_cost(self):
         result = _result([_turn(1, TokenUsage(uncached_input_tokens=10, output_tokens=1, total_cost_usd=0.1))])
         assert eval_result_to_task_dict(result)["judge_cost_usd"] is None
 
+    def test_unpriced_judge_is_skipped_not_fatal(self):
+        """An unpriced judge lowers the total; it must never raise or zero the row.
+
+        The pre-flight warns about this at dispatch. After that the run carries on
+        and the figures are a floor, which is the trade this framework makes
+        everywhere: cost degrades, the evaluation does not.
+        """
+        result = _result([_turn(1, TokenUsage(uncached_input_tokens=10, output_tokens=1, total_cost_usd=0.1))])
+        result.total_token_usage = TokenUsage(uncached_input_tokens=10, output_tokens=1, total_cost_usd=0.1)
+        result.success_criteria_results = [
+            JudgeCriterionResult(
+                criterion_type="llm_judge",
+                description="priced",
+                score=1.0,
+                token_usage=TokenUsage(uncached_input_tokens=5000, output_tokens=500, total_cost_usd=0.02),
+            ),
+            JudgeCriterionResult(
+                criterion_type="llm_judge",
+                description="unpriced",
+                score=1.0,
+                token_usage=TokenUsage(uncached_input_tokens=5000, output_tokens=500),
+            ),
+        ]
+        row = eval_result_to_task_dict(result)
+        assert row["judge_cost_usd"] == pytest.approx(0.02)
+        assert row["full_cost_usd"] == pytest.approx(0.12)
+
     @staticmethod
     def _simulated(**env) -> EvaluationResult:
         return _result(
@@ -220,6 +255,73 @@ def test_single_shot_row_has_no_simulator_cost(self):
         result = _result([_turn(1, TokenUsage(uncached_input_tokens=10, output_tokens=1, total_cost_usd=0.1))])
         assert eval_result_to_task_dict(result)["simulator_cost_usd"] is None
 
+    def test_unpriced_simulator_route_is_skipped_not_fatal(self):
+        """An unpriced simulator route leaves the rest of the row intact."""
+        result = self._simulated(bedrock_model="claude-sonnet-99")
+        result.total_token_usage = TokenUsage(uncached_input_tokens=10, output_tokens=1, total_cost_usd=0.1)
+        row = eval_result_to_task_dict(result)
+        assert row["simulator_cost_usd"] is None
+        # The agent's own spend still lands; only the simulator slice is missing.
+        assert row["full_cost_usd"] == pytest.approx(0.1)
+
+
+class TestFullCost:
+    """``full_cost_usd`` is the number to quote: agent + judge + simulator."""
+
+    def test_sums_every_component(self):
+        result = _result(
+            [_turn(1, TokenUsage(uncached_input_tokens=10, output_tokens=1, total_cost_usd=1.0))],
+            simulation=SimulationTelemetry(
+                n_trials=1,
+                replicate_index=0,
+                stop_reason="stop_token",
+                simulator_input_tokens=1_000_000,
+                simulator_output_tokens=0,
+                total_turns=2,
+            ),
+            environment_info={"bedrock_model": "claude-haiku-4-5-20251001"},
+        )
+        result.total_token_usage = TokenUsage(uncached_input_tokens=10, output_tokens=1, total_cost_usd=1.0)
+        result.success_criteria_results = [
+            JudgeCriterionResult(
+                criterion_type="llm_judge",
+                description="d",
+                score=1.0,
+                token_usage=TokenUsage(uncached_input_tokens=5000, output_tokens=500, total_cost_usd=0.25),
+            ),
+        ]
+        row = eval_result_to_task_dict(result)
+        # 1.0 agent + 0.25 judge + 1M input at haiku's $1/MTok.
+        assert row["full_cost_usd"] == pytest.approx(2.25)
+        # The agent bill stays comparable across harnesses.
+        assert row["total_cost_usd"] == pytest.approx(1.0)
+
+    def test_none_when_nothing_was_priced(self):
+        """Not 0.0 — an unpriced row must not read as a free one."""
+        assert eval_result_to_task_dict(_result([]))["full_cost_usd"] is None
+
+    def test_run_level_total_is_the_sum_of_both_halves(self):
+        from coder_eval.models import RunSummary
+
+        summary = RunSummary(
+            run_id="2026-07-29_00-00-00",
+            start_time=datetime(2026, 7, 29, 0, 0, 0),
+            end_time=datetime(2026, 7, 29, 1, 0, 0),
+            total_duration_seconds=3600.0,
+            tasks_run=1,
+            tasks_succeeded=1,
+            tasks_failed=0,
+            tasks_error=0,
+            task_results=[
+                {"task_id": "t", "total_cost_usd": 1.0, "judge_cost_usd": 0.2, "simulator_cost_usd": 0.05},
+            ],
+            framework_version="0.0.0-test",
+        )
+        assert summary.agent_cost_usd == pytest.approx(1.0)
+        assert summary.eval_overhead_cost_usd == pytest.approx(0.25)
+        assert summary.full_cost_usd == pytest.approx(1.25)
+        assert summary.model_dump()["full_cost_usd"] == pytest.approx(1.25)
+
 
 class TestErrorDiagnosticsOnTheRow:
     """Errors count as misses, so the rollup has to say why it lost those points.
diff --git a/tests/test_run_metrics.py b/tests/test_run_metrics.py
index e0aed2bc..327abbb8 100644
--- a/tests/test_run_metrics.py
+++ b/tests/test_run_metrics.py
@@ -137,7 +137,7 @@ def test_unpriced_row_is_counted_and_flagged(self):
                 _row(FinalStatus.TIMEOUT, total_tokens=8_000_000, total_cost_usd=None, cost_complete=False),
             ]
         )
-        assert summary.tasks_unpriced == 1
+        assert summary.tasks_cost_incomplete == 1
         assert summary.cost_complete is False
         # The priced row's cost still totals — a floor beats no number at all.
         assert summary.agent_cost_usd == pytest.approx(0.5)
@@ -152,13 +152,13 @@ def test_partially_priced_row_is_counted(self):
         summary = _summary(
             [_row(FinalStatus.TIMEOUT, total_tokens=5_000_000, total_cost_usd=1.25, cost_complete=False)]
         )
-        assert summary.tasks_unpriced == 1
+        assert summary.tasks_cost_incomplete == 1
         assert summary.cost_complete is False
 
     def test_zero_token_error_is_not_unpriced(self):
         """A row that died before the agent ran genuinely cost nothing."""
         summary = _summary([_row(FinalStatus.ERROR, total_tokens=0, total_cost_usd=None, cost_complete=True)])
-        assert summary.tasks_unpriced == 0
+        assert summary.tasks_cost_incomplete == 0
         assert summary.cost_complete is True
         assert summary.agent_cost_usd is None
 

From 090b8ce511ac61a8704cff467e0088348a5442c4 Mon Sep 17 00:00:00 2001
From: Bai Li <bai.li@uipath.com>
Date: Wed, 29 Jul 2026 11:15:12 -0700
Subject: [PATCH 19/20] refactor(cost): cut the commentary and drop unreachable
 rate-card keys

The cost and reporting code had grown more comment than code: 276 lines of
prose against 228 of implementation. Most of it argued a design decision to an
imagined reviewer rather than describing behaviour to a maintainer, which is
prose that goes stale the moment the decision is revisited and that no test
covers. The behavioural claims are kept, the arguments are not, and the
reasoning survives in the commit messages where it belongs.

Three rate-card keys can never match and are removed: claude-mythos-5 is
invitation-only and unreachable from this harness, and claude-opus-4-6-20250514
/ claude-sonnet-4-6-20250514 pair a 4.6 model with the Opus 4 release date,
where the whole 4.6 generation uses dateless ids.

Also silences a CodeQL "statement has no effect" on a bare `await turn` by
awaiting through wait_for, which additionally stops the test hanging if
cancellation ever regresses.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 src/coder_eval/agent.py                    |  19 ++--
 src/coder_eval/agents/claude_code_agent.py |   7 +-
 src/coder_eval/evaluation/judge_usage.py   |   7 +-
 src/coder_eval/models/experiment.py        |  13 +--
 src/coder_eval/orchestrator.py             |  32 +++----
 src/coder_eval/pricing.py                  | 102 ++++++---------------
 tests/test_agent_timeout.py                |   8 +-
 7 files changed, 60 insertions(+), 128 deletions(-)

diff --git a/src/coder_eval/agent.py b/src/coder_eval/agent.py
index aab36efd..4b4510a2 100644
--- a/src/coder_eval/agent.py
+++ b/src/coder_eval/agent.py
@@ -148,19 +148,12 @@ def _finalize_and_raise_crash(
         raise AgentCrashError(message)
 
     def _finalize_external_cancel(self, finalize: _FinalizeFn) -> None:
-        """Mark ERROR and finalize an externally-cancelled turn as a crash. Does NOT raise.
-
-        A cancel that is not this turn's own timeout was delivered from outside the
-        turn — the task-level watchdog kills the agent and cancels the task running
-        it. Everything the event stream delivered is still intact at this point and
-        ``finalize`` reduces it into a complete record, but only the ``crashed``
-        branch parks that record on ``pending_turn``; finalizing as ``COMPLETED``
-        drops it, and the frame is unwinding so the return value goes nowhere
-        either. So a turn killed from outside must finalize as a crash for its
-        telemetry to survive at all.
-
-        The caller re-raises the ``CancelledError`` afterwards — this helper only
-        preserves the record, it does not alter cancellation semantics.
+        """Finalize a turn cancelled from outside (the task watchdog) as a crash. Does NOT raise.
+
+        Only the ``crashed`` branch parks the record on ``pending_turn``; finalizing
+        as ``COMPLETED`` drops it, and the unwinding frame takes the return value
+        with it, so a killed turn's telemetry survives only via this path. The caller
+        re-raises the ``CancelledError`` afterwards.
         """
         self._state = AgentState.ERROR
         finalize(AgentEndStatus.CRASHED, crashed=True, crash_reason="turn cancelled")
diff --git a/src/coder_eval/agents/claude_code_agent.py b/src/coder_eval/agents/claude_code_agent.py
index 606f809d..01ce89ca 100644
--- a/src/coder_eval/agents/claude_code_agent.py
+++ b/src/coder_eval/agents/claude_code_agent.py
@@ -980,10 +980,9 @@ def _on_turn_timeout() -> None:
             if self._timed_out(state.timeout_hit, deadline):
                 assert timeout is not None
                 self._finalize_and_raise_timeout(state.finalize, timeout)
-            # Cancelled from outside this turn. Finalize as a crash so the turn's
-            # telemetry is parked on `pending_turn` for the caller to drain: without
-            # this the `finally` below finalizes as COMPLETED, which keeps no record,
-            # and the unwinding frame takes the only other copy with it.
+            # Cancelled from outside this turn: park the telemetry on `pending_turn`
+            # for the caller to drain. Otherwise the `finally` below finalizes as
+            # COMPLETED, which keeps no record.
             if not state.finalized:
                 self._finalize_external_cancel(state.finalize)
             raise
diff --git a/src/coder_eval/evaluation/judge_usage.py b/src/coder_eval/evaluation/judge_usage.py
index 534c9605..dfe51dd0 100644
--- a/src/coder_eval/evaluation/judge_usage.py
+++ b/src/coder_eval/evaluation/judge_usage.py
@@ -40,10 +40,9 @@ def token_usage_from_anthropic_dict(resp: dict[str, Any], *, model: str | None =
     ``usage`` block. Returns ``None`` when usage is missing or carries no tokens.
 
     ``model`` prices the call from the rate card. Neither judge backend returns a
-    cost, so without it the judge's spend is invisible in every rollup — a suite
-    with ~100 ``llm_judge`` rows books real Bedrock tokens against no dollar
-    figure anywhere. Left unpriced (``total_cost_usd=None``) when the model is
-    absent from the rate card, which the run-level unpriced count then surfaces.
+    cost, so without it the judge's spend is invisible in every rollup. Left
+    unpriced (``total_cost_usd=None``) when the model is absent from the card,
+    which ``RunSummary.tasks_cost_incomplete`` then surfaces.
     """
     u = resp.get("usage")
     if not isinstance(u, dict):
diff --git a/src/coder_eval/models/experiment.py b/src/coder_eval/models/experiment.py
index 161be48b..847a8475 100644
--- a/src/coder_eval/models/experiment.py
+++ b/src/coder_eval/models/experiment.py
@@ -197,10 +197,8 @@ class VariantAggregate(BaseModel):  # noqa: CE009 -- persisted result model; rou
     """Aggregated statistics for a single variant across all tasks.
 
     ``pass_rate`` uses the same denominator as ``RunSummary.pass_rate``: every task
-    the variant ran, errors included as misses. The variant tables previously
-    divided by ``tasks_run - tasks_error``, so an A/B whose variants errored at
-    different rates compared two different denominators and read the noisier
-    variant as the better one.
+    the variant ran, errors included as misses. Otherwise an A/B whose variants
+    error at different rates compares two different denominators.
     """
 
     variant_id: str
@@ -238,12 +236,7 @@ def _check_task_count_invariant(self) -> VariantAggregate:
     @computed_field  # type: ignore[prop-decorator]
     @property
     def pass_rate(self) -> float | None:
-        """``tasks_succeeded / tasks_run`` as a 0-1 fraction. ``None`` on an empty variant.
-
-        No variant-level ``error_share`` counterpart: the variant tables already
-        print ``Errors`` beside ``Pass Rate (n/m)``, and nothing else consumes the
-        experiment JSON, so a second computed field would be published unread.
-        """
+        """``tasks_succeeded / tasks_run`` as a 0-1 fraction. ``None`` on an empty variant."""
         return self.tasks_succeeded / self.tasks_run if self.tasks_run else None
 
 
diff --git a/src/coder_eval/orchestrator.py b/src/coder_eval/orchestrator.py
index b32e17ee..796520c6 100644
--- a/src/coder_eval/orchestrator.py
+++ b/src/coder_eval/orchestrator.py
@@ -530,11 +530,10 @@ def _kill_agent_subprocess_sync() -> None:
 
                 logger.error(f"Task timed out: {e}")
 
-                # Recover the turn that was in flight when the watchdog killed the
-                # agent. Nothing else on this path does: the cancel is delivered as
-                # a BaseException, so it never reaches the retry executor's
-                # per-attempt failure hook that drains the slot on a turn-level
-                # timeout. Runs here, before teardown clears the slot.
+                # Recover the turn in flight when the watchdog killed the agent.
+                # Nothing else on this path does: the cancel arrives as a
+                # BaseException, so it never reaches the retry executor's
+                # per-attempt hook that drains the slot on a turn-level timeout.
                 await self._drain_killed_turn()
             except BudgetExceededError as e:
                 # Map token-budget breaches and cost-budget breaches to distinct
@@ -622,27 +621,20 @@ def _kill_agent_subprocess_sync() -> None:
     async def _drain_killed_turn(self) -> None:
         """Move a hard-killed turn's partial record from the agent onto the result.
 
-        The task-level watchdog kills the agent and cancels the task running it, so
-        the in-flight turn never returns a record. What it managed to spend is real
-        and already recorded by the agent, which parks the partial on
-        ``pending_turn`` when it is cancelled from outside; this is the only reader
-        of that slot on the task-timeout path.
+        The only reader of ``pending_turn`` on the task-timeout path. Ordering
+        matters both ways: it must run before ``_cleanup`` (whose ``agent.stop()``
+        clears the slot) and before ``_finalize_result``, so the recovered turn
+        feeds token aggregation and command stats like any other.
 
-        Ordering matters in both directions. It runs before ``_cleanup``, whose
-        ``agent.stop()`` clears the slot, and before ``_finalize_result``, so the
-        recovered turn is picked up by token aggregation, command statistics and
-        model resolution exactly like a turn that ended on its own.
-
-        Best-effort by design: a task killed before its first turn has nothing
-        parked, and this runs on the way to a saved row, so it must not raise.
+        Best-effort: a task killed before its first turn has nothing parked, and
+        this runs on the way to a saved row, so it must not raise.
         """
         if self.agent is None or self.result is None:
             return
         try:
             partial = self.agent.pending_turn
-            # Type-checked because `pending_turn` is a slot any agent implementation
-            # fills, and a non-record here would fail validation during teardown and
-            # take the whole row down with it.
+            # `pending_turn` is a slot any agent implementation fills, so a non-record
+            # here would fail validation during teardown and take the row down with it.
             if not isinstance(partial, TurnRecord):
                 logger.debug("[%s] Hard-killed task preserved no partial turn", self.task.task_id)
                 return
diff --git a/src/coder_eval/pricing.py b/src/coder_eval/pricing.py
index 63d770a3..27132444 100644
--- a/src/coder_eval/pricing.py
+++ b/src/coder_eval/pricing.py
@@ -23,38 +23,29 @@ class ModelPricing:
 # Official vendor rate cards, verified 2026-07-29.
 # Key: CLI model name (before gateway mapping)
 _PRICING: dict[str, ModelPricing] = {
-    # Claude 5 Fable / Mythos: $10/$50.
     "claude-fable-5": ModelPricing(10.0, 50.0, 12.50, 1.0),
-    "claude-mythos-5": ModelPricing(10.0, 50.0, 12.50, 1.0),
-    # Opus 4.5 and later dropped to $5/$25. Opus 4.1 and Opus 4 keep the old
-    # $15/$75 — the version boundary is the price boundary, so do NOT assume a
-    # newer Opus costs more than an older one.
+    # Opus 4.5 and later dropped to $5/$25; 4.1 and 4 keep the old $15/$75. The
+    # version boundary is the price boundary: a newer Opus is not the dearer one.
     "claude-opus-5": ModelPricing(5.0, 25.0, 6.25, 0.50),
     "claude-opus-4-8": ModelPricing(5.0, 25.0, 6.25, 0.50),
     "claude-opus-4-7": ModelPricing(5.0, 25.0, 6.25, 0.50),
     "claude-opus-4-6": ModelPricing(5.0, 25.0, 6.25, 0.50),
-    "claude-opus-4-6-20250514": ModelPricing(5.0, 25.0, 6.25, 0.50),
     "claude-opus-4-5": ModelPricing(5.0, 25.0, 6.25, 0.50),
     "claude-opus-4-5-20251101": ModelPricing(5.0, 25.0, 6.25, 0.50),
-    # Claude 4.1 / 4 Opus (deprecated / retired) — still $15/$75.
     "claude-opus-4-1": ModelPricing(15.0, 75.0, 18.75, 1.50),
     "claude-opus-4": ModelPricing(15.0, 75.0, 18.75, 1.50),
     "claude-opus-4-20250514": ModelPricing(15.0, 75.0, 18.75, 1.50),
-    # Claude 5 Sonnet. Deliberately the STANDARD $3/$15 rather than the $2/$10
-    # introductory rate in effect through 2026-08-31: a static table cannot
-    # express a promo window, and of the two errors available this one overstates
-    # cost for a few weeks instead of understating it indefinitely afterwards.
+    # Standard $3/$15, not the $2/$10 promo running through 2026-08-31: a static
+    # table cannot express a window, and overstating for a few weeks beats
+    # understating indefinitely after it lapses.
     "claude-sonnet-5": ModelPricing(3.0, 15.0, 3.75, 0.30),
-    # Claude 4.6 / 4.5 / 4 Sonnet
     "claude-sonnet-4-6": ModelPricing(3.0, 15.0, 3.75, 0.30),
-    "claude-sonnet-4-6-20250514": ModelPricing(3.0, 15.0, 3.75, 0.30),
     "claude-sonnet-4-5": ModelPricing(3.0, 15.0, 3.75, 0.30),
     "claude-sonnet-4-5-20250929": ModelPricing(3.0, 15.0, 3.75, 0.30),
     "claude-sonnet-4-20250514": ModelPricing(3.0, 15.0, 3.75, 0.30),
-    # Claude 4.5 Haiku: $1/$5. (Not $0.80/$4 — those are Haiku 3.5's rates.)
+    # $1/$5. Not $0.80/$4 — those are Haiku 3.5's rates.
     "claude-haiku-4-5": ModelPricing(1.0, 5.0, 1.25, 0.10),
     "claude-haiku-4-5-20251001": ModelPricing(1.0, 5.0, 1.25, 0.10),
-    # Claude 3.5 Haiku (retired)
     "claude-haiku-3-5": ModelPricing(0.80, 4.0, 1.0, 0.08),
     # Claude 3.7 Sonnet
     "claude-3-7-sonnet-20250219": ModelPricing(3.0, 15.0, 3.75, 0.30),
@@ -67,56 +58,35 @@ class ModelPricing:
     "claude-3-sonnet-20240229": ModelPricing(3.0, 15.0, 3.75, 0.30),
     # Claude 3 Haiku
     "claude-3-haiku-20240307": ModelPricing(0.25, 1.25, 0.30, 0.03),
-    # OpenAI GPT-5 / Codex (direct or via Azure OpenAI). Released 2025-09-23.
-    # Source: https://developers.openai.com/api/docs/pricing (gpt-5-codex: $1.25/M
-    # in, $0.125/M cached in, $10/M out). OpenAI does not bill cache writes
-    # separately, so cache_write == input rate.
+    # OpenAI GPT-5 / Codex (direct or via Azure OpenAI). OpenAI bills no separate
+    # cache-write fee, so cache_write == input on every entry below.
     "gpt-5-codex": ModelPricing(1.25, 10.0, 1.25, 0.125),
     "gpt-5": ModelPricing(1.25, 10.0, 1.25, 0.125),
-    # Older codex tiers still on OpenAI's rate card — a task may pin any of them,
-    # and an unpriced model books its tokens against no money.
     "gpt-5.1-codex-max": ModelPricing(1.25, 10.0, 1.25, 0.125),
     "gpt-5.1-codex": ModelPricing(1.25, 10.0, 1.25, 0.125),
     "gpt-5.1-codex-mini": ModelPricing(0.25, 2.0, 0.25, 0.025),
-    # codex-mini-latest is the one OpenAI entry whose cached rate is NOT 10% of
-    # input — it is 25% ($0.375 against $1.50). Do not "normalize" it.
+    # The one OpenAI entry whose cached rate is 25% of input, not 10%.
     "codex-mini-latest": ModelPricing(1.50, 6.0, 1.50, 0.375),
-    # gpt-5.3-codex / gpt-5.2-codex: $1.75/M input, $0.175/M cached, $14/M output.
     "gpt-5.3-codex": ModelPricing(1.75, 14.0, 1.75, 0.175),
     "gpt-5.2-codex": ModelPricing(1.75, 14.0, 1.75, 0.175),
-    # gpt-5.4 (2026-03-05): $2.50/M input, $0.25/M cached, $15/M output.
     "gpt-5.4": ModelPricing(2.5, 15.0, 2.5, 0.25),
-    # gpt-5.5: $5/M input, $0.50/M cached, $30/M output.
-    # CAVEAT: this flat rate does NOT model gpt-5.5's long-context surcharge
-    # (2x input / 1.5x output once a session exceeds 272K input tokens), so cost
-    # for very-large-context runs reads low. Fine for typical eval tasks; revisit
-    # if benchmarking large-context Codex runs.
+    # CAVEAT: flat rate, so gpt-5.5's long-context surcharge (2x input / 1.5x
+    # output past 272K input tokens) is not modelled and reads low.
     "gpt-5.5": ModelPricing(5.0, 30.0, 5.0, 0.50),
-    # gpt-5.5-pro / gpt-5.4-pro: $30/M in, $180/M out; pro tiers offer NO prompt
-    # caching (cache_read nominal, never billed since pro doesn't cache).
+    # Pro tiers offer no prompt caching, so cache_read is nominal.
     "gpt-5.5-pro": ModelPricing(30.0, 180.0, 30.0, 3.0),
     "gpt-5.4-pro": ModelPricing(30.0, 180.0, 30.0, 3.0),
-    # gpt-5.4 economy tiers: mini $0.75/$4.50, nano $0.20/$1.25.
     "gpt-5.4-mini": ModelPricing(0.75, 4.5, 0.75, 0.075),
     "gpt-5.4-nano": ModelPricing(0.20, 1.25, 0.20, 0.02),
-    # GPT-5.6 family (2026-07-09): sol flagship / terra balanced (Codex default) / luna
-    # economy. Terra matches gpt-5.4's rate; sol matches gpt-5.5. cache_write ==
-    # input, as for every other OpenAI entry above: OpenAI bills no separate
-    # cache-write fee, so a 1.25x Anthropic-style write rate would overcharge.
+    # GPT-5.6: sol flagship / terra balanced (Codex default) / luna economy.
     "gpt-5.6-sol": ModelPricing(5.0, 30.0, 5.0, 0.50),
     "gpt-5.6-terra": ModelPricing(2.5, 15.0, 2.5, 0.25),
     "gpt-5.6-luna": ModelPricing(1.0, 6.0, 1.0, 0.10),
-    # Google Gemini (AntigravityAgent, via the Gemini Developer API). Per-MTok
-    # rates from ai.google.dev/gemini-api/docs/pricing. Gemini bills no separate
-    # cache-WRITE fee, so cache_write == input (the agent maps cache_creation_tokens
-    # to 0, so this value is effectively unused); cache_read is the cached-input
-    # rate (~10% of input). Keyed on the bare model id in agent.model — the ids
-    # below are the literal codes returned by the ListModels endpoint.
-    # Two CAVEATS, both of which read LOW rather than high:
-    #  - Pro's >200K-token tier costs more ($4/$18 in/out, $0.40 cached). Fine for
-    #    typical eval tasks; revisit if benchmarking very-large-context runs.
-    #  - Audio input is billed above the text/image/video rate on the Flash-Lite and
-    #    3-Flash-Preview tiers. Coding agents send no audio, so the text rate applies.
+    # Google Gemini (AntigravityAgent, via the Gemini Developer API), keyed on the
+    # literal ids the ListModels endpoint returns. No cache-write fee, so
+    # cache_write == input (unused: the agent maps cache_creation_tokens to 0).
+    # CAVEAT: Pro's >200K-token tier costs more ($4/$18, $0.40 cached), so a
+    # very-large-context run reads low.
     "gemini-3.6-flash": ModelPricing(1.5, 7.5, 1.5, 0.15),
     "gemini-3.5-flash": ModelPricing(1.5, 9.0, 1.5, 0.15),
     "gemini-3.5-flash-lite": ModelPricing(0.30, 2.5, 0.30, 0.03),
@@ -125,23 +95,19 @@ class ModelPricing:
     "gemini-3.1-flash-lite": ModelPricing(0.25, 1.5, 0.25, 0.025),
     "gemini-3.1-flash-lite-preview": ModelPricing(0.25, 1.5, 0.25, 0.025),
     "gemini-3-flash-preview": ModelPricing(0.50, 3.0, 0.50, 0.05),
-    # Gemini 3 Pro Preview has been dropped from the public rate card (superseded by
-    # 3.1 Pro). Its last published rate is kept so historical runs still price.
+    # Off the public card (superseded by 3.1 Pro); last published rate kept so
+    # historical runs still price.
     "gemini-3-pro-preview": ModelPricing(2.0, 12.0, 2.0, 0.20),
-    # Open-weight models on Bedrock (eu-north-1), driven via the LiteLLM backend.
-    # These are the Stockholm rates, a ~20% premium over the us-east-1 rate card —
-    # do NOT "correct" them against the US column of the AWS pricing page.
-    # Bedrock lists no prompt-cache read/write rate for these, so cache-creation
-    # is priced at the input rate and cache-read at 0 (conservative — see the
-    # per-provider cost-accounting caveat).
+    # Open-weight models on Bedrock, driven via the LiteLLM backend. These are the
+    # eu-north-1 rates, a ~20% premium over us-east-1 — do NOT "correct" them
+    # against the US column. Bedrock publishes no prompt-cache rate for these, so
+    # cache-creation is priced at input and cache-read at 0.
     "deepseek.v3.2": ModelPricing(0.74, 2.22, 0.74, 0.0),
     "zai.glm-5": ModelPricing(1.2, 3.84, 1.2, 0.0),
     "moonshotai.kimi-k2.5": ModelPricing(0.72, 3.6, 0.72, 0.0),
-    # OpenRouter models (cost-optimization path). These providers cache prompt
-    # prefixes IMPLICITLY (no cache_control, no cache-write fee), so cache-creation
-    # is priced at input (unused — cache_creation_tokens is always 0) and cache-read
-    # at OpenRouter's published input_cache_read rate. Rates per OpenRouter's
-    # /models endpoint (per-token x 1e6).
+    # OpenRouter models. These providers cache prefixes implicitly (no
+    # cache_control, no write fee), so cache-creation is priced at input (unused)
+    # and cache-read at OpenRouter's published input_cache_read rate.
     "moonshotai/kimi-k3": ModelPricing(3.0, 15.0, 3.0, 0.30),
     "z-ai/glm-5.2": ModelPricing(0.7168, 2.2528, 0.7168, 0.13312),
     "deepseek/deepseek-v4-pro": ModelPricing(0.435, 0.87, 0.435, 0.003625),
@@ -217,23 +183,15 @@ def _normalize_model(model: str) -> str:
 
 
 def is_priced(model: str) -> bool:
-    """Whether the rate card can price this model (after prefix normalization).
-
-    The rate card is a static table, so a model that shipped after the installed
-    framework version has no rate. It is the fallback for turns the agent's own
-    backend never priced (timed-out and killed partials), so an unpriced model
-    means those turns book their tokens against no money. Call this at run start
-    to make that a warning instead of a total that reads low with nothing on the
-    report to say so.
-    """
+    """Whether the rate card can price this model (after prefix normalization)."""
     return _lookup_rate(_normalize_model(model)) is not None
 
 
 def unpriced_models(models: Iterable[str | None]) -> list[str]:
     """Sorted, de-duplicated models from ``models`` that the rate card can't price.
 
-    Falsy entries are dropped: a task with no pinned model resolves its model at
-    the route level, which this pre-flight check cannot see.
+    Falsy entries are dropped: an unpinned model resolves at the route level, which
+    a pre-flight check cannot see.
     """
     return sorted({m for m in models if m and not is_priced(m)})
 
diff --git a/tests/test_agent_timeout.py b/tests/test_agent_timeout.py
index 6c7e29ed..5777dea0 100644
--- a/tests/test_agent_timeout.py
+++ b/tests/test_agent_timeout.py
@@ -280,10 +280,8 @@ def __init__(self, text: str, usage: dict[str, int]) -> None:
 async def test_external_cancel_parks_the_turn_it_interrupted():
     """A turn cancelled from outside must leave its telemetry on ``pending_turn``.
 
-    This is the task-level timeout's kill path: the watchdog kills the agent and
-    cancels the task awaiting ``communicate()``, so the turn never returns a record
-    and the frame that held one unwinds. Whatever it spent before the kill is real
-    and billed, so the pending slot is the only place it can survive.
+    The task-timeout kill path: the turn never returns a record and the frame that
+    held one unwinds, so the pending slot is the only place its spend can survive.
     """
     config = parse_agent_config(type=AgentKind.CLAUDE_CODE, permission_mode="acceptEdits", model="claude-sonnet-5")
     agent = ClaudeCodeAgent(config)
@@ -305,7 +303,7 @@ async def mock_query(prompt, options):
             await streaming.wait()
             turn.cancel()
             with pytest.raises(asyncio.CancelledError):
-                await turn
+                await asyncio.wait_for(turn, timeout=5)
 
     partial = agent.pending_turn
     assert partial is not None, "the interrupted turn's record was discarded"

From de662c144efff4fd546b5509595cf9f59d0f2d76 Mon Sep 17 00:00:00 2001
From: Bai Li <bai.li@uipath.com>
Date: Wed, 29 Jul 2026 11:29:29 -0700
Subject: [PATCH 20/20] refactor(cost)!: total_cost_usd means the whole bill
 everywhere
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

`total_cost_usd` meant subject-agent spend only, so every consumer that read
the obvious field for "what did this cost" got a number that excluded judge
and simulator spend. Rather than add a second field and ask each surface to
opt in, redefine the one they already read: at row and run level it is now
agent + judge + simulator. The agent-only slice stays available as
`agent_cost_usd` for harness-vs-harness comparison, where folding in judge
cost (identical across harnesses) would make two harnesses look closer than
they are.

This is why the evalboard needs no change: it sums `total_cost_usd` already,
and that field now carries the real total.

`TokenUsage.total_cost_usd` is untouched — it is the cost of those tokens, so
agent-only is correct, and `run_limits.max_usd` keeps gating on it since judge
and simulator spend is not known mid-run.

Renames the summing primitive `full_cost` -> `sum_costs` so the combinator
stops sharing a name with the metric, and documents the cost schema plus the
never-fail-on-missing-cost policy in REPORT_SCHEMA.md.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 docs/REPORT_SCHEMA.md                | 49 +++++++++++++++++++++++++++-
 src/coder_eval/models/__init__.py    | 12 +++----
 src/coder_eval/models/results.py     | 39 ++++++++++------------
 src/coder_eval/reports.py            |  9 +++--
 src/coder_eval/reports_experiment.py | 26 +++++++--------
 src/coder_eval/reports_html.py       |  8 ++---
 tests/test_cost_accounting_paths.py  | 33 +++++++++++--------
 tests/test_run_metrics.py            | 28 +++++++++++-----
 tests/test_token_usage.py            | 21 ++++++++++--
 9 files changed, 151 insertions(+), 74 deletions(-)

diff --git a/docs/REPORT_SCHEMA.md b/docs/REPORT_SCHEMA.md
index 3aceab87..76ec85df 100644
--- a/docs/REPORT_SCHEMA.md
+++ b/docs/REPORT_SCHEMA.md
@@ -51,6 +51,21 @@ run-level summary; full per-replicate detail lives in each `task.json`.
 | `framework_version` | `str` | Coder Eval version chip. |
 | `environment_info` | `dict` | Version/dependency info (may nest, e.g. `tool_plugins`). |
 
+These are **computed**, not stored — derived from the counts and rows above on every
+serialization, so they cannot drift from what they summarize. Read them rather than
+re-deriving your own; independent re-derivations are how two consumers end up
+publishing different numbers for the same run.
+
+| Key | Type | Meaning |
+| --- | --- | --- |
+| `pass_rate` | `float \| None` | `tasks_succeeded / tasks_run` — errors are in the denominator, counted as misses. `None` on an empty run (0/0 is unknown, not 0%). |
+| `error_share` | `float \| None` | `tasks_error / tasks_run`. Diagnostic only; never adjusts the rate. |
+| `total_cost_usd` | `float \| None` | **The bill**: agent + judge + simulator, summed over the rows. `None` when nothing could be priced. |
+| `agent_cost_usd` | `float \| None` | Subject-agent spend alone. The harness-vs-harness comparison figure — judge spend is a property of the suite's criteria and identical across harnesses, so leaving it in would make two harnesses look closer than they are. |
+| `eval_overhead_cost_usd` | `float \| None` | Judge + simulator spend. The other half of `total_cost_usd`. |
+| `tasks_cost_incomplete` | `int` | Rows whose recorded spend is missing money (unpriced model, or a hard kill that lost an in-flight turn). |
+| `cost_complete` | `bool` | `tasks_cost_incomplete == 0`. When false, every cost figure above is a **floor**, not the bill. A run is never failed for this — see [Missing cost is never fatal](#missing-cost-is-never-fatal). |
+
 ### `task_results[]` — the flat per-task row
 
 Each entry is an **untyped dict** (a denormalization, not a Pydantic model) with keys
@@ -58,7 +73,10 @@ including: `task_id`, `replicate_index`, `variant_id`, `status`
 ([`FinalStatus`](#finalstatus)), `weighted_score`, `duration`, `iteration_count`,
 `tags`, `task_path`, `model_used`, `reference_similarity`, the token buckets
 (`input_tokens` = uncached input, `output_tokens`, `cache_creation_input_tokens`,
-`cache_read_input_tokens`, `total_tokens`), `total_cost_usd`, `expected_commands`,
+`cache_read_input_tokens`, `total_tokens`), the cost fields
+(`total_cost_usd` = agent + judge + simulator, plus the `agent_cost_usd` /
+`judge_cost_usd` / `simulator_cost_usd` slices and the `cost_complete` flag),
+`expected_commands`,
 `actual_commands`, `commands_efficiency`, `agent_config`, `sdk_options`,
 `installed_tools`, turn accounting (`total_turns`, `visible_turns`, `expected_turns`,
 `max_turns_exhausted`, `has_final_reply`), and early-stop fields (`stopped_early`,
@@ -66,6 +84,29 @@ including: `task_id`, `replicate_index`, `variant_id`, `status`
 turn digest (`{iteration, duration_seconds, command_count, assistant_turn_count,
 crashed, crash_reason}`) — the full transcript is in `task.json`.
 
+### Missing cost is never fatal
+
+Pricing degrades; the evaluation does not. A model absent from the rate card, a turn
+the backend never priced, a hard-killed task that lost its in-flight spend: each one
+lowers a total and sets `cost_complete: false`. None of them raises, none of them
+books a zero, and none of them changes a run's exit code.
+
+The reasoning is that the two failure modes are not symmetric. A missing cost is
+recoverable after the fact — the token counts are on the record, so a corrected rate
+card reprices the run from its artifacts. A failed run is not: the tokens are already
+spent and the only way back is to run it again. So the framework warns loudly and
+keeps going.
+
+The warning fires up front. `check_pricing_coverage` walks every model the run pins
+(subject agents and judge criteria) before the first task dispatches, and logs the
+ones the card cannot price — early enough to fix the card and restart while it is
+still cheap. After that the run is on its own: totals become floors, and
+`tasks_cost_incomplete` says how many rows are behind that floor.
+
+Consumers should treat any cost field as a lower bound whenever `cost_complete` is
+false, and must not read `None` as `0.0` — "nothing could be priced" and "it was
+free" are different facts.
+
 ---
 
 ## `task.json` — `EvaluationResult`
@@ -247,6 +288,12 @@ respectively), checked after each completed agent turn — see
 - `TokenUsage.total_tokens` is not serialized; sum the buckets (or use the computed
   `input_tokens` + `output_tokens` + cache buckets).
 - `EarlyStopInfo` presence is itself the "stopped early" signal.
+- `total_cost_usd` is the whole bill (agent + judge + simulator) at both row and run
+  level; `agent_cost_usd` is the agent-only slice. `TokenUsage.total_cost_usd` is a
+  different thing: the cost of those tokens, so always agent-only. `run_limits.max_usd`
+  gates on that one, since judge and simulator spend is not known mid-run.
+- A cost of `None` means unpriced, not free, and any total is a floor while
+  `cost_complete` is false — see [Missing cost is never fatal](#missing-cost-is-never-fatal).
 
 ## See also
 
diff --git a/src/coder_eval/models/__init__.py b/src/coder_eval/models/__init__.py
index c50e5af1..084b0c2a 100644
--- a/src/coder_eval/models/__init__.py
+++ b/src/coder_eval/models/__init__.py
@@ -128,11 +128,11 @@
     ThresholdCheck,
     TurnRecord,
     eval_overhead_cost,
-    eval_result_full_cost,
-    full_cost,
+    eval_result_total_cost,
     judge_cost_usd,
     row_cost_incomplete,
     simulator_cost_usd,
+    sum_costs,
 )
 
 # Routing
@@ -297,12 +297,12 @@
     "TaskConfigRecord",
     "RunSummary",
     "SkippedTask",
-    # Row-level cost helpers, shared by RunSummary's computed fields and the
-    # reports so every surface agrees on which rows lost money.
+    # Cost helpers, shared by RunSummary's computed fields and the reports so
+    # every surface agrees on what a total costs and which rows lost money.
     "row_cost_incomplete",
     "eval_overhead_cost",
-    "full_cost",
-    "eval_result_full_cost",
+    "sum_costs",
+    "eval_result_total_cost",
     "judge_cost_usd",
     "simulator_cost_usd",
     # Judge defaults
diff --git a/src/coder_eval/models/results.py b/src/coder_eval/models/results.py
index 02409cf9..feecdbe5 100644
--- a/src/coder_eval/models/results.py
+++ b/src/coder_eval/models/results.py
@@ -860,8 +860,8 @@ def row_cost_incomplete(row: Mapping[str, Any]) -> bool:
     return row.get("cost_complete") is False
 
 
-def full_cost(*components: float | None) -> float | None:
-    """Total of whichever cost components were priced. ``None`` when none were.
+def sum_costs(*components: float | None) -> float | None:
+    """Add whichever cost components were priced. ``None`` when none were.
 
     The one way cost components are combined, so every surface adds them up the
     same way. Never raises and never invents a zero: an unpriced component is
@@ -876,7 +876,7 @@ def eval_overhead_cost(rows: Iterable[Mapping[str, Any]]) -> float | None:
 
     ``None`` rather than ``0.0`` so "no judge ran" stays distinct from "a judge ran free".
     """
-    return full_cost(*(row.get(key) for row in rows for key in ("judge_cost_usd", "simulator_cost_usd")))
+    return sum_costs(*(row.get(key) for row in rows for key in ("judge_cost_usd", "simulator_cost_usd")))
 
 
 def judge_cost_usd(result: EvaluationResult) -> float | None:
@@ -887,7 +887,7 @@ def judge_cost_usd(result: EvaluationResult) -> float | None:
     turn. ``None`` when no criterion reported cost.
     """
     usages = [u for cr in result.success_criteria_results if (u := getattr(cr, "token_usage", None)) is not None]
-    return full_cost(*(u.total_cost_usd for u in usages))
+    return sum_costs(*(u.total_cost_usd for u in usages))
 
 
 def simulator_cost_usd(result: EvaluationResult) -> float | None:
@@ -918,15 +918,15 @@ def simulator_cost_usd(result: EvaluationResult) -> float | None:
     )
 
 
-def eval_result_full_cost(result: EvaluationResult) -> float | None:
+def eval_result_total_cost(result: EvaluationResult) -> float | None:
     """Everything one evaluation cost: agent + judge + simulator.
 
     ``None`` when nothing could be priced, a floor when only part of it could. The
-    same figure the row projection publishes as ``full_cost_usd``, for the surfaces
+    same figure the row projection publishes as ``total_cost_usd``, for the surfaces
     that hold an ``EvaluationResult`` rather than a row dict.
     """
     agent = result.total_token_usage.total_cost_usd if result.total_token_usage else None
-    return full_cost(agent, judge_cost_usd(result), simulator_cost_usd(result))
+    return sum_costs(agent, judge_cost_usd(result), simulator_cost_usd(result))
 
 
 class RunSummary(BaseModel):
@@ -1042,28 +1042,25 @@ def cost_complete(self) -> bool:
     def agent_cost_usd(self) -> float | None:
         """Subject-agent spend across the run. ``None`` when no row reported cost.
 
-        Excludes evaluation machinery (``eval_overhead_cost_usd``); ``full_cost_usd``
-        is the two added together. A floor when ``cost_complete`` is False.
+        The comparison figure, not the bill: judge spend is a property of the suite's
+        criteria and identical across harnesses, so folding it in would make two
+        harnesses look closer than they are. ``total_cost_usd`` is the bill.
         """
-        return full_cost(*(row.get("total_cost_usd") for row in self.task_results))
+        return sum_costs(*(row.get("agent_cost_usd") for row in self.task_results))
 
     @computed_field  # type: ignore[prop-decorator]
     @property
     def eval_overhead_cost_usd(self) -> float | None:
-        """Judge + simulator spend across the run. ``None`` when neither reported cost.
-
-        Kept out of ``agent_cost_usd`` on purpose: the judge bill is a property of the
-        suite's criteria and identical across harnesses, so folding it in would make
-        two harnesses look closer than they are.
-        """
+        """Judge + simulator spend across the run. ``None`` when neither reported cost."""
         return eval_overhead_cost(self.task_results)
 
     @computed_field  # type: ignore[prop-decorator]
     @property
-    def full_cost_usd(self) -> float | None:
-        """What the run actually cost: agent + judge + simulator.
+    def total_cost_usd(self) -> float | None:
+        """What the run cost: agent + judge + simulator.
 
-        The number to quote for a run's bill. ``None`` when nothing could be priced,
-        and a floor rather than an error when only some of it could.
+        The number to quote for a run's bill, and what every surface means by "total
+        cost". ``None`` when nothing could be priced, and a floor rather than an error
+        when only some of it could.
         """
-        return full_cost(self.agent_cost_usd, self.eval_overhead_cost_usd)
+        return sum_costs(*(row.get("total_cost_usd") for row in self.task_results))
diff --git a/src/coder_eval/reports.py b/src/coder_eval/reports.py
index dce380e1..743dcf92 100644
--- a/src/coder_eval/reports.py
+++ b/src/coder_eval/reports.py
@@ -16,8 +16,8 @@
     TaskResult,
     ThresholdCheck,
     eval_overhead_cost,
-    full_cost,
     row_cost_incomplete,
+    sum_costs,
 )
 from .path_utils import build_task_run_dir
 
@@ -543,7 +543,7 @@ def _generate_token_usage_section(task_results: list[dict[str, Any]]) -> list[st
         total_cache_write = sum(t.get("cache_creation_input_tokens") or 0 for t in tasks_with_tokens)
         total_cache_read = sum(t.get("cache_read_input_tokens") or 0 for t in tasks_with_tokens)
         total_tokens = sum(t["total_tokens"] for t in tasks_with_tokens)
-        agent_cost = full_cost(*(t.get("total_cost_usd") for t in tasks_with_tokens))
+        agent_cost = sum_costs(*(t.get("agent_cost_usd") for t in tasks_with_tokens))
 
         # Same helpers RunSummary uses, so the report and run.json cannot disagree
         # about the bill. Worded cause-agnostically ("spend missing") because an
@@ -551,7 +551,7 @@ def _generate_token_usage_section(task_results: list[dict[str, Any]]) -> list[st
         # cannot always tell which applied.
         incomplete = [t for t in task_results if row_cost_incomplete(t)]
         overhead = eval_overhead_cost(task_results)
-        total_cost = full_cost(agent_cost, overhead)
+        total_cost = sum_costs(*(t.get("total_cost_usd") for t in task_results))
 
         lines.append(f"**Total Tokens**: {total_tokens:,} (input: {total_input:,}, output: {total_output:,})")
         if total_cache_write > 0 or total_cache_read > 0:
@@ -588,8 +588,7 @@ def _generate_token_usage_section(task_results: list[dict[str, Any]]) -> list[st
             cache_read = t.get("cache_read_input_tokens") or 0
             tokens = t.get("total_tokens", 0)
             # The row's whole bill, so the column sums to **Total Cost** above.
-            # Legacy rows predate full_cost_usd and carry only the agent figure.
-            cost = t.get("full_cost_usd", t.get("total_cost_usd"))
+            cost = t.get("total_cost_usd")
             cost_str = f"${cost:.4f}" if cost is not None else "N/A"
             row = (
                 f"| {t['task_id']} | {input_tok:,} | {output_tok:,} "
diff --git a/src/coder_eval/reports_experiment.py b/src/coder_eval/reports_experiment.py
index 88ec52e5..e362d271 100644
--- a/src/coder_eval/reports_experiment.py
+++ b/src/coder_eval/reports_experiment.py
@@ -13,9 +13,9 @@
     ExperimentResult,
     FinalStatus,
     TaskExperimentSummary,
-    full_cost,
     judge_cost_usd,
     simulator_cost_usd,
+    sum_costs,
 )
 from coder_eval.path_utils import replicate_subdir_name
 from coder_eval.reports import resolve_agent_settings
@@ -122,7 +122,7 @@ def eval_result_to_task_dict(
     agent_cost = result.total_token_usage.total_cost_usd if result.total_token_usage else None
     judge_cost = judge_cost_usd(result)
     simulator_cost = simulator_cost_usd(result)
-    row_full_cost = full_cost(agent_cost, judge_cost, simulator_cost)
+    row_total_cost = sum_costs(agent_cost, judge_cost, simulator_cost)
 
     expected_turns_value: int | None = None
     if result.task_config is not None:
@@ -163,22 +163,22 @@ def eval_result_to_task_dict(
             result.total_token_usage.cache_read_input_tokens if result.total_token_usage else None
         ),
         "total_tokens": (result.total_token_usage.total_tokens if result.total_token_usage else None),
-        # Subject-agent spend only. `total_cost_usd` predates the judge/simulator
-        # fields and the dashboard already reads it as the agent bill, so its
-        # meaning is unchanged and `full_cost_usd` below carries the whole figure.
-        "total_cost_usd": agent_cost,
+        # What the task cost: agent + judge + simulator. `total_cost_usd` means the
+        # whole bill on every surface, so a consumer that reads it gets the real
+        # number without adding anything up. None when nothing was priced at all.
+        "total_cost_usd": row_total_cost,
+        # Subject-agent spend alone, broken out for harness-vs-harness comparison:
+        # judge cost is a property of the suite's criteria and identical across
+        # harnesses, so leaving it in would make two harnesses look closer than they
+        # are. Rolled up as RunSummary.agent_cost_usd.
+        "agent_cost_usd": agent_cost,
         # False when the agent spend above is missing money, so it is a floor.
         # Rolled up as RunSummary.tasks_cost_incomplete / cost_complete.
         "cost_complete": _cost_complete(result),
-        # Evaluation-machinery spend, kept beside the agent bill rather than inside
-        # it: judge cost is a property of the suite's criteria and identical across
-        # harnesses, so folding it in would make two harnesses look closer than they
-        # are. Rolled up as RunSummary.eval_overhead_cost_usd.
+        # The two halves of the eval-machinery bill, rolled up as
+        # RunSummary.eval_overhead_cost_usd.
         "judge_cost_usd": judge_cost,
         "simulator_cost_usd": simulator_cost,
-        # What the task actually cost: agent + judge + simulator. None when nothing
-        # was priced at all. Published so no consumer has to add the three itself.
-        "full_cost_usd": row_full_cost,
         # Errors count as misses, so the rollup has to say why it lost those points.
         # Without these, triaging an errored run needs one task.json fetch per row.
         "error_message": (
diff --git a/src/coder_eval/reports_html.py b/src/coder_eval/reports_html.py
index c86c6bdb..3fef5d12 100644
--- a/src/coder_eval/reports_html.py
+++ b/src/coder_eval/reports_html.py
@@ -17,7 +17,7 @@
 from pathlib import Path
 from typing import TYPE_CHECKING, Any
 
-from coder_eval.models import FinalStatus, eval_result_full_cost, full_cost
+from coder_eval.models import FinalStatus, eval_result_total_cost, sum_costs
 
 
 if TYPE_CHECKING:
@@ -335,7 +335,7 @@ def _render_header(result: EvaluationResult) -> str:
     subtitle = _esc(result.task_description.strip().splitlines()[0] if result.task_description else "")
     turns_count = result.total_assistant_turns or 0
     cost_badge = ""
-    task_cost = eval_result_full_cost(result)
+    task_cost = eval_result_total_cost(result)
     if task_cost is not None:
         cost_badge = f'<span class="badge neutral">${task_cost:.4f}</span>'
     expected_turns_badge = ""
@@ -876,7 +876,7 @@ def _render_token_usage(result: EvaluationResult) -> str:
     if tu is None:
         return ""
     total = tu.total_tokens
-    task_cost = eval_result_full_cost(result)
+    task_cost = eval_result_total_cost(result)
     cost_str = f"${task_cost:.4f}" if task_cost is not None else "N/A"
     uncached_fmt = f"{tu.uncached_input_tokens:,}"
     cache_write_fmt = f"{tu.cache_creation_input_tokens:,}"
@@ -1178,7 +1178,7 @@ def _render_variant_token_usage(eval_results: list[EvaluationResult]) -> str:
     cache_write = sum(u.cache_creation_input_tokens for u in usages)
     cache_read = sum(u.cache_read_input_tokens for u in usages)
     total = input_tok + output_tok + cache_write + cache_read
-    variant_cost = full_cost(*(eval_result_full_cost(r) for r in eval_results))
+    variant_cost = sum_costs(*(eval_result_total_cost(r) for r in eval_results))
     cost_str = f"${variant_cost:.4f}" if variant_cost is not None else "N/A"
     return f"""
 <h2>Token Usage</h2>
diff --git a/tests/test_cost_accounting_paths.py b/tests/test_cost_accounting_paths.py
index eafeee42..9193f7dc 100644
--- a/tests/test_cost_accounting_paths.py
+++ b/tests/test_cost_accounting_paths.py
@@ -184,8 +184,9 @@ def test_judge_cost_rolls_up_onto_the_row(self):
         ]
         row = eval_result_to_task_dict(result)
         assert row["judge_cost_usd"] == pytest.approx(0.05)
-        # Kept out of the agent's bill.
-        assert row["total_cost_usd"] == pytest.approx(0.1)
+        # Folded into the row's bill, and kept out of the agent-only slice.
+        assert row["total_cost_usd"] == pytest.approx(0.15)
+        assert row["agent_cost_usd"] == pytest.approx(0.1)
 
     def test_no_judge_means_no_judge_cost(self):
         """None, not 0.0 — 'no judge ran' must stay distinct from 'a judge ran free'."""
@@ -217,7 +218,7 @@ def test_unpriced_judge_is_skipped_not_fatal(self):
         ]
         row = eval_result_to_task_dict(result)
         assert row["judge_cost_usd"] == pytest.approx(0.02)
-        assert row["full_cost_usd"] == pytest.approx(0.12)
+        assert row["total_cost_usd"] == pytest.approx(0.12)
 
     @staticmethod
     def _simulated(**env) -> EvaluationResult:
@@ -262,11 +263,11 @@ def test_unpriced_simulator_route_is_skipped_not_fatal(self):
         row = eval_result_to_task_dict(result)
         assert row["simulator_cost_usd"] is None
         # The agent's own spend still lands; only the simulator slice is missing.
-        assert row["full_cost_usd"] == pytest.approx(0.1)
+        assert row["total_cost_usd"] == pytest.approx(0.1)
 
 
-class TestFullCost:
-    """``full_cost_usd`` is the number to quote: agent + judge + simulator."""
+class TestTotalCost:
+    """``total_cost_usd`` is the whole bill: agent + judge + simulator."""
 
     def test_sums_every_component(self):
         result = _result(
@@ -292,13 +293,13 @@ def test_sums_every_component(self):
         ]
         row = eval_result_to_task_dict(result)
         # 1.0 agent + 0.25 judge + 1M input at haiku's $1/MTok.
-        assert row["full_cost_usd"] == pytest.approx(2.25)
-        # The agent bill stays comparable across harnesses.
-        assert row["total_cost_usd"] == pytest.approx(1.0)
+        assert row["total_cost_usd"] == pytest.approx(2.25)
+        # The agent slice stays broken out so harnesses stay comparable.
+        assert row["agent_cost_usd"] == pytest.approx(1.0)
 
     def test_none_when_nothing_was_priced(self):
         """Not 0.0 — an unpriced row must not read as a free one."""
-        assert eval_result_to_task_dict(_result([]))["full_cost_usd"] is None
+        assert eval_result_to_task_dict(_result([]))["total_cost_usd"] is None
 
     def test_run_level_total_is_the_sum_of_both_halves(self):
         from coder_eval.models import RunSummary
@@ -313,14 +314,20 @@ def test_run_level_total_is_the_sum_of_both_halves(self):
             tasks_failed=0,
             tasks_error=0,
             task_results=[
-                {"task_id": "t", "total_cost_usd": 1.0, "judge_cost_usd": 0.2, "simulator_cost_usd": 0.05},
+                {
+                    "task_id": "t",
+                    "total_cost_usd": 1.25,
+                    "agent_cost_usd": 1.0,
+                    "judge_cost_usd": 0.2,
+                    "simulator_cost_usd": 0.05,
+                },
             ],
             framework_version="0.0.0-test",
         )
         assert summary.agent_cost_usd == pytest.approx(1.0)
         assert summary.eval_overhead_cost_usd == pytest.approx(0.25)
-        assert summary.full_cost_usd == pytest.approx(1.25)
-        assert summary.model_dump()["full_cost_usd"] == pytest.approx(1.25)
+        assert summary.total_cost_usd == pytest.approx(1.25)
+        assert summary.model_dump()["total_cost_usd"] == pytest.approx(1.25)
 
 
 class TestErrorDiagnosticsOnTheRow:
diff --git a/tests/test_run_metrics.py b/tests/test_run_metrics.py
index 327abbb8..6ac5baff 100644
--- a/tests/test_run_metrics.py
+++ b/tests/test_run_metrics.py
@@ -22,12 +22,22 @@
 import pytest
 import yaml
 
-from coder_eval.models import FinalStatus, RunSummary
+from coder_eval.models import FinalStatus, RunSummary, sum_costs
 from coder_eval.pricing import is_priced, unpriced_models
 
 
 def _row(status: FinalStatus | str, **extra: Any) -> dict[str, Any]:
-    return {"task_id": f"t{id(extra)}", "status": status, **extra}
+    """A run.json row. ``total_cost_usd`` is derived the way the real projection derives it.
+
+    Callers give the three cost slices; the row's total is their sum, so a test can
+    never assert against a total the projection would not have written.
+    """
+    row: dict[str, Any] = {"task_id": f"t{id(extra)}", "status": status, **extra}
+    if any(k in extra for k in ("agent_cost_usd", "judge_cost_usd", "simulator_cost_usd")):
+        row["total_cost_usd"] = sum_costs(
+            extra.get("agent_cost_usd"), extra.get("judge_cost_usd"), extra.get("simulator_cost_usd")
+        )
+    return row
 
 
 def _summary(rows: list[dict[str, Any]]) -> RunSummary:
@@ -133,8 +143,8 @@ class TestCostCompleteness:
     def test_unpriced_row_is_counted_and_flagged(self):
         summary = _summary(
             [
-                _row(FinalStatus.SUCCESS, total_tokens=1000, total_cost_usd=0.5, cost_complete=True),
-                _row(FinalStatus.TIMEOUT, total_tokens=8_000_000, total_cost_usd=None, cost_complete=False),
+                _row(FinalStatus.SUCCESS, total_tokens=1000, agent_cost_usd=0.5, cost_complete=True),
+                _row(FinalStatus.TIMEOUT, total_tokens=8_000_000, agent_cost_usd=None, cost_complete=False),
             ]
         )
         assert summary.tasks_cost_incomplete == 1
@@ -150,14 +160,14 @@ def test_partially_priced_row_is_counted(self):
         it the row's cost is the silent 19% understatement.
         """
         summary = _summary(
-            [_row(FinalStatus.TIMEOUT, total_tokens=5_000_000, total_cost_usd=1.25, cost_complete=False)]
+            [_row(FinalStatus.TIMEOUT, total_tokens=5_000_000, agent_cost_usd=1.25, cost_complete=False)]
         )
         assert summary.tasks_cost_incomplete == 1
         assert summary.cost_complete is False
 
     def test_zero_token_error_is_not_unpriced(self):
         """A row that died before the agent ran genuinely cost nothing."""
-        summary = _summary([_row(FinalStatus.ERROR, total_tokens=0, total_cost_usd=None, cost_complete=True)])
+        summary = _summary([_row(FinalStatus.ERROR, total_tokens=0, agent_cost_usd=None, cost_complete=True)])
         assert summary.tasks_cost_incomplete == 0
         assert summary.cost_complete is True
         assert summary.agent_cost_usd is None
@@ -173,7 +183,7 @@ def test_eval_overhead_is_separate_from_the_agent_bill(self):
                 _row(
                     FinalStatus.SUCCESS,
                     total_tokens=1000,
-                    total_cost_usd=1.0,
+                    agent_cost_usd=1.0,
                     cost_complete=True,
                     judge_cost_usd=0.2,
                     simulator_cost_usd=0.05,
@@ -182,9 +192,11 @@ def test_eval_overhead_is_separate_from_the_agent_bill(self):
         )
         assert summary.agent_cost_usd == pytest.approx(1.0)
         assert summary.eval_overhead_cost_usd == pytest.approx(0.25)
+        # And the bill is the two together, which is what `total_cost_usd` means.
+        assert summary.total_cost_usd == pytest.approx(1.25)
 
     def test_no_overhead_reports_none(self):
-        summary = _summary([_row(FinalStatus.SUCCESS, total_tokens=1, total_cost_usd=0.1, cost_complete=True)])
+        summary = _summary([_row(FinalStatus.SUCCESS, total_tokens=1, agent_cost_usd=0.1, cost_complete=True)])
         assert summary.eval_overhead_cost_usd is None
 
 
diff --git a/tests/test_token_usage.py b/tests/test_token_usage.py
index 2b715583..71bf6035 100644
--- a/tests/test_token_usage.py
+++ b/tests/test_token_usage.py
@@ -929,8 +929,20 @@ def test_token_section_marks_partial_cost_as_floor(self):
         killed ones, which are also the ones that burned the most getting there.
         """
         task_results = [
-            {"task_id": "priced", "total_tokens": 1000, "total_cost_usd": 0.25, "cost_complete": True},
-            {"task_id": "unpriced", "total_tokens": 4_000_000, "total_cost_usd": None, "cost_complete": False},
+            {
+                "task_id": "priced",
+                "total_tokens": 1000,
+                "total_cost_usd": 0.25,
+                "agent_cost_usd": 0.25,
+                "cost_complete": True,
+            },
+            {
+                "task_id": "unpriced",
+                "total_tokens": 4_000_000,
+                "total_cost_usd": None,
+                "agent_cost_usd": None,
+                "cost_complete": False,
+            },
         ]
         joined = "\n".join(ReportGenerator._generate_token_usage_section(task_results))
 
@@ -943,7 +955,8 @@ def test_token_section_breaks_out_eval_overhead(self):
             {
                 "task_id": "task1",
                 "total_tokens": 1000,
-                "total_cost_usd": 1.0,
+                "total_cost_usd": 1.25,
+                "agent_cost_usd": 1.0,
                 "cost_complete": True,
                 "judge_cost_usd": 0.25,
             },
@@ -953,6 +966,8 @@ def test_token_section_breaks_out_eval_overhead(self):
         assert "**Agent Cost**: $1.0000" in joined
         assert "**Eval Overhead (judge + simulator)**: $0.2500" in joined
         assert "**Total Cost**: $1.2500" in joined
+        # And the per-task column carries the whole bill, so it sums to that total.
+        assert "| task1 | 0 | 0 | 0 | 0 | 1,000 | $1.2500 |" in joined
 
     def test_token_section_in_full_report(self):
         """Test that token section appears in generate_markdown when data is available."""