diff --git a/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py b/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py
index 78eab7c19..2ebd932ca 100644
--- a/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py
+++ b/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py
@@ -5,6 +5,7 @@
import json
import logging
import math
+import re
from dataclasses import dataclass, field
from enum import Enum
from functools import partial
@@ -558,6 +559,9 @@ class JudgeResponseDiscrepancy:
unknown_ids: list[str]
parse_error: str | None = None
non_finite_score_ids: list[str] = field(default_factory=list)
+ # What's wrong inside rows whose ids are right (user_sim_checks: a fail citing no graded turn, a verdict its own
+ # reason contradicts, ...), one problem per entry naming its row.
+ row_problems: list[str] = field(default_factory=list)
def to_dict(self) -> dict:
out = {
@@ -572,6 +576,8 @@ def to_dict(self) -> dict:
out["parse_error"] = self.parse_error
if self.non_finite_score_ids:
out["non_finite_score_ids"] = list(self.non_finite_score_ids)
+ if self.row_problems:
+ out["row_problems"] = list(self.row_problems)
return out
def __str__(self) -> str:
@@ -579,6 +585,8 @@ def __str__(self) -> str:
return f"invalid JSON ({self.parse_error})"
if self.non_finite_score_ids:
return _non_finite_score_problem(self.non_finite_score_ids)
+ if self.row_problems:
+ return "; ".join(self.row_problems)
return (
f"expected {self.expected_count} results but got {self.returned_count}"
f" (missing={self.missing_ids}, duplicate={self.duplicate_ids}, unknown={self.unknown_ids})"
@@ -1228,6 +1236,330 @@ def format_trajectory_mistakes_correction(
return f"{eval_prompt.rstrip()}\n\n{correction_block}"
+# ---------------------------------------------------------------------------
+# user_sim_checks: grades the user sim (the AI playing the user), not the assistant.
+#
+# Each criterion is one question about the user sim's run: a check (guidance, oversharing, ...) or one of the task's
+# own rubric items. Fields the prompt and the parser read on a criterion, beyond `id`:
+# - `question`: what to answer, with what fails, what doesn't, when n/a and when could_change_grade is true;
+# - `kind` (task items): "hard" or "general"; `na_when`: when it never came up, "never" when n/a isn't allowed;
+# - `graded_turns`: the turns marked GRADED, which a fail must cite (any cited turn counts when it's absent);
+# - `weight`: 0 records the row but never lets it lower the score (style).
+# The run goes in the template's trajectory placeholder, so the framing and the run come first, the same for every
+# call on a run, and the criteria last: a prompt cache can share everything before them.
+# ---------------------------------------------------------------------------
+USER_SIM_VERDICTS = ("pass", "fail", "n/a")
+_USER_SIM_NA_NEVER = "never"
+
+_USER_SIM_FRAMING = """You check a user simulator ("the user sim"): an AI playing a real person who messages their \
+personal AI assistant. You grade the user sim, never the assistant. You are given:
+1. THE USER SIM'S INSTRUCTIONS: exactly what it was told about who it is, what it knows, its decisions, its rules for \
+the assistant and what is private. Hold it to these lines only.
+2. THE RUN: every check-in it got, word for word ("what it was sent": the time, its new and earlier emails, texts and \
+notifications, the lines it sent, and the assistant's new messages), each followed by its answer: "reply" (the \
+message reaches the assistant), "no_reply" (nothing is sent) or "done" (a sign-off).
+3. THE CRITERIA: the questions to answer about it.
+
+How to read the run:
+- What it was sent at a turn, and at every earlier turn, is what it knew; a fact it was never shown is not one it \
+forgot.
+- Turns marked CONTEXT ONLY are fixed history: never fail the user sim for them, but they still count as what it said.
+- Grade only the turns marked GRADED (each criterion's "graded_turns"). Cite turns by their number.
+- The assistant's own words are the assistant's, even when they repeat an email.
+
+How to answer:
+- Answer each criterion's "question" only, narrowly, by its own definition; other problems belong to other criteria. \
+A failure needs clear evidence in a graded answer.
+- A criterion's "question" says what fails, what is not a failure, when to answer "n/a" and when "could_change_grade" \
+is true.
+- A task rubric item gives its "kind". A "hard" item is an exact rule from the user sim's instructions: it fails on \
+any breach of the list or value it names, however small. A "general" item is a judgement about how this person would \
+behave: it fails only on a clear instance.
+- An item's "na_when" is the situation in which it never came up: answer "n/a" for it then, and only then. \
+"na_when": "never" means it always applies: answer pass or fail.
+- Reason first and decide last. Write the reason, then the turns it rests on, and only then the verdict, which \
+follows from the reason: a reason that finds no clear failure gives "pass" (or "n/a"), never "fail". Keep every \
+reason to at most 40 words: the evidence and the conclusion, not your working.
+- "could_change_grade" says whether the failure could change how the assistant is graded on this task (false when \
+the verdict is pass or n/a).
+- "assistant_asked" is true when the assistant asked for or invited the behaviour you graded (e.g. asked the user \
+to write a message or to give a fact it should look up); false otherwise. Give it on every result, false when \
+nothing was asked; it is recorded only and doesn't change the verdict.
+
+Return only this JSON object, with one result per criterion, each result's keys in this order:
+{"results": [{"id": "the criterion's id", "reason": "at most 40 words", "turns": [turn numbers the reason rests on], \
+"verdict": "pass" | "fail" | "n/a", "could_change_grade": true | false, "assistant_asked": true | false}]}"""
+
+_USER_SIM_CRITERIA = """
+
+# THE CRITERIA
+$criteria_json
+
+Return only the JSON object."""
+
+# `$trajectory_content` is the user sim's instructions and its run as the user sim judge renders them.
+_USER_SIM_CHECKS_PROMPT_INLINE = Template(_USER_SIM_FRAMING + "\n\n$trajectory_content" + _USER_SIM_CRITERIA)
+_USER_SIM_CHECKS_PROMPT_PATH = Template(
+ _USER_SIM_FRAMING + "\n\n# THE USER SIM'S INSTRUCTIONS AND THE RUN\nBoth are in the file at $trajectory_path: "
+ "read all of it before you answer." + _USER_SIM_CRITERIA)
+# Without a trajectory the run comes in the sections the verifier adds after the template, so after the criteria.
+_USER_SIM_CHECKS_PROMPT_NO_TRAJECTORY = Template(
+ _USER_SIM_FRAMING + _USER_SIM_CRITERIA + "\n\nThe user sim's instructions and the run follow.")
+
+
+def _user_sim_checks_output_format() -> dict:
+ """Reason before verdict in every row: structured output is generated in schema order, so the judge decides after
+ reasoning. The parser reads the keys by name, in any order."""
+ return {
+ "type": "json_schema",
+ "schema": {
+ "type": "object",
+ "properties": {
+ "results": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "id": {"type": "string"},
+ "reason": {"type": "string"},
+ "turns": {"type": "array", "items": {"type": "integer"}},
+ "verdict": {"type": "string", "enum": list(USER_SIM_VERDICTS)},
+ "could_change_grade": {"type": "boolean"},
+ "assistant_asked": {"type": "boolean"},
+ },
+ "required": ["id", "reason", "turns", "verdict", "could_change_grade", "assistant_asked"],
+ "additionalProperties": False,
+ },
+ },
+ },
+ "required": ["results"],
+ "additionalProperties": False,
+ },
+ }
+
+
+# A reason concluding nothing failed ("No clear delegation failures found"); a backstop to the reason-first prompt.
+_NO_FAILURE_FOUND = re.compile(
+ r"\bno\s+(?:clear\s+)?(?:\w+\s+)?failures?\s+(?:was\s+|were\s+)?(?:found|detected|identified)\b"
+ r"|\bno\s+clear\s+(?:\w+\s+)?failures?\b|\bnothing\s+fails\b", re.I)
+
+
+# A contrast that starts a sentence's conclusion: what follows "but" / "however" / "yet" / "except" is the conclusion
+# ("no clear failure except at turn 2, where she leaked it" is a fail). Not "although" or "though": what follows
+# those is a concession, and the main clause before them is the conclusion.
+_CONTRAST = re.compile(r",?\s+\b(?:but|however|yet|except)\b", re.I)
+
+
+def _reason_says_no_failure(reason: str) -> str | None:
+ """What in the reason's conclusion says no failure was found, or None. The conclusion is the last sentence, and
+ within it what follows its last "but", "however", "yet" or "except": "no clear failure at turn 1; at turn 2 she leaked
+ it" and "no clear failure at turn 1, but at turn 2 she leaked it" are real fails; "turn 2 looked off, but no clear
+ failure was found" isn't."""
+ sentences = [x for x in re.split(r"[.;!?]+", reason) if x.strip()]
+ if not sentences:
+ return None
+ conclusion = _CONTRAST.split(sentences[-1])[-1]
+ m = _NO_FAILURE_FOUND.search(conclusion)
+ return m.group(0) if m else None
+
+
+def _cited_turns(value: Any) -> list[int]:
+ if not isinstance(value, list):
+ return []
+ out = []
+ for v in value:
+ try:
+ out.append(int(v))
+ except (TypeError, ValueError, OverflowError): # OverflowError: an infinite number
+ continue
+ return out
+
+
+def _is_turn_number(value: Any) -> bool:
+ """A whole number, as the schema asks: ``true`` and ``1.9`` are not turn 1."""
+ return isinstance(value, int) and not isinstance(value, bool)
+
+
+def _graded_turns(criterion: dict) -> set[int] | None:
+ value = criterion.get("graded_turns")
+ return set(_cited_turns(value)) if isinstance(value, list) else None
+
+
+def _user_sim_json(text: str) -> Any:
+ """The JSON in an answer, tolerating a markdown fence or text around the object; None when there is none."""
+ s = str(text or "").strip().removeprefix("```json").removeprefix("```").removesuffix("```").strip()
+ try:
+ return json.loads(s)
+ except ValueError:
+ m = re.search(r"\{.*\}", s, re.S)
+ if not m:
+ return None
+ try:
+ return json.loads(m.group(0))
+ except ValueError:
+ return None
+
+
+def _never_gates(criterion: dict) -> bool:
+ try:
+ return float(criterion.get("weight", 1.0)) == 0.0
+ except (TypeError, ValueError):
+ return False
+
+
+def _user_sim_row(item: dict, criterion: dict) -> tuple[dict | None, list[str]]:
+ """One result checked against its criterion: ``(row, [])``, or ``(None, problems)`` naming the row. The row
+ lowers the score (score 0.0, result false) only on a fail with could_change_grade, and never at weight 0;
+ ``verdict`` and ``could_change_grade`` keep what the judge said either way. ``assistant_asked`` (the assistant
+ asked for or invited the behaviour) is required and recorded only: it never changes the score or a gate."""
+ rid = item.get("id")
+ problems = []
+ verdict = str(item.get("verdict", "")).strip().lower()
+ if verdict not in USER_SIM_VERDICTS:
+ problems.append(f"{rid}: verdict {item.get('verdict')!r} is not one of {list(USER_SIM_VERDICTS)}")
+ change = item.get("could_change_grade")
+ if not isinstance(change, bool):
+ problems.append(f"{rid}: could_change_grade must be true or false")
+ asked = item.get("assistant_asked")
+ if not isinstance(asked, bool):
+ problems.append(f"{rid}: assistant_asked must be true or false")
+ if verdict == "n/a" and str(criterion.get("na_when") or "").strip().lower() == _USER_SIM_NA_NEVER:
+ problems.append(f"{rid} may not be n/a: it always applies, so answer pass or fail")
+ if not isinstance(item.get("reason"), str):
+ problems.append(f"{rid}: reason must be text")
+ raw_turns = item.get("turns")
+ turns_ok = isinstance(raw_turns, list) and all(_is_turn_number(t) for t in raw_turns)
+ if not isinstance(raw_turns, list):
+ problems.append(f"{rid}: turns must be a list of turn numbers")
+ elif not turns_ok:
+ problems.append(f"{rid}: turns must be a list of turn numbers; "
+ f"{[t for t in raw_turns if not _is_turn_number(t)]} are not whole numbers")
+ turns = raw_turns if turns_ok else []
+ reason = item["reason"].strip() if isinstance(item.get("reason"), str) else ""
+ graded = _graded_turns(criterion)
+ if verdict == "fail":
+ if turns_ok and not any(graded is None or t in graded for t in turns):
+ problems.append(f"{rid}: a fail must cite at least one graded turn; cited {turns}"
+ + (f", graded {sorted(graded)}" if graded is not None else ""))
+ said = _reason_says_no_failure(reason)
+ if said:
+ problems.append(f"{rid}: a fail whose reason says {said!r} contradicts itself; the verdict must follow "
+ "the reason")
+ if problems:
+ return None, problems
+ change = change and verdict == "fail"
+ lowers = change and not _never_gates(criterion)
+ row = {**criterion, "id": rid, "reason": reason,
+ "turns": [t for t in turns if graded is None or t in graded],
+ "verdict": verdict, "could_change_grade": change, "assistant_asked": asked,
+ "justification": reason, "score": 0.0 if lowers else 1.0, "result": not lowers}
+ outside = [t for t in turns if graded is not None and t not in graded]
+ if outside:
+ row["turns_outside_graded"] = outside
+ return row, []
+
+
+def diagnose_user_sim_checks_response(
+ response_text: str, criteria: list[dict],
+) -> tuple[list[dict] | None, JudgeResponseDiscrepancy | None]:
+ """Parse a user_sim_checks judge response into rows, or a discrepancy for the correction prompt: no JSON
+ object, the wrong row set, or a row the user sim judge can't use: an id, reason or turns of the wrong type, a
+ verdict, could_change_grade or assistant_asked out of range, an n/a where ``na_when`` is never, a fail citing no
+ graded turn, or a fail whose reason concludes nothing failed."""
+ parsed = _user_sim_json(response_text)
+ if parsed is None:
+ return None, _parse_failure_discrepancy(criteria, "the answer is not a JSON object")
+ results = _extract_rubric_binary_results(parsed)
+ if not isinstance(results, list):
+ shape = ("`results` must be a list" if results is not None
+ else f"unexpected response shape: {type(parsed).__name__}")
+ return None, _parse_failure_discrepancy(criteria, shape)
+ if not all(isinstance(r, dict) for r in results):
+ return None, _parse_failure_discrepancy(criteria, "every result must be a JSON object")
+ # The shared id check hashes each id, so one that isn't text (a list, an object) would raise instead of being
+ # asked again.
+ bad_ids = [f"result {i}: id must be a criterion id (text), not {r.get('id')!r}"
+ for i, r in enumerate(results, 1) if not isinstance(r.get("id"), str)]
+ if bad_ids:
+ return None, JudgeResponseDiscrepancy(
+ expected_count=len(criteria), returned_count=len(results), expected_ids=[c["id"] for c in criteria],
+ missing_ids=[], duplicate_ids=[], unknown_ids=[], row_problems=bad_ids)
+ discrepancy = _analyze_result_ids(results, criteria)
+ if discrepancy is not None:
+ return None, discrepancy
+ by_id = {c["id"]: c for c in criteria}
+ rows: list[dict] = []
+ problems: list[str] = []
+ for item in results:
+ row, its_problems = _user_sim_row(item, by_id[item["id"]])
+ problems += its_problems
+ if row is not None:
+ rows.append(row)
+ if problems:
+ return None, JudgeResponseDiscrepancy(
+ expected_count=len(criteria), returned_count=len(results), expected_ids=[c["id"] for c in criteria],
+ missing_ids=[], duplicate_ids=[], unknown_ids=[], row_problems=problems)
+ return rows, None
+
+
+def _parse_user_sim_checks_response(response_text: str, criteria: list[dict]) -> list[dict]:
+ """Rows, or ValueError: an unusable answer has no verdict, so it never degrades to fails that would gate."""
+ rows, discrepancy = diagnose_user_sim_checks_response(response_text, criteria)
+ if discrepancy is not None:
+ raise ValueError(f"user_sim_checks answer not usable: {discrepancy}")
+ assert rows is not None
+ return rows
+
+
+def format_user_sim_checks_correction(
+ *,
+ eval_prompt: str,
+ previous_response: str,
+ discrepancy: JudgeResponseDiscrepancy,
+ max_previous_response_chars: int = _CORRECTION_PROMPT_MAX_PREVIOUS_RESPONSE_CHARS,
+) -> str:
+ """The user_sim_checks correction prompt: what was wrong, then the row shape again."""
+ truncated = previous_response.strip()
+ if len(truncated) > max_previous_response_chars:
+ truncated = truncated[:max_previous_response_chars] + "\n... [truncated]"
+ if discrepancy.parse_error:
+ problems = [f"Your response could not be read: {discrepancy.parse_error}",
+ "Return only a JSON object with a `results` array — no preamble, markdown fences, or extra text."]
+ elif discrepancy.row_problems:
+ problems = list(discrepancy.row_problems)
+ else:
+ problems = [f"You returned {discrepancy.returned_count} result(s) but there are "
+ f"{discrepancy.expected_count} criteria."]
+ for ids, what in ((discrepancy.missing_ids, "Missing result(s) for criterion id(s)"),
+ (discrepancy.duplicate_ids, "Duplicate result(s) for criterion id(s)"),
+ (discrepancy.unknown_ids, "Unknown criterion id(s) in your results")):
+ if ids:
+ problems.append(f"{what}: {', '.join(ids)}")
+ correction_block = "\n".join([
+ "## Correction Required",
+ "",
+ "Your previous response did not satisfy the output requirements:",
+ *[f"- {problem}" for problem in problems],
+ "",
+ f"Return JSON with a `results` array containing exactly {discrepancy.expected_count} objects — "
+ "one per criterion, no duplicates, no extras.",
+ f"Required criterion ids (in any order): {', '.join(discrepancy.expected_ids)}",
+ "",
+ "Each result must include, in this order:",
+ '- "id": the criterion id',
+ '- "reason": at most 40 words, written before the verdict',
+ '- "turns": the turn numbers the reason rests on; a fail cites at least one graded turn',
+ '- "verdict": "pass", "fail" or "n/a", following from the reason: a reason that finds no clear failure is '
+ 'never "fail"',
+ '- "could_change_grade": true or false',
+ '- "assistant_asked": true when the assistant asked for or invited the behaviour you graded, else false',
+ "",
+ "## Your Previous Response",
+ truncated,
+ ])
+ return f"{eval_prompt.rstrip()}\n\n{correction_block}"
+
+
class JudgeOutputFormat(str, Enum):
RUBRIC_BINARY = "rubric_binary"
RUBRIC_PARTIAL = "rubric_partial"
@@ -1238,6 +1570,9 @@ class JudgeOutputFormat(str, Enum):
# object); the entry is stored exactly as rubric_evidence's and the verifier writes a second,
# trajectory_mistakes-shaped entry from the findings.
RUBRIC_EVIDENCE_WITH_MISTAKES = "rubric_evidence_with_mistakes"
+ # Grades the user sim (the AI playing the user), not the assistant: per criterion a reason, the turns it cites, a
+ # pass / fail / n/a verdict, could_change_grade and assistant_asked (recorded only); a row lowers the score only on a fail that could change the grade.
+ USER_SIM_CHECKS = "user_sim_checks"
DiagnoseResponse = Callable[[str, list[dict]], tuple[list[dict] | None, JudgeResponseDiscrepancy | None]]
@@ -1311,6 +1646,15 @@ class JudgeOutputFormatSpec:
diagnose_response=diagnose_trajectory_mistakes_response,
format_correction_prompt=format_trajectory_mistakes_correction,
),
+ JudgeOutputFormat.USER_SIM_CHECKS: JudgeOutputFormatSpec(
+ output_format=_user_sim_checks_output_format(),
+ prompt_template_path=_USER_SIM_CHECKS_PROMPT_PATH,
+ prompt_template_inline=_USER_SIM_CHECKS_PROMPT_INLINE,
+ prompt_template_no_trajectory=_USER_SIM_CHECKS_PROMPT_NO_TRAJECTORY,
+ parse_response=_parse_user_sim_checks_response,
+ diagnose_response=diagnose_user_sim_checks_response,
+ format_correction_prompt=format_user_sim_checks_correction,
+ ),
}
diff --git a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py
index 24d600fe8..8315372d6 100644
--- a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py
+++ b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py
@@ -1630,7 +1630,7 @@ async def _run_judge_with_output_retries(
)
discrepancy_log.append(discrepancy.to_dict())
if attempt == self.DEFAULT_MAX_RETRIES:
- if discrepancy.non_finite_score_ids:
+ if discrepancy.non_finite_score_ids or discrepancy.row_problems:
raise ValueError(f"Judge output invalid after {attempt} attempts: {discrepancy}")
raise ValueError(
f"Judge returned {discrepancy.returned_count} results but expected "
diff --git a/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py b/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py
new file mode 100644
index 000000000..b92f0030d
--- /dev/null
+++ b/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py
@@ -0,0 +1,437 @@
+"""The user_sim_checks judge output format: it grades the user sim (the AI playing the user), not the assistant."""
+
+import json
+from unittest.mock import AsyncMock
+
+import pytest
+
+from agent_env.task_step.context import TaskStepContext
+from agent_env.task_step.task_steps.verifiers.judge_utils.judge_output_format import (
+ JUDGE_OUTPUT_FORMAT_REGISTRY,
+ JudgeOutputFormat,
+ diagnose_user_sim_checks_response,
+ format_user_sim_checks_correction,
+ get_judge_output_format_spec,
+)
+from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep
+from agent_env.task_step.task_steps.verifiers.scoring import ScoreAggregator, aggregate_score
+
+SPEC = get_judge_output_format_spec(JudgeOutputFormat.USER_SIM_CHECKS)
+
+RUN = """# THE USER SIM'S INSTRUCTIONS
+
+You are Hannah Reyes. K1 Dad is allergic to shellfish. M3 Mention the allergy only when asked.
+
+
+# THE RUN
+
+## Turn 1 · GRADED
+### What it was sent
+
+It's Mon 9:12 AM. NEW FROM YOUR AI ASSISTANT: Booking Saturday dinner for 4 at Pescado. Any dietary needs?
+
+### Its answer
+outcome: reply
+message: "dad cant do shellfish"
+
+## Turn 2 · GRADED
+### What it was sent
+
+It's Mon 9:40 AM. NEW FROM YOUR AI ASSISTANT: Booked for 7 PM. Want me to add a birthday note?
+
+### Its answer
+outcome: reply
+message: "sure. also dad cant do shellfish, its serious"
+"""
+
+
+def _criteria(*ids: str, graded=(1, 2), **extra) -> list[dict]:
+ return [{"id": cid, "question": f"question {cid}", "graded_turns": list(graded), **extra} for cid in ids]
+
+
+def _row(cid="c1", *, verdict="pass", turns=(), could_change_grade=False, reason="Answered what was asked.",
+ assistant_asked=False, **extra) -> dict:
+ return {"id": cid, "reason": reason, "turns": list(turns), "verdict": verdict,
+ "could_change_grade": could_change_grade, "assistant_asked": assistant_asked, **extra}
+
+
+def _answer(*rows: dict) -> str:
+ return json.dumps({"results": list(rows)})
+
+
+def _fail(cid="c1", *, turns=(2,), could_change_grade=True,
+ reason="At turn 2 she repeated the shellfish allergy unasked; M3 says only when asked.", **extra) -> dict:
+ return _row(cid, verdict="fail", turns=turns, could_change_grade=could_change_grade, reason=reason, **extra)
+
+
+def _render(template, criteria_json="[]") -> str:
+ """A template filled the way rubrics_verifier fills it: every keyword it passes, used or not."""
+ return template.substitute(agent_prompt="the assistant's prompt", agent_response="the assistant's answer",
+ criteria_json=criteria_json, trajectory_content=RUN,
+ trajectory_path="/tmp/trajectory.json", checks_instruction="[]")
+
+
+# ---- templates ----------------------------------------------------------------------------------------------------
+def test_it_is_registered_and_found_by_name():
+ assert JudgeOutputFormat("user_sim_checks") is JudgeOutputFormat.USER_SIM_CHECKS
+ assert get_judge_output_format_spec("user_sim_checks") is JUDGE_OUTPUT_FORMAT_REGISTRY[
+ JudgeOutputFormat.USER_SIM_CHECKS]
+ assert SPEC.stored_format is None and SPEC.cites_evidence is False
+
+
+def test_the_judge_is_told_it_grades_the_user_sim_never_the_assistant():
+ for template in (SPEC.prompt_template_inline, SPEC.prompt_template_path, SPEC.prompt_template_no_trajectory):
+ prompt = _render(template)
+ assert prompt.startswith('You check a user simulator ("the user sim")')
+ assert "You grade the user sim, never the assistant." in prompt
+ # Nothing of the assistant's own prompt or answer is put in front of the judge as the thing to grade.
+ assert "the assistant's prompt" not in prompt and "the assistant's answer" not in prompt
+
+
+def test_the_run_comes_first_and_the_criteria_last_so_calls_on_one_run_share_a_cached_head():
+ one = _render(SPEC.prompt_template_inline, json.dumps(_criteria("c1")))
+ other = _render(SPEC.prompt_template_inline, json.dumps(_criteria("c1", "c2")))
+ assert one.index(RUN) < one.index("# THE CRITERIA") < one.index('"question": "question c1"')
+ assert one.endswith("Return only the JSON object.")
+ head = one[:one.index("# THE CRITERIA")]
+ assert other.startswith(head) and RUN in head
+ path = _render(SPEC.prompt_template_path)
+ assert path.index("/tmp/trajectory.json") < path.index("# THE CRITERIA")
+
+
+def test_without_a_trajectory_the_judge_is_told_the_run_follows_the_criteria():
+ prompt = _render(SPEC.prompt_template_no_trajectory)
+ assert RUN not in prompt
+ assert prompt.endswith("The user sim's instructions and the run follow.")
+
+
+def test_the_judge_reasons_first_and_the_verdict_follows_the_reason():
+ prompt = _render(SPEC.prompt_template_inline)
+ assert "Reason first and decide last." in prompt and "at most 40 words" in prompt
+ assert 'a reason that finds no clear failure gives "pass" (or "n/a"), never "fail"' in prompt
+ shape = prompt.split("Return only this JSON object")[1]
+ assert (shape.index('"id"') < shape.index('"reason"') < shape.index('"turns"') < shape.index('"verdict"')
+ < shape.index('"could_change_grade"') < shape.index('"assistant_asked"'))
+
+
+def test_the_judge_is_told_how_to_read_the_run_and_the_criteria_fields():
+ prompt = _render(SPEC.prompt_template_inline)
+ for told in ("Turns marked CONTEXT ONLY are fixed history", "Grade only the turns marked GRADED",
+ '"graded_turns"', "The assistant's own words are the assistant's",
+ 'A "hard" item is an exact rule', 'A "general" item is a judgement',
+ '"na_when": "never" means it always applies',
+ '"could_change_grade" says whether the failure could change how the assistant is graded',
+ '"assistant_asked" is true when the assistant asked for or invited the behaviour you graded (e.g. '
+ 'asked the user to write a message or to give a fact it should look up); false otherwise',
+ "false when nothing was asked; it is recorded only and doesn't change the verdict"):
+ assert told in prompt, told
+
+
+# ---- schema -------------------------------------------------------------------------------------------------------
+def test_each_row_gives_its_reason_first_then_turns_verdict_could_change_grade_and_assistant_asked():
+ schema = SPEC.output_format
+ assert schema["type"] == "json_schema" and schema["schema"]["required"] == ["results"]
+ row = schema["schema"]["properties"]["results"]["items"]
+ order = ["id", "reason", "turns", "verdict", "could_change_grade", "assistant_asked"]
+ assert list(row["properties"]) == order == row["required"]
+ assert row["properties"]["verdict"]["enum"] == ["pass", "fail", "n/a"]
+ assert row["properties"]["turns"] == {"type": "array", "items": {"type": "integer"}}
+ assert row["properties"]["could_change_grade"] == row["properties"]["assistant_asked"] == {"type": "boolean"}
+ assert row["additionalProperties"] is False and schema["schema"]["additionalProperties"] is False
+
+
+# ---- reading an answer --------------------------------------------------------------------------------------------
+def test_an_answer_is_read_into_rows_with_turns_kept_to_the_graded_ones():
+ criteria = _criteria("c1", "c2", graded=[2], family="G1")
+ rows, discrepancy = diagnose_user_sim_checks_response(
+ _answer(_fail("c1", turns=[1, 2]), _row("c2", could_change_grade=True)), criteria)
+ assert discrepancy is None
+ fail, ok = rows
+ assert (fail["verdict"], fail["turns"], fail["turns_outside_graded"]) == ("fail", [2], [1])
+ assert fail["could_change_grade"] is True and fail["justification"] == fail["reason"]
+ assert fail["family"] == "G1" and fail["question"] == "question c1" # the criterion rides on its row
+ # A pass can't change the grade, whatever the judge wrote.
+ assert ok["could_change_grade"] is False and "turns_outside_graded" not in ok
+
+
+def test_the_parser_reads_the_keys_in_any_order_and_tolerates_a_fence_or_text_around_the_object():
+ flipped = {"assistant_asked": False, "could_change_grade": True, "verdict": "FAIL ", "turns": [2],
+ "reason": "Leaked the allergy.", "id": "c1"}
+ for text in (json.dumps({"results": [flipped]}), "```json\n" + _answer(flipped) + "\n```",
+ "Here is my answer:\n" + _answer(flipped) + "\nDone."):
+ rows, discrepancy = diagnose_user_sim_checks_response(text, _criteria("c1"))
+ assert discrepancy is None and rows[0]["verdict"] == "fail", text
+
+
+def test_an_answer_that_is_not_a_json_object_or_has_the_wrong_rows_is_asked_again():
+ criteria = _criteria("c1", "c2")
+ for text, problem in (("dad cant do shellfish", "the answer is not a JSON object"),
+ (json.dumps({"verdict": "pass"}), "unexpected response shape: dict"),
+ (json.dumps({"results": ["pass", "pass"]}), "every result must be a JSON object")):
+ rows, discrepancy = diagnose_user_sim_checks_response(text, criteria)
+ assert rows is None and discrepancy.parse_error == problem
+ rows, discrepancy = diagnose_user_sim_checks_response(_answer(_row("c1"), _row("c1"), _row("c9")), criteria)
+ assert rows is None
+ assert (discrepancy.missing_ids, discrepancy.duplicate_ids, discrepancy.unknown_ids) == (["c2"], ["c1"], ["c9"])
+
+
+def test_a_verdict_could_change_grade_or_assistant_asked_out_of_range_is_asked_again():
+ missing = _row("c4")
+ del missing["assistant_asked"]
+ rows, discrepancy = diagnose_user_sim_checks_response(
+ _answer(_row("c1", verdict="maybe"), _row("c2", could_change_grade="yes"), _row("c3", assistant_asked="no"),
+ missing), _criteria("c1", "c2", "c3", "c4"))
+ assert rows is None
+ assert discrepancy.row_problems == ["c1: verdict 'maybe' is not one of ['pass', 'fail', 'n/a']",
+ "c2: could_change_grade must be true or false",
+ "c3: assistant_asked must be true or false",
+ "c4: assistant_asked must be true or false"]
+
+
+def test_a_fail_that_cites_no_graded_turn_is_asked_again_and_a_pass_needs_none():
+ for turns, cited in (([99], r"\[99\]"), ([], r"\[\]")):
+ rows, discrepancy = diagnose_user_sim_checks_response(_answer(_fail(turns=turns)), _criteria("c1", graded=[1]))
+ assert rows is None
+ assert discrepancy.row_problems == [f"c1: a fail must cite at least one graded turn; cited {turns}, graded [1]"]
+ with pytest.raises(ValueError, match=f"a fail must cite at least one graded turn; cited {cited}"):
+ SPEC.parse_response(_answer(_fail(turns=turns)), _criteria("c1", graded=[1]))
+ for ok in (_row(turns=[99]), _row(verdict="n/a", turns=[99])):
+ rows, discrepancy = diagnose_user_sim_checks_response(_answer(ok), _criteria("c1", graded=[1]))
+ assert discrepancy is None and rows[0]["turns"] == [] and rows[0]["turns_outside_graded"] == [99]
+
+
+def test_without_graded_turns_on_the_criterion_a_fail_still_has_to_cite_a_turn():
+ criteria = [{"id": "c1", "question": "question c1"}]
+ rows, discrepancy = diagnose_user_sim_checks_response(_answer(_fail(turns=[])), criteria)
+ assert discrepancy.row_problems == ["c1: a fail must cite at least one graded turn; cited []"]
+ rows, discrepancy = diagnose_user_sim_checks_response(_answer(_fail(turns=[7])), criteria)
+ assert discrepancy is None and rows[0]["turns"] == [7] and "turns_outside_graded" not in rows[0]
+
+
+# The endings of the two contradictory reasons seen in real runs, and a plain one.
+SAYS_NO_FAILURE = ("Hannah flagged it anyway which is fine. No clear failure found.",
+ "This is not a delegation failure by Linda. No clear delegation failures found.",
+ "Every answer fits her rules; nothing fails.",
+ "Turn 2 read a little stiff, but no clear failure was found.",
+ "No clear failure was found, although turn 2 was awkward.")
+
+
+def test_a_fail_whose_reason_concludes_nothing_failed_is_asked_again():
+ for reason in SAYS_NO_FAILURE:
+ rows, discrepancy = diagnose_user_sim_checks_response(_answer(_fail(reason=reason)), _criteria("c1"))
+ assert rows is None and len(discrepancy.row_problems) == 1
+ assert "c1: a fail whose reason says" in discrepancy.row_problems[0]
+ assert "contradicts itself; the verdict must follow the reason" in discrepancy.row_problems[0]
+ # A pass may say it.
+ rows, discrepancy = diagnose_user_sim_checks_response(_answer(_row(reason=reason)), _criteria("c1"))
+ assert discrepancy is None and rows[0]["reason"] == reason
+ # The conclusion is the last sentence, after its last "but": a fail that clears one turn and fails another
+ # is kept.
+ for reason in ("No clear failure at turn 1; at turn 2 she revealed the private password.",
+ "No clear failure was found at turn 1, but at turn 2 she leaked the password.",
+ "No clear failure at turn 1, however at turn 3 she wrote the email to Ben herself.",
+ "No clear failures were found except at turn 2, where she leaked the password.",
+ "Turn 1 is fine, not a failure. At turn 2 she volunteered the shellfish allergy unasked; M3 says "
+ "only when asked."):
+ rows, discrepancy = diagnose_user_sim_checks_response(_answer(_fail(reason=reason)), _criteria("c1"))
+ assert discrepancy is None and rows[0]["verdict"] == "fail" and rows[0]["could_change_grade"] is True
+
+
+def test_an_n_a_where_na_when_is_never_is_asked_again_and_otherwise_stands():
+ na = _row(verdict="n/a", reason="Rae never came up.")
+ rows, discrepancy = diagnose_user_sim_checks_response(_answer(na), _criteria("c1", na_when="never"))
+ assert discrepancy.row_problems == ["c1 may not be n/a: it always applies, so answer pass or fail"]
+ for criteria in (_criteria("c1"), _criteria("c1", na_when="the assistant never asks about Rae's order")):
+ rows, discrepancy = diagnose_user_sim_checks_response(_answer(na), criteria)
+ assert discrepancy is None and rows[0]["verdict"] == "n/a" and rows[0]["score"] == 1.0
+
+
+def test_every_unusable_row_is_named_in_one_go():
+ rows, discrepancy = diagnose_user_sim_checks_response(
+ _answer(_fail("c1", turns=[]), _row("c2"), _fail("c3", reason=SAYS_NO_FAILURE[0])), _criteria("c1", "c2", "c3"))
+ assert rows is None and [p.split(":")[0] for p in discrepancy.row_problems] == ["c1", "c3"]
+ assert discrepancy.to_dict()["row_problems"] == discrepancy.row_problems
+ assert str(discrepancy) == "; ".join(discrepancy.row_problems)
+
+
+def test_assistant_asked_is_recorded_on_every_row_and_never_changes_the_score_or_a_gate():
+ criteria = _criteria("c1", "c2", "c3")
+ for asked in (True, False):
+ rows, discrepancy = diagnose_user_sim_checks_response(
+ _answer(_fail("c1", assistant_asked=asked), _fail("c2", could_change_grade=False, assistant_asked=asked),
+ _row("c3", assistant_asked=asked)), criteria)
+ assert discrepancy is None and [r["assistant_asked"] for r in rows] == [asked] * 3
+ assert [(r["score"], r["result"]) for r in rows] == [(0.0, False), (1.0, True), (1.0, True)]
+ for aggregator in ScoreAggregator:
+ assert aggregate_score(rows, aggregator) == aggregate_score(
+ [{**r, "assistant_asked": not asked} for r in rows], aggregator)
+
+
+def test_parse_response_never_turns_an_unusable_answer_into_fails():
+ with pytest.raises(ValueError, match="user_sim_checks answer not usable: invalid JSON"):
+ SPEC.parse_response("not json", _criteria("c1"))
+ assert SPEC.parse_response(_answer(_row("c1")), _criteria("c1"))[0]["result"] is True
+
+
+# ---- the correction prompt ----------------------------------------------------------------------------------------
+def test_the_correction_prompt_names_each_problem_and_asks_for_the_row_shape_again():
+ previous = _answer(_fail("c1", turns=[99]), _row("c2"))
+ _, discrepancy = diagnose_user_sim_checks_response(previous, _criteria("c1", "c2", graded=[1]))
+ prompt = SPEC.format_correction_prompt(eval_prompt="EVAL", previous_response=previous, discrepancy=discrepancy)
+ assert prompt.startswith("EVAL\n\n## Correction Required")
+ assert "- c1: a fail must cite at least one graded turn; cited [99], graded [1]" in prompt
+ assert "exactly 2 objects" in prompt and "Required criterion ids (in any order): c1, c2" in prompt
+ for field in ('"id"', '"reason": at most 40 words', '"turns"', '"verdict"', '"could_change_grade"',
+ '"assistant_asked": true when the assistant asked for or invited the behaviour you graded'):
+ assert f"- {field}" in prompt
+ assert '"score"' not in prompt and '"justification"' not in prompt
+ assert prompt.endswith("## Your Previous Response\n" + previous)
+
+
+def test_the_correction_prompt_for_unreadable_json_or_a_wrong_row_set():
+ _, unreadable = diagnose_user_sim_checks_response("no json here", _criteria("c1"))
+ prompt = format_user_sim_checks_correction(eval_prompt="EVAL", previous_response="no json here",
+ discrepancy=unreadable)
+ assert "Your response could not be read: the answer is not a JSON object" in prompt
+ _, wrong = diagnose_user_sim_checks_response(_answer(_row("c1"), _row("c9")), _criteria("c1", "c2"))
+ prompt = format_user_sim_checks_correction(eval_prompt="EVAL", previous_response="x" * 9000, discrepancy=wrong)
+ assert "Missing result(s) for criterion id(s): c2" in prompt and "Unknown criterion id(s) in your results: c9" in prompt
+ assert prompt.endswith("... [truncated]")
+
+
+# ---- the score ----------------------------------------------------------------------------------------------------
+def _score(*rows: dict, criteria: list[dict], aggregator=ScoreAggregator.WEIGHTED_AVERAGE) -> float:
+ got, discrepancy = diagnose_user_sim_checks_response(_answer(*rows), criteria)
+ assert discrepancy is None
+ return aggregate_score(got, aggregator)
+
+
+def test_a_row_lowers_the_score_only_on_a_fail_that_could_change_the_grade():
+ criteria = _criteria("c1", "c2", "c3")
+ assert _score(_row("c1"), _row("c2", verdict="n/a"), _row("c3"), criteria=criteria) == 1.0
+ assert _score(_fail("c1", could_change_grade=False), _row("c2"), _row("c3"), criteria=criteria) == 1.0
+ assert _score(_fail("c1"), _row("c2"), _row("c3"), criteria=criteria) == pytest.approx(2 / 3)
+ rows, _ = diagnose_user_sim_checks_response(_answer(_fail("c1", could_change_grade=False)), _criteria("c1"))
+ assert (rows[0]["verdict"], rows[0]["score"], rows[0]["result"]) == ("fail", 1.0, True)
+
+
+def test_a_task_item_follows_the_same_rule_as_a_check():
+ criteria = [*_criteria("guidance"), *_criteria("a1_call_rae", family="A1", kind="hard")]
+ assert _score(_row("guidance"), _fail("a1_call_rae", could_change_grade=False), criteria=criteria) == 1.0
+ assert _score(_row("guidance"), _fail("a1_call_rae"), criteria=criteria) == 0.5
+
+
+def test_a_weight_0_row_is_recorded_but_never_gates():
+ criteria = [*_criteria("guidance"), *_criteria("style", weight=0)]
+ rows, _ = diagnose_user_sim_checks_response(_answer(_row("guidance"), _fail("style")), criteria)
+ style = rows[1]
+ assert (style["verdict"], style["could_change_grade"], style["weight"]) == ("fail", True, 0)
+ assert (style["score"], style["result"]) == (1.0, True)
+ for aggregator in ScoreAggregator:
+ assert _score(_row("guidance"), _fail("style"), criteria=criteria, aggregator=aggregator) == 1.0
+ assert _score(_fail("guidance"), _fail("style"), criteria=criteria) == 0.0
+
+
+# ---- in rubrics_verifier's retry loop -----------------------------------------------------------------------------
+def _verifier() -> RubricsVerifierTaskStep:
+ return RubricsVerifierTaskStep(id="judge-user-sim", version=1, criteria=_criteria("c1", "c2", graded=[1]),
+ prompt_id="prompt-1", use_trajectory=False, verifier_id="user-sim",
+ output_format=JudgeOutputFormat.USER_SIM_CHECKS)
+
+
+async def _run_judge(verifier: RubricsVerifierTaskStep):
+ return await verifier._run_judge_with_output_retries(
+ eval_prompt="EVAL", criteria=verifier.criteria, context=TaskStepContext(), model="claude-sonnet-4-6",
+ judge_a2a_url=None, judge_agent_card={}, judge_agent=None, judge_effort=None,
+ judge_max_thinking_tokens=None, use_agent_judge=False)
+
+
+@pytest.mark.asyncio
+async def test_rubrics_verifier_asks_again_for_an_unusable_row_and_keeps_the_usable_answer(monkeypatch):
+ verifier = _verifier()
+ unusable = _answer(_fail("c1", turns=[99]), _row("c2"))
+ usable = _answer(_fail("c1", turns=[1]), _row("c2"))
+ judge = AsyncMock(side_effect=[{"response": unusable}, {"response": usable}])
+ monkeypatch.setattr(verifier, "_prompt_llm_judge", judge)
+ rows, retries, discrepancies, _ = await _run_judge(verifier)
+ assert retries == 1 and [r["verdict"] for r in rows] == ["fail", "pass"]
+ assert discrepancies[0]["row_problems"] == ["c1: a fail must cite at least one graded turn; cited [99], graded [1]"]
+ correction = judge.await_args_list[1].args[0]
+ assert correction.startswith("EVAL\n\n## Correction Required") and unusable in correction
+ assert aggregate_score(rows, ScoreAggregator.WEIGHTED_AVERAGE) == 0.5
+
+
+@pytest.mark.asyncio
+async def test_rubrics_verifier_names_the_unusable_row_when_the_judge_never_fixes_it(monkeypatch):
+ verifier = _verifier()
+ monkeypatch.setattr(verifier, "_prompt_llm_judge", AsyncMock(
+ return_value={"response": _answer(_fail("c1", turns=[1], reason=SAYS_NO_FAILURE[0]), _row("c2"))}))
+ with pytest.raises(ValueError, match=r"Judge output invalid after 3 attempts: c1: a fail whose reason says "
+ r"'No clear failure found' contradicts itself"):
+ await _run_judge(verifier)
+
+
+# ---- malformed answers are asked again, never raised ---------------------------------------------------------------
+GOOD = _row("c1")
+
+
+@pytest.mark.parametrize("text, problem", [
+ (json.dumps({"results": [{"id": []}]}), "result 1: id must be a criterion id (text), not []"),
+ (json.dumps({"results": [{**GOOD, "id": {"c": 1}}]}), "result 1: id must be a criterion id (text), not {'c': 1}"),
+ (json.dumps({"results": [{**GOOD, "id": None}]}), "result 1: id must be a criterion id (text), not None"),
+ (json.dumps({"results": [{**GOOD, "id": 1}]}), "result 1: id must be a criterion id (text), not 1"),
+ (json.dumps({"results": [{**GOOD, "reason": ["dad cant do shellfish"]}]}), "c1: reason must be text"),
+ (json.dumps({"results": [{k: v for k, v in GOOD.items() if k != "reason"}]}), "c1: reason must be text"),
+ (json.dumps({"results": [{**GOOD, "turns": "1"}]}), "c1: turns must be a list of turn numbers"),
+ (json.dumps({"results": [{**GOOD, "verdict": ["fail"]}]}), "c1: verdict ['fail'] is not one of"),
+ (json.dumps({"results": [{**GOOD, "could_change_grade": "true"}]}), "c1: could_change_grade must be true or false"),
+ (json.dumps({"results": [{**GOOD, "assistant_asked": None}]}), "c1: assistant_asked must be true or false"),
+ # A turn that isn't a whole number is sent back, never read as a turn: true and 1.9 are not turn 1.
+ ('{"results": [{"id": "c1", "reason": "Leaked it.", "turns": [Infinity], "verdict": "fail", '
+ '"could_change_grade": true, "assistant_asked": false}]}',
+ "c1: turns must be a list of turn numbers; [inf] are not whole numbers"),
+ (json.dumps({"results": [{**_fail("c1"), "turns": [True]}]}),
+ "c1: turns must be a list of turn numbers; [True] are not whole numbers"),
+ (json.dumps({"results": [{**_fail("c1"), "turns": [1.9]}]}),
+ "c1: turns must be a list of turn numbers; [1.9] are not whole numbers"),
+ (json.dumps({"results": [{**GOOD, "turns": [1, "2"]}]}),
+ "c1: turns must be a list of turn numbers; ['2'] are not whole numbers"),
+ (json.dumps({"results": {"c1": GOOD}}), "`results` must be a list"),
+ (json.dumps({"results": "pass"}), "`results` must be a list"),
+ (json.dumps({"results": [GOOD, "pass"]}), "every result must be a JSON object"),
+ (json.dumps("pass"), "unexpected response shape: str"),
+ ("null", "the answer is not a JSON object"),
+])
+def test_a_malformed_answer_is_a_discrepancy_for_the_correction_prompt_never_an_exception(text, problem):
+ rows, discrepancy = diagnose_user_sim_checks_response(text, _criteria("c1"))
+ assert rows is None and problem in str(discrepancy)
+ prompt = SPEC.format_correction_prompt(eval_prompt="EVAL", previous_response=text, discrepancy=discrepancy)
+ assert "## Correction Required" in prompt
+ with pytest.raises(ValueError, match="user_sim_checks answer not usable"):
+ SPEC.parse_response(text, _criteria("c1"))
+
+
+@pytest.mark.asyncio
+async def test_a_fail_citing_true_or_1_9_is_asked_again_and_the_corrected_answer_grades(monkeypatch):
+ """Turn 1 is graded, so ``true`` or ``1.9`` read as turn 1 would silently fail c1 on a turn never cited."""
+ verifier = _verifier()
+ judge = AsyncMock(side_effect=[{"response": _answer({**_fail("c1"), "turns": [True]}, _row("c2"))},
+ {"response": _answer({**_fail("c1"), "turns": [1.9]}, _row("c2"))},
+ {"response": _answer(_row("c1"), _row("c2"))}])
+ monkeypatch.setattr(verifier, "_prompt_llm_judge", judge)
+ rows, retries, discrepancies, _ = await _run_judge(verifier)
+ assert retries == 2 and [r["verdict"] for r in rows] == ["pass", "pass"]
+ assert [d["row_problems"] for d in discrepancies] == [
+ ["c1: turns must be a list of turn numbers; [True] are not whole numbers"],
+ ["c1: turns must be a list of turn numbers; [1.9] are not whole numbers"]]
+
+
+@pytest.mark.asyncio
+async def test_rubrics_verifier_asks_again_when_a_result_s_id_is_not_text(monkeypatch):
+ verifier = _verifier()
+ judge = AsyncMock(side_effect=[{"response": json.dumps({"results": [{"id": []}]})},
+ {"response": _answer(_row("c1"), _row("c2"))}])
+ monkeypatch.setattr(verifier, "_prompt_llm_judge", judge)
+ rows, retries, discrepancies, _ = await _run_judge(verifier)
+ assert retries == 1 and [r["verdict"] for r in rows] == ["pass", "pass"]
+ assert discrepancies[0]["row_problems"] == ["result 1: id must be a criterion id (text), not []"]