From 2e562f81bcb9a5d65f448877f03170af559bc666 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 16 Nov 2026 10:29:15 -0800 Subject: [PATCH 1/7] feat(verifiers): user_sim_checks judge output format, for grading the user sim A new JudgeOutputFormat, USER_SIM_CHECKS, so the user sim judge (#682) can become a rubrics_verifier subclass. The judge grades the user sim, not the assistant: the framing and the run come first and the criteria last, so calls on one run can share a cached head. Each row gives reason, turns, verdict (pass / fail / n/a) and could_change_grade. Diagnosis ports the current judge's answer checks: a fail must cite a graded turn, a fail whose reason concludes nothing failed is asked again, and so is an n/a where na_when is never. A row lowers the score only on a fail that could change the grade; a weight 0 row is recorded and never gates. Additive: JudgeResponseDiscrepancy gains row_problems (empty by default), and rubrics_verifier's final error names them. Other formats are unchanged. Co-Authored-By: Claude Opus 5.5 --- .../judge_utils/judge_output_format.py | 302 +++++++++++++++ .../task_steps/verifiers/rubrics_verifier.py | 2 +- .../test_user_sim_checks_format.py | 345 ++++++++++++++++++ 3 files changed, 648 insertions(+), 1 deletion(-) create mode 100644 tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py diff --git a/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py b/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py index 78eab7c19..e6a0e37d8 100644 --- a/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py +++ b/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py @@ -5,6 +5,7 @@ import json import logging import math +import re from dataclasses import dataclass, field from enum import Enum from functools import partial @@ -558,6 +559,9 @@ class JudgeResponseDiscrepancy: unknown_ids: list[str] parse_error: str | None = None non_finite_score_ids: list[str] = field(default_factory=list) + # What's wrong inside rows whose ids are right (user_sim_checks: a fail citing no graded turn, a verdict its own + # reason contradicts, ...), one problem per entry naming its row. + row_problems: list[str] = field(default_factory=list) def to_dict(self) -> dict: out = { @@ -572,6 +576,8 @@ def to_dict(self) -> dict: out["parse_error"] = self.parse_error if self.non_finite_score_ids: out["non_finite_score_ids"] = list(self.non_finite_score_ids) + if self.row_problems: + out["row_problems"] = list(self.row_problems) return out def __str__(self) -> str: @@ -579,6 +585,8 @@ def __str__(self) -> str: return f"invalid JSON ({self.parse_error})" if self.non_finite_score_ids: return _non_finite_score_problem(self.non_finite_score_ids) + if self.row_problems: + return "; ".join(self.row_problems) return ( f"expected {self.expected_count} results but got {self.returned_count}" f" (missing={self.missing_ids}, duplicate={self.duplicate_ids}, unknown={self.unknown_ids})" @@ -1228,6 +1236,288 @@ def format_trajectory_mistakes_correction( return f"{eval_prompt.rstrip()}\n\n{correction_block}" +# --------------------------------------------------------------------------- +# user_sim_checks: grades the user sim (the AI playing the user), not the assistant. +# +# Each criterion is one question about the user sim's run: a check (guidance, oversharing, ...) or one of the task's +# own rubric items. Fields the prompt and the parser read on a criterion, beyond `id`: +# - `question`: what to answer, with what fails, what doesn't, when n/a and when could_change_grade is true; +# - `kind` (task items): "hard" or "general"; `na_when`: when it never came up, "never" when n/a isn't allowed; +# - `graded_turns`: the turns marked GRADED, which a fail must cite (any cited turn counts when it's absent); +# - `weight`: 0 records the row but never lets it lower the score (style). +# The run goes in the template's trajectory placeholder, so the framing and the run come first, the same for every +# call on a run, and the criteria last: a prompt cache can share everything before them. +# --------------------------------------------------------------------------- +USER_SIM_VERDICTS = ("pass", "fail", "n/a") +_USER_SIM_NA_NEVER = "never" + +_USER_SIM_FRAMING = """You check a user simulator ("the user sim"): an AI playing a real person who messages their \ +personal AI assistant. You grade the user sim, never the assistant. You are given: +1. THE USER SIM'S INSTRUCTIONS: exactly what it was told about who it is, what it knows, its decisions, its rules for \ +the assistant and what is private. Hold it to these lines only. +2. THE RUN: every check-in it got, word for word ("what it was sent": the time, its new and earlier emails, texts and \ +notifications, the lines it sent, and the assistant's new messages), each followed by its answer: "reply" (the \ +message reaches the assistant), "no_reply" (nothing is sent) or "done" (a sign-off). +3. THE CRITERIA: the questions to answer about it. + +How to read the run: +- What it was sent at a turn, and at every earlier turn, is what it knew; a fact it was never shown is not one it \ +forgot. +- Turns marked CONTEXT ONLY are fixed history: never fail the user sim for them, but they still count as what it said. +- Grade only the turns marked GRADED (each criterion's "graded_turns"). Cite turns by their number. +- The assistant's own words are the assistant's, even when they repeat an email. + +How to answer: +- Answer each criterion's "question" only, narrowly, by its own definition; other problems belong to other criteria. \ +A failure needs clear evidence in a graded answer. +- A criterion's "question" says what fails, what is not a failure, when to answer "n/a" and when "could_change_grade" \ +is true. +- A task rubric item gives its "kind". A "hard" item is an exact rule from the user sim's instructions: it fails on \ +any breach of the list or value it names, however small. A "general" item is a judgement about how this person would \ +behave: it fails only on a clear instance. +- An item's "na_when" is the situation in which it never came up: answer "n/a" for it then, and only then. \ +"na_when": "never" means it always applies: answer pass or fail. +- Reason first and decide last. Write the reason, then the turns it rests on, and only then the verdict, which \ +follows from the reason: a reason that finds no clear failure gives "pass" (or "n/a"), never "fail". Keep every \ +reason to at most 40 words: the evidence and the conclusion, not your working. +- "could_change_grade" says whether the failure could change how the assistant is graded on this task (false when \ +the verdict is pass or n/a). + +Return only this JSON object, with one result per criterion, each result's keys in this order: +{"results": [{"id": "the criterion's id", "reason": "at most 40 words", "turns": [turn numbers the reason rests on], \ +"verdict": "pass" | "fail" | "n/a", "could_change_grade": true | false}]}""" + +_USER_SIM_CRITERIA = """ + +# THE CRITERIA +$criteria_json + +Return only the JSON object.""" + +# `$trajectory_content` is the user sim's instructions and its run as the user sim judge renders them. +_USER_SIM_CHECKS_PROMPT_INLINE = Template(_USER_SIM_FRAMING + "\n\n$trajectory_content" + _USER_SIM_CRITERIA) +_USER_SIM_CHECKS_PROMPT_PATH = Template( + _USER_SIM_FRAMING + "\n\n# THE USER SIM'S INSTRUCTIONS AND THE RUN\nBoth are in the file at $trajectory_path: " + "read all of it before you answer." + _USER_SIM_CRITERIA) +# Without a trajectory the run comes in the sections the verifier adds after the template, so after the criteria. +_USER_SIM_CHECKS_PROMPT_NO_TRAJECTORY = Template( + _USER_SIM_FRAMING + _USER_SIM_CRITERIA + "\n\nThe user sim's instructions and the run follow.") + + +def _user_sim_checks_output_format() -> dict: + """Reason before verdict in every row: structured output is generated in schema order, so the judge decides after + reasoning. The parser reads the keys by name, in any order.""" + return { + "type": "json_schema", + "schema": { + "type": "object", + "properties": { + "results": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": {"type": "string"}, + "reason": {"type": "string"}, + "turns": {"type": "array", "items": {"type": "integer"}}, + "verdict": {"type": "string", "enum": list(USER_SIM_VERDICTS)}, + "could_change_grade": {"type": "boolean"}, + }, + "required": ["id", "reason", "turns", "verdict", "could_change_grade"], + "additionalProperties": False, + }, + }, + }, + "required": ["results"], + "additionalProperties": False, + }, + } + + +# A reason concluding nothing failed ("No clear delegation failures found"); a backstop to the reason-first prompt. +_NO_FAILURE_FOUND = re.compile( + r"\bno\s+(?:clear\s+)?(?:\w+\s+)?failures?\s+(?:was\s+|were\s+)?(?:found|detected|identified)\b" + r"|\bno\s+clear\s+(?:\w+\s+)?failures?\b|\bnothing\s+fails\b", re.I) + + +def _reason_says_no_failure(reason: str) -> str | None: + """What in the reason's conclusion says no failure was found, or None. Only its last sentence is its conclusion: + "no clear failure at turn 1; at turn 2 she leaked it" is a real fail.""" + sentences = [x for x in re.split(r"[.;!?]+", reason) if x.strip()] + m = _NO_FAILURE_FOUND.search(sentences[-1]) if sentences else None + return m.group(0) if m else None + + +def _cited_turns(value: Any) -> list[int]: + if not isinstance(value, list): + return [] + out = [] + for v in value: + try: + out.append(int(v)) + except (TypeError, ValueError): + continue + return out + + +def _graded_turns(criterion: dict) -> set[int] | None: + value = criterion.get("graded_turns") + return set(_cited_turns(value)) if isinstance(value, list) else None + + +def _user_sim_json(text: str) -> Any: + """The JSON in an answer, tolerating a markdown fence or text around the object; None when there is none.""" + s = str(text or "").strip().removeprefix("```json").removeprefix("```").removesuffix("```").strip() + try: + return json.loads(s) + except ValueError: + m = re.search(r"\{.*\}", s, re.S) + if not m: + return None + try: + return json.loads(m.group(0)) + except ValueError: + return None + + +def _never_gates(criterion: dict) -> bool: + try: + return float(criterion.get("weight", 1.0)) == 0.0 + except (TypeError, ValueError): + return False + + +def _user_sim_row(item: dict, criterion: dict) -> tuple[dict | None, list[str]]: + """One result checked against its criterion: ``(row, [])``, or ``(None, problems)`` naming the row. The row + lowers the score (score 0.0, result false) only on a fail with could_change_grade, and never at weight 0; + ``verdict`` and ``could_change_grade`` keep what the judge said either way. An ``assistant_asked`` bool (the + assistant asked for the behaviour) is kept when the judge gives one; it doesn't change the score.""" + rid = item.get("id") + problems = [] + verdict = str(item.get("verdict", "")).strip().lower() + if verdict not in USER_SIM_VERDICTS: + problems.append(f"{rid}: verdict {item.get('verdict')!r} is not one of {list(USER_SIM_VERDICTS)}") + change = item.get("could_change_grade") + if not isinstance(change, bool): + problems.append(f"{rid}: could_change_grade must be true or false") + if verdict == "n/a" and str(criterion.get("na_when") or "").strip().lower() == _USER_SIM_NA_NEVER: + problems.append(f"{rid} may not be n/a: it always applies, so answer pass or fail") + turns = _cited_turns(item.get("turns")) + reason = str(item.get("reason") or "").strip() + graded = _graded_turns(criterion) + if verdict == "fail": + if not any(graded is None or t in graded for t in turns): + problems.append(f"{rid}: a fail must cite at least one graded turn; cited {turns}" + + (f", graded {sorted(graded)}" if graded is not None else "")) + said = _reason_says_no_failure(reason) + if said: + problems.append(f"{rid}: a fail whose reason says {said!r} contradicts itself; the verdict must follow " + "the reason") + if problems: + return None, problems + change = change and verdict == "fail" + lowers = change and not _never_gates(criterion) + row = {**criterion, "id": rid, "reason": reason, + "turns": [t for t in turns if graded is None or t in graded], + "verdict": verdict, "could_change_grade": change, + "justification": reason, "score": 0.0 if lowers else 1.0, "result": not lowers} + outside = [t for t in turns if graded is not None and t not in graded] + if outside: + row["turns_outside_graded"] = outside + if isinstance(item.get("assistant_asked"), bool): + row["assistant_asked"] = item["assistant_asked"] + return row, [] + + +def diagnose_user_sim_checks_response( + response_text: str, criteria: list[dict], +) -> tuple[list[dict] | None, JudgeResponseDiscrepancy | None]: + """Parse a user_sim_checks judge response into rows, or a discrepancy for the correction prompt: no JSON + object, the wrong row set, or a row the user sim judge can't use (its verdict or could_change_grade invalid, an + n/a where ``na_when`` is never, a fail citing no graded turn, a fail whose reason concludes nothing failed).""" + parsed = _user_sim_json(response_text) + if parsed is None: + return None, _parse_failure_discrepancy(criteria, "the answer is not a JSON object") + results = _extract_rubric_binary_results(parsed) + if not isinstance(results, list): + return None, _parse_failure_discrepancy(criteria, f"unexpected response shape: {type(parsed).__name__}") + if not all(isinstance(r, dict) for r in results): + return None, _parse_failure_discrepancy(criteria, "every result must be a JSON object") + discrepancy = _analyze_result_ids(results, criteria) + if discrepancy is not None: + return None, discrepancy + by_id = {c["id"]: c for c in criteria} + rows: list[dict] = [] + problems: list[str] = [] + for item in results: + row, its_problems = _user_sim_row(item, by_id[item["id"]]) + problems += its_problems + if row is not None: + rows.append(row) + if problems: + return None, JudgeResponseDiscrepancy( + expected_count=len(criteria), returned_count=len(results), expected_ids=[c["id"] for c in criteria], + missing_ids=[], duplicate_ids=[], unknown_ids=[], row_problems=problems) + return rows, None + + +def _parse_user_sim_checks_response(response_text: str, criteria: list[dict]) -> list[dict]: + """Rows, or ValueError: an unusable answer has no verdict, so it never degrades to fails that would gate.""" + rows, discrepancy = diagnose_user_sim_checks_response(response_text, criteria) + if discrepancy is not None: + raise ValueError(f"user_sim_checks answer not usable: {discrepancy}") + assert rows is not None + return rows + + +def format_user_sim_checks_correction( + *, + eval_prompt: str, + previous_response: str, + discrepancy: JudgeResponseDiscrepancy, + max_previous_response_chars: int = _CORRECTION_PROMPT_MAX_PREVIOUS_RESPONSE_CHARS, +) -> str: + """The user_sim_checks correction prompt: what was wrong, then the row shape again.""" + truncated = previous_response.strip() + if len(truncated) > max_previous_response_chars: + truncated = truncated[:max_previous_response_chars] + "\n... [truncated]" + if discrepancy.parse_error: + problems = [f"Your response could not be read: {discrepancy.parse_error}", + "Return only a JSON object with a `results` array — no preamble, markdown fences, or extra text."] + elif discrepancy.row_problems: + problems = list(discrepancy.row_problems) + else: + problems = [f"You returned {discrepancy.returned_count} result(s) but there are " + f"{discrepancy.expected_count} criteria."] + for ids, what in ((discrepancy.missing_ids, "Missing result(s) for criterion id(s)"), + (discrepancy.duplicate_ids, "Duplicate result(s) for criterion id(s)"), + (discrepancy.unknown_ids, "Unknown criterion id(s) in your results")): + if ids: + problems.append(f"{what}: {', '.join(ids)}") + correction_block = "\n".join([ + "## Correction Required", + "", + "Your previous response did not satisfy the output requirements:", + *[f"- {problem}" for problem in problems], + "", + f"Return JSON with a `results` array containing exactly {discrepancy.expected_count} objects — " + "one per criterion, no duplicates, no extras.", + f"Required criterion ids (in any order): {', '.join(discrepancy.expected_ids)}", + "", + "Each result must include, in this order:", + '- "id": the criterion id', + '- "reason": at most 40 words, written before the verdict', + '- "turns": the turn numbers the reason rests on; a fail cites at least one graded turn', + '- "verdict": "pass", "fail" or "n/a", following from the reason: a reason that finds no clear failure is ' + 'never "fail"', + '- "could_change_grade": true or false', + "", + "## Your Previous Response", + truncated, + ]) + return f"{eval_prompt.rstrip()}\n\n{correction_block}" + + class JudgeOutputFormat(str, Enum): RUBRIC_BINARY = "rubric_binary" RUBRIC_PARTIAL = "rubric_partial" @@ -1238,6 +1528,9 @@ class JudgeOutputFormat(str, Enum): # object); the entry is stored exactly as rubric_evidence's and the verifier writes a second, # trajectory_mistakes-shaped entry from the findings. RUBRIC_EVIDENCE_WITH_MISTAKES = "rubric_evidence_with_mistakes" + # Grades the user sim (the AI playing the user), not the assistant: per criterion a reason, the turns it cites, a + # pass / fail / n/a verdict and could_change_grade; a row lowers the score only on a fail that could change the grade. + USER_SIM_CHECKS = "user_sim_checks" DiagnoseResponse = Callable[[str, list[dict]], tuple[list[dict] | None, JudgeResponseDiscrepancy | None]] @@ -1311,6 +1604,15 @@ class JudgeOutputFormatSpec: diagnose_response=diagnose_trajectory_mistakes_response, format_correction_prompt=format_trajectory_mistakes_correction, ), + JudgeOutputFormat.USER_SIM_CHECKS: JudgeOutputFormatSpec( + output_format=_user_sim_checks_output_format(), + prompt_template_path=_USER_SIM_CHECKS_PROMPT_PATH, + prompt_template_inline=_USER_SIM_CHECKS_PROMPT_INLINE, + prompt_template_no_trajectory=_USER_SIM_CHECKS_PROMPT_NO_TRAJECTORY, + parse_response=_parse_user_sim_checks_response, + diagnose_response=diagnose_user_sim_checks_response, + format_correction_prompt=format_user_sim_checks_correction, + ), } diff --git a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py index a7796c22c..9230e7529 100644 --- a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py @@ -1516,7 +1516,7 @@ async def _run_judge_with_output_retries( ) discrepancy_log.append(discrepancy.to_dict()) if attempt == self.DEFAULT_MAX_RETRIES: - if discrepancy.non_finite_score_ids: + if discrepancy.non_finite_score_ids or discrepancy.row_problems: raise ValueError(f"Judge output invalid after {attempt} attempts: {discrepancy}") raise ValueError( f"Judge returned {discrepancy.returned_count} results but expected " diff --git a/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py b/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py new file mode 100644 index 000000000..0b399732a --- /dev/null +++ b/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py @@ -0,0 +1,345 @@ +"""The user_sim_checks judge output format: it grades the user sim (the AI playing the user), not the assistant.""" + +import json +from unittest.mock import AsyncMock + +import pytest + +from agent_env.task_step.context import TaskStepContext +from agent_env.task_step.task_steps.verifiers.judge_utils.judge_output_format import ( + JUDGE_OUTPUT_FORMAT_REGISTRY, + JudgeOutputFormat, + diagnose_user_sim_checks_response, + format_user_sim_checks_correction, + get_judge_output_format_spec, +) +from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep +from agent_env.task_step.task_steps.verifiers.scoring import ScoreAggregator, aggregate_score + +SPEC = get_judge_output_format_spec(JudgeOutputFormat.USER_SIM_CHECKS) + +RUN = """# THE USER SIM'S INSTRUCTIONS + +You are Hannah Reyes. K1 Dad is allergic to shellfish. M3 Mention the allergy only when asked. + + +# THE RUN + +## Turn 1 · GRADED +### What it was sent + +It's Mon 9:12 AM. NEW FROM YOUR AI ASSISTANT: Booking Saturday dinner for 4 at Pescado. Any dietary needs? + +### Its answer +outcome: reply +message: "dad cant do shellfish" + +## Turn 2 · GRADED +### What it was sent + +It's Mon 9:40 AM. NEW FROM YOUR AI ASSISTANT: Booked for 7 PM. Want me to add a birthday note? + +### Its answer +outcome: reply +message: "sure. also dad cant do shellfish, its serious" +""" + + +def _criteria(*ids: str, graded=(1, 2), **extra) -> list[dict]: + return [{"id": cid, "question": f"question {cid}", "graded_turns": list(graded), **extra} for cid in ids] + + +def _row(cid="c1", *, verdict="pass", turns=(), could_change_grade=False, reason="Answered what was asked.", + **extra) -> dict: + return {"id": cid, "reason": reason, "turns": list(turns), "verdict": verdict, + "could_change_grade": could_change_grade, **extra} + + +def _answer(*rows: dict) -> str: + return json.dumps({"results": list(rows)}) + + +def _fail(cid="c1", *, turns=(2,), could_change_grade=True, + reason="At turn 2 she repeated the shellfish allergy unasked; M3 says only when asked.", **extra) -> dict: + return _row(cid, verdict="fail", turns=turns, could_change_grade=could_change_grade, reason=reason, **extra) + + +def _render(template, criteria_json="[]") -> str: + """A template filled the way rubrics_verifier fills it: every keyword it passes, used or not.""" + return template.substitute(agent_prompt="the assistant's prompt", agent_response="the assistant's answer", + criteria_json=criteria_json, trajectory_content=RUN, + trajectory_path="/tmp/trajectory.json", checks_instruction="[]") + + +# ---- templates ---------------------------------------------------------------------------------------------------- +def test_it_is_registered_and_found_by_name(): + assert JudgeOutputFormat("user_sim_checks") is JudgeOutputFormat.USER_SIM_CHECKS + assert get_judge_output_format_spec("user_sim_checks") is JUDGE_OUTPUT_FORMAT_REGISTRY[ + JudgeOutputFormat.USER_SIM_CHECKS] + assert SPEC.stored_format is None and SPEC.cites_evidence is False + + +def test_the_judge_is_told_it_grades_the_user_sim_never_the_assistant(): + for template in (SPEC.prompt_template_inline, SPEC.prompt_template_path, SPEC.prompt_template_no_trajectory): + prompt = _render(template) + assert prompt.startswith('You check a user simulator ("the user sim")') + assert "You grade the user sim, never the assistant." in prompt + # Nothing of the assistant's own prompt or answer is put in front of the judge as the thing to grade. + assert "the assistant's prompt" not in prompt and "the assistant's answer" not in prompt + + +def test_the_run_comes_first_and_the_criteria_last_so_calls_on_one_run_share_a_cached_head(): + one = _render(SPEC.prompt_template_inline, json.dumps(_criteria("c1"))) + other = _render(SPEC.prompt_template_inline, json.dumps(_criteria("c1", "c2"))) + assert one.index(RUN) < one.index("# THE CRITERIA") < one.index('"question": "question c1"') + assert one.endswith("Return only the JSON object.") + head = one[:one.index("# THE CRITERIA")] + assert other.startswith(head) and RUN in head + path = _render(SPEC.prompt_template_path) + assert path.index("/tmp/trajectory.json") < path.index("# THE CRITERIA") + + +def test_without_a_trajectory_the_judge_is_told_the_run_follows_the_criteria(): + prompt = _render(SPEC.prompt_template_no_trajectory) + assert RUN not in prompt + assert prompt.endswith("The user sim's instructions and the run follow.") + + +def test_the_judge_reasons_first_and_the_verdict_follows_the_reason(): + prompt = _render(SPEC.prompt_template_inline) + assert "Reason first and decide last." in prompt and "at most 40 words" in prompt + assert 'a reason that finds no clear failure gives "pass" (or "n/a"), never "fail"' in prompt + shape = prompt.split("Return only this JSON object")[1] + assert (shape.index('"id"') < shape.index('"reason"') < shape.index('"turns"') < shape.index('"verdict"') + < shape.index('"could_change_grade"')) + + +def test_the_judge_is_told_how_to_read_the_run_and_the_criteria_fields(): + prompt = _render(SPEC.prompt_template_inline) + for told in ("Turns marked CONTEXT ONLY are fixed history", "Grade only the turns marked GRADED", + '"graded_turns"', "The assistant's own words are the assistant's", + 'A "hard" item is an exact rule', 'A "general" item is a judgement', + '"na_when": "never" means it always applies', + '"could_change_grade" says whether the failure could change how the assistant is graded'): + assert told in prompt, told + + +# ---- schema ------------------------------------------------------------------------------------------------------- +def test_each_row_gives_its_reason_first_then_turns_verdict_and_could_change_grade(): + schema = SPEC.output_format + assert schema["type"] == "json_schema" and schema["schema"]["required"] == ["results"] + row = schema["schema"]["properties"]["results"]["items"] + order = ["id", "reason", "turns", "verdict", "could_change_grade"] + assert list(row["properties"]) == order == row["required"] + assert row["properties"]["verdict"]["enum"] == ["pass", "fail", "n/a"] + assert row["properties"]["turns"] == {"type": "array", "items": {"type": "integer"}} + assert row["properties"]["could_change_grade"] == {"type": "boolean"} + assert row["additionalProperties"] is False and schema["schema"]["additionalProperties"] is False + + +# ---- reading an answer -------------------------------------------------------------------------------------------- +def test_an_answer_is_read_into_rows_with_turns_kept_to_the_graded_ones(): + criteria = _criteria("c1", "c2", graded=[2], family="G1") + rows, discrepancy = diagnose_user_sim_checks_response( + _answer(_fail("c1", turns=[1, 2, "x"]), _row("c2", could_change_grade=True)), criteria) + assert discrepancy is None + fail, ok = rows + assert (fail["verdict"], fail["turns"], fail["turns_outside_graded"]) == ("fail", [2], [1]) + assert fail["could_change_grade"] is True and fail["justification"] == fail["reason"] + assert fail["family"] == "G1" and fail["question"] == "question c1" # the criterion rides on its row + # A pass can't change the grade, whatever the judge wrote. + assert ok["could_change_grade"] is False and "turns_outside_graded" not in ok + + +def test_the_parser_reads_the_keys_in_any_order_and_tolerates_a_fence_or_text_around_the_object(): + flipped = {"could_change_grade": True, "verdict": "FAIL ", "turns": [2], "reason": "Leaked the allergy.", "id": "c1"} + for text in (json.dumps({"results": [flipped]}), "```json\n" + _answer(flipped) + "\n```", + "Here is my answer:\n" + _answer(flipped) + "\nDone."): + rows, discrepancy = diagnose_user_sim_checks_response(text, _criteria("c1")) + assert discrepancy is None and rows[0]["verdict"] == "fail", text + + +def test_an_answer_that_is_not_a_json_object_or_has_the_wrong_rows_is_asked_again(): + criteria = _criteria("c1", "c2") + for text, problem in (("dad cant do shellfish", "the answer is not a JSON object"), + (json.dumps({"verdict": "pass"}), "unexpected response shape: dict"), + (json.dumps({"results": ["pass", "pass"]}), "every result must be a JSON object")): + rows, discrepancy = diagnose_user_sim_checks_response(text, criteria) + assert rows is None and discrepancy.parse_error == problem + rows, discrepancy = diagnose_user_sim_checks_response(_answer(_row("c1"), _row("c1"), _row("c9")), criteria) + assert rows is None + assert (discrepancy.missing_ids, discrepancy.duplicate_ids, discrepancy.unknown_ids) == (["c2"], ["c1"], ["c9"]) + + +def test_a_verdict_or_could_change_grade_out_of_range_is_asked_again(): + rows, discrepancy = diagnose_user_sim_checks_response( + _answer(_row("c1", verdict="maybe"), _row("c2", could_change_grade="yes")), _criteria("c1", "c2")) + assert rows is None + assert discrepancy.row_problems == ["c1: verdict 'maybe' is not one of ['pass', 'fail', 'n/a']", + "c2: could_change_grade must be true or false"] + + +def test_a_fail_that_cites_no_graded_turn_is_asked_again_and_a_pass_needs_none(): + for turns, cited in (([99], r"\[99\]"), ([], r"\[\]")): + rows, discrepancy = diagnose_user_sim_checks_response(_answer(_fail(turns=turns)), _criteria("c1", graded=[1])) + assert rows is None + assert discrepancy.row_problems == [f"c1: a fail must cite at least one graded turn; cited {turns}, graded [1]"] + with pytest.raises(ValueError, match=f"a fail must cite at least one graded turn; cited {cited}"): + SPEC.parse_response(_answer(_fail(turns=turns)), _criteria("c1", graded=[1])) + for ok in (_row(turns=[99]), _row(verdict="n/a", turns=[99])): + rows, discrepancy = diagnose_user_sim_checks_response(_answer(ok), _criteria("c1", graded=[1])) + assert discrepancy is None and rows[0]["turns"] == [] and rows[0]["turns_outside_graded"] == [99] + + +def test_without_graded_turns_on_the_criterion_a_fail_still_has_to_cite_a_turn(): + criteria = [{"id": "c1", "question": "question c1"}] + rows, discrepancy = diagnose_user_sim_checks_response(_answer(_fail(turns=[])), criteria) + assert discrepancy.row_problems == ["c1: a fail must cite at least one graded turn; cited []"] + rows, discrepancy = diagnose_user_sim_checks_response(_answer(_fail(turns=[7])), criteria) + assert discrepancy is None and rows[0]["turns"] == [7] and "turns_outside_graded" not in rows[0] + + +# The endings of the two contradictory reasons seen in real runs, and a plain one. +SAYS_NO_FAILURE = ("Hannah flagged it anyway which is fine. No clear failure found.", + "This is not a delegation failure by Linda. No clear delegation failures found.", + "Every answer fits her rules; nothing fails.") + + +def test_a_fail_whose_reason_concludes_nothing_failed_is_asked_again(): + for reason in SAYS_NO_FAILURE: + rows, discrepancy = diagnose_user_sim_checks_response(_answer(_fail(reason=reason)), _criteria("c1")) + assert rows is None and len(discrepancy.row_problems) == 1 + assert "c1: a fail whose reason says" in discrepancy.row_problems[0] + assert "contradicts itself; the verdict must follow the reason" in discrepancy.row_problems[0] + # A pass may say it. + rows, discrepancy = diagnose_user_sim_checks_response(_answer(_row(reason=reason)), _criteria("c1")) + assert discrepancy is None and rows[0]["reason"] == reason + # Only the last sentence is the conclusion: a fail that clears one turn and fails another is kept. + for reason in ("No clear failure at turn 1; at turn 2 she revealed the private password.", + "Turn 1 is fine, not a failure. At turn 2 she volunteered the shellfish allergy unasked; M3 says " + "only when asked."): + rows, discrepancy = diagnose_user_sim_checks_response(_answer(_fail(reason=reason)), _criteria("c1")) + assert discrepancy is None and rows[0]["verdict"] == "fail" and rows[0]["could_change_grade"] is True + + +def test_an_n_a_where_na_when_is_never_is_asked_again_and_otherwise_stands(): + na = _row(verdict="n/a", reason="Rae never came up.") + rows, discrepancy = diagnose_user_sim_checks_response(_answer(na), _criteria("c1", na_when="never")) + assert discrepancy.row_problems == ["c1 may not be n/a: it always applies, so answer pass or fail"] + for criteria in (_criteria("c1"), _criteria("c1", na_when="the assistant never asks about Rae's order")): + rows, discrepancy = diagnose_user_sim_checks_response(_answer(na), criteria) + assert discrepancy is None and rows[0]["verdict"] == "n/a" and rows[0]["score"] == 1.0 + + +def test_every_unusable_row_is_named_in_one_go(): + rows, discrepancy = diagnose_user_sim_checks_response( + _answer(_fail("c1", turns=[]), _row("c2"), _fail("c3", reason=SAYS_NO_FAILURE[0])), _criteria("c1", "c2", "c3")) + assert rows is None and [p.split(":")[0] for p in discrepancy.row_problems] == ["c1", "c3"] + assert discrepancy.to_dict()["row_problems"] == discrepancy.row_problems + assert str(discrepancy) == "; ".join(discrepancy.row_problems) + + +def test_assistant_asked_is_kept_when_given_and_never_changes_the_score(): + rows, discrepancy = diagnose_user_sim_checks_response( + _answer(_fail("c1", assistant_asked=True), _row("c2", assistant_asked="yes"), _row("c3")), + _criteria("c1", "c2", "c3")) + assert discrepancy is None + assert rows[0]["assistant_asked"] is True and rows[0]["score"] == 0.0 + assert "assistant_asked" not in rows[1] and "assistant_asked" not in rows[2] + + +def test_parse_response_never_turns_an_unusable_answer_into_fails(): + with pytest.raises(ValueError, match="user_sim_checks answer not usable: invalid JSON"): + SPEC.parse_response("not json", _criteria("c1")) + assert SPEC.parse_response(_answer(_row("c1")), _criteria("c1"))[0]["result"] is True + + +# ---- the correction prompt ---------------------------------------------------------------------------------------- +def test_the_correction_prompt_names_each_problem_and_asks_for_the_row_shape_again(): + previous = _answer(_fail("c1", turns=[99]), _row("c2")) + _, discrepancy = diagnose_user_sim_checks_response(previous, _criteria("c1", "c2", graded=[1])) + prompt = SPEC.format_correction_prompt(eval_prompt="EVAL", previous_response=previous, discrepancy=discrepancy) + assert prompt.startswith("EVAL\n\n## Correction Required") + assert "- c1: a fail must cite at least one graded turn; cited [99], graded [1]" in prompt + assert "exactly 2 objects" in prompt and "Required criterion ids (in any order): c1, c2" in prompt + for field in ('"id"', '"reason": at most 40 words', '"turns"', '"verdict"', '"could_change_grade"'): + assert f"- {field}" in prompt + assert '"score"' not in prompt and '"justification"' not in prompt + assert prompt.endswith("## Your Previous Response\n" + previous) + + +def test_the_correction_prompt_for_unreadable_json_or_a_wrong_row_set(): + _, unreadable = diagnose_user_sim_checks_response("no json here", _criteria("c1")) + prompt = format_user_sim_checks_correction(eval_prompt="EVAL", previous_response="no json here", + discrepancy=unreadable) + assert "Your response could not be read: the answer is not a JSON object" in prompt + _, wrong = diagnose_user_sim_checks_response(_answer(_row("c1"), _row("c9")), _criteria("c1", "c2")) + prompt = format_user_sim_checks_correction(eval_prompt="EVAL", previous_response="x" * 9000, discrepancy=wrong) + assert "Missing result(s) for criterion id(s): c2" in prompt and "Unknown criterion id(s) in your results: c9" in prompt + assert prompt.endswith("... [truncated]") + + +# ---- the score ---------------------------------------------------------------------------------------------------- +def _score(*rows: dict, criteria: list[dict], aggregator=ScoreAggregator.WEIGHTED_AVERAGE) -> float: + got, discrepancy = diagnose_user_sim_checks_response(_answer(*rows), criteria) + assert discrepancy is None + return aggregate_score(got, aggregator) + + +def test_a_row_lowers_the_score_only_on_a_fail_that_could_change_the_grade(): + criteria = _criteria("c1", "c2", "c3") + assert _score(_row("c1"), _row("c2", verdict="n/a"), _row("c3"), criteria=criteria) == 1.0 + assert _score(_fail("c1", could_change_grade=False), _row("c2"), _row("c3"), criteria=criteria) == 1.0 + assert _score(_fail("c1"), _row("c2"), _row("c3"), criteria=criteria) == pytest.approx(2 / 3) + rows, _ = diagnose_user_sim_checks_response(_answer(_fail("c1", could_change_grade=False)), _criteria("c1")) + assert (rows[0]["verdict"], rows[0]["score"], rows[0]["result"]) == ("fail", 1.0, True) + + +def test_a_weight_0_row_is_recorded_but_never_gates(): + criteria = [*_criteria("guidance"), *_criteria("style", weight=0)] + rows, _ = diagnose_user_sim_checks_response(_answer(_row("guidance"), _fail("style")), criteria) + style = rows[1] + assert (style["verdict"], style["could_change_grade"], style["weight"]) == ("fail", True, 0) + assert (style["score"], style["result"]) == (1.0, True) + for aggregator in ScoreAggregator: + assert _score(_row("guidance"), _fail("style"), criteria=criteria, aggregator=aggregator) == 1.0 + assert _score(_fail("guidance"), _fail("style"), criteria=criteria) == 0.0 + + +# ---- in rubrics_verifier's retry loop ----------------------------------------------------------------------------- +def _verifier() -> RubricsVerifierTaskStep: + return RubricsVerifierTaskStep(id="judge-user-sim", version=1, criteria=_criteria("c1", "c2", graded=[1]), + prompt_id="prompt-1", use_trajectory=False, verifier_id="user-sim", + output_format=JudgeOutputFormat.USER_SIM_CHECKS) + + +async def _run_judge(verifier: RubricsVerifierTaskStep): + return await verifier._run_judge_with_output_retries( + eval_prompt="EVAL", criteria=verifier.criteria, context=TaskStepContext(), model="claude-sonnet-4-6", + judge_a2a_url=None, judge_agent_card={}, judge_agent=None, judge_effort=None, + judge_max_thinking_tokens=None, use_agent_judge=False) + + +@pytest.mark.asyncio +async def test_rubrics_verifier_asks_again_for_an_unusable_row_and_keeps_the_usable_answer(monkeypatch): + verifier = _verifier() + unusable = _answer(_fail("c1", turns=[99]), _row("c2")) + usable = _answer(_fail("c1", turns=[1]), _row("c2")) + judge = AsyncMock(side_effect=[{"response": unusable}, {"response": usable}]) + monkeypatch.setattr(verifier, "_prompt_llm_judge", judge) + rows, retries, discrepancies, _ = await _run_judge(verifier) + assert retries == 1 and [r["verdict"] for r in rows] == ["fail", "pass"] + assert discrepancies[0]["row_problems"] == ["c1: a fail must cite at least one graded turn; cited [99], graded [1]"] + correction = judge.await_args_list[1].args[0] + assert correction.startswith("EVAL\n\n## Correction Required") and unusable in correction + assert aggregate_score(rows, ScoreAggregator.WEIGHTED_AVERAGE) == 0.5 + + +@pytest.mark.asyncio +async def test_rubrics_verifier_names_the_unusable_row_when_the_judge_never_fixes_it(monkeypatch): + verifier = _verifier() + monkeypatch.setattr(verifier, "_prompt_llm_judge", AsyncMock( + return_value={"response": _answer(_fail("c1", turns=[1], reason=SAYS_NO_FAILURE[0]), _row("c2"))})) + with pytest.raises(ValueError, match=r"Judge output invalid after 3 attempts: c1: a fail whose reason says " + r"'No clear failure found' contradicts itself"): + await _run_judge(verifier) From 4db2603055617ad1f2809722239f73f9bdbb1718 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 16 Nov 2026 11:05:00 -0800 Subject: [PATCH 2/7] feat(verifiers): user_sim_checks rows give assistant_asked, recorded only Every row now gives assistant_asked: true when the assistant asked for or invited the behaviour the row grades (asked the user to write a message, or to give a fact it should look up), false otherwise. It is in the schema and the prompt, required on every row (a missing or non-bool one is asked again), and recorded only: it never changes a row's score or a gate. A test also shows that a task item follows the same rule as a check: a fail lowers the score only with could_change_grade. Co-Authored-By: Claude Opus 5.5 --- .../judge_utils/judge_output_format.py | 24 +++++--- .../test_user_sim_checks_format.py | 58 +++++++++++++------ 2 files changed, 54 insertions(+), 28 deletions(-) diff --git a/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py b/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py index e6a0e37d8..cffa0ae63 100644 --- a/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py +++ b/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py @@ -1282,10 +1282,13 @@ def format_trajectory_mistakes_correction( reason to at most 40 words: the evidence and the conclusion, not your working. - "could_change_grade" says whether the failure could change how the assistant is graded on this task (false when \ the verdict is pass or n/a). +- "assistant_asked" is true when the assistant asked for or invited the behaviour you graded (e.g. asked the user \ +to write a message or to give a fact it should look up); false otherwise. Give it on every result, false when \ +nothing was asked; it is recorded only and doesn't change the verdict. Return only this JSON object, with one result per criterion, each result's keys in this order: {"results": [{"id": "the criterion's id", "reason": "at most 40 words", "turns": [turn numbers the reason rests on], \ -"verdict": "pass" | "fail" | "n/a", "could_change_grade": true | false}]}""" +"verdict": "pass" | "fail" | "n/a", "could_change_grade": true | false, "assistant_asked": true | false}]}""" _USER_SIM_CRITERIA = """ @@ -1322,8 +1325,9 @@ def _user_sim_checks_output_format() -> dict: "turns": {"type": "array", "items": {"type": "integer"}}, "verdict": {"type": "string", "enum": list(USER_SIM_VERDICTS)}, "could_change_grade": {"type": "boolean"}, + "assistant_asked": {"type": "boolean"}, }, - "required": ["id", "reason", "turns", "verdict", "could_change_grade"], + "required": ["id", "reason", "turns", "verdict", "could_change_grade", "assistant_asked"], "additionalProperties": False, }, }, @@ -1390,8 +1394,8 @@ def _never_gates(criterion: dict) -> bool: def _user_sim_row(item: dict, criterion: dict) -> tuple[dict | None, list[str]]: """One result checked against its criterion: ``(row, [])``, or ``(None, problems)`` naming the row. The row lowers the score (score 0.0, result false) only on a fail with could_change_grade, and never at weight 0; - ``verdict`` and ``could_change_grade`` keep what the judge said either way. An ``assistant_asked`` bool (the - assistant asked for the behaviour) is kept when the judge gives one; it doesn't change the score.""" + ``verdict`` and ``could_change_grade`` keep what the judge said either way. ``assistant_asked`` (the assistant + asked for or invited the behaviour) is required and recorded only: it never changes the score or a gate.""" rid = item.get("id") problems = [] verdict = str(item.get("verdict", "")).strip().lower() @@ -1400,6 +1404,9 @@ def _user_sim_row(item: dict, criterion: dict) -> tuple[dict | None, list[str]]: change = item.get("could_change_grade") if not isinstance(change, bool): problems.append(f"{rid}: could_change_grade must be true or false") + asked = item.get("assistant_asked") + if not isinstance(asked, bool): + problems.append(f"{rid}: assistant_asked must be true or false") if verdict == "n/a" and str(criterion.get("na_when") or "").strip().lower() == _USER_SIM_NA_NEVER: problems.append(f"{rid} may not be n/a: it always applies, so answer pass or fail") turns = _cited_turns(item.get("turns")) @@ -1419,13 +1426,11 @@ def _user_sim_row(item: dict, criterion: dict) -> tuple[dict | None, list[str]]: lowers = change and not _never_gates(criterion) row = {**criterion, "id": rid, "reason": reason, "turns": [t for t in turns if graded is None or t in graded], - "verdict": verdict, "could_change_grade": change, + "verdict": verdict, "could_change_grade": change, "assistant_asked": asked, "justification": reason, "score": 0.0 if lowers else 1.0, "result": not lowers} outside = [t for t in turns if graded is not None and t not in graded] if outside: row["turns_outside_graded"] = outside - if isinstance(item.get("assistant_asked"), bool): - row["assistant_asked"] = item["assistant_asked"] return row, [] @@ -1434,7 +1439,7 @@ def diagnose_user_sim_checks_response( ) -> tuple[list[dict] | None, JudgeResponseDiscrepancy | None]: """Parse a user_sim_checks judge response into rows, or a discrepancy for the correction prompt: no JSON object, the wrong row set, or a row the user sim judge can't use (its verdict or could_change_grade invalid, an - n/a where ``na_when`` is never, a fail citing no graded turn, a fail whose reason concludes nothing failed).""" + n/a where ``na_when`` is never, an assistant_asked that isn't true or false, a fail citing no graded turn, a fail whose reason concludes nothing failed).""" parsed = _user_sim_json(response_text) if parsed is None: return None, _parse_failure_discrepancy(criteria, "the answer is not a JSON object") @@ -1511,6 +1516,7 @@ def format_user_sim_checks_correction( '- "verdict": "pass", "fail" or "n/a", following from the reason: a reason that finds no clear failure is ' 'never "fail"', '- "could_change_grade": true or false', + '- "assistant_asked": true when the assistant asked for or invited the behaviour you graded, else false', "", "## Your Previous Response", truncated, @@ -1529,7 +1535,7 @@ class JudgeOutputFormat(str, Enum): # trajectory_mistakes-shaped entry from the findings. RUBRIC_EVIDENCE_WITH_MISTAKES = "rubric_evidence_with_mistakes" # Grades the user sim (the AI playing the user), not the assistant: per criterion a reason, the turns it cites, a - # pass / fail / n/a verdict and could_change_grade; a row lowers the score only on a fail that could change the grade. + # pass / fail / n/a verdict, could_change_grade and assistant_asked (recorded only); a row lowers the score only on a fail that could change the grade. USER_SIM_CHECKS = "user_sim_checks" diff --git a/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py b/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py index 0b399732a..8f01b83eb 100644 --- a/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py +++ b/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py @@ -50,9 +50,9 @@ def _criteria(*ids: str, graded=(1, 2), **extra) -> list[dict]: def _row(cid="c1", *, verdict="pass", turns=(), could_change_grade=False, reason="Answered what was asked.", - **extra) -> dict: + assistant_asked=False, **extra) -> dict: return {"id": cid, "reason": reason, "turns": list(turns), "verdict": verdict, - "could_change_grade": could_change_grade, **extra} + "could_change_grade": could_change_grade, "assistant_asked": assistant_asked, **extra} def _answer(*rows: dict) -> str: @@ -111,7 +111,7 @@ def test_the_judge_reasons_first_and_the_verdict_follows_the_reason(): assert 'a reason that finds no clear failure gives "pass" (or "n/a"), never "fail"' in prompt shape = prompt.split("Return only this JSON object")[1] assert (shape.index('"id"') < shape.index('"reason"') < shape.index('"turns"') < shape.index('"verdict"') - < shape.index('"could_change_grade"')) + < shape.index('"could_change_grade"') < shape.index('"assistant_asked"')) def test_the_judge_is_told_how_to_read_the_run_and_the_criteria_fields(): @@ -120,20 +120,23 @@ def test_the_judge_is_told_how_to_read_the_run_and_the_criteria_fields(): '"graded_turns"', "The assistant's own words are the assistant's", 'A "hard" item is an exact rule', 'A "general" item is a judgement', '"na_when": "never" means it always applies', - '"could_change_grade" says whether the failure could change how the assistant is graded'): + '"could_change_grade" says whether the failure could change how the assistant is graded', + '"assistant_asked" is true when the assistant asked for or invited the behaviour you graded (e.g. ' + 'asked the user to write a message or to give a fact it should look up); false otherwise', + "false when nothing was asked; it is recorded only and doesn't change the verdict"): assert told in prompt, told # ---- schema ------------------------------------------------------------------------------------------------------- -def test_each_row_gives_its_reason_first_then_turns_verdict_and_could_change_grade(): +def test_each_row_gives_its_reason_first_then_turns_verdict_could_change_grade_and_assistant_asked(): schema = SPEC.output_format assert schema["type"] == "json_schema" and schema["schema"]["required"] == ["results"] row = schema["schema"]["properties"]["results"]["items"] - order = ["id", "reason", "turns", "verdict", "could_change_grade"] + order = ["id", "reason", "turns", "verdict", "could_change_grade", "assistant_asked"] assert list(row["properties"]) == order == row["required"] assert row["properties"]["verdict"]["enum"] == ["pass", "fail", "n/a"] assert row["properties"]["turns"] == {"type": "array", "items": {"type": "integer"}} - assert row["properties"]["could_change_grade"] == {"type": "boolean"} + assert row["properties"]["could_change_grade"] == row["properties"]["assistant_asked"] == {"type": "boolean"} assert row["additionalProperties"] is False and schema["schema"]["additionalProperties"] is False @@ -152,7 +155,8 @@ def test_an_answer_is_read_into_rows_with_turns_kept_to_the_graded_ones(): def test_the_parser_reads_the_keys_in_any_order_and_tolerates_a_fence_or_text_around_the_object(): - flipped = {"could_change_grade": True, "verdict": "FAIL ", "turns": [2], "reason": "Leaked the allergy.", "id": "c1"} + flipped = {"assistant_asked": False, "could_change_grade": True, "verdict": "FAIL ", "turns": [2], + "reason": "Leaked the allergy.", "id": "c1"} for text in (json.dumps({"results": [flipped]}), "```json\n" + _answer(flipped) + "\n```", "Here is my answer:\n" + _answer(flipped) + "\nDone."): rows, discrepancy = diagnose_user_sim_checks_response(text, _criteria("c1")) @@ -171,12 +175,17 @@ def test_an_answer_that_is_not_a_json_object_or_has_the_wrong_rows_is_asked_agai assert (discrepancy.missing_ids, discrepancy.duplicate_ids, discrepancy.unknown_ids) == (["c2"], ["c1"], ["c9"]) -def test_a_verdict_or_could_change_grade_out_of_range_is_asked_again(): +def test_a_verdict_could_change_grade_or_assistant_asked_out_of_range_is_asked_again(): + missing = _row("c4") + del missing["assistant_asked"] rows, discrepancy = diagnose_user_sim_checks_response( - _answer(_row("c1", verdict="maybe"), _row("c2", could_change_grade="yes")), _criteria("c1", "c2")) + _answer(_row("c1", verdict="maybe"), _row("c2", could_change_grade="yes"), _row("c3", assistant_asked="no"), + missing), _criteria("c1", "c2", "c3", "c4")) assert rows is None assert discrepancy.row_problems == ["c1: verdict 'maybe' is not one of ['pass', 'fail', 'n/a']", - "c2: could_change_grade must be true or false"] + "c2: could_change_grade must be true or false", + "c3: assistant_asked must be true or false", + "c4: assistant_asked must be true or false"] def test_a_fail_that_cites_no_graded_turn_is_asked_again_and_a_pass_needs_none(): @@ -239,13 +248,17 @@ def test_every_unusable_row_is_named_in_one_go(): assert str(discrepancy) == "; ".join(discrepancy.row_problems) -def test_assistant_asked_is_kept_when_given_and_never_changes_the_score(): - rows, discrepancy = diagnose_user_sim_checks_response( - _answer(_fail("c1", assistant_asked=True), _row("c2", assistant_asked="yes"), _row("c3")), - _criteria("c1", "c2", "c3")) - assert discrepancy is None - assert rows[0]["assistant_asked"] is True and rows[0]["score"] == 0.0 - assert "assistant_asked" not in rows[1] and "assistant_asked" not in rows[2] +def test_assistant_asked_is_recorded_on_every_row_and_never_changes_the_score_or_a_gate(): + criteria = _criteria("c1", "c2", "c3") + for asked in (True, False): + rows, discrepancy = diagnose_user_sim_checks_response( + _answer(_fail("c1", assistant_asked=asked), _fail("c2", could_change_grade=False, assistant_asked=asked), + _row("c3", assistant_asked=asked)), criteria) + assert discrepancy is None and [r["assistant_asked"] for r in rows] == [asked] * 3 + assert [(r["score"], r["result"]) for r in rows] == [(0.0, False), (1.0, True), (1.0, True)] + for aggregator in ScoreAggregator: + assert aggregate_score(rows, aggregator) == aggregate_score( + [{**r, "assistant_asked": not asked} for r in rows], aggregator) def test_parse_response_never_turns_an_unusable_answer_into_fails(): @@ -262,7 +275,8 @@ def test_the_correction_prompt_names_each_problem_and_asks_for_the_row_shape_aga assert prompt.startswith("EVAL\n\n## Correction Required") assert "- c1: a fail must cite at least one graded turn; cited [99], graded [1]" in prompt assert "exactly 2 objects" in prompt and "Required criterion ids (in any order): c1, c2" in prompt - for field in ('"id"', '"reason": at most 40 words', '"turns"', '"verdict"', '"could_change_grade"'): + for field in ('"id"', '"reason": at most 40 words', '"turns"', '"verdict"', '"could_change_grade"', + '"assistant_asked": true when the assistant asked for or invited the behaviour you graded'): assert f"- {field}" in prompt assert '"score"' not in prompt and '"justification"' not in prompt assert prompt.endswith("## Your Previous Response\n" + previous) @@ -295,6 +309,12 @@ def test_a_row_lowers_the_score_only_on_a_fail_that_could_change_the_grade(): assert (rows[0]["verdict"], rows[0]["score"], rows[0]["result"]) == ("fail", 1.0, True) +def test_a_task_item_follows_the_same_rule_as_a_check(): + criteria = [*_criteria("guidance"), *_criteria("a1_call_rae", family="A1", kind="hard")] + assert _score(_row("guidance"), _fail("a1_call_rae", could_change_grade=False), criteria=criteria) == 1.0 + assert _score(_row("guidance"), _fail("a1_call_rae"), criteria=criteria) == 0.5 + + def test_a_weight_0_row_is_recorded_but_never_gates(): criteria = [*_criteria("guidance"), *_criteria("style", weight=0)] rows, _ = diagnose_user_sim_checks_response(_answer(_row("guidance"), _fail("style")), criteria) From 3e07f442e91871e8498616e1427ac4fa44501ca0 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 16 Nov 2026 13:49:10 -0800 Subject: [PATCH 3/7] fix(verifiers): a malformed user_sim_checks answer is asked again, never raised A result whose id isn't text ({"results": [{"id": []}]}) reached the shared id check, which hashes each id and raised TypeError, failing the task instead of sending the answer back for correction. Each id is now checked first and a non-text one is a row problem. The same goes for a reason that isn't text and turns that aren't a list. An infinite turn number (OverflowError) is dropped like any other entry that isn't a turn, and `results` that isn't a list is named as such. Co-Authored-By: Claude Opus 5.5 --- .../judge_utils/judge_output_format.py | 25 ++++++++--- .../test_user_sim_checks_format.py | 44 +++++++++++++++++++ 2 files changed, 64 insertions(+), 5 deletions(-) diff --git a/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py b/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py index cffa0ae63..974652d9a 100644 --- a/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py +++ b/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py @@ -1359,7 +1359,7 @@ def _cited_turns(value: Any) -> list[int]: for v in value: try: out.append(int(v)) - except (TypeError, ValueError): + except (TypeError, ValueError, OverflowError): # OverflowError: an infinite number continue return out @@ -1409,8 +1409,12 @@ def _user_sim_row(item: dict, criterion: dict) -> tuple[dict | None, list[str]]: problems.append(f"{rid}: assistant_asked must be true or false") if verdict == "n/a" and str(criterion.get("na_when") or "").strip().lower() == _USER_SIM_NA_NEVER: problems.append(f"{rid} may not be n/a: it always applies, so answer pass or fail") + if not isinstance(item.get("reason"), str): + problems.append(f"{rid}: reason must be text") + if not isinstance(item.get("turns"), list): + problems.append(f"{rid}: turns must be a list of turn numbers") turns = _cited_turns(item.get("turns")) - reason = str(item.get("reason") or "").strip() + reason = item["reason"].strip() if isinstance(item.get("reason"), str) else "" graded = _graded_turns(criterion) if verdict == "fail": if not any(graded is None or t in graded for t in turns): @@ -1438,16 +1442,27 @@ def diagnose_user_sim_checks_response( response_text: str, criteria: list[dict], ) -> tuple[list[dict] | None, JudgeResponseDiscrepancy | None]: """Parse a user_sim_checks judge response into rows, or a discrepancy for the correction prompt: no JSON - object, the wrong row set, or a row the user sim judge can't use (its verdict or could_change_grade invalid, an - n/a where ``na_when`` is never, an assistant_asked that isn't true or false, a fail citing no graded turn, a fail whose reason concludes nothing failed).""" + object, the wrong row set, or a row the user sim judge can't use: an id, reason or turns of the wrong type, a + verdict, could_change_grade or assistant_asked out of range, an n/a where ``na_when`` is never, a fail citing no + graded turn, or a fail whose reason concludes nothing failed.""" parsed = _user_sim_json(response_text) if parsed is None: return None, _parse_failure_discrepancy(criteria, "the answer is not a JSON object") results = _extract_rubric_binary_results(parsed) if not isinstance(results, list): - return None, _parse_failure_discrepancy(criteria, f"unexpected response shape: {type(parsed).__name__}") + shape = ("`results` must be a list" if results is not None + else f"unexpected response shape: {type(parsed).__name__}") + return None, _parse_failure_discrepancy(criteria, shape) if not all(isinstance(r, dict) for r in results): return None, _parse_failure_discrepancy(criteria, "every result must be a JSON object") + # The shared id check hashes each id, so one that isn't text (a list, an object) would raise instead of being + # asked again. + bad_ids = [f"result {i}: id must be a criterion id (text), not {r.get('id')!r}" + for i, r in enumerate(results, 1) if not isinstance(r.get("id"), str)] + if bad_ids: + return None, JudgeResponseDiscrepancy( + expected_count=len(criteria), returned_count=len(results), expected_ids=[c["id"] for c in criteria], + missing_ids=[], duplicate_ids=[], unknown_ids=[], row_problems=bad_ids) discrepancy = _analyze_result_ids(results, criteria) if discrepancy is not None: return None, discrepancy diff --git a/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py b/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py index 8f01b83eb..f4f96211e 100644 --- a/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py +++ b/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py @@ -363,3 +363,47 @@ async def test_rubrics_verifier_names_the_unusable_row_when_the_judge_never_fixe with pytest.raises(ValueError, match=r"Judge output invalid after 3 attempts: c1: a fail whose reason says " r"'No clear failure found' contradicts itself"): await _run_judge(verifier) + + +# ---- malformed answers are asked again, never raised --------------------------------------------------------------- +GOOD = _row("c1") + + +@pytest.mark.parametrize("text, problem", [ + (json.dumps({"results": [{"id": []}]}), "result 1: id must be a criterion id (text), not []"), + (json.dumps({"results": [{**GOOD, "id": {"c": 1}}]}), "result 1: id must be a criterion id (text), not {'c': 1}"), + (json.dumps({"results": [{**GOOD, "id": None}]}), "result 1: id must be a criterion id (text), not None"), + (json.dumps({"results": [{**GOOD, "id": 1}]}), "result 1: id must be a criterion id (text), not 1"), + (json.dumps({"results": [{**GOOD, "reason": ["dad cant do shellfish"]}]}), "c1: reason must be text"), + (json.dumps({"results": [{k: v for k, v in GOOD.items() if k != "reason"}]}), "c1: reason must be text"), + (json.dumps({"results": [{**GOOD, "turns": "1"}]}), "c1: turns must be a list of turn numbers"), + (json.dumps({"results": [{**GOOD, "verdict": ["fail"]}]}), "c1: verdict ['fail'] is not one of"), + (json.dumps({"results": [{**GOOD, "could_change_grade": "true"}]}), "c1: could_change_grade must be true or false"), + (json.dumps({"results": [{**GOOD, "assistant_asked": None}]}), "c1: assistant_asked must be true or false"), + # An infinite turn number is dropped like any other that isn't one, so the fail cites none. + ('{"results": [{"id": "c1", "reason": "Leaked it.", "turns": [Infinity], "verdict": "fail", ' + '"could_change_grade": true, "assistant_asked": false}]}', "c1: a fail must cite at least one graded turn"), + (json.dumps({"results": {"c1": GOOD}}), "`results` must be a list"), + (json.dumps({"results": "pass"}), "`results` must be a list"), + (json.dumps({"results": [GOOD, "pass"]}), "every result must be a JSON object"), + (json.dumps("pass"), "unexpected response shape: str"), + ("null", "the answer is not a JSON object"), +]) +def test_a_malformed_answer_is_a_discrepancy_for_the_correction_prompt_never_an_exception(text, problem): + rows, discrepancy = diagnose_user_sim_checks_response(text, _criteria("c1")) + assert rows is None and problem in str(discrepancy) + prompt = SPEC.format_correction_prompt(eval_prompt="EVAL", previous_response=text, discrepancy=discrepancy) + assert "## Correction Required" in prompt + with pytest.raises(ValueError, match="user_sim_checks answer not usable"): + SPEC.parse_response(text, _criteria("c1")) + + +@pytest.mark.asyncio +async def test_rubrics_verifier_asks_again_when_a_result_s_id_is_not_text(monkeypatch): + verifier = _verifier() + judge = AsyncMock(side_effect=[{"response": json.dumps({"results": [{"id": []}]})}, + {"response": _answer(_row("c1"), _row("c2"))}]) + monkeypatch.setattr(verifier, "_prompt_llm_judge", judge) + rows, retries, discrepancies, _ = await _run_judge(verifier) + assert retries == 1 and [r["verdict"] for r in rows] == ["pass", "pass"] + assert discrepancies[0]["row_problems"] == ["result 1: id must be a criterion id (text), not []"] From 01a773f7dc96506b539cfc906f44a073d9b52d77 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 16 Nov 2026 14:29:30 -0800 Subject: [PATCH 4/7] fix(verifiers): a user_sim_checks turn that isn't a whole number is asked again, not read as a turn int() read `true` and `1.9` as turn 1, so a fail could gate on a turn the judge never cited. Any cited turn that isn't an integer is now a row problem for the correction prompt. Co-Authored-By: Claude Opus 5.5 --- .../judge_utils/judge_output_format.py | 16 +++++++++-- .../test_user_sim_checks_format.py | 28 +++++++++++++++++-- 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py b/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py index 974652d9a..c413a25c6 100644 --- a/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py +++ b/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py @@ -1364,6 +1364,11 @@ def _cited_turns(value: Any) -> list[int]: return out +def _is_turn_number(value: Any) -> bool: + """A whole number, as the schema asks: ``true`` and ``1.9`` are not turn 1.""" + return isinstance(value, int) and not isinstance(value, bool) + + def _graded_turns(criterion: dict) -> set[int] | None: value = criterion.get("graded_turns") return set(_cited_turns(value)) if isinstance(value, list) else None @@ -1411,13 +1416,18 @@ def _user_sim_row(item: dict, criterion: dict) -> tuple[dict | None, list[str]]: problems.append(f"{rid} may not be n/a: it always applies, so answer pass or fail") if not isinstance(item.get("reason"), str): problems.append(f"{rid}: reason must be text") - if not isinstance(item.get("turns"), list): + raw_turns = item.get("turns") + turns_ok = isinstance(raw_turns, list) and all(_is_turn_number(t) for t in raw_turns) + if not isinstance(raw_turns, list): problems.append(f"{rid}: turns must be a list of turn numbers") - turns = _cited_turns(item.get("turns")) + elif not turns_ok: + problems.append(f"{rid}: turns must be a list of turn numbers; " + f"{[t for t in raw_turns if not _is_turn_number(t)]} are not whole numbers") + turns = raw_turns if turns_ok else [] reason = item["reason"].strip() if isinstance(item.get("reason"), str) else "" graded = _graded_turns(criterion) if verdict == "fail": - if not any(graded is None or t in graded for t in turns): + if turns_ok and not any(graded is None or t in graded for t in turns): problems.append(f"{rid}: a fail must cite at least one graded turn; cited {turns}" + (f", graded {sorted(graded)}" if graded is not None else "")) said = _reason_says_no_failure(reason) diff --git a/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py b/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py index f4f96211e..66bf3bf18 100644 --- a/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py +++ b/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py @@ -144,7 +144,7 @@ def test_each_row_gives_its_reason_first_then_turns_verdict_could_change_grade_a def test_an_answer_is_read_into_rows_with_turns_kept_to_the_graded_ones(): criteria = _criteria("c1", "c2", graded=[2], family="G1") rows, discrepancy = diagnose_user_sim_checks_response( - _answer(_fail("c1", turns=[1, 2, "x"]), _row("c2", could_change_grade=True)), criteria) + _answer(_fail("c1", turns=[1, 2]), _row("c2", could_change_grade=True)), criteria) assert discrepancy is None fail, ok = rows assert (fail["verdict"], fail["turns"], fail["turns_outside_graded"]) == ("fail", [2], [1]) @@ -380,9 +380,16 @@ async def test_rubrics_verifier_names_the_unusable_row_when_the_judge_never_fixe (json.dumps({"results": [{**GOOD, "verdict": ["fail"]}]}), "c1: verdict ['fail'] is not one of"), (json.dumps({"results": [{**GOOD, "could_change_grade": "true"}]}), "c1: could_change_grade must be true or false"), (json.dumps({"results": [{**GOOD, "assistant_asked": None}]}), "c1: assistant_asked must be true or false"), - # An infinite turn number is dropped like any other that isn't one, so the fail cites none. + # A turn that isn't a whole number is sent back, never read as a turn: true and 1.9 are not turn 1. ('{"results": [{"id": "c1", "reason": "Leaked it.", "turns": [Infinity], "verdict": "fail", ' - '"could_change_grade": true, "assistant_asked": false}]}', "c1: a fail must cite at least one graded turn"), + '"could_change_grade": true, "assistant_asked": false}]}', + "c1: turns must be a list of turn numbers; [inf] are not whole numbers"), + (json.dumps({"results": [{**_fail("c1"), "turns": [True]}]}), + "c1: turns must be a list of turn numbers; [True] are not whole numbers"), + (json.dumps({"results": [{**_fail("c1"), "turns": [1.9]}]}), + "c1: turns must be a list of turn numbers; [1.9] are not whole numbers"), + (json.dumps({"results": [{**GOOD, "turns": [1, "2"]}]}), + "c1: turns must be a list of turn numbers; ['2'] are not whole numbers"), (json.dumps({"results": {"c1": GOOD}}), "`results` must be a list"), (json.dumps({"results": "pass"}), "`results` must be a list"), (json.dumps({"results": [GOOD, "pass"]}), "every result must be a JSON object"), @@ -398,6 +405,21 @@ def test_a_malformed_answer_is_a_discrepancy_for_the_correction_prompt_never_an_ SPEC.parse_response(text, _criteria("c1")) +@pytest.mark.asyncio +async def test_a_fail_citing_true_or_1_9_is_asked_again_and_the_corrected_answer_grades(monkeypatch): + """Turn 1 is graded, so ``true`` or ``1.9`` read as turn 1 would silently fail c1 on a turn never cited.""" + verifier = _verifier() + judge = AsyncMock(side_effect=[{"response": _answer({**_fail("c1"), "turns": [True]}, _row("c2"))}, + {"response": _answer({**_fail("c1"), "turns": [1.9]}, _row("c2"))}, + {"response": _answer(_row("c1"), _row("c2"))}]) + monkeypatch.setattr(verifier, "_prompt_llm_judge", judge) + rows, retries, discrepancies, _ = await _run_judge(verifier) + assert retries == 2 and [r["verdict"] for r in rows] == ["pass", "pass"] + assert [d["row_problems"] for d in discrepancies] == [ + ["c1: turns must be a list of turn numbers; [True] are not whole numbers"], + ["c1: turns must be a list of turn numbers; [1.9] are not whole numbers"]] + + @pytest.mark.asyncio async def test_rubrics_verifier_asks_again_when_a_result_s_id_is_not_text(monkeypatch): verifier = _verifier() From b0c65585912b661d3c0d5959608cfd9b824b9637 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 16 Nov 2026 14:51:56 -0800 Subject: [PATCH 5/7] fix(verifiers): a user_sim_checks reason's conclusion is what follows its last "but" "No clear failure was found at turn 1, but at turn 2 she leaked the password" supports a fail; it was sent back because the no-failure phrase matched anywhere in the last sentence. The conclusion is now the last sentence after its last contrast word. Co-Authored-By: Claude Opus 5.5 --- .../verifiers/judge_utils/judge_output_format.py | 15 ++++++++++++--- .../judge_utils/test_user_sim_checks_format.py | 8 ++++++-- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py b/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py index c413a25c6..151fb8338 100644 --- a/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py +++ b/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py @@ -1344,11 +1344,20 @@ def _user_sim_checks_output_format() -> dict: r"|\bno\s+clear\s+(?:\w+\s+)?failures?\b|\bnothing\s+fails\b", re.I) +# A contrast inside a sentence: what follows it is the sentence's conclusion. +_CONTRAST = re.compile(r",?\s+\b(?:but|however|yet|though|although|except)\b", re.I) + + def _reason_says_no_failure(reason: str) -> str | None: - """What in the reason's conclusion says no failure was found, or None. Only its last sentence is its conclusion: - "no clear failure at turn 1; at turn 2 she leaked it" is a real fail.""" + """What in the reason's conclusion says no failure was found, or None. The conclusion is the last sentence, and + within it what follows its last contrast ("but", "however"...): "no clear failure at turn 1; at turn 2 she leaked + it" and "no clear failure at turn 1, but at turn 2 she leaked it" are real fails; "turn 2 looked off, but no clear + failure was found" isn't.""" sentences = [x for x in re.split(r"[.;!?]+", reason) if x.strip()] - m = _NO_FAILURE_FOUND.search(sentences[-1]) if sentences else None + if not sentences: + return None + conclusion = _CONTRAST.split(sentences[-1])[-1] + m = _NO_FAILURE_FOUND.search(conclusion) return m.group(0) if m else None diff --git a/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py b/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py index 66bf3bf18..d7416adc8 100644 --- a/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py +++ b/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py @@ -211,7 +211,8 @@ def test_without_graded_turns_on_the_criterion_a_fail_still_has_to_cite_a_turn() # The endings of the two contradictory reasons seen in real runs, and a plain one. SAYS_NO_FAILURE = ("Hannah flagged it anyway which is fine. No clear failure found.", "This is not a delegation failure by Linda. No clear delegation failures found.", - "Every answer fits her rules; nothing fails.") + "Every answer fits her rules; nothing fails.", + "Turn 2 read a little stiff, but no clear failure was found.") def test_a_fail_whose_reason_concludes_nothing_failed_is_asked_again(): @@ -223,8 +224,11 @@ def test_a_fail_whose_reason_concludes_nothing_failed_is_asked_again(): # A pass may say it. rows, discrepancy = diagnose_user_sim_checks_response(_answer(_row(reason=reason)), _criteria("c1")) assert discrepancy is None and rows[0]["reason"] == reason - # Only the last sentence is the conclusion: a fail that clears one turn and fails another is kept. + # The conclusion is the last sentence, after its last "but": a fail that clears one turn and fails another + # is kept. for reason in ("No clear failure at turn 1; at turn 2 she revealed the private password.", + "No clear failure was found at turn 1, but at turn 2 she leaked the password.", + "No clear failure at turn 1, however at turn 3 she wrote the email to Ben herself.", "Turn 1 is fine, not a failure. At turn 2 she volunteered the shellfish allergy unasked; M3 says " "only when asked."): rows, discrepancy = diagnose_user_sim_checks_response(_answer(_fail(reason=reason)), _criteria("c1")) From cc53d9bebab338b9307fada230d81c5c0aa73f13 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 16 Nov 2026 14:58:13 -0800 Subject: [PATCH 6/7] fix(verifiers): 'although' doesn't start a user_sim_checks reason's conclusion "No clear failure was found, although turn 2 was awkward" concludes nothing failed; only what follows but / however / yet is read as the conclusion. Co-Authored-By: Claude Opus 5.5 --- .../verifiers/judge_utils/judge_output_format.py | 7 ++++--- .../verifiers/judge_utils/test_user_sim_checks_format.py | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py b/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py index 151fb8338..445a04311 100644 --- a/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py +++ b/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py @@ -1344,13 +1344,14 @@ def _user_sim_checks_output_format() -> dict: r"|\bno\s+clear\s+(?:\w+\s+)?failures?\b|\bnothing\s+fails\b", re.I) -# A contrast inside a sentence: what follows it is the sentence's conclusion. -_CONTRAST = re.compile(r",?\s+\b(?:but|however|yet|though|although|except)\b", re.I) +# A contrast that starts a sentence's conclusion: what follows "but" / "however" / "yet" is the conclusion. Not +# "although" or "though": what follows those is a concession, and the main clause before them is the conclusion. +_CONTRAST = re.compile(r",?\s+\b(?:but|however|yet)\b", re.I) def _reason_says_no_failure(reason: str) -> str | None: """What in the reason's conclusion says no failure was found, or None. The conclusion is the last sentence, and - within it what follows its last contrast ("but", "however"...): "no clear failure at turn 1; at turn 2 she leaked + within it what follows its last "but", "however" or "yet": "no clear failure at turn 1; at turn 2 she leaked it" and "no clear failure at turn 1, but at turn 2 she leaked it" are real fails; "turn 2 looked off, but no clear failure was found" isn't.""" sentences = [x for x in re.split(r"[.;!?]+", reason) if x.strip()] diff --git a/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py b/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py index d7416adc8..24c126ab5 100644 --- a/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py +++ b/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py @@ -212,7 +212,8 @@ def test_without_graded_turns_on_the_criterion_a_fail_still_has_to_cite_a_turn() SAYS_NO_FAILURE = ("Hannah flagged it anyway which is fine. No clear failure found.", "This is not a delegation failure by Linda. No clear delegation failures found.", "Every answer fits her rules; nothing fails.", - "Turn 2 read a little stiff, but no clear failure was found.") + "Turn 2 read a little stiff, but no clear failure was found.", + "No clear failure was found, although turn 2 was awkward.") def test_a_fail_whose_reason_concludes_nothing_failed_is_asked_again(): From 82a3f2df5a30f286ad6f797d902ec6f431cddcd2 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 16 Nov 2026 15:11:59 -0800 Subject: [PATCH 7/7] fix(verifiers): 'except' starts a user_sim_checks reason's conclusion again "No clear failures were found except at turn 2, where she leaked the password" is a fail. Co-Authored-By: Claude Opus 5.5 --- .../verifiers/judge_utils/judge_output_format.py | 9 +++++---- .../verifiers/judge_utils/test_user_sim_checks_format.py | 1 + 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py b/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py index 445a04311..2ebd932ca 100644 --- a/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py +++ b/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py @@ -1344,14 +1344,15 @@ def _user_sim_checks_output_format() -> dict: r"|\bno\s+clear\s+(?:\w+\s+)?failures?\b|\bnothing\s+fails\b", re.I) -# A contrast that starts a sentence's conclusion: what follows "but" / "however" / "yet" is the conclusion. Not -# "although" or "though": what follows those is a concession, and the main clause before them is the conclusion. -_CONTRAST = re.compile(r",?\s+\b(?:but|however|yet)\b", re.I) +# A contrast that starts a sentence's conclusion: what follows "but" / "however" / "yet" / "except" is the conclusion +# ("no clear failure except at turn 2, where she leaked it" is a fail). Not "although" or "though": what follows +# those is a concession, and the main clause before them is the conclusion. +_CONTRAST = re.compile(r",?\s+\b(?:but|however|yet|except)\b", re.I) def _reason_says_no_failure(reason: str) -> str | None: """What in the reason's conclusion says no failure was found, or None. The conclusion is the last sentence, and - within it what follows its last "but", "however" or "yet": "no clear failure at turn 1; at turn 2 she leaked + within it what follows its last "but", "however", "yet" or "except": "no clear failure at turn 1; at turn 2 she leaked it" and "no clear failure at turn 1, but at turn 2 she leaked it" are real fails; "turn 2 looked off, but no clear failure was found" isn't.""" sentences = [x for x in re.split(r"[.;!?]+", reason) if x.strip()] diff --git a/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py b/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py index 24c126ab5..b92f0030d 100644 --- a/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py +++ b/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py @@ -230,6 +230,7 @@ def test_a_fail_whose_reason_concludes_nothing_failed_is_asked_again(): for reason in ("No clear failure at turn 1; at turn 2 she revealed the private password.", "No clear failure was found at turn 1, but at turn 2 she leaked the password.", "No clear failure at turn 1, however at turn 3 she wrote the email to Ben herself.", + "No clear failures were found except at turn 2, where she leaked the password.", "Turn 1 is fine, not a failure. At turn 2 she volunteered the shellfish allergy unasked; M3 says " "only when asked."): rows, discrepancy = diagnose_user_sim_checks_response(_answer(_fail(reason=reason)), _criteria("c1"))