diff --git a/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py b/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py index 78eab7c19..2ebd932ca 100644 --- a/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py +++ b/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py @@ -5,6 +5,7 @@ import json import logging import math +import re from dataclasses import dataclass, field from enum import Enum from functools import partial @@ -558,6 +559,9 @@ class JudgeResponseDiscrepancy: unknown_ids: list[str] parse_error: str | None = None non_finite_score_ids: list[str] = field(default_factory=list) + # What's wrong inside rows whose ids are right (user_sim_checks: a fail citing no graded turn, a verdict its own + # reason contradicts, ...), one problem per entry naming its row. + row_problems: list[str] = field(default_factory=list) def to_dict(self) -> dict: out = { @@ -572,6 +576,8 @@ def to_dict(self) -> dict: out["parse_error"] = self.parse_error if self.non_finite_score_ids: out["non_finite_score_ids"] = list(self.non_finite_score_ids) + if self.row_problems: + out["row_problems"] = list(self.row_problems) return out def __str__(self) -> str: @@ -579,6 +585,8 @@ def __str__(self) -> str: return f"invalid JSON ({self.parse_error})" if self.non_finite_score_ids: return _non_finite_score_problem(self.non_finite_score_ids) + if self.row_problems: + return "; ".join(self.row_problems) return ( f"expected {self.expected_count} results but got {self.returned_count}" f" (missing={self.missing_ids}, duplicate={self.duplicate_ids}, unknown={self.unknown_ids})" @@ -1228,6 +1236,330 @@ def format_trajectory_mistakes_correction( return f"{eval_prompt.rstrip()}\n\n{correction_block}" +# --------------------------------------------------------------------------- +# user_sim_checks: grades the user sim (the AI playing the user), not the assistant. +# +# Each criterion is one question about the user sim's run: a check (guidance, oversharing, ...) or one of the task's +# own rubric items. Fields the prompt and the parser read on a criterion, beyond `id`: +# - `question`: what to answer, with what fails, what doesn't, when n/a and when could_change_grade is true; +# - `kind` (task items): "hard" or "general"; `na_when`: when it never came up, "never" when n/a isn't allowed; +# - `graded_turns`: the turns marked GRADED, which a fail must cite (any cited turn counts when it's absent); +# - `weight`: 0 records the row but never lets it lower the score (style). +# The run goes in the template's trajectory placeholder, so the framing and the run come first, the same for every +# call on a run, and the criteria last: a prompt cache can share everything before them. +# --------------------------------------------------------------------------- +USER_SIM_VERDICTS = ("pass", "fail", "n/a") +_USER_SIM_NA_NEVER = "never" + +_USER_SIM_FRAMING = """You check a user simulator ("the user sim"): an AI playing a real person who messages their \ +personal AI assistant. You grade the user sim, never the assistant. You are given: +1. THE USER SIM'S INSTRUCTIONS: exactly what it was told about who it is, what it knows, its decisions, its rules for \ +the assistant and what is private. Hold it to these lines only. +2. THE RUN: every check-in it got, word for word ("what it was sent": the time, its new and earlier emails, texts and \ +notifications, the lines it sent, and the assistant's new messages), each followed by its answer: "reply" (the \ +message reaches the assistant), "no_reply" (nothing is sent) or "done" (a sign-off). +3. THE CRITERIA: the questions to answer about it. + +How to read the run: +- What it was sent at a turn, and at every earlier turn, is what it knew; a fact it was never shown is not one it \ +forgot. +- Turns marked CONTEXT ONLY are fixed history: never fail the user sim for them, but they still count as what it said. +- Grade only the turns marked GRADED (each criterion's "graded_turns"). Cite turns by their number. +- The assistant's own words are the assistant's, even when they repeat an email. + +How to answer: +- Answer each criterion's "question" only, narrowly, by its own definition; other problems belong to other criteria. \ +A failure needs clear evidence in a graded answer. +- A criterion's "question" says what fails, what is not a failure, when to answer "n/a" and when "could_change_grade" \ +is true. +- A task rubric item gives its "kind". A "hard" item is an exact rule from the user sim's instructions: it fails on \ +any breach of the list or value it names, however small. A "general" item is a judgement about how this person would \ +behave: it fails only on a clear instance. +- An item's "na_when" is the situation in which it never came up: answer "n/a" for it then, and only then. \ +"na_when": "never" means it always applies: answer pass or fail. +- Reason first and decide last. Write the reason, then the turns it rests on, and only then the verdict, which \ +follows from the reason: a reason that finds no clear failure gives "pass" (or "n/a"), never "fail". Keep every \ +reason to at most 40 words: the evidence and the conclusion, not your working. +- "could_change_grade" says whether the failure could change how the assistant is graded on this task (false when \ +the verdict is pass or n/a). +- "assistant_asked" is true when the assistant asked for or invited the behaviour you graded (e.g. asked the user \ +to write a message or to give a fact it should look up); false otherwise. Give it on every result, false when \ +nothing was asked; it is recorded only and doesn't change the verdict. + +Return only this JSON object, with one result per criterion, each result's keys in this order: +{"results": [{"id": "the criterion's id", "reason": "at most 40 words", "turns": [turn numbers the reason rests on], \ +"verdict": "pass" | "fail" | "n/a", "could_change_grade": true | false, "assistant_asked": true | false}]}""" + +_USER_SIM_CRITERIA = """ + +# THE CRITERIA +$criteria_json + +Return only the JSON object.""" + +# `$trajectory_content` is the user sim's instructions and its run as the user sim judge renders them. +_USER_SIM_CHECKS_PROMPT_INLINE = Template(_USER_SIM_FRAMING + "\n\n$trajectory_content" + _USER_SIM_CRITERIA) +_USER_SIM_CHECKS_PROMPT_PATH = Template( + _USER_SIM_FRAMING + "\n\n# THE USER SIM'S INSTRUCTIONS AND THE RUN\nBoth are in the file at $trajectory_path: " + "read all of it before you answer." + _USER_SIM_CRITERIA) +# Without a trajectory the run comes in the sections the verifier adds after the template, so after the criteria. +_USER_SIM_CHECKS_PROMPT_NO_TRAJECTORY = Template( + _USER_SIM_FRAMING + _USER_SIM_CRITERIA + "\n\nThe user sim's instructions and the run follow.") + + +def _user_sim_checks_output_format() -> dict: + """Reason before verdict in every row: structured output is generated in schema order, so the judge decides after + reasoning. The parser reads the keys by name, in any order.""" + return { + "type": "json_schema", + "schema": { + "type": "object", + "properties": { + "results": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": {"type": "string"}, + "reason": {"type": "string"}, + "turns": {"type": "array", "items": {"type": "integer"}}, + "verdict": {"type": "string", "enum": list(USER_SIM_VERDICTS)}, + "could_change_grade": {"type": "boolean"}, + "assistant_asked": {"type": "boolean"}, + }, + "required": ["id", "reason", "turns", "verdict", "could_change_grade", "assistant_asked"], + "additionalProperties": False, + }, + }, + }, + "required": ["results"], + "additionalProperties": False, + }, + } + + +# A reason concluding nothing failed ("No clear delegation failures found"); a backstop to the reason-first prompt. +_NO_FAILURE_FOUND = re.compile( + r"\bno\s+(?:clear\s+)?(?:\w+\s+)?failures?\s+(?:was\s+|were\s+)?(?:found|detected|identified)\b" + r"|\bno\s+clear\s+(?:\w+\s+)?failures?\b|\bnothing\s+fails\b", re.I) + + +# A contrast that starts a sentence's conclusion: what follows "but" / "however" / "yet" / "except" is the conclusion +# ("no clear failure except at turn 2, where she leaked it" is a fail). Not "although" or "though": what follows +# those is a concession, and the main clause before them is the conclusion. +_CONTRAST = re.compile(r",?\s+\b(?:but|however|yet|except)\b", re.I) + + +def _reason_says_no_failure(reason: str) -> str | None: + """What in the reason's conclusion says no failure was found, or None. The conclusion is the last sentence, and + within it what follows its last "but", "however", "yet" or "except": "no clear failure at turn 1; at turn 2 she leaked + it" and "no clear failure at turn 1, but at turn 2 she leaked it" are real fails; "turn 2 looked off, but no clear + failure was found" isn't.""" + sentences = [x for x in re.split(r"[.;!?]+", reason) if x.strip()] + if not sentences: + return None + conclusion = _CONTRAST.split(sentences[-1])[-1] + m = _NO_FAILURE_FOUND.search(conclusion) + return m.group(0) if m else None + + +def _cited_turns(value: Any) -> list[int]: + if not isinstance(value, list): + return [] + out = [] + for v in value: + try: + out.append(int(v)) + except (TypeError, ValueError, OverflowError): # OverflowError: an infinite number + continue + return out + + +def _is_turn_number(value: Any) -> bool: + """A whole number, as the schema asks: ``true`` and ``1.9`` are not turn 1.""" + return isinstance(value, int) and not isinstance(value, bool) + + +def _graded_turns(criterion: dict) -> set[int] | None: + value = criterion.get("graded_turns") + return set(_cited_turns(value)) if isinstance(value, list) else None + + +def _user_sim_json(text: str) -> Any: + """The JSON in an answer, tolerating a markdown fence or text around the object; None when there is none.""" + s = str(text or "").strip().removeprefix("```json").removeprefix("```").removesuffix("```").strip() + try: + return json.loads(s) + except ValueError: + m = re.search(r"\{.*\}", s, re.S) + if not m: + return None + try: + return json.loads(m.group(0)) + except ValueError: + return None + + +def _never_gates(criterion: dict) -> bool: + try: + return float(criterion.get("weight", 1.0)) == 0.0 + except (TypeError, ValueError): + return False + + +def _user_sim_row(item: dict, criterion: dict) -> tuple[dict | None, list[str]]: + """One result checked against its criterion: ``(row, [])``, or ``(None, problems)`` naming the row. The row + lowers the score (score 0.0, result false) only on a fail with could_change_grade, and never at weight 0; + ``verdict`` and ``could_change_grade`` keep what the judge said either way. ``assistant_asked`` (the assistant + asked for or invited the behaviour) is required and recorded only: it never changes the score or a gate.""" + rid = item.get("id") + problems = [] + verdict = str(item.get("verdict", "")).strip().lower() + if verdict not in USER_SIM_VERDICTS: + problems.append(f"{rid}: verdict {item.get('verdict')!r} is not one of {list(USER_SIM_VERDICTS)}") + change = item.get("could_change_grade") + if not isinstance(change, bool): + problems.append(f"{rid}: could_change_grade must be true or false") + asked = item.get("assistant_asked") + if not isinstance(asked, bool): + problems.append(f"{rid}: assistant_asked must be true or false") + if verdict == "n/a" and str(criterion.get("na_when") or "").strip().lower() == _USER_SIM_NA_NEVER: + problems.append(f"{rid} may not be n/a: it always applies, so answer pass or fail") + if not isinstance(item.get("reason"), str): + problems.append(f"{rid}: reason must be text") + raw_turns = item.get("turns") + turns_ok = isinstance(raw_turns, list) and all(_is_turn_number(t) for t in raw_turns) + if not isinstance(raw_turns, list): + problems.append(f"{rid}: turns must be a list of turn numbers") + elif not turns_ok: + problems.append(f"{rid}: turns must be a list of turn numbers; " + f"{[t for t in raw_turns if not _is_turn_number(t)]} are not whole numbers") + turns = raw_turns if turns_ok else [] + reason = item["reason"].strip() if isinstance(item.get("reason"), str) else "" + graded = _graded_turns(criterion) + if verdict == "fail": + if turns_ok and not any(graded is None or t in graded for t in turns): + problems.append(f"{rid}: a fail must cite at least one graded turn; cited {turns}" + + (f", graded {sorted(graded)}" if graded is not None else "")) + said = _reason_says_no_failure(reason) + if said: + problems.append(f"{rid}: a fail whose reason says {said!r} contradicts itself; the verdict must follow " + "the reason") + if problems: + return None, problems + change = change and verdict == "fail" + lowers = change and not _never_gates(criterion) + row = {**criterion, "id": rid, "reason": reason, + "turns": [t for t in turns if graded is None or t in graded], + "verdict": verdict, "could_change_grade": change, "assistant_asked": asked, + "justification": reason, "score": 0.0 if lowers else 1.0, "result": not lowers} + outside = [t for t in turns if graded is not None and t not in graded] + if outside: + row["turns_outside_graded"] = outside + return row, [] + + +def diagnose_user_sim_checks_response( + response_text: str, criteria: list[dict], +) -> tuple[list[dict] | None, JudgeResponseDiscrepancy | None]: + """Parse a user_sim_checks judge response into rows, or a discrepancy for the correction prompt: no JSON + object, the wrong row set, or a row the user sim judge can't use: an id, reason or turns of the wrong type, a + verdict, could_change_grade or assistant_asked out of range, an n/a where ``na_when`` is never, a fail citing no + graded turn, or a fail whose reason concludes nothing failed.""" + parsed = _user_sim_json(response_text) + if parsed is None: + return None, _parse_failure_discrepancy(criteria, "the answer is not a JSON object") + results = _extract_rubric_binary_results(parsed) + if not isinstance(results, list): + shape = ("`results` must be a list" if results is not None + else f"unexpected response shape: {type(parsed).__name__}") + return None, _parse_failure_discrepancy(criteria, shape) + if not all(isinstance(r, dict) for r in results): + return None, _parse_failure_discrepancy(criteria, "every result must be a JSON object") + # The shared id check hashes each id, so one that isn't text (a list, an object) would raise instead of being + # asked again. + bad_ids = [f"result {i}: id must be a criterion id (text), not {r.get('id')!r}" + for i, r in enumerate(results, 1) if not isinstance(r.get("id"), str)] + if bad_ids: + return None, JudgeResponseDiscrepancy( + expected_count=len(criteria), returned_count=len(results), expected_ids=[c["id"] for c in criteria], + missing_ids=[], duplicate_ids=[], unknown_ids=[], row_problems=bad_ids) + discrepancy = _analyze_result_ids(results, criteria) + if discrepancy is not None: + return None, discrepancy + by_id = {c["id"]: c for c in criteria} + rows: list[dict] = [] + problems: list[str] = [] + for item in results: + row, its_problems = _user_sim_row(item, by_id[item["id"]]) + problems += its_problems + if row is not None: + rows.append(row) + if problems: + return None, JudgeResponseDiscrepancy( + expected_count=len(criteria), returned_count=len(results), expected_ids=[c["id"] for c in criteria], + missing_ids=[], duplicate_ids=[], unknown_ids=[], row_problems=problems) + return rows, None + + +def _parse_user_sim_checks_response(response_text: str, criteria: list[dict]) -> list[dict]: + """Rows, or ValueError: an unusable answer has no verdict, so it never degrades to fails that would gate.""" + rows, discrepancy = diagnose_user_sim_checks_response(response_text, criteria) + if discrepancy is not None: + raise ValueError(f"user_sim_checks answer not usable: {discrepancy}") + assert rows is not None + return rows + + +def format_user_sim_checks_correction( + *, + eval_prompt: str, + previous_response: str, + discrepancy: JudgeResponseDiscrepancy, + max_previous_response_chars: int = _CORRECTION_PROMPT_MAX_PREVIOUS_RESPONSE_CHARS, +) -> str: + """The user_sim_checks correction prompt: what was wrong, then the row shape again.""" + truncated = previous_response.strip() + if len(truncated) > max_previous_response_chars: + truncated = truncated[:max_previous_response_chars] + "\n... [truncated]" + if discrepancy.parse_error: + problems = [f"Your response could not be read: {discrepancy.parse_error}", + "Return only a JSON object with a `results` array — no preamble, markdown fences, or extra text."] + elif discrepancy.row_problems: + problems = list(discrepancy.row_problems) + else: + problems = [f"You returned {discrepancy.returned_count} result(s) but there are " + f"{discrepancy.expected_count} criteria."] + for ids, what in ((discrepancy.missing_ids, "Missing result(s) for criterion id(s)"), + (discrepancy.duplicate_ids, "Duplicate result(s) for criterion id(s)"), + (discrepancy.unknown_ids, "Unknown criterion id(s) in your results")): + if ids: + problems.append(f"{what}: {', '.join(ids)}") + correction_block = "\n".join([ + "## Correction Required", + "", + "Your previous response did not satisfy the output requirements:", + *[f"- {problem}" for problem in problems], + "", + f"Return JSON with a `results` array containing exactly {discrepancy.expected_count} objects — " + "one per criterion, no duplicates, no extras.", + f"Required criterion ids (in any order): {', '.join(discrepancy.expected_ids)}", + "", + "Each result must include, in this order:", + '- "id": the criterion id', + '- "reason": at most 40 words, written before the verdict', + '- "turns": the turn numbers the reason rests on; a fail cites at least one graded turn', + '- "verdict": "pass", "fail" or "n/a", following from the reason: a reason that finds no clear failure is ' + 'never "fail"', + '- "could_change_grade": true or false', + '- "assistant_asked": true when the assistant asked for or invited the behaviour you graded, else false', + "", + "## Your Previous Response", + truncated, + ]) + return f"{eval_prompt.rstrip()}\n\n{correction_block}" + + class JudgeOutputFormat(str, Enum): RUBRIC_BINARY = "rubric_binary" RUBRIC_PARTIAL = "rubric_partial" @@ -1238,6 +1570,9 @@ class JudgeOutputFormat(str, Enum): # object); the entry is stored exactly as rubric_evidence's and the verifier writes a second, # trajectory_mistakes-shaped entry from the findings. RUBRIC_EVIDENCE_WITH_MISTAKES = "rubric_evidence_with_mistakes" + # Grades the user sim (the AI playing the user), not the assistant: per criterion a reason, the turns it cites, a + # pass / fail / n/a verdict, could_change_grade and assistant_asked (recorded only); a row lowers the score only on a fail that could change the grade. + USER_SIM_CHECKS = "user_sim_checks" DiagnoseResponse = Callable[[str, list[dict]], tuple[list[dict] | None, JudgeResponseDiscrepancy | None]] @@ -1311,6 +1646,15 @@ class JudgeOutputFormatSpec: diagnose_response=diagnose_trajectory_mistakes_response, format_correction_prompt=format_trajectory_mistakes_correction, ), + JudgeOutputFormat.USER_SIM_CHECKS: JudgeOutputFormatSpec( + output_format=_user_sim_checks_output_format(), + prompt_template_path=_USER_SIM_CHECKS_PROMPT_PATH, + prompt_template_inline=_USER_SIM_CHECKS_PROMPT_INLINE, + prompt_template_no_trajectory=_USER_SIM_CHECKS_PROMPT_NO_TRAJECTORY, + parse_response=_parse_user_sim_checks_response, + diagnose_response=diagnose_user_sim_checks_response, + format_correction_prompt=format_user_sim_checks_correction, + ), } diff --git a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py index 24d600fe8..8315372d6 100644 --- a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py @@ -1630,7 +1630,7 @@ async def _run_judge_with_output_retries( ) discrepancy_log.append(discrepancy.to_dict()) if attempt == self.DEFAULT_MAX_RETRIES: - if discrepancy.non_finite_score_ids: + if discrepancy.non_finite_score_ids or discrepancy.row_problems: raise ValueError(f"Judge output invalid after {attempt} attempts: {discrepancy}") raise ValueError( f"Judge returned {discrepancy.returned_count} results but expected " diff --git a/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py b/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py new file mode 100644 index 000000000..b92f0030d --- /dev/null +++ b/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py @@ -0,0 +1,437 @@ +"""The user_sim_checks judge output format: it grades the user sim (the AI playing the user), not the assistant.""" + +import json +from unittest.mock import AsyncMock + +import pytest + +from agent_env.task_step.context import TaskStepContext +from agent_env.task_step.task_steps.verifiers.judge_utils.judge_output_format import ( + JUDGE_OUTPUT_FORMAT_REGISTRY, + JudgeOutputFormat, + diagnose_user_sim_checks_response, + format_user_sim_checks_correction, + get_judge_output_format_spec, +) +from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep +from agent_env.task_step.task_steps.verifiers.scoring import ScoreAggregator, aggregate_score + +SPEC = get_judge_output_format_spec(JudgeOutputFormat.USER_SIM_CHECKS) + +RUN = """# THE USER SIM'S INSTRUCTIONS + +You are Hannah Reyes. K1 Dad is allergic to shellfish. M3 Mention the allergy only when asked. + + +# THE RUN + +## Turn 1 · GRADED +### What it was sent + +It's Mon 9:12 AM. NEW FROM YOUR AI ASSISTANT: Booking Saturday dinner for 4 at Pescado. Any dietary needs? + +### Its answer +outcome: reply +message: "dad cant do shellfish" + +## Turn 2 · GRADED +### What it was sent + +It's Mon 9:40 AM. NEW FROM YOUR AI ASSISTANT: Booked for 7 PM. Want me to add a birthday note? + +### Its answer +outcome: reply +message: "sure. also dad cant do shellfish, its serious" +""" + + +def _criteria(*ids: str, graded=(1, 2), **extra) -> list[dict]: + return [{"id": cid, "question": f"question {cid}", "graded_turns": list(graded), **extra} for cid in ids] + + +def _row(cid="c1", *, verdict="pass", turns=(), could_change_grade=False, reason="Answered what was asked.", + assistant_asked=False, **extra) -> dict: + return {"id": cid, "reason": reason, "turns": list(turns), "verdict": verdict, + "could_change_grade": could_change_grade, "assistant_asked": assistant_asked, **extra} + + +def _answer(*rows: dict) -> str: + return json.dumps({"results": list(rows)}) + + +def _fail(cid="c1", *, turns=(2,), could_change_grade=True, + reason="At turn 2 she repeated the shellfish allergy unasked; M3 says only when asked.", **extra) -> dict: + return _row(cid, verdict="fail", turns=turns, could_change_grade=could_change_grade, reason=reason, **extra) + + +def _render(template, criteria_json="[]") -> str: + """A template filled the way rubrics_verifier fills it: every keyword it passes, used or not.""" + return template.substitute(agent_prompt="the assistant's prompt", agent_response="the assistant's answer", + criteria_json=criteria_json, trajectory_content=RUN, + trajectory_path="/tmp/trajectory.json", checks_instruction="[]") + + +# ---- templates ---------------------------------------------------------------------------------------------------- +def test_it_is_registered_and_found_by_name(): + assert JudgeOutputFormat("user_sim_checks") is JudgeOutputFormat.USER_SIM_CHECKS + assert get_judge_output_format_spec("user_sim_checks") is JUDGE_OUTPUT_FORMAT_REGISTRY[ + JudgeOutputFormat.USER_SIM_CHECKS] + assert SPEC.stored_format is None and SPEC.cites_evidence is False + + +def test_the_judge_is_told_it_grades_the_user_sim_never_the_assistant(): + for template in (SPEC.prompt_template_inline, SPEC.prompt_template_path, SPEC.prompt_template_no_trajectory): + prompt = _render(template) + assert prompt.startswith('You check a user simulator ("the user sim")') + assert "You grade the user sim, never the assistant." in prompt + # Nothing of the assistant's own prompt or answer is put in front of the judge as the thing to grade. + assert "the assistant's prompt" not in prompt and "the assistant's answer" not in prompt + + +def test_the_run_comes_first_and_the_criteria_last_so_calls_on_one_run_share_a_cached_head(): + one = _render(SPEC.prompt_template_inline, json.dumps(_criteria("c1"))) + other = _render(SPEC.prompt_template_inline, json.dumps(_criteria("c1", "c2"))) + assert one.index(RUN) < one.index("# THE CRITERIA") < one.index('"question": "question c1"') + assert one.endswith("Return only the JSON object.") + head = one[:one.index("# THE CRITERIA")] + assert other.startswith(head) and RUN in head + path = _render(SPEC.prompt_template_path) + assert path.index("/tmp/trajectory.json") < path.index("# THE CRITERIA") + + +def test_without_a_trajectory_the_judge_is_told_the_run_follows_the_criteria(): + prompt = _render(SPEC.prompt_template_no_trajectory) + assert RUN not in prompt + assert prompt.endswith("The user sim's instructions and the run follow.") + + +def test_the_judge_reasons_first_and_the_verdict_follows_the_reason(): + prompt = _render(SPEC.prompt_template_inline) + assert "Reason first and decide last." in prompt and "at most 40 words" in prompt + assert 'a reason that finds no clear failure gives "pass" (or "n/a"), never "fail"' in prompt + shape = prompt.split("Return only this JSON object")[1] + assert (shape.index('"id"') < shape.index('"reason"') < shape.index('"turns"') < shape.index('"verdict"') + < shape.index('"could_change_grade"') < shape.index('"assistant_asked"')) + + +def test_the_judge_is_told_how_to_read_the_run_and_the_criteria_fields(): + prompt = _render(SPEC.prompt_template_inline) + for told in ("Turns marked CONTEXT ONLY are fixed history", "Grade only the turns marked GRADED", + '"graded_turns"', "The assistant's own words are the assistant's", + 'A "hard" item is an exact rule', 'A "general" item is a judgement', + '"na_when": "never" means it always applies', + '"could_change_grade" says whether the failure could change how the assistant is graded', + '"assistant_asked" is true when the assistant asked for or invited the behaviour you graded (e.g. ' + 'asked the user to write a message or to give a fact it should look up); false otherwise', + "false when nothing was asked; it is recorded only and doesn't change the verdict"): + assert told in prompt, told + + +# ---- schema ------------------------------------------------------------------------------------------------------- +def test_each_row_gives_its_reason_first_then_turns_verdict_could_change_grade_and_assistant_asked(): + schema = SPEC.output_format + assert schema["type"] == "json_schema" and schema["schema"]["required"] == ["results"] + row = schema["schema"]["properties"]["results"]["items"] + order = ["id", "reason", "turns", "verdict", "could_change_grade", "assistant_asked"] + assert list(row["properties"]) == order == row["required"] + assert row["properties"]["verdict"]["enum"] == ["pass", "fail", "n/a"] + assert row["properties"]["turns"] == {"type": "array", "items": {"type": "integer"}} + assert row["properties"]["could_change_grade"] == row["properties"]["assistant_asked"] == {"type": "boolean"} + assert row["additionalProperties"] is False and schema["schema"]["additionalProperties"] is False + + +# ---- reading an answer -------------------------------------------------------------------------------------------- +def test_an_answer_is_read_into_rows_with_turns_kept_to_the_graded_ones(): + criteria = _criteria("c1", "c2", graded=[2], family="G1") + rows, discrepancy = diagnose_user_sim_checks_response( + _answer(_fail("c1", turns=[1, 2]), _row("c2", could_change_grade=True)), criteria) + assert discrepancy is None + fail, ok = rows + assert (fail["verdict"], fail["turns"], fail["turns_outside_graded"]) == ("fail", [2], [1]) + assert fail["could_change_grade"] is True and fail["justification"] == fail["reason"] + assert fail["family"] == "G1" and fail["question"] == "question c1" # the criterion rides on its row + # A pass can't change the grade, whatever the judge wrote. + assert ok["could_change_grade"] is False and "turns_outside_graded" not in ok + + +def test_the_parser_reads_the_keys_in_any_order_and_tolerates_a_fence_or_text_around_the_object(): + flipped = {"assistant_asked": False, "could_change_grade": True, "verdict": "FAIL ", "turns": [2], + "reason": "Leaked the allergy.", "id": "c1"} + for text in (json.dumps({"results": [flipped]}), "```json\n" + _answer(flipped) + "\n```", + "Here is my answer:\n" + _answer(flipped) + "\nDone."): + rows, discrepancy = diagnose_user_sim_checks_response(text, _criteria("c1")) + assert discrepancy is None and rows[0]["verdict"] == "fail", text + + +def test_an_answer_that_is_not_a_json_object_or_has_the_wrong_rows_is_asked_again(): + criteria = _criteria("c1", "c2") + for text, problem in (("dad cant do shellfish", "the answer is not a JSON object"), + (json.dumps({"verdict": "pass"}), "unexpected response shape: dict"), + (json.dumps({"results": ["pass", "pass"]}), "every result must be a JSON object")): + rows, discrepancy = diagnose_user_sim_checks_response(text, criteria) + assert rows is None and discrepancy.parse_error == problem + rows, discrepancy = diagnose_user_sim_checks_response(_answer(_row("c1"), _row("c1"), _row("c9")), criteria) + assert rows is None + assert (discrepancy.missing_ids, discrepancy.duplicate_ids, discrepancy.unknown_ids) == (["c2"], ["c1"], ["c9"]) + + +def test_a_verdict_could_change_grade_or_assistant_asked_out_of_range_is_asked_again(): + missing = _row("c4") + del missing["assistant_asked"] + rows, discrepancy = diagnose_user_sim_checks_response( + _answer(_row("c1", verdict="maybe"), _row("c2", could_change_grade="yes"), _row("c3", assistant_asked="no"), + missing), _criteria("c1", "c2", "c3", "c4")) + assert rows is None + assert discrepancy.row_problems == ["c1: verdict 'maybe' is not one of ['pass', 'fail', 'n/a']", + "c2: could_change_grade must be true or false", + "c3: assistant_asked must be true or false", + "c4: assistant_asked must be true or false"] + + +def test_a_fail_that_cites_no_graded_turn_is_asked_again_and_a_pass_needs_none(): + for turns, cited in (([99], r"\[99\]"), ([], r"\[\]")): + rows, discrepancy = diagnose_user_sim_checks_response(_answer(_fail(turns=turns)), _criteria("c1", graded=[1])) + assert rows is None + assert discrepancy.row_problems == [f"c1: a fail must cite at least one graded turn; cited {turns}, graded [1]"] + with pytest.raises(ValueError, match=f"a fail must cite at least one graded turn; cited {cited}"): + SPEC.parse_response(_answer(_fail(turns=turns)), _criteria("c1", graded=[1])) + for ok in (_row(turns=[99]), _row(verdict="n/a", turns=[99])): + rows, discrepancy = diagnose_user_sim_checks_response(_answer(ok), _criteria("c1", graded=[1])) + assert discrepancy is None and rows[0]["turns"] == [] and rows[0]["turns_outside_graded"] == [99] + + +def test_without_graded_turns_on_the_criterion_a_fail_still_has_to_cite_a_turn(): + criteria = [{"id": "c1", "question": "question c1"}] + rows, discrepancy = diagnose_user_sim_checks_response(_answer(_fail(turns=[])), criteria) + assert discrepancy.row_problems == ["c1: a fail must cite at least one graded turn; cited []"] + rows, discrepancy = diagnose_user_sim_checks_response(_answer(_fail(turns=[7])), criteria) + assert discrepancy is None and rows[0]["turns"] == [7] and "turns_outside_graded" not in rows[0] + + +# The endings of the two contradictory reasons seen in real runs, and a plain one. +SAYS_NO_FAILURE = ("Hannah flagged it anyway which is fine. No clear failure found.", + "This is not a delegation failure by Linda. No clear delegation failures found.", + "Every answer fits her rules; nothing fails.", + "Turn 2 read a little stiff, but no clear failure was found.", + "No clear failure was found, although turn 2 was awkward.") + + +def test_a_fail_whose_reason_concludes_nothing_failed_is_asked_again(): + for reason in SAYS_NO_FAILURE: + rows, discrepancy = diagnose_user_sim_checks_response(_answer(_fail(reason=reason)), _criteria("c1")) + assert rows is None and len(discrepancy.row_problems) == 1 + assert "c1: a fail whose reason says" in discrepancy.row_problems[0] + assert "contradicts itself; the verdict must follow the reason" in discrepancy.row_problems[0] + # A pass may say it. + rows, discrepancy = diagnose_user_sim_checks_response(_answer(_row(reason=reason)), _criteria("c1")) + assert discrepancy is None and rows[0]["reason"] == reason + # The conclusion is the last sentence, after its last "but": a fail that clears one turn and fails another + # is kept. + for reason in ("No clear failure at turn 1; at turn 2 she revealed the private password.", + "No clear failure was found at turn 1, but at turn 2 she leaked the password.", + "No clear failure at turn 1, however at turn 3 she wrote the email to Ben herself.", + "No clear failures were found except at turn 2, where she leaked the password.", + "Turn 1 is fine, not a failure. At turn 2 she volunteered the shellfish allergy unasked; M3 says " + "only when asked."): + rows, discrepancy = diagnose_user_sim_checks_response(_answer(_fail(reason=reason)), _criteria("c1")) + assert discrepancy is None and rows[0]["verdict"] == "fail" and rows[0]["could_change_grade"] is True + + +def test_an_n_a_where_na_when_is_never_is_asked_again_and_otherwise_stands(): + na = _row(verdict="n/a", reason="Rae never came up.") + rows, discrepancy = diagnose_user_sim_checks_response(_answer(na), _criteria("c1", na_when="never")) + assert discrepancy.row_problems == ["c1 may not be n/a: it always applies, so answer pass or fail"] + for criteria in (_criteria("c1"), _criteria("c1", na_when="the assistant never asks about Rae's order")): + rows, discrepancy = diagnose_user_sim_checks_response(_answer(na), criteria) + assert discrepancy is None and rows[0]["verdict"] == "n/a" and rows[0]["score"] == 1.0 + + +def test_every_unusable_row_is_named_in_one_go(): + rows, discrepancy = diagnose_user_sim_checks_response( + _answer(_fail("c1", turns=[]), _row("c2"), _fail("c3", reason=SAYS_NO_FAILURE[0])), _criteria("c1", "c2", "c3")) + assert rows is None and [p.split(":")[0] for p in discrepancy.row_problems] == ["c1", "c3"] + assert discrepancy.to_dict()["row_problems"] == discrepancy.row_problems + assert str(discrepancy) == "; ".join(discrepancy.row_problems) + + +def test_assistant_asked_is_recorded_on_every_row_and_never_changes_the_score_or_a_gate(): + criteria = _criteria("c1", "c2", "c3") + for asked in (True, False): + rows, discrepancy = diagnose_user_sim_checks_response( + _answer(_fail("c1", assistant_asked=asked), _fail("c2", could_change_grade=False, assistant_asked=asked), + _row("c3", assistant_asked=asked)), criteria) + assert discrepancy is None and [r["assistant_asked"] for r in rows] == [asked] * 3 + assert [(r["score"], r["result"]) for r in rows] == [(0.0, False), (1.0, True), (1.0, True)] + for aggregator in ScoreAggregator: + assert aggregate_score(rows, aggregator) == aggregate_score( + [{**r, "assistant_asked": not asked} for r in rows], aggregator) + + +def test_parse_response_never_turns_an_unusable_answer_into_fails(): + with pytest.raises(ValueError, match="user_sim_checks answer not usable: invalid JSON"): + SPEC.parse_response("not json", _criteria("c1")) + assert SPEC.parse_response(_answer(_row("c1")), _criteria("c1"))[0]["result"] is True + + +# ---- the correction prompt ---------------------------------------------------------------------------------------- +def test_the_correction_prompt_names_each_problem_and_asks_for_the_row_shape_again(): + previous = _answer(_fail("c1", turns=[99]), _row("c2")) + _, discrepancy = diagnose_user_sim_checks_response(previous, _criteria("c1", "c2", graded=[1])) + prompt = SPEC.format_correction_prompt(eval_prompt="EVAL", previous_response=previous, discrepancy=discrepancy) + assert prompt.startswith("EVAL\n\n## Correction Required") + assert "- c1: a fail must cite at least one graded turn; cited [99], graded [1]" in prompt + assert "exactly 2 objects" in prompt and "Required criterion ids (in any order): c1, c2" in prompt + for field in ('"id"', '"reason": at most 40 words', '"turns"', '"verdict"', '"could_change_grade"', + '"assistant_asked": true when the assistant asked for or invited the behaviour you graded'): + assert f"- {field}" in prompt + assert '"score"' not in prompt and '"justification"' not in prompt + assert prompt.endswith("## Your Previous Response\n" + previous) + + +def test_the_correction_prompt_for_unreadable_json_or_a_wrong_row_set(): + _, unreadable = diagnose_user_sim_checks_response("no json here", _criteria("c1")) + prompt = format_user_sim_checks_correction(eval_prompt="EVAL", previous_response="no json here", + discrepancy=unreadable) + assert "Your response could not be read: the answer is not a JSON object" in prompt + _, wrong = diagnose_user_sim_checks_response(_answer(_row("c1"), _row("c9")), _criteria("c1", "c2")) + prompt = format_user_sim_checks_correction(eval_prompt="EVAL", previous_response="x" * 9000, discrepancy=wrong) + assert "Missing result(s) for criterion id(s): c2" in prompt and "Unknown criterion id(s) in your results: c9" in prompt + assert prompt.endswith("... [truncated]") + + +# ---- the score ---------------------------------------------------------------------------------------------------- +def _score(*rows: dict, criteria: list[dict], aggregator=ScoreAggregator.WEIGHTED_AVERAGE) -> float: + got, discrepancy = diagnose_user_sim_checks_response(_answer(*rows), criteria) + assert discrepancy is None + return aggregate_score(got, aggregator) + + +def test_a_row_lowers_the_score_only_on_a_fail_that_could_change_the_grade(): + criteria = _criteria("c1", "c2", "c3") + assert _score(_row("c1"), _row("c2", verdict="n/a"), _row("c3"), criteria=criteria) == 1.0 + assert _score(_fail("c1", could_change_grade=False), _row("c2"), _row("c3"), criteria=criteria) == 1.0 + assert _score(_fail("c1"), _row("c2"), _row("c3"), criteria=criteria) == pytest.approx(2 / 3) + rows, _ = diagnose_user_sim_checks_response(_answer(_fail("c1", could_change_grade=False)), _criteria("c1")) + assert (rows[0]["verdict"], rows[0]["score"], rows[0]["result"]) == ("fail", 1.0, True) + + +def test_a_task_item_follows_the_same_rule_as_a_check(): + criteria = [*_criteria("guidance"), *_criteria("a1_call_rae", family="A1", kind="hard")] + assert _score(_row("guidance"), _fail("a1_call_rae", could_change_grade=False), criteria=criteria) == 1.0 + assert _score(_row("guidance"), _fail("a1_call_rae"), criteria=criteria) == 0.5 + + +def test_a_weight_0_row_is_recorded_but_never_gates(): + criteria = [*_criteria("guidance"), *_criteria("style", weight=0)] + rows, _ = diagnose_user_sim_checks_response(_answer(_row("guidance"), _fail("style")), criteria) + style = rows[1] + assert (style["verdict"], style["could_change_grade"], style["weight"]) == ("fail", True, 0) + assert (style["score"], style["result"]) == (1.0, True) + for aggregator in ScoreAggregator: + assert _score(_row("guidance"), _fail("style"), criteria=criteria, aggregator=aggregator) == 1.0 + assert _score(_fail("guidance"), _fail("style"), criteria=criteria) == 0.0 + + +# ---- in rubrics_verifier's retry loop ----------------------------------------------------------------------------- +def _verifier() -> RubricsVerifierTaskStep: + return RubricsVerifierTaskStep(id="judge-user-sim", version=1, criteria=_criteria("c1", "c2", graded=[1]), + prompt_id="prompt-1", use_trajectory=False, verifier_id="user-sim", + output_format=JudgeOutputFormat.USER_SIM_CHECKS) + + +async def _run_judge(verifier: RubricsVerifierTaskStep): + return await verifier._run_judge_with_output_retries( + eval_prompt="EVAL", criteria=verifier.criteria, context=TaskStepContext(), model="claude-sonnet-4-6", + judge_a2a_url=None, judge_agent_card={}, judge_agent=None, judge_effort=None, + judge_max_thinking_tokens=None, use_agent_judge=False) + + +@pytest.mark.asyncio +async def test_rubrics_verifier_asks_again_for_an_unusable_row_and_keeps_the_usable_answer(monkeypatch): + verifier = _verifier() + unusable = _answer(_fail("c1", turns=[99]), _row("c2")) + usable = _answer(_fail("c1", turns=[1]), _row("c2")) + judge = AsyncMock(side_effect=[{"response": unusable}, {"response": usable}]) + monkeypatch.setattr(verifier, "_prompt_llm_judge", judge) + rows, retries, discrepancies, _ = await _run_judge(verifier) + assert retries == 1 and [r["verdict"] for r in rows] == ["fail", "pass"] + assert discrepancies[0]["row_problems"] == ["c1: a fail must cite at least one graded turn; cited [99], graded [1]"] + correction = judge.await_args_list[1].args[0] + assert correction.startswith("EVAL\n\n## Correction Required") and unusable in correction + assert aggregate_score(rows, ScoreAggregator.WEIGHTED_AVERAGE) == 0.5 + + +@pytest.mark.asyncio +async def test_rubrics_verifier_names_the_unusable_row_when_the_judge_never_fixes_it(monkeypatch): + verifier = _verifier() + monkeypatch.setattr(verifier, "_prompt_llm_judge", AsyncMock( + return_value={"response": _answer(_fail("c1", turns=[1], reason=SAYS_NO_FAILURE[0]), _row("c2"))})) + with pytest.raises(ValueError, match=r"Judge output invalid after 3 attempts: c1: a fail whose reason says " + r"'No clear failure found' contradicts itself"): + await _run_judge(verifier) + + +# ---- malformed answers are asked again, never raised --------------------------------------------------------------- +GOOD = _row("c1") + + +@pytest.mark.parametrize("text, problem", [ + (json.dumps({"results": [{"id": []}]}), "result 1: id must be a criterion id (text), not []"), + (json.dumps({"results": [{**GOOD, "id": {"c": 1}}]}), "result 1: id must be a criterion id (text), not {'c': 1}"), + (json.dumps({"results": [{**GOOD, "id": None}]}), "result 1: id must be a criterion id (text), not None"), + (json.dumps({"results": [{**GOOD, "id": 1}]}), "result 1: id must be a criterion id (text), not 1"), + (json.dumps({"results": [{**GOOD, "reason": ["dad cant do shellfish"]}]}), "c1: reason must be text"), + (json.dumps({"results": [{k: v for k, v in GOOD.items() if k != "reason"}]}), "c1: reason must be text"), + (json.dumps({"results": [{**GOOD, "turns": "1"}]}), "c1: turns must be a list of turn numbers"), + (json.dumps({"results": [{**GOOD, "verdict": ["fail"]}]}), "c1: verdict ['fail'] is not one of"), + (json.dumps({"results": [{**GOOD, "could_change_grade": "true"}]}), "c1: could_change_grade must be true or false"), + (json.dumps({"results": [{**GOOD, "assistant_asked": None}]}), "c1: assistant_asked must be true or false"), + # A turn that isn't a whole number is sent back, never read as a turn: true and 1.9 are not turn 1. + ('{"results": [{"id": "c1", "reason": "Leaked it.", "turns": [Infinity], "verdict": "fail", ' + '"could_change_grade": true, "assistant_asked": false}]}', + "c1: turns must be a list of turn numbers; [inf] are not whole numbers"), + (json.dumps({"results": [{**_fail("c1"), "turns": [True]}]}), + "c1: turns must be a list of turn numbers; [True] are not whole numbers"), + (json.dumps({"results": [{**_fail("c1"), "turns": [1.9]}]}), + "c1: turns must be a list of turn numbers; [1.9] are not whole numbers"), + (json.dumps({"results": [{**GOOD, "turns": [1, "2"]}]}), + "c1: turns must be a list of turn numbers; ['2'] are not whole numbers"), + (json.dumps({"results": {"c1": GOOD}}), "`results` must be a list"), + (json.dumps({"results": "pass"}), "`results` must be a list"), + (json.dumps({"results": [GOOD, "pass"]}), "every result must be a JSON object"), + (json.dumps("pass"), "unexpected response shape: str"), + ("null", "the answer is not a JSON object"), +]) +def test_a_malformed_answer_is_a_discrepancy_for_the_correction_prompt_never_an_exception(text, problem): + rows, discrepancy = diagnose_user_sim_checks_response(text, _criteria("c1")) + assert rows is None and problem in str(discrepancy) + prompt = SPEC.format_correction_prompt(eval_prompt="EVAL", previous_response=text, discrepancy=discrepancy) + assert "## Correction Required" in prompt + with pytest.raises(ValueError, match="user_sim_checks answer not usable"): + SPEC.parse_response(text, _criteria("c1")) + + +@pytest.mark.asyncio +async def test_a_fail_citing_true_or_1_9_is_asked_again_and_the_corrected_answer_grades(monkeypatch): + """Turn 1 is graded, so ``true`` or ``1.9`` read as turn 1 would silently fail c1 on a turn never cited.""" + verifier = _verifier() + judge = AsyncMock(side_effect=[{"response": _answer({**_fail("c1"), "turns": [True]}, _row("c2"))}, + {"response": _answer({**_fail("c1"), "turns": [1.9]}, _row("c2"))}, + {"response": _answer(_row("c1"), _row("c2"))}]) + monkeypatch.setattr(verifier, "_prompt_llm_judge", judge) + rows, retries, discrepancies, _ = await _run_judge(verifier) + assert retries == 2 and [r["verdict"] for r in rows] == ["pass", "pass"] + assert [d["row_problems"] for d in discrepancies] == [ + ["c1: turns must be a list of turn numbers; [True] are not whole numbers"], + ["c1: turns must be a list of turn numbers; [1.9] are not whole numbers"]] + + +@pytest.mark.asyncio +async def test_rubrics_verifier_asks_again_when_a_result_s_id_is_not_text(monkeypatch): + verifier = _verifier() + judge = AsyncMock(side_effect=[{"response": json.dumps({"results": [{"id": []}]})}, + {"response": _answer(_row("c1"), _row("c2"))}]) + monkeypatch.setattr(verifier, "_prompt_llm_judge", judge) + rows, retries, discrepancies, _ = await _run_judge(verifier) + assert retries == 1 and [r["verdict"] for r in rows] == ["pass", "pass"] + assert discrepancies[0]["row_problems"] == ["result 1: id must be a criterion id (text), not []"]