diff --git a/src/agent_env/task/episode/rules.py b/src/agent_env/task/episode/rules.py index 661190ef4..f6cd5045a 100644 --- a/src/agent_env/task/episode/rules.py +++ b/src/agent_env/task/episode/rules.py @@ -158,7 +158,7 @@ def _rubrics_verifier(step: RubricsVerifierTaskStep, outside: Outside) -> list[s "browser_release_context", "browser_set_cookies", "browser_sign_in", "browser_set_local_storage", "build_mcp_cli", "combine_universe_verdicts", "configure_scheduled_checks", "deploy_env", "deploy_sandbox", "end_browser_recording", "end_computer_recording", "start_browser_recording", "start_computer_recording", "env_outcome_verifier", - "modify_env_tool_access", "reset_env", "review", + "modify_env_tool_access", "reset_env", "review", "user_sim_judge", "run_container_unit_tests_verifier", "run_docker_container", "set_env_clock_rate", "snapshot_env", "snapshot_computer_env", "sync_env_clock", "validation_gate_aggregator", "verify_env_card", "verify_env_core_protocol", "verify_mcp_tool_schema", diff --git a/src/agent_env/task/task.py b/src/agent_env/task/task.py index df78a58f4..d356c1bdc 100644 --- a/src/agent_env/task/task.py +++ b/src/agent_env/task/task.py @@ -135,7 +135,10 @@ def _sub(value): out: list["TaskStep"] = [] for step in steps: data = _sub(step.to_dict()) - out.append(attach_retry_config(type(step).from_dict(data), data)) + seeded = attach_retry_config(type(step).from_dict(data), data) + if (keep := getattr(seeded, "seeded_from", None)) is not None: # a step that keeps something as written + keep(step) + out.append(seeded) return out diff --git a/src/agent_env/task_step/registry.py b/src/agent_env/task_step/registry.py index 94df3ede2..c4b5c5b9d 100644 --- a/src/agent_env/task_step/registry.py +++ b/src/agent_env/task_step/registry.py @@ -73,6 +73,7 @@ def _builtin_registry() -> dict[str, type["TaskStep"]]: from agent_env.task_step.task_steps.configure_scheduled_checks import ConfigureScheduledChecksTaskStep from agent_env.task_step.task_steps.install_agent import InstallAgentTaskStep from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep + from agent_env.task_step.task_steps.verifiers.user_sim_judge import UserSimJudge from agent_env.task_step.task_steps.verifiers.always_on_agent_verifier import AlwaysOnAgentVerifier from agent_env.task_step.task_steps.verifiers.run_container_unit_tests_verifier import RunContainerUnitTestsVerifierTaskStep from agent_env.task_step.task_steps.run_docker_container import RunDockerContainerTaskStep @@ -148,6 +149,7 @@ def _builtin_registry() -> dict[str, type["TaskStep"]]: _get_type(ConfigureScheduledChecksTaskStep): ConfigureScheduledChecksTaskStep, _get_type(InstallAgentTaskStep): InstallAgentTaskStep, _get_type(RubricsVerifierTaskStep): RubricsVerifierTaskStep, + _get_type(UserSimJudge): UserSimJudge, _get_type(AlwaysOnAgentVerifier): AlwaysOnAgentVerifier, _get_type(RunContainerUnitTestsVerifierTaskStep): RunContainerUnitTestsVerifierTaskStep, _get_type(RunDockerContainerTaskStep): RunDockerContainerTaskStep, diff --git a/src/agent_env/task_step/task_steps/agent/deploy_user_sim_agent.py b/src/agent_env/task_step/task_steps/agent/deploy_user_sim_agent.py index 3224d2fe6..34a81e810 100644 --- a/src/agent_env/task_step/task_steps/agent/deploy_user_sim_agent.py +++ b/src/agent_env/task_step/task_steps/agent/deploy_user_sim_agent.py @@ -33,6 +33,8 @@ - ``prompt_set``: the system prompt and model are on the agent, so always_on sets neither; ``multi_day``: the done line the prompt was written with, which always_on refuses to run against a run of the other kind. - ``model``, ``model_source``, ``prompt_sha256``, ``user_version``, ``setup``: for the run's record. +- ``persona_sha256``: ``user_sim_common.persona_fingerprint`` of the persona and ``opening_prompt`` as written, which + user_sim_judge compares a rubric's ``rubric_persona_sha256`` with. - ``tagged_lines``: each tag the system prompt shows with its line as written there (``{"W1": "...", "D9": "..."}``), so an answer's ``used`` resolves against the lines this run's user sim saw, whatever later edits to the persona move. @@ -102,7 +104,8 @@ from agent_env.entity.scopes import FieldScope from agent_env.task_step.context import TaskStepContext from agent_env.task_step.task_steps.agent import user_sim_versions -from agent_env.task_step.task_steps.agent.user_sim_common import attached, opening_parts, opening_problems, opening_text +from agent_env.task_step.task_steps.agent.user_sim_common import (attached, opening_parts, opening_problems, opening_text, + persona_fingerprint, recorded_opening) from agent_env.task_step.task_steps.deploy_agent import DeployAgentTaskStep logger = logging.getLogger(__name__) @@ -188,6 +191,31 @@ def persona_problems(persona: Any) -> list[str]: return out +def person(key: str, persona: dict, opening: list[dict], seed: dict, multi_day: Optional[bool]) -> dict: + """The user as the built-in user sim's ``setup`` takes them: ``persona``'s fields, the ``opening`` parts with the + run's ```` placeholders filled in (as always_on fills them), and the done line ``multi_day`` picks. The step + writes its prompt from this, and user_sim_judge rebuilds that prompt from the step's record (``recorded_person``).""" + from agent_env.task_step.task_steps.prompt_agent import apply_seed + + name = persona["display_name"] + parts = apply_seed(opening, seed) + text, files = opening_text(parts), attached(parts) + return {"key": key, "name": name, "first": persona.get("first_name") or name.split()[0], + "description": persona.get("description"), "permissions": persona.get("permissions"), + "keep_private": persona.get("keep_private"), + "opening": {"text": text, **({"files": files} if files else {})} if text else None, + "done_ends_run": not multi_day} + + +def recorded_person(us: dict, seed: dict, multi_day: Optional[bool] = None) -> dict: + """``person`` from the step's ``metadata["user_sims"]`` record, with its ``opening_parts`` when it has them, so + ``setup`` gives the prompt the step wrote; ``multi_day`` is the record's, else the one given.""" + persona = {**{k: us.get(k) for k in PERSONA_KEYS}, + "display_name": us.get("display_name") or us.get("name") or "the user"} + return person(us.get("name"), persona, recorded_opening(us), seed, + us["multi_day"] if us.get("multi_day") is not None else multi_day) + + class DeployUserSimAgentTaskStep(DeployAgentTaskStep): """Deploys the agent that plays the task's user, its persona set at deploy, and records the user under ``agent_name`` in ``metadata["user_sims"]``.""" @@ -224,9 +252,16 @@ def __init__(self, id: str, version: Optional[int], *, persona: Optional[dict] = self.model = model self.multi_day = multi_day self.opening_prompt = opening_prompt + # Who the user is as the task writes it, the seed not filled in: what the judge's rubric_persona_sha256 is. + self.persona_sha256 = persona_fingerprint(persona, opening_prompt) _OWN = ("persona", "model", "multi_day", "opening_prompt") + def seeded_from(self, written: "DeployUserSimAgentTaskStep") -> None: + """This step is ``written`` with the run's seed filled in (``substitute_seed_in_steps``): keep the fingerprint + of the persona and opening as written.""" + self.persona_sha256 = written.persona_sha256 + def to_dict(self) -> dict: base = super().to_dict() base.pop("system_prompt", None) @@ -250,20 +285,8 @@ def _resolve_agent_ref(self, context: TaskStepContext) -> tuple[str, Optional[in def person(self, context: TaskStepContext) -> dict: """The user as the built-in user sim's ``setup`` takes them, as always_on hands them over.""" - name = self.persona["display_name"] - parts = self._opening(context) - text, files = opening_text(parts), attached(parts) - return {"key": self.agent_name, "name": name, "first": self.persona.get("first_name") or name.split()[0], - "description": self.persona.get("description"), "permissions": self.persona.get("permissions"), - "keep_private": self.persona.get("keep_private"), - "opening": {"text": text, **({"files": files} if files else {})} if text else None, - "done_ends_run": not self.multi_day} - - def _opening(self, context: TaskStepContext) -> list[dict]: - """``opening_prompt`` as parts with the run's ```` placeholders filled in, as always_on fills them.""" - from agent_env.task_step.task_steps.prompt_agent import apply_seed - - return apply_seed(opening_parts(self.opening_prompt), context.metadata.get("seed") or {}) + return person(self.agent_name, self.persona, opening_parts(self.opening_prompt), + context.metadata.get("seed") or {}, self.multi_day) def _setup(self, context: TaskStepContext) -> dict: return user_sim_versions.setup(user_sim_versions.VERSION, self.person(context)) @@ -319,6 +342,8 @@ async def _set_user(self, context: TaskStepContext, agent, setup: dict) -> TaskS **({"opening_parts": opening_parts(self.opening_prompt)} if isinstance(self.opening_prompt, list) else {}), "user_version": user_sim_versions.VERSION, "model": model, "model_source": source, "prompt_sha256": hashlib.sha256(prompt.encode()).hexdigest(), "prompt_set": True, + # Who the user is, as written: what user_sim_judge's rubric_persona_sha256 is compared with. + "persona_sha256": self.persona_sha256, # Each tag the prompt shows with its line, so an answer's ``used`` ("D9") names what this run saw. "tagged_lines": user_sim_versions.tagged_lines(self.person(context)), "multi_day": self.multi_day, diff --git a/src/agent_env/task_step/task_steps/agent/rate_user_satisfaction.py b/src/agent_env/task_step/task_steps/agent/rate_user_satisfaction.py index 16db2efc9..12ba5baa9 100644 --- a/src/agent_env/task_step/task_steps/agent/rate_user_satisfaction.py +++ b/src/agent_env/task_step/task_steps/agent/rate_user_satisfaction.py @@ -133,22 +133,7 @@ def from_dict(cls, data: dict) -> RateUserSatisfactionTaskStep: # ---- the load check --------------------------------------------------------------------------------------------- def _run_step(self, steps: list[TaskStep]) -> tuple[Optional[PromptAgentTaskStep], list[str]]: """The step this one rates, or why there's none.""" - where = f"{self.type} '{self.id}'" - if self.run_step_id is not None: - run = next((s for s in steps if s.id == self.run_step_id), None) - if run is None: - return None, [f"{where}: run_step_id '{self.run_step_id}' names no step of the task"] - if not isinstance(run, PromptAgentTaskStep): - return None, [f"{where}: run_step_id '{self.run_step_id}' is a {run.type} step; name the always_on or " - "prompt_agent step to rate"] - return run, [] - runs = [s for s in steps if isinstance(s, PromptAgentTaskStep)] - if not runs: - return None, [f"{where}: the task has no always_on or prompt_agent step to rate"] - if len(runs) > 1: - return None, [f"{where}: the task has several always_on and prompt_agent steps " - f"({', '.join(s.id for s in runs)}); set run_step_id to the one to rate"] - return runs[0], [] + return run_step_of(self, steps) def task_problems(self, steps: list[TaskStep]) -> list[str]: run, found = self._run_step(steps) @@ -159,8 +144,7 @@ def task_problems(self, steps: list[TaskStep]) -> list[str]: found.append(f"{where} comes before step '{run.id}', the run it rates; put it after") if self.depends_on is not None and run.id not in {d.task_step_id for d in self.depends_on}: found.append(f"{where}: its depends_on must name '{run.id}', the run it rates, so it runs once that's over") - user = _user_of(run) - if user is None or not any(isinstance(s, DeployUserSimAgentTaskStep) and s.agent_name == user for s in steps): + if not plays_built_in_user_sim(run, steps): found.append(f"{where}: the user of {run.type} step '{run.id}' isn't a user sim a deploy_user_sim_agent " "step deploys; only that user sim can rate the run") earlier = next((s for s in steps[:steps.index(self)] if isinstance(s, RateUserSatisfactionTaskStep) @@ -294,6 +278,34 @@ async def _trajectory(self, agent: DeployedAgent, session: _Session, task_id: st return None +# ---- the load check, shared with user_sim_judge ---------------------------------------------------------------------- +def run_step_of(step: TaskStep, steps: list[TaskStep], does: str = "rate") -> tuple[Optional[PromptAgentTaskStep], list[str]]: + """The always_on or prompt_agent step ``step`` works on (its ``run_step_id``'s, else the task's only one), or why + there's none. ``does`` names what ``step`` does to the run, in the messages.""" + where = f"{step.type} '{step.id}'" + if step.run_step_id is not None: + run = next((s for s in steps if s.id == step.run_step_id), None) + if run is None: + return None, [f"{where}: run_step_id '{step.run_step_id}' names no step of the task"] + if not isinstance(run, PromptAgentTaskStep): + return None, [f"{where}: run_step_id '{step.run_step_id}' is a {run.type} step; name the always_on or " + f"prompt_agent step to {does}"] + return run, [] + runs = [s for s in steps if isinstance(s, PromptAgentTaskStep)] + if not runs: + return None, [f"{where}: the task has no always_on or prompt_agent step to {does}"] + if len(runs) > 1: + return None, [f"{where}: the task has several always_on and prompt_agent steps " + f"({', '.join(s.id for s in runs)}); set run_step_id to the one to {does}"] + return runs[0], [] + + +def plays_built_in_user_sim(run: PromptAgentTaskStep, steps: list[TaskStep]) -> bool: + """Whether ``run``'s user is a user sim a deploy_user_sim_agent step of the task deploys.""" + user = _user_of(run) + return user is not None and any(isinstance(s, DeployUserSimAgentTaskStep) and s.agent_name == user for s in steps) + + # ---- the run, as the run step recorded it ---------------------------------------------------------------------------- def _user_of(run: PromptAgentTaskStep) -> Optional[str]: """The agent a run step talks to as its user: always_on's ``user_sim``; prompt_agent's ``user_agent_name``, unless diff --git a/src/agent_env/task_step/task_steps/agent/user_sim_common.py b/src/agent_env/task_step/task_steps/agent/user_sim_common.py index 7cd09efa2..28581486d 100644 --- a/src/agent_env/task_step/task_steps/agent/user_sim_common.py +++ b/src/agent_env/task_step/task_steps/agent/user_sim_common.py @@ -2,6 +2,8 @@ from __future__ import annotations import copy +import hashlib +import json from datetime import datetime, timezone from typing import Any, Optional @@ -20,6 +22,22 @@ def person_of(us: dict) -> dict: "permissions": us.get("permissions"), "keep_private": us.get("keep_private")} +def persona_fingerprint(persona: dict, opening_prompt: Any) -> str: + """Which user sim a run had: the hex sha256 of who the user is and what they first sent, as written (the seed not + filled in). deploy_user_sim_agent records it as ``persona_sha256``, and user_sim_judge grades a rubric written for + a fingerprint only on a run that has it. psi-benchmark-tasks's converter computes it too, so the recipe is fixed:: + + sha256(json.dumps({"description": persona.get("description"), "permissions": persona.get("permissions"), + "keep_private": persona.get("keep_private"), "opening_prompt": opening_prompt}, + sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")).hexdigest() + + ``opening_prompt`` is the step's as written: its text, or its list of parts.""" + body = json.dumps({"description": persona.get("description"), "permissions": persona.get("permissions"), + "keep_private": persona.get("keep_private"), "opening_prompt": opening_prompt}, + sort_keys=True, separators=(",", ":"), ensure_ascii=False) + return hashlib.sha256(body.encode("utf-8")).hexdigest() + + def task_zone(us: dict, clocks: list[dict], where: str) -> str: """The user's time zone, else the one sync_env_clock gave the world; refused when the two disagree. ``where`` names the step in the error.""" diff --git a/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py b/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py index dd23920b1..961fd6f7d 100644 --- a/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py +++ b/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py @@ -1662,6 +1662,10 @@ class JudgeOutputFormatSpec: parse_response=_parse_user_sim_checks_response, diagnose_response=diagnose_user_sim_checks_response, format_correction_prompt=format_user_sim_checks_correction, + # Each row stands alone (no per-call reasoning, checks or mistakes), so a judge may ask its rows over several + # calls; and a short pass reason is normal here, so bulk verdicts aren't flagged. + batchable=True, + checks_bulk_verdicts=False, ), } diff --git a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py new file mode 100644 index 000000000..7cf7d1453 --- /dev/null +++ b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py @@ -0,0 +1,720 @@ +"""The user_sim_judge step: after a run with the built-in user sim, grade the user sim (the AI playing the user), not +the assistant, and fail the task when the user sim slipped in a way that could change the assistant's grade, so the +run is done again rather than scored. A task writer adds it after the run's step. + +It is a rubrics_verifier (``RubricsVerifierTaskStep``) with its own criteria and its own context. What a task can set +on it, beyond every step's ``id``, ``type``, ``version``, ``depends_on`` and ``retry_config`` (``SETTINGS``): + +- ``run_step_id``, required: the always_on or prompt_agent step whose user sim it judges; +- ``rubric_items`` and ``rubric_persona_sha256`` (below); +- ``judge_model``: unset, rubrics_verifier's default, the config's judge model; ``judge_timeout_seconds``: unset, + rubrics_verifier's; +- ``fail_task_when_not_passing``: the gate (below), on unless set to false; +- ``verifier_id``: the key of its entry. + +Every step's ``fail_task_on_error`` must stay true, its default. The rest is fixed, and a task that sets it is refused +when it loads: the judge asks all seven checks, every check but style can fail a run (``checks`` and ``error_checks`` +are refused), and each rubrics_verifier setting the judge fixes (``PINNED``: the direct judge, ``use_trajectory`` off, +``output_format`` ``user_sim_checks``, no batch size, no criteria of its own ...) may only be given its fixed value. The +model call, its key and base, its retries and the calls' concurrency are rubrics_verifier's, and a run's +``user_overrides`` change them as they do for every judge (``judge_model``; ``use_agent_judge`` sends the same prompt to +an agent judge, without the system message and the cache marks). + +It judges either kind of run the built-in user sim plays (``deploy_user_sim_agent``), the one ``run_step_id`` names: + +- an ``always_on`` run: every ``user_sim_read`` in the timeline stored at ``metadata["always_on"][step]["object_url"]`` + (else the record's ``timeline_tail``, which keeps answers but not what the user sim was sent); +- a ``prompt_agent`` chat run: each short entry in ``metadata["user_sim_reads"][step]``, read whole from its + ``object_url`` (else the entry, which keeps the answer only). + +The user sim's instructions are rebuilt from its ``metadata["user_sims"]`` record by ``deploy_user_sim_agent``'s own +path (``recorded_person``, then ``user_sim_versions.setup``), its opening's files included, and checked against the +record's ``prompt_sha256``. ``used`` is never shown to the judge: it is the user sim's own pulse check, not evidence. + +What the judge reads: a system message saying it grades the user sim, the format's framing, then the run +(``judge_trajectory_content``: the rebuilt instructions and every check-in, ``user_sim_judge_core.render_run``), then +the criteria. Everything up to the end of the run is the same in every call on a run, so it is marked for the prompt +cache, and the run's first call goes alone so the rest read it from the cache. + +The criteria (``user_sim_judge_core.plan_run``): each of the seven checks (``user_sim_judge_core.CHECKS``) is one +criterion and one call, and a task's ``rubric_items`` ride in the call of the check that owns their family, at most +``MAX_ITEMS_PER_CALL`` to a call. Each row has a weight: 1 for every check but style +(``user_sim_judge_core.DEFAULT_ERROR_CHECKS``) and for an item that can make the run a user sim error (its +``can_error``, unset as its family's check), else 0. A weight-0 row is recorded and never gates. + +A rubric item is the shape of a criterion in a task's ``usersim.yaml`` (psi-benchmark-tasks), which that repo turns +into this step's settings; this step reads no file: + +- ``id`` and ``family`` (A1-A5, F1-F4, G1-G5), both required; ids are unique, and none is a check's name. +- ``kind``: ``hard`` (an exact rule from the user sim's instructions: the judge fails it on any breach), ``general`` + (a judgement: it fails only on a clear instance; the default) or ``auto`` (never sent to the model: ``metric`` + ``max_words`` or ``max_sentences`` is counted on each graded answer, and any answer over ``max`` fails it, naming + the turn; see ``user_sim_judge_core.count_sentences``). If none is over but the record kept only the start of one, + or dropped some answers (always_on's tail, when its stored timeline can't be read), the item is ``error``: those + lengths can't be counted. +- ``check``: the text the judge applies. ``text`` is its old name and still loads. +- ``line``: instead of ``check``, a tag of the user sim's instructions ("P1", "T1", "C2", "D3"): the judge applies + that line's text from the run's ``metadata["user_sims"][name]["tagged_lines"]``, and the item's row records ``line`` + and ``line_text``. A hard or general item has exactly one of ``check`` and ``line``; an auto item, counted rather + than judged, takes no ``line``. A tag names the same line only for the same persona, so a line item needs + ``rubric_persona_sha256``, and grades only on a run whose fingerprint matches it: on a run that recorded none, it + is ``error``. A tag the run doesn't have makes the item ``error`` ("the user sim's instructions have no line P1"). + Neither makes a call. +- ``name`` and ``short``: a title and one line for reviewers, on the item's row but never shown to the judge. +- ``na_when``: the situation in which it never came up, told to the judge. ``always`` makes it n/a with no call, + recording ``static_na_reason`` (required with it); ``never`` means it always applies, and an n/a is asked again. + An auto item is counted, not judged, so it takes only ``always``; any other ``na_when`` is refused when the task loads. +- ``can_error``: whether its fail can make the run a user sim error (weight 1, or 0); unset, as the check that owns + its family. + +A task item never restates a persona line. The persona is the one place that says what the user sim knows and must +not say, and the judge reads all of it with every check. Task items are for counted limits (``auto``) and judgment +calls specific to the task, or they point at a persona line with ``line``. + +For example:: + + rubric_persona_sha256: 3f1c... # the run's persona_sha256, all 64 hex digits + rubric_items: + - {id: a1_call_rae, family: A1, kind: hard, name: "Rae: yes if it fits", + check: "If the assistant asks about Rae's order, Tess says yes only if it fits, gives both floors ...", + na_when: "The assistant didn't raise Rae's order."} + - {id: f2_her_length, family: F2, kind: auto, check: "Every reply is at most 25 words.", metric: max_words, max: 25} + - {id: g1_engagement, family: G1, kind: hard, line: P1} + +``rubric_persona_sha256`` is the fingerprint of the user sim the items were written for: +``user_sim_common.persona_fingerprint`` of its persona and ``opening_prompt``, which deploy_user_sim_agent records as +the run's ``persona_sha256``. When it's set and differs from the run's, the task items aren't graded and the entry's +``rubric_persona`` (``status``, ``rubric``, ``run``, ``note``) says so; the general checks still run. A run that +recorded no ``persona_sha256`` is ``unknown``, and the items are graded. Edits to the task that leave the persona and +the opening as they were keep the items graded. + +What it writes, all in one place, ``metadata["verifications"][verifier_id]``: + +- ``format`` (``user_sim_checks``), ``results`` and ``score``, as rubrics_verifier writes them. Each row: ``id`` (a + check's name or an item's id); what a reviewer reads (a check's ``title`` and ``families``; an item's ``family``, + ``kind``, ``check`` or ``line`` and ``line_text``, ``name``, ``short``, ``na_when``, ``asked_with``: the call it + rode in, or ``computed``, ``static_na``, ``no_line``); its ``weight``; then its ``verdict`` (pass, fail, n/a, or + error when undecided), ``reason``, ``turns``, ``could_change_grade``, ``assistant_asked`` (whether the assistant + asked for or invited what was graded: recorded only, never part of the gate; None on a row no model graded), + ``score`` and ``result``. A row lowers the score only on a fail that could change the grade at a weight other + than 0, so ``score`` is 0 when ``user_sim_error`` is yes, else 1. ``can_be_reward`` is False: the score checks the + run, so it is never the task's reward. +- ``user_sim_error``: ``value`` yes / no / unknown, ``because`` and ``unknown`` (the row ids), ``fails_task``, and + ``not_failing`` (why a yes didn't fail the task). ``flag`` when ``value`` is unknown: "user sim judge undecided", or + "the judge couldn't decide any check — check its setup" when every check is undecided. +- ``run_step_id``, ``run_kind``, ``user_sim``, ``judge_model``; ``instructions`` (``source``, ``prompt_sha256``, + ``recorded_prompt_sha256``, ``matches``); ``rubric_persona`` (when ``rubric_persona_sha256`` is set); ``notes`` + (what the judge was told about gaps in the record); ``graded_turns``, ``turns``; ``calls``, ``usage``, ``cost``; and + rubrics_verifier's ``judge_output_retries`` / ``judge_output_discrepancies`` / ``judge_batches``. +- ``skipped`` instead, with ``score`` None and no rows, for a run that isn't judged: one that ended in an error (the + always_on record's or the prompt response's ``error_type``: every verifier skips its prompt anyway), a chat run + whose user sim never read, and a run with no graded turn. + +The gate, ``fail_task_when_not_passing`` (on by default): the step raises ``VerifierNotPassingError``, naming the failing rows, when ``user_sim_error`` is +yes and the rebuilt instructions match the recorded ``prompt_sha256``. The task then ends FAILED with no score, and +the run is done again. Whoever caused the slip, the run is spoiled for grading the assistant, so ``assistant_asked`` +never changes it. Off, the verdicts are recorded only. When the gate is on, every step that can be the task's reward +(one with a ``verifier_id`` and ``can_be_reward``) must depend on this step, directly or through others, or the task is +refused when it loads. + +A judge that can't reach its model fails the run at once, with ``JudgeSetupError`` ("the user sim judge can't reach +its model: ..."): agent-env's ``ConfigError`` (no key, missing config) and LiteLLM's ``AuthenticationError``, +``PermissionDeniedError``, ``NotFoundError`` (an unknown model), ``BadRequestError`` and its kinds, +``UnprocessableEntityError`` and ``BudgetExceededError`` (``_setup_error``). Every other failure is taken as flaky +(``Timeout``, ``RateLimitError``, ``ServiceUnavailableError``, ``InternalServerError``, ``BadGatewayError``, +``APIConnectionError``, an agent judge's ``TimeoutError``, ``JudgeTimeoutError`` or ``AgentUnreachableError``, an answer +still unusable after its retries), and unknown never fails the task: a run is failed +only on a slip the judge found, so a flaky judge never throws away a good run. A judge answer the format can't use is +asked again with the format's correction, up to 3 more times (``DEFAULT_MAX_RETRIES``, 4, counts the first ask; +rubrics_verifier tries a failed model call as many times). A call still without a usable answer leaves its rows +``error``, ``user_sim_error`` unknown and the entry flagged; the step doesn't raise, and nothing the assistant is +scored on changes. + +When the task loads it is refused without ``run_step_id``, when that step isn't an always_on or prompt_agent step +(``rate_user_satisfaction.run_step_of``), when the run's user isn't a user sim a deploy_user_sim_agent step deploys, +when this step doesn't come after the run (its ``depends_on``, when set, must name it), when a reward step doesn't +depend on it (above), when ``fail_task_on_error`` is false, when it sets ``checks``, ``error_checks`` or a fixed +rubrics_verifier setting to another value, and when it still sets ``mark_user_sim_error``, which +``fail_task_when_not_passing`` replaces. +""" +from __future__ import annotations + +import asyncio +import copy +import hashlib +import json +import logging +import re +from dataclasses import dataclass, field +from typing import Any, ClassVar, Optional + +from agent_env.task_step.context import PromptResponse, StepContextError, TaskStepContext, VerifierNotPassingError +from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_steps.verifiers import user_sim_judge_core as core +from agent_env.task_step.task_steps.verifiers.judge_utils.judge_output_format import JudgeOutputFormat +from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep + +logger = logging.getLogger(__name__) + +# Not imported: always_on and prompt_agent import the verifiers' peers. Each is the key its module names. +ALWAYS_ON_KEY = "always_on" # always_on.ALWAYS_ON_KEY +READS_KEY = "user_sim_reads" # prompt_agent's _BuiltInUserSim +USER_SIMS_KEY = "user_sims" # deploy_user_sim_agent.USER_SIMS_KEY +ALWAYS_ON, CHAT = "always_on", "prompt_agent" +SYSTEM = ("You grade a user simulator: an AI playing a real person who messages their personal AI assistant. You " + "grade the user sim, never the assistant.") +COST_DECIMALS = 6 # the entry keeps the judge's cost to a millionth of a dollar +FLAG_UNDECIDED = "user sim judge undecided" +FLAG_NONE_DECIDED = "the judge couldn't decide any check — check its setup" +# The judge's own settings: what a task may set on it, beyond every step's (id, type, version, depends_on ...). +SETTINGS = ("run_step_id", "rubric_items", "rubric_persona_sha256", "judge_model", "judge_timeout_seconds", + "fail_task_when_not_passing", "verifier_id") +# rubrics_verifier settings the judge fixes, with the value it fixes them at: a task may give only that value. +PINNED: dict[str, Any] = { + "use_trajectory": False, "use_agent_judge": False, "output_format": "user_sim_checks", "score_aggregator": "all_pass", + **dict.fromkeys(("criteria", "prompt_id", "default_model", "default_model_api_base", "agent_name", + "trajectory_filter", "max_thinking_tokens", "effort", "judge_a2a_agent_id", "judge_sandbox_type", + "grading_policy_prompt", "context_facts_for_judge", "grade_up_to_turn", "run_context_for_judge", + "judge_batch_size", "judge_batch_concurrency", "judge_timeout_retries", "judge_max_turns"))} +NOT_FAILING_OFF = "fail_task_when_not_passing is off: verdicts are recorded only" +NOT_FAILING_UNCONFIRMED = ("the judge's copy of the user sim's instructions isn't confirmed to be the one it ran with, " + "so it doesn't fail the task") +FLAG_ASSISTANT = "this answer looks like the assistant's (very long, or markdown)" +FLAG_LAST = "after the run's last wake: this answer never reached the assistant" +FLAG_HELD = "its reply was still waiting to be delivered when the run ended: it never reached the assistant" +# always_on.ASSISTANT_LIKE_MESSAGE_CHARS: a chat run's reads carry no flag of their own, so it's computed the same way. +ASSISTANT_LIKE_MESSAGE_CHARS = 1200 +SCHEMA_ERRORS_SHOWN_CHARS = 300 # of a malformed answer's schema errors, in the judge's turn note +USER_SIM_ERROR_SHOWN_CHARS = 200 # of a user sim agent error, in the judge's turn note +ERROR_SHOWN_CHARS = 300 # of why a call got no usable answer, on its rows +# Why a line item isn't graded on a run that recorded no persona fingerprint: its tag may name another line now. +LINE_UNCHECKED = ("the run recorded no persona fingerprint, so its tags can't be checked against the persona the item " + "was written for") + + +@dataclass +class _ThisRun: + """One run's judging: what it asks and what its calls cost. Held by the run's own copy of the step, never by the + step a task shares across runs.""" + plan: core.Plan + text: str # the run as the judge reads it + first: Optional[asyncio.Event] = None # set once the run's first call is over + first_error: Optional[BaseException] = None # what the first call raised, if it did + calls: int = 0 + usage: list[Any] = field(default_factory=list) + cost: float = 0.0 + + +class UserSimJudge(RubricsVerifierTaskStep): + type: ClassVar[str] = "user_sim_judge" + entity_refs = () + context_paths = () + field_scopes = TaskStep.field_scopes + can_be_reward: ClassVar[bool] = False + # Each call's answer is asked for up to this many times: the first ask and 3 corrections. rubrics_verifier tries a + # failed model call the same number of times. + DEFAULT_MAX_RETRIES: ClassVar[int] = 4 + + def __init__( + self, + id: str, + version: Optional[int], + run_step_id: Optional[str] = None, + judge_model: Optional[str] = None, + rubric_items: Optional[list[dict]] = None, + rubric_persona_sha256: Optional[str] = None, + fail_task_when_not_passing: bool = True, + verifier_id: Optional[str] = None, + judge_timeout_seconds: Optional[int] = None, + depends_on: Optional[list[TaskStepDependency]] = None, + fail_task_on_error: bool = True, + ): + """``run_step_id`` is required (it is a keyword with a default only so the refusal can say what's missing).""" + if not isinstance(run_step_id, str) or not run_step_id.strip(): + raise ValueError(f"user_sim_judge '{id}': run_step_id is required: name the always_on or prompt_agent " + "step whose user sim it judges") + if fail_task_on_error is not True: + raise ValueError(f"user_sim_judge '{id}': fail_task_on_error can't be false: a judge that fails has to " + "fail the task, or the steps after it would score a run nobody checked. Leave it out " + "(true is the default); set fail_task_when_not_passing to false to record the verdicts " + "only.") + super().__init__( + id, version, criteria=[], prompt_id=None, default_model=judge_model, verifier_id=verifier_id, + use_agent_judge=False, use_trajectory=False, output_format=JudgeOutputFormat.USER_SIM_CHECKS, + judge_timeout_seconds=judge_timeout_seconds, depends_on=depends_on, fail_task_on_error=fail_task_on_error, + fail_task_when_not_passing=fail_task_when_not_passing is True) + self.fail_task_when_not_passing = fail_task_when_not_passing # as given, so config_problems sees a non-bool + self.run_step_id = run_step_id + self.judge_model = judge_model # None: rubrics_verifier's default, the config's judge model + self.rubric_items = list(rubric_items or []) + self.rubric_persona_sha256 = rubric_persona_sha256 + self._run: Optional[_ThisRun] = None + + # ---- settings ---------------------------------------------------------------------------------------------- + def config_problems(self) -> list[str]: + return _problems(self.judge_model, self.rubric_items, self.fail_task_when_not_passing, + self.rubric_persona_sha256) + + def task_problems(self, steps: list[TaskStep]) -> list[str]: + """The run it judges must be an always_on or prompt_agent step's, played by a built-in user sim, and come + before it; with the gate on, every step that can be the task's reward must depend on it.""" + from agent_env.task.task import _ancestor_ids, _dependency_ids # here: the task module imports the steps + from agent_env.task_step.task_steps.agent.rate_user_satisfaction import plays_built_in_user_sim, run_step_of + + run, found = run_step_of(self, steps, does="judge") + if run is None: + return found + where = f"{self.type} '{self.id}'" + if steps.index(run) > steps.index(self): + found.append(f"{where} comes before step '{run.id}', the run it judges; put it after") + if self.depends_on is not None and run.id not in {d.task_step_id for d in self.depends_on}: + found.append(f"{where}: its depends_on must name '{run.id}', the run it judges, so it runs once that's over") + if not plays_built_in_user_sim(run, steps): + found.append(f"{where}: the user of {run.type} step '{run.id}' isn't a user sim a deploy_user_sim_agent " + "step deploys; the judge grades only the built-in user sim") + if self.fail_task_when_not_passing is True: + deps = _dependency_ids(steps) + for step in steps: + if (step is not self and getattr(step, "verifier_id", None) and getattr(step, "can_be_reward", True) + and self.id not in _ancestor_ids(step.id, deps)): + found.append(f"step '{step.id}' ({step.type}) can be the task's reward but doesn't depend on " + f"{where}, so a run the user sim spoiled could still be scored; make it depend on " + f"'{self.id}', or set fail_task_when_not_passing to false to record the verdicts only") + return found + + def to_dict(self) -> dict: + base = TaskStep.to_dict(self) + base.update(run_step_id=self.run_step_id, judge_model=self.judge_model, rubric_items=self.rubric_items, + rubric_persona_sha256=self.rubric_persona_sha256, + fail_task_when_not_passing=self.fail_task_when_not_passing, verifier_id=self.verifier_id, + judge_timeout_seconds=self.judge_timeout_seconds) + return base + + @classmethod + def from_dict(cls, data: dict) -> "UserSimJudge": + where = f"user_sim_judge '{data.get('id')}'" + if "mark_user_sim_error" in data: + raise ValueError( + f"{where}: mark_user_sim_error is gone; fail_task_when_not_passing (on by default) takes its place: " + "a run whose user sim slipped in a way that could change the grade fails the task, so it is run " + "again. Remove mark_user_sim_error, and set fail_task_when_not_passing to false to record the " + "verdicts only.") + if gone := [k for k in ("checks", "error_checks") if k in data]: + raise ValueError( + f"{where}: {' and '.join(gone)} can't be set: the judge always asks all seven checks, and every " + "check but style can fail a run. Remove them; a task item's can_error says whether it can.") + if pinned := [k for k, v in PINNED.items() if k in data and data[k] != v]: + raise ValueError( + f"{where}: {', '.join(pinned)} can't be set on the user sim judge, which fixes " + f"{'them' if len(pinned) > 1 else 'it'} ({', '.join(f'{k}={PINNED[k]!r}' for k in pinned)}). " + "Remove " + ("them" if len(pinned) > 1 else "it") + "; its settings are " + ", ".join(SETTINGS) + + ".") + return cls(**cls._base_from_dict(data), run_step_id=data.get("run_step_id"), + judge_model=data.get("judge_model"), rubric_items=data.get("rubric_items"), + rubric_persona_sha256=data.get("rubric_persona_sha256"), + fail_task_when_not_passing=data.get("fail_task_when_not_passing", True), + verifier_id=data.get("verifier_id"), judge_timeout_seconds=data.get("judge_timeout_seconds")) + + # ---- the run ----------------------------------------------------------------------------------------------- + async def execute(self, context: TaskStepContext) -> TaskStepContext: + context = await self._grade(context) + error = context.metadata["verifications"][self.verifier_id]["user_sim_error"] + if error.get("fails_task"): + raise VerifierNotPassingError( + f"user_sim_judge '{self.id}': the user sim slipped in a way that could change the assistant's grade " + f"({', '.join(error['because'])}), so the run doesn't count; run it again") + return context + + async def _grade(self, context: TaskStepContext) -> TaskStepContext: + problems = self.config_problems() + if problems: + raise ValueError(f"user_sim_judge '{self.id}': " + "; ".join(problems)) + items = [core.RubricItem.from_dict(i) for i in self.rubric_items] + kind, step_id = self._find_run(context) + response = _prompt_response(context, step_id) + model = (context.metadata.get("user_overrides") or {}).get("judge_model") or self.default_model + entry: dict[str, Any] = {"run_step_id": step_id, "run_kind": kind, "judge_model": model} + record = (context.metadata.get(ALWAYS_ON_KEY) or {}).get(step_id) if kind == ALWAYS_ON else None + error_type = (record or {}).get("error_type") or (response.error_type if response else None) + skipped = None + if error_type: + skipped = f"the run ended with error_type={error_type}" + elif kind == CHAT and not (context.metadata.get(READS_KEY) or {}).get(step_id): + skipped = "the user sim never read: the run has no user_sim_reads" + else: + run, about = await asyncio.to_thread(_read_run, kind, step_id, record, context) + entry.update(user_sim=about["user_sim"], instructions=about["instructions"]) + if self.rubric_persona_sha256 is not None and items: + us = (context.metadata.get(USER_SIMS_KEY) or {}).get(about["user_sim"]) or {} + entry["rubric_persona"] = _rubric_persona(self.rubric_persona_sha256, us.get("persona_sha256")) + if entry["rubric_persona"]["status"] == "mismatch": + items = [] + graded = [t.n for t in run.turns if t.graded] + entry.update(graded_turns=graded, turns=len(run.turns), notes=list(run.notes)) + if not graded: + skipped = "no graded turns" + elif response is None: + raise StepContextError(f"user_sim_judge '{self.id}': step '{step_id}' left no prompt response") + if skipped: + logger.warning(f"user_sim_judge '{self.id}': skipped, {skipped}") + context.metadata.setdefault("verifications", {})[self.verifier_id] = { + **self._entry_format_fields(), "results": [], "score": None, **entry, "skipped": skipped, + "user_sim_error": {**core.NOT_JUDGED, "fails_task": False}} + return context + + unusable = LINE_UNCHECKED if (entry.get("rubric_persona") or {}).get("status") == "unknown" else "" + plan = core.plan_run(run, checks=list(core.CHECK_NAMES), items=items, + error_checks=list(core.DEFAULT_ERROR_CHECKS), lines_unusable=unusable) + judge = copy.copy(self) # this run's own copy: the step is shared by every run of the task + judge.prompt_id, judge._run = response.prompt_id, _ThisRun(plan=plan, text=core.run_text(run)) + if plan.criteria: + await RubricsVerifierTaskStep._grade(judge, context) + else: # nothing for a model: only rows settled without one + judge._write_entries(context, [], {}, unjudged=plan.unjudged) + self._finish(context.metadata["verifications"][self.verifier_id], entry, judge._run) + return context + + def _finish(self, written: dict, entry: dict, this: _ThisRun) -> None: + """The written entry with its rows in the plan's order and shape, the run's verdict, and the run's details.""" + rows = core.finish_rows(written["results"], this.plan) + error = core.user_sim_error(rows) + error["fails_task"] = False + if error["value"] == "yes": + if self.fail_task_when_not_passing is not True: + error["not_failing"] = NOT_FAILING_OFF + elif not (entry.get("instructions") or {}).get("matches"): + error["not_failing"] = NOT_FAILING_UNCONFIRMED + else: + error["fails_task"] = True + written.update(results=rows, score=0.0 if any(r.get("result") is False for r in rows) else 1.0, **entry, + user_sim_error=error, calls=this.calls, usage=core.usage_total(this.usage), + cost=round(this.cost, COST_DECIMALS)) + if error["value"] == "unknown": + judged = [r for r in rows if r["id"] in core.CHECKS] + written["flag"] = FLAG_NONE_DECIDED if judged and all(r["verdict"] == "error" for r in judged) \ + else FLAG_UNDECIDED + logger.info(f"user_sim_judge '{self.id}': {entry['run_kind']} '{entry['run_step_id']}' user_sim_error=" + f"{error['value']} ({', '.join(error['because'] + error['unknown']) or 'no row'}), fails_task=" + f"{error['fails_task']}; " + ", ".join(f"{r['id']}={r['verdict']}" for r in rows)) + + def _find_run(self, context: TaskStepContext) -> tuple[str, str]: + """The kind of run ``run_step_id`` left and its step id: always_on when it left an always_on record, else a + chat (a chat whose user sim never read leaves a response and no reads, and is skipped).""" + records = context.metadata.get(ALWAYS_ON_KEY) or {} + reads = context.metadata.get(READS_KEY) or {} + if self.run_step_id in records: + return ALWAYS_ON, self.run_step_id + if self.run_step_id in reads or _prompt_response(context, self.run_step_id) is not None: + return CHAT, self.run_step_id + raise StepContextError(f"user_sim_judge '{self.id}': step '{self.run_step_id}' left no run to judge " + f"(always_on records: {sorted(records)}; prompt_agent user sim reads: {sorted(reads)})") + + # ---- rubrics_verifier's hooks, on the run's own copy --------------------------------------------------------- + def _this(self) -> _ThisRun: + if self._run is None: + raise RuntimeError(f"user_sim_judge '{self.id}': the judge's hooks run only on a run's own copy of the step") + return self._run + + def _resolved_criteria(self, context: TaskStepContext) -> list[dict]: + return [dict(c) for c in self._this().plan.criteria] + + async def judge_trajectory_content(self, context: TaskStepContext, prompt_response: PromptResponse) -> str: + return self._this().text + + async def unjudged_rows(self, context: TaskStepContext, prompt_response: PromptResponse) -> list[dict]: + return list(self._this().plan.unjudged) + + def _criteria_batches(self, criteria: list[dict], size: Optional[int]) -> list[list[dict]]: + """One call per check, with the task items riding with it, then the calls that ask only items.""" + out, start = [], 0 + for n in self._this().plan.batches: + out.append(criteria[start:start + n]) + start += n + return out + + async def _prompt_llm_judge(self, eval_prompt: str, *, model: str, context: TaskStepContext, + image_blocks: Optional[list[dict]] = None, system: Optional[str] = None, + cached_prefix: Optional[str] = None) -> dict: + """rubrics_verifier's call, with the system message and everything up to the end of the run marked for the + prompt cache. The run's first call goes alone, so the others read the run from the cache.""" + this = self._this() + end = eval_prompt.find(this.text) + head = eval_prompt[:end + len(this.text)] if end >= 0 else None + + async def call() -> dict: + got = await super(UserSimJudge, self)._prompt_llm_judge( + eval_prompt, model=model, context=context, image_blocks=image_blocks, system=SYSTEM, + cached_prefix=head) + this.calls += 1 + this.usage.append(got.get("usage")) + this.cost += got.get("cost") or 0.0 + return got + + if this.first is None: + this.first = asyncio.Event() + try: + return await call() + except Exception as e: + this.first_error = e + raise + finally: + this.first.set() + await this.first.wait() + if this.first_error is not None and _setup_error(this.first_error) is not None: + raise RuntimeError("the run's first judge call couldn't reach its model") from this.first_error + return await call() + + async def _run_judge_with_output_retries(self, *, criteria: list[dict], **kwargs: Any + ) -> tuple[list[dict], int, list[dict], Optional[str]]: + """rubrics_verifier's answer retries; a call still without a usable answer leaves its criteria undecided + (``error``) rather than failing the step: an unknown never fails the task.""" + try: + return await super()._run_judge_with_output_retries(criteria=criteria, **kwargs) + except (ValueError, RuntimeError, TimeoutError) as e: # TimeoutError: an agent judge's (run-wide use_agent_judge) + if (setup := _setup_error(e)) is not None: + raise JudgeSetupError(f"the user sim judge can't reach its model: {type(setup).__name__}: " + f"{str(setup)[:ERROR_SHOWN_CHARS]}") from e + cause = _root(e) + error = (f"no usable answer after {self.DEFAULT_MAX_RETRIES} tries: {type(cause).__name__}: " + f"{str(cause)[:ERROR_SHOWN_CHARS]}") + logger.warning(f"user_sim_judge '{self.id}': {len(criteria)} criteria undecided, {error}") + return [core.undecided_row(c, error) for c in criteria], 0, [], None + + +class JudgeSetupError(RuntimeError): + """The judge can't reach its model as it is set up (no key, a bad key, a model that doesn't exist, a request the + model refuses): the run fails at once, rather than every check going unknown.""" + + +def _chain(e: BaseException) -> list[BaseException]: + out: list[BaseException] = [] + while e is not None and e not in out: + out.append(e) + e = e.__cause__ or e.__context__ + return out + + +def _root(e: BaseException) -> BaseException: + """What the judge call itself raised: rubrics_verifier wraps a model call that kept failing in a RuntimeError.""" + return _chain(e)[-1] + + +def _setup_error(e: BaseException) -> Optional[BaseException]: + """The setup error behind a failed judge call, or None for a flaky one. Setup: agent-env's ConfigError (no key, + missing config) and LiteLLM's AuthenticationError, PermissionDeniedError, NotFoundError (an unknown model), + BadRequestError (with its kinds: an unknown provider, unsupported params, a prompt over the context window), + UnprocessableEntityError and BudgetExceededError. Anything else is flaky (Timeout, RateLimitError, + ServiceUnavailableError, InternalServerError, BadGatewayError, APIConnectionError, an agent judge's TimeoutError, + JudgeTimeoutError or AgentUnreachableError, an answer still unusable after its retries) and leaves the call's rows + undecided.""" + import litellm + + from agent_env.config.errors import ConfigError + + setup = (ConfigError, litellm.AuthenticationError, litellm.PermissionDeniedError, litellm.NotFoundError, + litellm.BadRequestError, litellm.UnprocessableEntityError, litellm.BudgetExceededError) + return next((x for x in _chain(e) if isinstance(x, setup)), None) + + +def _problems(model: Any, items: Any, gate: Any, rubric_persona: Any = None) -> list[str]: + out = [] + if rubric_persona is not None and not (isinstance(rubric_persona, str) + and re.fullmatch(r"[0-9a-f]{64}", rubric_persona)): + out.append(f"rubric_persona_sha256 must be a persona fingerprint (64 lowercase hex digits), not " + f"{rubric_persona!r}") + if model is not None and (not isinstance(model, str) or not model.strip()): + out.append("judge_model must be a model name") + if not isinstance(gate, bool): + out.append("fail_task_when_not_passing must be true or false") + if not isinstance(items, list): + out.append("rubric_items must be a list of {id, family, kind, check, ...}") + else: + out += core.item_problems(items) + out += [f"rubric_items[{i}] ({item['id']}): id '{item['id']}' is a check's name; give the item another id" + for i, item in enumerate(items) if isinstance(item, dict) and isinstance(item.get("id"), str) + and item["id"] in core.CHECKS] + if rubric_persona is None: + out += [f"rubric_items[{i}] ({item.get('id')}): a line item needs rubric_persona_sha256, so its tag is " + "graded only on runs of the persona it was written for" + for i, item in enumerate(items) if isinstance(item, dict) and item.get("line") is not None] + return out + + +def _rubric_persona(rubric: str, run: Optional[str]) -> dict: + """Whether the task items were written for this run's user sim: ``status`` match, mismatch (the items aren't + graded) or unknown (the run recorded no ``persona_sha256``; they are).""" + out: dict[str, Any] = {"rubric": rubric, "run": run} + if run is None: + return {**out, "status": "unknown", + "note": "the run recorded no persona fingerprint, so the task items are graded as written"} + if run != rubric: + return {**out, "status": "mismatch", + "note": "the task items were written for a different persona or opening than this run's user sim had, " + "so they weren't graded; the general checks were"} + return {**out, "status": "match"} + + +def _prompt_response(context: TaskStepContext, step_id: str) -> Optional[PromptResponse]: + return next((p for p in reversed(context.prompt_responses) if p.step_id == step_id), None) + + +# ---- the run as the judge reads it ---------------------------------------------------------------------------------- +def _read_run(kind: str, step_id: str, record: Optional[dict], context: TaskStepContext) -> tuple[core.UserSimRun, dict]: + """The user sim's run, and what the entry records about it: ``user_sim`` (its name) and ``instructions``.""" + metadata = context.metadata + if kind == ALWAYS_ON: + reads, notes, missing = _always_on_reads(record) + name = (record.get("user_sim") or {}).get("name") + multi_day = record.get("multi_day") + else: + reads, notes = _chat_reads(metadata.get(READS_KEY, {}).get(step_id) or []) + missing = 0 # a chat run keeps a summary of every read + name = _chat_user_sim(metadata, step_id) + multi_day = False + us = (metadata.get(USER_SIMS_KEY) or {}).get(name) if name else None + instructions, check, more = _instructions(us, metadata.get("seed") or {}, multi_day) + turns = [_turn(n, e) for n, e in enumerate(reads, 1)] + more += _reply_limit_note(turns, (us or {}).get("max_replies")) + run = core.UserSimRun(name=name or "the user", instructions=instructions, turns=turns, notes=notes + more, + tagged_lines=(us or {}).get("tagged_lines"), missing_answers=missing) + return run, {"user_sim": name, "instructions": check} + + +def _reply_limit_note(turns: list[core.Turn], max_replies: Any) -> list[str]: + """A note when the user sim reached its ``max_replies``, which its instructions never mention: the ending check + needs it.""" + replies = [t.n for t in turns if t.outcome == "reply" and t.message.strip()] + if isinstance(max_replies, bool) or not isinstance(max_replies, int) or not 0 < max_replies <= len(replies): + return [] + return [f"the run ended at the user sim's reply limit: its reply at turn {replies[max_replies - 1]} was the last of " + f"its {max_replies} and it was never asked again (its instructions don't mention the limit)"] + + +def _load(url: str) -> dict: + from agent_env.config import get_config + + return json.loads(get_config().get_object_store_at(url).get(url)) + + +def _always_on_reads(record: dict) -> tuple[list[dict], list[str], int]: + """Every ``user_sim_read`` in the always_on step's stored timeline, else its tail; with notes for the judge and how + many of the run's reads the tail dropped (by the record's ``counts.user_sim_reads``).""" + if record.get("object_url"): + try: + timeline = _load(record["object_url"])["timeline"] + return _with_delivery(timeline, timeline), [], 0 + except Exception as e: + note = (f"the run's stored record could not be read ({type(e).__name__}); only its last events, without " + "what the user sim was sent, are shown") + else: + note = "the run's full record wasn't stored; only its last events, without what the user sim was sent, are shown" + from agent_env.task_step.task_steps.always_on import PENDING_TEXT_CHARS + + tail = record.get("timeline_tail") or [] + reads = [] + for e in _with_delivery(tail, tail): + message = e.get("message") or "" # the tail keeps each string's first PENDING_TEXT_CHARS, then "…" + reads.append({**e, "message_cut": len(message) == PENDING_TEXT_CHARS + 1 and message.endswith("…")}) + total = (record.get("counts") or {}).get("user_sim_reads") + missing = total - len(reads) if isinstance(total, int) and not isinstance(total, bool) and total > len(reads) else 0 + notes = [note] + ([f"the first {missing} of the run's {total} check-ins aren't kept, so they aren't shown"] + if missing else []) + return reads, notes, missing + + +def _with_delivery(reads_from: list[dict], timeline: list[dict]) -> list[dict]: + """The ``user_sim_read`` events, each reply marked ``held`` when no ``user_reply_delivered`` event carries it (its + ``virtual_time`` is the reply's ``deliver_at``): it was still waiting when the run ended. A delivery comes after its + read, so a tail that has the read has its delivery too.""" + delivered = {e.get("virtual_time") for e in timeline if e.get("event") == "user_reply_delivered"} + return [{**e, "held": bool(e.get("deliver_at")) and not e.get("last_read") and e["deliver_at"] not in delivered} + for e in reads_from if e.get("event") == "user_sim_read"] + + +def _chat_reads(summaries: list[dict]) -> tuple[list[dict], list[str]]: + """Each read of a chat run, whole from its ``object_url``, else its summary, which keeps only the answer.""" + reads, missing = [], 0 + for summary in summaries: + full = None + if summary.get("object_url"): + try: + full = _load(summary["object_url"]) + except Exception as e: + logger.warning(f"user_sim_judge: could not read {summary['object_url']}: {type(e).__name__}: {e}") + if full is None: + missing += 1 + read = {**summary, **(full or {})} + message = read.get("message") or "" + read["looks_like_assistant"] = (len(message) > ASSISTANT_LIKE_MESSAGE_CHARS + or bool(re.search(r"^\s*(#|\|)", message, re.M))) + reads.append(read) + notes = [f"{missing} of the run's {len(summaries)} check-ins weren't stored whole, so what the user sim was sent " + "at them isn't shown"] if missing else [] + return reads, notes + + +def _chat_user_sim(metadata: dict, step_id: str) -> Optional[str]: + """Which user sim played a chat run: its conversation's user, else the run's only user sim.""" + users = metadata.get(USER_SIMS_KEY) or {} + conversation_id = (metadata.get("a2a_conversations") or {}).get(step_id) + if conversation_id: + try: + from agent_env.a2a_agent import conversation_store + + source = (conversation_store.get_conversation(conversation_id) or {}).get("source_agent_name") + name = next((k for k, u in users.items() if source in (k, u.get("agent_name"))), None) + if name: + return name + except Exception as e: + logger.warning(f"user_sim_judge: could not read conversation {conversation_id}: {type(e).__name__}: {e}") + return next(iter(users)) if len(users) == 1 else None + + +def _turn(n: int, e: dict) -> core.Turn: + flags = [] + if e.get("looks_like_assistant"): + flags.append(FLAG_ASSISTANT) + if e.get("schema_errors"): + flags.append("its answer was malformed, so nothing was sent: " + "; ".join(e["schema_errors"])[:SCHEMA_ERRORS_SHOWN_CHARS]) + elif e.get("retried"): + flags.append("its first answer was malformed and it answered again") + if e.get("user_sim_error"): + flags.append(f"the user sim agent failed at this check-in: {str(e['user_sim_error'])[:USER_SIM_ERROR_SHOWN_CHARS]}") + if e.get("last_read"): + flags.append(FLAG_LAST) + elif e.get("held"): + flags.append(FLAG_HELD) + when = e.get("virtual_time") or (e.get("read") or {}).get("time") + return core.Turn(n=n, outcome=e.get("outcome") or "no_reply", message=e.get("message") or "", + input=e.get("input"), flags=flags, time=None if e.get("input") or not when else f"{when} UTC", + cut=bool(e.get("message_cut"))) + + +def _instructions(us: Optional[dict], seed: dict, multi_day: Optional[bool]) -> tuple[Optional[str], dict, list[str]]: + """The user sim's system prompt, rebuilt from its ``user_sims`` record as deploy_user_sim_agent wrote it, what the + entry records of the check against the record's ``prompt_sha256``, and notes for the judge.""" + if not us: + return None, {"source": "missing", "matches": False}, [ + "the user sim's settings were not found, so its instructions aren't shown"] + from agent_env.task_step.task_steps.agent import user_sim_versions + from agent_env.task_step.task_steps.agent.deploy_user_sim_agent import recorded_person + + try: + person = recorded_person(us, seed, multi_day) + prompt = user_sim_versions.setup(us.get("user_version") or user_sim_versions.VERSION, person)["system_prompt"] + except Exception as e: + return None, {"source": "missing", "matches": False, "error": f"{type(e).__name__}: {e}"}, [ + f"the user sim's instructions could not be rebuilt ({type(e).__name__}), so they aren't shown"] + sha = hashlib.sha256(prompt.encode()).hexdigest() + recorded = us.get("prompt_sha256") + check = {"source": "rebuilt", "prompt_sha256": sha, "recorded_prompt_sha256": recorded, + "matches": recorded == sha} + if recorded is None: + return prompt, check, ["these instructions were rebuilt from the user sim's settings, and the run recorded no " + "hash of the ones it got, so they may differ from what it got"] + if recorded != sha: + return prompt, check, ["these instructions were rebuilt from the user sim's settings and differ from the ones " + "it got, so a line may be missing or changed"] + if not person["opening"]: + return prompt, check, ["the opening request wasn't recorded, so it is missing from these instructions"] + return prompt, check, [] diff --git a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py new file mode 100644 index 000000000..6e4036826 --- /dev/null +++ b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py @@ -0,0 +1,709 @@ +"""What the user_sim_judge step asks about a run, and how it reads the rows. Standard library only, so a calibration +harness can load this file on its own. + +The judge grades the user sim (the AI playing the user), not the assistant. Each check asks one narrow question over +the user sim's whole run and covers families of the User Simulator Rubric Taxonomy (PSI Bench SSOT): + +| check | families | +| guidance | A1, A3 | +| oversharing | G1, G2 | +| delegation | A4 | +| style | F1, F2, F3, F4 | +| rogue | G3, G5 | +| made_up | A2 | +| ending | G4 | + +A5 (satisfaction) has no check: the separate ``rate_user_satisfaction`` step covers it, not this judge. A task's own +rubric items (``RubricItem``) may name any family. ``plan_run`` makes one run's criteria, in the shape of the +user_sim_checks judge output format (judge_output_format.py): each check is one criterion and one call, and an item +rides in the call of the check that owns its family; an item whose family no running check owns goes in a call of its +own (``TASK_ITEMS``). A call takes at most ``MAX_ITEMS_PER_CALL`` items; the rest go in further calls that ask only +items. ``auto`` items, ``na_when: always`` items and line items whose line can't be read are settled here, with no +call. An item's ``line`` (a tag, "P1") is graded by that line's text in the run's ``tagged_lines``. + +Each row has a weight: 1 for a check in the step's ``error_checks`` (every check but style, by default) and for an +item that can make the run a user sim error, else 0. ``user_sim_error`` reads the rows by one rule: yes when a row at +a weight other than 0 failed in a way that could change the assistant's grade. +""" +from __future__ import annotations + +import json +import re +from dataclasses import dataclass, field, replace +from typing import Any, Callable, Optional + +# ---- the rubric families ------------------------------------------------------------------------------------------ +FAMILIES: dict[str, str] = { + "A1": "Requirement provision: gives each requirement the way this person would; answers what it knows, points to " + "where look-up facts live, admits what it doesn't know, gives its stated call on a decision, corrects " + "mistakes it can see if this person would.", + "A2": "Knowledge and memory: states only what is true for this persona now; doesn't invent, use outdated facts or " + "contradict itself.", + "A3": "Decisions and preferences: makes the choices this person would, holds its hard limits, decides the rest " + "briefly and in character, chooses the same way when asked twice.", + "A4": "Delegation: gives or withholds permission the way this person would.", + "A5": "Satisfaction judgment: rates the outcome the way this person would.", + "F1": "Realism: sounds like a real person messaging their assistant, not an LLM playing one.", + "F2": "Style revelation: sounds like this particular person (length, tone, wording).", + "F3": "Reactions: reacts the way this person would, only after things happen.", + "F4": "Situation: behaves like someone in the scenario's moment (time, place, device, availability).", + "G1": "Leaking and privacy: keeps hidden information hidden.", + "G2": "Restraint: doesn't make the task easier than a real person would.", + "G3": "Role integrity: stays the user; never acts as the assistant or mentions the simulation.", + "G4": "Ending: ends the conversation at the right time.", + "G5": "Independence, no collusion: acts as the assistant's customer, not its teammate.", +} + +TASK_ITEMS = "task_items" +# An item's ``asked_with`` when no call asked it: an auto item counted here, an ``na_when: always`` item. +COMPUTED, STATIC_NA = "computed", "static_na" + + +@dataclass(frozen=True) +class Check: + name: str + title: str + families: tuple[str, ...] + question: str + fails: tuple[str, ...] + not_fails: tuple[str, ...] + na: str = "" + grade: str = "" + + +CHECKS: dict[str, Check] = {c.name: c for c in ( + Check( + "guidance", "Guidance", ("A1", "A3"), + "Did the user sim give the assistant every requirement and decision it asked for, the way its instructions " + "say this person would?", + fails=( + "The assistant asked for something its instructions say this person knows, decides or points to, and the " + "answer withheld it, deflected it (\"it's in the chat\" for something she should say herself), handed the " + "choice back (\"you pick\" where her instructions make the call), or left unanswered a question the assistant " + "needs answered.", + "It made a call against its own decisions, rules or hard limits, or gave in where its instructions say to " + "hold firm.", + "Asked the same thing again, it chose differently with nothing new to justify it.", + "Asked again for something it already gave, it didn't give it again.", + "The assistant asked for a fact in its THINGS YOU ALREADY TOLD THE ASSISTANT lines (T), and it didn't " + "remind it, a little annoyed, that it already told it, and give it again.", + "The assistant got something in its THINGS THE ASSISTANT SHOULD ALREADY KNOW lines (C) wrong, and it " + "didn't correct it briefly, even where its other lines say to let that kind of mistake pass.", + "Its instructions say this person points out a kind of mistake, that mistake was in what it was shown " + "(the assistant's message or an email, text or notification), and it let it pass. Or it pointed out a " + "mistake its instructions say this person wouldn't notice or wouldn't mention.", + "Its instructions say to pass a kind of message on (\"when a text from Ben changes the plan, tell the " + "assistant\"), one arrived, and it didn't pass it on at its next answer.", + ), + not_fails=( + "\"Don't know\" for a fact no line of its instructions gives.", + "Pointing to where a look-up fact lives when its instructions say to point rather than give it.", + "Silence or a brief reply to a message that asks nothing.", + "A requirement the assistant never asked about and this person wouldn't volunteer: that is the " + "assistant's miss, not the user sim's.", + "Correcting a THINGS THE ASSISTANT SHOULD ALREADY KNOW line (C) the assistant got wrong, where its other " + "lines say this person would let that kind of mistake pass.", + ), + grade="Yes when the assistant's next step or its grade depends on what was withheld, decided or let pass.", + ), + Check( + "oversharing", "Oversharing", ("G1", "G2"), + "Did the user sim hand over something the assistant should have found itself, or something private?", + fails=( + "It volunteered, unasked, a fact its instructions say the assistant must look up, must find itself, or " + "that it doesn't mention unless asked, or something the assistant should already know from its notes.", + "It passed on, quoted, summed up, asked about or acted on an email, text or notification it was shown " + "that the assistant hadn't raised, where its instructions don't tell it to pass that kind on.", + "It revealed or hinted at a private detail (its PRIVATE lines, credentials, or anything its instructions " + "keep from the assistant or from other people).", + "Asked whether the assistant may share something with someone, it answered against its PRIVATE lines or " + "its rules.", + "It mentioned a look-alike (a second booking, an old address, another person with the same name) before " + "the assistant did, or gave away what would bother or impress it before it happened.", + "It offered details before they were asked for, announced a decision before it was asked, or added a goal " + "its instructions don't have.", + ), + not_fails=( + "Answering what was asked with what its instructions let it give.", + "Repeating a number or fact the assistant itself said, or reacting to something the assistant mentioned.", + "Passing something on that its instructions tell it to pass on.", + "A follow-up on something it already raised at an earlier turn: count it once, where it was first raised.", + ), + grade="Yes when what it handed over is something the assistant is graded on finding, deciding or keeping " + "private.", + ), + Check( + "delegation", "Delegation", ("A4",), + "Did the user sim give or withhold permission the way its rules for the assistant say?", + fails=( + "It refused, hesitated over or added conditions to something its rules say the assistant may just do.", + "It approved something its rules say the assistant must never do.", + "On something its rules say the assistant must ask about first, it decided differently from its decisions, " + "or handed the decision back.", + "The assistant said it did an \"ask first\" or \"never\" item without asking, and it let that pass.", + "Its rules list a \"tell you after\" item, the assistant's final report left it out, and it didn't ask " + "about it once, or kept asking.", + ), + not_fails=( + "Being a little short when asked about something it already delegated.", + "Saying yes to an \"ask first\" item its decisions allow.", + ), + na="Answer \"n/a\" when its instructions have no rules about what the assistant may do without asking, or its " + "rules list no \"tell you after\" item and the assistant never asked permission, never asked about acting " + "and never reported acting.", + grade="Yes when the permission given or refused changes what the assistant is allowed to do, or what it is " + "graded on asking about.", + ), + Check( + "style", "Style", ("F1", "F2", "F3", "F4"), + "Does the user sim sound and react like this person all run, or does it drift?", + fails=( + "Its messages are clearly off from its HOW YOU WRITE lines and example messages on one dimension or more: " + "length, tone, punctuation, capitals, emoji or wording (too long or too polished for this person, " + "lists or headings, thanking every turn, repeating back what the assistant said, answering every question " + "in neat order where this person wouldn't, every message about the same length).", + "It starts in character and drifts later in the run: say from which turn.", + "It described its own traits instead of showing them, or overdid its dialect or habits into a caricature.", + "Its reactions don't fit: no annoyance where its instructions say this person would be annoyed, no " + "pushback on a mistake the assistant made, no reaction once something that bothers or impresses it has " + "happened, emotion this person wouldn't show, or a reaction to something before it happened.", + "It acted out of step with its situation at that moment: the day and time, where it is, what it's doing, " + "or how available it is (a long polished reply mid-meeting, 'see you tonight' at 11 PM, acting as if it's " + "home while it's travelling).", + ), + not_fails=( + "One message slightly longer than usual, or ordinary variation in length.", + "Leaving out a style habit in a very short answer.", + ), + na="Answer \"n/a\" when it sent no message at any graded turn.", + grade="Usually no. Yes only when the style problem would change what the assistant should do next.", + ), + Check( + "rogue", "Rogue", ("G3", "G5"), + "Did the user sim stop acting as the user?", + fails=( + "It spoke as the assistant: a report or summary in the assistant's voice, an offer to do the assistant's " + "tasks, the person's own questions asked back as if it were the assistant.", + "It drafted a full message for a third party, for sending.", + "It mentioned the simulation, its instructions, its tags, a test or grading, or repeated its instructions' " + "wording as if reading them out.", + "It spoke for someone else, or acted as anyone but this person.", + "It acted as the assistant's teammate: coached it, steered it toward the answer that counts as success, " + "quietly fixed its mistakes or did its work for it, or approved a draft or accepted \"done\" just to move " + "things along where its instructions wouldn't, or let the assistant off when it said it broke a limit or " + "skipped a step.", + ), + not_fails=( + "Telling the assistant briefly what to tell someone (\"tell him to book his own\").", + "Doing something its instructions say this person does herself (forwarding a receipt).", + ), + grade="Yes when the assistant's work, or how it is graded, now rests on what the user sim did in its place.", + ), + Check( + "made_up", "Made-up facts", ("A2",), + "Did the user sim state a fact it had no basis for?", + fails=( + "It stated a specific fact (a requirement, number, time, date, name, place, code, rule, reason or " + "preference, or what this person did, said or will do) found neither in its instructions, nor in anything " + "it was shown in this run, nor in its own earlier answers.", + "It claimed an earlier exchange that didn't happen (\"i already said...\", \"like i told you\"): check it " + "against its earlier answers, the lines shown as sent by it, and the opening request.", + "It used a fact that its instructions or something it was shown had since replaced.", + "It contradicted something it said earlier in the run.", + ), + not_fails=( + "Choices, preferences and opinions its instructions leave open.", + "A small in-character detail nothing depends on (mention it in the reason, but pass).", + "Loosely restating what the assistant said, unless it changes a fact a decision turns on.", + "Pointing to where something might be (\"check the chat\").", + ), + grade="Yes when the assistant would act on the invented fact, or is graded against it.", + ), + Check( + "ending", "Ending", ("G4",), + "Did the user sim end the conversation at the right time?", + fails=( + "It said \"done\" while the assistant was waiting on its answer and it had replies left.", + "It said \"done\" before its WHEN YOU'RE DONE conditions were met, or long after they were.", + ), + not_fails=( + "A short sign-off when its reply limit is reached.", + ), + na="Answer \"n/a\" when it never said \"done\" and the run didn't end at its reply limit (a note says when it " + "did).", + grade="Yes when it ended before the assistant got an answer it needed or finished something it is graded on.", + ), +)} +CHECK_NAMES = tuple(CHECKS) +FAMILY_OWNER: dict[str, str] = {f: c.name for c in CHECKS.values() for f in c.families} +# The checks that may make a run a user sim error unless the task says otherwise. Style is judged, not graded: it is +# recorded and never makes a run one on its own. +DEFAULT_ERROR_CHECKS = ("guidance", "oversharing", "delegation", "rogue", "made_up", "ending") + +# ---- the run the judge reads -------------------------------------------------------------------------------------- +@dataclass +class Turn: + """One check-in: what the user sim was sent (``input``, None when the record didn't keep it) and its answer.""" + n: int + outcome: str + message: str = "" + input: Optional[str] = None + time: Optional[str] = None + graded: bool = True + flags: list[str] = field(default_factory=list) + cut: bool = False # ``message`` is only the start of what it sent (the record kept a shortened copy) + + +@dataclass +class UserSimRun: + name: str + instructions: Optional[str] + turns: list[Turn] + notes: list[str] = field(default_factory=list) + # Each tag the user sim's instructions showed, with its line (deploy_user_sim_agent's record); None: not recorded. + tagged_lines: Optional[dict[str, str]] = None + missing_answers: int = 0 # the run's answers its record didn't keep (always_on's tail drops the earliest) + + +# ---- a task's own rubric items ------------------------------------------------------------------------------------ +HARD, GENERAL, AUTO = "hard", "general", "auto" +KINDS = (HARD, GENERAL, AUTO) +NA_ALWAYS, NA_NEVER = "always", "never" +MAX_WORDS, MAX_SENTENCES = "max_words", "max_sentences" +METRICS = (MAX_WORDS, MAX_SENTENCES) +MAX_ITEMS_PER_CALL = 12 # task items asked in one call; a check's or the task's further items get calls of their own +ITEM_FIELDS = ("id", "family", "kind", "check", "text", "line", "name", "short", "na_when", "static_na_reason", "metric", + "max", "can_error") +# A line item's tag: a section letter and the line's place in it, as the user sim's instructions show it ("P1", "D10"). +LINE_TAG = re.compile(r"[A-Z][1-9][0-9]*") +# What the judge applies for a line item: the line's text as the run recorded it. +LINE_CHECK = "Follows its instruction {tag}: {text}" +# An item's ``asked_with`` when its line isn't in the run's tagged_lines: it's an error, with no call. +NO_LINE = "no_line" + + +@dataclass(frozen=True) +class RubricItem: + """A task's own rubric item: one rule the judge checks, under one family. ``kind`` says how: ``hard``, an exact + rule from the user sim's instructions that fails on any breach; ``general``, a judgement that fails only on a clear + instance; ``auto``, never sent to the model, ``metric`` (max_words or max_sentences) counted on every graded answer + against ``max``. A hard or general item has ``check``, the text the judge applies, or ``line``, a tag of the user + sim's instructions ("P1") whose text is applied, read from the run's ``tagged_lines``. ``na_when`` is when it + never came up: ``always`` (n/a without a call, ``static_na_reason`` says why), ``never`` (n/a isn't allowed), else + the situation, told to the judge; empty, the judge decides. An auto item takes only ``always``: it is counted, + never judged.""" + id: str + family: str + check: str = "" + kind: str = GENERAL + name: str = "" + short: str = "" + na_when: str = "" + static_na_reason: str = "" + metric: str = "" + max: Optional[int] = None + can_error: Optional[bool] = None # None: as the check that owns its family + line: str = "" # a tag of the user sim's instructions, graded by that line's text instead of ``check`` + + @property + def static_na(self) -> bool: + return self.na_when.strip().lower() == NA_ALWAYS + + @property + def na_allowed(self) -> bool: + return self.na_when.strip().lower() != NA_NEVER + + @property + def asks_the_model(self) -> bool: + return self.kind != AUTO and not self.static_na + + @classmethod + def from_dict(cls, data: dict) -> "RubricItem": + """An item ``item_problems`` passed. ``text`` is the old name of ``check``.""" + return cls(id=str(data["id"]), family=str(data["family"]).strip().upper(), + check=str(data.get("check") if data.get("check") is not None else data.get("text") or "").strip(), + line=str(data.get("line") or "").strip(), + kind=str(data.get("kind") or GENERAL).strip().lower(), name=str(data.get("name") or ""), + short=str(data.get("short") or ""), na_when=str(data.get("na_when") or "").strip(), + static_na_reason=str(data.get("static_na_reason") or "").strip(), + metric=str(data.get("metric") or ""), max=data.get("max"), can_error=data.get("can_error")) + + def to_dict(self) -> dict: + out: dict[str, Any] = {"id": self.id, "family": self.family, "kind": self.kind} + for k in ("name", "short", "check", "line", "na_when", "static_na_reason", "metric"): + if getattr(self, k): + out[k] = getattr(self, k) + if self.max is not None: + out["max"] = self.max + if self.can_error is not None: + out["can_error"] = self.can_error + return out + + +def item_problems(items: list[dict]) -> list[str]: + """What's wrong with a task's rubric items, each problem naming the item; empty when they're fine.""" + out, seen = [], set() + for i, item in enumerate(items): + if not isinstance(item, dict): + out.append(f"rubric_items[{i}] must be a mapping with an id, a family and a check") + continue + at = f"rubric_items[{i}]" + (f" ({item['id']})" if isinstance(item.get("id"), str) else "") + for key in ("id", "family"): + if not isinstance(item.get(key), str) or not item[key].strip(): + out.append(f"{at} needs a{'n' if key == 'id' else ''} {key}") + if isinstance(item.get("family"), str) and item["family"].strip() and ( + item["family"].strip().upper() not in FAMILIES): + out.append(f"{at}: family {item['family']!r} is not one of {list(FAMILIES)}") + if isinstance(item.get("id"), str) and item["id"].strip(): + if item["id"] in seen: + out.append(f"{at}: id {item['id']!r} is used twice") + seen.add(item["id"]) + kind = GENERAL if item.get("kind") is None else item["kind"] + if not isinstance(kind, str) or kind.strip().lower() not in KINDS: + out.append(f"{at}: kind {kind!r} is not one of {list(KINDS)}") + kind = None + else: + kind = kind.strip().lower() + for key in ("check", "text", "line", "name", "short", "na_when", "static_na_reason"): + if item.get(key) is not None and not isinstance(item[key], str): + out.append(f"{at}: {key} must be text") + if item.get("check") is not None and item.get("text") is not None: + out.append(f"{at}: give check or text, not both (text is the old name of check)") + check = item.get("check") if item.get("check") is not None else item.get("text") + line = item.get("line") + if line is not None: + if isinstance(line, str) and not LINE_TAG.fullmatch(line.strip()): + out.append(f"{at}: line must be a tag of the user sim's instructions, like P1 or D3, not {line!r}") + if check is not None: + out.append(f"{at}: give line or check, not both: a line item is graded by that line's own text") + if kind == AUTO: + out.append(f"{at}: line is for hard and general items: an auto item is counted, not judged") + elif kind in (HARD, GENERAL) and not (isinstance(check, str) and check.strip()): + out.append(f"{at}: a {kind} item needs a check (the text the judge applies) or a line (a tag of the user " + "sim's instructions, like P1)") + if kind == AUTO: + if item.get("metric") not in METRICS: + out.append(f"{at}: an auto item's metric must be one of {list(METRICS)}, not {item.get('metric')!r}") + limit = item.get("max") + if isinstance(limit, bool) or not isinstance(limit, int) or limit < 1: + out.append(f"{at}: an auto item needs max, a whole number of at least 1, not {limit!r}") + elif kind is not None: + extra = [k for k in ("metric", "max") if item.get(k) is not None] + if extra: + out.append(f"{at}: {' and '.join(extra)} are for kind auto only") + na_when = item.get("na_when") + always = isinstance(na_when, str) and na_when.strip().lower() == NA_ALWAYS + reason = item.get("static_na_reason") + if always and not (isinstance(reason, str) and reason.strip()): + out.append(f"{at}: na_when always needs a static_na_reason saying why it can't apply") + if reason is not None and not always: + out.append(f"{at}: static_na_reason is only for na_when always") + if kind == AUTO and isinstance(na_when, str) and na_when.strip() and not always: + out.append(f"{at}: auto items are counted, so na_when can only be 'always' (with a static_na_reason), " + f"not {na_when!r}") + if "can_error" in item and not isinstance(item["can_error"], bool): + out.append(f"{at}: can_error must be true or false") + unknown = sorted(set(item) - set(ITEM_FIELDS)) + if unknown: + out.append(f"{at}: unknown fields {unknown}") + return out + + +# ---- the auto items ----------------------------------------------------------------------------------------------- +# A sentence ends at a run of . ! or ? followed by a space or the end, or at a line break; a piece with no letter or +# digit (an emoji, "...") isn't one. So "$15.50" stays one sentence and "Mr. Lee" counts two: simple on purpose. +_SENTENCE_END = re.compile(r"[.!?]+(?=\s|$)|\n+") + + +def count_words(message: str) -> int: + """Words in a message: its pieces between whitespace, an emoji on its own counting as one.""" + return len(message.split()) + + +def count_sentences(message: str) -> int: + return sum(1 for piece in _SENTENCE_END.split(message) if re.search(r"\w", piece)) + + +COUNTERS: dict[str, tuple[Callable[[str], int], str]] = {MAX_WORDS: (count_words, "words"), + MAX_SENTENCES: (count_sentences, "sentences")} + + +def sent_answers(run: "UserSimRun") -> list["Turn"]: + """The graded answers an auto item counts: each graded turn's message that isn't a no_reply.""" + return [t for t in run.turns if t.graded and t.outcome != "no_reply" and (t.message or "").strip()] + + +def auto_result(item: RubricItem, run: "UserSimRun") -> dict: + """An auto item, computed: fail when any graded answer is over ``max``, naming each turn that is.""" + count, unit = COUNTERS[item.metric] + counts = [(t.n, count(t.message)) for t in sent_answers(run)] + if not counts: + return {"verdict": "n/a", "reason": "it sent no message at a graded turn", "turns": []} + over = [(n, c) for n, c in counts if c > item.max] + if over: # a shortened answer over the max is over it whole too + listed = ", ".join(f"turn {n}: {c}" for n, c in over) + return {"verdict": "fail", "reason": f"over the {item.max} {unit} max at {listed}", + "turns": [n for n, _ in over]} + if run.missing_answers: + return {"verdict": "error", "turns": [], + "error": f"{run.missing_answers} of the run's answers weren't kept, so they can't be counted"} + cut = [t.n for t in sent_answers(run) if t.cut] + if cut: + return {"verdict": "error", "turns": cut, + "error": f"only the start of the answer at turn{'s' if len(cut) > 1 else ''} " + f"{', '.join(map(str, cut))} was kept, so its full length can't be counted"} + n, c = max(counts, key=lambda x: x[1]) + return {"verdict": "pass", "reason": f"every answer at most {item.max} {unit}; the longest, turn {n}, has {c}", + "turns": []} + + +def resolve_lines(items: list[RubricItem], tagged_lines: Optional[dict[str, str]], + unusable: str = "") -> tuple[list[RubricItem], dict[str, dict]]: + """Each item with its ``line``'s text as its check (``LINE_CHECK``), and, by id, the line items that can't be + graded: their ``error``, never a guess. ``unusable`` says why no line can be trusted to be the one an item was + written for. An ``na_when: always`` item isn't graded, so its line isn't needed.""" + out, missing = [], {} + for item in items: + if not item.line or item.static_na: + out.append(item) + elif unusable: + missing[item.id] = {"verdict": "error", "turns": [], "line": item.line, + "error": f"line {item.line} can't be graded: {unusable}"} + elif tagged_lines is None: + missing[item.id] = {"verdict": "error", "turns": [], "line": item.line, + "error": f"the run recorded no tagged_lines for the user sim, so its line {item.line} " + "can't be read"} + elif item.line not in tagged_lines: + missing[item.id] = {"verdict": "error", "turns": [], "line": item.line, + "error": f"the user sim's instructions have no line {item.line}"} + else: + text = tagged_lines[item.line] + out.append(replace(item, check=LINE_CHECK.format(tag=item.line, text=text.replace("\n", "\n ")))) + return out, missing + + +def render_run(run: UserSimRun) -> tuple[str, str]: + """The run as the judge reads it, in two parts so a prompt cache can share the first across calls: the + instructions and every turn's input up to the last graded turn's answers stay the same for all checks.""" + head = ["# THE USER SIM'S INSTRUCTIONS"] + head.append("\n" + (run.instructions or "(not recorded)").strip() + "\n") + head += ["", "# THE RUN"] + head += [f"Note: {n}" for n in run.notes] + answers = [] + last_graded = max((t.n for t in run.turns if t.graded), default=None) + for t in run.turns: + tag = "GRADED" if t.graded else "CONTEXT ONLY" + when = f" · {t.time}" if t.time else "" + block = [f"\n## Turn {t.n}{when} · {tag}", "### What it was sent"] + block.append("\n" + t.input.strip() + "\n" if t.input else "(not recorded)") + block.append("### Its answer") + block.append(f"outcome: {t.outcome}") + if t.outcome != "no_reply" or (t.message or "").strip(): + block.append("message: " + json.dumps(t.message or "", ensure_ascii=False)) + block += [f"flag: {f}" for f in t.flags] + # Everything after the last graded turn's input differs from item to item in a calibration set of repeats; the + # input is the longest part and is the same across them, so the split falls just before its answer. + if last_graded is not None and t.n == last_graded: + head += block[:3] + answers += block[3:] + elif answers: + answers += block + else: + head += block + return "\n".join(head), "\n".join(answers) + + + + +def run_text(run: UserSimRun) -> str: + """The run as one text, the part of every call's prompt that its calls share (``render_run``'s two parts).""" + return "\n".join(part for part in render_run(run) if part) + + +# ---- what the judge is asked -------------------------------------------------------------------------------------- +def call_check(call: str) -> Optional[str]: + """The check a call asks; None for a call that asks only task items (``TASK_ITEMS``, or a check's or the task's + further items, ``#2``).""" + return call if call in CHECKS else None + + +def plan_calls(checks: list[str], items: list[RubricItem], + per_call: int = MAX_ITEMS_PER_CALL) -> dict[str, list[RubricItem]]: + """Which call asks what: each running check with the task items its families own, the first ``per_call`` of + them; items no running check owns go in one more call, ``TASK_ITEMS``. Items past ``per_call`` go, ``per_call`` + at a time, in calls that ask only items (``#2``, ``task_items#2``...). Only items that ask the model.""" + grouped: dict[str, list[RubricItem]] = {c: [] for c in checks} + for item in items: + if not item.asks_the_model: + continue + owner = FAMILY_OWNER.get(item.family) + grouped.setdefault(owner if owner in grouped else TASK_ITEMS, []).append(item) + per_call = max(1, per_call) + calls: dict[str, list[RubricItem]] = {} + for call, its in grouped.items(): + if call == TASK_ITEMS and not its: + continue + calls[call] = its[:per_call] + for part, start in enumerate(range(per_call, len(its), per_call), 2): + calls[f"{call}#{part}"] = its[start:start + per_call] + return calls + + +def check_title(name: str) -> str: + c = CHECKS[name] + return f"{c.title} ({', '.join(c.families)})" + + +def check_question(name: str) -> str: + """A check's question as the judge reads it: what it asks, what fails, what doesn't, when it's n/a and when a fail + could change the assistant's grade.""" + c = CHECKS[name] + lines = [c.question, "", "It fails when, at a graded turn:", *[f"- {f}" for f in c.fails], + "", "Not a failure:", *[f"- {f}" for f in c.not_fails], "", + c.na or "Answer \"pass\" or \"fail\"; \"n/a\" only if no turn is graded."] + if c.grade: + lines += ["", f"could_change_grade: {c.grade}"] + return "\n".join(lines) + + +def family_label(family: str) -> str: + return f"{family} ({FAMILIES[family].split(':')[0]})" + + +# ---- one run's criteria and rows ---------------------------------------------------------------------------------- +# The fields a row's verdict is in, as a judge's row (judge_output_format's user_sim_checks) or one computed here has +# them; a stored row is these, after what a reviewer reads about it. +VERDICT_FIELDS = ("verdict", "reason", "turns", "turns_outside_graded", "could_change_grade", "assistant_asked", + "error", "score", "result", "justification") + + +@dataclass +class Plan: + """What one run's judge is asked, and the rows no model grades. + + ``criteria``: user_sim_checks criteria in call order, each check followed by the task items riding with it; what + the judge reads of each (``id``, ``check`` or ``item``, ``family``, ``kind``, ``question``, ``na_when``, + ``graded_turns``, ``weight``). ``batches``: how many of them each call asks, in order. ``unjudged``: the rows no + model grades (auto items, ``na_when: always`` items, line items whose line can't be read). ``about``: by row id, + what a reviewer reads on the row and the judge never sees (an item's ``name``, ``short``, ``asked_with`` ...).""" + criteria: list[dict] = field(default_factory=list) + batches: list[int] = field(default_factory=list) + unjudged: list[dict] = field(default_factory=list) + about: dict[str, dict] = field(default_factory=dict) + order: list[str] = field(default_factory=list) # the row ids: the checks in order, then the task's items + + +def item_weight(item: RubricItem, error_checks: list[str]) -> float: + """1 when a fail of the item can make the run a user sim error, else 0: its ``can_error``, unset as the check + that owns its family (an error check, or not).""" + can = item.can_error if item.can_error is not None else FAMILY_OWNER.get(item.family) in error_checks + return 1.0 if can else 0.0 + + +def computed_row(got: dict, weight: float) -> dict: + """A row no model graded (``got``: verdict, reason or error, turns), scored as a judge's: it lowers the score only + on a fail at a weight other than 0. A computed fail could change the grade; ``assistant_asked`` is None, since no + judge looked.""" + fail = got["verdict"] == "fail" + lowers = fail and weight != 0 + return {**got, "could_change_grade": fail, "assistant_asked": None, + "justification": got.get("reason") or got.get("error") or "", + "score": 0.0 if lowers else 1.0, "result": not lowers} + + +def plan_run(run: UserSimRun, *, checks: list[str], items: list[RubricItem], error_checks: list[str], + lines_unusable: str = "") -> Plan: + """The criteria the judge is asked about ``run`` and the rows settled here: each check in ``checks`` (weight 1 + when it is in ``error_checks``, else 0) with the task items its families own, at most ``MAX_ITEMS_PER_CALL`` to a + call; an auto item counted, an ``na_when: always`` item n/a, a ``line`` item graded by its line's text in + ``run.tagged_lines``, or an error when the run has no such tag or ``lines_unusable`` says why none can be + trusted.""" + graded = sorted(t.n for t in run.turns if t.graded) + resolved, no_line = resolve_lines(list(items), run.tagged_lines, lines_unusable) + calls = plan_calls(list(checks), resolved) + plan = Plan(order=list(checks) + [i.id for i in items]) + for name in checks: + plan.about[name] = {"title": CHECKS[name].title, "families": list(CHECKS[name].families), + "weight": 1.0 if name in error_checks else 0.0} + lines = run.tagged_lines or {} + for item in items: + about = {k: v for k, v in item.to_dict().items() if k != "id"} + if item.line and item.asks_the_model and item.id not in no_line: + about["line_text"] = lines[item.line] + plan.about[item.id] = {**about, "weight": item_weight(item, error_checks)} + for call, its in calls.items(): + size = 0 + if call_check(call): + plan.criteria.append({"id": call, "check": check_title(call), "question": check_question(call), + "graded_turns": graded, "weight": plan.about[call]["weight"]}) + size += 1 + for item in its: + criterion = {"id": item.id, "item": item.id, "family": family_label(item.family), "kind": item.kind, + "question": item.check, "graded_turns": graded, "weight": plan.about[item.id]["weight"]} + if item.na_when: + criterion["na_when"] = item.na_when + plan.criteria.append(criterion) + plan.about[item.id]["asked_with"] = call + size += 1 + if size: + plan.batches.append(size) + for item in items: + if item.id in no_line: + got, asked_with = no_line[item.id], NO_LINE + elif item.kind == AUTO and not item.static_na: + got, asked_with = auto_result(item, run), COMPUTED + elif item.static_na: + got, asked_with = {"verdict": "n/a", "reason": item.static_na_reason, "turns": []}, STATIC_NA + else: + continue + got = {k: v for k, v in got.items() if k != "line"} + plan.about[item.id]["asked_with"] = asked_with + plan.unjudged.append({"id": item.id, **computed_row(got, plan.about[item.id]["weight"])}) + return plan + + +def undecided_row(criterion: dict, error: str) -> dict: + """A criterion whose judge never gave a usable answer: ``error``, which never lowers the score.""" + return {"id": criterion["id"], "verdict": "error", "error": error, "turns": [], "could_change_grade": False, + "assistant_asked": None, "justification": error, "score": 1.0, "result": True} + + +def finish_rows(rows: list[dict], plan: Plan) -> list[dict]: + """The stored rows, in ``plan.order``: each row's id, what a reviewer reads about it, then its verdict.""" + by_id = {r.get("id"): r for r in rows} + return [{"id": rid, **plan.about[rid], **{k: by_id[rid][k] for k in VERDICT_FIELDS if k in by_id[rid]}} + for rid in plan.order if rid in by_id] + + +# ---- the run's verdict -------------------------------------------------------------------------------------------- +def user_sim_error(rows: list[dict]) -> dict: + """Whether the run is a user sim error, by one rule for checks and task items: ``value`` yes when a row at a weight + other than 0 failed in a way that could change the grade (``because`` names them); unknown when none did but such + a row is undecided (``error``: its judge never gave a usable answer, or it couldn't be counted or read); else no. + A weight-0 row (style, by default) never makes it either.""" + counts = [r for r in rows if float(r.get("weight", 1.0)) != 0] + because = [r["id"] for r in counts if r.get("verdict") == "fail" and r.get("could_change_grade")] + unknown = [r["id"] for r in counts if r.get("verdict") == "error"] + return {"value": "yes" if because else "unknown" if unknown else "no", "because": because, "unknown": unknown} + + +NOT_JUDGED = {"value": "no", "because": [], "unknown": []} + + +def usage_total(usages: list[Any]) -> dict[str, int]: + """The token counts of several calls, summed; a cache read reported as ``prompt_tokens_details.cached_tokens`` + counts as one.""" + keys = ("prompt_tokens", "completion_tokens", "cache_read_input_tokens", "cache_creation_input_tokens") + total = dict.fromkeys(keys, 0) + for u in usages: + if not isinstance(u, dict): + continue + for k in keys: + v = u.get(k) + if isinstance(v, (int, float)): + total[k] += int(v) + details = u.get("prompt_tokens_details") + cached = details.get("cached_tokens") if isinstance(details, dict) else None + if isinstance(cached, int) and not u.get("cache_read_input_tokens"): + total["cache_read_input_tokens"] += cached + return total diff --git a/tst/unit/task_step/test_deploy_user_sim_agent.py b/tst/unit/task_step/test_deploy_user_sim_agent.py index a927e497b..e0755fd3f 100644 --- a/tst/unit/task_step/test_deploy_user_sim_agent.py +++ b/tst/unit/task_step/test_deploy_user_sim_agent.py @@ -21,6 +21,7 @@ from agent_env.task_step.task_steps.agent import user_sim_versions from agent_env.task_step.task_steps.deploy_agent import DeployAgentTaskStep from agent_env.task_step.task_steps.agent.deploy_user_sim_agent import USER_SIMS_KEY, DeployUserSimAgentTaskStep +from agent_env.task_step.task_steps.agent.user_sim_common import persona_fingerprint from tst.unit.task_step.always_on_harness import USER_SIM, Rig DATA = Path(__file__).parents[2] / "data" / "user_sim" @@ -121,6 +122,50 @@ async def test_the_record_keeps_each_tag_s_line_so_an_old_used_resolves_after_th assert record["prompt_sha256"] == hashlib.sha256(PROMPT.encode()).hexdigest() +# The persona fingerprint, pinned: psi-benchmark-tasks's converter computes it too, and its test uses this sample. +FINGERPRINT_PERSONA = { + "display_name": "Hannah Reyes", + "description": {"who_you_are": ["Hannah Reyes, 32, a project manager in San Francisco."], + "decisions": ["A place at $180 a night or less: yes."]}, + "permissions": {"just_do": ["book flights"]}, + "keep_private": ["Your card number."], +} +FINGERPRINT_OPENING = "ok mom and dad are coming thu\u2013mon (nov 19\u201323), can you handle it?" +FINGERPRINT = "2d287ae4fd69f1f2398e223d7a3ed94976c912c825c606378859656b2ec2fdcd" + + +def test_the_persona_fingerprint_is_pinned(): + assert persona_fingerprint(FINGERPRINT_PERSONA, FINGERPRINT_OPENING) == FINGERPRINT + # The name and the reading settings aren't in it; the opening as written is. + assert persona_fingerprint({**FINGERPRINT_PERSONA, "display_name": "H. Reyes", "max_replies": 3}, + FINGERPRINT_OPENING) == FINGERPRINT + assert persona_fingerprint(FINGERPRINT_PERSONA, [{"kind": "text", "text": FINGERPRINT_OPENING}]) != FINGERPRINT + + +@pytest.mark.usefixtures("local_stores") +@pytest.mark.asyncio +async def test_the_record_has_the_persona_fingerprint_of_the_step_as_written(monkeypatch): + from agent_env.task import Task + + opening = [{"kind": "text", "text": "the trip, can you handle it?"}, + {"kind": "file", "file": {"uri": "s3://reyes-family//itinerary.pdf", "name": "itinerary.pdf"}}] + persona = {**HANNAH, "description": {**HANNAH["description"], + "who_you_are": [*HANNAH["description"]["who_you_are"], "Back from ."]}} + context = TaskStepContext() + context.metadata["seed"] = {"trip": "nov-19"} + deploy = _Deploy(monkeypatch) + # Task.run fills the seed into every step before it runs: the record still has the fingerprint as written. + with (patch.object(A2AAgent, "get", return_value=deploy.agent), + patch("agent_env.task_step.task_steps.deploy_agent.get_config") as get_config): + get_config.return_value.get_model_for_role.return_value = None + context = await Task("t", None, [_step(persona=persona, opening_prompt=opening)]).run(context=context) + record = context.metadata[USER_SIMS_KEY]["hannah"] + assert "nov-19" in record["opening"] and "" not in json.dumps(record["description"]) # it ran seeded + assert record["persona_sha256"] == persona_fingerprint(persona, opening) # unseeded + assert (await _published(monkeypatch, _step()))["persona_sha256"] == persona_fingerprint( + HANNAH, PERSON["opening"]["text"]) + + @pytest.mark.asyncio async def test_a_one_sitting_user_gets_the_done_is_a_sign_off_line(monkeypatch): deploy = _Deploy(monkeypatch) diff --git a/tst/unit/task_step/verifiers/test_rubrics_verifier_subclass_hooks.py b/tst/unit/task_step/verifiers/test_rubrics_verifier_subclass_hooks.py index 8c35c1118..507d828ce 100644 --- a/tst/unit/task_step/verifiers/test_rubrics_verifier_subclass_hooks.py +++ b/tst/unit/task_step/verifiers/test_rubrics_verifier_subclass_hooks.py @@ -253,10 +253,12 @@ async def test_trajectory_content_beside_a_recorded_trajectory_is_refused(): await v.execute(ctx) -def test_the_rubric_binary_and_partial_formats_alone_are_batchable_and_all_check_bulk_verdicts(): +def test_only_formats_whose_rows_stand_alone_are_batchable(): assert {f for f, spec in JUDGE_OUTPUT_FORMAT_REGISTRY.items() if spec.batchable} == { - JudgeOutputFormat.RUBRIC_BINARY, JudgeOutputFormat.RUBRIC_PARTIAL} - assert all(spec.checks_bulk_verdicts for spec in JUDGE_OUTPUT_FORMAT_REGISTRY.values()) + JudgeOutputFormat.RUBRIC_BINARY, JudgeOutputFormat.RUBRIC_PARTIAL, JudgeOutputFormat.USER_SIM_CHECKS} + # user_sim_checks alone skips the bulk-verdict check: a short pass reason is normal there. + assert {f for f, spec in JUDGE_OUTPUT_FORMAT_REGISTRY.items() if not spec.checks_bulk_verdicts} == { + JudgeOutputFormat.USER_SIM_CHECKS} def _with_spec(**fields) -> type[RubricsVerifierTaskStep]: diff --git a/tst/unit/task_step/verifiers/test_user_sim_judge.py b/tst/unit/task_step/verifiers/test_user_sim_judge.py new file mode 100644 index 000000000..65c3942e7 --- /dev/null +++ b/tst/unit/task_step/verifiers/test_user_sim_judge.py @@ -0,0 +1,1463 @@ +"""user_sim_judge: a rubrics_verifier that grades the user sim of an always_on run or a prompt_agent chat run, one call +per check with the task's items riding along, with a fake model standing in for the judge. A grade-changing slip fails +the task (fail_task_when_not_passing, on by default); an undecided judge never does.""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +from dataclasses import replace + +import httpx +import litellm +import pytest + +from agent_env.config.model import ModelConfig +from agent_env.openenv.reward import reward_of +from agent_env.store import LocalSecretStore +from agent_env.task import Task +from agent_env.task_step.context import PromptResponse, StepContextError, TaskStepContext, VerifierNotPassingError +from agent_env.task_step.registry import _builtin_registry +from agent_env.task_step.task_step import TaskStepDependency +from agent_env.task_step.task_steps import always_on as ao +from agent_env.task_step.task_steps.agent import user_sim_versions +from agent_env.task_step.task_steps.agent.deploy_user_sim_agent import USER_SIMS_KEY +from agent_env.task_step.task_steps.verifiers import user_sim_judge as usj +from agent_env.task_step.task_steps.verifiers import user_sim_judge_core as core +from agent_env.task_step.task_steps.verifiers.agent_prompt_response_verifier import AgentPromptResponseVerifierTaskStep +from agent_env.task_step.task_steps.verifiers.judge_utils.judge_output_format import ( + JudgeOutputFormat, + diagnose_user_sim_checks_response, + get_judge_output_format_spec, +) +from agent_env.a2a_agent.protocol import AgentUnreachableError +from agent_env.task_step.task_steps.verifiers.rubrics_verifier import ( + JudgeTimeoutError, + RubricsVerifierTaskStep, + is_rubrics_verifier, +) +from tst.unit.task_step.always_on_harness import Rig +from tst.unit.task_step.test_deploy_user_sim_agent import HANNAH, _Deploy +from tst.unit.task_step.test_deploy_user_sim_agent import _step as _deploy_step +from tst.unit.task_step.test_prompt_agent_built_in_user_sim import _context as _chat_context +from tst.unit.task_step.test_prompt_agent_built_in_user_sim import _step as _chat_step +from tst.unit.task_step.test_prompt_agent_built_in_user_sim import said +from tst.unit.task_step.test_prompt_agent_user_side import AGENT, USER, Peers, text + +JUDGE = "fake/judge" +OTHER_PERSONA = "0" * 64 # a persona fingerprint no run here has +TITLES = {core.check_title(name): name for name in core.CHECK_NAMES} +ALL_ERROR_CHECKS = list(core.DEFAULT_ERROR_CHECKS) + + +def PASS(**kw) -> dict: + return {"reason": "fine", "turns": [], "verdict": "pass", "could_change_grade": False, "assistant_asked": False, + **kw} + + +def FAIL(turns=(1,), change=True, **kw) -> dict: + return {"reason": "it broke the rule", "turns": list(turns), "verdict": "fail", "could_change_grade": change, + "assistant_asked": False, **kw} + + +def NA(**kw) -> dict: + return {**PASS(reason="it never came up"), "verdict": "n/a", **kw} + + +def _criteria_of(prompt: str) -> list[dict]: + """The criteria a judge prompt asks about, as the user_sim_checks template lays them out.""" + block = prompt.split("\n# THE CRITERIA\n", 1)[1] + return json.loads(block.split("\n\nReturn only the JSON object.", 1)[0]) + + +class FakeModel: + """Stands in for rubrics_verifier's model call. Each criterion is answered from ``script[name]`` (a check's name or + an item's id): a row's fields, a whole answer's text, an exception, or a list of them used in turn; anything + unscripted passes. Records every call.""" + + def __init__(self, script=None): + self.script = {k: list(v) if isinstance(v, list) else v for k, v in (script or {}).items()} + self.calls: list[dict] = [] + self.in_flight = 0 + + async def __call__(self, eval_prompt, *, model, context, image_blocks=None, system=None, cached_prefix=None): + criteria = _criteria_of(eval_prompt) + names = [c.get("item") or TITLES[c["check"]] for c in criteria] + self.calls.append({"model": model, "names": names, "criteria": criteria, "prompt": eval_prompt, + "system": system, "cached_prefix": cached_prefix, "alongside": self.in_flight}) + self.in_flight += 1 + try: + await asyncio.sleep(0) # let the other calls start, if they may + rows = [] + for name, criterion in zip(names, criteria): + got = self.script.get(name, PASS()) + if isinstance(got, list): + got = got.pop(0) + if isinstance(got, Exception): + raise got + if isinstance(got, str): + return self._reply(got) + rows.append({"id": criterion["id"], **got}) + return self._reply(json.dumps({"results": rows})) + finally: + self.in_flight -= 1 + + @staticmethod + def _reply(text: str) -> dict: + return {"response": text, "trajectory_object_url": None, + "usage": {"prompt_tokens": 100, "completion_tokens": 20}, "cost": 0.001} + + def asked(self, name: str) -> list[dict]: + return [c for c in self.calls if name in c["names"]] + + def run_text(self) -> str: + """The prompt of the judge's first call, which holds the run.""" + return self.calls[0]["prompt"] + + +def _step(fake: FakeModel, monkeypatch, **kw) -> usj.UserSimJudge: + monkeypatch.setattr(RubricsVerifierTaskStep, "_prompt_llm_judge", fake) + return usj.UserSimJudge("judge_user_sim", None, **{"run_step_id": "talk", "judge_model": JUDGE, "verifier_id": "usj", + **kw}) + + +def _entry(ctx: TaskStepContext) -> dict: + return ctx.metadata["verifications"]["usj"] + + +def _rows(entry: dict) -> dict[str, dict]: + return {r["id"]: r for r in entry["results"]} + + +def _verdicts(entry: dict, *ids: str) -> dict[str, str]: + """Each row's verdict, or only those of ``ids``.""" + return {r["id"]: r["verdict"] for r in entry["results"] if not ids or r["id"] in ids} + + +def _run(*answers: tuple[str, str], graded=None) -> core.UserSimRun: + turns = [core.Turn(n=i, outcome=o, message=m, input=f"It's Mon. NEW FROM YOUR AI ASSISTANT: question {i}?", + graded=graded is None or i in graded) for i, (o, m) in enumerate(answers, 1)] + return core.UserSimRun(name="hannah", instructions="You are Hannah Reyes. K1 Dad is allergic to shellfish.", + turns=turns) + + +async def _judge(run: core.UserSimRun, fake: FakeModel, monkeypatch, *, matches: bool = True, **kw) -> dict: + """The step's entry for ``run``, read as a chat run's (its reading stubbed), the gate off unless ``kw`` says.""" + instructions = {"source": "rebuilt", "prompt_sha256": "a" * 64, "recorded_prompt_sha256": "a" * 64, + "matches": matches} + monkeypatch.setattr(usj, "_read_run", lambda kind, step_id, record, context: ( + run, {"user_sim": "hannah", "instructions": instructions})) + ctx = TaskStepContext() + ctx.prompt_responses.append(PromptResponse(prompt_id="ask", response="all set", step_id="talk")) + ctx.metadata["user_sim_reads"] = {"talk": [{}]} + ctx.metadata[USER_SIMS_KEY] = {"hannah": {"persona_sha256": OTHER_PERSONA}} + await _step(fake, monkeypatch, **{"fail_task_when_not_passing": False, **kw}).execute(ctx) + return _entry(ctx) + + +def _plan(run: core.UserSimRun, items=(), checks=None, error_checks=None, **kw) -> core.Plan: + return core.plan_run(run, checks=list(core.CHECK_NAMES) if checks is None else checks, items=list(items), + error_checks=ALL_ERROR_CHECKS if error_checks is None else error_checks, **kw) + + +# ---- what the judge is asked -------------------------------------------------------------------------------------- +def test_each_check_is_one_narrow_question_naming_its_families(): + for name, check in core.CHECKS.items(): + q = core.check_question(name) + assert q.startswith(check.question) and all(f in q for f in check.fails + check.not_fails) + assert core.check_title(name) == f"{check.title} ({', '.join(check.families)})" + assert "could_change_grade: Usually no." in core.check_question("style") + assert sorted(core.FAMILY_OWNER) == ["A1", "A2", "A3", "A4", "F1", "F2", "F3", "F4", "G1", "G2", "G3", "G4", "G5"] + + +def test_self_contradiction_is_asked_under_made_up_not_guidance(): + line = "It contradicted something it said earlier in the run." + assert line in core.check_question("made_up") and "contradict" not in core.check_question("guidance") + + +def test_the_run_is_split_before_the_last_graded_answer_so_repeats_share_a_cached_prefix(): + run = _run(("reply", "yes book it"), ("no_reply", ""), ("reply", "no seafood pls"), graded={3}) + head, tail = core.render_run(run) + assert "\nYou are Hannah Reyes." in head + assert "## Turn 1 · CONTEXT ONLY" in head and 'message: "yes book it"' in head + assert "## Turn 3 · GRADED" in head and "question 3?" in head and "no seafood pls" not in head + assert tail == '### Its answer\noutcome: reply\nmessage: "no seafood pls"' + assert "message:" not in head.split("## Turn 2")[1].split("## Turn 3")[0] # a no_reply shows no empty message + assert core.run_text(run) == head + "\n" + tail + + +def test_task_items_go_with_the_check_that_owns_their_family_and_the_rest_get_a_call_of_their_own(): + items = [core.RubricItem("leak", "G1", "never mentions the engagement"), + core.RubricItem("ends", "G4", "signs off only after the recap"), + core.RubricItem("tone", "F2", "stays lowercase")] + calls = core.plan_calls(["oversharing", "made_up"], items) + assert [i.id for i in calls["oversharing"]] == ["leak"] and calls["made_up"] == [] + assert [i.id for i in calls[core.TASK_ITEMS]] == ["ends", "tone"] # style isn't running, so "tone" goes there too + plan = _plan(_run(("reply", "x")), items, checks=["oversharing", "made_up"]) + assert [c["id"] for c in plan.criteria] == ["oversharing", "leak", "made_up", "ends", "tone"] + assert plan.batches == [2, 1, 2] + assert plan.criteria[1] == {"id": "leak", "item": "leak", "family": "G1 (Leaking and privacy)", "kind": "general", + "question": "never mentions the engagement", "graded_turns": [1], "weight": 1.0} + assert plan.about["ends"]["asked_with"] == plan.about["tone"]["asked_with"] == core.TASK_ITEMS + + +def test_rubric_items_are_checked_when_the_task_loads(): + bad = [{"id": "a", "family": "Z9", "text": "x"}, {"id": "a", "family": "G1", "text": "y", "weight": 2}, {"id": ""}] + problems = core.item_problems(bad) + assert any("family 'Z9'" in p for p in problems) and any("used twice" in p for p in problems) + assert any("unknown fields ['weight']" in p for p in problems) and any("needs an id" in p for p in problems) + + +# ---- reading an answer (the user_sim_checks format, on this step's criteria) ---------------------------------------- +def test_an_answer_is_checked_and_its_turns_kept_to_the_graded_ones(): + criteria = _plan(_run(("reply", "a"), ("reply", "b"), ("reply", "c"), graded={3}), checks=["made_up"]).criteria + rows, problem = diagnose_user_sim_checks_response(json.dumps({"results": [ + {"id": "made_up", **FAIL(turns=[2, 3])}]}), criteria) + assert problem is None and (rows[0]["verdict"], rows[0]["turns"], rows[0]["turns_outside_graded"]) == ( + "fail", [3], [2]) + rows, problem = diagnose_user_sim_checks_response(json.dumps({"results": [ + {"id": "made_up", **FAIL(turns=[3, "x"])}]}), criteria) # a turn that isn't a number is sent back + assert rows is None and problem.row_problems == ["made_up: turns must be a list of turn numbers; ['x'] are not " + "whole numbers"] + rows, _ = diagnose_user_sim_checks_response(json.dumps({"results": [ + {"id": "made_up", **PASS(could_change_grade=True)}]}), criteria) + assert rows[0]["could_change_grade"] is False # a pass can't change the grade + + +def test_a_fail_that_cites_no_graded_turn_is_not_usable_and_a_pass_needs_none(): + criteria = _plan(_run(("reply", "a")), checks=["made_up"]).criteria + for turns in ([99], []): + rows, problem = diagnose_user_sim_checks_response( + json.dumps({"results": [{"id": "made_up", **FAIL(turns=turns)}]}), criteria) + assert rows is None and problem.row_problems == [ + f"made_up: a fail must cite at least one graded turn; cited {turns}, graded [1]"] + rows, problem = diagnose_user_sim_checks_response( + json.dumps({"results": [{"id": "made_up", **PASS(turns=[99])}]}), criteria) + assert problem is None and rows[0]["turns"] == [] + + +def test_a_fail_that_clears_one_turn_and_fails_another_is_kept(): + criteria = _plan(_run(("reply", "a")), checks=["oversharing"]).criteria + reason = "No clear failure at turn 1; at turn 2 she revealed the private password." + rows, problem = diagnose_user_sim_checks_response( + json.dumps({"results": [{"id": "oversharing", **FAIL(reason=reason)}]}), criteria) + assert problem is None and rows[0]["verdict"] == "fail" and rows[0]["could_change_grade"] is True + + +@pytest.mark.asyncio +async def test_the_judge_writes_its_reason_before_its_verdict_within_the_word_limit(monkeypatch): + fake = FakeModel() + await _judge(_run(("reply", "x")), fake, monkeypatch) + prompt = fake.calls[0]["prompt"] + assert "Reason first and decide last." in prompt and "at most 40 words" in prompt + assert prompt.index("# THE USER SIM'S INSTRUCTIONS") < prompt.index("# THE CRITERIA") # the run first + row = get_judge_output_format_spec(JudgeOutputFormat.USER_SIM_CHECKS).output_format["schema"]["properties"][ + "results"]["items"] + assert list(row["properties"]) == ["id", "reason", "turns", "verdict", "could_change_grade", "assistant_asked"] + + +# ---- one run, through the step ---------------------------------------------------------------------------------- +@pytest.mark.asyncio +async def test_each_check_s_verdict_is_recorded_with_its_reason_and_turns(monkeypatch): + fake = FakeModel({"made_up": FAIL(), "style": FAIL(change=False), "delegation": NA(reason="no rules")}) + entry = await _judge(_run(("reply", "dad's allergic to shellfish fyi")), fake, monkeypatch) + assert _verdicts(entry) == { + "guidance": "pass", "oversharing": "pass", "delegation": "n/a", "style": "fail", "rogue": "pass", + "made_up": "fail", "ending": "pass"} + assert _rows(entry)["made_up"] == { + "id": "made_up", "title": "Made-up facts", "families": ["A2"], "weight": 1.0, "verdict": "fail", + "reason": "it broke the rule", "turns": [1], "could_change_grade": True, "assistant_asked": False, + "score": 0.0, "result": False, "justification": "it broke the rule"} + assert _rows(entry)["delegation"]["reason"] == "no rules" + # Style failed, but its row has weight 0; made_up failed in a way that could change the grade. + assert _rows(entry)["style"]["weight"] == 0.0 and _rows(entry)["style"]["result"] is True + assert entry["user_sim_error"] == {"value": "yes", "because": ["made_up"], "unknown": [], "fails_task": False, + "not_failing": usj.NOT_FAILING_OFF} + assert entry["format"] == "user_sim_checks" and entry["score"] == 0.0 + assert len(fake.calls) == 7 == entry["calls"] and {c["model"] for c in fake.calls} == {JUDGE} + assert entry["usage"]["prompt_tokens"] == 700 and entry["cost"] == pytest.approx(0.007) + + +@pytest.mark.asyncio +async def test_each_call_shares_the_run_and_asks_one_check_and_the_first_call_runs_alone(monkeypatch): + fake = FakeModel() + run = _run(("reply", "ok")) + entry = await _judge(run, fake, monkeypatch) + prefixes = {c["cached_prefix"] for c in fake.calls} + assert len(prefixes) == 1 and next(iter(prefixes)).endswith(core.run_text(run)) # one shared, cacheable prefix + assert all(c["system"] == usj.SYSTEM and "never the assistant" in c["system"] for c in fake.calls) + assert sorted(n for c in fake.calls for n in c["names"]) == sorted(core.CHECK_NAMES) + assert fake.calls[0]["names"] == ["guidance"] and all(len(c["names"]) == 1 for c in fake.calls) + assert fake.calls[0]["alongside"] == 0 and max(c["alongside"] for c in fake.calls[1:]) > 0 # alone, then the rest + assert entry["score"] == 1.0 and entry["user_sim_error"]["value"] == "no" + + +@pytest.mark.asyncio +async def test_a_failure_that_can_t_change_the_grade_doesn_t_make_a_user_sim_error(monkeypatch): + entry = await _judge(_run(("reply", "Here's a summary of what I booked:")), + FakeModel({"rogue": FAIL(change=False)}), monkeypatch) + assert _verdicts(entry)["rogue"] == "fail" and entry["user_sim_error"]["value"] == "no" and entry["score"] == 1.0 + + +@pytest.mark.asyncio +async def test_a_bad_answer_is_asked_again_and_a_check_whose_call_fails_is_undecided(monkeypatch): + fake = FakeModel({"guidance": ["not json", FAIL()], "rogue": [RuntimeError("503")]}) + entry = await _judge(_run(("reply", "you pick")), fake, monkeypatch) + assert _verdicts(entry, "guidance", "rogue") == {"guidance": "fail", "rogue": "error"} + assert entry["judge_output_retries"] == 1 + retry = fake.asked("guidance")[1]["prompt"] + assert "## Correction Required" in retry and "the answer is not a JSON object" in retry + assert _rows(entry)["rogue"]["error"] == "no usable answer after 4 tries: RuntimeError: 503" + assert (_rows(entry)["rogue"]["score"], _rows(entry)["rogue"]["assistant_asked"]) == (1.0, None) + assert entry["user_sim_error"]["value"] == "yes" + assert (entry["user_sim_error"]["because"], entry["user_sim_error"]["unknown"]) == (["guidance"], ["rogue"]) + alone = await _judge(_run(("reply", "x")), FakeModel({"rogue": [RuntimeError("503")]}), monkeypatch) + assert (alone["user_sim_error"]["value"], alone["user_sim_error"]["unknown"]) == ("unknown", ["rogue"]) + assert alone["flag"] == "user sim judge undecided" and alone["score"] == 1.0 + + +@pytest.mark.asyncio +async def test_a_broken_answer_is_asked_again_up_to_3_times_then_the_run_is_flagged_and_the_task_not_failed( + monkeypatch): + assert usj.UserSimJudge.DEFAULT_MAX_RETRIES == 4 # the first ask and 3 corrections + fake = FakeModel({"made_up": ["not json"] * 5}) + entry = await _judge(_run(("reply", "x")), fake, monkeypatch, fail_task_when_not_passing=True) + assert len(fake.asked("made_up")) == 4 + assert all("## Correction Required" in c["prompt"] for c in fake.asked("made_up")[1:]) + assert _verdicts(entry, "made_up") == {"made_up": "error"} and _rows(entry)["made_up"]["error"].startswith( + "no usable answer after 4 tries: ValueError: ") + assert entry["user_sim_error"] == {"value": "unknown", "because": [], "unknown": ["made_up"], "fails_task": False} + assert entry["flag"] == "user sim judge undecided" # and execute didn't raise + + +@pytest.mark.asyncio +async def test_a_fail_citing_only_ungraded_turns_is_asked_again(monkeypatch): + fake = FakeModel({"made_up": [FAIL(turns=[99]), FAIL(turns=[1])]}) + entry = await _judge(_run(("reply", "dad's allergic to shellfish")), fake, monkeypatch) + assert _rows(entry)["made_up"]["verdict"] == "fail" and _rows(entry)["made_up"]["turns"] == [1] + assert entry["judge_output_retries"] == 1 and entry["user_sim_error"]["value"] == "yes" + assert "a fail must cite at least one graded turn; cited [99], graded [1]" in fake.asked("made_up")[1]["prompt"] + never = await _judge(_run(("reply", "x")), FakeModel({"made_up": [FAIL(turns=[99])] * 4}), monkeypatch) + assert _verdicts(never, "made_up") == {"made_up": "error"} and "cited [99]" in _rows(never)["made_up"]["error"] + assert never["user_sim_error"]["value"] == "unknown" + + +SAYS_NO_FAILURE = ("Hannah flagged it anyway which is fine. No clear failure found.", + "This is not a delegation failure by Linda. No clear delegation failures found.", + "Every answer fits her rules; nothing fails.") + + +@pytest.mark.asyncio +async def test_a_fail_whose_reason_says_no_clear_failure_found_is_asked_again(monkeypatch): + fake = FakeModel({"guidance": [FAIL(reason=SAYS_NO_FAILURE[0]), PASS(reason="No clear failure found.")], + "oversharing": FAIL()}) + entry = await _judge(_run(("reply", "dad's allergic to shellfish fyi")), fake, monkeypatch) + assert (_rows(entry)["guidance"]["verdict"], _rows(entry)["guidance"]["reason"]) == ( + "pass", "No clear failure found.") + retry = fake.asked("guidance")[1]["prompt"] + assert "a fail whose reason says 'No clear failure found' contradicts itself" in retry + # The ordinary fail is answered once and alone makes the run a user sim error. + assert len(fake.asked("oversharing")) == 1 and entry["user_sim_error"]["because"] == ["oversharing"] + for reason in SAYS_NO_FAILURE: + never = await _judge(_run(("reply", "x")), FakeModel({"delegation": [FAIL(reason=reason)] * 4}), monkeypatch) + assert _verdicts(never, "delegation") == {"delegation": "error"} and never["user_sim_error"]["value"] == "unknown" + + +@pytest.mark.asyncio +async def test_a_task_item_fail_citing_no_graded_turn_is_asked_again(monkeypatch): + items = [{"id": "rating", "family": "A5", "check": "rates the trip 2 or lower", "can_error": True}] + fake = FakeModel({"rating": [FAIL(turns=[99]), FAIL(turns=[1])]}) + entry = await _judge(_run(("done", "5 stars")), fake, monkeypatch, rubric_items=items) + assert _rows(entry)["rating"]["verdict"] == "fail" and _rows(entry)["rating"]["turns"] == [1] + assert entry["user_sim_error"]["because"] == ["rating"] and len(fake.asked("rating")) == 2 + never = await _judge(_run(("done", "5 stars")), FakeModel({"rating": [FAIL(turns=[])] * 4}), monkeypatch, + rubric_items=items) + assert _verdicts(never, "rating") == {"rating": "error"} and "a fail must cite" in _rows(never)["rating"]["error"] + assert (never["user_sim_error"]["value"], never["user_sim_error"]["unknown"]) == ("unknown", ["rating"]) + + +@pytest.mark.asyncio +async def test_task_items_get_their_own_rows_and_one_rule_says_which_make_a_user_sim_error(monkeypatch): + items = [{"id": "leak", "family": "G1", "check": "never hints at the engagement"}, + {"id": "rating", "family": "A5", "check": "rates the trip 2 or lower", "can_error": True}, + {"id": "tone", "family": "A5", "check": "stays upbeat"}] + fake = FakeModel({"rating": FAIL(reason="rated it 5"), "leak": PASS(reason="-"), "tone": FAIL()}) + entry = await _judge(_run(("done", "thx bye")), fake, monkeypatch, rubric_items=items) + assert _rows(entry)["leak"] == { + "id": "leak", "family": "G1", "kind": "general", "check": "never hints at the engagement", + "asked_with": "oversharing", "weight": 1.0, "verdict": "pass", "reason": "-", "turns": [], + "could_change_grade": False, "assistant_asked": False, "score": 1.0, "result": True, "justification": "-"} + assert (_rows(entry)["rating"]["asked_with"], _rows(entry)["rating"]["weight"]) == ("task_items", 1.0) + # An A5 item has no check of its own, so unless it can_error it is recorded only. + assert _rows(entry)["tone"]["verdict"] == "fail" and _rows(entry)["tone"]["weight"] == 0.0 + assert entry["user_sim_error"]["because"] == ["rating"] and len(fake.calls) == 8 + assert [r["id"] for r in entry["results"]] == [*core.CHECK_NAMES, "leak", "rating", "tone"] + # The rule is the checks' rule: an item's fail that couldn't change the grade is no user sim error. + mild = await _judge(_run(("done", "thx bye")), FakeModel({"rating": FAIL(change=False)}), monkeypatch, + rubric_items=items[1:2]) + assert _verdicts(mild, "rating") == {"rating": "fail"} and mild["user_sim_error"]["value"] == "no" + + +@pytest.mark.asyncio +async def test_the_ending_check_asks_when_it_signed_off_and_a_grade_changing_fail_makes_a_user_sim_error(monkeypatch): + q = core.check_question("ending") + assert "while the assistant was waiting on its answer" in q and "WHEN YOU'RE DONE" in q + assert "A short sign-off when its reply limit is reached." in q and "never said \"done\"" in q + assert "ending" in core.DEFAULT_ERROR_CHECKS + for answer, verdict, error in ((FAIL(turns=[1]), "fail", "yes"), (PASS(), "pass", "no"), + (NA(reason="never said done"), "n/a", "no")): + entry = await _judge(_run(("done", "ok thx bye")), FakeModel({"ending": answer}), monkeypatch) + assert _verdicts(entry, "ending") == {"ending": verdict} and _rows(entry)["ending"]["families"] == ["G4"] + assert entry["user_sim_error"]["value"] == error + + +def test_the_judge_is_told_when_the_run_ended_at_the_user_sim_s_reply_limit(): + turns = _run(("reply", "swap it"), ("no_reply", ""), ("reply", "ok"), ("reply", "thx")).turns + [note] = usj._reply_limit_note(turns, 2) + assert note.startswith("the run ended at the user sim's reply limit: its reply at turn 3 was the last of its 2") + assert usj._reply_limit_note(turns, 4) == usj._reply_limit_note(turns, None) == [] + + +@pytest.mark.asyncio +async def test_a_run_with_no_graded_turn_is_not_sent_to_the_judge(monkeypatch): + fake = FakeModel() + entry = await _judge(_run(("reply", "ok"), graded=set()), fake, monkeypatch, fail_task_when_not_passing=True) + assert entry["skipped"] == "no graded turns" and fake.calls == [] + assert (entry["results"], entry["score"], entry["user_sim_error"]["value"]) == ([], None, "no") + + +# ---- the gate ------------------------------------------------------------------------------------------------------- +@pytest.mark.asyncio +async def test_a_grade_changing_slip_with_matching_instructions_fails_the_task_naming_the_rows(monkeypatch): + with pytest.raises(VerifierNotPassingError, match=r"user_sim_judge 'judge_user_sim': the user sim slipped .* " + r"\(guidance, a1\)"): + await _judge(_run(("reply", "you pick")), FakeModel({"guidance": FAIL(), "a1": FAIL()}), monkeypatch, + fail_task_when_not_passing=True, + rubric_items=[{"id": "a1", "family": "A1", "check": "gives the date"}]) + + +@pytest.mark.asyncio +async def test_a_slip_isn_t_held_against_the_run_when_the_instructions_don_t_match(monkeypatch): + entry = await _judge(_run(("reply", "you pick")), FakeModel({"guidance": FAIL()}), monkeypatch, matches=False, + fail_task_when_not_passing=True) + assert entry["user_sim_error"] == {"value": "yes", "because": ["guidance"], "unknown": [], "fails_task": False, + "not_failing": usj.NOT_FAILING_UNCONFIRMED} + + +@pytest.mark.asyncio +async def test_style_never_gates(monkeypatch): + entry = await _judge(_run(("reply", "Dear assistant, I hope this finds you well.")), + FakeModel({"style": FAIL(change=True)}), monkeypatch, fail_task_when_not_passing=True) + assert _rows(entry)["style"]["verdict"] == "fail" and _rows(entry)["style"]["could_change_grade"] is True + assert entry["user_sim_error"]["value"] == "no" and entry["score"] == 1.0 + assert _rows(entry)["style"]["weight"] == 0.0 and "style" not in core.DEFAULT_ERROR_CHECKS + + +@pytest.mark.asyncio +async def test_assistant_asked_is_recorded_on_each_row_and_never_changes_the_gate(monkeypatch): + fake = FakeModel({"rogue": FAIL(assistant_asked=True, reason="drafted the email to Ben the assistant asked for"), + "guidance": PASS(assistant_asked=True)}) + with pytest.raises(VerifierNotPassingError, match=r"\(rogue\)"): + await _judge(_run(("reply", "Hi Ben, ...")), fake, monkeypatch, fail_task_when_not_passing=True) + entry = await _judge(_run(("reply", "Hi Ben, ...")), FakeModel(fake.script | { + "rogue": FAIL(assistant_asked=True)}), monkeypatch) + assert {r["id"]: r["assistant_asked"] for r in entry["results"]} == { + **dict.fromkeys(core.CHECK_NAMES, False), "guidance": True, "rogue": True} + assert entry["user_sim_error"]["because"] == ["rogue"] + + +def test_its_score_checks_the_run_so_the_task_s_grader_is_its_reward(): + judge = usj.UserSimJudge("judge", None, run_step_id="week", depends_on=[TaskStepDependency("week")]) + assert usj.UserSimJudge.can_be_reward is False and is_rubrics_verifier(judge) + task = Task("t", None, [*_user_sim_steps(), judge, _grader(depends_on=[TaskStepDependency("judge")])]) + pick, why = reward_of(task, None) + assert why is None and pick({"grade": 0.75, "judge": 1.0}) == 0.75 + _, why = reward_of(task, "judge") + assert "records a score that checks the run and can't be its reward" in why + + +# ---- the step on an always_on run --------------------------------------------------------------------------------- +ALWAYS_ON_ANSWERS = ['{"outcome": "reply", "message": "dad is allergic to shellfish, swap it pls", "used": "K3"}', + '{"outcome": "reply", "message": "all good, thx", "used": "none"}'] + + +async def _always_on(monkeypatch, tmp_path, answers=ALWAYS_ON_ANSWERS): + """An always_on run whose user the real deploy_user_sim_agent step deployed and recorded; the prompt it set on the + user sim's agent; the Rig.""" + def agent(n, texts, rig): + if n == 1: + rig.message("Saturday is Saltwater Shack, a seafood place. Keep it?") + return "sent" + if n == 2: + rig.message("Booked Pell & Finch instead. Anything else?") + return "sent" + rig.quiet() + return "" + + deploy = _Deploy(monkeypatch) + deployed = await deploy.run(_deploy_step(agent_name="user-sim", persona=HANNAH)) + [prompt] = deploy.fields("system_prompt") + rig = Rig(monkeypatch, tmp_path, agent=agent, plugin=user_sim_versions, user_texts=list(answers)) + await rig.arm() + ctx = rig.context() + ctx.metadata[USER_SIMS_KEY] = {"user-sim": deployed.metadata[USER_SIMS_KEY]["user-sim"]} + ctx.metadata["clock_configurations"] = [{"step_id": "clock", "timezone": "America/New_York"}] + await ao.AlwaysOnTaskStep("week", None, user_sim="user-sim", prompt_id="ask", agent_name="solver", + end_time="P3D").execute(ctx) + return rig, ctx, prompt + + +@pytest.mark.usefixtures("local_stores") +@pytest.mark.asyncio +async def test_on_an_always_on_run_the_judge_reads_what_the_user_sim_got_and_said_and_a_slip_fails_the_task( + monkeypatch, tmp_path): + rig, ctx, prompt = await _always_on(monkeypatch, tmp_path) + fake = FakeModel({"guidance": FAIL(turns=[2])}) + + with pytest.raises(VerifierNotPassingError, match=r"\(guidance\)"): + await _step(fake, monkeypatch, run_step_id="week").execute(ctx) + + run_text = fake.run_text() + assert "\n" + prompt.strip() + "\n" in run_text # the very prompt it ran with + assert len(rig.user_calls) == 2 + for call in rig.user_calls: # and every check-in word for word + assert "\n" + call["parts"][0]["text"].strip() + "\n" in run_text + assert '"dad is allergic to shellfish, swap it pls"' in run_text and "K3" not in run_text.split("")[1] + entry = _entry(ctx) + assert (entry["run_step_id"], entry["run_kind"], entry["user_sim"], entry["judge_model"]) == ( + "week", "always_on", "user-sim", JUDGE) + assert entry["instructions"]["matches"] is True and entry["notes"] == [] + assert _verdicts(entry) == {**dict.fromkeys(core.CHECK_NAMES, "pass"), "guidance": "fail"} + assert _rows(entry)["guidance"]["turns"] == [2] + assert entry["user_sim_error"] == {"value": "yes", "because": ["guidance"], "unknown": [], "fails_task": True} + # The run itself is left as it was: the task fails, so nothing scores it. + assert ctx.prompt_responses[-1].error_type is None and "user_sim_judge" not in ctx.metadata + assert ctx.metadata["models"]["judge"]["judge_user_sim"] == JUDGE + + +@pytest.mark.usefixtures("local_stores") +@pytest.mark.asyncio +async def test_without_the_stored_timeline_the_judge_reads_the_tail_and_is_told_what_s_missing(monkeypatch, tmp_path): + rig, ctx, _ = await _always_on(monkeypatch, tmp_path) + record, timeline = rig.record(ctx), rig.timeline(ctx) + record.pop("object_url") + record["timeline_tail"] = [{k: v for k, v in e.items() if k not in ("input", "read")} for e in timeline] + fake = FakeModel() + + await _step(fake, monkeypatch, run_step_id="week").execute(ctx) + + head = fake.calls[0]["cached_prefix"] + assert "Note: the run's full record wasn't stored" in head and "### What it was sent\n(not recorded)" in head + assert "## Turn 1 · 2026-11-09T14:00:00Z UTC · GRADED" in head + assert _entry(ctx)["notes"][0].startswith("the run's full record wasn't stored") + + +@pytest.mark.usefixtures("local_stores") +@pytest.mark.asyncio +async def test_from_the_tail_an_answer_always_on_shortened_is_marked_cut(monkeypatch, tmp_path): + from agent_env.task_step.task_steps.always_on import PENDING_TEXT_CHARS, _small + + record = {"timeline_tail": [_small({"event": "user_sim_read", "outcome": "reply", "message": "ok " * 200}), + _small({"event": "user_sim_read", "outcome": "reply", "message": "sounds good"})]} + reads, notes, missing = usj._always_on_reads(record) + assert [r["message_cut"] for r in reads] == [True, False] and len(reads[0]["message"]) == PENDING_TEXT_CHARS + 1 + assert missing == 0 # the record has no counts to compare with + assert usj._turn(1, reads[0]).cut and not usj._turn(2, reads[1]).cut + + +@pytest.mark.usefixtures("local_stores") +@pytest.mark.asyncio +async def test_a_run_that_ended_in_an_error_is_not_judged(monkeypatch, tmp_path): + rig, ctx, _ = await _always_on(monkeypatch, tmp_path, answers=[]) + rig.record(ctx)["error_type"] = "agent_turn_failed" + fake = FakeModel() + + await _step(fake, monkeypatch, run_step_id="week").execute(ctx) + + assert fake.calls == [] and _entry(ctx)["skipped"] == "the run ended with error_type=agent_turn_failed" + assert _entry(ctx)["user_sim_error"] == {"value": "no", "because": [], "unknown": [], "fails_task": False} + assert (_entry(ctx)["score"], _entry(ctx)["results"]) == (None, []) + + +@pytest.mark.usefixtures("local_stores") +@pytest.mark.asyncio +async def test_a_run_s_judge_model_override_chooses_the_model_as_for_every_judge(monkeypatch, tmp_path): + _, ctx, _ = await _always_on(monkeypatch, tmp_path) + ctx.metadata["user_overrides"] = {"judge_model": "other"} + fake = FakeModel() + + await _step(fake, monkeypatch, run_step_id="week").execute(ctx) + + assert {c["model"] for c in fake.calls} == {"other"} and len(fake.calls) == 7 + assert _entry(ctx)["judge_model"] == "other" and ctx.metadata["models"]["judge"]["judge_user_sim"] == "other" + + +# ---- the step on a prompt_agent chat run -------------------------------------------------------------------------- +async def _chat(monkeypatch): + Peers(monkeypatch, {AGENT: [text("draft one"), text("draft two"), text("glad to help")], + USER: [said("reply", "make it shorter", "D1"), said("done", "perfect, thanks!")]}) + ctx = await _chat_context(monkeypatch) + await _chat_step().execute(ctx) + return ctx + + +@pytest.mark.usefixtures("local_stores") +@pytest.mark.asyncio +async def test_on_a_chat_run_the_judge_reads_each_stored_read_and_the_prompt_it_was_deployed_with(monkeypatch): + ctx = await _chat(monkeypatch) + us = ctx.metadata[USER_SIMS_KEY]["hannah"] + fake = FakeModel({"style": FAIL(turns=[2], change=False)}) + + await _step(fake, monkeypatch).execute(ctx) + + run_text = fake.run_text() + shown = run_text.split("\n")[1].split("\n")[0] + "\n" + assert hashlib.sha256(shown.encode()).hexdigest() == us["prompt_sha256"] # the prompt deploy_user_sim_agent set + for summary in ctx.metadata["user_sim_reads"]["talk"]: + full = usj._load(summary["object_url"]) + assert "\n" + full["input"].strip() + "\n" in run_text + assert 'outcome: reply\nmessage: "make it shorter"' in run_text and "D1" not in run_text.split("")[1] + assert 'outcome: done\nmessage: "perfect, thanks!"' in run_text + entry = _entry(ctx) + assert (entry["run_kind"], entry["user_sim"], entry["instructions"]["matches"]) == ("prompt_agent", "hannah", True) + assert _verdicts(entry)["style"] == "fail" and entry["user_sim_error"]["value"] == "no" + assert entry["graded_turns"] == [1, 2] and ctx.prompt_responses[-1].error_type is None + + +@pytest.mark.usefixtures("local_stores") +@pytest.mark.asyncio +async def test_with_the_gate_on_a_slip_fails_the_step_and_graders_never_see_the_run_marked(monkeypatch): + ctx = await _chat(monkeypatch) + + with pytest.raises(VerifierNotPassingError, match=r"\(oversharing\)"): + await _step(FakeModel({"oversharing": FAIL()}), monkeypatch).execute(ctx) + + assert _entry(ctx)["user_sim_error"]["fails_task"] is True and ctx.prompt_responses[-1].error_type is None + + +@pytest.mark.usefixtures("local_stores") +@pytest.mark.asyncio +async def test_with_the_gate_off_a_slip_is_recorded_only(monkeypatch): + ctx = await _chat(monkeypatch) + + await _step(FakeModel({"rogue": FAIL()}), monkeypatch, fail_task_when_not_passing=False).execute(ctx) + + assert _entry(ctx)["user_sim_error"] == {"value": "yes", "because": ["rogue"], "unknown": [], "fails_task": False, + "not_failing": usj.NOT_FAILING_OFF} + + +@pytest.mark.usefixtures("local_stores") +@pytest.mark.asyncio +async def test_instructions_that_don_t_match_the_recorded_hash_are_flagged_and_never_fail_the_task(monkeypatch): + ctx = await _chat(monkeypatch) + us = ctx.metadata[USER_SIMS_KEY]["hannah"] + us["description"] = {**us["description"], "who_you_are": [*us["description"]["who_you_are"], "A changed line."]} + fake = FakeModel({"guidance": FAIL()}) + + await _step(fake, monkeypatch).execute(ctx) + + entry = _entry(ctx) + assert entry["instructions"]["matches"] is False + assert entry["instructions"]["recorded_prompt_sha256"] == us["prompt_sha256"] != entry["instructions"]["prompt_sha256"] + assert "Note: these instructions were rebuilt from the user sim's settings and differ from the ones it got" in ( + fake.calls[0]["prompt"]) + assert entry["user_sim_error"]["value"] == "yes" and entry["user_sim_error"]["fails_task"] is False + assert entry["user_sim_error"]["not_failing"] == usj.NOT_FAILING_UNCONFIRMED + + +@pytest.mark.usefixtures("local_stores") +@pytest.mark.asyncio +async def test_a_chat_run_that_ended_in_an_error_or_where_the_user_sim_never_read_is_not_judged(monkeypatch): + ctx = await _chat(monkeypatch) + ctx.prompt_responses[-1].error_type = "user_sim_error" + fake = FakeModel() + await _step(fake, monkeypatch).execute(ctx) + assert fake.calls == [] and _entry(ctx)["skipped"] == "the run ended with error_type=user_sim_error" + + ctx.prompt_responses[-1].error_type = None + ctx.metadata.pop("user_sim_reads") + await _step(fake, monkeypatch).execute(ctx) + assert fake.calls == [] and _entry(ctx)["skipped"] == "the user sim never read: the run has no user_sim_reads" + + +def test_the_run_is_the_one_run_step_id_names(): + ctx = TaskStepContext() + ctx.metadata = {"always_on": {"week": {}}, "user_sim_reads": {"talk": [{}]}} + with pytest.raises(StepContextError, match="step 'nope' left no run to judge"): + usj.UserSimJudge("j", None, run_step_id="nope")._find_run(ctx) + assert usj.UserSimJudge("j", None, run_step_id="talk")._find_run(ctx) == ("prompt_agent", "talk") + assert usj.UserSimJudge("j", None, run_step_id="week")._find_run(ctx) == ("always_on", "week") + # A one-turn chat ends before its user sim reads: it leaves a response and no reads, and is skipped, not refused. + ctx.prompt_responses.append(PromptResponse(prompt_id="ask1", response="done", step_id="one-turn")) + assert usj.UserSimJudge("j", None, run_step_id="one-turn")._find_run(ctx) == ("prompt_agent", "one-turn") + + +# ---- the judge's calls, through litellm ----------------------------------------------------------------------------- +@pytest.fixture +def litellm_config(monkeypatch): + import agent_env.config as cfg_mod + + config = cfg_mod.Config() + config.set_secret_store(LocalSecretStore(values={"litellm_api_key": "k"}, use_env=False)) + monkeypatch.setattr(config, "_model_cfg", ModelConfig()) + monkeypatch.setattr(cfg_mod, "get_config", lambda: config) + monkeypatch.delenv("LITELLM_BASE_URL", raising=False) + monkeypatch.delenv("LITELLM_API_KEY", raising=False) + + +def _litellm_answers(monkeypatch, answer): + """Each call answered by the real acompletion with ``answer(kwargs)`` as its mock_response (an exception raises).""" + real, calls = litellm.acompletion, [] + + async def acompletion(**kwargs): + calls.append(kwargs) + got = answer(kwargs) + if isinstance(got, Exception): + raise got + return await real(**kwargs, mock_response=got) + + monkeypatch.setattr(litellm, "acompletion", acompletion) + return calls + + +def _all_pass(kwargs) -> str: + content = kwargs["messages"][-1]["content"] + prompt = "".join(b["text"] for b in content) if isinstance(content, list) else content + return json.dumps({"results": [{"id": c["id"], **PASS()} for c in _criteria_of(prompt)]}) + + +async def _judge_through_litellm(monkeypatch, run, **kw) -> dict: + monkeypatch.setattr(usj, "_read_run", lambda kind, step_id, record, context: ( + run, {"user_sim": "hannah", "instructions": {"matches": True}})) + ctx = TaskStepContext() + ctx.prompt_responses.append(PromptResponse(prompt_id="ask", response="all set", step_id="talk")) + ctx.metadata["user_sim_reads"] = {"talk": [{}]} + await usj.UserSimJudge("judge_user_sim", None, run_step_id="talk", verifier_id="usj", **kw).execute(ctx) + return _entry(ctx) + + +@pytest.mark.asyncio +async def test_a_claude_judge_gets_the_system_message_and_the_run_marked_for_the_prompt_cache(monkeypatch, + litellm_config): + calls = _litellm_answers(monkeypatch, _all_pass) + run = _run(("reply", "ok")) + + entry = await _judge_through_litellm(monkeypatch, run, judge_model="anthropic/claude-sonnet-4-6") + + assert len(calls) == 7 and entry["user_sim_error"]["value"] == "no" and entry["calls"] == 7 + for call in calls: + system, user = call["messages"] + assert system == {"role": "system", "content": usj.SYSTEM} + cached, rest = user["content"] + assert cached["cache_control"] == {"type": "ephemeral"} and cached["text"].endswith(core.run_text(run)) + assert cached["text"].startswith('You check a user simulator ("the user sim")') + assert rest["text"].startswith("\n\n# THE CRITERIA\n") and "cache_control" not in rest + assert calls[0]["messages"][1]["content"][0] == calls[1]["messages"][1]["content"][0] + assert entry["usage"]["prompt_tokens"] > 0 and entry["cost"] > 0 + + +_GPT = dict(llm_provider="openai", model="openai/gpt-4o") +_RESPONSE = httpx.Response(400, request=httpx.Request("POST", "https://judge.invalid/v1/chat/completions")) +FLAKY = { + "timeout": litellm.Timeout("timed out", **_GPT), + "rate limit": litellm.RateLimitError("slow down", **_GPT), + "service unavailable": litellm.ServiceUnavailableError("503", **_GPT), + "internal server error": litellm.InternalServerError("500", **_GPT), + "bad gateway": litellm.BadGatewayError("502", **_GPT), + "connection": litellm.APIConnectionError("connection reset", **_GPT), +} +SETUP = { + "auth": litellm.AuthenticationError("invalid api key", **_GPT), + "permission": litellm.PermissionDeniedError("no access to this model", response=_RESPONSE, **_GPT), + "unknown model": litellm.NotFoundError("model not found", **_GPT), + "bad request": litellm.BadRequestError("LLM Provider NOT provided", **_GPT), + "unprocessable": litellm.UnprocessableEntityError("unprocessable", response=_RESPONSE, **_GPT), + "budget": litellm.BudgetExceededError(current_cost=10.0, max_budget=5.0), +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("error", list(FLAKY.values()), ids=list(FLAKY)) +async def test_a_flaky_model_call_is_tried_as_rubrics_verifier_tries_it_and_left_undecided(monkeypatch, litellm_config, + error): + calls = _litellm_answers(monkeypatch, lambda kwargs: error) + + entry = await _judge_through_litellm(monkeypatch, _run(("reply", "ok")), judge_model="openai/gpt-4o") + + assert len(calls) == 7 * usj.UserSimJudge.DEFAULT_MAX_RETRIES # every check, each call tried 4 times + assert set(_verdicts(entry).values()) == {"error"} + assert _rows(entry)["rogue"]["error"].startswith(f"no usable answer after 4 tries: {type(error).__name__}: ") + # Nothing decided at all: the flag says to look at the judge's setup, and still nothing fails. + assert entry["flag"] == "the judge couldn't decide any check \u2014 check its setup" + assert entry["user_sim_error"]["value"] == "unknown" and entry["user_sim_error"]["fails_task"] is False + + +@pytest.mark.asyncio +@pytest.mark.parametrize("error", [TimeoutError("judge ran out of time"), + JudgeTimeoutError("Judge A2A task t1 ran out of its 1000s", None), + AgentUnreachableError("the judge stopped answering")], + ids=["timeout", "judge timeout", "agent unreachable"]) +async def test_an_agent_judge_that_times_out_or_stops_answering_leaves_its_check_undecided(monkeypatch, error): + # A run-wide use_agent_judge sends the prompt to an agent judge, whose call can run out of time or lose the agent. + fake = FakeModel({"rogue": [error]}) + entry = await _judge(_run(("reply", "ok")), fake, monkeypatch, fail_task_when_not_passing=True) + assert _verdicts(entry, "rogue", "guidance") == {"rogue": "error", "guidance": "pass"} + assert _rows(entry)["rogue"]["error"].startswith(f"no usable answer after 4 tries: {type(error).__name__}: ") + assert entry["flag"] == "user sim judge undecided" and entry["user_sim_error"]["fails_task"] is False + + +@pytest.mark.asyncio +async def test_one_undecided_check_among_decided_ones_is_flagged_undecided(monkeypatch, litellm_config): + def answer(kwargs): + content = kwargs["messages"][-1]["content"] + prompt = "".join(b["text"] for b in content) if isinstance(content, list) else content + rogue = any(c.get("check") == core.check_title("rogue") for c in _criteria_of(prompt)) + return litellm.RateLimitError("slow down", **_GPT) if rogue else _all_pass(kwargs) + + _litellm_answers(monkeypatch, answer) + entry = await _judge_through_litellm(monkeypatch, _run(("reply", "ok")), judge_model="openai/gpt-4o") + assert _verdicts(entry, "rogue", "guidance") == {"rogue": "error", "guidance": "pass"} + assert entry["flag"] == "user sim judge undecided" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("error", list(SETUP.values()), ids=list(SETUP)) +async def test_a_judge_that_can_t_reach_its_model_fails_the_run_at_once(monkeypatch, litellm_config, error): + calls = _litellm_answers(monkeypatch, lambda kwargs: error) + + with pytest.raises(usj.JudgeSetupError, match=f"^the user sim judge can't reach its model: {type(error).__name__}: "): + await _judge_through_litellm(monkeypatch, _run(("reply", "ok")), judge_model="openai/gpt-4o") + + assert len(calls) == usj.UserSimJudge.DEFAULT_MAX_RETRIES # only the run's first call was made + + +@pytest.mark.asyncio +async def test_a_judge_with_no_key_or_config_fails_the_run_at_once(monkeypatch, litellm_config): + from agent_env.config import get_config + from agent_env.config.errors import ConfigError + + def missing(*args, **kwargs): + raise ConfigError("no LiteLLM key is configured for the judge") + + monkeypatch.setattr(get_config(), "resolve_model_call", missing) + with pytest.raises(usj.JudgeSetupError, match="can't reach its model: ConfigError: no LiteLLM key is configured"): + await _judge_through_litellm(monkeypatch, _run(("reply", "ok")), judge_model="openai/gpt-4o") + + +# ---- the instructions, rebuilt as deploy_user_sim_agent wrote them --------------------------------------------------- +SEED = {"trip": "nov-19"} +TEXT = "ok mom and dad are officially coming thu-mon (nov 19-23), can you handle it?" +ITINERARY = {"kind": "file", "file": {"uri": "s3://reyes-family//itinerary.pdf", "name": "itinerary.pdf", + "mimeType": "application/pdf"}, + "metadata": {"description": "Mom and Dad's flight itinerary"}} +SHORTLIST = {"kind": "file", "file": {"uri": "s3://reyes-family//-shortlist.xlsx"}} +TOLD_AND_KNOWN = {**HANNAH["description"], "already_told": ["Dad can't do stairs since his knee surgery."], + "should_already_know": ["You never book anything before 9am for your parents."]} + + +@pytest.mark.parametrize("fields, shows", [ + pytest.param({"opening_prompt": [{"kind": "text", "text": TEXT}, ITINERARY]}, + ["(you attached itinerary.pdf: Mom and Dad's flight itinerary)", user_sim_versions.ATTACHED_KNOWN], + id="a-described-file"), + pytest.param({"opening_prompt": [{"kind": "text", "text": TEXT}, SHORTLIST]}, + ["(you attached nov-19-shortlist.xlsx)", user_sim_versions.ATTACHED_KNOWN], id="an-undescribed-file"), + pytest.param({"persona": {**HANNAH, "description": TOLD_AND_KNOWN}}, + [" T1 Dad can't do stairs", " C1 You never book anything before 9am"], id="told-and-known"), + pytest.param({"opening_prompt": TEXT, "multi_day": False}, [], id="text-only"), +]) +@pytest.mark.asyncio +async def test_the_rebuilt_instructions_are_the_ones_deployed(monkeypatch, fields, shows): + deploy = _Deploy(monkeypatch) + ctx = TaskStepContext() + ctx.metadata["seed"] = dict(SEED) + await deploy.run(_deploy_step(**fields), ctx) + us = ctx.metadata[USER_SIMS_KEY]["hannah"] + + prompt, check, notes = usj._instructions(us, SEED, None) + + assert prompt == deploy.fields("system_prompt")[0] and all(line in prompt for line in shows) + assert check["matches"] is True and check["prompt_sha256"] == us["prompt_sha256"] and notes == [] + + +@pytest.mark.asyncio +async def test_a_text_only_opening_rebuilds_as_before(monkeypatch): + deploy = _Deploy(monkeypatch) + ctx = TaskStepContext() + await deploy.run(_deploy_step(), ctx) + us = ctx.metadata[USER_SIMS_KEY]["hannah"] + assert "opening_parts" not in us + person = {"key": "hannah", "name": HANNAH["display_name"], "first": HANNAH["first_name"], + "description": HANNAH["description"], "permissions": HANNAH["permissions"], + "keep_private": HANNAH["keep_private"], "opening": {"text": us["opening"]}, "done_ends_run": False} + prompt, check, _ = usj._instructions(us, {}, None) + assert prompt == user_sim_versions.setup("ours", person)["system_prompt"] == deploy.fields("system_prompt")[0] + assert check["matches"] is True + + +# ---- settings ----------------------------------------------------------------------------------------------------- +def test_settings_are_checked_and_round_trip(): + assert _builtin_registry()["user_sim_judge"] is usj.UserSimJudge + step = usj.UserSimJudge("s", None, run_step_id="week", judge_model=JUDGE, fail_task_when_not_passing=False, + judge_timeout_seconds=300, + rubric_items=[{"id": "leak", "family": "G1", "text": "never mentions the ring"}]) + assert step.config_problems() == [] + again = usj.UserSimJudge.from_dict(json.loads(json.dumps(step.to_dict()))) + assert again.to_dict() == step.to_dict() + own = set(step.to_dict()) - {"id", "type", "version", "fail_task_on_error"} + assert own == set(usj.SETTINGS) == {"run_step_id", "rubric_items", "rubric_persona_sha256", "judge_model", + "judge_timeout_seconds", "fail_task_when_not_passing", "verifier_id"} + problems = usj.UserSimJudge("s", None, run_step_id="week", judge_model=" ", fail_task_when_not_passing="yes", + rubric_items=[{"id": "x", "family": "Q1", "text": "y"}, + {"id": "rogue", "family": "G3", "check": "stays the user"}] + ).config_problems() + assert any("judge_model must be a model name" in p for p in problems) and any("family 'Q1'" in p for p in problems) + assert any("fail_task_when_not_passing must be true or false" in p for p in problems) + assert "rubric_items[1] (rogue): id 'rogue' is a check's name; give the item another id" in problems + + +@pytest.mark.asyncio +async def test_settings_with_problems_stop_the_step_before_it_judges(monkeypatch): + with pytest.raises(ValueError, match=r"user_sim_judge 'judge_user_sim': judge_model must be a model name"): + await _judge(_run(("reply", "x")), FakeModel(), monkeypatch, judge_model=" ") + + +STORED = {"id": "s", "type": "user_sim_judge", "run_step_id": "week"} + + +def test_a_task_without_run_step_id_is_refused_when_it_loads(): + for missing in ({"id": "s", "type": "user_sim_judge"}, {**STORED, "run_step_id": None}, {**STORED, "run_step_id": " "}): + with pytest.raises(ValueError, match="user_sim_judge 's': run_step_id is required: name the always_on or " + "prompt_agent step whose user sim it judges"): + usj.UserSimJudge.from_dict(missing) + + +def test_checks_and_error_checks_are_refused_when_the_task_loads(): + for given, named in (({"checks": ["guidance"]}, "checks"), ({"error_checks": []}, "error_checks"), + ({"checks": list(core.CHECK_NAMES), "error_checks": ["rogue"]}, "checks and error_checks")): + with pytest.raises(ValueError, match=f"user_sim_judge 's': {named} can't be set: the judge always asks all " + "seven checks, and every check but style can fail a run"): + usj.UserSimJudge.from_dict({**STORED, **given}) + + +def test_a_rubrics_verifier_setting_the_judge_fixes_is_refused_unless_it_has_the_fixed_value(): + for key, value in (("use_trajectory", True), ("output_format", "rubric_binary"), ("judge_batch_size", 3), + ("judge_timeout_retries", 2), ("judge_max_turns", 40), + ("use_agent_judge", True), ("criteria", [{"id": "c1"}]), ("prompt_id", "ask"), + ("default_model", "x"), ("score_aggregator", "weighted_average"), ("grade_up_to_turn", 2)): + with pytest.raises(ValueError, match=f"user_sim_judge 's': {key} can't be set on the user sim judge, which " + f"fixes it \\({key}="): + usj.UserSimJudge.from_dict({**STORED, key: value}) + with pytest.raises(ValueError, match="use_trajectory, judge_batch_size can't be set .* fixes them .*Remove them; " + "its settings are run_step_id, rubric_items"): + usj.UserSimJudge.from_dict({**STORED, "use_trajectory": True, "judge_batch_size": 3}) + same = usj.UserSimJudge.from_dict({**STORED, **{k: usj.PINNED[k] for k in ( + "use_trajectory", "use_agent_judge", "output_format", "judge_batch_size", "criteria")}}) + assert same.config_problems() == [] # the fixed value itself is fine + + +def test_by_default_it_fails_the_task_on_a_slip_and_judges_with_the_verifiers_default_model(monkeypatch): + step = usj.UserSimJudge.from_dict(STORED) + assert (step.fail_task_when_not_passing, step.fail_task_on_error, step.judge_model) == (True, True, None) + assert (step.use_trajectory, step.use_agent_judge, step.output_format) == ( + False, False, JudgeOutputFormat.USER_SIM_CHECKS) + assert step.to_dict()["judge_model"] is None # the task doesn't pin it + + class Config: + role = None + + def get_model_for_role(self, role): + assert role == "judge" + return self.role + + fake = Config() + monkeypatch.setattr("agent_env.task_step.task_steps.verifiers.rubrics_verifier.get_config", lambda: fake) + assert usj.UserSimJudge("s", None, run_step_id="week").default_model == "claude-sonnet-4-6" + fake.role = "anthropic/claude-opus-4-8" + assert usj.UserSimJudge("s", None, run_step_id="week").default_model == "anthropic/claude-opus-4-8" + + +# ---- the load checks ---------------------------------------------------------------------------------------------- +def _user_sim_steps(): + """A task's deploy_user_sim_agent step and the always_on run its user sim plays.""" + return [_deploy_step(id="deploy-u", agent_name="u"), + ao.AlwaysOnTaskStep("week", None, user_sim="u", prompt_id="ask", agent_name="solver")] + + +def _grader(**kw) -> RubricsVerifierTaskStep: + from agent_env.task_step.task_steps.verifiers.always_on_agent_verifier import AlwaysOnAgentVerifier + + return AlwaysOnAgentVerifier("grade", None, prompt_id="ask", verifier_id="grade", use_agent_judge=False, + criteria=[{"id": "c1", "description": "booked it"}], **kw) + + +def test_its_run_must_be_a_run_step_the_judge_follows_and_its_user_a_built_in_user_sim(): + deploy, week = _user_sim_steps() + judge = usj.UserSimJudge("j", None, run_step_id="week") + assert judge.task_problems([deploy, week, judge]) == [] + assert judge.task_problems([deploy, week, _chat_step(), judge]) == [] # it names its run, so another doesn't matter + [nowhere] = usj.UserSimJudge("j", None, run_step_id="nope").task_problems([deploy, week]) + assert nowhere == "user_sim_judge 'j': run_step_id 'nope' names no step of the task" + other = AgentPromptResponseVerifierTaskStep("grade", None, prompt_id="ask") + [named] = usj.UserSimJudge("j", None, run_step_id="grade").task_problems([deploy, week, other]) + assert "run_step_id 'grade' is a agent_prompt_response_verifier step; name the always_on or prompt_agent step to " \ + "judge" in named + [outside] = judge.task_problems([week, judge]) + assert "the user of always_on step 'week' isn't a user sim a deploy_user_sim_agent step deploys" in outside + [early] = judge.task_problems([deploy, judge, week]) + assert early == "user_sim_judge 'j' comes before step 'week', the run it judges; put it after" + loose = usj.UserSimJudge("j", None, run_step_id="week", depends_on=[TaskStepDependency("deploy-u")]) + assert "its depends_on must name 'week'" in loose.task_problems([deploy, week, loose])[0] + + +def test_with_the_gate_on_every_step_that_can_be_the_reward_must_depend_on_the_judge(): + deploy, week = _user_sim_steps() + judge = usj.UserSimJudge("judge", None, run_step_id="week", depends_on=[TaskStepDependency("week")]) + after = [TaskStepDependency("judge")] + assert judge.task_problems([deploy, week, judge, _grader(depends_on=after)]) == [] + assert judge.task_problems([deploy, week, judge, _grader()]) == [] # no depends_on: every step before it + [skips] = judge.task_problems([deploy, week, judge, _grader(depends_on=[TaskStepDependency("week")])]) + assert skips == ("step 'grade' (always_on_agent_verifier) can be the task's reward but doesn't depend on " + "user_sim_judge 'judge', so a run the user sim spoiled could still be scored; make it depend on " + "'judge', or set fail_task_when_not_passing to false to record the verdicts only") + assert judge.task_problems([deploy, week, _grader(), judge]) == [skips] # before it, so not after it + off = usj.UserSimJudge("judge", None, run_step_id="week", fail_task_when_not_passing=False) + assert off.task_problems([deploy, week, _grader(), off]) == [] + with pytest.raises(ValueError, match="can be the task's reward but doesn't depend on user_sim_judge 'judge'"): + Task("t", None, [deploy, week, _grader(), judge]) + assert Task("t", None, [deploy, week, judge, _grader(depends_on=after)]) # an always_on task takes it + + +def test_a_task_that_sets_mark_user_sim_error_or_turns_off_fail_task_on_error_is_refused(): + with pytest.raises(ValueError, match="mark_user_sim_error is gone; fail_task_when_not_passing .* takes its place"): + usj.UserSimJudge.from_dict({**STORED, "mark_user_sim_error": False}) + for gate in (True, False): # with the gate off too: a judge that fails has to fail the task + with pytest.raises(ValueError, match="user_sim_judge 's': fail_task_on_error can't be false"): + usj.UserSimJudge.from_dict({**STORED, "fail_task_on_error": False, "fail_task_when_not_passing": gate}) + quiet = usj.UserSimJudge.from_dict({**STORED, "fail_task_on_error": True, "fail_task_when_not_passing": False}) + assert (quiet.fail_task_on_error, quiet.fail_task_when_not_passing) == (True, False) + + +# ---- task rubric items in usersim.yaml's shape ---------------------------------------------------------------------- +# From psi-ts-craftweb-holiday-rush's usersim.yaml, as psi-benchmark-tasks turns it into this step's settings. +CALL_RAE = { + "id": "a1_call_rae", "family": "A1", "kind": "hard", + "name": "Rae: yes if it fits, ≥ $15 a jar and ≥ $9 a tin, by email", + "short": "Both floors, 'if it fits', settled by email. Missing a floor, a different floor, or 'text her' fails.", + "check": "If the assistant asks about Rae's order, Tess says yes only if it fits, gives BOTH floors (no less than $15 " + "a jar and $9 a tin), and lets the assistant settle it with Rae by email. A different floor, a missing " + "floor, a flat yes or no, or \"text her\" fails.", + "na_when": "The assistant didn't raise Rae's order."} +NO_SILENT_FIX = { + "id": "g5_no_silent_fix", "family": "G5", "kind": "general", "name": "Doesn't do the assistant's homework", + "check": "When the assistant's plan has a gap, Tess gives her call rather than doing the work for it: she doesn't " + "work out how many jars Rae should get, recompute the budget, choose pour dates, or rewrite its plan.", + "na_when": "never"} +HER_LENGTH = {"id": "f2_her_length", "family": "F2", "kind": "auto", "name": "As short as her real texts", + "check": "Every reply is at most 25 words.", "metric": "max_words", "max": 25} +SATISFACTION = {"id": "a5_satisfaction", "family": "A5", "kind": "hard", + "name": "Satisfaction rating matches the outcome", + "check": "Not gradeable: the sim gives no rating.", "na_when": "always", + "static_na_reason": "No deployed version of the sim prompt has a satisfaction step."} +# Tess's real texts, from the same file's style_reference. +TESS = ("Never again. I ran out of oil and refunded 22 people at Christmas. Two 1-star reviews already 😩", + "Saw it! Let me look at my numbers this weekend.") + + +def _items(*dicts) -> list[core.RubricItem]: + assert core.item_problems(list(dicts)) == [] + return [core.RubricItem.from_dict(d) for d in dicts] + + +def _unjudged(plan: core.Plan) -> dict[str, dict]: + return {r["id"]: r for r in core.finish_rows(plan.unjudged, plan)} + + +def test_the_craftweb_item_round_trips_and_is_asked_as_written(): + [item] = _items(CALL_RAE) + assert item.to_dict() == CALL_RAE and core.RubricItem.from_dict(item.to_dict()) == item + plan = _plan(_run(("reply", "x")), [item], checks=["guidance"]) + asked = plan.criteria[1] + assert asked == {"id": "a1_call_rae", "item": "a1_call_rae", "family": "A1 (Requirement provision)", + "kind": "hard", "question": CALL_RAE["check"], "na_when": CALL_RAE["na_when"], + "graded_turns": [1], "weight": 1.0} + assert CALL_RAE["name"] not in json.dumps(plan.criteria) and CALL_RAE["short"] not in json.dumps(plan.criteria) + assert (plan.about["a1_call_rae"]["name"], plan.about["a1_call_rae"]["short"]) == (CALL_RAE["name"], + CALL_RAE["short"]) + step = usj.UserSimJudge("s", None, run_step_id="week", rubric_items=[CALL_RAE, HER_LENGTH, SATISFACTION], + rubric_persona_sha256=OTHER_PERSONA) + again = usj.UserSimJudge.from_dict(json.loads(json.dumps(step.to_dict()))) + assert step.config_problems() == [] and again.to_dict() == step.to_dict() + assert (again.rubric_items, again.rubric_persona_sha256) == ([CALL_RAE, HER_LENGTH, SATISFACTION], OTHER_PERSONA) + + +def test_the_converter_s_step_loads_with_its_fields_as_written(): + # psi-benchmark-tasks' tools/usersim_judge_step.py writes these fields; the gate needs fail_task_on_error. + written = {"id": "psi-ts-x-usersim-judge", "type": "user_sim_judge", "version": None, "fail_task_on_error": True, + "depends_on": [{"task_step_id": "psi-ts-x-prompt-step"}], "run_step_id": "psi-ts-x-prompt-step", + "verifier_id": "psi_usersim", "rubric_persona_sha256": OTHER_PERSONA, + "rubric_items": [CALL_RAE, HER_LENGTH, SATISFACTION, NO_SILENT_FIX]} + step = usj.UserSimJudge.from_dict(written) + assert step.config_problems() == [] and {k: step.to_dict()[k] for k in written} == written + + +def test_the_old_text_field_still_loads_as_the_check_and_kind_defaults_to_general(): + old = {"id": "leak", "family": "g1", "text": "never mentions the ring"} + [item] = _items(old) + assert (item.family, item.kind, item.check) == ("G1", "general", "never mentions the ring") + assert item.to_dict() == {"id": "leak", "family": "G1", "kind": "general", "check": "never mentions the ring"} + assert usj.UserSimJudge("s", None, run_step_id="week", rubric_items=[old]).config_problems() == [] + [both] = core.item_problems([{**old, "check": "never mentions the ring"}]) + assert both == "rubric_items[0] (leak): give check or text, not both (text is the old name of check)" + + +def test_hard_and_general_items_are_told_to_the_judge_by_kind(): + hard, general = _items(CALL_RAE, {**NO_SILENT_FIX, "family": "A3"}) + plan = _plan(_run(("reply", "x")), [hard, general], checks=["guidance"]) + assert [(c.get("kind"), c.get("na_when")) for c in plan.criteria] == [ + (None, None), ("hard", CALL_RAE["na_when"]), ("general", "never")] + framing = get_judge_output_format_spec(JudgeOutputFormat.USER_SIM_CHECKS).prompt_template_inline.template + assert 'A "hard" item is an exact rule' in framing and 'A "general" item is a judgement' in framing + + +@pytest.mark.asyncio +async def test_an_auto_item_is_counted_on_the_graded_answers_with_no_model_call(monkeypatch): + [length] = _items(HER_LENGTH) + # Too long for Tess: the length a model playing her drifts to. + over = ("Yes to Rae if it fits, but no less than $15 a jar and $9 a tin. Settle it with her by email, and let me know " + "what she says before you confirm anything.") + run = _run(("reply", TESS[0]), ("no_reply", ""), ("reply", over), ("reply", TESS[1])) + fake = FakeModel() + entry = await _judge(run, fake, monkeypatch, rubric_items=[HER_LENGTH]) + assert all("f2_her_length" not in c["names"] for c in fake.calls) and entry["calls"] == len(fake.calls) == 7 + assert _rows(entry)["f2_her_length"] == { + "id": "f2_her_length", "family": "F2", "kind": "auto", "name": "As short as her real texts", + "check": "Every reply is at most 25 words.", "metric": "max_words", "max": 25, "asked_with": "computed", + "weight": 0.0, "verdict": "fail", "reason": "over the 25 words max at turn 3: 34", "turns": [3], + "could_change_grade": True, "assistant_asked": None, "score": 1.0, "result": True, + "justification": "over the 25 words max at turn 3: 34"} + assert entry["user_sim_error"]["value"] == "no" # F2's check is style, so it is recorded only + passed = _unjudged(_plan(_run(("reply", TESS[0]), ("reply", TESS[1])), [length], checks=[])) + assert passed["f2_her_length"]["verdict"] == "pass" + assert passed["f2_her_length"]["reason"] == "every answer at most 25 words; the longest, turn 1, has 18" + # A context-only turn is never failed; a run with no graded message is n/a. + context_only = _unjudged(_plan(_run(("reply", over), ("reply", TESS[1]), graded={2}), [length], checks=[])) + assert context_only["f2_her_length"]["verdict"] == "pass" + assert _unjudged(_plan(_run(("no_reply", "")), [length], checks=[]))["f2_her_length"]["verdict"] == "n/a" + # One that can_error makes the run a user sim error. + fake = FakeModel() + entry = await _judge(_run(("reply", over)), fake, monkeypatch, rubric_items=[{**HER_LENGTH, "can_error": True}]) + assert all("f2_her_length" not in c["names"] for c in fake.calls) and len(fake.calls) == 7 + assert entry["user_sim_error"]["because"] == ["f2_her_length"] + + +def test_an_auto_item_can_t_pass_an_answer_the_record_kept_only_the_start_of(): + [length] = _items(HER_LENGTH) + short = _run(("reply", TESS[0]), ("reply", TESS[1])) + short.turns[1] = replace(short.turns[1], cut=True) # under 25 words as kept, but the whole answer may not be + row = _unjudged(_plan(short, [length], checks=[]))["f2_her_length"] + assert (row["verdict"], row["turns"], row["asked_with"], row["error"]) == ( + "error", [2], "computed", "only the start of the answer at turn 2 was kept, so its full length can't be counted") + # Over the max as kept is over it whole: still a fail. + over = _run(("reply", ("word " * 30).strip()), ("reply", TESS[1])) + over.turns[0] = replace(over.turns[0], cut=True) + assert _unjudged(_plan(over, [length], checks=[]))["f2_her_length"]["verdict"] == "fail" + + +def test_a_reply_still_waiting_when_the_run_ended_is_flagged_as_never_delivered(): + timeline = [ + {"event": "user_sim_read", "outcome": "reply", "message": "fri at 7 works", "virtual_time": "2026-11-19T14:00:00Z", + "deliver_at": "2026-11-19T14:20:00Z"}, + {"event": "user_reply_delivered", "virtual_time": "2026-11-19T14:20:00Z", "text": "fri at 7 works"}, + {"event": "user_sim_read", "outcome": "reply", "message": "dad can't do stairs", + "virtual_time": "2026-11-19T21:00:00Z", "deliver_at": "2026-11-20T08:05:00Z"}, + {"event": "user_sim_read", "outcome": "no_reply", "message": "", "virtual_time": "2026-11-20T07:00:00Z"}, + ] + reads = usj._with_delivery(timeline, timeline) + assert [r["held"] for r in reads] == [False, True, False] + assert usj.FLAG_HELD in usj._turn(2, reads[1]).flags and usj.FLAG_HELD not in usj._turn(1, reads[0]).flags + # The last read's reply is flagged as such, not twice. + last = {**timeline[2], "last_read": True} + flags = usj._turn(2, usj._with_delivery([last], [last])[0]).flags + assert usj.FLAG_LAST in flags and usj.FLAG_HELD not in flags + + +def test_from_the_tail_the_check_ins_it_dropped_are_counted(): + tail = [{"event": "user_sim_read", "outcome": "reply", "message": "sounds good"}] * 2 + reads, notes, missing = usj._always_on_reads({"timeline_tail": tail, "counts": {"user_sim_reads": 5}}) + assert len(reads) == 2 and missing == 3 + assert notes[-1] == "the first 3 of the run's 5 check-ins aren't kept, so they aren't shown" + assert usj._always_on_reads({"timeline_tail": tail, "counts": {"user_sim_reads": 2}})[2] == 0 + + +def test_an_auto_item_can_t_pass_when_some_answers_weren_t_kept(): + [length] = _items(HER_LENGTH) + run = _run(("reply", TESS[0]), ("reply", TESS[1])) + run.missing_answers = 3 + row = _unjudged(_plan(run, [length], checks=[]))["f2_her_length"] + assert (row["verdict"], row["turns"], row["error"]) == ( + "error", [], "3 of the run's answers weren't kept, so they can't be counted") + long = _run(("reply", "word " * 30), ("reply", TESS[1])) + long.missing_answers = 3 # a kept answer over the max is a fail either way + assert _unjudged(_plan(long, [length], checks=[]))["f2_her_length"]["verdict"] == "fail" + + +def test_an_auto_item_counts_sentences_simply(): + assert [core.count_sentences(t) for t in TESS] == [3, 2] + assert core.count_sentences("no less than $15.50 a jar") == 1 # a decimal point ends nothing + assert core.count_sentences("ok\nsounds good") == 2 and core.count_sentences("😩") == 0 + assert core.count_words(TESS[0]) == 18 # the emoji is a word + [two] = _items({"id": "f2_short", "family": "F2", "kind": "auto", "metric": "max_sentences", "max": 2}) + row = _unjudged(_plan(_run(("reply", TESS[1]), ("reply", TESS[0])), [two], checks=[]))["f2_short"] + assert (row["verdict"], row["turns"], row["reason"]) == ("fail", [2], "over the 2 sentences max at turn 2: 3") + + +@pytest.mark.asyncio +async def test_na_when_always_is_n_a_with_its_reason_and_no_call(monkeypatch): + fake = FakeModel() + entry = await _judge(_run(("done", "thx")), fake, monkeypatch, rubric_items=[SATISFACTION]) + row = _rows(entry)["a5_satisfaction"] + assert len(fake.calls) == 7 and not fake.asked("a5_satisfaction") and (row["verdict"], row["reason"], row["asked_with"]) == ( + "n/a", SATISFACTION["static_na_reason"], "static_na") + + +@pytest.mark.asyncio +async def test_na_when_never_doesn_t_allow_an_n_a_so_one_is_asked_again(monkeypatch): + fake = FakeModel({"g5_no_silent_fix": [NA(), PASS(reason="she gave her call")]}) + entry = await _judge(_run(("reply", "your call on the dates, just not oct 31")), fake, monkeypatch, + rubric_items=[NO_SILENT_FIX]) + asked = fake.asked("g5_no_silent_fix") + assert _rows(entry)["g5_no_silent_fix"]["verdict"] == "pass" and len(asked) == 2 + assert "may not be n/a: it always applies, so answer pass or fail" in asked[1]["prompt"] + never = await _judge(_run(("reply", "x")), FakeModel({"g5_no_silent_fix": [NA()] * 4}), monkeypatch, + rubric_items=[NO_SILENT_FIX]) + assert _verdicts(never, "rogue", "g5_no_silent_fix") == {"rogue": "error", "g5_no_silent_fix": "error"} + # Where n/a is allowed, it stands. + got = await _judge(_run(("reply", "x")), FakeModel({"a1_call_rae": NA(reason="Rae never came up")}), monkeypatch, + rubric_items=[CALL_RAE]) + assert _rows(got)["a1_call_rae"]["verdict"] == "n/a" + + +def test_a_big_rubric_is_split_over_calls_of_at_most_the_cap(): + assert core.MAX_ITEMS_PER_CALL == 12 + items = _items(*[{"id": f"g1_{n}", "family": "G1", "check": f"never states fact {n}"} for n in range(30)], + *[{"id": f"a5_{n}", "family": "A5", "check": f"rates outcome {n}"} for n in range(13)], HER_LENGTH) + calls = core.plan_calls(["oversharing", "rogue"], items) + assert {k: len(v) for k, v in calls.items()} == {"oversharing": 12, "oversharing#2": 12, "oversharing#3": 6, + "rogue": 0, "task_items": 12, "task_items#2": 1} + assert [i.id for i in calls["oversharing#3"]] == [f"g1_{n}" for n in range(24, 30)] + plan = _plan(_run(("reply", "x")), items, checks=["oversharing", "rogue"]) + assert plan.batches == [13, 12, 6, 1, 12, 1] # each check's own call holds the check and its first 12 items + assert [c["id"] for c in plan.criteria if "check" in c] == ["oversharing", "rogue"] # each check is asked once + + +@pytest.mark.asyncio +async def test_split_items_get_their_rows_and_the_check_is_asked_once(monkeypatch): + items = [{"id": f"g1_{n}", "family": "G1", "check": f"never states fact {n}"} for n in range(14)] + fake = FakeModel({"g1_13": FAIL()}) + entry = await _judge(_run(("reply", "x")), fake, monkeypatch, rubric_items=items) + split = [c for c in fake.calls if any(n.startswith("g1_") for n in c["names"])] + assert [len(c["names"]) for c in split] == [13, 2] and split[0]["names"][0] == "oversharing" + assert len(fake.asked("oversharing")) == 1 and "These are criteria 15–16 of 21" in split[1]["prompt"] + assert _rows(entry)["g1_13"]["asked_with"] == "oversharing#2" and _rows(entry)["g1_13"]["verdict"] == "fail" + assert [r["id"] for r in entry["results"]] == [*core.CHECK_NAMES, *[f"g1_{n}" for n in range(14)]] + assert entry["user_sim_error"]["because"] == ["g1_13"] # G1's owner, oversharing, is an error check + + +def test_each_bad_item_is_refused_when_the_task_loads_with_a_clear_error(): + cases = [ + ({"family": "A1", "check": "x"}, "rubric_items[0] needs an id"), + ({"id": "a", "check": "x"}, "rubric_items[0] (a) needs a family"), + ({"id": "a", "family": "B1", "check": "x"}, "rubric_items[0] (a): family 'B1' is not one of"), + ({"id": "a", "family": "A1", "kind": "strict", "check": "x"}, "kind 'strict' is not one of ['hard', 'general', " + "'auto']"), + ({"id": "a", "family": "F2", "kind": "auto", "max": 25}, "an auto item's metric must be one of"), + ({"id": "a", "family": "F2", "kind": "auto", "metric": "max_chars", "max": 25}, "not 'max_chars'"), + ({"id": "a", "family": "F2", "kind": "auto", "metric": "max_words"}, "an auto item needs max"), + ({"id": "a", "family": "F2", "kind": "auto", "metric": "max_words", "max": 0}, "not 0"), + ({"id": "a", "family": "F2", "kind": "auto", "metric": "max_words", "max": "25"}, "not '25'"), + ({"id": "a", "family": "F2", "kind": "general", "check": "x", "metric": "max_words", "max": 25}, + "metric and max are for kind auto only"), + ({"id": "a", "family": "A5", "check": "x", "na_when": "always"}, "na_when always needs a static_na_reason"), + ({"id": "a", "family": "A5", "check": "x", "static_na_reason": "y"}, "static_na_reason is only for na_when " + "always"), + ({"id": "a", "family": "A1", "kind": "hard", "check": " "}, "a hard item needs a check"), + ({"id": "a", "family": "A1"}, "a general item needs a check"), + ({"id": "a", "family": "A1", "check": "x", "na-when": "never"}, "unknown fields ['na-when']"), + ({"id": "a", "family": "F2", "kind": "auto", "metric": "max_words", "max": 25, "na_when": "never"}, + "auto items are counted, so na_when can only be 'always' (with a static_na_reason), not 'never'"), + ({"id": "a", "family": "F2", "kind": "auto", "metric": "max_words", "max": 25, + "na_when": "The assistant never asked her anything."}, + "auto items are counted, so na_when can only be 'always'"), + ] + for item, error in cases: + problems = core.item_problems([item]) + assert any(error in p for p in problems), (item, problems) + assert any(error in p for p in usj.UserSimJudge("s", None, run_step_id="week", + rubric_items=[item]).config_problems()) + dup = core.item_problems([CALL_RAE, {**CALL_RAE, "family": "A3"}]) + assert dup == ["rubric_items[1] (a1_call_rae): id 'a1_call_rae' is used twice"] + for bad in (7, "3f1c", "A" * 64): + assert any("rubric_persona_sha256 must be a persona fingerprint" in p for p in usj.UserSimJudge( + "s", None, run_step_id="week", rubric_items=[CALL_RAE], rubric_persona_sha256=bad).config_problems()) + + +def test_an_auto_item_loads_with_no_na_when_or_with_always(): + auto = {"id": "a", "family": "F2", "kind": "auto", "metric": "max_words", "max": 25} + assert core.item_problems([auto]) == [] + assert core.item_problems([{**auto, "na_when": "always", "static_na_reason": "She never texts here."}]) == [] + + +# ---- the rubric's persona ------------------------------------------------------------------------------------------ +@pytest.mark.usefixtures("local_stores") +@pytest.mark.asyncio +async def test_items_written_for_another_persona_aren_t_graded_and_the_checks_still_run(monkeypatch): + ctx = await _chat(monkeypatch) + run = ctx.metadata[USER_SIMS_KEY]["hannah"]["persona_sha256"] + fake = FakeModel({"guidance": FAIL()}) + + with pytest.raises(VerifierNotPassingError, match=r"\(guidance\)"): + await _step(fake, monkeypatch, rubric_items=[CALL_RAE, HER_LENGTH], + rubric_persona_sha256=OTHER_PERSONA).execute(ctx) + + entry = _entry(ctx) + assert entry["rubric_persona"] == { + "rubric": OTHER_PERSONA, "run": run, "status": "mismatch", + "note": "the task items were written for a different persona or opening than this run's user sim had, so they " + "weren't graded; the general checks were"} + assert list(_rows(entry)) == list(core.CHECK_NAMES) and len(fake.calls) == 7 + assert all("a1_call_rae" not in c["names"] for c in fake.calls) + assert entry["user_sim_error"]["because"] == ["guidance"] + + +@pytest.mark.usefixtures("local_stores") +@pytest.mark.asyncio +@pytest.mark.parametrize("recorded, status", [(True, "match"), (False, "unknown")]) +async def test_items_for_this_persona_or_a_run_with_no_fingerprint_are_graded(monkeypatch, recorded, status): + ctx = await _chat(monkeypatch) + us = ctx.metadata[USER_SIMS_KEY]["hannah"] + fingerprint = us["persona_sha256"] if recorded else OTHER_PERSONA + if not recorded: + del us["persona_sha256"] # a run from before the step recorded it + + await _step(FakeModel(), monkeypatch, rubric_items=[HER_LENGTH], + rubric_persona_sha256=fingerprint).execute(ctx) + + entry = _entry(ctx) + assert entry["rubric_persona"]["status"] == status and _rows(entry)["f2_her_length"]["verdict"] == "pass" + assert entry["rubric_persona"]["rubric"] == fingerprint + if not recorded: + assert entry["rubric_persona"]["run"] is None and "graded as written" in entry["rubric_persona"]["note"] + # Without a rubric_persona_sha256 nothing is compared. + await _step(FakeModel(), monkeypatch, rubric_items=[HER_LENGTH]).execute(ctx) + assert "rubric_persona" not in _entry(ctx) and _rows(_entry(ctx))["f2_her_length"]["verdict"] == "pass" + + +# ---- items that point at a persona line ------------------------------------------------------------------------------ +ENGAGEMENT = {"id": "g1_engagement", "family": "G1", "kind": "hard", "line": "P1"} +RING = "Ben proposed last month. Never mention or hint at it." + + +def _lined_run(tagged_lines=None) -> core.UserSimRun: + return replace(_run(("reply", "just a family dinner")), tagged_lines=tagged_lines) + + +def test_the_guidance_question_holds_it_to_its_already_told_and_should_already_know_lines(): + q = core.check_question("guidance") + assert "THINGS YOU ALREADY TOLD THE ASSISTANT lines (T)" in q and "remind it, a little annoyed" in q + assert "THINGS THE ASSISTANT SHOULD ALREADY KNOW lines (C) wrong, and it didn't correct it briefly" in q + assert "even where its other lines say to let that kind of mistake pass" in q + assert "Correcting a THINGS THE ASSISTANT SHOULD ALREADY KNOW line (C)" in q.split("Not a failure:")[1] + + +@pytest.mark.asyncio +async def test_a_line_item_is_graded_by_that_line_s_text_from_the_run_and_records_it(monkeypatch): + [item] = _items(ENGAGEMENT) + assert item.to_dict() == ENGAGEMENT and item.check == "" + fake = FakeModel({"g1_engagement": FAIL()}) + entry = await _judge(_lined_run({"W1": "Hannah.", "P1": RING}), fake, monkeypatch, + rubric_items=[ENGAGEMENT], rubric_persona_sha256=OTHER_PERSONA) + assert fake.asked("g1_engagement")[0]["criteria"][1]["question"] == f"Follows its instruction P1: {RING}" + row = _rows(entry)["g1_engagement"] + assert (row["line"], row["line_text"], row["asked_with"], row["verdict"], row["kind"]) == ( + "P1", RING, "oversharing", "fail", "hard") + assert entry["user_sim_error"]["because"] == ["g1_engagement"] + + +def test_a_line_s_later_lines_stay_under_the_item(): + [item] = _items({**ENGAGEMENT, "line": "W2"}) + plan = _plan(_lined_run({"W2": 'Earlier you texted the assistant:\n"sort the weekend"'}), [item], + checks=["oversharing"]) + assert plan.criteria[1]["question"] == 'Follows its instruction W2: Earlier you texted the assistant:\n "sort the ' \ + 'weekend"' + + +@pytest.mark.parametrize("tagged_lines, error", [ + ({"P1": RING}, "the user sim's instructions have no line P2"), + (None, "the run recorded no tagged_lines for the user sim, so its line P2 can't be read"), +]) +def test_a_line_the_run_doesn_t_have_is_an_error_never_a_guess(tagged_lines, error): + [item] = _items({**ENGAGEMENT, "line": "P2"}) + plan = _plan(_lined_run(tagged_lines), [item], checks=["oversharing"]) + row = _unjudged(plan)["g1_engagement"] + assert (row["verdict"], row["turns"], row["line"], row["error"], row["asked_with"]) == ( + "error", [], "P2", error, core.NO_LINE) + assert [c["id"] for c in plan.criteria] == ["oversharing"] # it makes no call + assert core.user_sim_error(core.finish_rows(plan.unjudged, plan)) == { + "value": "unknown", "because": [], "unknown": ["g1_engagement"]} + + +def test_a_line_item_is_refused_with_a_check_on_an_auto_item_or_without_the_persona_fingerprint(): + cases = [ + ({**ENGAGEMENT, "check": "Never mentions the ring."}, "give line or check, not both"), + ({**ENGAGEMENT, "text": "Never mentions the ring."}, "give line or check, not both"), + ({**ENGAGEMENT, "kind": "auto", "metric": "max_words", "max": 25}, + "line is for hard and general items: an auto item is counted, not judged"), + ({**ENGAGEMENT, "line": "p1"}, "line must be a tag of the user sim's instructions, like P1 or D3, not 'p1'"), + ({**ENGAGEMENT, "line": "P"}, "not 'P'"), + ({**ENGAGEMENT, "line": 1}, "line must be text"), + ] + for item, error in cases: + problems = core.item_problems([item]) + assert any(error in p for p in problems), (item, problems) + [no_check] = core.item_problems([{"id": "a", "family": "A1"}]) + assert no_check == ("rubric_items[0] (a): a general item needs a check (the text the judge applies) or a line (a " + "tag of the user sim's instructions, like P1)") + assert usj.UserSimJudge("s", None, run_step_id="week", rubric_items=[ENGAGEMENT], + rubric_persona_sha256=OTHER_PERSONA).config_problems() == [] + assert usj.UserSimJudge("s", None, run_step_id="week", rubric_items=[ENGAGEMENT]).config_problems() == [ + "rubric_items[0] (g1_engagement): a line item needs rubric_persona_sha256, so its tag is graded only on runs " + "of the persona it was written for"] + + +@pytest.mark.usefixtures("local_stores") +@pytest.mark.asyncio +async def test_on_a_run_a_line_item_reads_the_deploy_record_s_tagged_lines(monkeypatch): + ctx = await _chat(monkeypatch) + us = ctx.metadata[USER_SIMS_KEY]["hannah"] + fake = FakeModel() + + await _step(fake, monkeypatch, rubric_items=[ENGAGEMENT], + rubric_persona_sha256=us["persona_sha256"]).execute(ctx) + + [call] = fake.asked("g1_engagement") + assert call["criteria"][1]["question"] == f"Follows its instruction P1: {us['tagged_lines']['P1']}" + row = _rows(_entry(ctx))["g1_engagement"] + assert (row["line"], row["line_text"], row["asked_with"]) == ("P1", us["tagged_lines"]["P1"], "oversharing") + + +@pytest.mark.usefixtures("local_stores") +@pytest.mark.asyncio +async def test_on_a_run_with_no_persona_fingerprint_a_line_item_is_an_error(monkeypatch): + ctx = await _chat(monkeypatch) + del ctx.metadata[USER_SIMS_KEY]["hannah"]["persona_sha256"] + fake = FakeModel() + + await _step(fake, monkeypatch, rubric_items=[ENGAGEMENT, HER_LENGTH], + rubric_persona_sha256=OTHER_PERSONA).execute(ctx) + + rows = _rows(_entry(ctx)) + assert rows["g1_engagement"]["verdict"] == "error" and rows["f2_her_length"]["verdict"] == "pass" + assert rows["g1_engagement"]["error"] == "line P1 can't be graded: " + usj.LINE_UNCHECKED + assert not fake.asked("g1_engagement") and len(fake.calls) == 7 + assert _entry(ctx)["user_sim_error"]["value"] == "unknown" and _entry(ctx)["flag"] == "user sim judge undecided"