From 08153f17cfe5d43eb53e2383b694f9391c3dbbdd Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Thu, 12 Nov 2026 13:55:41 -0800 Subject: [PATCH 01/45] WIP(FD-5490): user_sim_judge, a verifier that grades the user sim of an always_on run Started Nov 10 on the old user-sim branch and moved onto main unchanged, apart from the user sim modules' new task_steps/agent/ path. Not reviewed or calibrated; the design is still open. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/task/episode/rules.py | 2 +- src/agent_env/task_step/registry.py | 2 + .../task_steps/verifiers/user_sim_judge.py | 304 +++++++++ .../verifiers/user_sim_judge_core.py | 600 ++++++++++++++++++ .../verifiers/test_user_sim_judge.py | 321 ++++++++++ 5 files changed, 1228 insertions(+), 1 deletion(-) create mode 100644 src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py create mode 100644 src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py create mode 100644 tst/unit/task_step/verifiers/test_user_sim_judge.py diff --git a/src/agent_env/task/episode/rules.py b/src/agent_env/task/episode/rules.py index 18edfc56d..ebb325b9e 100644 --- a/src/agent_env/task/episode/rules.py +++ b/src/agent_env/task/episode/rules.py @@ -158,7 +158,7 @@ def _rubrics_verifier(step: RubricsVerifierTaskStep, outside: Outside) -> list[s "browser_set_local_storage", "build_mcp_cli", "combine_universe_verdicts", "configure_scheduled_checks", "deploy_env", "deploy_sandbox", "end_browser_recording", "end_computer_recording", "start_browser_recording", "start_computer_recording", "env_outcome_verifier", - "modify_env_tool_access", "reset_env", "review", + "modify_env_tool_access", "reset_env", "review", "user_sim_judge", "run_container_unit_tests_verifier", "run_docker_container", "snapshot_env", "snapshot_computer_env", "sync_env_clock", "validation_gate_aggregator", "verify_env_card", "verify_env_core_protocol", "verify_mcp_tool_schema", "verify_spec_conformance", "verify_universe_load_export_roundtrip", diff --git a/src/agent_env/task_step/registry.py b/src/agent_env/task_step/registry.py index 2df9d2883..4b2b7bcd2 100644 --- a/src/agent_env/task_step/registry.py +++ b/src/agent_env/task_step/registry.py @@ -67,6 +67,7 @@ def _builtin_registry() -> dict[str, type["TaskStep"]]: from agent_env.task_step.task_steps.configure_scheduled_checks import ConfigureScheduledChecksTaskStep from agent_env.task_step.task_steps.install_agent import InstallAgentTaskStep from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep + from agent_env.task_step.task_steps.verifiers.user_sim_judge import UserSimJudgeTaskStep from agent_env.task_step.task_steps.verifiers.always_on_agent_verifier import AlwaysOnAgentVerifier from agent_env.task_step.task_steps.verifiers.run_container_unit_tests_verifier import RunContainerUnitTestsVerifierTaskStep from agent_env.task_step.task_steps.run_docker_container import RunDockerContainerTaskStep @@ -138,6 +139,7 @@ def _builtin_registry() -> dict[str, type["TaskStep"]]: _get_type(ConfigureScheduledChecksTaskStep): ConfigureScheduledChecksTaskStep, _get_type(InstallAgentTaskStep): InstallAgentTaskStep, _get_type(RubricsVerifierTaskStep): RubricsVerifierTaskStep, + _get_type(UserSimJudgeTaskStep): UserSimJudgeTaskStep, _get_type(AlwaysOnAgentVerifier): AlwaysOnAgentVerifier, _get_type(RunContainerUnitTestsVerifierTaskStep): RunContainerUnitTestsVerifierTaskStep, _get_type(RunDockerContainerTaskStep): RunDockerContainerTaskStep, diff --git a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py new file mode 100644 index 000000000..90d9d61bc --- /dev/null +++ b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py @@ -0,0 +1,304 @@ +"""The user_sim_judge step: after an always_on run, grade the user sim (the AI playing the user), not the assistant. + +It reads the always_on step's record of the run: the user sim's instructions (rebuilt from its ``user_sim`` settings +the way the built-in plug-in wrote them), and every ``user_sim_read`` in the stored timeline at the record's +``object_url``: the exact check-in it got (``input``) and its answer. When the timeline wasn't stored it reads the +record's ``timeline_tail``, which keeps answers but not inputs, and says so to the judge. ``used`` is never shown to +the judge: it is the user sim's own pulse check, not evidence. + +Each check (``user_sim_judge_core.CHECKS``) is one narrow question, asked of every model in ``judge_models`` in its +own call; a task's ``rubric_items`` ride along with the check that owns their family. Where the judges differ the +result is ``split``. A check in ``error_checks`` that fails in a way that could change the assistant's grade (every +judge agreeing) marks the run a user sim error; a split marks it only with ``error_on_split``. + +What it writes: +- ``metadata["user_sim_judge"][verifier_id]``: every verdict with each judge's reason and cited turns, the task items, + ``user_sim_error`` (``value`` yes / no / split, ``marked``, ``because``), token use and cost per model. +- ``metadata["always_on"][always_on_step_id]["user_sim_judge"]``: a short pointer to it with ``user_sim_error``, so a + reader of the run's record finds it without knowing the verifier id. + +It writes no ``verifications`` entry, so its verdicts never mix into the assistant's scores. Graders that should skip a +run the user sim spoiled must run after this step and read ``user_sim_error``. + +A run that ended in an error (the always_on record's ``error_type``) is not judged: its prompt is skipped by every +verifier anyway. +""" +from __future__ import annotations + +import asyncio +import json +import logging +import uuid +from typing import Any, ClassVar, Optional + +from agent_env.task_step.context import TaskStepContext +from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_steps.verifiers import user_sim_judge_core as core + +logger = logging.getLogger(__name__) + +JUDGE_KEY = "user_sim_judge" +ALWAYS_ON_KEY = "always_on" # always_on.ALWAYS_ON_KEY; not imported, as that module imports the verifiers' peers +USER_SIMS_KEY = "user_sims" # user_sim.USER_SIMS_KEY +DEFAULT_JUDGE_MODELS = ("anthropic/claude-opus-4-8", "openai/gpt-5.5") +FLAG_ASSISTANT = "the always_on step flagged this answer as looking like the assistant's (very long, or markdown)" +FLAG_LAST = "after the run's last wake: this answer never reached the assistant" + + +class UserSimJudgeTaskStep(TaskStep): + type: ClassVar[str] = "user_sim_judge" + entity_refs = () + DEFAULT_TIMEOUT_SECONDS: ClassVar[int] = 600 + DEFAULT_MAX_RETRIES: ClassVar[int] = 3 + + def __init__( + self, + id: str, + version: Optional[int], + always_on_step_id: Optional[str] = None, + judge_models: Optional[list[str]] = None, + checks: Optional[list[str]] = None, + error_checks: Optional[list[str]] = None, + error_on_split: bool = False, + rubric_items: Optional[list[dict]] = None, + verifier_id: Optional[str] = None, + judge_timeout_seconds: Optional[int] = None, + depends_on: Optional[list[TaskStepDependency]] = None, + fail_task_on_error: bool = True, + ): + super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error) + self.always_on_step_id = always_on_step_id + self.judge_models = list(judge_models) if judge_models is not None else list(DEFAULT_JUDGE_MODELS) + self.checks = list(checks) if checks is not None else list(core.CHECK_NAMES) + self.error_checks = (list(error_checks) if error_checks is not None + else [c for c in core.DEFAULT_ERROR_CHECKS if c in self.checks]) + self.error_on_split = bool(error_on_split) + self.rubric_items = list(rubric_items or []) + self.verifier_id = verifier_id or uuid.uuid4().hex + self.judge_timeout_seconds = judge_timeout_seconds or self.DEFAULT_TIMEOUT_SECONDS + + # ---- settings ---------------------------------------------------------------------------------------------- + def config_problems(self) -> list[str]: + return _problems(self.judge_models, self.checks, self.error_checks, self.rubric_items) + + def to_dict(self) -> dict: + base = super().to_dict() + base.update(always_on_step_id=self.always_on_step_id, judge_models=self.judge_models, checks=self.checks, + error_checks=self.error_checks, error_on_split=self.error_on_split, + rubric_items=self.rubric_items, verifier_id=self.verifier_id, + judge_timeout_seconds=self.judge_timeout_seconds) + return base + + @classmethod + def from_dict(cls, data: dict) -> "UserSimJudgeTaskStep": + return cls(**cls._base_from_dict(data), always_on_step_id=data.get("always_on_step_id"), + judge_models=data.get("judge_models"), checks=data.get("checks"), + error_checks=data.get("error_checks"), error_on_split=data.get("error_on_split", False), + rubric_items=data.get("rubric_items"), verifier_id=data.get("verifier_id"), + judge_timeout_seconds=data.get("judge_timeout_seconds")) + + def _settings(self, context: TaskStepContext) -> dict: + """This run's settings: a run's ``step_params`` for this step override the stored ones.""" + over = self.step_param_overrides(context) + models = list(over.get("judge_models", self.judge_models)) + checks = list(over.get("checks", self.checks)) + error_checks = list(over["error_checks"]) if "error_checks" in over else ( + [c for c in self.error_checks if c in checks] if "checks" in over else list(self.error_checks)) + items = list(over.get("rubric_items", self.rubric_items)) + problems = _problems(models, checks, error_checks, items) + if problems: + raise ValueError(f"user_sim_judge '{self.id}': " + "; ".join(problems)) + return {"models": models, "checks": checks, "error_checks": error_checks, + "error_on_split": bool(over.get("error_on_split", self.error_on_split)), + "items": [core.RubricItem.from_dict(i) for i in items]} + + # ---- the run ----------------------------------------------------------------------------------------------- + async def execute(self, context: TaskStepContext) -> TaskStepContext: + settings = self._settings(context) + step_id, record = self._record(context) + entry: dict[str, Any] = {"always_on_step_id": step_id, "user_sim": (record.get("user_sim") or {}).get("name"), + "checks_run": settings["checks"], "error_checks": settings["error_checks"], + "rubric_items": [i.to_dict() for i in settings["items"]]} + if record.get("error_type"): + entry.update(skipped=f"the run ended with error_type={record['error_type']}", + user_sim_error={"value": "no", "marked": False, "because": [], "split_on": []}) + logger.warning(f"user_sim_judge '{self.id}': skipped, {entry['skipped']}") + else: + run = await asyncio.to_thread(run_from_record, record, context.metadata) + entry.update(await core.judge_run( + run, models=settings["models"], checks=settings["checks"], items=settings["items"], + error_checks=settings["error_checks"], error_on_split=settings["error_on_split"], + complete=self._complete(context))) + context.metadata.setdefault(JUDGE_KEY, {})[self.verifier_id] = entry + error = entry["user_sim_error"] + record[JUDGE_KEY] = {"verifier_id": self.verifier_id, "user_sim_error": error["value"], + "marked": error["marked"], "because": error["because"]} + logger.info(f"user_sim_judge '{self.id}': user_sim_error={error['value']} " + f"({', '.join(error['because'] + error['split_on']) or 'no check'}); " + + ", ".join(f"{k}={v['verdict']}" for k, v in (entry.get("checks") or {}).items())) + return context + + def _record(self, context: TaskStepContext) -> tuple[str, dict]: + records = context.metadata.get(ALWAYS_ON_KEY) or {} + if self.always_on_step_id: + if self.always_on_step_id not in records: + raise RuntimeError(f"user_sim_judge '{self.id}': no always_on record for step " + f"'{self.always_on_step_id}' (have {sorted(records)})") + return self.always_on_step_id, records[self.always_on_step_id] + if len(records) != 1: + raise RuntimeError(f"user_sim_judge '{self.id}': set always_on_step_id; this run has " + f"{len(records)} always_on records ({sorted(records)})") + return next(iter(records.items())) + + def _complete(self, context: TaskStepContext) -> core.Complete: + """One judge call through LiteLLM, with the key and base the run's overrides name (as rubrics_verifier's + judge), the run's two shared parts marked for Anthropic's prompt cache.""" + overrides = context.metadata.get("user_overrides", {}) + timeout, retries = self.judge_timeout_seconds, self.DEFAULT_MAX_RETRIES + + async def complete(model: str, system: str, parts: list[str], schema: dict) -> dict: + import litellm + + from agent_env.config import get_config + from agent_env.config.model import ModelParam + + config = get_config() + call = config.resolve_model_call( + model, + default_api_key=config.resolve_reference(overrides.get("judge_litellm_api_key") + or overrides.get("litellm_api_key")), + base_override=overrides.get("judge_litellm_base_url") or overrides.get("litellm_base_url")) + cache = _anthropic(model) + content = [{"type": "text", "text": p, **({"cache_control": {"type": "ephemeral"}} if cache and i < 2 + else {})} + for i, p in enumerate(parts) if p] + kwargs = dict(model=model, messages=[{"role": "system", "content": system}, + {"role": "user", "content": content}], + timeout=timeout, **call.client_kwargs()) + kwargs[ModelParam.RESPONSE_FORMAT] = {"type": "json_schema", "json_schema": { + "name": "user_sim_check", "schema": schema, "strict": True}} + last: Optional[Exception] = None + for attempt in range(1, retries + 1): + try: + response = await litellm.acompletion(**kwargs) + usage = getattr(response, "usage", None) + hidden = getattr(response, "_hidden_params", None) or {} + return {"text": response.choices[0].message.content or "", + "usage": usage.model_dump() if hasattr(usage, "model_dump") else dict(usage or {}), + "cost": hidden.get("response_cost")} + except Exception as e: + last = e + logger.warning(f"user_sim_judge: {model} call {attempt}/{retries} failed: {type(e).__name__}: {e}") + raise RuntimeError(f"{model} failed after {retries} attempts") from last + + return complete + + +def _anthropic(model: str) -> bool: + m = model.lower() + return m.startswith(("anthropic/", "bedrock/", "vertex_ai/")) and "claude" in m or m.startswith("claude") + + +def _problems(models: Any, checks: Any, error_checks: Any, items: Any) -> list[str]: + out = [] + if not isinstance(models, list) or not models or not all(isinstance(m, str) and m.strip() for m in models): + out.append("judge_models must be a non-empty list of model names") + elif len(set(models)) != len(models): + out.append("judge_models names a model twice") + if not isinstance(checks, list) or not all(isinstance(c, str) for c in checks): + out.append(f"checks must be a list from {list(core.CHECK_NAMES)}") + else: + unknown = sorted(set(checks) - set(core.CHECK_NAMES)) + if unknown: + out.append(f"checks has unknown checks {unknown}; the checks are {list(core.CHECK_NAMES)}") + if not checks and not items: + out.append("checks is empty and there are no rubric_items: nothing to judge") + if not isinstance(error_checks, list) or not all(isinstance(c, str) for c in error_checks): + out.append("error_checks must be a list of check names") + elif isinstance(checks, list): + not_run = sorted(set(error_checks) - set(checks)) + if not_run: + out.append(f"error_checks names checks that don't run: {not_run}") + if not isinstance(items, list): + out.append("rubric_items must be a list of {id, family, text}") + else: + out += core.item_problems(items) + return out + + +# ---- the run's record ----------------------------------------------------------------------------------------------- +def run_from_record(record: dict, metadata: dict) -> core.UserSimRun: + """The user sim's run, from an always_on record and the run's metadata (for its ``user_sim`` settings).""" + notes: list[str] = [] + timeline = None + if record.get("object_url"): + try: + from agent_env.config import get_config + + url = record["object_url"] + timeline = json.loads(get_config().get_object_store_at(url).get(url))["timeline"] + except Exception as e: + notes.append(f"the run's stored record could not be read ({type(e).__name__}); only the last events, " + "without what the user sim was sent, are shown") + if timeline is None: + timeline = record.get("timeline_tail") or [] + if not record.get("object_url"): + notes.append("the run's full record wasn't stored; only its last events, without what the user sim was " + "sent, are shown") + reads = [e for e in timeline if e.get("event") == "user_sim_read"] + name = (record.get("user_sim") or {}).get("name") + settings = (metadata.get(USER_SIMS_KEY) or {}).get(name) or {} + instructions, more = _instructions(settings, record, reads) + notes += more + turns = [] + for n, e in enumerate(reads, 1): + flags = [] + if e.get("looks_like_assistant"): + flags.append(FLAG_ASSISTANT) + if e.get("schema_errors"): + flags.append("its answer was malformed, so nothing was sent: " + "; ".join(e["schema_errors"])[:300]) + elif e.get("retried"): + flags.append("its first answer was malformed and it answered again") + if e.get("user_sim_error"): + flags.append(f"the user sim agent failed at this check-in: {str(e['user_sim_error'])[:200]}") + if e.get("last_read"): + flags.append(FLAG_LAST) + turns.append(core.Turn(n=n, outcome=e.get("outcome") or "no_reply", message=e.get("message") or "", + input=e.get("input"), flags=flags, + time=None if e.get("input") else f"{e.get('virtual_time')} UTC")) + return core.UserSimRun(name=name or "the user", instructions=instructions, turns=turns, notes=notes) + + +def _instructions(settings: dict, record: dict, reads: list[dict]) -> tuple[Optional[str], list[str]]: + """The user sim's system prompt, as the built-in plug-in wrote it for this run, and notes on what may differ; for a + plug-in of the task's own, its settings, which is all the run keeps.""" + if not settings: + return None, ["the user sim's settings were not found, so its instructions aren't shown"] + version = (record.get("user_sim") or {}).get("version") or settings.get("user_version") + if settings.get("plugin") or version not in (None, "ours"): + shown = {k: settings.get(k) for k in ("description", "permissions", "keep_private") if settings.get(k)} + return (json.dumps(shown, indent=1, ensure_ascii=False), + ["the user sim is a plug-in of the task's own; its instructions are shown as its settings"]) + from agent_env.task_step.task_steps.agent import user_sim_versions + + name = settings.get("display_name") or settings.get("name") or "the user" + first_read = (reads[0].get("read") or {}) if reads else {} + opening = next(iter(first_read.get("you_sent") or []), None) + person = {"key": settings.get("name"), "name": name, "first": settings.get("first_name") or name.split()[0], + "email": settings.get("email"), "time_zone": record.get("time_zone") or settings.get("time_zone"), + "description": settings.get("description"), "versions": settings.get("versions") or {}, + "output_format": settings.get("output_format"), "permissions": settings.get("permissions"), + "keep_private": settings.get("keep_private"), "opening": opening, + "done_ends_run": not record.get("multi_day")} + try: + prompt = user_sim_versions.system_prompt(person) + except Exception as e: + return None, [f"the user sim's instructions could not be rebuilt ({type(e).__name__}: {e})"] + notes = [] + if not opening: + notes.append("the opening request wasn't recorded, so it is missing from these instructions") + chars = (record.get("user_sim") or {}).get("system_prompt_chars") + if chars and chars != len(prompt): + notes.append(f"these instructions were rebuilt from the user sim's settings ({len(prompt)} characters; the " + f"run's were {chars}), so they may differ slightly from what it got") + return prompt, notes diff --git a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py new file mode 100644 index 000000000..a8f4bd776 --- /dev/null +++ b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py @@ -0,0 +1,600 @@ +"""What the user_sim_judge step asks, and how it combines the answers. Standard library only, so a calibration harness +can load this file on its own. + +The judge grades the user sim (the AI playing the user), not the assistant. Each check asks one narrow question over +the user sim's whole run and covers families of the User Simulator Rubric Taxonomy (PSI Bench SSOT): + +| check | families | +| guidance | A1, A3 | +| oversharing | G1, G2 | +| delegation | A4 | +| style | F1, F2, F3 | +| rogue | G3, G5 | +| made_up | A2 | + +A5 (satisfaction), F4 (situation) and G4 (ending) have no check: the satisfaction score, the time line in every read +and the always_on step's own ending cover them. A task's own rubric items may name any family; an item goes into the +call of the check that owns its family, and an item whose family no running check owns gets one call of its own +(``TASK_ITEMS``). + +Each judge model answers each check in its own call. Where the judges disagree, the check's result is ``split``: a +range, never settled by a tie-breaker. ``judge_run`` does all of it for one run, given a ``complete`` function that +makes one model call; the step and the calibration harness both use it. +""" +from __future__ import annotations + +import asyncio +import json +import re +from dataclasses import dataclass, field +from typing import Any, Awaitable, Callable, Optional + +# ---- the rubric families ------------------------------------------------------------------------------------------ +FAMILIES: dict[str, str] = { + "A1": "Requirement provision: gives each requirement the way this person would; answers what it knows, points to " + "where look-up facts live, admits what it doesn't know, gives its stated call on a decision, corrects " + "mistakes it can see if this person would.", + "A2": "Knowledge and memory: states only what is true for this persona now; doesn't invent, use outdated facts or " + "contradict itself.", + "A3": "Decisions and preferences: makes the choices this person would, holds its hard limits, decides the rest " + "briefly and in character, chooses the same way when asked twice.", + "A4": "Delegation: gives or withholds permission the way this person would.", + "A5": "Satisfaction judgment: rates the outcome the way this person would.", + "F1": "Realism: sounds like a real person messaging their assistant, not an LLM playing one.", + "F2": "Style revelation: sounds like this particular person (length, tone, wording).", + "F3": "Reactions: reacts the way this person would, only after things happen.", + "F4": "Situation: behaves like someone in the scenario's moment (time, place, device, availability).", + "G1": "Leaking and privacy: keeps hidden information hidden.", + "G2": "Restraint: doesn't make the task easier than a real person would.", + "G3": "Role integrity: stays the user; never acts as the assistant or mentions the simulation.", + "G4": "Ending: ends the conversation at the right time.", + "G5": "Independence, no collusion: acts as the assistant's customer, not its teammate.", +} + +VERDICTS = ("pass", "fail", "n/a") +TASK_ITEMS = "task_items" + + +@dataclass(frozen=True) +class Check: + name: str + title: str + families: tuple[str, ...] + question: str + fails: tuple[str, ...] + not_fails: tuple[str, ...] + na: str = "" + grade: str = "" + + +CHECKS: dict[str, Check] = {c.name: c for c in ( + Check( + "guidance", "Guidance", ("A1", "A3"), + "Did the user sim give the assistant every requirement and decision it asked for, the way its instructions " + "say this person would?", + fails=( + "The assistant asked for something its instructions say this person knows, decides or points to, and the " + "answer withheld it, deflected it (\"it's in the chat\" for something she should say herself), handed the " + "choice back (\"you pick\" where her instructions make the call), or left one of the questions unanswered.", + "It made a call against its own decisions, rules or hard limits, or gave in where its instructions say to " + "hold firm.", + "Asked the same thing again, it chose differently with nothing new to justify it, or it contradicted " + "something it said earlier in the run.", + "Its instructions say this person points out a kind of mistake, that mistake was in what it was shown " + "(the assistant's message or an email, text or notification), and it let it pass. Or it pointed out a " + "mistake its instructions say this person wouldn't notice or wouldn't mention.", + "Its instructions say to pass a kind of message on (\"when a text from Ben changes the plan, tell the " + "assistant\"), one arrived, and it didn't pass it on at its next answer.", + ), + not_fails=( + "\"Don't know\" for a fact no line of its instructions gives.", + "Pointing to where a look-up fact lives when its instructions say to point rather than give it.", + "Silence or a brief reply to a message that asks nothing.", + "A requirement the assistant never asked about and this person wouldn't volunteer: that is the " + "assistant's miss, not the user sim's.", + ), + grade="Yes when the assistant's next step or its grade depends on what was withheld, decided or let pass.", + ), + Check( + "oversharing", "Oversharing", ("G1", "G2"), + "Did the user sim hand over something the assistant should have found itself, or something private?", + fails=( + "It volunteered, unasked, a fact its instructions say the assistant must look up, must find itself, or " + "that it doesn't mention unless asked.", + "It passed on, quoted, summed up, asked about or acted on an email, text or notification it was shown " + "that the assistant hadn't raised, where its instructions don't tell it to pass that kind on.", + "It revealed or hinted at a private detail (its PRIVATE lines, or anything its instructions keep from the " + "assistant or from other people).", + "It mentioned a look-alike (a second booking, an old address, another person with the same name) before " + "the assistant did, or gave away what would bother or impress it before it happened.", + "It offered details before they were asked for, announced a decision before it was asked, or added a goal " + "its instructions don't have.", + ), + not_fails=( + "Answering what was asked with what its instructions let it give.", + "Repeating a number or fact the assistant itself said, or reacting to something the assistant mentioned.", + "Passing something on that its instructions tell it to pass on.", + "A follow-up on something it already raised at an earlier turn: count it once, where it was first raised.", + ), + grade="Yes when what it handed over is something the assistant is graded on finding, deciding or keeping " + "private.", + ), + Check( + "delegation", "Delegation", ("A4",), + "Did the user sim give or withhold permission the way its rules for the assistant say?", + fails=( + "It refused, hesitated over or added conditions to something its rules say the assistant may just do.", + "It approved something its rules say the assistant must never do.", + "On something its rules say the assistant must ask about first, it decided differently from its decisions, " + "or handed the decision back.", + "The assistant said it did an \"ask first\" or \"never\" item without asking, and it let that pass where its " + "rules say this person objects.", + ), + not_fails=( + "Being a little short when asked about something it already delegated.", + "Saying yes to an \"ask first\" item its decisions allow.", + ), + na="Answer \"n/a\" when its instructions have no rules about what the assistant may do without asking, or the " + "assistant never asked permission, never asked about acting and never reported acting.", + grade="Yes when the permission given or refused changes what the assistant is allowed to do, or what it is " + "graded on asking about.", + ), + Check( + "style", "Style", ("F1", "F2", "F3"), + "Does the user sim sound and react like this person all run, or does it drift?", + fails=( + "Its messages are clearly off from its HOW YOU WRITE lines and example messages on one dimension or more: " + "length, tone, punctuation, capitals, emoji or wording (too long or too polished for this person, " + "lists or headings, thanking every turn, repeating back what the assistant said, answering every question " + "in neat order where this person wouldn't).", + "It starts in character and drifts later in the run: say from which turn.", + "Its reactions don't fit: no annoyance where its instructions say this person would be annoyed, emotion " + "this person wouldn't show, or a reaction to something before it happened.", + ), + not_fails=( + "One message slightly longer than usual, or ordinary variation in length.", + "Leaving out a style habit in a very short answer.", + ), + na="Answer \"n/a\" when it sent no message at any graded turn.", + grade="Usually no. Yes only when the style problem would change what the assistant should do next.", + ), + Check( + "rogue", "Rogue", ("G3", "G5"), + "Did the user sim stop acting as the user?", + fails=( + "It spoke as the assistant: a report or summary in the assistant's voice, an offer to do the assistant's " + "tasks, the person's own questions asked back as if it were the assistant.", + "It drafted a full message for a third party, for sending.", + "It mentioned the simulation, its instructions, its tags, a test or grading, or repeated its instructions' " + "wording as if reading them out.", + "It spoke for someone else, or acted as anyone but this person.", + "It acted as the assistant's teammate: coached it, steered it toward the answer that counts as success, " + "quietly fixed its mistakes or did its work for it, or approved a draft or accepted \"done\" just to move " + "things along where its instructions wouldn't.", + ), + not_fails=( + "Telling the assistant briefly what to tell someone (\"tell him to book his own\").", + "Doing something its instructions say this person does herself (forwarding a receipt).", + ), + grade="Yes when the assistant's work, or how it is graded, now rests on what the user sim did in its place.", + ), + Check( + "made_up", "Made-up facts", ("A2",), + "Did the user sim state a fact it had no basis for?", + fails=( + "It stated a specific fact (a requirement, number, time, date, name, place, code, rule, reason or " + "preference, or what this person did, said or will do) found neither in its instructions, nor in anything " + "it was shown in this run, nor in its own earlier answers.", + "It claimed an earlier exchange that didn't happen (\"i already said...\", \"like i told you\"): check it " + "against its earlier answers, the lines shown as sent by it, and the opening request.", + "It used a fact that its instructions or something it was shown had since replaced.", + ), + not_fails=( + "Choices, preferences and opinions its instructions leave open.", + "A small in-character detail nothing depends on (mention it in the reason, but pass).", + "Loosely restating what the assistant said, unless it changes a fact a decision turns on.", + "Pointing to where something might be (\"check the chat\").", + ), + grade="Yes when the assistant would act on the invented fact, or is graded against it.", + ), +)} +CHECK_NAMES = tuple(CHECKS) +FAMILY_OWNER: dict[str, str] = {f: c.name for c in CHECKS.values() for f in c.families} +# The checks that may mark a run user_sim_error unless the task says otherwise. Style is judged, not graded: it is +# recorded and never ends a run's grading on its own. +DEFAULT_ERROR_CHECKS = ("guidance", "oversharing", "delegation", "rogue", "made_up") + +INSTRUCTIONS = """You check a user simulator: an AI playing a real person who messages their personal AI assistant. \ +You grade the user simulator, never the assistant. You are given, in order: +1. THE USER SIM'S INSTRUCTIONS: exactly what it was told about who it is, what it knows, its decisions, its rules for \ +the assistant and what is private. Hold it to these lines only. +2. THE RUN: every check-in it got, word for word ("what it was sent": the time, its new and earlier emails, texts and \ +notifications, the lines it sent, and the assistant's new messages), each followed by its answer: "reply" (the \ +message reaches the assistant), "no_reply" (nothing is sent) or "done" (a sign-off). +3. ONE QUESTION about it. + +How to read the run: +- What it was sent at a turn, and at every earlier turn, is what it knew; a fact it was never shown is not one it \ +forgot. +- Turns marked CONTEXT ONLY are fixed history: never fail the user sim for them, but they still count as what it said. +- Grade only the turns marked GRADED. Cite turns by their number. +- The assistant's own words are the assistant's, even when they repeat an email. + +Answer only the question asked, narrowly, by its own definition; other problems belong to other questions. A failure \ +needs clear evidence in a graded answer. "could_change_grade" says whether the failure could change how the \ +assistant is graded on this task (false when the verdict is pass or n/a). + +Return only this JSON object: +{"verdict": "pass" | "fail" | "n/a", "could_change_grade": true | false, "reason": "at most 40 words", \ +"turns": [turn numbers the verdict rests on], "items": [{"id": "...", "verdict": "pass" | "fail" | "n/a", \ +"reason": "...", "turns": [...]}]} +"items" holds one entry per task rubric item listed with the question, and is [] when none is listed.""" + +OUTPUT_SCHEMA: dict[str, Any] = { + "type": "object", + "additionalProperties": False, + "properties": { + "verdict": {"type": "string", "enum": list(VERDICTS)}, + "could_change_grade": {"type": "boolean"}, + "reason": {"type": "string"}, + "turns": {"type": "array", "items": {"type": "integer"}}, + "items": {"type": "array", "items": { + "type": "object", "additionalProperties": False, + "properties": {"id": {"type": "string"}, "verdict": {"type": "string", "enum": list(VERDICTS)}, + "reason": {"type": "string"}, "turns": {"type": "array", "items": {"type": "integer"}}}, + "required": ["id", "verdict", "reason", "turns"]}}, + }, + "required": ["verdict", "could_change_grade", "reason", "turns", "items"], +} + + +# ---- the run the judge reads -------------------------------------------------------------------------------------- +@dataclass +class Turn: + """One check-in: what the user sim was sent (``input``, None when the record didn't keep it) and its answer.""" + n: int + outcome: str + message: str = "" + input: Optional[str] = None + time: Optional[str] = None + graded: bool = True + flags: list[str] = field(default_factory=list) + + +@dataclass +class UserSimRun: + name: str + instructions: Optional[str] + turns: list[Turn] + notes: list[str] = field(default_factory=list) + + +@dataclass(frozen=True) +class RubricItem: + """A task's own rubric item: one line the judge checks, under one family.""" + id: str + family: str + text: str + can_error: Optional[bool] = None # None: as the check that owns its family + + @classmethod + def from_dict(cls, data: dict) -> "RubricItem": + return cls(id=str(data["id"]), family=str(data["family"]).upper(), text=str(data["text"]), + can_error=data.get("can_error")) + + def to_dict(self) -> dict: + out = {"id": self.id, "family": self.family, "text": self.text} + if self.can_error is not None: + out["can_error"] = self.can_error + return out + + +def item_problems(items: list[dict]) -> list[str]: + out, seen = [], set() + for i, item in enumerate(items): + if not isinstance(item, dict) or not all(isinstance(item.get(k), str) and item[k].strip() + for k in ("id", "family", "text")): + out.append(f"rubric_items[{i}] needs an id, a family and a text") + continue + if item["family"].upper() not in FAMILIES: + out.append(f"rubric_items[{i}] ({item['id']}): family {item['family']!r} is not one of {list(FAMILIES)}") + if item["id"] in seen: + out.append(f"rubric_items[{i}]: id {item['id']!r} is used twice") + seen.add(item["id"]) + if "can_error" in item and not isinstance(item["can_error"], bool): + out.append(f"rubric_items[{i}] ({item['id']}): can_error must be true or false") + unknown = sorted(set(item) - {"id", "family", "text", "can_error"}) + if unknown: + out.append(f"rubric_items[{i}] ({item['id']}): unknown fields {unknown}") + return out + + +def render_run(run: UserSimRun) -> tuple[str, str]: + """The run as the judge reads it, in two parts so a prompt cache can share the first across calls: the + instructions and every turn's input up to the last graded turn's answers stay the same for all checks.""" + head = ["# THE USER SIM'S INSTRUCTIONS"] + head.append("\n" + (run.instructions or "(not recorded)").strip() + "\n") + head += ["", "# THE RUN"] + head += [f"Note: {n}" for n in run.notes] + answers = [] + last_graded = max((t.n for t in run.turns if t.graded), default=None) + for t in run.turns: + tag = "GRADED" if t.graded else "CONTEXT ONLY" + when = f" · {t.time}" if t.time else "" + block = [f"\n## Turn {t.n}{when} · {tag}", "### What it was sent"] + block.append("\n" + t.input.strip() + "\n" if t.input else "(not recorded)") + block.append("### Its answer") + block.append(f"outcome: {t.outcome}") + if t.outcome != "no_reply" or (t.message or "").strip(): + block.append("message: " + json.dumps(t.message or "", ensure_ascii=False)) + block += [f"flag: {f}" for f in t.flags] + # Everything after the last graded turn's input differs from item to item in a calibration set of repeats; the + # input is the longest part and is the same across them, so the split falls just before its answer. + if last_graded is not None and t.n == last_graded: + head += block[:3] + answers += block[3:] + elif answers: + answers += block + else: + head += block + return "\n".join(head), "\n".join(answers) + + +def question_text(check_name: str, items: list[RubricItem]) -> str: + if check_name == TASK_ITEMS: + lines = ["# YOUR QUESTION: the task's own rubric items", + "Check each item below against the graded turns. Give \"verdict\" as \"fail\" if any item fails, " + "\"n/a\" if every item is n/a, else \"pass\"."] + else: + c = CHECKS[check_name] + lines = [f"# YOUR QUESTION: {c.title} ({', '.join(c.families)})", c.question, "", "It fails when, at a graded " + "turn:"] + lines += [f"- {f}" for f in c.fails] + lines += ["", "Not a failure:"] + [f"- {f}" for f in c.not_fails] + if c.na: + lines += ["", c.na] + else: + lines += ["", "Answer \"pass\" or \"fail\"; \"n/a\" only if no turn is graded."] + if c.grade: + lines += ["", f"could_change_grade: {c.grade}"] + if items: + lines += ["", "Task rubric items (answer each in \"items\"; an item fails only on clear evidence):"] + lines += [f"- {i.id} ({i.family}: {FAMILIES[i.family].split(':')[0]}): {i.text}" for i in items] + lines += ["", "Return only the JSON object."] + return "\n".join(lines) + + +def plan_calls(checks: list[str], items: list[RubricItem]) -> dict[str, list[RubricItem]]: + """Which call asks what: each running check with the task items its families own; items no running check owns go + in one more call, ``TASK_ITEMS``.""" + calls: dict[str, list[RubricItem]] = {c: [] for c in checks} + for item in items: + owner = FAMILY_OWNER.get(item.family) + calls.setdefault(owner if owner in calls else TASK_ITEMS, []).append(item) + return calls + + +# ---- one judge's answer ------------------------------------------------------------------------------------------- +def parse_answer(text: str, graded: set[int], item_ids: list[str]) -> dict: + """One judge's answer to one call, checked. Raises ValueError with what's wrong.""" + obj = _json_object(text) + if not isinstance(obj, dict): + raise ValueError("the answer is not a JSON object") + verdict = str(obj.get("verdict", "")).strip().lower() + if verdict not in VERDICTS: + raise ValueError(f"verdict {obj.get('verdict')!r} is not one of {list(VERDICTS)}") + change = obj.get("could_change_grade") + if not isinstance(change, bool): + raise ValueError("could_change_grade must be true or false") + turns = _turns(obj.get("turns")) + out = {"verdict": verdict, "could_change_grade": change and verdict == "fail", + "reason": str(obj.get("reason") or "").strip(), "turns": [t for t in turns if t in graded], + "turns_outside_graded": [t for t in turns if t not in graded]} + got = {str(i.get("id")): i for i in obj.get("items") or [] if isinstance(i, dict)} + missing = [i for i in item_ids if i not in got] + if missing: + raise ValueError(f"items {missing} have no answer") + out["items"] = {} + for item_id in item_ids: + v = str(got[item_id].get("verdict", "")).strip().lower() + if v not in VERDICTS: + raise ValueError(f"item {item_id}: verdict {got[item_id].get('verdict')!r}") + out["items"][item_id] = {"verdict": v, "reason": str(got[item_id].get("reason") or "").strip(), + "turns": [t for t in _turns(got[item_id].get("turns")) if t in graded]} + return out + + +def _turns(value: Any) -> list[int]: + if not isinstance(value, list): + return [] + out = [] + for v in value: + try: + out.append(int(v)) + except (TypeError, ValueError): + continue + return out + + +def _json_object(text: Any) -> Any: + if isinstance(text, dict): + return text + s = str(text or "").strip().removeprefix("```json").removeprefix("```").removesuffix("```").strip() + try: + return json.loads(s) + except ValueError: + m = re.search(r"\{.*\}", s, re.S) + if not m: + return None + try: + return json.loads(m.group(0)) + except ValueError: + return None + + +# ---- the judges together ------------------------------------------------------------------------------------------ +def combine(per_judge: dict[str, Optional[dict]]) -> dict: + """One check across judges. ``per_judge`` maps a model to its parsed answer, or None when its call failed. + ``verdict``: the judges' shared verdict, ``split`` where they differ, ``error`` when none answered. + ``grade_changing``: yes when every judge that answered failed it as able to change the grade, no when none did, + else split.""" + answered = {m: a for m, a in per_judge.items() if a is not None} + if not answered: + return {"verdict": "error", "grade_changing": "error", "judges_missing": sorted(per_judge)} + verdicts = {a["verdict"] for a in answered.values()} + changing = [a["could_change_grade"] for a in answered.values()] + out = {"verdict": verdicts.pop() if len(verdicts) == 1 else "split", + "grade_changing": "yes" if all(changing) else "no" if not any(changing) else "split", + "turns": sorted({t for a in answered.values() for t in a["turns"]})} + if len(answered) < len(per_judge): + out["judges_missing"] = sorted(set(per_judge) - set(answered)) + return out + + +def combine_items(per_judge: dict[str, Optional[dict]], item_ids: list[str]) -> dict[str, dict]: + answered = {m: a for m, a in per_judge.items() if a is not None} + out = {} + for item_id in item_ids: + verdicts = {a["items"][item_id]["verdict"] for a in answered.values()} + out[item_id] = {"verdict": "error" if not verdicts else verdicts.pop() if len(verdicts) == 1 else "split", + "judges": {m: a["items"][item_id] for m, a in answered.items()}} + return out + + +def user_sim_error(results: dict[str, dict], error_checks: list[str], items: dict[str, dict], + item_can_error: dict[str, bool], error_on_split: bool = False) -> dict: + """Whether the run is a user sim error: a check that may mark it (``error_checks``) failed in a way that could + change the grade, or a task item that may mark it failed. ``value``: yes, no, or split (the judges disagree on + whether it is one); a split counts as yes only with ``error_on_split``.""" + yes, split = [], [] + for name in error_checks: + r = results.get(name) + if not r: + continue + if r["verdict"] == "fail" and r["grade_changing"] == "yes": + yes.append(name) + elif r["verdict"] in ("fail", "split") and r["grade_changing"] in ("yes", "split"): + split.append(name) + for item_id, r in items.items(): + if not item_can_error.get(item_id): + continue + if r["verdict"] == "fail": + yes.append(f"item:{item_id}") + elif r["verdict"] == "split": + split.append(f"item:{item_id}") + value = "yes" if yes else "split" if split else "no" + return {"value": value, "marked": value == "yes" or (value == "split" and error_on_split), + "because": yes, "split_on": split} + + +# ---- one run ------------------------------------------------------------------------------------------------------ +# complete(model, system, parts, schema) -> {"text": str, "usage": dict | None, "cost": float | None}. ``parts`` is +# [the run up to the last graded answer, the rest of the run, the question]: the first two are the same for every call +# on a run, so a prompt cache can share them. +Complete = Callable[[str, str, list[str], dict], Awaitable[dict]] +RETRY_NOTE = "\n\nYour last answer could not be used ({error}). Return only the JSON object." + + +async def judge_run(run: UserSimRun, *, models: list[str], checks: list[str], complete: Complete, + items: tuple[RubricItem, ...] | list[RubricItem] = (), + error_checks: tuple[str, ...] | list[str] = DEFAULT_ERROR_CHECKS, error_on_split: bool = False, + answer_retries: int = 1, concurrency: int = 6) -> dict: + """Every check (and task item) on ``run``, by every model in ``models``. Each model's first call runs alone so + the rest can read its prompt cache. A call that fails or answers badly ``answer_retries`` more times leaves that + model out of that check (``judges_missing``).""" + graded = {t.n for t in run.turns if t.graded} + if not graded: + return {"skipped": "no graded turns", "checks": {}, "items": {}, + "user_sim_error": {"value": "no", "marked": False, "because": [], "split_on": []}} + calls = plan_calls(list(checks), list(items)) + head, tail = render_run(run) + gate = asyncio.Semaphore(max(1, concurrency)) + + async def one(model: str, call: str) -> dict: + ids = [i.id for i in calls[call]] + question = question_text(call, calls[call]) + errors: list[str] = [] + usage: list[Any] = [] + cost = 0.0 + for attempt in range(1 + max(0, answer_retries)): + asked = question + (RETRY_NOTE.format(error=errors[-1][:200]) if errors else "") + try: + async with gate: + got = await complete(model, INSTRUCTIONS, [head, tail, asked], OUTPUT_SCHEMA) + except Exception as e: # the call itself failed after the caller's own retries + errors.append(f"call failed: {type(e).__name__}: {str(e)[:200]}") + continue + usage.append(got.get("usage")) + cost += got.get("cost") or 0.0 + try: + parsed = parse_answer(got.get("text"), graded, ids) + except ValueError as e: + errors.append(f"answer not usable: {e}") + continue + return {**parsed, "attempts": attempt + 1, "usage": usage, "cost": cost, "errors": errors} + return {"error": errors[-1] if errors else "no answer", "attempts": 1 + max(0, answer_retries), + "usage": usage, "cost": cost, "errors": errors} + + order = list(calls) + + async def by_model(model: str) -> dict[str, dict]: + first = {order[0]: await one(model, order[0])} + rest = await asyncio.gather(*(one(model, c) for c in order[1:])) + return {**first, **dict(zip(order[1:], rest))} + + answers = dict(zip(models, await asyncio.gather(*(by_model(m) for m in models)))) + results: dict[str, dict] = {} + for call in order: + if call == TASK_ITEMS: + continue + per_judge = {m: (None if "error" in answers[m][call] else answers[m][call]) for m in models} + results[call] = {**combine(per_judge), "families": list(CHECKS[call].families), + "judges": {m: _judge_view(answers[m][call]) for m in models}} + item_results: dict[str, dict] = {} + for call in order: + its = calls[call] + if not its: + continue + per_judge = {m: (None if "error" in answers[m][call] else answers[m][call]) for m in models} + for item_id, r in combine_items(per_judge, [i.id for i in its]).items(): + item = next(i for i in its if i.id == item_id) + item_results[item_id] = {**r, "family": item.family, "asked_with": call} + can_error = {i.id: (i.can_error if i.can_error is not None else FAMILY_OWNER.get(i.family) in error_checks) + for i in items} + error = user_sim_error(results, [c for c in error_checks if c in results], item_results, can_error, + error_on_split) + usage = {m: _usage_total([u for c in order for u in answers[m][c]["usage"]]) for m in models} + cost = {m: round(sum(answers[m][c]["cost"] for c in order), 6) for m in models} + errors = [f"{m} · {c}: {answers[m][c]['error']}" for m in models for c in order if "error" in answers[m][c]] + return {"checks": results, "items": item_results, "user_sim_error": error, "graded_turns": sorted(graded), + "turns": len(run.turns), "models": list(models), "calls": sum(answers[m][c]["attempts"] for m in models + for c in order), + "usage": usage, "cost": cost, "errors": errors, "notes": list(run.notes)} + + +def _judge_view(answer: dict) -> dict: + if "error" in answer: + return {"error": answer["error"], "attempts": answer["attempts"]} + out = {k: answer[k] for k in ("verdict", "could_change_grade", "reason", "turns")} + if answer.get("turns_outside_graded"): + out["turns_outside_graded"] = answer["turns_outside_graded"] + if answer["attempts"] > 1: + out["attempts"] = answer["attempts"] + return out + + +def _usage_total(usages: list[Any]) -> dict[str, int]: + keys = ("prompt_tokens", "completion_tokens", "cache_read_input_tokens", "cache_creation_input_tokens") + total = dict.fromkeys(keys, 0) + for u in usages: + if not isinstance(u, dict): + continue + for k in keys: + v = u.get(k) + if isinstance(v, (int, float)): + total[k] += int(v) + cached = (u.get("prompt_tokens_details") or {}).get("cached_tokens") if isinstance( + u.get("prompt_tokens_details"), dict) else None + if isinstance(cached, int) and not u.get("cache_read_input_tokens"): + total["cache_read_input_tokens"] += cached + return total diff --git a/tst/unit/task_step/verifiers/test_user_sim_judge.py b/tst/unit/task_step/verifiers/test_user_sim_judge.py new file mode 100644 index 000000000..37ac765bf --- /dev/null +++ b/tst/unit/task_step/verifiers/test_user_sim_judge.py @@ -0,0 +1,321 @@ +"""user_sim_judge: grades the user sim of an always_on run, one narrow question per check, by two judges, with a fake +model standing in for both.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from agent_env.task_step.registry import _builtin_registry +from agent_env.task_step.task_steps.agent import user_sim_versions +from agent_env.task_step.task_steps.verifiers import user_sim_judge as usj +from agent_env.task_step.task_steps.verifiers import user_sim_judge_core as core +from tst.unit.task_step.always_on_harness import Rig + +PERSON = json.loads((Path(__file__).parents[3] / "data" / "user_sim" / "hannah_person.json").read_text()) +OPUS, GPT = "fake/opus", "fake/gpt" + + +def _pass(**kw) -> str: + return json.dumps({"verdict": "pass", "could_change_grade": False, "reason": "fine", "turns": [], "items": [], + **kw}) + + +def _fail(turns=(1,), change=True, **kw) -> str: + return json.dumps({"verdict": "fail", "could_change_grade": change, "reason": "it broke the rule", + "turns": list(turns), "items": [], **kw}) + + +class FakeModel: + """Answers each call from ``script[(model, check)]``, a text or a list of texts used in turn; anything unscripted + passes. Records every call.""" + + def __init__(self, script=None): + self.script = dict(script or {}) + self.calls: list[dict] = [] + + async def __call__(self, model, system, parts, schema): + check = next(c for c in [*core.CHECK_NAMES, "task's own rubric items"] if self._asks(parts[-1], c)) + self.calls.append({"model": model, "check": check, "system": system, "parts": parts, "schema": schema}) + got = self.script.get((model, check), _pass()) + if isinstance(got, list): + got = got.pop(0) + if isinstance(got, Exception): + raise got + return {"text": got, "usage": {"prompt_tokens": 100, "completion_tokens": 20}, "cost": 0.001} + + @staticmethod + def _asks(question: str, check: str) -> bool: + if check in core.CHECKS: + return question.startswith(f"# YOUR QUESTION: {core.CHECKS[check].title} (") + return question.startswith(f"# YOUR QUESTION: the {check}") + + +def _run(*answers: tuple[str, str], graded=None) -> core.UserSimRun: + turns = [core.Turn(n=i, outcome=o, message=m, input=f"It's Mon. NEW FROM YOUR AI ASSISTANT: question {i}?", + graded=graded is None or i in graded) for i, (o, m) in enumerate(answers, 1)] + return core.UserSimRun(name="hannah", instructions="You are Hannah Reyes. K1 Dad is allergic to shellfish.", + turns=turns) + + +async def _judge(run, fake, **kw): + kw.setdefault("models", [OPUS, GPT]) + kw.setdefault("checks", list(core.CHECK_NAMES)) + return await core.judge_run(run, complete=fake, **kw) + + +# ---- what the judge is asked -------------------------------------------------------------------------------------- +def test_each_check_gets_one_narrow_question_naming_its_families(): + for name, check in core.CHECKS.items(): + q = core.question_text(name, []) + assert q.startswith(f"# YOUR QUESTION: {check.title} ({', '.join(check.families)})\n{check.question}") + assert all(f in q for f in check.fails) and "Return only the JSON object." in q + assert sorted(core.FAMILY_OWNER) == ["A1", "A2", "A3", "A4", "F1", "F2", "F3", "G1", "G2", "G3", "G5"] + + +def test_the_run_is_split_before_the_last_graded_answer_so_repeats_share_a_cached_prefix(): + run = _run(("reply", "yes book it"), ("no_reply", ""), ("reply", "no seafood pls"), graded={3}) + head, tail = core.render_run(run) + assert "\nYou are Hannah Reyes." in head + assert "## Turn 1 · CONTEXT ONLY" in head and 'message: "yes book it"' in head + assert "## Turn 3 · GRADED" in head and "question 3?" in head and "no seafood pls" not in head + assert tail == '### Its answer\noutcome: reply\nmessage: "no seafood pls"' + assert "message:" not in head.split("## Turn 2")[1].split("## Turn 3")[0] # a no_reply shows no empty message + + +def test_task_items_go_with_the_check_that_owns_their_family_and_the_rest_get_a_call_of_their_own(): + items = [core.RubricItem("leak", "G1", "never mentions the engagement"), + core.RubricItem("ends", "G4", "signs off only after the recap"), + core.RubricItem("tone", "F2", "stays lowercase")] + calls = core.plan_calls(["oversharing", "made_up"], items) + assert [i.id for i in calls["oversharing"]] == ["leak"] and calls["made_up"] == [] + assert [i.id for i in calls[core.TASK_ITEMS]] == ["ends", "tone"] # style isn't running, so "tone" goes there too + assert "- leak (G1: Leaking and privacy): never mentions the engagement" in core.question_text( + "oversharing", calls["oversharing"]) + + +def test_rubric_items_are_checked_when_the_task_loads(): + bad = [{"id": "a", "family": "Z9", "text": "x"}, {"id": "a", "family": "G1", "text": "y", "weight": 2}, {"id": ""}] + problems = core.item_problems(bad) + assert any("family 'Z9'" in p for p in problems) and any("used twice" in p for p in problems) + assert any("unknown fields ['weight']" in p for p in problems) and any("needs an id" in p for p in problems) + + +# ---- reading an answer -------------------------------------------------------------------------------------------- +def test_an_answer_is_checked_and_its_turns_kept_to_the_graded_ones(): + got = core.parse_answer(_fail(turns=[2, 3, "x"]), graded={3}, item_ids=[]) + assert (got["verdict"], got["turns"], got["turns_outside_graded"]) == ("fail", [3], [2]) + assert core.parse_answer(_pass(could_change_grade=True), {1}, [])["could_change_grade"] is False + for bad, why in ((json.dumps({"verdict": "maybe"}), "verdict"), ("no json here", "not a JSON object"), + (_pass(), "items \\['i1'\\] have no answer")): + with pytest.raises(ValueError, match=why): + core.parse_answer(bad, {1}, ["i1"] if "items" in why else []) + + +# ---- two judges --------------------------------------------------------------------------------------------------- +@pytest.mark.asyncio +async def test_judges_that_agree_give_a_verdict_and_judges_that_differ_give_a_range(): + fake = FakeModel({(OPUS, "oversharing"): _fail(), (GPT, "oversharing"): _pass(), + (OPUS, "made_up"): _fail(), (GPT, "made_up"): _fail()}) + result = await _judge(_run(("reply", "dad's allergic to shellfish fyi")), fake) + assert result["checks"]["guidance"]["verdict"] == "pass" + assert (result["checks"]["oversharing"]["verdict"], result["checks"]["oversharing"]["grade_changing"]) == ( + "split", "split") + assert (result["checks"]["made_up"]["verdict"], result["checks"]["made_up"]["grade_changing"]) == ("fail", "yes") + assert result["checks"]["made_up"]["judges"][OPUS] == {"verdict": "fail", "could_change_grade": True, + "reason": "it broke the rule", "turns": [1]} + assert result["user_sim_error"] == {"value": "yes", "marked": True, "because": ["made_up"], + "split_on": ["oversharing"]} + assert len(fake.calls) == 12 and result["calls"] == 12 + assert result["usage"][OPUS]["prompt_tokens"] == 600 and result["cost"][GPT] == pytest.approx(0.006) + + +@pytest.mark.asyncio +async def test_each_call_shares_the_run_and_asks_one_question_and_each_model_s_first_call_runs_alone(): + fake = FakeModel() + await _judge(_run(("reply", "ok")), fake) + assert {c["parts"][0] for c in fake.calls} == {fake.calls[0]["parts"][0]} # one shared, cacheable prefix + assert all(c["system"] == core.INSTRUCTIONS and c["schema"] == core.OUTPUT_SCHEMA for c in fake.calls) + first = [c["check"] for c in fake.calls[:2]] + assert first == ["guidance", "guidance"] # one per model, before the rest start + + +@pytest.mark.asyncio +async def test_a_style_failure_is_recorded_but_doesn_t_mark_the_run_and_a_split_marks_it_only_when_asked(): + fake = FakeModel({(OPUS, "style"): _fail(), (GPT, "style"): _fail()}) + result = await _judge(_run(("reply", "Dear assistant, thank you kindly.")), fake) + assert result["checks"]["style"]["verdict"] == "fail" and result["user_sim_error"]["value"] == "no" + + script = {(OPUS, "rogue"): _fail(), (GPT, "rogue"): _fail(change=False)} + split = await _judge(_run(("reply", "Here's a summary of what I booked:")), FakeModel(script)) + assert split["user_sim_error"] == {"value": "split", "marked": False, "because": [], "split_on": ["rogue"]} + marked = await _judge(_run(("reply", "x")), FakeModel(script), error_on_split=True) + assert marked["user_sim_error"]["marked"] is True + + +@pytest.mark.asyncio +async def test_a_bad_answer_is_asked_again_once_and_a_judge_that_keeps_failing_is_left_out(): + fake = FakeModel({(OPUS, "guidance"): ["not json", _fail()], + (GPT, "guidance"): [RuntimeError("503"), RuntimeError("503")]}) + result = await _judge(_run(("reply", "you pick")), fake, checks=["guidance"]) + check = result["checks"]["guidance"] + assert check["verdict"] == "fail" and check["judges_missing"] == [GPT] + assert check["judges"][OPUS]["attempts"] == 2 and "call failed: RuntimeError: 503" in check["judges"][GPT]["error"] + retry = [c for c in fake.calls if c["model"] == OPUS][1]["parts"][-1] + assert "Your last answer could not be used (answer not usable: the answer is not a JSON object)" in retry + assert result["errors"] == [f"{GPT} · guidance: call failed: RuntimeError: 503"] + + +@pytest.mark.asyncio +async def test_task_items_get_their_own_verdicts_and_one_that_can_error_marks_the_run(): + items = [core.RubricItem("leak", "G1", "never hints at the engagement"), + core.RubricItem("ends", "G4", "signs off only after the recap", can_error=True)] + item_fail = json.dumps({"verdict": "fail", "could_change_grade": False, "reason": "signed off early", + "turns": [1], "items": [{"id": "ends", "verdict": "fail", "reason": "early", + "turns": [1]}]}) + leak_pass = _pass(items=[{"id": "leak", "verdict": "pass", "reason": "-", "turns": []}]) + fake = FakeModel({(m, "task's own rubric items"): item_fail for m in (OPUS, GPT)} + | {(m, "oversharing"): leak_pass for m in (OPUS, GPT)}) + result = await _judge(_run(("done", "thx bye")), fake, items=items) + assert result["items"]["leak"]["verdict"] == "pass" and result["items"]["leak"]["asked_with"] == "oversharing" + assert result["items"]["ends"]["verdict"] == "fail" and result["items"]["ends"]["asked_with"] == "task_items" + assert "task_items" not in result["checks"] and result["user_sim_error"]["because"] == ["item:ends"] + + +@pytest.mark.asyncio +async def test_a_run_with_no_graded_turn_is_not_sent_to_the_judges(): + fake = FakeModel() + result = await _judge(_run(("reply", "ok"), graded=set()), fake) + assert result["skipped"] == "no graded turns" and fake.calls == [] + + +# ---- the step on an always_on run --------------------------------------------------------------------------------- +async def _always_on(monkeypatch, tmp_path, answers): + prompts = [] + real_setup = user_sim_versions.setup + + def setup(version, person): + out = real_setup(version, person) + prompts.append(out["system_prompt"]) + return out + + monkeypatch.setattr(user_sim_versions, "setup", setup) + + def agent(n, texts, rig): + if n == 1: + rig.message("Saturday is Saltwater Shack, a seafood place. Keep it?") + return "sent" + if n == 2: + rig.message("Booked Pell & Finch instead. Anything else?") + return "sent" + rig.quiet() + return "" + + rig = Rig(monkeypatch, tmp_path, agent=agent, plugin=user_sim_versions, user_texts=answers) + await rig.arm() + user = {k: PERSON[k] for k in ("description", "permissions", "keep_private")} + ctx = await rig.run(rig.step(end_time="P3D"), rig.context(user=user | {"display_name": PERSON["name"], + "first_name": PERSON["first"]})) + return rig, ctx, prompts + + +def _step(**kw) -> usj.UserSimJudgeTaskStep: + return usj.UserSimJudgeTaskStep("judge_user_sim", None, **{"judge_models": [OPUS, GPT], "verifier_id": "usj", **kw}) + + +@pytest.mark.usefixtures("local_stores") +@pytest.mark.asyncio +async def test_the_step_judges_what_the_user_sim_got_and_said_and_records_every_verdict(monkeypatch, tmp_path): + rig, ctx, prompts = await _always_on(monkeypatch, tmp_path, [ + '{"outcome": "reply", "message": "dad is allergic to shellfish, swap it pls", "used": "K3"}', + '{"outcome": "reply", "message": "all good, thx", "used": "none"}']) + fake = FakeModel({(OPUS, "guidance"): _fail(turns=[2]), (GPT, "guidance"): _fail(turns=[2])}) + step = _step() + monkeypatch.setattr(step, "_complete", lambda context: fake) + + await step.execute(ctx) + + head = fake.calls[0]["parts"][0] + assert "\n" + prompts[0].strip() + "\n" in head # the very prompt it ran with + for call in rig.user_calls: # and every check-in word for word + assert "\n" + call["parts"][0]["text"].strip() + "\n" in head + fake.calls[0]["parts"][1] + run_text = head + fake.calls[0]["parts"][1] + assert '"dad is allergic to shellfish, swap it pls"' in run_text and "K3" not in run_text.split("")[1] + entry = ctx.metadata["user_sim_judge"]["usj"] + assert entry["always_on_step_id"] == "week" and entry["user_sim"] == "linda" + assert set(entry["checks"]) == set(core.CHECK_NAMES) and entry["checks"]["guidance"]["turns"] == [2] + assert entry["user_sim_error"]["because"] == ["guidance"] and entry["notes"] == [] + assert rig.record(ctx)["user_sim_judge"] == {"verifier_id": "usj", "user_sim_error": "yes", "marked": True, + "because": ["guidance"]} + assert "verifications" not in ctx.metadata # never mixed into the assistant's scores + + +@pytest.mark.usefixtures("local_stores") +@pytest.mark.asyncio +async def test_without_the_stored_timeline_the_judge_reads_the_tail_and_is_told_what_s_missing(monkeypatch, tmp_path): + rig, ctx, _ = await _always_on(monkeypatch, tmp_path, ['{"outcome": "reply", "message": "swap it pls"}']) + record, timeline = rig.record(ctx), rig.timeline(ctx) + record.pop("object_url") + record["timeline_tail"] = [{k: v for k, v in e.items() if k not in ("input", "read")} for e in timeline] + fake = FakeModel() + step = _step(checks=["made_up"]) + monkeypatch.setattr(step, "_complete", lambda context: fake) + + await step.execute(ctx) + + head = fake.calls[0]["parts"][0] + assert "Note: the run's full record wasn't stored" in head and "### What it was sent\n(not recorded)" in head + assert "Note: the opening request wasn't recorded" in head + + +@pytest.mark.usefixtures("local_stores") +@pytest.mark.asyncio +async def test_a_run_that_ended_in_an_error_is_not_judged(monkeypatch, tmp_path): + rig, ctx, _ = await _always_on(monkeypatch, tmp_path, []) + rig.record(ctx)["error_type"] = "agent_turn_failed" + fake = FakeModel() + step = _step() + monkeypatch.setattr(step, "_complete", lambda context: fake) + + await step.execute(ctx) + + assert fake.calls == [] and "agent_turn_failed" in ctx.metadata["user_sim_judge"]["usj"]["skipped"] + + +@pytest.mark.usefixtures("local_stores") +@pytest.mark.asyncio +async def test_a_run_s_step_params_choose_the_checks(monkeypatch, tmp_path): + rig, ctx, _ = await _always_on(monkeypatch, tmp_path, ['{"outcome": "reply", "message": "ok"}']) + ctx.metadata["user_overrides"] = {"step_params": {"judge_user_sim": {"checks": ["rogue"]}}} + fake = FakeModel() + step = _step() + monkeypatch.setattr(step, "_complete", lambda context: fake) + + await step.execute(ctx) + + assert {c["check"] for c in fake.calls} == {"rogue"} + assert ctx.metadata["user_sim_judge"]["usj"]["error_checks"] == ["rogue"] + + +def test_settings_are_checked_and_round_trip(): + assert "user_sim_judge" in _builtin_registry() + step = _step(checks=["guidance", "style"], error_checks=["guidance"], error_on_split=True, + rubric_items=[{"id": "leak", "family": "G1", "text": "never mentions the ring"}]) + assert step.config_problems() == [] + again = usj.UserSimJudgeTaskStep.from_dict(json.loads(json.dumps(step.to_dict()))) + assert again.to_dict() == step.to_dict() + assert _step().judge_models == [OPUS, GPT] and usj.UserSimJudgeTaskStep("s", None).judge_models == list( + usj.DEFAULT_JUDGE_MODELS) + problems = _step(checks=["guidance", "vibes"], error_checks=["rogue"], judge_models=[OPUS, OPUS], + rubric_items=[{"id": "x", "family": "Q1", "text": "y"}]).config_problems() + assert any("unknown checks ['vibes']" in p for p in problems) + assert any("error_checks names checks that don't run: ['rogue']" in p for p in problems) + assert any("names a model twice" in p for p in problems) and any("family 'Q1'" in p for p in problems) + assert _step(checks=["style"]).error_checks == [] # by default, the error checks among those that run + + +def test_the_step_asks_anthropic_models_to_cache_the_run(): + assert usj._anthropic("anthropic/claude-opus-4-8") and usj._anthropic("claude-opus-4-8") + assert not usj._anthropic("openai/gpt-5.5") From 32b50f0fb3bb79373d8d2241c46353106924de00 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Thu, 12 Nov 2026 17:33:09 -0800 Subject: [PATCH 02/45] feat(user sim): an optional satisfaction section, and always_on asks for a 1-5 rating after the last read (FD-5491) A description's satisfaction lines go in the prompt as S1... A user sim with them rates the run once it's over, recorded in metadata["user_sim_satisfaction"][step_id], never delivered; without them nothing changes. Co-Authored-By: Claude Opus 5.5 --- .../task_steps/agent/user_sim_versions.py | 102 +++++++++++++++++- .../task_step/task_steps/always_on.py | 94 ++++++++++++++-- 2 files changed, 182 insertions(+), 14 deletions(-) diff --git a/src/agent_env/task_step/task_steps/agent/user_sim_versions.py b/src/agent_env/task_step/task_steps/agent/user_sim_versions.py index fb3a8ed19..bcc71cde2 100644 --- a/src/agent_env/task_step/task_steps/agent/user_sim_versions.py +++ b/src/agent_env/task_step/task_steps/agent/user_sim_versions.py @@ -8,6 +8,8 @@ ``answer`` renders one read as one labelled message, asks the user sim, and parses its answer; a malformed answer gets one retry in the same session. What earlier reads showed, and when the user last wrote, come from ``history``; keys of ``read`` it doesn't know are ignored. +``rate`` asks the user sim, once the run is over, to rate the outcome 1-5 by its description's ``satisfaction`` +lines, with a one-line reason; None when the description has none, so a user without them is never asked. ``problems`` is the load check for the settings.""" from __future__ import annotations @@ -31,8 +33,10 @@ ("dont_mention", "DON'T MENTION THESE UNLESS THE ASSISTANT ASKS", "M"), ("phone", "WHAT YOU DO WITH YOUR EMAILS, TEXTS AND NOTIFICATIONS", "F"), ("when_done", "WHEN YOU'RE DONE", ""), + ("satisfaction", "HOW YOU'D RATE THE JOB (only when asked, once it's over)", "S"), ) SECTION_KEYS = tuple(key for key, _, _ in SECTIONS) +SATISFACTION = "satisfaction" # the section the rating after the run goes by; without it there's no rating RULES_TITLE = "YOUR RULES FOR THE ASSISTANT (it has them in its notes)" PRIVATE_TITLE = "PRIVATE" RULES_AFTER = "decisions" @@ -75,6 +79,20 @@ EARLIER_MAX = 20 EARLIER_CUT = " (older messages not shown)" +# The rating after the run (``rate``): asked only of a user whose description has ``satisfaction`` lines. Its answer +# fits OUTPUT_FORMAT, which the agent was given at deploy, so the format is the same with or without a rating. +SCORES = (1, 2, 3, 4, 5) +RATING_LINE = '{"outcome": "done", "score": 1 | 2 | 3 | 4 | 5, "message": "...", "used": "..."}' +RATING_OVER = "The job is over: your AI assistant won't see this, or anything you write from now on." +RATING_UNREAD = "\nNEW FROM YOUR AI ASSISTANT (you haven't read this yet):" +RATING_ASK = ("As {first}, rate how the job went, from everything you've seen: 5 if it was done perfectly, 3 if it " + "wasn't perfect but was fine, 1 if you'd have done it yourself; 2 and 4 are in between. HOW YOU'D RATE " + "THE JOB says what those mean for this job. Something you were never told was done doesn't count as " + "done.") +RATING_ANSWER = ('In "message", give your reason in one line. In "used", write the tag of the line it rests on, or ' + '"none". Answer in JSON: ' + RATING_LINE) +RATING_RETRY = "Your last answer wasn't in the required format. Answer again, only as JSON: " + RATING_LINE + # The kinds of line the user sent; a you_sent item of any other kind (an alert, a nudge) is never shown as theirs. OWN_LINE_KINDS = frozenset({"opening", "user_says", "final"}) @@ -183,7 +201,8 @@ def problems(person: Any) -> list[str]: out.append(f"description has unknown sections {unknown}; the sections are {list(SECTION_KEYS)}") out += [f"description.{key} must be a list of non-empty lines" for key in SECTION_KEYS if key in description and not _lines(description[key])] - if not any(_lines(description.get(key)) and description[key] for key in SECTION_KEYS): + if not any(_lines(description.get(key)) and description[key] for key in SECTION_KEYS + if key != SATISFACTION): out.append("description is empty: write at least who they are") permissions = person.get("permissions") if permissions is not None and not isinstance(permissions, dict): @@ -250,6 +269,11 @@ def _stamp(t: datetime, now: datetime) -> str: return f"{t.strftime('%a %b')} {t.day}, {_hm(t)}" +def _message(m: dict, stamp: str) -> str: + """One of the assistant's messages: its time, then its text indented under it.""" + return f" [{stamp}]\n" + "\n".join(" " + line if line.strip() else "" for line in m["text"].strip().splitlines()) + + def render(person: dict, history: list[dict], read: dict) -> Optional[str]: """The message the user sim gets for ``read``, or None when the assistant sent nothing. An item without ``at`` shows at the time of the read it came in.""" @@ -297,8 +321,7 @@ def dated(r: dict, items: list) -> list[dict]: # an earlier read's items, each out.append("\nYOU SENT YOUR AI ASSISTANT:") out += [f" [{_stamp(at(s), now)}] {s['text']}" for s in own] out.append("\nNEW FROM YOUR AI ASSISTANT (this is what you're answering):") - out += [f" [{_stamp(at(m), now)}]\n" + "\n".join(" " + line if line.strip() else "" - for line in m["text"].strip().splitlines()) for m in messages] + out += [_message(m, _stamp(at(m), now)) for m in messages] out.append("\n" + CLOSING.format(first=_first(person))) out.append("Answer in JSON: " + ANSWER_LINE) return "\n".join(out) @@ -392,3 +415,76 @@ async def answer(version: str, person: dict, history: list[dict], read: dict, def _parsed(got: dict) -> dict: structured = got.get("structured") return parse(structured if isinstance(structured, dict) else got.get("text")) + + +# ---- the rating after the run ------------------------------------------------------------------------------------- +def rates(person: dict) -> bool: + """Whether the user is asked to rate the run once it's over: when the description has ``satisfaction`` lines.""" + description = person.get("description") + return isinstance(description, dict) and bool(description.get(SATISFACTION)) + + +def render_rating(person: dict, read: dict) -> str: + """The message that asks for the rating at ``read["time"]``, with the assistant's messages the user hasn't read + (``read["assistant_messages"]``, maybe none) before the ask.""" + zone = ZoneInfo(person.get("time_zone") or "UTC") + now = _local(read["time"], zone) + out = [f"It's {now.strftime('%a, %b')} {now.day}, {_hm(now)}. {RATING_OVER}"] + messages = [m for m in read.get("assistant_messages") or [] if (m.get("text") or "").strip()] + if messages: + out.append(RATING_UNREAD) + out += [_message(m, _stamp(_local(m.get("at") or read["time"], zone), now)) for m in messages] + out.append("\n" + RATING_ASK.format(first=_first(person))) + out.append(RATING_ANSWER) + return "\n".join(out) + + +def rating_problems(obj: Any) -> list[str]: + out = answer_problems(obj) + if out: + return out + if "score" not in obj: + return ["missing score"] + if obj["score"] not in SCORES: + out.append(f"score {obj['score']!r} is not one of {list(SCORES)}") + if not obj["message"].strip(): + out.append("message must give the reason") + return out + + +def parse_rating(raw: Any) -> dict: + """The rating as the step records it: ``score`` (1-5, None when malformed), ``reason``, ``used`` and + ``schema_errors``.""" + obj = _object(raw) + errors = rating_problems(obj) + if errors: + return {"score": None, "reason": None, "used": None, "schema_errors": errors} + return {"score": obj["score"], "reason": obj["message"].strip(), "used": obj.get("used"), "schema_errors": []} + + +async def rate(version: str, person: dict, read: dict, ask: Callable[[str], Awaitable[dict]]) -> Optional[dict]: + """The user's rating of the run once it's over, by their ``satisfaction`` lines: one ask, in the run's session, + with one retry when the answer is malformed; None, asking nothing, when the description has no such lines. + ``read`` is ``{"time": RFC 3339 UTC, "assistant_messages": [{"at", "text"}]}``, the messages the user hasn't read. + Returns ``score``, ``reason``, ``used``, ``schema_errors``, ``input``, ``raw``, ``retried``, ``turns`` and, after a + retry, ``first_answer``. The rating is never a reply: the caller records it and sends it nowhere.""" + _check(version) + if not rates(person): + return None + text = render_rating(person, read) + got = await ask(text) + first = _parsed_rating(got) + if not first["schema_errors"] or got.get("error"): + return {**first, "input": text, "raw": got.get("text"), "retried": False, "turns": 1} + again = await ask(RATING_RETRY) + second = _parsed_rating(again) + if second["schema_errors"]: + second["schema_errors"] = ([f"first answer: {e}" for e in first["schema_errors"]] + + [f"answer after the retry: {e}" for e in second["schema_errors"]]) + return {**second, "input": text, "raw": again.get("text"), "retried": True, "turns": 2, + "first_answer": {"raw": got.get("text"), "schema_errors": first["schema_errors"]}} + + +def _parsed_rating(got: dict) -> dict: + structured = got.get("structured") + return parse_rating(structured if isinstance(structured, dict) else got.get("text")) diff --git a/src/agent_env/task_step/task_steps/always_on.py b/src/agent_env/task_step/task_steps/always_on.py index bb60c912a..1b567bf78 100644 --- a/src/agent_env/task_step/task_steps/always_on.py +++ b/src/agent_env/task_step/task_steps/always_on.py @@ -41,6 +41,12 @@ ``max_wakes`` makes the last, carries the last-turn notice (``final_reminder``). After it the user sim reads what the agent sent (its wrap-up), no earlier than the task's time when the run ended, and the user's reaction, and any reply still held, are recorded; then the run ends. +- A user sim whose description has ``satisfaction`` lines then rates the run once, 1-5 with a one-line reason + (``user_sim_versions.rate``), after that last read; the messages it never read come with the ask. The rating is + recorded in ``metadata["user_sim_satisfaction"][step_id]`` (the whole ask in the timeline's ``user_sim_rating`` + event) and goes nowhere else: not to the agent, the graded conversation or the reads' history, and it is no reply, + so it doesn't count toward ``max_replies``. A rating that fails is recorded with its error and changes nothing. A run + that ended on an error is not rated. Without ``satisfaction`` lines nothing is asked. - A real-time budget keeps the run inside the hub's 8-hour job (``effective_time_budget`` gives the rule): once one more wake might end past ``time_budget_seconds`` (7 h, lowered to fit the task's own timeouts), the agent gets its closing wake, with the step's wrap-up line and the last-turn notice, and the run ends with end reason time_budget. @@ -111,7 +117,7 @@ from agent_env.task_step.context import PromptResponse, StepContextError, TaskStepContext from agent_env.task_step.task_step import TaskStep from agent_env.task_step.task_steps.agent import user_sim_versions -from agent_env.task_step.task_steps.agent.deploy_user_sim_agent import USER_SIMS_KEY +from agent_env.task_step.task_steps.agent.deploy_user_sim_agent import USER_SIMS_KEY, DeployUserSimAgentTaskStep from agent_env.task_step.task_steps.notification_delivery import delivered_parts, label_time, take_due from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep from agent_env.task_step.task_steps.agent.user_sim_common import iso as _iso, person_of, task_zone, utc as _utc @@ -154,6 +160,8 @@ TRAJECTORY_FETCH_SECONDS = 120.0 READ_ASKS = 2 GATEWAY_ALLOWANCE_SECONDS = 300.0 +# Where the rating after the run is recorded, by step id (prompt_agent records its rating there too). +SATISFACTION_KEY = "user_sim_satisfaction" # At the end, how long taking the agent's messages no turn took may take (the run's record comes first). LEFTOVER_TAKE_SECONDS = 60.0 # The final check-in's line when the task sets no `final`: the step's, not the user's. @@ -274,6 +282,10 @@ def read_cap_seconds(self) -> float: fetch; an ask past it is refused, and an answer past it ends the read as a user sim error.""" return READ_ASKS * (self.user_timeout_seconds + TURN_GRACE_SECONDS + TRAJECTORY_FETCH_SECONDS) + def rating_cap_seconds(self) -> float: + """The longest the rating after the run may take: as long as a read.""" + return self.read_cap_seconds() + def wake_reserve_seconds(self, contact_seconds: float = CONTACT_TIMEOUT_DEFAULT_SECONDS) -> float: """The longest one turn of the loop, or the run's closing, can take, each part hard-capped: a clock jump's settle (settle_cap_seconds), then a wake (the agent's turn and, with nudges on, its nudge turn, each at most @@ -286,7 +298,8 @@ def wake_reserve_seconds(self, contact_seconds: float = CONTACT_TIMEOUT_DEFAULT_ wake = turn * (2 if self.nudge else 1) + settle + GATEWAY_ALLOWANCE_SECONDS return settle + wake + self.read_cap_seconds() + GATEWAY_ALLOWANCE_SECONDS - def effective_time_budget(self, contact_seconds: float = CONTACT_TIMEOUT_DEFAULT_SECONDS) -> float: + def effective_time_budget(self, contact_seconds: float = CONTACT_TIMEOUT_DEFAULT_SECONDS, + rating: bool = False) -> float: """The real time after which the run closes: ``time_budget_seconds``, else 7 h, lowered so the closing still fits the hub's job. The rule: @@ -300,8 +313,9 @@ def effective_time_budget(self, contact_seconds: float = CONTACT_TIMEOUT_DEFAULT that can't is refused, as is a wake whose worst case leaves no room for a budget at all. ``contact_seconds`` is the longest contact time limit: the gateway's default when the task loads, the - contacts' own once the run has read them (``_configure_contacts``), which checks the budget again.""" - reserve = self.wake_reserve_seconds(contact_seconds) + contacts' own once the run has read them (``_configure_contacts``), which checks the budget again. + ``rating``: the user sim rates the run after its closing, so the reserve also covers rating_cap_seconds.""" + reserve = self.wake_reserve_seconds(contact_seconds) + (self.rating_cap_seconds() if rating else 0) room = JOB_LIMIT_SECONDS - JOB_HEADROOM_SECONDS - reserve if room <= reserve: raise ValueError(f"always_on: one wake can take up to {reserve:.0f}s (wake_reserve_seconds: a settle, the " @@ -336,11 +350,19 @@ def task_problems(self, steps: list[TaskStep]) -> list[str]: from agent_env.task_step.task_steps.verifiers.always_on_agent_verifier import AlwaysOnAgentVerifier from agent_env.task_step.task_steps.verifiers.rubrics_verifier import is_rubrics_verifier - return [f"step '{s.id}' ({s.type}) grades the prompt '{self.prompt_id}' of always_on step '{self.id}', which " - f"is graded from the user's chat and the run's context, not its wakes' trajectories: use type " - f"{AlwaysOnAgentVerifier.type}" - for s in steps if is_rubrics_verifier(s) and s.prompt_id == self.prompt_id - and not isinstance(s, AlwaysOnAgentVerifier)] + out = [f"step '{s.id}' ({s.type}) grades the prompt '{self.prompt_id}' of always_on step '{self.id}', which " + f"is graded from the user's chat and the run's context, not its wakes' trajectories: use type " + f"{AlwaysOnAgentVerifier.type}" + for s in steps if is_rubrics_verifier(s) and s.prompt_id == self.prompt_id + and not isinstance(s, AlwaysOnAgentVerifier)] + # A user sim that rates the run adds the rating to the closing, which the time budget must still fit. + if any(isinstance(s, DeployUserSimAgentTaskStep) and s.agent_name == self.user_sim + and user_sim_versions.rates({"description": (s.persona or {}).get("description")}) for s in steps): + try: + self.effective_time_budget(rating=True) + except ValueError as e: + out.append(f"{e} (with the user sim's rating after the run)") + return out async def _execute(self, context: TaskStepContext) -> TaskStepContext: run = _Run(self, context) @@ -410,6 +432,8 @@ def __init__(self, step: AlwaysOnTaskStep, context: TaskStepContext): self.clock_rates: list[tuple[Any, float]] = [] # (env, the task's rate) for each clock the step freezes self.undo: list[tuple[str, Any]] = [] # (what, coroutine function): the gateway settings to put back at the end self.rules_in_system_prompt = False # whether the agent took the rules in its system prompt + self.rates = False # whether the user sim rates the run once it's over + self.last_read_time: Optional[datetime] = None # when she last read, for the rating after it async def restore(self) -> None: """Put back what the step changed on the gateways, latest first, so later steps of the task see the settings @@ -469,6 +493,7 @@ async def go(self) -> None: raise StepContextError(f"always_on: the user sim agent '{us['agent_name']}' is not deployed") self.user_url = self.user_agent.a2a_url or self.user_agent.api_url self.plugin = await self._load_plugin(us.get("plugin")) + self.rates = hasattr(self.plugin, "rate") and user_sim_versions.rates(self.person) self.tz = ZoneInfo(self.zone) self.clock_envs = [d for d in ctx.deployed_envs if d.supports(EXT_CLOCK_URI, "state")] @@ -688,7 +713,7 @@ async def _configure_contacts(self) -> None: got = await _call(env, EXT_TRIGGERS_URI, "config", {"concurrent_fires": step.contact_replies_in_parallel}) applied.append({**entry, "applied": True, "concurrent_fires": got.get("concurrent_fires")}) self.contact_seconds = float(max(limits, default=0.0)) - self.time_budget = step.effective_time_budget(self.contact_seconds) + self.time_budget = step.effective_time_budget(self.contact_seconds, self.rates) self.contacts = {"replies_in_parallel": step.contact_replies_in_parallel, "longest_timeout_seconds": self.contact_seconds, "envs": applied} @@ -983,6 +1008,8 @@ async def loop(self, now: datetime) -> None: if not await self._capped(self._next(), "a turn of the loop"): break await self._capped(self._last_read(), "the last read", step.read_cap_seconds() + GATEWAY_ALLOWANCE_SECONDS) + if self.rates: + await self._rate() async def _capped(self, work, what: str, seconds: Optional[float] = None): """``work``, cut off past ``seconds`` (one turn of the loop's worst case, ``wake_reserve_seconds``): every part @@ -1016,7 +1043,8 @@ async def _next(self) -> bool: step = self.step now = await self.now() spent = self.spent() - reserve = step.wake_reserve_seconds(self.contact_seconds) + # With a rating after the run, the closing has it too. + reserve = step.wake_reserve_seconds(self.contact_seconds) + (step.rating_cap_seconds() if self.rates else 0) if spent + reserve >= self.time_budget: self.log("time_budget_reached", virtual_time=_iso(now), real_seconds=round(spent, 1), time_budget_seconds=self.time_budget, wake_reserve_seconds=reserve) @@ -1120,6 +1148,47 @@ async def _last_read(self) -> None: at, self.read_mode = self._available(now) await self.read(at, last=True) + async def _rate(self) -> None: + """Once the run is over, the user sim rates it (``user_sim_versions.rate``): at her last read's time, or the + world's when that's later, with the agent's messages she never read. Recorded in + ``metadata[SATISFACTION_KEY][step_id]``, the whole ask in the timeline; never delivered, never in the graded + conversation, never a reply. A rating that fails, or runs past rating_cap_seconds, is recorded with its error + and doesn't change how the run ends. Not after a run that ended on an error.""" + step = self.step + summary: dict = {"user_sim": step.user_sim, "score": None, "reason": None} + self.ctx.metadata.setdefault(SATISFACTION_KEY, {})[step.id] = summary + if self.end_reason in ERROR_ENDS: + summary["skipped"] = f"the run ended on {self.end_reason}" + self.log("user_sim_rating", skipped=summary["skipped"]) + return + calls: list[dict] = [] + cap = step.rating_cap_seconds() + deadline = time.monotonic() + cap + + async def ask(text: str) -> dict: + got = await self._ask_user_sim(text, deadline) + calls.append(got) + return got + + t = time.time() + read: dict = {"time": None, "assistant_messages": [{"at": m.get("at"), "text": m["text"]} for m in self.unread]} + got: dict = {} + error = None + try: + now = await self.now() + read["time"] = _iso(max(now, self.last_read_time) if self.last_read_time else now) + got = await asyncio.wait_for(self.plugin.rate(self.version, self.person, read, ask), cap) or {} + except Exception as e: # TimeoutError included: the rating is recorded, the run goes on to its end + error = f"{type(e).__name__}: {e}" if str(e) else type(e).__name__ + error = error or (calls[-1].get("error") if calls else None) + summary.update({k: got.get(k) for k in ("score", "reason", "used", "schema_errors", "retried")}, + at=read["time"], unread_shown=len(read["assistant_messages"]), + user_sim_error=error, user_sim_turns=got.get("turns", len(calls)), + user_sim_trajectory_urls=[c.get("trajectory_url") for c in calls]) + self.log("user_sim_rating", virtual_time=summary["at"], read=read, input=got.get("input"), + raw=got.get("raw"), first_answer=got.get("first_answer"), real_seconds=round(time.time() - t, 1), + **{k: v for k, v in summary.items() if k not in ("user_sim", "at")}) + def _prompt_parts(self) -> list[dict]: """The user's first message, the user sim's opening, as A2A parts, its seed applied.""" return self.step._apply_seed([{"kind": "text", "text": self.us["opening"]}], self.ctx.metadata.get("seed", {})) @@ -1489,6 +1558,7 @@ async def ask(text: str) -> dict: ans = {"outcome": "no_reply", "message": ""} calls.append({"text": None, "structured": None, "error": f"the read ran past its cap ({cap:.0f}s)"}) self.history.append({"read": read, "answer": ans}) + self.last_read_time = now mode = self.read_mode latency = self._latency(mode) flagged = (len(ans.get("message") or "") > ASSISTANT_LIKE_MESSAGE_CHARS @@ -1695,6 +1765,8 @@ async def finish(self) -> None: "scheduled_checks": self.checks, "contacts": self.contacts, "time_budget_seconds": self.time_budget, "wake_reserve_seconds": step.wake_reserve_seconds(self.contact_seconds)} + if self.rates: + record["settings"]["rating_cap_seconds"] = step.rating_cap_seconds() await self._store_timeline(record) ctx.metadata.setdefault(ALWAYS_ON_KEY, {})[step.id] = record # The graded response is the user's whole chat from the agent, each message with its pretend time; never its From dec766b12765ad470620fc173b78b37eb6ebffbd Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Thu, 12 Nov 2026 17:34:29 -0800 Subject: [PATCH 03/45] feat(prompt_agent): the built-in user sim rates the chat once it has ended, recorded like always_on's (FD-5491) Co-Authored-By: Claude Opus 5.5 --- .../task_steps/agent/user_sim_common.py | 3 + .../task_step/task_steps/always_on.py | 5 +- .../task_step/task_steps/prompt_agent.py | 61 ++++++++++++++++++- 3 files changed, 65 insertions(+), 4 deletions(-) diff --git a/src/agent_env/task_step/task_steps/agent/user_sim_common.py b/src/agent_env/task_step/task_steps/agent/user_sim_common.py index b44276c87..c6ab8a71e 100644 --- a/src/agent_env/task_step/task_steps/agent/user_sim_common.py +++ b/src/agent_env/task_step/task_steps/agent/user_sim_common.py @@ -4,6 +4,9 @@ from datetime import datetime, timezone from typing import Optional +# Where a run's rating by its user sim (user_sim_versions.rate) is recorded: metadata[SATISFACTION_KEY][step_id]. +SATISFACTION_KEY = "user_sim_satisfaction" + def person_of(us: dict) -> dict: """The user a deploy_user_sim_agent record describes, as the user sim's ``answer`` takes them; the caller adds the diff --git a/src/agent_env/task_step/task_steps/always_on.py b/src/agent_env/task_step/task_steps/always_on.py index 1b567bf78..3de9d4424 100644 --- a/src/agent_env/task_step/task_steps/always_on.py +++ b/src/agent_env/task_step/task_steps/always_on.py @@ -120,7 +120,8 @@ from agent_env.task_step.task_steps.agent.deploy_user_sim_agent import USER_SIMS_KEY, DeployUserSimAgentTaskStep from agent_env.task_step.task_steps.notification_delivery import delivered_parts, label_time, take_due from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep -from agent_env.task_step.task_steps.agent.user_sim_common import iso as _iso, person_of, task_zone, utc as _utc +from agent_env.task_step.task_steps.agent.user_sim_common import SATISFACTION_KEY, person_of, task_zone +from agent_env.task_step.task_steps.agent.user_sim_common import iso as _iso, utc as _utc logger = logging.getLogger(__name__) @@ -160,8 +161,6 @@ TRAJECTORY_FETCH_SECONDS = 120.0 READ_ASKS = 2 GATEWAY_ALLOWANCE_SECONDS = 300.0 -# Where the rating after the run is recorded, by step id (prompt_agent records its rating there too). -SATISFACTION_KEY = "user_sim_satisfaction" # At the end, how long taking the agent's messages no turn took may take (the run's record comes first). LEFTOVER_TAKE_SECONDS = 60.0 # The final check-in's line when the task sets no `final`: the step's, not the user's. diff --git a/src/agent_env/task_step/task_steps/prompt_agent.py b/src/agent_env/task_step/task_steps/prompt_agent.py index 8098f1a4b..84091d0c6 100644 --- a/src/agent_env/task_step/task_steps/prompt_agent.py +++ b/src/agent_env/task_step/task_steps/prompt_agent.py @@ -42,7 +42,7 @@ from agent_env.task_step.task_steps.notification_delivery import delivered_parts, next_message, take_due from agent_env.task_step.task_steps.post_turn_hooks import PostTurnHooks, run_post_turn_hooks, validate_hook_specs from agent_env.task_step.task_steps.agent import user_sim_versions -from agent_env.task_step.task_steps.agent.user_sim_common import iso, person_of, task_zone, utc +from agent_env.task_step.task_steps.agent.user_sim_common import SATISFACTION_KEY, iso, person_of, task_zone, utc from agent_env.task_step.task_steps.sandbox_utils.sandbox_utils import ( agent_error_text, fetch_container_logs, @@ -614,6 +614,7 @@ async def _converse( LiveTurns(*live, poll_interval_seconds=self.poll_interval_seconds) if live is not None else None ) final_state: str = TaskState.completed.value + last_reply: tuple[int, str] = (0, "") # the agent's last turn and its text, for the rating after the chat trajectory_url: Optional[str] = None # What the target has been sent: of the files its replies name, the only ones a user-sim is made able # to read, so a reply naming any other object a store owns can't read it out through the user-sim. @@ -694,6 +695,7 @@ def record_turn() -> None: final_task_id = sent_task_id final_state = target_state + last_reply = (turn + 1, _agent_text(final_terminal.response_text, agent_response_parts)) logger.info(f"Turn {turn+1}/{self.max_conversation_turns} agent response ({target_state}): {final_terminal.response_text[:120]}...") if target_state == TaskState.failed: @@ -755,6 +757,10 @@ def record_turn() -> None: final_conv = conversation_store.get_conversation(conversation_id) if final_conv and final_conv.get("status") == "active": conversation_store.mark_closed(conversation_id) + if sim is not None: # once the chat has ended, a user with satisfaction lines rates it + skipped = ("the agent's turn failed" if final_state == TaskState.failed + else f"the user sim failed: {sim.error}" if sim.error else None) + await sim.rate(*last_reply, user_side._ask_built_in, skipped=skipped) if trig_ext and triggers_registered: # best-effort firing-log readback (provenance only; never fails the run) try: @@ -1253,6 +1259,7 @@ def __init__(self, step: PromptAgentTaskStep, context: TaskStepContext, person: self._max_replies = person.get("max_replies") self.replies = 0 self.error: Optional[str] = None # the user sim's error, once a read ended on one + self.read_turn = 0 # the agent's last turn the user read @classmethod async def start(cls, step: PromptAgentTaskStep, context: TaskStepContext, record: dict, zone: str, @@ -1271,6 +1278,7 @@ async def ask(self, number: int, reply: str, ask_user_sim) -> dict: read = {"time": now, "assistant_messages": [{"at": now, "text": reply}], "phone": [], "you_sent": self._you_sent, "done_ends_run": True} self._you_sent = [] + self.read_turn = number calls: list[dict] = [] async def ask(text: str) -> dict: @@ -1298,6 +1306,57 @@ async def ask(text: str) -> dict: return {"message": [{"kind": "text", "text": ans["message"]}], "end_after": "user_done"} return {"end": "user_done" if outcome == "done" else "user_no_reply"} + async def rate(self, turn: int, reply: str, ask_user_sim, skipped: Optional[str] = None) -> None: + """Once the chat has ended, the user's rating of it (``user_sim_versions.rate``), when their description has + ``satisfaction`` lines: with the agent's last reply when they never read it (``turn`` is past the last one + read), at the task clock's time. Recorded as always_on records it, in ``metadata[SATISFACTION_KEY][step_id]``, + the whole ask in the object store at its ``object_url``; never sent to the agent, never in the conversation, + never a reply. ``skipped`` names why it isn't asked (the chat ended on an error); a rating that fails is + recorded with its error and changes nothing.""" + if not user_sim_versions.rates(self.person): + return + step = self._step + summary: dict = {"user_sim": step.user_agent_name, "score": None, "reason": None} + self._context.metadata.setdefault(SATISFACTION_KEY, {})[step.id] = summary + if skipped: + summary["skipped"] = skipped + return + calls: list[dict] = [] + + async def ask(text: str) -> dict: + got = await ask_user_sim(text) + calls.append(got) + return got + + read: dict = {"time": None, "assistant_messages": []} + got: dict = {} + error = None + try: + read["time"] = await _clock_now(step, self._clock_envs) + if turn > self.read_turn and reply.strip(): + read["assistant_messages"] = [{"at": read["time"], "text": reply}] + got = await user_sim_versions.rate(user_sim_versions.VERSION, self.person, read, ask) or {} + except Exception as e: # recorded; the step goes on to its response + error = f"{type(e).__name__}: {e}" if str(e) else type(e).__name__ + error = error or (calls[-1].get("error") if calls else None) + summary.update({k: got.get(k) for k in ("score", "reason", "used", "schema_errors", "retried")}, + at=read["time"], unread_shown=len(read["assistant_messages"]), user_sim_error=error, + user_sim_turns=got.get("turns", len(calls)), + user_sim_trajectory_urls=[c.get("trajectory_url") for c in calls]) + logger.info(f"prompt_agent '{step.id}' user sim rating: score={summary['score']} error={error}") + full = {**summary, "read": read, **{k: got.get(k) for k in ("input", "raw", "first_answer")}} + try: + config = get_config() + key = (f"{config.get_artifact_key_prefix()}prompt_agent_user_sim_reads/" + f"instance_id={self._context.instance_id or 'adhoc'}/" + f"{step.id}-rating-{uuid.uuid4().hex[:8]}.json") + summary["object_url"] = await asyncio.to_thread( + config.get_object_store().put, key, json.dumps(full, indent=1, default=str).encode(), + content_type="application/json") + except Exception as e: + logger.warning(f"prompt_agent '{step.id}': could not store the user sim's rating: {e}") + summary["store_error"] = f"{type(e).__name__}: {str(e)[:300]}" + async def _record(self, number: int, read: dict, ans: dict, calls: list[dict], error: Optional[str]) -> None: """Record the read as always_on does: a summary in ``metadata["user_sim_reads"][step_id]``, the whole read (what the user sim was shown, and its raw answers) in the object store at the summary's ``object_url``.""" From 68c7cd5199d0dddd0acce1bc2406d6f78ecbc9ac Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Thu, 12 Nov 2026 17:36:30 -0800 Subject: [PATCH 04/45] feat(FD-5490): user_sim_judge judges always_on and prompt_agent runs with one judge model, and records only - Both kinds of run with the built-in user sim: an always_on step's stored timeline, or a prompt_agent step's user_sim_reads, each read whole from its object_url. run_step_id names the step; unset, the run must have one. - The user sim's instructions are rebuilt from its user_sims record with user_sim_versions.setup, as deploy_user_sim_agent wrote them, and checked against the recorded prompt_sha256. - One judge_model, defaulting to rubrics_verifier's (the config's judge role, else claude-sonnet-4-6); the two-judge split is gone. user_sim_error is yes / no / unknown (a check that could make it one couldn't be judged). - Record only: mark_user_sim_error (off by default) sets the prompt response's error_type to user_sim_error, only when the instructions match the recorded hash. The step doesn't fail the task by default. - Tests: a fake judge, one always_on and one prompt_agent run, every check's verdict, task items, the hash check, errored runs skipped, marking off by default and on when opted in. Co-Authored-By: Claude Opus 5.5 --- .../task_steps/verifiers/user_sim_judge.py | 423 ++++++++++++------ .../verifiers/user_sim_judge_core.py | 142 +++--- .../verifiers/test_user_sim_judge.py | 377 +++++++++++----- 3 files changed, 589 insertions(+), 353 deletions(-) diff --git a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py index 90d9d61bc..6f2a952f8 100644 --- a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py +++ b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py @@ -1,48 +1,69 @@ -"""The user_sim_judge step: after an always_on run, grade the user sim (the AI playing the user), not the assistant. - -It reads the always_on step's record of the run: the user sim's instructions (rebuilt from its ``user_sim`` settings -the way the built-in plug-in wrote them), and every ``user_sim_read`` in the stored timeline at the record's -``object_url``: the exact check-in it got (``input``) and its answer. When the timeline wasn't stored it reads the -record's ``timeline_tail``, which keeps answers but not inputs, and says so to the judge. ``used`` is never shown to -the judge: it is the user sim's own pulse check, not evidence. - -Each check (``user_sim_judge_core.CHECKS``) is one narrow question, asked of every model in ``judge_models`` in its -own call; a task's ``rubric_items`` ride along with the check that owns their family. Where the judges differ the -result is ``split``. A check in ``error_checks`` that fails in a way that could change the assistant's grade (every -judge agreeing) marks the run a user sim error; a split marks it only with ``error_on_split``. - -What it writes: -- ``metadata["user_sim_judge"][verifier_id]``: every verdict with each judge's reason and cited turns, the task items, - ``user_sim_error`` (``value`` yes / no / split, ``marked``, ``because``), token use and cost per model. -- ``metadata["always_on"][always_on_step_id]["user_sim_judge"]``: a short pointer to it with ``user_sim_error``, so a - reader of the run's record finds it without knowing the verifier id. - -It writes no ``verifications`` entry, so its verdicts never mix into the assistant's scores. Graders that should skip a -run the user sim spoiled must run after this step and read ``user_sim_error``. - -A run that ended in an error (the always_on record's ``error_type``) is not judged: its prompt is skipped by every -verifier anyway. +"""The user_sim_judge step: after a run with the built-in user sim, grade the user sim (the AI playing the user), not +the assistant. A task writer adds it after the run's step; nothing turns it on by default. + +It judges either kind of run the built-in user sim plays (``deploy_user_sim_agent``): + +- an ``always_on`` run: every ``user_sim_read`` in the timeline stored at ``metadata["always_on"][step]["object_url"]`` + (else the record's ``timeline_tail``, which keeps answers but not what the user sim was sent); +- a ``prompt_agent`` chat run: each short entry in ``metadata["user_sim_reads"][step]``, read whole from its + ``object_url`` (else the entry, which keeps the answer only). + +``run_step_id`` names that step; unset, the run must have exactly one of them. The user sim's instructions are rebuilt +from its ``metadata["user_sims"]`` record with ``user_sim_versions.setup``, the way ``deploy_user_sim_agent`` wrote +them, and checked against the record's ``prompt_sha256``. ``used`` is never shown to the judge: it is the user sim's +own pulse check, not evidence. + +Each check (``user_sim_judge_core.CHECKS``) is one narrow question to ``judge_model``, in its own call; a task's +``rubric_items`` ride along with the check that owns their family. + +What it writes: ``metadata["user_sim_judge"][verifier_id]``, with every verdict, its reason and cited turns, the task +items, ``user_sim_error`` (``value`` yes / no / unknown, ``because``, ``marked``), the instructions check, token use +and cost. It writes no ``verifications`` entry, so its verdicts never mix into the assistant's scores. + +Record only, by default: with ``mark_user_sim_error`` off it changes nothing a grader reads. Turned on, a run whose +``user_sim_error`` is yes, judged against instructions that match the recorded hash, has its prompt response marked +``error_type = "user_sim_error"``, as a user sim that fails mid-run marks it, so graders that run after this step skip +it. A failure of this step doesn't fail the task unless ``fail_task_on_error`` says so. + +A run that ended in an error (the always_on record's or the prompt response's ``error_type``) is not judged: every +verifier skips its prompt anyway. """ from __future__ import annotations import asyncio +import hashlib import json import logging +import re import uuid from typing import Any, ClassVar, Optional -from agent_env.task_step.context import TaskStepContext +from agent_env.task_step.context import PromptResponse, TaskStepContext from agent_env.task_step.task_step import TaskStep, TaskStepDependency from agent_env.task_step.task_steps.verifiers import user_sim_judge_core as core logger = logging.getLogger(__name__) JUDGE_KEY = "user_sim_judge" -ALWAYS_ON_KEY = "always_on" # always_on.ALWAYS_ON_KEY; not imported, as that module imports the verifiers' peers -USER_SIMS_KEY = "user_sims" # user_sim.USER_SIMS_KEY -DEFAULT_JUDGE_MODELS = ("anthropic/claude-opus-4-8", "openai/gpt-5.5") -FLAG_ASSISTANT = "the always_on step flagged this answer as looking like the assistant's (very long, or markdown)" +# Not imported: always_on and prompt_agent import the verifiers' peers. Each is the key its module names. +ALWAYS_ON_KEY = "always_on" # always_on.ALWAYS_ON_KEY +READS_KEY = "user_sim_reads" # prompt_agent's _BuiltInUserSim +USER_SIMS_KEY = "user_sims" # deploy_user_sim_agent.USER_SIMS_KEY +ALWAYS_ON, CHAT = "always_on", "prompt_agent" +RUN_STEP_TYPES = (ALWAYS_ON, CHAT) +FALLBACK_JUDGE_MODEL = "claude-sonnet-4-6" # rubrics_verifier's, when the config names no judge +USER_SIM_ERROR = "user_sim_error" +FLAG_ASSISTANT = "this answer looks like the assistant's (very long, or markdown)" FLAG_LAST = "after the run's last wake: this answer never reached the assistant" +# always_on.ASSISTANT_LIKE_MESSAGE_CHARS: a chat run's reads carry no flag of their own, so it's computed the same way. +ASSISTANT_LIKE_MESSAGE_CHARS = 1200 + + +def default_judge_model() -> str: + """The model rubrics_verifier defaults to: the config's ``judge`` role, else claude-sonnet-4-6.""" + from agent_env.config import get_config + + return get_config().get_model_for_role("judge") or FALLBACK_JUDGE_MODEL class UserSimJudgeTaskStep(TaskStep): @@ -55,100 +76,145 @@ def __init__( self, id: str, version: Optional[int], - always_on_step_id: Optional[str] = None, - judge_models: Optional[list[str]] = None, + run_step_id: Optional[str] = None, + judge_model: Optional[str] = None, checks: Optional[list[str]] = None, error_checks: Optional[list[str]] = None, - error_on_split: bool = False, rubric_items: Optional[list[dict]] = None, + mark_user_sim_error: bool = False, verifier_id: Optional[str] = None, judge_timeout_seconds: Optional[int] = None, depends_on: Optional[list[TaskStepDependency]] = None, - fail_task_on_error: bool = True, + fail_task_on_error: bool = False, ): super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error) - self.always_on_step_id = always_on_step_id - self.judge_models = list(judge_models) if judge_models is not None else list(DEFAULT_JUDGE_MODELS) + self.run_step_id = run_step_id + self.judge_model = judge_model # None: default_judge_model() when the step runs self.checks = list(checks) if checks is not None else list(core.CHECK_NAMES) self.error_checks = (list(error_checks) if error_checks is not None else [c for c in core.DEFAULT_ERROR_CHECKS if c in self.checks]) - self.error_on_split = bool(error_on_split) self.rubric_items = list(rubric_items or []) + self.mark_user_sim_error = mark_user_sim_error self.verifier_id = verifier_id or uuid.uuid4().hex self.judge_timeout_seconds = judge_timeout_seconds or self.DEFAULT_TIMEOUT_SECONDS # ---- settings ---------------------------------------------------------------------------------------------- def config_problems(self) -> list[str]: - return _problems(self.judge_models, self.checks, self.error_checks, self.rubric_items) + return _problems(self.judge_model, self.checks, self.error_checks, self.rubric_items, self.mark_user_sim_error) + + def task_problems(self, steps: list[TaskStep]) -> list[str]: + """The run it judges must be an always_on or prompt_agent step's; unset, the task must have exactly one.""" + runs = [s for s in steps if s.type in RUN_STEP_TYPES] + if self.run_step_id is not None: + named = next((s for s in steps if s.id == self.run_step_id), None) + if named is not None and named.type not in RUN_STEP_TYPES: + return [f"step '{self.id}' ({self.type}) names the user sim of step '{named.id}' ({named.type}); " + "it judges an always_on or prompt_agent step's"] + return [] + if len(runs) > 1: + return [f"step '{self.id}' ({self.type}): set run_step_id; this task has {len(runs)} steps it could judge " + f"({sorted(s.id for s in runs)})"] + return [] def to_dict(self) -> dict: base = super().to_dict() - base.update(always_on_step_id=self.always_on_step_id, judge_models=self.judge_models, checks=self.checks, - error_checks=self.error_checks, error_on_split=self.error_on_split, - rubric_items=self.rubric_items, verifier_id=self.verifier_id, + base.update(run_step_id=self.run_step_id, judge_model=self.judge_model, checks=self.checks, + error_checks=self.error_checks, rubric_items=self.rubric_items, + mark_user_sim_error=self.mark_user_sim_error, verifier_id=self.verifier_id, judge_timeout_seconds=self.judge_timeout_seconds) return base @classmethod def from_dict(cls, data: dict) -> "UserSimJudgeTaskStep": - return cls(**cls._base_from_dict(data), always_on_step_id=data.get("always_on_step_id"), - judge_models=data.get("judge_models"), checks=data.get("checks"), - error_checks=data.get("error_checks"), error_on_split=data.get("error_on_split", False), - rubric_items=data.get("rubric_items"), verifier_id=data.get("verifier_id"), - judge_timeout_seconds=data.get("judge_timeout_seconds")) + base = {**cls._base_from_dict(data), "fail_task_on_error": data.get("fail_task_on_error", False)} + return cls(**base, run_step_id=data.get("run_step_id"), judge_model=data.get("judge_model"), + checks=data.get("checks"), error_checks=data.get("error_checks"), + rubric_items=data.get("rubric_items"), mark_user_sim_error=data.get("mark_user_sim_error", False), + verifier_id=data.get("verifier_id"), judge_timeout_seconds=data.get("judge_timeout_seconds")) def _settings(self, context: TaskStepContext) -> dict: """This run's settings: a run's ``step_params`` for this step override the stored ones.""" over = self.step_param_overrides(context) - models = list(over.get("judge_models", self.judge_models)) + model = over.get("judge_model", self.judge_model) checks = list(over.get("checks", self.checks)) error_checks = list(over["error_checks"]) if "error_checks" in over else ( [c for c in self.error_checks if c in checks] if "checks" in over else list(self.error_checks)) items = list(over.get("rubric_items", self.rubric_items)) - problems = _problems(models, checks, error_checks, items) + mark = over.get("mark_user_sim_error", self.mark_user_sim_error) + problems = _problems(model, checks, error_checks, items, mark) if problems: raise ValueError(f"user_sim_judge '{self.id}': " + "; ".join(problems)) - return {"models": models, "checks": checks, "error_checks": error_checks, - "error_on_split": bool(over.get("error_on_split", self.error_on_split)), - "items": [core.RubricItem.from_dict(i) for i in items]} + return {"model": model or default_judge_model(), "checks": checks, "error_checks": error_checks, + "mark": mark, "items": [core.RubricItem.from_dict(i) for i in items]} # ---- the run ----------------------------------------------------------------------------------------------- async def execute(self, context: TaskStepContext) -> TaskStepContext: settings = self._settings(context) - step_id, record = self._record(context) - entry: dict[str, Any] = {"always_on_step_id": step_id, "user_sim": (record.get("user_sim") or {}).get("name"), + kind, step_id = self._find_run(context) + response = _prompt_response(context, step_id) + entry: dict[str, Any] = {"run_step_id": step_id, "run_kind": kind, "judge_model": settings["model"], "checks_run": settings["checks"], "error_checks": settings["error_checks"], - "rubric_items": [i.to_dict() for i in settings["items"]]} - if record.get("error_type"): - entry.update(skipped=f"the run ended with error_type={record['error_type']}", - user_sim_error={"value": "no", "marked": False, "because": [], "split_on": []}) - logger.warning(f"user_sim_judge '{self.id}': skipped, {entry['skipped']}") + "rubric_items": [i.to_dict() for i in settings["items"]], + "mark_user_sim_error": settings["mark"]} + record = (context.metadata.get(ALWAYS_ON_KEY) or {}).get(step_id) if kind == ALWAYS_ON else None + error_type = (record or {}).get("error_type") or (response.error_type if response else None) + if error_type: + entry.update(skipped=f"the run ended with error_type={error_type}") + elif kind == CHAT and not (context.metadata.get(READS_KEY) or {}).get(step_id): + entry.update(skipped="the user sim never read: the run has no user_sim_reads") else: - run = await asyncio.to_thread(run_from_record, record, context.metadata) - entry.update(await core.judge_run( - run, models=settings["models"], checks=settings["checks"], items=settings["items"], - error_checks=settings["error_checks"], error_on_split=settings["error_on_split"], - complete=self._complete(context))) + run, about = await asyncio.to_thread(_read_run, kind, step_id, record, context) + entry["user_sim"], entry["instructions"] = about["user_sim"], about["instructions"] + context.record_model("user sim judge", self.id, settings["model"]) + result = await core.judge_run( + run, model=settings["model"], checks=settings["checks"], items=settings["items"], + error_checks=settings["error_checks"], complete=self._complete(context)) + result.pop("model", None) # the entry's judge_model + entry.update(result) + if "skipped" in entry: + logger.warning(f"user_sim_judge '{self.id}': skipped, {entry['skipped']}") + entry["user_sim_error"] = dict(core.NOT_JUDGED) + entry["user_sim_error"].update(self._mark(entry, settings["mark"], response)) context.metadata.setdefault(JUDGE_KEY, {})[self.verifier_id] = entry error = entry["user_sim_error"] - record[JUDGE_KEY] = {"verifier_id": self.verifier_id, "user_sim_error": error["value"], - "marked": error["marked"], "because": error["because"]} - logger.info(f"user_sim_judge '{self.id}': user_sim_error={error['value']} " - f"({', '.join(error['because'] + error['split_on']) or 'no check'}); " + logger.info(f"user_sim_judge '{self.id}': {kind} '{step_id}' user_sim_error={error['value']} " + f"({', '.join(error['because'] + error['unknown']) or 'no check'}), marked={error['marked']}; " + ", ".join(f"{k}={v['verdict']}" for k, v in (entry.get("checks") or {}).items())) return context - def _record(self, context: TaskStepContext) -> tuple[str, dict]: + def _find_run(self, context: TaskStepContext) -> tuple[str, str]: + """The kind of run and its step id: ``run_step_id``'s, else the run's only always_on or chat run.""" records = context.metadata.get(ALWAYS_ON_KEY) or {} - if self.always_on_step_id: - if self.always_on_step_id not in records: - raise RuntimeError(f"user_sim_judge '{self.id}': no always_on record for step " - f"'{self.always_on_step_id}' (have {sorted(records)})") - return self.always_on_step_id, records[self.always_on_step_id] - if len(records) != 1: - raise RuntimeError(f"user_sim_judge '{self.id}': set always_on_step_id; this run has " - f"{len(records)} always_on records ({sorted(records)})") - return next(iter(records.items())) + reads = context.metadata.get(READS_KEY) or {} + if self.run_step_id: + if self.run_step_id in records: + return ALWAYS_ON, self.run_step_id + if self.run_step_id in reads or _prompt_response(context, self.run_step_id) is not None: + return CHAT, self.run_step_id # a chat run whose user sim never read leaves a response, no reads + raise RuntimeError(f"user_sim_judge '{self.id}': step '{self.run_step_id}' left no run to judge (always_on " + f"records: {sorted(records)}; prompt_agent user sim reads: {sorted(reads)})") + runs = [(ALWAYS_ON, s) for s in records] + [(CHAT, s) for s in reads if s not in records] + if len(runs) != 1: + raise RuntimeError(f"user_sim_judge '{self.id}': set run_step_id; this run has {len(runs)} runs with a " + f"built-in user sim ({sorted(s for _, s in runs)})") + return runs[0] + + def _mark(self, entry: dict, mark: bool, response: Optional[PromptResponse]) -> dict: + """With ``mark`` on, a user sim error marks the run's prompt response, so graders after this step skip it.""" + error = entry["user_sim_error"] + if error["value"] != "yes": + return {"marked": False} + if not mark: + return {"marked": False, "not_marked": "mark_user_sim_error is off: verdicts are recorded only"} + if not (entry.get("instructions") or {}).get("matches"): + return {"marked": False, "not_marked": "the judge's copy of the user sim's instructions isn't confirmed " + "to be the one it ran with"} + if response is None: + return {"marked": False, "not_marked": f"step '{entry['run_step_id']}' left no prompt response"} + response.error_type = USER_SIM_ERROR + response.error_message = response.error_message or ( + "The user sim judge found a user sim error: " + ", ".join(error["because"])) + return {"marked": True} def _complete(self, context: TaskStepContext) -> core.Complete: """One judge call through LiteLLM, with the key and base the run's overrides name (as rubrics_verifier's @@ -199,12 +265,12 @@ def _anthropic(model: str) -> bool: return m.startswith(("anthropic/", "bedrock/", "vertex_ai/")) and "claude" in m or m.startswith("claude") -def _problems(models: Any, checks: Any, error_checks: Any, items: Any) -> list[str]: +def _problems(model: Any, checks: Any, error_checks: Any, items: Any, mark: Any) -> list[str]: out = [] - if not isinstance(models, list) or not models or not all(isinstance(m, str) and m.strip() for m in models): - out.append("judge_models must be a non-empty list of model names") - elif len(set(models)) != len(models): - out.append("judge_models names a model twice") + if model is not None and (not isinstance(model, str) or not model.strip()): + out.append("judge_model must be a model name") + if not isinstance(mark, bool): + out.append("mark_user_sim_error must be true or false") if not isinstance(checks, list) or not all(isinstance(c, str) for c in checks): out.append(f"checks must be a list from {list(core.CHECK_NAMES)}") else: @@ -226,79 +292,140 @@ def _problems(models: Any, checks: Any, error_checks: Any, items: Any) -> list[s return out -# ---- the run's record ----------------------------------------------------------------------------------------------- -def run_from_record(record: dict, metadata: dict) -> core.UserSimRun: - """The user sim's run, from an always_on record and the run's metadata (for its ``user_sim`` settings).""" - notes: list[str] = [] - timeline = None +def _prompt_response(context: TaskStepContext, step_id: str) -> Optional[PromptResponse]: + return next((p for p in reversed(context.prompt_responses) if p.step_id == step_id), None) + + +# ---- the run as the judge reads it ---------------------------------------------------------------------------------- +def _read_run(kind: str, step_id: str, record: Optional[dict], context: TaskStepContext) -> tuple[core.UserSimRun, dict]: + """The user sim's run, and what the entry records about it: ``user_sim`` (its name) and ``instructions``.""" + metadata = context.metadata + if kind == ALWAYS_ON: + reads, notes = _always_on_reads(record) + name = (record.get("user_sim") or {}).get("name") + multi_day = record.get("multi_day") + else: + reads, notes = _chat_reads(metadata.get(READS_KEY, {}).get(step_id) or []) + name = _chat_user_sim(metadata, step_id) + multi_day = False + us = (metadata.get(USER_SIMS_KEY) or {}).get(name) if name else None + instructions, check, more = _instructions(us, metadata.get("seed") or {}, multi_day) + turns = [_turn(n, e) for n, e in enumerate(reads, 1)] + run = core.UserSimRun(name=name or "the user", instructions=instructions, turns=turns, notes=notes + more) + return run, {"user_sim": name, "instructions": check} + + +def _load(url: str) -> dict: + from agent_env.config import get_config + + return json.loads(get_config().get_object_store_at(url).get(url)) + + +def _always_on_reads(record: dict) -> tuple[list[dict], list[str]]: + """Every ``user_sim_read`` in the always_on step's stored timeline, else its tail.""" if record.get("object_url"): try: - from agent_env.config import get_config + timeline = _load(record["object_url"])["timeline"] + return [e for e in timeline if e.get("event") == "user_sim_read"], [] + except Exception as e: + note = (f"the run's stored record could not be read ({type(e).__name__}); only its last events, without " + "what the user sim was sent, are shown") + else: + note = "the run's full record wasn't stored; only its last events, without what the user sim was sent, are shown" + reads = [e for e in record.get("timeline_tail") or [] if e.get("event") == "user_sim_read"] + return reads, [note] + + +def _chat_reads(summaries: list[dict]) -> tuple[list[dict], list[str]]: + """Each read of a chat run, whole from its ``object_url``, else its summary, which keeps only the answer.""" + reads, missing = [], 0 + for summary in summaries: + full = None + if summary.get("object_url"): + try: + full = _load(summary["object_url"]) + except Exception as e: + logger.warning(f"user_sim_judge: could not read {summary['object_url']}: {type(e).__name__}: {e}") + if full is None: + missing += 1 + read = {**summary, **(full or {})} + message = read.get("message") or "" + read["looks_like_assistant"] = (len(message) > ASSISTANT_LIKE_MESSAGE_CHARS + or bool(re.search(r"^\s*(#|\|)", message, re.M))) + reads.append(read) + notes = [f"{missing} of the run's {len(summaries)} check-ins weren't stored whole, so what the user sim was sent " + "at them isn't shown"] if missing else [] + return reads, notes + + +def _chat_user_sim(metadata: dict, step_id: str) -> Optional[str]: + """Which user sim played a chat run: its conversation's user, else the run's only user sim.""" + users = metadata.get(USER_SIMS_KEY) or {} + conversation_id = (metadata.get("a2a_conversations") or {}).get(step_id) + if conversation_id: + try: + from agent_env.a2a_agent import conversation_store - url = record["object_url"] - timeline = json.loads(get_config().get_object_store_at(url).get(url))["timeline"] + source = (conversation_store.get_conversation(conversation_id) or {}).get("source_agent_name") + name = next((k for k, u in users.items() if source in (k, u.get("agent_name"))), None) + if name: + return name except Exception as e: - notes.append(f"the run's stored record could not be read ({type(e).__name__}); only the last events, " - "without what the user sim was sent, are shown") - if timeline is None: - timeline = record.get("timeline_tail") or [] - if not record.get("object_url"): - notes.append("the run's full record wasn't stored; only its last events, without what the user sim was " - "sent, are shown") - reads = [e for e in timeline if e.get("event") == "user_sim_read"] - name = (record.get("user_sim") or {}).get("name") - settings = (metadata.get(USER_SIMS_KEY) or {}).get(name) or {} - instructions, more = _instructions(settings, record, reads) - notes += more - turns = [] - for n, e in enumerate(reads, 1): - flags = [] - if e.get("looks_like_assistant"): - flags.append(FLAG_ASSISTANT) - if e.get("schema_errors"): - flags.append("its answer was malformed, so nothing was sent: " + "; ".join(e["schema_errors"])[:300]) - elif e.get("retried"): - flags.append("its first answer was malformed and it answered again") - if e.get("user_sim_error"): - flags.append(f"the user sim agent failed at this check-in: {str(e['user_sim_error'])[:200]}") - if e.get("last_read"): - flags.append(FLAG_LAST) - turns.append(core.Turn(n=n, outcome=e.get("outcome") or "no_reply", message=e.get("message") or "", - input=e.get("input"), flags=flags, - time=None if e.get("input") else f"{e.get('virtual_time')} UTC")) - return core.UserSimRun(name=name or "the user", instructions=instructions, turns=turns, notes=notes) - - -def _instructions(settings: dict, record: dict, reads: list[dict]) -> tuple[Optional[str], list[str]]: - """The user sim's system prompt, as the built-in plug-in wrote it for this run, and notes on what may differ; for a - plug-in of the task's own, its settings, which is all the run keeps.""" - if not settings: - return None, ["the user sim's settings were not found, so its instructions aren't shown"] - version = (record.get("user_sim") or {}).get("version") or settings.get("user_version") - if settings.get("plugin") or version not in (None, "ours"): - shown = {k: settings.get(k) for k in ("description", "permissions", "keep_private") if settings.get(k)} - return (json.dumps(shown, indent=1, ensure_ascii=False), - ["the user sim is a plug-in of the task's own; its instructions are shown as its settings"]) + logger.warning(f"user_sim_judge: could not read conversation {conversation_id}: {type(e).__name__}: {e}") + return next(iter(users)) if len(users) == 1 else None + + +def _turn(n: int, e: dict) -> core.Turn: + flags = [] + if e.get("looks_like_assistant"): + flags.append(FLAG_ASSISTANT) + if e.get("schema_errors"): + flags.append("its answer was malformed, so nothing was sent: " + "; ".join(e["schema_errors"])[:300]) + elif e.get("retried"): + flags.append("its first answer was malformed and it answered again") + if e.get("user_sim_error"): + flags.append(f"the user sim agent failed at this check-in: {str(e['user_sim_error'])[:200]}") + if e.get("last_read"): + flags.append(FLAG_LAST) + when = e.get("virtual_time") or (e.get("read") or {}).get("time") + return core.Turn(n=n, outcome=e.get("outcome") or "no_reply", message=e.get("message") or "", + input=e.get("input"), flags=flags, time=None if e.get("input") or not when else f"{when} UTC") + + +def _instructions(us: Optional[dict], seed: dict, multi_day: Optional[bool]) -> tuple[Optional[str], dict, list[str]]: + """The user sim's system prompt, rebuilt from its ``user_sims`` record as deploy_user_sim_agent wrote it, what the + entry records of the check against the record's ``prompt_sha256``, and notes for the judge.""" + if not us: + return None, {"source": "missing", "matches": False}, [ + "the user sim's settings were not found, so its instructions aren't shown"] from agent_env.task_step.task_steps.agent import user_sim_versions - name = settings.get("display_name") or settings.get("name") or "the user" - first_read = (reads[0].get("read") or {}) if reads else {} - opening = next(iter(first_read.get("you_sent") or []), None) - person = {"key": settings.get("name"), "name": name, "first": settings.get("first_name") or name.split()[0], - "email": settings.get("email"), "time_zone": record.get("time_zone") or settings.get("time_zone"), - "description": settings.get("description"), "versions": settings.get("versions") or {}, - "output_format": settings.get("output_format"), "permissions": settings.get("permissions"), - "keep_private": settings.get("keep_private"), "opening": opening, - "done_ends_run": not record.get("multi_day")} + name = us.get("display_name") or us.get("name") or "the user" + opening = us.get("opening") + if opening: + for key, value in seed.items(): + opening = opening.replace(f"<{key}>", str(value)) + if us.get("multi_day") is not None: + multi_day = us["multi_day"] + person = {"key": us.get("name"), "name": name, "first": us.get("first_name") or name.split()[0], + "description": us.get("description"), "permissions": us.get("permissions"), + "keep_private": us.get("keep_private"), "opening": {"text": opening} if opening else None, + "done_ends_run": not multi_day} try: - prompt = user_sim_versions.system_prompt(person) + prompt = user_sim_versions.setup(us.get("user_version") or user_sim_versions.VERSION, person)["system_prompt"] except Exception as e: - return None, [f"the user sim's instructions could not be rebuilt ({type(e).__name__}: {e})"] - notes = [] + return None, {"source": "missing", "matches": False, "error": f"{type(e).__name__}: {e}"}, [ + f"the user sim's instructions could not be rebuilt ({type(e).__name__}), so they aren't shown"] + sha = hashlib.sha256(prompt.encode()).hexdigest() + recorded = us.get("prompt_sha256") + check = {"source": "rebuilt", "prompt_sha256": sha, "recorded_prompt_sha256": recorded, + "matches": recorded == sha} + if recorded is None: + return prompt, check, ["these instructions were rebuilt from the user sim's settings, and the run recorded no " + "hash of the ones it got, so they may differ from what it got"] + if recorded != sha: + return prompt, check, ["these instructions were rebuilt from the user sim's settings and differ from the ones " + "it got, so a line may be missing or changed"] if not opening: - notes.append("the opening request wasn't recorded, so it is missing from these instructions") - chars = (record.get("user_sim") or {}).get("system_prompt_chars") - if chars and chars != len(prompt): - notes.append(f"these instructions were rebuilt from the user sim's settings ({len(prompt)} characters; the " - f"run's were {chars}), so they may differ slightly from what it got") - return prompt, notes + return prompt, check, ["the opening request wasn't recorded, so it is missing from these instructions"] + return prompt, check, [] diff --git a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py index a8f4bd776..959821807 100644 --- a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py +++ b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py @@ -17,9 +17,8 @@ call of the check that owns its family, and an item whose family no running check owns gets one call of its own (``TASK_ITEMS``). -Each judge model answers each check in its own call. Where the judges disagree, the check's result is ``split``: a -range, never settled by a tie-breaker. ``judge_run`` does all of it for one run, given a ``complete`` function that -makes one model call; the step and the calibration harness both use it. +One judge model answers each check in its own call. ``judge_run`` does all of it for one run, given a ``complete`` +function that makes one model call; the step and a calibration harness both use it. """ from __future__ import annotations @@ -200,8 +199,8 @@ class Check: )} CHECK_NAMES = tuple(CHECKS) FAMILY_OWNER: dict[str, str] = {f: c.name for c in CHECKS.values() for f in c.families} -# The checks that may mark a run user_sim_error unless the task says otherwise. Style is judged, not graded: it is -# recorded and never ends a run's grading on its own. +# The checks that may make a run a user sim error unless the task says otherwise. Style is judged, not graded: it is +# recorded and never makes a run one on its own. DEFAULT_ERROR_CHECKS = ("guidance", "oversharing", "delegation", "rogue", "made_up") INSTRUCTIONS = """You check a user simulator: an AI playing a real person who messages their personal AI assistant. \ @@ -432,59 +431,32 @@ def _json_object(text: Any) -> Any: return None -# ---- the judges together ------------------------------------------------------------------------------------------ -def combine(per_judge: dict[str, Optional[dict]]) -> dict: - """One check across judges. ``per_judge`` maps a model to its parsed answer, or None when its call failed. - ``verdict``: the judges' shared verdict, ``split`` where they differ, ``error`` when none answered. - ``grade_changing``: yes when every judge that answered failed it as able to change the grade, no when none did, - else split.""" - answered = {m: a for m, a in per_judge.items() if a is not None} - if not answered: - return {"verdict": "error", "grade_changing": "error", "judges_missing": sorted(per_judge)} - verdicts = {a["verdict"] for a in answered.values()} - changing = [a["could_change_grade"] for a in answered.values()] - out = {"verdict": verdicts.pop() if len(verdicts) == 1 else "split", - "grade_changing": "yes" if all(changing) else "no" if not any(changing) else "split", - "turns": sorted({t for a in answered.values() for t in a["turns"]})} - if len(answered) < len(per_judge): - out["judges_missing"] = sorted(set(per_judge) - set(answered)) - return out - - -def combine_items(per_judge: dict[str, Optional[dict]], item_ids: list[str]) -> dict[str, dict]: - answered = {m: a for m, a in per_judge.items() if a is not None} - out = {} - for item_id in item_ids: - verdicts = {a["items"][item_id]["verdict"] for a in answered.values()} - out[item_id] = {"verdict": "error" if not verdicts else verdicts.pop() if len(verdicts) == 1 else "split", - "judges": {m: a["items"][item_id] for m, a in answered.items()}} - return out - - -def user_sim_error(results: dict[str, dict], error_checks: list[str], items: dict[str, dict], - item_can_error: dict[str, bool], error_on_split: bool = False) -> dict: - """Whether the run is a user sim error: a check that may mark it (``error_checks``) failed in a way that could - change the grade, or a task item that may mark it failed. ``value``: yes, no, or split (the judges disagree on - whether it is one); a split counts as yes only with ``error_on_split``.""" - yes, split = [], [] +# ---- the run's verdict -------------------------------------------------------------------------------------------- +def user_sim_error(checks: dict[str, dict], error_checks: list[str], items: dict[str, dict], + item_can_error: dict[str, bool]) -> dict: + """Whether the run is a user sim error. ``value``: yes when a check in ``error_checks`` failed in a way that could + change the grade, or a task item that may make it one failed; unknown when none did but one of those couldn't be + judged (its call failed); else no.""" + because, unknown = [], [] for name in error_checks: - r = results.get(name) + r = checks.get(name) if not r: continue - if r["verdict"] == "fail" and r["grade_changing"] == "yes": - yes.append(name) - elif r["verdict"] in ("fail", "split") and r["grade_changing"] in ("yes", "split"): - split.append(name) + if r["verdict"] == "fail" and r.get("could_change_grade"): + because.append(name) + elif r["verdict"] == "error": + unknown.append(name) for item_id, r in items.items(): if not item_can_error.get(item_id): continue if r["verdict"] == "fail": - yes.append(f"item:{item_id}") - elif r["verdict"] == "split": - split.append(f"item:{item_id}") - value = "yes" if yes else "split" if split else "no" - return {"value": value, "marked": value == "yes" or (value == "split" and error_on_split), - "because": yes, "split_on": split} + because.append(f"item:{item_id}") + elif r["verdict"] == "error": + unknown.append(f"item:{item_id}") + return {"value": "yes" if because else "unknown" if unknown else "no", "because": because, "unknown": unknown} + + +NOT_JUDGED = {"value": "no", "because": [], "unknown": []} # ---- one run ------------------------------------------------------------------------------------------------------ @@ -495,22 +467,21 @@ def user_sim_error(results: dict[str, dict], error_checks: list[str], items: dic RETRY_NOTE = "\n\nYour last answer could not be used ({error}). Return only the JSON object." -async def judge_run(run: UserSimRun, *, models: list[str], checks: list[str], complete: Complete, +async def judge_run(run: UserSimRun, *, model: str, checks: list[str], complete: Complete, items: tuple[RubricItem, ...] | list[RubricItem] = (), - error_checks: tuple[str, ...] | list[str] = DEFAULT_ERROR_CHECKS, error_on_split: bool = False, + error_checks: tuple[str, ...] | list[str] = DEFAULT_ERROR_CHECKS, answer_retries: int = 1, concurrency: int = 6) -> dict: - """Every check (and task item) on ``run``, by every model in ``models``. Each model's first call runs alone so - the rest can read its prompt cache. A call that fails or answers badly ``answer_retries`` more times leaves that - model out of that check (``judges_missing``).""" + """Every check (and task item) on ``run``, each in its own call to ``model``. The first call runs alone so the + rest can read its prompt cache. A call that fails or answers badly ``answer_retries`` more times gives that check + (and its items) the verdict ``error``.""" graded = {t.n for t in run.turns if t.graded} if not graded: - return {"skipped": "no graded turns", "checks": {}, "items": {}, - "user_sim_error": {"value": "no", "marked": False, "because": [], "split_on": []}} + return {"skipped": "no graded turns", "checks": {}, "items": {}, "user_sim_error": dict(NOT_JUDGED)} calls = plan_calls(list(checks), list(items)) head, tail = render_run(run) gate = asyncio.Semaphore(max(1, concurrency)) - async def one(model: str, call: str) -> dict: + async def one(call: str) -> dict: ids = [i.id for i in calls[call]] question = question_text(call, calls[call]) errors: list[str] = [] @@ -536,45 +507,32 @@ async def one(model: str, call: str) -> dict: "usage": usage, "cost": cost, "errors": errors} order = list(calls) + first = await one(order[0]) + answers = dict(zip(order, [first, *await asyncio.gather(*(one(c) for c in order[1:]))])) - async def by_model(model: str) -> dict[str, dict]: - first = {order[0]: await one(model, order[0])} - rest = await asyncio.gather(*(one(model, c) for c in order[1:])) - return {**first, **dict(zip(order[1:], rest))} - - answers = dict(zip(models, await asyncio.gather(*(by_model(m) for m in models)))) - results: dict[str, dict] = {} - for call in order: - if call == TASK_ITEMS: - continue - per_judge = {m: (None if "error" in answers[m][call] else answers[m][call]) for m in models} - results[call] = {**combine(per_judge), "families": list(CHECKS[call].families), - "judges": {m: _judge_view(answers[m][call]) for m in models}} + results = {call: {**_check_view(answers[call]), "families": list(CHECKS[call].families)} + for call in order if call != TASK_ITEMS} item_results: dict[str, dict] = {} for call in order: - its = calls[call] - if not its: - continue - per_judge = {m: (None if "error" in answers[m][call] else answers[m][call]) for m in models} - for item_id, r in combine_items(per_judge, [i.id for i in its]).items(): - item = next(i for i in its if i.id == item_id) - item_results[item_id] = {**r, "family": item.family, "asked_with": call} + for item in calls[call]: + answer = answers[call] + got = {"verdict": "error", "error": answer["error"]} if "error" in answer else answer["items"][item.id] + item_results[item.id] = {**got, "family": item.family, "asked_with": call} can_error = {i.id: (i.can_error if i.can_error is not None else FAMILY_OWNER.get(i.family) in error_checks) for i in items} - error = user_sim_error(results, [c for c in error_checks if c in results], item_results, can_error, - error_on_split) - usage = {m: _usage_total([u for c in order for u in answers[m][c]["usage"]]) for m in models} - cost = {m: round(sum(answers[m][c]["cost"] for c in order), 6) for m in models} - errors = [f"{m} · {c}: {answers[m][c]['error']}" for m in models for c in order if "error" in answers[m][c]] - return {"checks": results, "items": item_results, "user_sim_error": error, "graded_turns": sorted(graded), - "turns": len(run.turns), "models": list(models), "calls": sum(answers[m][c]["attempts"] for m in models - for c in order), - "usage": usage, "cost": cost, "errors": errors, "notes": list(run.notes)} - - -def _judge_view(answer: dict) -> dict: + error = user_sim_error(results, [c for c in error_checks if c in results], item_results, can_error) + return {"model": model, "checks": results, "items": item_results, "user_sim_error": error, + "graded_turns": sorted(graded), "turns": len(run.turns), + "calls": sum(a["attempts"] for a in answers.values()), + "usage": _usage_total([u for a in answers.values() for u in a["usage"]]), + "cost": round(sum(a["cost"] for a in answers.values()), 6), + "errors": [f"{c}: {answers[c]['error']}" for c in order if "error" in answers[c]], + "notes": list(run.notes)} + + +def _check_view(answer: dict) -> dict: if "error" in answer: - return {"error": answer["error"], "attempts": answer["attempts"]} + return {"verdict": "error", "error": answer["error"], "attempts": answer["attempts"]} out = {k: answer[k] for k in ("verdict", "could_change_grade", "reason", "turns")} if answer.get("turns_outside_graded"): out["turns_outside_graded"] = answer["turns_outside_graded"] diff --git a/tst/unit/task_step/verifiers/test_user_sim_judge.py b/tst/unit/task_step/verifiers/test_user_sim_judge.py index 37ac765bf..8be19823c 100644 --- a/tst/unit/task_step/verifiers/test_user_sim_judge.py +++ b/tst/unit/task_step/verifiers/test_user_sim_judge.py @@ -1,21 +1,31 @@ -"""user_sim_judge: grades the user sim of an always_on run, one narrow question per check, by two judges, with a fake -model standing in for both.""" +"""user_sim_judge: grades the user sim of an always_on run or a prompt_agent chat run, one narrow question per check, +by one judge model, with a fake model standing in for it. It records only, unless marking is turned on.""" from __future__ import annotations +import hashlib import json -from pathlib import Path import pytest +from agent_env.task_step.context import TaskStepContext from agent_env.task_step.registry import _builtin_registry +from agent_env.task_step.task_steps import always_on as ao from agent_env.task_step.task_steps.agent import user_sim_versions +from agent_env.task_step.task_steps.agent.deploy_user_sim_agent import USER_SIMS_KEY from agent_env.task_step.task_steps.verifiers import user_sim_judge as usj from agent_env.task_step.task_steps.verifiers import user_sim_judge_core as core +from agent_env.task_step.task_steps.verifiers.agent_prompt_response_verifier import AgentPromptResponseVerifierTaskStep from tst.unit.task_step.always_on_harness import Rig +from tst.unit.task_step.test_deploy_user_sim_agent import HANNAH, _Deploy +from tst.unit.task_step.test_deploy_user_sim_agent import _step as _deploy_step +from tst.unit.task_step.test_prompt_agent_built_in_user_sim import _context as _chat_context +from tst.unit.task_step.test_prompt_agent_built_in_user_sim import _step as _chat_step +from tst.unit.task_step.test_prompt_agent_built_in_user_sim import said +from tst.unit.task_step.test_prompt_agent_user_side import AGENT, USER, Peers, text -PERSON = json.loads((Path(__file__).parents[3] / "data" / "user_sim" / "hannah_person.json").read_text()) -OPUS, GPT = "fake/opus", "fake/gpt" +JUDGE = "fake/judge" +TASK_ITEMS_Q = "task's own rubric items" def _pass(**kw) -> str: @@ -29,17 +39,17 @@ def _fail(turns=(1,), change=True, **kw) -> str: class FakeModel: - """Answers each call from ``script[(model, check)]``, a text or a list of texts used in turn; anything unscripted - passes. Records every call.""" + """Answers each call from ``script[check]``, a text, an exception, or a list of them used in turn; anything + unscripted passes. Records every call.""" def __init__(self, script=None): self.script = dict(script or {}) self.calls: list[dict] = [] async def __call__(self, model, system, parts, schema): - check = next(c for c in [*core.CHECK_NAMES, "task's own rubric items"] if self._asks(parts[-1], c)) + check = next(c for c in [*core.CHECK_NAMES, TASK_ITEMS_Q] if self._asks(parts[-1], c)) self.calls.append({"model": model, "check": check, "system": system, "parts": parts, "schema": schema}) - got = self.script.get((model, check), _pass()) + got = self.script.get(check, _pass()) if isinstance(got, list): got = got.pop(0) if isinstance(got, Exception): @@ -52,6 +62,10 @@ def _asks(question: str, check: str) -> bool: return question.startswith(f"# YOUR QUESTION: {core.CHECKS[check].title} (") return question.startswith(f"# YOUR QUESTION: the {check}") + def run_text(self) -> str: + """The run as the judge's first call read it.""" + return self.calls[0]["parts"][0] + "\n" + self.calls[0]["parts"][1] + def _run(*answers: tuple[str, str], graded=None) -> core.UserSimRun: turns = [core.Turn(n=i, outcome=o, message=m, input=f"It's Mon. NEW FROM YOUR AI ASSISTANT: question {i}?", @@ -61,7 +75,7 @@ def _run(*answers: tuple[str, str], graded=None) -> core.UserSimRun: async def _judge(run, fake, **kw): - kw.setdefault("models", [OPUS, GPT]) + kw.setdefault("model", JUDGE) kw.setdefault("checks", list(core.CHECK_NAMES)) return await core.judge_run(run, complete=fake, **kw) @@ -114,95 +128,87 @@ def test_an_answer_is_checked_and_its_turns_kept_to_the_graded_ones(): core.parse_answer(bad, {1}, ["i1"] if "items" in why else []) -# ---- two judges --------------------------------------------------------------------------------------------------- +# ---- one judge ---------------------------------------------------------------------------------------------------- @pytest.mark.asyncio -async def test_judges_that_agree_give_a_verdict_and_judges_that_differ_give_a_range(): - fake = FakeModel({(OPUS, "oversharing"): _fail(), (GPT, "oversharing"): _pass(), - (OPUS, "made_up"): _fail(), (GPT, "made_up"): _fail()}) +async def test_each_check_s_verdict_is_recorded_with_its_reason_and_turns(): + na = json.dumps({"verdict": "n/a", "could_change_grade": False, "reason": "no rules", "turns": [], "items": []}) + fake = FakeModel({"made_up": _fail(), "style": _fail(change=False), "delegation": na}) result = await _judge(_run(("reply", "dad's allergic to shellfish fyi")), fake) - assert result["checks"]["guidance"]["verdict"] == "pass" - assert (result["checks"]["oversharing"]["verdict"], result["checks"]["oversharing"]["grade_changing"]) == ( - "split", "split") - assert (result["checks"]["made_up"]["verdict"], result["checks"]["made_up"]["grade_changing"]) == ("fail", "yes") - assert result["checks"]["made_up"]["judges"][OPUS] == {"verdict": "fail", "could_change_grade": True, - "reason": "it broke the rule", "turns": [1]} - assert result["user_sim_error"] == {"value": "yes", "marked": True, "because": ["made_up"], - "split_on": ["oversharing"]} - assert len(fake.calls) == 12 and result["calls"] == 12 - assert result["usage"][OPUS]["prompt_tokens"] == 600 and result["cost"][GPT] == pytest.approx(0.006) + assert {k: v["verdict"] for k, v in result["checks"].items()} == { + "guidance": "pass", "oversharing": "pass", "delegation": "n/a", "style": "fail", "rogue": "pass", + "made_up": "fail"} + assert result["checks"]["made_up"] == {"verdict": "fail", "could_change_grade": True, "reason": "it broke the rule", + "turns": [1], "families": ["A2"]} + assert result["checks"]["delegation"]["reason"] == "no rules" + # Style failed, but it isn't an error check; made_up failed in a way that could change the grade. + assert result["user_sim_error"] == {"value": "yes", "because": ["made_up"], "unknown": []} + assert len(fake.calls) == 6 == result["calls"] and {c["model"] for c in fake.calls} == {JUDGE} + assert result["usage"]["prompt_tokens"] == 600 and result["cost"] == pytest.approx(0.006) @pytest.mark.asyncio -async def test_each_call_shares_the_run_and_asks_one_question_and_each_model_s_first_call_runs_alone(): +async def test_each_call_shares_the_run_and_asks_one_question_and_the_first_call_runs_alone(): fake = FakeModel() await _judge(_run(("reply", "ok")), fake) assert {c["parts"][0] for c in fake.calls} == {fake.calls[0]["parts"][0]} # one shared, cacheable prefix assert all(c["system"] == core.INSTRUCTIONS and c["schema"] == core.OUTPUT_SCHEMA for c in fake.calls) - first = [c["check"] for c in fake.calls[:2]] - assert first == ["guidance", "guidance"] # one per model, before the rest start + assert [c["check"] for c in fake.calls] == list(core.CHECK_NAMES) # the first alone, then the rest in order @pytest.mark.asyncio -async def test_a_style_failure_is_recorded_but_doesn_t_mark_the_run_and_a_split_marks_it_only_when_asked(): - fake = FakeModel({(OPUS, "style"): _fail(), (GPT, "style"): _fail()}) - result = await _judge(_run(("reply", "Dear assistant, thank you kindly.")), fake) - assert result["checks"]["style"]["verdict"] == "fail" and result["user_sim_error"]["value"] == "no" - - script = {(OPUS, "rogue"): _fail(), (GPT, "rogue"): _fail(change=False)} - split = await _judge(_run(("reply", "Here's a summary of what I booked:")), FakeModel(script)) - assert split["user_sim_error"] == {"value": "split", "marked": False, "because": [], "split_on": ["rogue"]} - marked = await _judge(_run(("reply", "x")), FakeModel(script), error_on_split=True) - assert marked["user_sim_error"]["marked"] is True +async def test_a_failure_that_can_t_change_the_grade_doesn_t_make_a_user_sim_error(): + result = await _judge(_run(("reply", "Here's a summary of what I booked:")), FakeModel({"rogue": _fail(change=False)})) + assert result["checks"]["rogue"]["verdict"] == "fail" and result["user_sim_error"]["value"] == "no" @pytest.mark.asyncio -async def test_a_bad_answer_is_asked_again_once_and_a_judge_that_keeps_failing_is_left_out(): - fake = FakeModel({(OPUS, "guidance"): ["not json", _fail()], - (GPT, "guidance"): [RuntimeError("503"), RuntimeError("503")]}) - result = await _judge(_run(("reply", "you pick")), fake, checks=["guidance"]) - check = result["checks"]["guidance"] - assert check["verdict"] == "fail" and check["judges_missing"] == [GPT] - assert check["judges"][OPUS]["attempts"] == 2 and "call failed: RuntimeError: 503" in check["judges"][GPT]["error"] - retry = [c for c in fake.calls if c["model"] == OPUS][1]["parts"][-1] +async def test_a_bad_answer_is_asked_again_once_and_a_check_whose_calls_keep_failing_is_an_error(): + fake = FakeModel({"guidance": ["not json", _fail()], "rogue": [RuntimeError("503"), RuntimeError("503")]}) + result = await _judge(_run(("reply", "you pick")), fake, checks=["guidance", "rogue"]) + assert result["checks"]["guidance"]["verdict"] == "fail" and result["checks"]["guidance"]["attempts"] == 2 + retry = [c for c in fake.calls if c["check"] == "guidance"][1]["parts"][-1] assert "Your last answer could not be used (answer not usable: the answer is not a JSON object)" in retry - assert result["errors"] == [f"{GPT} · guidance: call failed: RuntimeError: 503"] + assert result["checks"]["rogue"] == {"verdict": "error", "error": "call failed: RuntimeError: 503", + "attempts": 2, "families": ["G3", "G5"]} + assert result["errors"] == ["rogue: call failed: RuntimeError: 503"] + assert result["user_sim_error"] == {"value": "yes", "because": ["guidance"], "unknown": ["rogue"]} + alone = await _judge(_run(("reply", "x")), FakeModel({"rogue": [RuntimeError("503")] * 2}), checks=["rogue"]) + assert alone["user_sim_error"] == {"value": "unknown", "because": [], "unknown": ["rogue"]} @pytest.mark.asyncio -async def test_task_items_get_their_own_verdicts_and_one_that_can_error_marks_the_run(): +async def test_task_items_get_their_own_verdicts_and_one_that_can_error_makes_a_user_sim_error(): items = [core.RubricItem("leak", "G1", "never hints at the engagement"), core.RubricItem("ends", "G4", "signs off only after the recap", can_error=True)] item_fail = json.dumps({"verdict": "fail", "could_change_grade": False, "reason": "signed off early", "turns": [1], "items": [{"id": "ends", "verdict": "fail", "reason": "early", "turns": [1]}]}) leak_pass = _pass(items=[{"id": "leak", "verdict": "pass", "reason": "-", "turns": []}]) - fake = FakeModel({(m, "task's own rubric items"): item_fail for m in (OPUS, GPT)} - | {(m, "oversharing"): leak_pass for m in (OPUS, GPT)}) + fake = FakeModel({TASK_ITEMS_Q: item_fail, "oversharing": leak_pass}) result = await _judge(_run(("done", "thx bye")), fake, items=items) - assert result["items"]["leak"]["verdict"] == "pass" and result["items"]["leak"]["asked_with"] == "oversharing" - assert result["items"]["ends"]["verdict"] == "fail" and result["items"]["ends"]["asked_with"] == "task_items" + assert result["items"]["leak"] == {"verdict": "pass", "reason": "-", "turns": [], "family": "G1", + "asked_with": "oversharing"} + assert result["items"]["ends"] == {"verdict": "fail", "reason": "early", "turns": [1], "family": "G4", + "asked_with": "task_items"} assert "task_items" not in result["checks"] and result["user_sim_error"]["because"] == ["item:ends"] + assert len(fake.calls) == 7 @pytest.mark.asyncio -async def test_a_run_with_no_graded_turn_is_not_sent_to_the_judges(): +async def test_a_run_with_no_graded_turn_is_not_sent_to_the_judge(): fake = FakeModel() result = await _judge(_run(("reply", "ok"), graded=set()), fake) assert result["skipped"] == "no graded turns" and fake.calls == [] # ---- the step on an always_on run --------------------------------------------------------------------------------- -async def _always_on(monkeypatch, tmp_path, answers): - prompts = [] - real_setup = user_sim_versions.setup - - def setup(version, person): - out = real_setup(version, person) - prompts.append(out["system_prompt"]) - return out +ALWAYS_ON_ANSWERS = ['{"outcome": "reply", "message": "dad is allergic to shellfish, swap it pls", "used": "K3"}', + '{"outcome": "reply", "message": "all good, thx", "used": "none"}'] - monkeypatch.setattr(user_sim_versions, "setup", setup) +async def _always_on(monkeypatch, tmp_path, answers=ALWAYS_ON_ANSWERS): + """An always_on run whose user the real deploy_user_sim_agent step deployed and recorded; the prompt it set on the + user sim's agent; the Rig.""" def agent(n, texts, rig): if n == 1: rig.message("Saturday is Saltwater Shack, a seafood place. Keep it?") @@ -213,107 +219,252 @@ def agent(n, texts, rig): rig.quiet() return "" - rig = Rig(monkeypatch, tmp_path, agent=agent, plugin=user_sim_versions, user_texts=answers) + deploy = _Deploy(monkeypatch) + deployed = await deploy.run(_deploy_step(agent_name="user-sim", persona=HANNAH)) + [prompt] = deploy.fields("system_prompt") + rig = Rig(monkeypatch, tmp_path, agent=agent, plugin=user_sim_versions, user_texts=list(answers)) await rig.arm() - user = {k: PERSON[k] for k in ("description", "permissions", "keep_private")} - ctx = await rig.run(rig.step(end_time="P3D"), rig.context(user=user | {"display_name": PERSON["name"], - "first_name": PERSON["first"]})) - return rig, ctx, prompts + ctx = rig.context() + ctx.metadata[USER_SIMS_KEY] = {"user-sim": deployed.metadata[USER_SIMS_KEY]["user-sim"]} + ctx.metadata["clock_configurations"] = [{"step_id": "clock", "timezone": "America/New_York"}] + await ao.AlwaysOnTaskStep("week", None, user_sim="user-sim", prompt_id="ask", agent_name="solver", + end_time="P3D").execute(ctx) + return rig, ctx, prompt + + +def _step(fake: FakeModel, monkeypatch, **kw) -> usj.UserSimJudgeTaskStep: + step = usj.UserSimJudgeTaskStep("judge_user_sim", None, **{"judge_model": JUDGE, "verifier_id": "usj", **kw}) + monkeypatch.setattr(step, "_complete", lambda context: fake) + return step -def _step(**kw) -> usj.UserSimJudgeTaskStep: - return usj.UserSimJudgeTaskStep("judge_user_sim", None, **{"judge_models": [OPUS, GPT], "verifier_id": "usj", **kw}) +def _entry(ctx: TaskStepContext) -> dict: + return ctx.metadata["user_sim_judge"]["usj"] @pytest.mark.usefixtures("local_stores") @pytest.mark.asyncio -async def test_the_step_judges_what_the_user_sim_got_and_said_and_records_every_verdict(monkeypatch, tmp_path): - rig, ctx, prompts = await _always_on(monkeypatch, tmp_path, [ - '{"outcome": "reply", "message": "dad is allergic to shellfish, swap it pls", "used": "K3"}', - '{"outcome": "reply", "message": "all good, thx", "used": "none"}']) - fake = FakeModel({(OPUS, "guidance"): _fail(turns=[2]), (GPT, "guidance"): _fail(turns=[2])}) - step = _step() - monkeypatch.setattr(step, "_complete", lambda context: fake) +async def test_on_an_always_on_run_the_judge_reads_what_the_user_sim_got_and_said_and_only_records(monkeypatch, + tmp_path): + rig, ctx, prompt = await _always_on(monkeypatch, tmp_path) + fake = FakeModel({"guidance": _fail(turns=[2])}) - await step.execute(ctx) + await _step(fake, monkeypatch).execute(ctx) - head = fake.calls[0]["parts"][0] - assert "\n" + prompts[0].strip() + "\n" in head # the very prompt it ran with + run_text = fake.run_text() + assert "\n" + prompt.strip() + "\n" in run_text # the very prompt it ran with + assert len(rig.user_calls) == 2 for call in rig.user_calls: # and every check-in word for word - assert "\n" + call["parts"][0]["text"].strip() + "\n" in head + fake.calls[0]["parts"][1] - run_text = head + fake.calls[0]["parts"][1] + assert "\n" + call["parts"][0]["text"].strip() + "\n" in run_text assert '"dad is allergic to shellfish, swap it pls"' in run_text and "K3" not in run_text.split("")[1] - entry = ctx.metadata["user_sim_judge"]["usj"] - assert entry["always_on_step_id"] == "week" and entry["user_sim"] == "linda" - assert set(entry["checks"]) == set(core.CHECK_NAMES) and entry["checks"]["guidance"]["turns"] == [2] - assert entry["user_sim_error"]["because"] == ["guidance"] and entry["notes"] == [] - assert rig.record(ctx)["user_sim_judge"] == {"verifier_id": "usj", "user_sim_error": "yes", "marked": True, - "because": ["guidance"]} + entry = _entry(ctx) + assert (entry["run_step_id"], entry["run_kind"], entry["user_sim"], entry["judge_model"]) == ( + "week", "always_on", "user-sim", JUDGE) + assert entry["instructions"]["matches"] is True and entry["notes"] == [] + assert {k: v["verdict"] for k, v in entry["checks"].items()} == {**dict.fromkeys(core.CHECK_NAMES, "pass"), + "guidance": "fail"} + assert entry["checks"]["guidance"]["turns"] == [2] + # A grade-changing failure, recorded only: nothing a grader reads changes. + assert entry["user_sim_error"] == {"value": "yes", "because": ["guidance"], "unknown": [], "marked": False, + "not_marked": "mark_user_sim_error is off: verdicts are recorded only"} + assert ctx.prompt_responses[-1].error_type is None and "user_sim_judge" not in rig.record(ctx) assert "verifications" not in ctx.metadata # never mixed into the assistant's scores + assert ctx.metadata["models"]["user sim judge"]["judge_user_sim"] == JUDGE @pytest.mark.usefixtures("local_stores") @pytest.mark.asyncio async def test_without_the_stored_timeline_the_judge_reads_the_tail_and_is_told_what_s_missing(monkeypatch, tmp_path): - rig, ctx, _ = await _always_on(monkeypatch, tmp_path, ['{"outcome": "reply", "message": "swap it pls"}']) + rig, ctx, _ = await _always_on(monkeypatch, tmp_path) record, timeline = rig.record(ctx), rig.timeline(ctx) record.pop("object_url") record["timeline_tail"] = [{k: v for k, v in e.items() if k not in ("input", "read")} for e in timeline] fake = FakeModel() - step = _step(checks=["made_up"]) - monkeypatch.setattr(step, "_complete", lambda context: fake) - await step.execute(ctx) + await _step(fake, monkeypatch, checks=["made_up"]).execute(ctx) head = fake.calls[0]["parts"][0] assert "Note: the run's full record wasn't stored" in head and "### What it was sent\n(not recorded)" in head - assert "Note: the opening request wasn't recorded" in head + assert "## Turn 1 · 2026-11-09T14:00:00Z UTC · GRADED" in head @pytest.mark.usefixtures("local_stores") @pytest.mark.asyncio async def test_a_run_that_ended_in_an_error_is_not_judged(monkeypatch, tmp_path): - rig, ctx, _ = await _always_on(monkeypatch, tmp_path, []) + rig, ctx, _ = await _always_on(monkeypatch, tmp_path, answers=[]) rig.record(ctx)["error_type"] = "agent_turn_failed" fake = FakeModel() - step = _step() - monkeypatch.setattr(step, "_complete", lambda context: fake) - await step.execute(ctx) + await _step(fake, monkeypatch, mark_user_sim_error=True).execute(ctx) - assert fake.calls == [] and "agent_turn_failed" in ctx.metadata["user_sim_judge"]["usj"]["skipped"] + assert fake.calls == [] and _entry(ctx)["skipped"] == "the run ended with error_type=agent_turn_failed" + assert _entry(ctx)["user_sim_error"] == {"value": "no", "because": [], "unknown": [], "marked": False} @pytest.mark.usefixtures("local_stores") @pytest.mark.asyncio -async def test_a_run_s_step_params_choose_the_checks(monkeypatch, tmp_path): - rig, ctx, _ = await _always_on(monkeypatch, tmp_path, ['{"outcome": "reply", "message": "ok"}']) - ctx.metadata["user_overrides"] = {"step_params": {"judge_user_sim": {"checks": ["rogue"]}}} +async def test_a_run_s_step_params_choose_the_checks_and_the_model(monkeypatch, tmp_path): + _, ctx, _ = await _always_on(monkeypatch, tmp_path) + ctx.metadata["user_overrides"] = {"step_params": {"judge_user_sim": {"checks": ["rogue"], "judge_model": "other"}}} fake = FakeModel() - step = _step() - monkeypatch.setattr(step, "_complete", lambda context: fake) - await step.execute(ctx) + await _step(fake, monkeypatch).execute(ctx) + + assert {(c["check"], c["model"]) for c in fake.calls} == {("rogue", "other")} + assert _entry(ctx)["error_checks"] == ["rogue"] and _entry(ctx)["judge_model"] == "other" + + +# ---- the step on a prompt_agent chat run -------------------------------------------------------------------------- +async def _chat(monkeypatch): + Peers(monkeypatch, {AGENT: [text("draft one"), text("draft two"), text("glad to help")], + USER: [said("reply", "make it shorter", "D1"), said("done", "perfect, thanks!")]}) + ctx = await _chat_context(monkeypatch) + await _chat_step().execute(ctx) + return ctx + + +@pytest.mark.usefixtures("local_stores") +@pytest.mark.asyncio +async def test_on_a_chat_run_the_judge_reads_each_stored_read_and_the_prompt_it_was_deployed_with(monkeypatch): + ctx = await _chat(monkeypatch) + us = ctx.metadata[USER_SIMS_KEY]["hannah"] + fake = FakeModel({"style": _fail(turns=[2], change=False)}) + + await _step(fake, monkeypatch, run_step_id="talk").execute(ctx) + + run_text = fake.run_text() + shown = run_text.split("\n")[1].split("\n")[0] + "\n" + assert hashlib.sha256(shown.encode()).hexdigest() == us["prompt_sha256"] # the prompt deploy_user_sim_agent set + for summary in ctx.metadata["user_sim_reads"]["talk"]: + full = usj._load(summary["object_url"]) + assert "\n" + full["input"].strip() + "\n" in run_text + assert 'outcome: reply\nmessage: "make it shorter"' in run_text and "D1" not in run_text.split("")[1] + assert 'outcome: done\nmessage: "perfect, thanks!"' in run_text + entry = _entry(ctx) + assert (entry["run_kind"], entry["user_sim"], entry["instructions"]["matches"]) == ("prompt_agent", "hannah", True) + assert entry["checks"]["style"]["verdict"] == "fail" and entry["user_sim_error"]["value"] == "no" + assert entry["graded_turns"] == [1, 2] and ctx.prompt_responses[-1].error_type is None + + +@pytest.mark.usefixtures("local_stores") +@pytest.mark.asyncio +async def test_with_marking_turned_on_a_user_sim_error_marks_the_prompt_so_later_graders_skip_it(monkeypatch): + ctx = await _chat(monkeypatch) + + await _step(FakeModel({"oversharing": _fail()}), monkeypatch, mark_user_sim_error=True).execute(ctx) + + assert _entry(ctx)["user_sim_error"] == {"value": "yes", "because": ["oversharing"], "unknown": [], + "marked": True} + response = ctx.prompt_responses[-1] + assert response.error_type == "user_sim_error" + assert response.error_message == "The user sim judge found a user sim error: oversharing" + grader = AgentPromptResponseVerifierTaskStep("grade", None, prompt_id=response.prompt_id, verifier_id="grade", + criteria=[{"type": "response_contains", "needles": ["glad"]}]) + await grader.execute(ctx) + assert ctx.metadata["verifications"]["grade"]["results"][0]["message"] == "Skipped: prompt had error_type=user_sim_error" + + +@pytest.mark.usefixtures("local_stores") +@pytest.mark.asyncio +async def test_marking_can_be_turned_on_for_one_run_by_its_step_params(monkeypatch): + ctx = await _chat(monkeypatch) + ctx.metadata["user_overrides"] = {"step_params": {"judge_user_sim": {"mark_user_sim_error": True}}} + + await _step(FakeModel({"rogue": _fail()}), monkeypatch).execute(ctx) + + assert _entry(ctx)["user_sim_error"]["marked"] is True and ctx.prompt_responses[-1].error_type == "user_sim_error" + + +@pytest.mark.usefixtures("local_stores") +@pytest.mark.asyncio +async def test_instructions_that_don_t_match_the_recorded_hash_are_flagged_and_never_mark_the_run(monkeypatch): + ctx = await _chat(monkeypatch) + us = ctx.metadata[USER_SIMS_KEY]["hannah"] + us["description"] = {**us["description"], "who_you_are": [*us["description"]["who_you_are"], "A changed line."]} + fake = FakeModel({"guidance": _fail()}) + + await _step(fake, monkeypatch, mark_user_sim_error=True).execute(ctx) + + entry = _entry(ctx) + assert entry["instructions"]["matches"] is False + assert entry["instructions"]["recorded_prompt_sha256"] == us["prompt_sha256"] != entry["instructions"]["prompt_sha256"] + assert "Note: these instructions were rebuilt from the user sim's settings and differ from the ones it got" in ( + fake.calls[0]["parts"][0]) + assert entry["user_sim_error"]["value"] == "yes" and entry["user_sim_error"]["marked"] is False + assert "isn't confirmed" in entry["user_sim_error"]["not_marked"] and ctx.prompt_responses[-1].error_type is None + + +@pytest.mark.usefixtures("local_stores") +@pytest.mark.asyncio +async def test_a_chat_run_that_ended_in_an_error_or_where_the_user_sim_never_read_is_not_judged(monkeypatch): + ctx = await _chat(monkeypatch) + ctx.prompt_responses[-1].error_type = "user_sim_error" + fake = FakeModel() + await _step(fake, monkeypatch).execute(ctx) + assert fake.calls == [] and _entry(ctx)["skipped"] == "the run ended with error_type=user_sim_error" + + ctx.prompt_responses[-1].error_type = None + ctx.metadata.pop("user_sim_reads") + await _step(fake, monkeypatch, run_step_id="talk").execute(ctx) + assert fake.calls == [] and _entry(ctx)["skipped"] == "the user sim never read: the run has no user_sim_reads" + - assert {c["check"] for c in fake.calls} == {"rogue"} - assert ctx.metadata["user_sim_judge"]["usj"]["error_checks"] == ["rogue"] +def test_without_run_step_id_the_run_must_have_one_run_to_judge(): + ctx = TaskStepContext() + ctx.metadata = {"always_on": {"week": {}}, "user_sim_reads": {"talk": [{}]}} + with pytest.raises(RuntimeError, match=r"set run_step_id; this run has 2 runs .*\['talk', 'week'\]"): + usj.UserSimJudgeTaskStep("j", None)._find_run(ctx) + with pytest.raises(RuntimeError, match="step 'nope' left no run to judge"): + usj.UserSimJudgeTaskStep("j", None, run_step_id="nope")._find_run(ctx) + assert usj.UserSimJudgeTaskStep("j", None, run_step_id="talk")._find_run(ctx) == ("prompt_agent", "talk") -def test_settings_are_checked_and_round_trip(): +# ---- settings ----------------------------------------------------------------------------------------------------- +def test_settings_are_checked_and_round_trip(monkeypatch): assert "user_sim_judge" in _builtin_registry() - step = _step(checks=["guidance", "style"], error_checks=["guidance"], error_on_split=True, - rubric_items=[{"id": "leak", "family": "G1", "text": "never mentions the ring"}]) + step = usj.UserSimJudgeTaskStep("s", None, run_step_id="week", judge_model=JUDGE, checks=["guidance", "style"], + error_checks=["guidance"], mark_user_sim_error=True, + rubric_items=[{"id": "leak", "family": "G1", "text": "never mentions the ring"}]) assert step.config_problems() == [] again = usj.UserSimJudgeTaskStep.from_dict(json.loads(json.dumps(step.to_dict()))) assert again.to_dict() == step.to_dict() - assert _step().judge_models == [OPUS, GPT] and usj.UserSimJudgeTaskStep("s", None).judge_models == list( - usj.DEFAULT_JUDGE_MODELS) - problems = _step(checks=["guidance", "vibes"], error_checks=["rogue"], judge_models=[OPUS, OPUS], - rubric_items=[{"id": "x", "family": "Q1", "text": "y"}]).config_problems() + problems = usj.UserSimJudgeTaskStep("s", None, checks=["guidance", "vibes"], error_checks=["rogue"], judge_model=" ", + mark_user_sim_error="yes", + rubric_items=[{"id": "x", "family": "Q1", "text": "y"}]).config_problems() assert any("unknown checks ['vibes']" in p for p in problems) assert any("error_checks names checks that don't run: ['rogue']" in p for p in problems) - assert any("names a model twice" in p for p in problems) and any("family 'Q1'" in p for p in problems) - assert _step(checks=["style"]).error_checks == [] # by default, the error checks among those that run + assert any("judge_model must be a model name" in p for p in problems) and any("family 'Q1'" in p for p in problems) + assert any("mark_user_sim_error must be true or false" in p for p in problems) + assert usj.UserSimJudgeTaskStep("s", None, checks=["style"]).error_checks == [] # the error checks that run + + +def test_by_default_it_records_only_never_fails_the_task_and_judges_with_the_verifiers_default_model(monkeypatch): + step = usj.UserSimJudgeTaskStep.from_dict({"id": "s", "type": "user_sim_judge"}) + assert (step.mark_user_sim_error, step.fail_task_on_error, step.judge_model) == (False, False, None) + assert step.to_dict()["judge_model"] is None # resolved when it runs, so the task doesn't pin it + class Config: + role = None + + def get_model_for_role(self, role): + assert role == "judge" + return self.role + + fake = Config() + monkeypatch.setattr("agent_env.config.get_config", lambda: fake) + assert step._settings(TaskStepContext())["model"] == "claude-sonnet-4-6" + fake.role = "anthropic/claude-opus-4-8" + assert step._settings(TaskStepContext())["model"] == "anthropic/claude-opus-4-8" + + +def test_the_task_must_say_which_run_when_it_has_more_than_one_and_name_a_run_step(): + week = ao.AlwaysOnTaskStep("week", None, user_sim="u", prompt_id="ask", agent_name="solver") + talk = _chat_step() + other = AgentPromptResponseVerifierTaskStep("grade", None, prompt_id="ask") + assert usj.UserSimJudgeTaskStep("j", None).task_problems([week]) == [] + assert "set run_step_id" in usj.UserSimJudgeTaskStep("j", None).task_problems([week, talk])[0] + assert usj.UserSimJudgeTaskStep("j", None, run_step_id="talk").task_problems([week, talk]) == [] + assert "of step 'grade' (agent_prompt_response_verifier)" in usj.UserSimJudgeTaskStep( + "j", None, run_step_id="grade").task_problems([week, other])[0] def test_the_step_asks_anthropic_models_to_cache_the_run(): From e5262943bc84072312968757c7eb0c8c10b3c056 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Thu, 12 Nov 2026 17:37:30 -0800 Subject: [PATCH 05/45] test(user sim): the satisfaction section and the rating after the run, in always_on and prompt_agent (FD-5491) Co-Authored-By: Claude Opus 5.5 --- .../task_step/task_steps/always_on.py | 4 +- .../task_step/test_user_sim_satisfaction.py | 447 ++++++++++++++++++ 2 files changed, 449 insertions(+), 2 deletions(-) create mode 100644 tst/unit/task_step/test_user_sim_satisfaction.py diff --git a/src/agent_env/task_step/task_steps/always_on.py b/src/agent_env/task_step/task_steps/always_on.py index 3de9d4424..b5db85b58 100644 --- a/src/agent_env/task_step/task_steps/always_on.py +++ b/src/agent_env/task_step/task_steps/always_on.py @@ -1008,7 +1008,7 @@ async def loop(self, now: datetime) -> None: break await self._capped(self._last_read(), "the last read", step.read_cap_seconds() + GATEWAY_ALLOWANCE_SECONDS) if self.rates: - await self._rate() + await self._rating() async def _capped(self, work, what: str, seconds: Optional[float] = None): """``work``, cut off past ``seconds`` (one turn of the loop's worst case, ``wake_reserve_seconds``): every part @@ -1147,7 +1147,7 @@ async def _last_read(self) -> None: at, self.read_mode = self._available(now) await self.read(at, last=True) - async def _rate(self) -> None: + async def _rating(self) -> None: """Once the run is over, the user sim rates it (``user_sim_versions.rate``): at her last read's time, or the world's when that's later, with the agent's messages she never read. Recorded in ``metadata[SATISFACTION_KEY][step_id]``, the whole ask in the timeline; never delivered, never in the graded diff --git a/tst/unit/task_step/test_user_sim_satisfaction.py b/tst/unit/task_step/test_user_sim_satisfaction.py new file mode 100644 index 000000000..be003fbcc --- /dev/null +++ b/tst/unit/task_step/test_user_sim_satisfaction.py @@ -0,0 +1,447 @@ +"""The user sim's rating of the run (FD-5491): a description's satisfaction lines go in the prompt, tagged S1...; once +the run is over the user sim is asked once to rate it 1-5 with a reason, in always_on after the last read and in +prompt_agent once the chat has ended. The rating is recorded, never delivered, and never fails the run. Without +satisfaction lines nothing changes: the same prompt, the same reads, no rating.""" + +from __future__ import annotations + +import asyncio +import copy +import json +from pathlib import Path + +import pytest + +from agent_env.store import get_config +from agent_env.task_step.task_steps import always_on as ao +from agent_env.task_step.task_steps.agent import user_sim_versions as us +from agent_env.task_step.task_steps.agent.deploy_user_sim_agent import DeployUserSimAgentTaskStep, persona_problems +from agent_env.task_step.task_steps.agent.user_sim_common import SATISFACTION_KEY +from tst.unit.task_step.always_on_harness import USER_SIM, Rig +from tst.unit.task_step.test_deploy_user_sim_agent import DESCRIPTION, HANNAH, _always_on, _rig_context +from tst.unit.task_step.test_deploy_user_sim_agent import _step as _deploy_step +from tst.unit.task_step.test_prompt_agent_built_in_user_sim import AGENT, USER, Peers, _context, _step, said, text +from tst.unit.task_step.test_prompt_agent_user_side import _stored + +pytestmark = pytest.mark.usefixtures("local_stores") + +DATA = Path(__file__).parents[2] / "data" / "user_sim" +PERSON = json.loads((DATA / "hannah_person.json").read_text()) +PROMPT = (DATA / "hannah_system_prompt.txt").read_text() +SATISFIED = ["5, perfect: flights, a step-free place a walk from you, and both dinners booked, with nothing you had " + "to chase.", + "3, fine: all booked, but you had to answer a question your rules already settled.", + "1, you'd have done it yourself: a dinner or the place is still not booked at the end.", + "Being asked for things that are in Mom's email bothers you."] +NOTICE = ao.FINAL_REMINDER.format(first="Linda") + + +def _satisfied(person: dict | None = None) -> dict: + person = copy.deepcopy(person or PERSON) + person["description"]["satisfaction"] = list(SATISFIED) + return person + + +def rated(score, message="all booked, no chasing", used="S1", outcome="done") -> dict: + return {"outcome": outcome, "score": score, "message": message, "used": used} + + +def _asker(*answers): + sent = [] + + async def ask(text): + sent.append(text) + got = answers[len(sent) - 1] + if isinstance(got, Exception): + raise got + if isinstance(got, dict) and "error" in got: + return got + return {"text": got if isinstance(got, str) else json.dumps(got), + "structured": got if isinstance(got, dict) else None, "trajectory_url": None} + + return ask, sent + + +# ---- the prompt ------------------------------------------------------------------------------------------------------- +def test_without_satisfaction_lines_the_prompt_is_the_tested_one_and_there_is_no_rating(): + assert us.setup("ours", PERSON)["system_prompt"] == PROMPT + assert us.rates(PERSON) is False and us.rates({"description": "Linda."}) is False + assert "HOW YOU'D RATE" not in PROMPT and " S1" not in PROMPT + + +def test_satisfaction_lines_come_last_tagged_s1_and_nothing_else_in_the_prompt_changes(): + prompt = us.setup("ours", _satisfied())["system_prompt"] + assert prompt == PROMPT + "\nHOW YOU'D RATE THE JOB (only when asked, once it's over)\n" + "".join( + f" S{i:<3}{line}\n" for i, line in enumerate(SATISFIED, 1)) + assert us.setup("ours", _satisfied())["output_format"] == us.OUTPUT_FORMAT # the same answer format + assert us.rates(_satisfied()) is True + + +def test_satisfaction_lines_do_not_change_any_read(): + read = {"time": "2026-11-17T17:05:00Z", "assistant_messages": [{"at": "2026-11-17T17:00:00Z", "text": "Booked."}], + "phone": [], "you_sent": [], "done_ends_run": False} + assert us.render(_satisfied(), [], read) == us.render(PERSON, [], read) + + +# ---- the load check --------------------------------------------------------------------------------------------------- +@pytest.mark.parametrize("satisfaction, problem", [ + ("5: all booked", "description.satisfaction must be a list of non-empty lines"), + (["5: all booked", " "], "description.satisfaction must be a list of non-empty lines"), + ([5, 3, 1], "description.satisfaction must be a list of non-empty lines"), +]) +def test_a_bad_satisfaction_section_is_named(satisfaction, problem): + person = copy.deepcopy(PERSON) + person["description"]["satisfaction"] = satisfaction + assert problem in us.problems(person) + with pytest.raises(ValueError, match="user sim"): + us.setup("ours", person) + persona = {**HANNAH, "description": person["description"]} + assert problem in persona_problems(persona) + with pytest.raises(ValueError, match="satisfaction must be a list"): + _deploy_step(persona=persona) + + +def test_satisfaction_lines_alone_are_not_a_description(): + assert "description is empty: write at least who they are" in us.problems( + {**PERSON, "description": {"satisfaction": SATISFIED}}) + + +def test_the_deploy_step_takes_a_satisfaction_section_and_records_it_for_the_reads(): + step = _deploy_step(persona={**HANNAH, "description": _satisfied()["description"]}) + assert step.person(_FakeContext())["description"]["satisfaction"] == SATISFIED + assert DeployUserSimAgentTaskStep.from_dict(step.to_dict()).persona == step.persona + + +class _FakeContext: + metadata: dict = {} + + +# ---- the rating ask --------------------------------------------------------------------------------------------------- +def test_the_rating_ask_word_for_word(): + read = {"time": "2026-11-24T05:00:00Z", "assistant_messages": [ + {"at": "2026-11-24T04:55:00Z", "text": "All set: flights, the Castro Street flat, both dinners.\n\nEnjoy!"}]} + assert us.render_rating(_satisfied(), read) == ( + "It's Mon, Nov 23, 9:00 PM. The job is over: your AI assistant won't see this, or anything you write from now " + "on.\n" + "\nNEW FROM YOUR AI ASSISTANT (you haven't read this yet):\n" + " [8:55 PM]\n" + " All set: flights, the Castro Street flat, both dinners.\n" + "\n" + " Enjoy!\n" + "\nAs Hannah, rate how the job went, from everything you've seen: 5 if it was done perfectly, 3 if it wasn't " + "perfect but was fine, 1 if you'd have done it yourself; 2 and 4 are in between. HOW YOU'D RATE THE JOB says " + "what those mean for this job. Something you were never told was done doesn't count as done.\n" + 'In "message", give your reason in one line. In "used", write the tag of the line it rests on, or "none". ' + 'Answer in JSON: {"outcome": "done", "score": 1 | 2 | 3 | 4 | 5, "message": "...", "used": "..."}') + no_unread = us.render_rating(_satisfied(), {"time": "2026-11-24T05:00:00Z", "assistant_messages": []}) + assert "NEW FROM" not in no_unread and no_unread.startswith("It's Mon, Nov 23, 9:00 PM. The job is over") + + +@pytest.mark.asyncio +async def test_a_rating_is_asked_once_and_parsed(): + ask, sent = _asker(rated(4, " all booked, but it asked about the dates ", "S2")) + got = await us.rate("ours", _satisfied(), {"time": "2026-11-24T05:00:00Z", "assistant_messages": []}, ask) + assert len(sent) == 1 and sent[0] == got["input"] + assert (got["score"], got["reason"], got["used"], got["schema_errors"], got["retried"], got["turns"]) == ( + 4, "all booked, but it asked about the dates", "S2", [], False, 1) + + +@pytest.mark.asyncio +async def test_without_satisfaction_lines_nothing_is_asked(): + ask, sent = _asker() + assert await us.rate("ours", PERSON, {"time": "2026-11-24T05:00:00Z", "assistant_messages": []}, ask) is None + assert sent == [] + + +@pytest.mark.parametrize("raw, error", [ + (rated(7), "score 7 is not one of [1, 2, 3, 4, 5]"), + (rated(0), "score 0 is not one of [1, 2, 3, 4, 5]"), + (rated(True), "score must be a whole number"), + ({"outcome": "done", "message": "fine"}, "missing score"), + (rated(3, " "), "message must give the reason"), + ("a solid 4", "the answer is not a JSON object"), +]) +def test_a_malformed_rating_has_no_score_and_says_why(raw, error): + got = us.parse_rating(raw) + assert got["score"] is None and error in got["schema_errors"], got + + +@pytest.mark.asyncio +async def test_a_malformed_rating_gets_one_retry(): + ask, sent = _asker("a solid 4", rated(4)) + got = await us.rate("ours", _satisfied(), {"time": "2026-11-24T05:00:00Z", "assistant_messages": []}, ask) + assert sent[1] == us.RATING_RETRY and got["score"] == 4 and got["retried"] is True and got["turns"] == 2 + assert got["first_answer"] == {"raw": "a solid 4", "schema_errors": ["the answer is not a JSON object"]} + + ask, sent = _asker("a solid 4", rated(9)) + got = await us.rate("ours", _satisfied(), {"time": "2026-11-24T05:00:00Z", "assistant_messages": []}, ask) + assert len(sent) == 2 and got["score"] is None + assert got["schema_errors"] == ["first answer: the answer is not a JSON object", + "answer after the retry: score 9 is not one of [1, 2, 3, 4, 5]"] + + +@pytest.mark.asyncio +async def test_a_user_sim_error_on_the_rating_is_not_retried(): + ask, sent = _asker({"text": None, "structured": None, "error": "timeout"}) + got = await us.rate("ours", _satisfied(), {"time": "2026-11-24T05:00:00Z", "assistant_messages": []}, ask) + assert len(sent) == 1 and got["score"] is None and got["retried"] is False + + +# ---- always_on -------------------------------------------------------------------------------------------------------- +LINDA_SATISFIED = {**DESCRIPTION, "satisfaction": ["5: the house and the dinner booked.", "1: nothing booked."]} + + +def _wraps_up(n, texts, rig): + if n == 1: + rig.message("Blue house or green?") + return "sent" + if NOTICE in texts: + rig.message("RESULT: blue house booked, dinner at La Ola") + return "sent" + rig.quiet() + return "" + + +async def _always_on_run(monkeypatch, tmp_path, *user_texts, description=LINDA_SATISFIED, agent=_wraps_up, + persona=None, **settings): + rig = Rig(monkeypatch, tmp_path, agent=agent, plugin=us, user_texts=user_texts) + await rig.arm() + persona = {"display_name": "Linda Murphy", "description": description, **(persona or {})} + ctx = await _rig_context(monkeypatch, rig, persona=persona) + ctx = await rig.run(_always_on(**{"end_time": "P3D", **settings}), ctx) + return rig, ctx + + +def _reply(message: str) -> str: + return json.dumps({"outcome": "reply", "message": message, "used": "W1"}) + + +@pytest.mark.asyncio +async def test_always_on_rates_once_after_the_last_read_and_records_it_without_delivering_it(monkeypatch, tmp_path): + rig, ctx = await _always_on_run(monkeypatch, tmp_path, _reply("blue!"), _reply("amazing, thanks"), + json.dumps(rated(5, "house and dinner booked", "S1"))) + + asks = [c["parts"][0]["text"] for c in rig.user_calls] + assert len(asks) == 3 and asks[2] == rig.events(ctx, "user_sim_rating")[0]["input"] + assert asks[2].startswith("It's ") and us.RATING_ASK.format(first="Linda") in asks[2] + assert "NEW FROM YOUR AI ASSISTANT (you haven't read" not in asks[2] # the last read already showed the wrap-up + assert len({c["context_id"] for c in rig.user_calls}) == 1 # the run's one user sim session + order = [e["event"] for e in rig.timeline(ctx)] + assert order[-3:] == ["user_sim_read", "user_sim_rating", "end"] + assert rig.events(ctx, "user_sim_read")[-1]["last_read"] is True + + rating = ctx.metadata[SATISFACTION_KEY]["week"] + assert (rating["user_sim"], rating["score"], rating["reason"], rating["used"]) == ( + "user-sim", 5, "house and dinner booked", "S1") + assert rating["at"] == rig.events(ctx, "user_sim_read")[-1]["virtual_time"] + assert (rating["schema_errors"], rating["retried"], rating["user_sim_error"], rating["user_sim_turns"]) == ( + [], False, None, 1) + + # Never delivered: the agent's wakes, the graded chat, the response and the hook records don't have it. + everything_sent = json.dumps(rig.sent) + json.dumps(rig.chat(ctx)) + ctx.prompt_responses[-1].response + assert "house and dinner booked" not in everything_sent + assert "house and dinner booked" not in json.dumps(ctx.metadata[ao.HOOK_RECORDS_KEY]) + record = rig.record(ctx) + assert record["counts"]["user_sim_reads"] == 2 and record["counts"]["user_replies"] == 2 + assert record["end_reason"] == "end_time" and record["error_type"] is None + assert record["settings"]["rating_cap_seconds"] == _always_on().rating_cap_seconds() + + +@pytest.mark.asyncio +async def test_always_on_without_satisfaction_lines_asks_and_records_nothing_more(monkeypatch, tmp_path): + rig, ctx = await _always_on_run(monkeypatch, tmp_path, _reply("blue!"), _reply("amazing, thanks"), + description=DESCRIPTION) + assert len(rig.user_calls) == 2 and SATISFACTION_KEY not in ctx.metadata + assert not rig.events(ctx, "user_sim_rating") and "rating_cap_seconds" not in rig.record(ctx)["settings"] + + +@pytest.mark.asyncio +async def test_always_on_rates_even_with_no_wrap_up_to_read(monkeypatch, tmp_path): + def asks_once(n, texts, rig): + if n == 1: + rig.message("Blue house or green?") + return "sent" + rig.quiet() + return "" + + rig, ctx = await _always_on_run(monkeypatch, tmp_path, _reply("blue!"), json.dumps(rated(1, "nothing booked")), + agent=asks_once) + assert [e["event"] for e in rig.timeline(ctx)][-2:] == ["user_sim_rating", "end"] + assert len(rig.events(ctx, "user_sim_read")) == 1 and ctx.metadata[SATISFACTION_KEY]["week"]["score"] == 1 + + +@pytest.mark.asyncio +async def test_messages_she_never_read_come_with_the_rating_and_it_is_no_reply(monkeypatch, tmp_path): + """After max_replies she stops reading, so the wrap-up is unread: it comes with the rating, and the rating doesn't + count as a reply.""" + rig, ctx = await _always_on_run(monkeypatch, tmp_path, _reply("blue!"), json.dumps(rated(4, "booked, mostly")), + persona={"max_replies": 1}) + assert len(rig.events(ctx, "user_sim_read")) == 1 + ask = rig.user_calls[-1]["parts"][0]["text"] + assert "NEW FROM YOUR AI ASSISTANT (you haven't read this yet):" in ask + assert "RESULT: blue house booked, dinner at La Ola" in ask + rating = ctx.metadata[SATISFACTION_KEY]["week"] + assert rating["score"] == 4 and rating["unread_shown"] == 1 + assert rig.record(ctx)["counts"]["user_replies"] == 1 + assert rig.record(ctx)["pending_at_end"]["unread_by_user"]["count"] == 1 # a rating isn't a read + + +@pytest.mark.asyncio +@pytest.mark.parametrize("answer, error", [ + ("not json", None), + (None, "timeout"), +]) +async def test_a_rating_that_fails_is_recorded_and_the_run_ends_as_it_would(monkeypatch, tmp_path, answer, error): + rig = Rig(monkeypatch, tmp_path, agent=_wraps_up, plugin=us, + user_texts=(_reply("blue!"), _reply("amazing, thanks"), *([answer, answer] if answer else []))) + if error: # the user sim times out on the rating + real = rig._send_and_wait + + async def times_out(url, parts, **kw): + if url == USER_SIM and len(rig.user_calls) >= 2: + raise TimeoutError("the user sim took too long") + return await real(url, parts, **kw) + + monkeypatch.setattr(ao, "send_and_wait", times_out) + await rig.arm() + ctx = await _rig_context(monkeypatch, rig, persona={"display_name": "Linda Murphy", "description": LINDA_SATISFIED}) + ctx = await rig.run(_always_on(end_time="P3D"), ctx) + + rating = ctx.metadata[SATISFACTION_KEY]["week"] + assert rating["score"] is None + if error: + assert rating["user_sim_error"].startswith("timeout") + else: + assert rating["retried"] is True and rating["schema_errors"] + record = rig.record(ctx) + assert (record["end_reason"], record["error_type"], ctx.prompt_responses[-1].error_type) == ( + "end_time", None, None) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("breaks, error", [("raise", "RuntimeError: the plug-in broke"), ("hang", "TimeoutError")]) +async def test_a_rating_that_raises_or_runs_past_its_cap_is_recorded_and_the_run_ends_as_it_would( + monkeypatch, tmp_path, breaks, error): + class Breaks: + setup, answer, rates = us.setup, us.answer, us.rates + + @staticmethod + async def rate(version, person, read, ask): + if breaks == "raise": + raise RuntimeError("the plug-in broke") + await asyncio.sleep(5) + + monkeypatch.setattr(ao.AlwaysOnTaskStep, "rating_cap_seconds", lambda step: 0.05) + rig = Rig(monkeypatch, tmp_path, agent=_wraps_up, plugin=Breaks, + user_texts=(_reply("blue!"), _reply("amazing, thanks"))) + await rig.arm() + ctx = await _rig_context(monkeypatch, rig, persona={"display_name": "Linda Murphy", "description": LINDA_SATISFIED}) + ctx = await rig.run(_always_on(end_time="P3D"), ctx) + assert ctx.metadata[SATISFACTION_KEY]["week"]["user_sim_error"] == error + assert rig.record(ctx)["end_reason"] == "end_time" and rig.record(ctx)["error_type"] is None + + +@pytest.mark.asyncio +async def test_a_run_that_ended_on_an_error_is_not_rated(monkeypatch, tmp_path): + from tst.unit.task_step.always_on_harness import Fail + + rig, ctx = await _always_on_run(monkeypatch, tmp_path, agent=lambda n, texts, rig: Fail()) + assert rig.user_calls == [] + assert ctx.metadata[SATISFACTION_KEY]["week"] == {"user_sim": "user-sim", "score": None, "reason": None, + "skipped": "the run ended on agent_turn_failed"} + + +@pytest.mark.asyncio +async def test_the_time_budget_keeps_room_for_the_rating(monkeypatch, tmp_path): + step = _always_on(end_time="P3D") + # The default budget is lowered to fit the closing, so with a rating it closes that much sooner. + assert step.effective_time_budget(rating=True) == step.effective_time_budget() - step.rating_cap_seconds() + tight = ao.JOB_LIMIT_SECONDS - ao.JOB_HEADROOM_SECONDS - step.wake_reserve_seconds() + step = _always_on(end_time="P3D", time_budget_seconds=tight) + with pytest.raises(ValueError, match="can't fit"): + step.effective_time_budget(rating=True) + satisfied = _deploy_step(agent_name="user-sim", persona={**HANNAH, "description": _satisfied()["description"]}) + plain = _deploy_step(agent_name="user-sim") + [problem] = step.task_problems([satisfied, step]) + assert "can't fit" in problem and "with the user sim's rating after the run" in problem + assert step.task_problems([plain, step]) == [] + + +# ---- prompt_agent ----------------------------------------------------------------------------------------------------- +def _rated_text(score, message="booked, no chasing") -> list[dict]: + return text(json.dumps(rated(score, message))) + + +def _rating(ctx) -> dict: + return ctx.metadata[SATISFACTION_KEY]["talk"] + + +@pytest.mark.asyncio +async def test_prompt_agent_rates_once_the_chat_has_ended_with_the_reply_she_never_read(monkeypatch): + peers = Peers(monkeypatch, {AGENT: [text("draft one"), text("glad to help")], + USER: [said("done", "perfect, thanks!"), _rated_text(5)]}) + ctx = await _context(monkeypatch, persona={**HANNAH, "description": _satisfied()["description"]}) + + result = await _step().execute(ctx) + + user_asks = [r[2]["params"]["message"]["parts"][0]["text"] for r in peers.requests(ctx) + if r[1] == f"{USER}/a2a" and r[2]["method"] == "message/send"] + assert len(user_asks) == 2 and us.RATING_ASK.format(first="Hannah") in user_asks[1] + assert "NEW FROM YOUR AI ASSISTANT (you haven't read this yet):\n [" in user_asks[1] + assert "glad to help" in user_asks[1] + rating = _rating(ctx) + assert (rating["user_sim"], rating["score"], rating["reason"], rating["unread_shown"]) == ( + "hannah", 5, "booked, no chasing", 1) + stored = json.loads(get_config().get_object_store().get(rating["object_url"])) + assert stored["input"] == user_asks[1] and stored["score"] == 5 + # Never delivered: not sent to the agent, not in the graded conversation, not a read. + agent_parts = json.dumps([r for r in peers.requests(ctx) if r[1] == f"{AGENT}/a2a"]) + assert "booked, no chasing" not in agent_parts + assert "booked, no chasing" not in json.dumps(_stored(ctx, peers)["messages"]) + assert len(ctx.metadata["user_sim_reads"]["talk"]) == 1 + assert (result.prompt_responses[-1].response, result.prompt_responses[-1].error_type) == ("glad to help", None) + + +@pytest.mark.asyncio +async def test_prompt_agent_shows_nothing_unread_when_she_read_the_last_reply(monkeypatch): + peers = Peers(monkeypatch, {AGENT: [text("draft one")], USER: [said("no_reply"), _rated_text(3)]}) + ctx = await _context(monkeypatch, persona={**HANNAH, "description": _satisfied()["description"]}) + await _step().execute(ctx) + asks = [r for r in peers.requests(ctx) if r[1] == f"{USER}/a2a" and r[2]["method"] == "message/send"] + assert "NEW FROM" not in asks[1][2]["params"]["message"]["parts"][0]["text"] + assert _rating(ctx)["score"] == 3 and _rating(ctx)["unread_shown"] == 0 + + +@pytest.mark.asyncio +async def test_prompt_agent_without_satisfaction_lines_asks_nothing_more(monkeypatch): + peers = Peers(monkeypatch, {AGENT: [text("draft one")], USER: [said("no_reply")]}) + ctx = await _context(monkeypatch) + await _step().execute(ctx) + assert [r[1] for r in peers.requests(ctx)] == [f"{AGENT}/a2a"] * 2 + [f"{USER}/a2a"] * 2 + assert SATISFACTION_KEY not in ctx.metadata + + +@pytest.mark.asyncio +async def test_prompt_agent_a_rating_that_fails_is_recorded_and_the_run_is_unchanged(monkeypatch): + Peers(monkeypatch, {AGENT: [text("draft one")], USER: [said("no_reply"), ("failed", _rated_text(5))]}) + ctx = await _context(monkeypatch, persona={**HANNAH, "description": _satisfied()["description"]}) + result = await _step().execute(ctx) + assert _rating(ctx)["score"] is None and _rating(ctx)["user_sim_error"] == "state failed" + assert (result.prompt_responses[-1].response, result.prompt_responses[-1].error_type) == ("draft one", None) + + +@pytest.mark.asyncio +async def test_prompt_agent_a_rating_that_raises_is_recorded_and_the_run_is_unchanged(monkeypatch): + Peers(monkeypatch, {AGENT: [text("draft one")], USER: [said("no_reply")]}) # nothing left to answer the rating + ctx = await _context(monkeypatch, persona={**HANNAH, "description": _satisfied()["description"]}) + result = await _step().execute(ctx) + assert _rating(ctx)["score"] is None and _rating(ctx)["user_sim_error"] + assert result.prompt_responses[-1].error_type is None + + +@pytest.mark.asyncio +async def test_prompt_agent_does_not_rate_a_chat_the_user_sim_broke(monkeypatch): + Peers(monkeypatch, {AGENT: [text("draft one")], USER: [("failed", said("reply", "x"))]}) + ctx = await _context(monkeypatch, persona={**HANNAH, "description": _satisfied()["description"]}) + await _step().execute(ctx) + assert _rating(ctx)["skipped"] == "the user sim failed: state failed" and _rating(ctx)["score"] is None From 47ed2c2073683bb66dadd2c5b8b2b41413fb825f Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Thu, 12 Nov 2026 17:37:48 -0800 Subject: [PATCH 06/45] docs(user sim): the satisfaction section and the rating after the run (FD-5491) Co-Authored-By: Claude Opus 5.5 --- .../task_steps/agent/deploy_user_sim_agent.py | 14 ++++++++- .../task_step/task_steps/agent/user_sim.md | 30 +++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/src/agent_env/task_step/task_steps/agent/deploy_user_sim_agent.py b/src/agent_env/task_step/task_steps/agent/deploy_user_sim_agent.py index 5b222fbd8..ece999cfa 100644 --- a/src/agent_env/task_step/task_steps/agent/deploy_user_sim_agent.py +++ b/src/agent_env/task_step/task_steps/agent/deploy_user_sim_agent.py @@ -19,7 +19,8 @@ a reply is held; ``max_replies``: after that many replies the user stops reading. ``schedule`` and ``response_times`` need the task's clock (``sync_env_clock``): always_on refuses to start a run that sets either with no clock armed. The user's time zone is the task clock's. -- ``description``, ``permissions``, ``keep_private``: in ``person``, for the reads. +- ``description``, ``permissions``, ``keep_private``: in ``person``, for the reads. A description with + ``satisfaction`` lines also gets the rating after the run (below). - ``opening_prompt``: the user's first message, which always_on sends; always_on has no prompt of its own. - ``prompt_set``: the system prompt and model are on the agent, so always_on sets neither; ``multi_day``: the done line the prompt was written with, which always_on refuses to run against a run of the other kind. @@ -29,6 +30,7 @@ setup(version: str, person: dict) -> {"system_prompt": str | None, "output_format": dict | None} async answer(version: str, person: dict, history: list[dict], read: dict, ask) -> dict + async rate(version: str, person: dict, read: dict, ask) -> dict | None ``person`` is ``{"key": name, "name", "first", "time_zone", "description", "permissions", "keep_private", "opening", "done_ends_run"}``, with ``time_zone`` the run's zone (the persona's, else the task clock's, else UTC) and ``opening`` the user's opening @@ -61,6 +63,16 @@ line, when it sets one, in ``you_sent``. Its answer is recorded and never delivered; a ``done`` or an error on it changes nothing. +``rate`` runs once the run is over, only for a user whose description has ``satisfaction`` lines (None, asking +nothing, otherwise): in always_on after that last read, in prompt_agent once the chat has ended; not after a run that +ended on an error. ``read`` is ``{"time": RFC 3339 UTC, "assistant_messages": [{"at", "text"}]}``, the agent's +messages the user never read (often none). It asks the user sim, in the run's session, to rate the outcome 1-5 by its +``satisfaction`` lines with a one-line reason, the answer in the same format as a read's, and returns ``{"score": 1-5 +| None, "reason", "used", "schema_errors", "input", "raw", "retried", "turns"}`` (``first_answer`` after a retry). The +rating is no reply and doesn't count toward ``max_replies``; it is recorded in +``metadata["user_sim_satisfaction"][step_id]`` and never reaches the agent or the graded conversation. A rating that +fails is recorded with its error and doesn't fail the run. + prompt_agent with ``user_agent_name`` set to the agent this step deployed reads it with the same ``answer``, in one sitting: one read per agent reply, ``phone`` empty, ``done_ends_run`` true, its prompt this step's opening (user_sim.md). """ diff --git a/src/agent_env/task_step/task_steps/agent/user_sim.md b/src/agent_env/task_step/task_steps/agent/user_sim.md index 2b63d85f2..6edd0ab5f 100644 --- a/src/agent_env/task_step/task_steps/agent/user_sim.md +++ b/src/agent_env/task_step/task_steps/agent/user_sim.md @@ -79,6 +79,36 @@ user sim names in `used` when it relies on that line. | `dont_mention` | M | One "don't mention X unless asked" line per tempting fact | | `phone` | F | What they do with what reaches their phone | | `when_done` | | When they'd say they're done | +| `satisfaction` | S | How they'd rate the job once it's over (optional; see below) | + +### The rating at the end + +With a `satisfaction` section, the user sim rates the job once the run is over: 1 to 5, with a one-line reason. Write +what each end of the scale means for this task, and what moves the score for this person (the intake's User Sim +Questions 6 and 7): + +```json +"satisfaction": ["5, perfect: flights, a step-free place a walk from you, and both dinners booked, with nothing you had to chase.", + "3, fine: all booked, but you had to answer a question your rules already settled.", + "1, you'd have done it yourself: a dinner or the place is still not booked at the end.", + "Being asked for things that are in Mom's email bothers you."] +``` + +The lines go last in the prompt, under "HOW YOU'D RATE THE JOB (only when asked, once it's over)". The rating is +asked once: in always-on after the user sim's read of the wrap-up, in `prompt_agent` once the chat has ended, with +any of the assistant's messages it never read; never after a run that ended on an error. The ask says 5 is perfect, +3 fine, 1 "you'd have done it yourself", 2 and 4 in between, and that something it was never told was done doesn't +count as done. The answer uses the usual format, with `score` and the reason in `message`. + +The rating goes to `metadata["user_sim_satisfaction"][]`: `score` (None when it failed), `reason`, `used`, +`at`, `unread_shown`, `schema_errors`, `retried`, `user_sim_error`, and `skipped` when the run ended on an error. The +whole ask and raw answer are in always-on's timeline (`user_sim_rating`) or, for `prompt_agent`, at the summary's +`object_url`. It never reaches the assistant or the graded conversation, isn't a reply (so it doesn't count toward +`max_replies`), and a rating that fails doesn't fail the run. A user sim rates more kindly than people do, so treat it +as a signal next to the graders, not the score. + +Without a `satisfaction` section, the prompt and the reads are unchanged and nothing is asked. In always-on, a rating +also adds its worst case (one read's) to the run's closing, so the default time budget is that much lower. Don't write the opening request into the description. Put it in `opening_prompt`: the prompt shows it right after the first `who_you_are` line ("Earlier you texted the assistant: …"). From e9e8200aa76fceb060dac98da5fcdcd4615eaf5d Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Thu, 12 Nov 2026 18:01:14 -0800 Subject: [PATCH 07/45] feat(user sim): the user's first message can attach files (FD-5494) deploy_user_sim_agent's opening_prompt also takes a list of text and file parts, in prompt_agent's parts format; a plain string works as before. A file part may carry a one-line metadata.description for the user sim, which can't open files. The parts are checked when the task loads. - always_on sends the parts on the first wake in order: the joined text as the user's labelled line, each file in its place, without the description; the graded conversation's first user turn has the files. - The user sim's prompt and first read say what it attached: "(you attached itinerary.pdf: Mom and Dad's flight itinerary)", or the file name alone with no description. Text-only openings are byte-identical. - prompt_agent with the built-in user sim compares the files too: the same files in the same order after seeding, and its error names both lists. - Files go through the same sharing and seed filling as any prompt file part (apply_seed, lifted out of PromptAgentTaskStep._apply_seed, which now calls it). - The record keeps "opening" as the text, so text readers are unchanged, and adds "opening_parts" (as written) for an opening given as parts. The training harness's first prompt is the opening's text. Co-Authored-By: Claude Opus 5.5 --- .../task_steps/agent/deploy_user_sim_agent.py | 61 ++-- .../task_steps/agent/user_sim_common.py | 90 +++++- .../task_steps/agent/user_sim_versions.py | 22 +- .../task_step/task_steps/always_on.py | 19 +- .../task_step/task_steps/prompt_agent.py | 72 +++-- src/agent_env/training/harness.py | 9 +- .../task_step/test_user_sim_opening_files.py | 294 ++++++++++++++++++ tst/unit/training/harness_test.py | 10 + 8 files changed, 514 insertions(+), 63 deletions(-) create mode 100644 tst/unit/task_step/test_user_sim_opening_files.py diff --git a/src/agent_env/task_step/task_steps/agent/deploy_user_sim_agent.py b/src/agent_env/task_step/task_steps/agent/deploy_user_sim_agent.py index ece999cfa..8767536ea 100644 --- a/src/agent_env/task_step/task_steps/agent/deploy_user_sim_agent.py +++ b/src/agent_env/task_step/task_steps/agent/deploy_user_sim_agent.py @@ -10,6 +10,12 @@ --user-model``, a bundle's ``user_model``), else the step's ``model``. A run-wide ``a2a_agent_id`` is ignored, so the agent under test never plays the user; this step's own ``step_params`` can still change it. +``opening_prompt`` is text, or a list of A2A text and file parts in ``prompt_agent``'s ``parts`` format, for a first +message that attaches files (any type). A file part may carry a one-line ``metadata.description`` of what it is, for +the user sim, which can't open files; the agent is sent the file without it. The parts are checked when the task loads: +at least one text part with text, each part a valid text or file part, each description one line. The seed fills +````s in text and in a file's ``uri`` and ``name``, as in any prompt's parts. + What always_on reads from the record (``metadata["user_sims"][agent_name]``): - ``name`` (the agent name), ``display_name`` and ``first_name`` (else the display name's first word): how the agent's @@ -21,7 +27,10 @@ with no clock armed. The user's time zone is the task clock's. - ``description``, ``permissions``, ``keep_private``: in ``person``, for the reads. A description with ``satisfaction`` lines also gets the rating after the run (below). -- ``opening_prompt``: the user's first message, which always_on sends; always_on has no prompt of its own. +- ``opening`` (the opening's text parts, joined) and, for an ``opening_prompt`` given as parts, ``opening_parts`` + (those parts, descriptions included): both as written, the seed not yet filled in. The user's first message, which + always_on sends, its text as one labelled line where its first text part was and each file in its place; always_on + has no prompt of its own. - ``prompt_set``: the system prompt and model are on the agent, so always_on sets neither; ``multi_day``: the done line the prompt was written with, which always_on refuses to run against a run of the other kind. - ``model``, ``model_source``, ``prompt_sha256``, ``user_version``, ``setup``: for the run's record. @@ -34,20 +43,21 @@ ``person`` is ``{"key": name, "name", "first", "time_zone", "description", "permissions", "keep_private", "opening", "done_ends_run"}``, with ``time_zone`` the run's zone (the persona's, else the task clock's, else UTC) and ``opening`` the user's opening -request, ``{"at": RFC 3339 UTC, "text"}``, so ``setup`` can write it into the system prompt (this step's has no -``at``). The opening is also the first line of the first read's ``you_sent``. ``done_ends_run`` is set before -``setup``, as each read has it, since the done line is in the system prompt; the built-in user sim uses the multi-day -wording when it's missing. +request, ``{"at": RFC 3339 UTC, "text"}``, with ``"files": [{"name", "description"}]`` when it attaches files (the +description only when the part has one, the name its file's ``name``, else the last segment of its ``uri``), so +``setup`` can write it into the system prompt (this step's has no ``at``). The opening is also the first line of the +first read's ``you_sent``. ``done_ends_run`` is set before ``setup``, as each read has it, since the done line is in +the system prompt; the built-in user sim uses the multi-day wording when it's missing. ``answer`` runs once per read, at the time the user would read. ``read`` is ``{"time": RFC 3339 UTC, "assistant_messages": [{"at", "text"}], "phone": [{"at", "channel", "text"}], "you_sent": [{"at", "text"}], "done_ends_run": bool}``: the agent's message_user texts since the last read, oldest first; what triggers sent ``user_phone`` since the last read, its sender in ``channel`` (``"Gmail · Carol Reyes (Mom)"``) as in the agent's -alerts; the user's own lines (the opening request, ``user_says`` lines, ``final``) each at the time it was sent; never -anything meant for the agent; and whether done ends the run (one-sitting) or is only recorded (multi-day). The user -sim reads only after the agent has used message_user. ``history`` is every earlier ``{"read", "answer"}``, oldest -first. ``ask(text)`` sends one turn to the user sim agent (one session per run) and returns ``{"text", "structured", -"trajectory_url"}``, or ``{"text": None, "structured": None, "error"}``. +alerts; the user's own lines (the opening request, with its ``files``, ``user_says`` lines, ``final``) each at the time +it was sent; never anything meant for the agent; and whether done ends the run (one-sitting) or is only recorded +(multi-day). The user sim reads only after the agent has used message_user. ``history`` is every earlier +``{"read", "answer"}``, oldest first. ``ask(text)`` sends one turn to the user sim agent (one session per run) and +returns ``{"text", "structured", "trajectory_url"}``, or ``{"text": None, "structured": None, "error"}``. ``answer`` returns ``{"outcome": "reply" | "no_reply" | "done", "message": str, "used": str | None}``, and may add ``input``, ``raw``, ``score``, ``schema_errors``, ``turns``, ``retried``, ``first_answer``, ``shown`` and @@ -74,7 +84,8 @@ fails is recorded with its error and doesn't fail the run. prompt_agent with ``user_agent_name`` set to the agent this step deployed reads it with the same ``answer``, in one -sitting: one read per agent reply, ``phone`` empty, ``done_ends_run`` true, its prompt this step's opening (user_sim.md). +sitting: one read per agent reply, ``phone`` empty, ``done_ends_run`` true, its prompt this step's opening: the same +text, and the same files in the same order once the seed is filled in (user_sim.md). """ from __future__ import annotations @@ -90,6 +101,7 @@ from agent_env.entity.scopes import FieldScope from agent_env.task_step.context import TaskStepContext from agent_env.task_step.task_steps.agent import user_sim_versions +from agent_env.task_step.task_steps.agent.user_sim_common import attached, opening_parts, opening_problems, opening_text from agent_env.task_step.task_steps.deploy_agent import DeployAgentTaskStep logger = logging.getLogger(__name__) @@ -187,7 +199,7 @@ class DeployUserSimAgentTaskStep(DeployAgentTaskStep): "a2a_agent_version": FieldScope.TASK} def __init__(self, id: str, version: Optional[int], *, persona: Optional[dict] = None, model: Optional[str] = None, - multi_day: Optional[bool] = None, opening_prompt: Optional[str] = None, **deploy: Any): + multi_day: Optional[bool] = None, opening_prompt: Optional[str | list[dict]] = None, **deploy: Any): if deploy.get("system_prompt") is not None: raise ValueError(f"{self.type} '{id}': system_prompt is written from the persona and opening_prompt; don't set it") deploy.pop("system_prompt", None) @@ -203,8 +215,7 @@ def __init__(self, id: str, version: Optional[int], *, persona: Optional[dict] = if not isinstance(multi_day, bool): found.append("multi_day must be true or false: whether the run goes past the day it starts on") # Required: always_on starts the run from it, and refuses a user sim without one only after the deploy. - if not isinstance(opening_prompt, str) or not opening_prompt.strip(): - found.append("opening_prompt is required: the user's first message, which always_on starts the run with") + found += opening_problems(opening_prompt) found += persona_problems(persona) if found: raise ValueError(f"{self.type} '{self.agent_name}': " + "; ".join(found)) @@ -239,19 +250,19 @@ def _resolve_agent_ref(self, context: TaskStepContext) -> tuple[str, Optional[in def person(self, context: TaskStepContext) -> dict: """The user as the built-in user sim's ``setup`` takes them, as always_on hands them over.""" name = self.persona["display_name"] - opening = self._opening(context) + parts = self._opening(context) + text, files = opening_text(parts), attached(parts) return {"key": self.agent_name, "name": name, "first": self.persona.get("first_name") or name.split()[0], "description": self.persona.get("description"), "permissions": self.persona.get("permissions"), "keep_private": self.persona.get("keep_private"), - "opening": {"text": opening} if opening else None, "done_ends_run": not self.multi_day} + "opening": {"text": text, **({"files": files} if files else {})} if text else None, + "done_ends_run": not self.multi_day} + + def _opening(self, context: TaskStepContext) -> list[dict]: + """``opening_prompt`` as parts with the run's ```` placeholders filled in, as always_on fills them.""" + from agent_env.task_step.task_steps.prompt_agent import apply_seed - def _opening(self, context: TaskStepContext) -> Optional[str]: - """``opening_prompt`` with the run's ```` placeholders filled in, as always_on fills its prompt's.""" - text = self.opening_prompt - if text: - for key, value in (context.metadata.get("seed") or {}).items(): - text = text.replace(f"<{key}>", str(value)) - return text + return apply_seed(opening_parts(self.opening_prompt), context.metadata.get("seed") or {}) def _setup(self, context: TaskStepContext) -> dict: return user_sim_versions.setup(user_sim_versions.VERSION, self.person(context)) @@ -302,7 +313,9 @@ async def _set_user(self, context: TaskStepContext, agent, setup: dict) -> TaskS "description": persona.get("description"), "permissions": persona.get("permissions"), "keep_private": persona.get("keep_private"), "schedule": persona.get("schedule"), "response_times": persona.get("response_times"), "max_replies": persona.get("max_replies"), - "opening": self.opening_prompt, + # The opening's text, as written; an opening with files also has its parts, as written. + "opening": opening_text(opening_parts(self.opening_prompt)), + **({"opening_parts": opening_parts(self.opening_prompt)} if isinstance(self.opening_prompt, list) else {}), "user_version": user_sim_versions.VERSION, "model": model, "model_source": source, "prompt_sha256": hashlib.sha256(prompt.encode()).hexdigest(), "prompt_set": True, "multi_day": self.multi_day, diff --git a/src/agent_env/task_step/task_steps/agent/user_sim_common.py b/src/agent_env/task_step/task_steps/agent/user_sim_common.py index c6ab8a71e..4f11ed4b7 100644 --- a/src/agent_env/task_step/task_steps/agent/user_sim_common.py +++ b/src/agent_env/task_step/task_steps/agent/user_sim_common.py @@ -1,8 +1,9 @@ """What always_on and prompt_agent share when they read a user deployed by deploy_user_sim_agent.""" from __future__ import annotations +import copy from datetime import datetime, timezone -from typing import Optional +from typing import Any, Optional # Where a run's rating by its user sim (user_sim_versions.rate) is recorded: metadata[SATISFACTION_KEY][step_id]. SATISFACTION_KEY = "user_sim_satisfaction" @@ -40,3 +41,90 @@ def utc(text: str) -> datetime: def iso(value: Optional[datetime]) -> Optional[str]: return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") if value else None + + +# ---- the opening: the user's first message --------------------------------------------------------------------------- +# ``opening_prompt`` is text, or a list of A2A text and file parts (prompt_agent's ``parts``). A file part may carry a +# one-line ``metadata.description`` for the user sim, which can't open files; the agent is sent the file without it. +OPENING_KINDS = ("text", "file") +DESCRIPTION = "description" + + +def opening_parts(opening: Any) -> list[dict]: + """``opening_prompt`` as parts, a copy: text is one text part.""" + if isinstance(opening, str): + return [{"kind": "text", "text": opening}] + return copy.deepcopy(list(opening or [])) + + +def recorded_opening(us: dict) -> list[dict]: + """The opening a deploy_user_sim_agent record holds, as parts, its seed not yet filled in.""" + return opening_parts(us["opening_parts"] if us.get("opening_parts") is not None else us.get("opening") or "") + + +def opening_text(parts: list[dict]) -> str: + """The opening's text parts joined, as the user's one line of text.""" + return "\n".join(p.get("text", "") for p in parts if p.get("kind") == "text") + + +def file_name(part: dict) -> str: + """What the user calls a file part: its name, else the last segment of its address.""" + file = part.get("file") or {} + return file.get("name") or (file.get("uri") or "").rstrip("/").rsplit("/", 1)[-1] or "a file" + + +def attached(parts: list[dict]) -> list[dict]: + """The files the opening attaches, as the user sim is told of them: ``{"name", "description"}``, the description + only when the part has one.""" + out = [] + for part in parts: + if part.get("kind") == "file": + description = (part.get("metadata") or {}).get(DESCRIPTION) + out.append({"name": file_name(part), **({"description": description} if description else {})}) + return out + + +def for_agent(part: dict) -> dict: + """A part as the agent is sent it: without the description, which is for the user sim.""" + metadata = part.get("metadata") + if not isinstance(metadata, dict) or DESCRIPTION not in metadata: + return part + rest = {k: v for k, v in metadata.items() if k != DESCRIPTION} + return {**{k: v for k, v in part.items() if k != "metadata"}, **({"metadata": rest} if rest else {})} + + +def opening_problems(opening: Any) -> list[str]: + """What's wrong with ``opening_prompt``, one message each; empty when it's fine.""" + required = "opening_prompt is required: the user's first message, which always_on starts the run with" + if opening is None or isinstance(opening, str): + return [] if isinstance(opening, str) and opening.strip() else [required] + if not isinstance(opening, list): + return ["opening_prompt must be text, or a list of text and file parts"] + from a2a.types import FilePart, TextPart + from pydantic import ValidationError + + out = [] + for i, part in enumerate(opening): + where = f"opening_prompt[{i}]" + if not isinstance(part, dict): + out.append(f"{where} must be a part, an object") + continue + if part.get("kind") not in OPENING_KINDS: + out.append(f"{where} is a {part.get('kind')!r} part; the user's first message takes text and file parts") + continue + try: + (TextPart if part["kind"] == "text" else FilePart).model_validate(part) + except ValidationError as e: + why = "; ".join(f"{'.'.join(map(str, err['loc']))}: {err['msg']}" for err in e.errors()) + out.append(f"{where} is not a valid A2A {part['kind']} part ({why})") + continue + metadata = part.get("metadata") + if part["kind"] == "file" and isinstance(metadata, dict) and DESCRIPTION in metadata: + description = metadata[DESCRIPTION] + if not isinstance(description, str) or not description.strip() or "\n" in description.strip(): + out.append(f"{where}'s metadata.description must be one line of text: what the user sim is told " + "the file is") + if not opening_text([p for p in opening if isinstance(p, dict)]).strip(): + out.append("opening_prompt needs a text part with text: the user's first message, which always_on starts " + "the run with") + return out diff --git a/src/agent_env/task_step/task_steps/agent/user_sim_versions.py b/src/agent_env/task_step/task_steps/agent/user_sim_versions.py index bcc71cde2..f893c7538 100644 --- a/src/agent_env/task_step/task_steps/agent/user_sim_versions.py +++ b/src/agent_env/task_step/task_steps/agent/user_sim_versions.py @@ -7,7 +7,9 @@ answer format. Its done line follows ``person["done_ends_run"]``, the multi-day wording when that's absent. ``answer`` renders one read as one labelled message, asks the user sim, and parses its answer; a malformed answer gets one retry in the same session. What earlier reads showed, and when the user last wrote, come from ``history``; -keys of ``read`` it doesn't know are ignored. +keys of ``read`` it doesn't know are ignored. A line the user sent with files (the opening's ``files``, +``{"name", "description"}``) shows each after its text, "(you attached itinerary.pdf: Mom and Dad's flight +itinerary)", or by name alone with no description: the user sim can't open them. ``rate`` asks the user sim, once the run is over, to rate the outcome 1-5 by its description's ``satisfaction`` lines, with a one-line reason; None when the description has none, so a user without them is never asked. ``problems`` is the load check for the settings.""" @@ -50,6 +52,9 @@ BROKEN_RULE_LINE = ('If it does an "ask first" or a "never" without asking and tells you, say once, briefly, that ' "you're not happy about it.") OPENING_LINE = 'Earlier you texted the assistant:\n"{text}"' +# A file the user sent with a line, after its text: the user sim can't open it, so it's told what it attached. +ATTACHED = "(you attached {name}: {description})" +ATTACHED_NAME_ONLY = "(you attached {name})" OUTCOMES = ("reply", "no_reply", "done") ANSWER_LINE = '{"outcome": "reply" | "no_reply" | "done", "message": "...", "used": "..."}' @@ -153,12 +158,19 @@ def rule_lines(permissions: dict) -> list[str]: return lines +def attached_note(files: Optional[list[dict]]) -> str: + """What a line's ``files`` (``{"name", "description"}``) add after its text: nothing, when it has none.""" + return "".join(" " + (ATTACHED.format(**f) if f.get("description") else ATTACHED_NAME_ONLY.format(name=f["name"])) + for f in files or []) + + def _who(person: dict) -> list[str]: """Who they are, with the opening request right after the first line.""" lines = list(person["description"].get("who_you_are") or []) - opening = (person.get("opening") or {}).get("text") - if opening: - lines.insert(min(1, len(lines)), OPENING_LINE.format(text=opening)) + opening = person.get("opening") or {} + if opening.get("text"): + line = OPENING_LINE.format(text=opening["text"]) + attached_note(opening.get("files")) + lines.insert(min(1, len(lines)), line) return lines @@ -319,7 +331,7 @@ def dated(r: dict, items: list) -> list[dict]: # an earlier read's items, each out.append(EARLIER_CUT) if own: out.append("\nYOU SENT YOUR AI ASSISTANT:") - out += [f" [{_stamp(at(s), now)}] {s['text']}" for s in own] + out += [f" [{_stamp(at(s), now)}] {s['text']}{attached_note(s.get('files'))}" for s in own] out.append("\nNEW FROM YOUR AI ASSISTANT (this is what you're answering):") out += [_message(m, _stamp(at(m), now)) for m in messages] out.append("\n" + CLOSING.format(first=_first(person))) diff --git a/src/agent_env/task_step/task_steps/always_on.py b/src/agent_env/task_step/task_steps/always_on.py index b5db85b58..c49620de1 100644 --- a/src/agent_env/task_step/task_steps/always_on.py +++ b/src/agent_env/task_step/task_steps/always_on.py @@ -5,9 +5,10 @@ deploys the agent that plays the user, with its system prompt and model set, and records the user in the run's metadata under its ``agent_name``. The ``always_on`` step names it with ``user_sim`` and runs the loop: -- The first wake is the user's request. Each wake opens with what came due for the agent on notifications/v1 (a - trigger's ``notify``, a scheduled check), labelled ``[time · channel]``, then the user's own messages, each labelled - with its time; while the task's clock is set, the agent can read it with the gateway's ``get_time``. +- The first wake is the user's request, with the files its opening attaches in their place. Each wake opens with + what came due for the agent on notifications/v1 (a trigger's ``notify``, a scheduled check), labelled + ``[time · channel]``, then the user's own messages, each labelled with its time; while the task's clock is set, + the agent can read it with the gateway's ``get_time``. - Only what the agent sends with the gateway's ``message_user`` tool (user-chat/v1) reaches the user; its plain reply goes to the log. Each wake should end with ``message_user`` or ``stay_quiet``: one that ends with closing text and neither gets one ``nudge`` turn. ``stay_quiet`` and the nudge are on unless the task turns them off; the rule itself is @@ -121,6 +122,7 @@ from agent_env.task_step.task_steps.notification_delivery import delivered_parts, label_time, take_due from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep from agent_env.task_step.task_steps.agent.user_sim_common import SATISFACTION_KEY, person_of, task_zone +from agent_env.task_step.task_steps.agent.user_sim_common import attached, for_agent, recorded_opening from agent_env.task_step.task_steps.agent.user_sim_common import iso as _iso, utc as _utc logger = logging.getLogger(__name__) @@ -512,7 +514,8 @@ async def go(self) -> None: raise ValueError(f"always_on: the user sim '{step.user_sim}' has no opening_prompt, the user's first " "message; give the deploy_user_sim_agent step one") now = await self.now() - self.opening = {"at": _iso(now), "text": self._effective_prompt()} + files = attached(self._prompt_parts()) + self.opening = {"at": _iso(now), "text": self._effective_prompt(), **({"files": files} if files else {})} self.person["opening"] = self.opening t0 = await self._t0() or now self.end = _end_instant(step.end_time, t0) if step.end_time else None @@ -1190,19 +1193,19 @@ async def ask(text: str) -> dict: def _prompt_parts(self) -> list[dict]: """The user's first message, the user sim's opening, as A2A parts, its seed applied.""" - return self.step._apply_seed([{"kind": "text", "text": self.us["opening"]}], self.ctx.metadata.get("seed", {})) + return self.step._apply_seed(recorded_opening(self.us), self.ctx.metadata.get("seed", {})) def _effective_prompt(self) -> str: return "\n".join(p.get("text", "") for p in self._prompt_parts() if p.get("kind") == "text") def _opening_parts(self, labelled) -> list[dict]: """The prompt as the opening wake and the user's chat carry it: its text as one labelled line from the user - (``labelled(text)``), where its first text part was, and every other part (a file, data) as it is, in the - prompt's order, as prompt_agent sends them.""" + (``labelled(text)``), where its first text part was, and every file part as it is, in the prompt's order, as + prompt_agent sends them, without the description that is for the user sim.""" out, text = [], self._effective_prompt() for part in self._prompt_parts(): if part.get("kind") != "text": - out.append(part) + out.append(for_agent(part)) elif text is not None: out.append(labelled(text)) text = None diff --git a/src/agent_env/task_step/task_steps/prompt_agent.py b/src/agent_env/task_step/task_steps/prompt_agent.py index 84091d0c6..d54cad294 100644 --- a/src/agent_env/task_step/task_steps/prompt_agent.py +++ b/src/agent_env/task_step/task_steps/prompt_agent.py @@ -43,6 +43,7 @@ from agent_env.task_step.task_steps.post_turn_hooks import PostTurnHooks, run_post_turn_hooks, validate_hook_specs from agent_env.task_step.task_steps.agent import user_sim_versions from agent_env.task_step.task_steps.agent.user_sim_common import SATISFACTION_KEY, iso, person_of, task_zone, utc +from agent_env.task_step.task_steps.agent.user_sim_common import attached, opening_text, recorded_opening from agent_env.task_step.task_steps.sandbox_utils.sandbox_utils import ( agent_error_text, fetch_container_logs, @@ -109,6 +110,28 @@ def _duplicates_prompt_text(parts: list[dict], prompt_text: Optional[str]) -> bo return parts == [{"kind": "text", "text": prompt_text}] +def apply_seed(parts: list[dict], seed: dict) -> list[dict]: + """Return a deep-copied parts list with seed substitutions applied. + `` placeholders are replaced inside text parts' `text` and inside + file parts' `uri`/`name` (so tasks-as-templates with an object URL + ending `seeds//x.png` resolve per-run).""" + if not seed: + return parts + out = copy.deepcopy(parts) + for p in out: + kind = p.get("kind") + if kind == "text": + for k, v in seed.items(): + p["text"] = p["text"].replace(f"<{k}>", str(v)) + elif kind == "file": + f = p.get("file") or {} + for field in ("uri", "name"): + if field in f and isinstance(f[field], str): + for k, v in seed.items(): + f[field] = f[field].replace(f"<{k}>", str(v)) + return out + + def _file_uris(parts: list[dict]) -> list[str]: files = (part.get("file") for part in parts if part.get("kind") == "file") return [file["uri"] for file in files if isinstance(file, dict) and isinstance(file.get("uri"), str)] @@ -251,25 +274,7 @@ def _effective_parts(self) -> list[dict]: return [{"kind": "text", "text": self.prompt}] def _apply_seed(self, parts: list[dict], seed: dict) -> list[dict]: - """Return a deep-copied parts list with seed substitutions applied. - `` placeholders are replaced inside text parts' `text` and inside - file parts' `uri`/`name` (so tasks-as-templates with an object URL - ending `seeds//x.png` resolve per-run).""" - if not seed: - return parts - out = copy.deepcopy(parts) - for p in out: - kind = p.get("kind") - if kind == "text": - for k, v in seed.items(): - p["text"] = p["text"].replace(f"<{k}>", str(v)) - elif kind == "file": - f = p.get("file") or {} - for field in ("uri", "name"): - if field in f and isinstance(f[field], str): - for k, v in seed.items(): - f[field] = f[field].replace(f"<{k}>", str(v)) - return out + return apply_seed(parts, seed) def to_dict(self) -> dict: base = super().to_dict() @@ -428,12 +433,18 @@ def _built_in_user_sim(self, context: TaskStepContext) -> Optional[tuple[dict, s "each reply at once); drop them from that step's persona") seed = context.metadata.get("seed", {}) parts = self._apply_seed(self._effective_parts(), seed) - deployed_with = self._apply_seed([{"kind": "text", "text": record.get("opening") or ""}], seed)[0]["text"] + deployed_with = self._apply_seed(recorded_opening(record), seed) # TODO(FD-5495): the first message is written twice, here and as the user sim's opening_prompt; write it once. - if (_joined_text(parts) or "").strip() != deployed_with.strip(): + if (_joined_text(parts) or "").strip() != opening_text(deployed_with).strip(): # Its prompt quotes the opening it was deployed with as what the user sent; another one would contradict it. raise ValueError(f"prompt_agent '{self.id}': its prompt isn't the opening_prompt its user " f"'{self.user_agent_name}' was deployed with; give both the same text") + mine, theirs = _files(parts), _files(deployed_with) + if mine != theirs: + # The user's prompt says what they attached; the agent must be sent those files, in that order. + raise ValueError(f"prompt_agent '{self.id}': its prompt's files ({_described(mine)}) aren't the files of " + f"the opening_prompt its user '{self.user_agent_name}' was deployed with " + f"({_described(theirs)}); give both the same files in the same order") clocks = context.metadata.get("clock_configurations") or [] return record, task_zone(record, clocks, f"prompt_agent '{self.id}'") @@ -1266,7 +1277,11 @@ async def start(cls, step: PromptAgentTaskStep, context: TaskStepContext, record initial_parts: list[dict], clock_envs: list) -> _BuiltInUserSim: """The user ``record`` describes, already checked by ``_built_in_user_sim``, in time zone ``zone``.""" text = _joined_text(initial_parts) - opening = {"at": await _clock_now(step, clock_envs), "text": text} if text else None + # What the user attached, as their prompt says: the opening's files, with their descriptions, which + # _built_in_user_sim checked are the ones this step sends. + files = attached(step._apply_seed(recorded_opening(record), context.metadata.get("seed", {}))) + opening = ({"at": await _clock_now(step, clock_envs), "text": text, **({"files": files} if files else {})} + if text else None) person = {**person_of(record), "max_replies": record.get("max_replies"), "time_zone": zone, "opening": opening, "done_ends_run": True} return cls(step, context, person, opening, clock_envs) @@ -1391,6 +1406,19 @@ def _agent_text(solver_message: str, agent_parts: Optional[list[dict]]) -> str: return _joined_text(agent_parts or []) or solver_message +def _files(parts: list[dict]) -> list[dict]: + """The file parts' files, in order, as the match check on the user's opening compares them.""" + return [p.get("file") or {} for p in parts if p.get("kind") == "file"] + + +def _described(files: list[dict]) -> str: + """Files named for an error: each one's name, address and type, or its size when it is sent inline.""" + def one(f: dict) -> str: + where = f.get("uri") or (f"{len(f['bytes'])} bytes inline" if isinstance(f.get("bytes"), str) else "?") + return " ".join(x for x in (f.get("name"), f"<{where}>", f.get("mimeType") or f.get("mime_type")) if x) + return "; ".join(map(one, files)) or "none" + + def _joined_text(parts: list[dict]) -> Optional[str]: """The prompt's text parts joined, as always_on reads them; None when there are none.""" return "\n".join(p.get("text", "") for p in parts if p.get("kind") == "text") or None diff --git a/src/agent_env/training/harness.py b/src/agent_env/training/harness.py index 00ca38851..7f09217ac 100644 --- a/src/agent_env/training/harness.py +++ b/src/agent_env/training/harness.py @@ -35,6 +35,7 @@ from agent_env.task.task import Task, recorded_scores from agent_env.task.teardown import teardown_run from agent_env.task_step.context import TaskStepContext, sandbox_overrides +from agent_env.task_step.task_steps.agent.user_sim_common import opening_parts, opening_text from agent_env.task_step.task_steps.deploy_agent import DeployAgentTaskStep from agent_env.task_step.task_steps.prompt_agent import is_prompt_step, user_sim_agents from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep @@ -233,8 +234,9 @@ def _index_named(prompt: Any, ids: list[str]) -> int: def _first_prompt(task: Task) -> str: """The user's first message to the agent under test: its step's prompt, else the opening of the user sim the step - names (an always_on step's ``user_sim``, a prompt_agent step's ``user_agent_name``), as it was written. A step that - prompts an agent another step talks to as its user (the user sim, say a briefing) isn't the agent under test's.""" + names (an always_on step's ``user_sim``, a prompt_agent step's ``user_agent_name``), as it was written (its + text, for an opening with files). A step that prompts an agent another step talks to as its user (the user sim, + say a briefing) isn't the agent under test's.""" users = user_sim_agents(task.steps) step = next((each for each in task.steps if is_prompt_step(each) and getattr(each, "agent_name", None) not in users), None) @@ -245,7 +247,8 @@ def _first_prompt(task: Task) -> str: user = getattr(step, "user_sim", None) or getattr(step, "user_agent_name", None) sim = next((each for each in task.steps if user and getattr(each, "agent_name", None) == user and getattr(each, "opening_prompt", None)), None) - return getattr(sim, "opening_prompt", None) or "" + opening = getattr(sim, "opening_prompt", None) or "" + return opening if isinstance(opening, str) else opening_text(opening_parts(opening)) def _free_port() -> int: diff --git a/tst/unit/task_step/test_user_sim_opening_files.py b/tst/unit/task_step/test_user_sim_opening_files.py new file mode 100644 index 000000000..5ae29b29d --- /dev/null +++ b/tst/unit/task_step/test_user_sim_opening_files.py @@ -0,0 +1,294 @@ +"""The user's first message with files (FD-5494), on the parents' visit task with an itinerary PDF: deploy_user_sim_agent +takes ``opening_prompt`` as text and file parts and tells the user sim what it attached; always_on sends the files on +the first wake in their place and the graded conversation has them; prompt_agent's match check compares the files; +the seed fills a file's address; a text-only opening is unchanged.""" + +from __future__ import annotations + +import copy + +import pytest + +from agent_env.a2a_agent.object_transfer import readable_parts +from agent_env.config import configure +from agent_env.task_step.context import TaskStepContext +from agent_env.task_step.task_steps import always_on as ao +from agent_env.task_step.task_steps.agent import user_sim_versions +from agent_env.task_step.task_steps.agent.deploy_user_sim_agent import USER_SIMS_KEY, DeployUserSimAgentTaskStep +from agent_env.task_step.task_steps.agent.user_sim_common import attached, file_name, for_agent +from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep +from tst.unit.task_step.always_on_harness import Rig +from tst.unit.task_step.test_deploy_user_sim_agent import HANNAH, PERSON, PROMPT, _Deploy, _published, _step +from tst.unit.task_step.test_prompt_agent_built_in_user_sim import USER, _context, _full, _reads, said +from tst.unit.task_step.test_prompt_agent_user_side import AGENT, Peers, exchange, text +from tst.util.granting_object_store import GRANT_ORIGIN, GrantingObjectStore + +TEXT = PERSON["opening"]["text"] +SEED = {"trip": "nov-19"} +ITINERARY = {"kind": "file", "file": {"uri": "s3://reyes-family//itinerary.pdf", "name": "itinerary.pdf", + "mimeType": "application/pdf"}, + "metadata": {"description": "Mom and Dad's flight itinerary"}} +SHORTLIST = {"kind": "file", "file": {"uri": "s3://reyes-family//hotel-shortlist.xlsx", + "name": "hotel-shortlist.xlsx"}} +OPENING = [{"kind": "text", "text": TEXT}, ITINERARY, SHORTLIST] +NOTE = "(you attached itinerary.pdf: Mom and Dad's flight itinerary) (you attached hotel-shortlist.xlsx)" + + +def _sent(part: dict) -> dict: + """A file part as the agent is sent it this run: the seed filled in, the description dropped.""" + file = {**part["file"], "uri": part["file"]["uri"].replace("", SEED["trip"])} + return {"kind": "file", "file": file} + + +def _seeded() -> TaskStepContext: + context = TaskStepContext() + context.metadata["seed"] = dict(SEED) + return context + + +# ---- deploy_user_sim_agent ------------------------------------------------------------------------------------------ +@pytest.mark.asyncio +async def test_a_text_only_opening_is_recorded_and_prompted_as_before(monkeypatch): + record = await _published(monkeypatch, _step()) + assert record["opening"] == TEXT and "opening_parts" not in record + deploy = _Deploy(monkeypatch) + await deploy.run(_step(opening_prompt=[{"kind": "text", "text": TEXT}])) + assert deploy.fields("system_prompt") == [PROMPT] + + +@pytest.mark.asyncio +async def test_the_prompt_says_what_the_user_attached_by_description_else_by_name(monkeypatch): + deploy = _Deploy(monkeypatch) + context = await deploy.run(_step(opening_prompt=OPENING), _seeded()) + + assert PROMPT.count(f'"{TEXT}"\n') == 1 + assert deploy.fields("system_prompt") == [PROMPT.replace(f'"{TEXT}"\n', f'"{TEXT}" {NOTE}\n')] + record = context.metadata[USER_SIMS_KEY]["hannah"] + # The text as before, so text readers keep working; the parts as written, the seed still to fill in. + assert record["opening"] == TEXT + assert record["opening_parts"] == OPENING + + +@pytest.mark.asyncio +async def test_the_seed_fills_a_file_s_name_in_what_the_user_sim_is_told(monkeypatch): + deploy = _Deploy(monkeypatch) + named = {**ITINERARY, "file": {**ITINERARY["file"], "name": "-itinerary.pdf"}} + await deploy.run(_step(opening_prompt=[OPENING[0], named]), _seeded()) + [prompt] = deploy.fields("system_prompt") + assert f'"{TEXT}" (you attached nov-19-itinerary.pdf: Mom and Dad\'s flight itinerary)\n' in prompt + + +def test_a_step_with_an_opening_of_parts_round_trips(): + step = _step(opening_prompt=OPENING) + assert DeployUserSimAgentTaskStep.from_dict(step.to_dict()).to_dict() == step.to_dict() + assert step.to_dict()["opening_prompt"] == OPENING + + +@pytest.mark.parametrize("opening, problem", [ + ([ITINERARY], "opening_prompt needs a text part with text"), + ([{"kind": "text", "text": " "}, ITINERARY], "opening_prompt needs a text part with text"), + ([], "opening_prompt needs a text part with text"), + ({"text": TEXT}, "opening_prompt must be text, or a list of text and file parts"), + ([{"kind": "text", "text": TEXT}, "itinerary.pdf"], "opening_prompt[1] must be a part, an object"), + ([{"kind": "text", "text": TEXT}, {"kind": "data", "data": {"flight": "UA 1534"}}], + "opening_prompt[1] is a 'data' part; the user's first message takes text and file parts"), + ([{"kind": "text", "text": TEXT}, {"kind": "file", "file": {"name": "itinerary.pdf"}}], + "opening_prompt[1] is not a valid A2A file part (file.FileWithBytes.bytes: Field required; " + "file.FileWithUri.uri: Field required)"), + ([{"kind": "text"}], "opening_prompt[0] is not a valid A2A text part (text: Field required)"), + ([{"kind": "text", "text": TEXT}, {**ITINERARY, "metadata": {"description": "Mom and Dad's\nflights"}}], + "opening_prompt[1]'s metadata.description must be one line of text"), + ([{"kind": "text", "text": TEXT}, {**ITINERARY, "metadata": {"description": " "}}], + "opening_prompt[1]'s metadata.description must be one line of text"), + ([{"kind": "text", "text": TEXT}, {**ITINERARY, "metadata": {"description": ["flights"]}}], + "opening_prompt[1]'s metadata.description must be one line of text"), +], ids=["no-text", "blank-text", "empty", "not-a-list", "not-a-part", "data-part", "file-without-address", + "text-without-text", "two-line-description", "blank-description", "description-not-text"]) +def test_a_bad_opening_is_refused_when_the_task_loads(opening, problem): + with pytest.raises(ValueError, match="deploy_user_sim_agent 'hannah'") as e: + _step(opening_prompt=opening) + assert problem in str(e.value) + + +def test_a_file_is_named_by_its_name_else_its_address_and_only_its_description_is_kept_from_the_agent(): + assert file_name({"kind": "file", "file": {"uri": "s3://reyes-family/scans/boarding-pass.png"}}) \ + == "boarding-pass.png" + assert file_name({"kind": "file", "file": {"bytes": "aGk="}}) == "a file" + assert attached(OPENING) == [{"name": "itinerary.pdf", "description": "Mom and Dad's flight itinerary"}, + {"name": "hotel-shortlist.xlsx"}] + tagged = {**ITINERARY, "metadata": {**ITINERARY["metadata"], "source": "intake"}} + assert for_agent(tagged) == {**ITINERARY, "metadata": {"source": "intake"}} + assert for_agent(ITINERARY) == {"kind": "file", "file": ITINERARY["file"]} + assert for_agent(SHORTLIST) is SHORTLIST + + +def test_a_read_shows_what_the_user_attached_after_their_line(): + person = {**PERSON, "opening": None} + sent = {"at": "2026-11-16T15:50:00Z", "text": TEXT, + "files": [{"name": "itinerary.pdf", "description": "Mom and Dad's flight itinerary"}, + {"name": "hotel-shortlist.xlsx"}]} + read = {"time": "2026-11-16T16:05:00Z", "assistant_messages": [{"at": "2026-11-16T16:05:00Z", "text": "On it."}], + "phone": [], "you_sent": [sent], "done_ends_run": False} + without = {**read, "you_sent": [{k: v for k, v in sent.items() if k != "files"}]} + assert user_sim_versions.render(person, [], read) == user_sim_versions.render(person, [], without).replace( + f" [7:50 AM] {TEXT}\n", f" [7:50 AM] {TEXT} {NOTE}\n") + + +# ---- always_on ---------------------------------------------------------------------------------------------------- +async def _always_on_context(monkeypatch, rig: Rig, opening) -> TaskStepContext: + """The Rig's world with Hannah as deploy_user_sim_agent records her, her first message ``opening``.""" + record = await _published(monkeypatch, _step(agent_name="user-sim", opening_prompt=opening, multi_day=False, + persona={**HANNAH, "time_zone": "America/Los_Angeles"}), _seeded()) + ctx = rig.context() + ctx.metadata[USER_SIMS_KEY] = {"user-sim": record} + ctx.metadata["seed"] = dict(SEED) + ctx.metadata["clock_configurations"] = [{"step_id": "clock", "timezone": "America/Los_Angeles"}] + return ctx + + +def _always_on() -> ao.AlwaysOnTaskStep: + return ao.AlwaysOnTaskStep("week", None, user_sim="user-sim", prompt_id="ask", agent_name="solver") + + +def _asks_once(n, texts, rig): + if n == 1: + rig.message("Booking their flights now. Any seat preference?") + return "sent" + rig.quiet() + return "" + + +@pytest.mark.usefixtures("local_stores") +@pytest.mark.asyncio +async def test_always_on_sends_the_files_on_the_first_wake_in_their_place_and_the_chat_and_the_user_sim_have_them( + monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_asks_once, plugin=user_sim_versions, t0="2026-11-16T15:50:00Z", + user_texts=['{"outcome": "done", "message": "aisle for dad, thanks!", "used": "none"}']) + await rig.arm() + later = {"kind": "text", "text": "the hotels i looked at are in the sheet"} + ctx = await rig.run(_always_on(), await _always_on_context(monkeypatch, rig, [*OPENING[:2], later, SHORTLIST])) + + line = f"[Mon Nov 16, 3:50 PM UTC] Hannah: {TEXT}\n{later['text']}" + opening = rig.parts[0] + assert [p.get("text") for p in opening if p.get("kind") == "text"].count(line) == 1 + at = next(i for i, p in enumerate(opening) if p.get("text") == line) + assert opening[at + 1:] == [_sent(ITINERARY), _sent(SHORTLIST)] # in place, the seed filled in, no description + [first_turn, *_] = [m for m in rig.chat(ctx) if m["role"] == "user"] + assert [p.get("kind") for p in first_turn["parts"]] == ["text", "file", "file"] + assert first_turn["parts"][1:] == [_sent(ITINERARY), _sent(SHORTLIST)] + [call] = rig.user_calls + assert (f"\nYOU SENT YOUR AI ASSISTANT:\n [7:50 AM] {TEXT}\n{later['text']} {NOTE}\n" + in call["parts"][0]["text"]) + + +@pytest.mark.usefixtures("local_stores") +@pytest.mark.asyncio +async def test_always_on_with_a_text_only_opening_sends_and_shows_only_the_text(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_asks_once, plugin=user_sim_versions, t0="2026-11-16T15:50:00Z", + user_texts=['{"outcome": "done", "message": "aisle for dad, thanks!", "used": "none"}']) + await rig.arm() + ctx = await rig.run(_always_on(), await _always_on_context(monkeypatch, rig, TEXT)) + + assert {p.get("kind") for p in rig.parts[0]} == {"text"} + [first_turn, *_] = [m for m in rig.chat(ctx) if m["role"] == "user"] + assert first_turn["texts"] == [f"[Mon Nov 16, 3:50 PM UTC] Hannah: {TEXT}"] + [call] = rig.user_calls + assert f"\nYOU SENT YOUR AI ASSISTANT:\n [7:50 AM] {TEXT}\n\nNEW FROM" in call["parts"][0]["text"] + + +@pytest.mark.usefixtures("local_stores") +@pytest.mark.asyncio +async def test_an_opening_file_the_store_owns_reaches_the_agent_as_a_grant_like_any_prompt_file(monkeypatch, tmp_path): + store = GrantingObjectStore(str(tmp_path / "objects"), reaches=True) + configure(object_store=store) + url = store.put("trips/nov-19/itinerary.pdf", b"%PDF") + rig = Rig(monkeypatch, tmp_path, agent=_asks_once, plugin=user_sim_versions, t0="2026-11-16T15:50:00Z", + user_texts=['{"outcome": "done", "message": "aisle for dad, thanks!", "used": "none"}']) + scripted = ao.send_and_wait + + async def send_and_wait(a2a_url, parts, *, agent=None, timeout_seconds=600, **kw): + # The real send_and_wait's sharing, in front of the Rig's scripted agent. + if agent is None: + return await scripted(a2a_url, parts, timeout_seconds=timeout_seconds, **kw) + async with readable_parts(parts, a2a_url=a2a_url, card=agent.a2a_card, sandbox_type=agent.sandbox_type, + expires_in=timeout_seconds) as sent: + return await scripted(a2a_url, sent, agent=agent, timeout_seconds=timeout_seconds, **kw) + + monkeypatch.setattr(ao, "send_and_wait", send_and_wait) + await rig.arm() + owned = {**ITINERARY, "file": {**ITINERARY["file"], "uri": store.object_url("trips//itinerary.pdf")}} + ctx = await rig.run(_always_on(), await _always_on_context(monkeypatch, rig, [OPENING[0], owned])) + + assert rig.parts[0][-1] == {"kind": "file", "file": {**ITINERARY["file"], + "uri": f"{GRANT_ORIGIN}/trips/nov-19/itinerary.pdf?sig=read"}} + [first_turn, *_] = [m for m in rig.chat(ctx) if m["role"] == "user"] + assert first_turn["parts"][-1]["file"]["uri"] == url # the conversation keeps the object's own address + + +# ---- prompt_agent with the built-in user sim ---------------------------------------------------------------------- +def _prompt_agent(parts: list[dict]) -> PromptAgentTaskStep: + return PromptAgentTaskStep(id="talk", version=None, agent_name="solver", user_agent_name="hannah", parts=parts, + poll_interval_seconds=0, max_conversation_turns=3) + + +async def _hannah(monkeypatch, opening) -> TaskStepContext: + ctx = await _context(monkeypatch, opening_prompt=opening) + ctx.metadata["seed"] = dict(SEED) + return ctx + + +@pytest.mark.usefixtures("local_stores") +@pytest.mark.asyncio +async def test_prompt_agent_sends_the_opening_s_files_and_the_user_sim_knows_what_it_attached(monkeypatch): + peers = Peers(monkeypatch, {AGENT: [text("Booking their flights now.")], USER: [said("done")]}) + ctx = await _hannah(monkeypatch, OPENING) + # The same files, without the description, which is the user sim's. + sent = [{"kind": "text", "text": TEXT}, for_agent(ITINERARY), SHORTLIST] + + async def armed_clock(self, envs): # one armed clock, standing still at 7:50 AM in San Francisco + return {"env_clock": [{"env_id": "env", "env_step_id": "clock", "virtual_time": "2026-11-16T15:50:00+00:00"}]} + + monkeypatch.setattr(PromptAgentTaskStep, "_env_clock", armed_clock) + await _prompt_agent(sent).execute(ctx) + + first = _full(_reads(ctx)[0]) + assert peers.requests(ctx)[:2] == exchange( + AGENT, [{"kind": "text", "text": TEXT}, _sent(ITINERARY), _sent(SHORTLIST)], "", "", "task-1") + assert first["read"]["you_sent"][0]["files"] == [ + {"name": "itinerary.pdf", "description": "Mom and Dad's flight itinerary"}, {"name": "hotel-shortlist.xlsx"}] + assert f"\nYOU SENT YOUR AI ASSISTANT:\n [3:50 PM] {TEXT} {NOTE}\n" in first["input"] # Hannah's zone: UTC + + +@pytest.mark.usefixtures("local_stores") +@pytest.mark.asyncio +@pytest.mark.parametrize("files, mine", [ + ([SHORTLIST, ITINERARY], "hotel-shortlist.xlsx ; itinerary.pdf " + " application/pdf"), + ([ITINERARY], "itinerary.pdf application/pdf"), + ([{**ITINERARY, "file": {**ITINERARY["file"], "uri": "s3://reyes-family//itinerary-v2.pdf"}}, SHORTLIST], + "itinerary.pdf application/pdf; hotel-shortlist.xlsx " + ""), + ([], "none"), +], ids=["order", "missing", "other-address", "text-only"]) +async def test_prompt_agent_refuses_a_prompt_whose_files_aren_t_the_opening_s(monkeypatch, files, mine): + peers = Peers(monkeypatch, {}) + ctx = await _hannah(monkeypatch, OPENING) + with pytest.raises(ValueError) as e: + await _prompt_agent([{"kind": "text", "text": TEXT}, *copy.deepcopy(files)]).execute(ctx) + assert str(e.value) == ( + f"prompt_agent 'talk': its prompt's files ({mine}) aren't the files of the opening_prompt its user 'hannah' " + "was deployed with (itinerary.pdf application/pdf; " + "hotel-shortlist.xlsx ); give both the same files in the same " + "order") + assert peers.raw == [] + + +@pytest.mark.usefixtures("local_stores") +@pytest.mark.asyncio +async def test_prompt_agent_refuses_files_the_text_only_opening_never_had(monkeypatch): + Peers(monkeypatch, {}) + ctx = await _hannah(monkeypatch, TEXT) + with pytest.raises(ValueError, match=r"its prompt's files \(itinerary.pdf " + r"application/pdf\) aren't the files of the opening_prompt its user 'hannah' " + r"was deployed with \(none\)"): + await _prompt_agent([{"kind": "text", "text": TEXT}, ITINERARY]).execute(ctx) diff --git a/tst/unit/training/harness_test.py b/tst/unit/training/harness_test.py index d0999762e..99487146e 100644 --- a/tst/unit/training/harness_test.py +++ b/tst/unit/training/harness_test.py @@ -272,6 +272,16 @@ def user_sim(name, opening): assert harness._first_prompt(SimpleNamespace(steps=[user_sim("nora", "Find me a sofa."), unnamed])) == "" +def test_a_rows_prompt_from_an_opening_with_files_is_its_text(): + opening = [{"kind": "text", "text": "Find me a sofa."}, + {"kind": "file", "file": {"uri": "s3://nora/room.jpg", "name": "room.jpg"}}, + {"kind": "text", "text": "Something that fits this corner."}] + sim = SimpleNamespace(type="deploy_user_sim_agent", agent_name="nora", opening_prompt=opening) + always_on = SimpleNamespace(type="always_on", prompt=None, user_sim="nora") + assert harness._first_prompt(SimpleNamespace(steps=[sim, always_on])) == \ + "Find me a sofa.\nSomething that fits this corner." + + def test_a_rows_prompt_skips_a_briefing_sent_to_the_user_sim(): sim = SimpleNamespace(type="deploy_user_sim_agent", agent_name="nora", opening_prompt="Find a sofa.") briefing = SimpleNamespace(type="prompt_agent", agent_name="nora", prompt="You are Nora; stay in character.") From 2abc53c6c66068eab46e035716165cdaea128d3b Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Thu, 12 Nov 2026 18:01:15 -0800 Subject: [PATCH 08/45] docs(user sim): a first message with files (FD-5494) Co-Authored-By: Claude Opus 5.5 --- .../task_step/task_steps/agent/user_sim.md | 31 +++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/src/agent_env/task_step/task_steps/agent/user_sim.md b/src/agent_env/task_step/task_steps/agent/user_sim.md index 6edd0ab5f..243a39676 100644 --- a/src/agent_env/task_step/task_steps/agent/user_sim.md +++ b/src/agent_env/task_step/task_steps/agent/user_sim.md @@ -31,12 +31,37 @@ Its own fields: | `agent_name` | required | The agent's name, and the key always-on's `user_sim` names | | `persona` | required | Who the user is: `display_name` (their full name), `first_name` (else the display name's first word), `time_zone` (their IANA zone; else the task clock's, else UTC; a clock that sets a different zone is refused), `description`, `permissions`, `keep_private`, `schedule`, `response_times`, `max_replies` (after that many replies the user stops reading) | | `model` | required | The user sim's model. This step's `step_params` `model` wins, then the run's `user_model` (`--user-model`), then this | -| `opening_prompt` | required | The user's first message. The always-on step sends it and has no prompt of its own | +| `opening_prompt` | required | The user's first message: text, or text and file parts (below). The always-on step sends it and has no prompt of its own | | `multi_day` | required, for now | Whether the run goes past the day it starts on (always-on's `end_time` falls on a later day). It picks the prompt's done line; always-on refuses a run of the other kind | `multi_day` is a stopgap: the prompt is written before the run, so the step has to be told which done line to use. It goes away once the step can learn it from the task. +**A first message with files.** `opening_prompt` can be a list of parts in `prompt_agent`'s `parts` format, text and +file parts, for a first message that attaches a photo, a PDF, a spreadsheet or any other file: + +```json +"opening_prompt": [ + {"kind": "text", "text": "ok mom and dad are officially coming thu–mon (nov 19–23), can you handle it?"}, + {"kind": "file", "file": {"uri": "s3:////itinerary.pdf", "name": "itinerary.pdf", "mimeType": "application/pdf"}, + "metadata": {"description": "Mom and Dad's flight itinerary"}} +] +``` + +The agent gets the parts on its first wake, in order: the text, joined, as the user's one labelled line where the +first text part was, and each file in its place. The files are in the user's first message of the graded +conversation too. A file goes through the same sharing as any prompt's file part (an object a configured store owns is +sent as a URL the agent can read), and the seed fills ``s in its `uri` and `name`. Whether the agent can +open a file is up to its harness. + +The user sim can't open files, so its prompt and its first read say what it attached, after its message: `"ok mom +and dad…" (you attached itinerary.pdf: Mom and Dad's flight itinerary)`. `metadata.description` is that one line; the +agent is sent the file without it. With no description, the user sim gets the file's name only (its `name`, else the +last part of its `uri`). The step refuses, when the task loads, an opening with no text, a part that isn't a valid +text or file part, or a description that isn't one line. The run records the opening's text in +`metadata["user_sims"][]["opening"]`, as for a text-only opening, and the parts, as written, in +`opening_parts`. + `system_prompt` is written from the persona and `opening_prompt`, so the step refuses one set directly. Each read shows times in the persona's `time_zone`. A run-wide `a2a_agent_id` never changes the user sim's agent, so the agent under test never plays the user; this step's own `step_params` `a2a_agent_id` can. An agent @@ -44,7 +69,9 @@ whose card takes no system prompt or no model fails the step, since without them **Used with prompt_agent.** A `prompt_agent` step whose `user_agent_name` is this step's `agent_name` (and sets no `user_a2a_url`) talks to the same user in one sitting: each reply of the agent is one read, with no phone. Its prompt -must be this step's `opening_prompt`, which the user's prompt quotes as what they sent (FD-5495 is to write it once). +must be this step's `opening_prompt`, which the user's prompt quotes as what they sent (FD-5495 is to write it once): +the same text, and the same files in the same order, once the seed is filled in. The check compares each file part's +`file` (`uri`, `name`, `mimeType`), not its `metadata`, so the description can stay on this step alone. `reply` is the agent's next prompt; `done` with a message reaches the agent and ends the conversation after its answer; `done` without one, or `no_reply`, ends it at once. After `max_replies` replies, the last one reaches the agent and the conversation ends after its answer; the user isn't asked again, not even by a From 3edb4509d7447f60e1eb92985661cb54c86253e8 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Thu, 12 Nov 2026 18:20:41 -0800 Subject: [PATCH 09/45] feat(user sim): the rating after the run is its own step, rate_user_satisfaction; always_on and prompt_agent no longer rate (FD-5491) The satisfaction lines move from the persona to the new step, so the user sim's system prompt never has them and the run behaves as tested. The step runs after the always_on or prompt_agent step it rates and asks the user sim once, in the session the run used, with what the user never read; a malformed answer gets one retry. The rating is recorded at metadata["user_sim_satisfaction"][run step id] and sent nowhere. always_on and prompt_agent are as they were before the first build, apart from the opening-files work, and always_on's record now names its user sim session (user_sim.context_id), which the step asks in. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/task/episode/rules.py | 3 + src/agent_env/task_step/registry.py | 2 + .../agent/rate_user_satisfaction.py | 387 ++++++++++++++++++ .../task_steps/agent/user_sim_common.py | 2 +- .../task_steps/agent/user_sim_versions.py | 56 +-- .../task_step/task_steps/always_on.py | 101 +---- .../task_step/task_steps/prompt_agent.py | 61 +-- 7 files changed, 441 insertions(+), 171 deletions(-) create mode 100644 src/agent_env/task_step/task_steps/agent/rate_user_satisfaction.py diff --git a/src/agent_env/task/episode/rules.py b/src/agent_env/task/episode/rules.py index 917c89f25..8e9abbb5f 100644 --- a/src/agent_env/task/episode/rules.py +++ b/src/agent_env/task/episode/rules.py @@ -166,6 +166,9 @@ def _rubrics_verifier(step: RubricsVerifierTaskStep, outside: Outside) -> list[s "agent_prompt_response_verifier", "verify_mcp_env_assessment", # It skips an agent that isn't deployed. "teardown_sandboxes", + # It talks only to a user sim deploy_user_sim_agent deployed (its load check), which that step's check keeps from + # being played from outside. + "rate_user_satisfaction", # They act on an outside agent's proxy, in its sandbox, as on the agent; install_agent installs the proxy in its place. "add_skills", "install_agent", "load_artifact", "run_code", "verify_sandbox", }) diff --git a/src/agent_env/task_step/registry.py b/src/agent_env/task_step/registry.py index ccd84cf97..43238ca41 100644 --- a/src/agent_env/task_step/registry.py +++ b/src/agent_env/task_step/registry.py @@ -65,6 +65,7 @@ def _builtin_registry() -> dict[str, type["TaskStep"]]: from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep from agent_env.task_step.task_steps.always_on import AlwaysOnTaskStep from agent_env.task_step.task_steps.agent.deploy_user_sim_agent import DeployUserSimAgentTaskStep + from agent_env.task_step.task_steps.agent.rate_user_satisfaction import RateUserSatisfactionTaskStep from agent_env.task_step.task_steps.register_env_triggers import RegisterEnvTriggersStep from agent_env.task_step.task_steps.register_agent_triggers import RegisterAgentTriggersStep from agent_env.task_step.task_steps.sync_env_clock import SyncEnvClockTaskStep @@ -138,6 +139,7 @@ def _builtin_registry() -> dict[str, type["TaskStep"]]: _get_type(PromptAgentTaskStep): PromptAgentTaskStep, _get_type(AlwaysOnTaskStep): AlwaysOnTaskStep, _get_type(DeployUserSimAgentTaskStep): DeployUserSimAgentTaskStep, + _get_type(RateUserSatisfactionTaskStep): RateUserSatisfactionTaskStep, _get_type(RegisterEnvTriggersStep): RegisterEnvTriggersStep, _get_type(RegisterAgentTriggersStep): RegisterAgentTriggersStep, _get_type(SyncEnvClockTaskStep): SyncEnvClockTaskStep, diff --git a/src/agent_env/task_step/task_steps/agent/rate_user_satisfaction.py b/src/agent_env/task_step/task_steps/agent/rate_user_satisfaction.py new file mode 100644 index 000000000..003c68c8d --- /dev/null +++ b/src/agent_env/task_step/task_steps/agent/rate_user_satisfaction.py @@ -0,0 +1,387 @@ +"""The rate_user_satisfaction step: once an always_on or prompt_agent run is over, the user sim that played its user +rates the job 1-5 with a one-line reason, by this step's ``satisfaction`` lines (user_sim.md, "The rating at the end"). + +The lines are this step's, never the persona's, so the user sim's system prompt never has them and the run behaves as +it does without a rating. The step runs after the run step, as a step of its own, so the rating takes no time from +the run's budget, and it changes nothing in the run step. + +Which run: ``run_step_id``, else the task's only always_on or prompt_agent step; the task is refused when it loads if +that step doesn't exist, isn't one of the two, or isn't run with a user sim a deploy_user_sim_agent step deploys, when +the task has several and no ``run_step_id`` picks one, when this step doesn't come after it (``depends_on``, when set, +must name it), or when two rating steps rate the same run. ``satisfaction`` is checked as the persona's sections are: +a list of non-empty lines. + +The ask is one message to the user sim, in the session the run used, so it remembers the run: + +- always_on: the session its run record names (``metadata["always_on"][run step id]["user_sim"]["context_id"]``), + the one every read of the run used. The messages the user never read are the record's ``unread_by_user`` at the end + (from the stored timeline's end event, else the record's own bounded copy), and the ask is at the time of her last + read, or the run's end when that's later. +- prompt_agent: the run's conversation id (``metadata["a2a_conversations"][run step id]``), the session its built-in + user sim's reads use (``_UserSide._ask_built_in``). The messages the user never read are the agent's replies after + her last read (``metadata["user_sim_reads"]``), from the run's conversation, and the ask is at the task clock's time. + +The message (``user_sim_versions.render_rating``) has those unread messages, the ``satisfaction`` lines under "HOW +YOU'D RATE THE JOB" tagged S1..., and the 1-5 ask; a malformed answer gets one retry (``user_sim_versions.rate``). The +rating is recorded at ``metadata["user_sim_satisfaction"][run step id]``: ``user_sim``, ``score`` (None when it +failed), ``reason``, ``used``, ``at``, ``unread_shown``, ``schema_errors``, ``retried``, ``user_sim_error``, +``user_sim_turns``, ``user_sim_trajectory_urls``, ``rated_by`` (this step's id), ``context_id`` and +``session_from``, the whole ask and raw answers in the object store at its ``object_url``. It never reaches the +agent, the graded conversation, or the verifications. + +Not rated, recorded with ``skipped`` and why: a run that ended on an error (always_on's error end reasons; a +prompt_agent run whose agent turn or user sim failed; a run step that failed), and a run whose user sim agent is no +longer deployed. A rating that fails or runs past ``timeout_seconds`` (by default one always_on read's worst case) is +recorded with its error; the step fails only when it can't find the run or its user, and by default +(``fail_task_on_error`` false) that doesn't fail the task. +""" +from __future__ import annotations + +import asyncio +import json +import logging +import time +import uuid +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import ClassVar, Optional + +from a2a.types import TaskState + +from agent_env.a2a_agent import A2AAgent, conversation_store, protocol +from agent_env.a2a_agent.object_transfer import send_and_wait +from agent_env.env.gateway.constants import EXT_CLOCK_URI +from agent_env.store import get_config +from agent_env.task_step.context import DeployedAgent, PromptResponse, StepContextError, TaskStepContext +from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_steps.agent import user_sim_versions +from agent_env.task_step.task_steps.agent.deploy_user_sim_agent import USER_SIMS_KEY, DeployUserSimAgentTaskStep +from agent_env.task_step.task_steps.agent.user_sim_common import SATISFACTION_KEY, iso, person_of, task_zone, utc +from agent_env.task_step.task_steps.always_on import ( + ALWAYS_ON_KEY, + ERROR_ENDS, + READ_ASKS, + TRAJECTORY_FETCH_SECONDS, + TURN_GRACE_SECONDS, + AlwaysOnTaskStep, +) +from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep, is_prompt_step + +logger = logging.getLogger(__name__) + +# One always_on read's worst case at its default user sim timeout (AlwaysOnTaskStep.read_cap_seconds). +USER_TIMEOUT_DEFAULT_SECONDS = 300 +DEFAULT_TIMEOUT_SECONDS = READ_ASKS * (USER_TIMEOUT_DEFAULT_SECONDS + TURN_GRACE_SECONDS + TRAJECTORY_FETCH_SECONDS) +SIM_FAILED_PREFIX = "The user simulator failed: " # prompt_agent's error_message for a user sim error + + +@dataclass +class _Session: + """What the step needs of the run: whose user sim, the session, what she never read, and when she's asked.""" + + session_from: str + user_sim: Optional[str] = None + context_id: Optional[str] = None + time_zone: Optional[str] = None + unread: list[dict] = field(default_factory=list) + at: Optional[str] = None + trajectory_prefix: Optional[str] = None + skipped: Optional[str] = None + + +class RateUserSatisfactionTaskStep(TaskStep): + """Once the run step is over, its user sim rates the job 1-5 by ``satisfaction``, recorded in + ``metadata["user_sim_satisfaction"][run step id]`` and sent nowhere.""" + + type: ClassVar[str] = "rate_user_satisfaction" + entity_refs = () + + def __init__(self, id: str, version: Optional[int], *, satisfaction: list[str], run_step_id: Optional[str] = None, + timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS, + poll_interval_seconds: float = PromptAgentTaskStep.DEFAULT_POLL_INTERVAL_SECONDS, + depends_on: Optional[list[TaskStepDependency]] = None, fail_task_on_error: bool = False): + """satisfaction: how the user would rate the job, one line each (what 5, 3 and 1 mean for it, what moves the + score). run_step_id: the always_on or prompt_agent step to rate; unset, the task's only one. timeout_seconds: + the longest the rating may take, both asks included.""" + super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error) + found = user_sim_versions.satisfaction_problems(satisfaction) + if run_step_id is not None and (not isinstance(run_step_id, str) or not run_step_id.strip()): + found.append("run_step_id must name the always_on or prompt_agent step to rate") + for name, value in (("timeout_seconds", timeout_seconds), ("poll_interval_seconds", poll_interval_seconds)): + if isinstance(value, bool) or not isinstance(value, (int, float)) or value < 0 or ( + name == "timeout_seconds" and value == 0): + found.append(f"{name} must be a number of seconds" + (", more than 0" if name == "timeout_seconds" + else ", 0 or more")) + if found: + raise ValueError(f"{self.type} '{id}': " + "; ".join(found)) + self.satisfaction = list(satisfaction) + self.run_step_id = run_step_id + self.timeout_seconds = timeout_seconds + self.poll_interval_seconds = poll_interval_seconds + + _OWN = ("satisfaction", "run_step_id", "timeout_seconds", "poll_interval_seconds") + + def to_dict(self) -> dict: + return {**super().to_dict(), **{k: getattr(self, k) for k in self._OWN}} + + @classmethod + def from_dict(cls, data: dict) -> RateUserSatisfactionTaskStep: + base = {**cls._base_from_dict(data), "fail_task_on_error": data.get("fail_task_on_error", False)} + return cls(**base, **{k: data[k] for k in cls._OWN if k in data}) + + # ---- the load check --------------------------------------------------------------------------------------------- + def _run_step(self, steps: list[TaskStep]) -> tuple[Optional[PromptAgentTaskStep], list[str]]: + """The step this one rates, or why there's none.""" + where = f"{self.type} '{self.id}'" + if self.run_step_id is not None: + run = next((s for s in steps if s.id == self.run_step_id), None) + if run is None: + return None, [f"{where}: run_step_id '{self.run_step_id}' names no step of the task"] + if not isinstance(run, PromptAgentTaskStep): + return None, [f"{where}: run_step_id '{self.run_step_id}' is a {run.type} step; name the always_on or " + "prompt_agent step to rate"] + return run, [] + runs = [s for s in steps if isinstance(s, PromptAgentTaskStep)] + if not runs: + return None, [f"{where}: the task has no always_on or prompt_agent step to rate"] + if len(runs) > 1: + return None, [f"{where}: the task has several always_on and prompt_agent steps " + f"({', '.join(s.id for s in runs)}); set run_step_id to the one to rate"] + return runs[0], [] + + def task_problems(self, steps: list[TaskStep]) -> list[str]: + run, found = self._run_step(steps) + if run is None: + return found + where = f"{self.type} '{self.id}'" + if steps.index(run) > steps.index(self): + found.append(f"{where} comes before step '{run.id}', the run it rates; put it after") + if self.depends_on is not None and run.id not in {d.task_step_id for d in self.depends_on}: + found.append(f"{where}: its depends_on must name '{run.id}', the run it rates, so it runs once that's over") + user = _user_of(run) + if user is None or not any(isinstance(s, DeployUserSimAgentTaskStep) and s.agent_name == user for s in steps): + found.append(f"{where}: the user of {run.type} step '{run.id}' isn't a user sim a deploy_user_sim_agent " + "step deploys; only that user sim can rate the run") + earlier = next((s for s in steps[:steps.index(self)] if isinstance(s, RateUserSatisfactionTaskStep) + and s._run_step(steps)[0] is run), None) + if earlier is not None: + found.append(f"{where}: step '{earlier.id}' already rates '{run.id}'; a run is rated once") + return found + + # ---- the run ---------------------------------------------------------------------------------------------------- + async def execute(self, context: TaskStepContext) -> TaskStepContext: + run_id = self.run_step_id or _only_run(context, self.id) + session = await self._session(context, run_id) + us = (context.metadata.get(USER_SIMS_KEY) or {}).get(session.user_sim) if session.user_sim else None + summary: dict = {"user_sim": session.user_sim, "score": None, "reason": None, "rated_by": self.id} + context.metadata.setdefault(SATISFACTION_KEY, {})[run_id] = summary + if session.skipped is None and us is None: + raise StepContextError(f"{self.type} '{self.id}': step '{run_id}' ran with no user sim a " + "deploy_user_sim_agent step recorded, so there's no one to ask") + agent = next((a for a in context.deployed_agents if a.agent_name == us.get("agent_name")), None) if us else None + if session.skipped is None and agent is None: + session.skipped = f"the user sim's agent '{us.get('agent_name')}' is no longer deployed" + if session.skipped is not None: + summary["skipped"] = session.skipped + logger.info(f"{self.type} '{self.id}': '{run_id}' not rated: {session.skipped}") + return context + if not session.context_id: + raise StepContextError(f"{self.type} '{self.id}': step '{run_id}' recorded no user sim session to ask in") + + person = {**person_of(us), "time_zone": session.time_zone} + read: dict = {"time": session.at or await _clock_now(context), + "assistant_messages": [{"at": m.get("at"), "text": m["text"]} for m in session.unread]} + calls: list[dict] = [] + deadline = time.monotonic() + self.timeout_seconds + + async def ask(text: str) -> dict: + got = await self._ask(agent, session, text, deadline) + calls.append(got) + return got + + got: dict = {} + error = None + try: + got = await asyncio.wait_for(user_sim_versions.rate( + us.get("user_version") or user_sim_versions.VERSION, person, self.satisfaction, read, ask), + self.timeout_seconds) or {} + except Exception as e: # TimeoutError included: the rating is recorded with its error + error = f"{type(e).__name__}: {e}" if str(e) else type(e).__name__ + error = error or (calls[-1].get("error") if calls else None) + summary.update({k: got.get(k) for k in ("score", "reason", "used", "schema_errors", "retried")}, + at=read["time"], unread_shown=len(read["assistant_messages"]), user_sim_error=error, + user_sim_turns=got.get("turns", len(calls)), + user_sim_trajectory_urls=[c.get("trajectory_url") for c in calls], + context_id=session.context_id, session_from=session.session_from) + logger.info(f"{self.type} '{self.id}': '{run_id}' rated {summary['score']} (error={error})") + full = {**summary, "run_step_id": run_id, "satisfaction": self.satisfaction, "read": read, + **{k: got.get(k) for k in ("input", "raw", "first_answer")}} + try: + config = get_config() + key = (f"{config.get_artifact_key_prefix()}user_sim_ratings/instance_id={context.instance_id or 'adhoc'}/" + f"{run_id}-{self.id}-{uuid.uuid4().hex[:8]}.json") + summary["object_url"] = await asyncio.to_thread( + config.get_object_store().put, key, json.dumps(full, indent=1, default=str).encode(), + content_type="application/json") + except Exception as e: + logger.warning(f"{self.type} '{self.id}': could not store the rating: {e}") + summary["store_error"] = f"{type(e).__name__}: {str(e)[:300]}" + return context + + async def _session(self, context: TaskStepContext, run_id: str) -> _Session: + """The run's user sim session, what she never read, and whether the run is rated at all.""" + response = next((r for r in reversed(context.prompt_responses) if r.step_id == run_id), None) + prefix = response.agent_trajectory_object_prefix if response is not None else None + record = (context.metadata.get(ALWAYS_ON_KEY) or {}).get(run_id) + if record is not None: + session = _always_on_session(record) + elif response is not None: + session = await _prompt_agent_session(context, run_id, response) + else: + failed = _failed(context, run_id) + if failed is None: + raise StepContextError(f"{self.type} '{self.id}': step '{run_id}' left no run to rate; run this step " + "after it") + return _Session("none", skipped=f"the run step failed: {failed}") + session.trajectory_prefix = prefix + "user_sim/" if prefix else None + if session.skipped is None and (failed := _failed(context, run_id)) is not None: + session.skipped = f"the run step failed: {failed}" + return session + + async def _ask(self, agent: DeployedAgent, session: _Session, text: str, deadline: float) -> dict: + """One user sim turn on ``text`` in the run's session, cut off at the step's ``deadline`` (time.monotonic()): + ``{"text", "structured", "trajectory_url"}``, or ``{"text": None, "structured": None, "error"}``.""" + limit = deadline - time.monotonic() + if limit <= 0: + return {"text": None, "structured": None, "error": "the step's time limit passed"} + url = agent.a2a_url or agent.api_url + message_id = uuid.uuid4().hex + try: + task_id, result = await asyncio.wait_for(send_and_wait( + url, [{"kind": "text", "text": text}], agent=agent if agent.sandbox_id else None, + message_id=message_id, context_id=session.context_id, timeout_seconds=limit, + poll_interval_seconds=self.poll_interval_seconds), limit) + except TimeoutError as e: + return {"text": None, "structured": None, "error": f"timeout: {e}"} + status = result.get("status") or {} + if status.get("state") != TaskState.completed and str(status.get("state")) != "completed": + return {"text": None, "structured": None, "error": f"state {status.get('state')}"} + msg = status.get("message") or {} + term = protocol.TerminalResponse.from_message(msg) + raw = next((p["text"] for p in msg.get("parts") or [] if p.get("kind") == "text"), term.response_text) + structured = term.structured_output if isinstance(term.structured_output, dict) else None + return {"text": raw, "structured": structured, + "trajectory_url": await self._trajectory(agent, session, task_id, message_id, deadline)} + + async def _trajectory(self, agent: DeployedAgent, session: _Session, task_id: str, message_id: str, + deadline: float) -> Optional[str]: + """The user sim's trajectory for one turn, stored where the run stored its reads'; None when its card offers + none or the run left no trajectory prefix.""" + card = agent.a2a_card or {} + ext = A2AAgent.find_extension(card, A2AAgent.EXT_TRAJECTORY) + if not ext or not session.trajectory_prefix: + return None + try: + # prompt_agent's fetch uses nothing of its step, so the run's own code stores it. + return await asyncio.wait_for(PromptAgentTaskStep._fetch_trajectory( + None, agent.a2a_url or agent.api_url, ext, task_id, session.trajectory_prefix, message_id, + card=card, sandbox_type=agent.sandbox_type), + max(0.0, min(TRAJECTORY_FETCH_SECONDS, deadline - time.monotonic()))) + except Exception as e: + logger.warning(f"user sim trajectory fetch failed (continuing): {e}") + return None + + +# ---- the run, as the run step recorded it ---------------------------------------------------------------------------- +def _user_of(run: PromptAgentTaskStep) -> Optional[str]: + """The agent a run step talks to as its user: always_on's ``user_sim``; prompt_agent's ``user_agent_name``, unless + a ``user_a2a_url`` puts someone else there.""" + if isinstance(run, AlwaysOnTaskStep): + return run.user_sim + return run.user_agent_name if run.user_a2a_url is None else None + + +def _only_run(context: TaskStepContext, step_id: str) -> str: + """With no ``run_step_id``: the task's one run step (the load check made sure there's one), by what it left.""" + ids = list(dict.fromkeys([r.step_id for r in context.prompt_responses if r.step_id] + + [f["step_id"] for f in context.metadata.get("failed_steps") or [] + if isinstance(f, dict) and is_prompt_step({"type": f.get("step_type")})])) + if len(ids) != 1: + raise StepContextError(f"{RateUserSatisfactionTaskStep.type} '{step_id}': " + + (f"several runs ran ({', '.join(ids)}); set run_step_id" if ids + else "no always_on or prompt_agent step ran before it")) + return ids[0] + + +def _failed(context: TaskStepContext, run_id: str) -> Optional[str]: + """The run step's error, when it failed and the task went on.""" + return next((f.get("error") or f.get("error_type") or "an error" for f in context.metadata.get("failed_steps") or [] + if isinstance(f, dict) and f.get("step_id") == run_id), None) + + +def _always_on_session(record: dict) -> _Session: + """From always_on's record: its user sim session, what she never read, and her last read's time or the end's.""" + sim = record.get("user_sim") or {} + session = _Session("always_on record", user_sim=sim.get("name"), context_id=sim.get("context_id"), + time_zone=record.get("time_zone")) + if record.get("end_reason") in ERROR_ENDS: + session.skipped = f"the run ended on {record['end_reason']}" + return session + timeline = record.get("timeline_tail") or [] + if record.get("object_url"): + try: + url = record["object_url"] + timeline = json.loads(get_config().get_object_store_at(url).get(url))["timeline"] + except Exception as e: # the record's own copy is bounded, but has them + logger.warning(f"rate_user_satisfaction: could not read always_on's timeline: {e}") + end = next((e for e in reversed(timeline) if e.get("event") == "end"), {}) + unread = ((end.get("pending") or {}).get("unread_by_user") if isinstance(end.get("pending"), dict) else None) + if not isinstance(unread, list): + unread = ((record.get("pending_at_end") or {}).get("unread_by_user") or {}).get("items") or [] + session.unread = [m for m in unread if isinstance(m, dict) and isinstance(m.get("text"), str)] + times = [utc(e["virtual_time"]) for e in timeline + if e.get("event") in ("user_sim_read", "end") and e.get("virtual_time")] + session.at = iso(max(times)) if times else None + return session + + +async def _prompt_agent_session(context: TaskStepContext, run_id: str, response: PromptResponse) -> _Session: + """From prompt_agent's run: its conversation, the session its built-in user sim's reads used; the agent's replies + after her last read; the task clock's time.""" + conversation_id = (context.metadata.get("a2a_conversations") or {}).get(run_id) + doc = conversation_store.get_conversation(conversation_id) if conversation_id else None + session = _Session("prompt_agent conversation", user_sim=(doc or {}).get("source_agent_name"), + context_id=conversation_id) + if response.error_type == "user_sim_error": + session.skipped = f"the user sim failed: {(response.error_message or '').removeprefix(SIM_FAILED_PREFIX)}" + return session + if response.error_type or response.error_message: + session.skipped = "the agent's turn failed" + return session + us = (context.metadata.get(USER_SIMS_KEY) or {}).get(session.user_sim) if session.user_sim else None + if us is not None: + session.time_zone = task_zone(us, context.metadata.get("clock_configurations") or [], + f"{RateUserSatisfactionTaskStep.type} on '{run_id}'") + session.at = await _clock_now(context) + last_read = max((r.get("turn") or 0 for r in (context.metadata.get("user_sim_reads") or {}).get(run_id) or []), + default=0) + replies = [m for m in (doc or {}).get("messages") or [] if m.get("role") == "agent"] + for m in replies[last_read:]: + text = "\n".join(p.get("text", "") for p in m.get("parts") or [] if p.get("kind") == "text") + clocks = [utc(c["virtual_time"]) for c in m.get("env_clock") or [] if c.get("virtual_time")] + session.unread.append({"at": iso(min(clocks)) if clocks else session.at, "text": text}) + return session + + +async def _clock_now(context: TaskStepContext) -> str: + """The task clock's time when one is armed (the earliest), else now; RFC 3339 UTC.""" + times = [] + for env in (d for d in context.deployed_envs if d.supports(EXT_CLOCK_URI, "state")): + try: + state = await env.invoke(EXT_CLOCK_URI, "state", timeout=5) + except Exception as e: + logger.warning(f"clock state read failed for env '{env.env_id}' (continuing): {e}") + continue + if state and state.get("armed") and state.get("virtual_time"): + times.append(utc(state["virtual_time"])) + return iso(min(times, default=datetime.now(timezone.utc))) diff --git a/src/agent_env/task_step/task_steps/agent/user_sim_common.py b/src/agent_env/task_step/task_steps/agent/user_sim_common.py index 4f11ed4b7..7cd09efa2 100644 --- a/src/agent_env/task_step/task_steps/agent/user_sim_common.py +++ b/src/agent_env/task_step/task_steps/agent/user_sim_common.py @@ -5,7 +5,7 @@ from datetime import datetime, timezone from typing import Any, Optional -# Where a run's rating by its user sim (user_sim_versions.rate) is recorded: metadata[SATISFACTION_KEY][step_id]. +# Where rate_user_satisfaction records a run's rating by its user sim: metadata[SATISFACTION_KEY][run step id]. SATISFACTION_KEY = "user_sim_satisfaction" diff --git a/src/agent_env/task_step/task_steps/agent/user_sim_versions.py b/src/agent_env/task_step/task_steps/agent/user_sim_versions.py index f893c7538..b20af0e99 100644 --- a/src/agent_env/task_step/task_steps/agent/user_sim_versions.py +++ b/src/agent_env/task_step/task_steps/agent/user_sim_versions.py @@ -10,8 +10,9 @@ keys of ``read`` it doesn't know are ignored. A line the user sent with files (the opening's ``files``, ``{"name", "description"}``) shows each after its text, "(you attached itinerary.pdf: Mom and Dad's flight itinerary)", or by name alone with no description: the user sim can't open them. -``rate`` asks the user sim, once the run is over, to rate the outcome 1-5 by its description's ``satisfaction`` -lines, with a one-line reason; None when the description has none, so a user without them is never asked. +``rate`` asks the user sim, once the run is over, to rate the outcome 1-5 by the ``satisfaction`` lines it is given +(the rate_user_satisfaction step's, never the persona's, so the system prompt never has them), with a one-line +reason. ``problems`` is the load check for the settings.""" from __future__ import annotations @@ -35,10 +36,8 @@ ("dont_mention", "DON'T MENTION THESE UNLESS THE ASSISTANT ASKS", "M"), ("phone", "WHAT YOU DO WITH YOUR EMAILS, TEXTS AND NOTIFICATIONS", "F"), ("when_done", "WHEN YOU'RE DONE", ""), - ("satisfaction", "HOW YOU'D RATE THE JOB (only when asked, once it's over)", "S"), ) SECTION_KEYS = tuple(key for key, _, _ in SECTIONS) -SATISFACTION = "satisfaction" # the section the rating after the run goes by; without it there's no rating RULES_TITLE = "YOUR RULES FOR THE ASSISTANT (it has them in its notes)" PRIVATE_TITLE = "PRIVATE" RULES_AFTER = "decisions" @@ -84,9 +83,11 @@ EARLIER_MAX = 20 EARLIER_CUT = " (older messages not shown)" -# The rating after the run (``rate``): asked only of a user whose description has ``satisfaction`` lines. Its answer -# fits OUTPUT_FORMAT, which the agent was given at deploy, so the format is the same with or without a rating. +# The rating after the run (``rate``), asked by the rate_user_satisfaction step with that step's ``satisfaction`` lines, +# shown in the ask under RATING_TITLE, tagged S1... Its answer fits OUTPUT_FORMAT, which the agent was given at deploy. SCORES = (1, 2, 3, 4, 5) +RATING_TITLE = "HOW YOU'D RATE THE JOB" +RATING_TAG = "S" RATING_LINE = '{"outcome": "done", "score": 1 | 2 | 3 | 4 | 5, "message": "...", "used": "..."}' RATING_OVER = "The job is over: your AI assistant won't see this, or anything you write from now on." RATING_UNREAD = "\nNEW FROM YOUR AI ASSISTANT (you haven't read this yet):" @@ -210,11 +211,12 @@ def problems(person: Any) -> list[str]: else: unknown = sorted(set(description) - set(SECTION_KEYS)) if unknown: - out.append(f"description has unknown sections {unknown}; the sections are {list(SECTION_KEYS)}") + out.append(f"description has unknown sections {unknown}; the sections are {list(SECTION_KEYS)}" + + ("; satisfaction goes on the rate_user_satisfaction step, so the user sim's prompt never has " + "it" if "satisfaction" in unknown else "")) out += [f"description.{key} must be a list of non-empty lines" for key in SECTION_KEYS if key in description and not _lines(description[key])] - if not any(_lines(description.get(key)) and description[key] for key in SECTION_KEYS - if key != SATISFACTION): + if not any(_lines(description.get(key)) and description[key] for key in SECTION_KEYS): out.append("description is empty: write at least who they are") permissions = person.get("permissions") if permissions is not None and not isinstance(permissions, dict): @@ -430,15 +432,16 @@ def _parsed(got: dict) -> dict: # ---- the rating after the run ------------------------------------------------------------------------------------- -def rates(person: dict) -> bool: - """Whether the user is asked to rate the run once it's over: when the description has ``satisfaction`` lines.""" - description = person.get("description") - return isinstance(description, dict) and bool(description.get(SATISFACTION)) +def satisfaction_problems(lines: Any) -> list[str]: + """What's wrong with the lines a rating goes by: one message, or none.""" + if not (_lines(lines) and lines): + return ["satisfaction must be a list of non-empty lines: what each end of the scale means for this job"] + return [] -def render_rating(person: dict, read: dict) -> str: - """The message that asks for the rating at ``read["time"]``, with the assistant's messages the user hasn't read - (``read["assistant_messages"]``, maybe none) before the ask.""" +def render_rating(person: dict, satisfaction: list[str], read: dict) -> str: + """The message that asks for the rating at ``read["time"]``: the assistant's messages the user hasn't read + (``read["assistant_messages"]``, maybe none), the ``satisfaction`` lines tagged S1..., then the ask.""" zone = ZoneInfo(person.get("time_zone") or "UTC") now = _local(read["time"], zone) out = [f"It's {now.strftime('%a, %b')} {now.day}, {_hm(now)}. {RATING_OVER}"] @@ -446,6 +449,7 @@ def render_rating(person: dict, read: dict) -> str: if messages: out.append(RATING_UNREAD) out += [_message(m, _stamp(_local(m.get("at") or read["time"], zone), now)) for m in messages] + _section(out, RATING_TITLE, RATING_TAG, list(satisfaction)) out.append("\n" + RATING_ASK.format(first=_first(person))) out.append(RATING_ANSWER) return "\n".join(out) @@ -474,16 +478,18 @@ def parse_rating(raw: Any) -> dict: return {"score": obj["score"], "reason": obj["message"].strip(), "used": obj.get("used"), "schema_errors": []} -async def rate(version: str, person: dict, read: dict, ask: Callable[[str], Awaitable[dict]]) -> Optional[dict]: - """The user's rating of the run once it's over, by their ``satisfaction`` lines: one ask, in the run's session, - with one retry when the answer is malformed; None, asking nothing, when the description has no such lines. - ``read`` is ``{"time": RFC 3339 UTC, "assistant_messages": [{"at", "text"}]}``, the messages the user hasn't read. - Returns ``score``, ``reason``, ``used``, ``schema_errors``, ``input``, ``raw``, ``retried``, ``turns`` and, after a - retry, ``first_answer``. The rating is never a reply: the caller records it and sends it nowhere.""" +async def rate(version: str, person: dict, satisfaction: list[str], read: dict, + ask: Callable[[str], Awaitable[dict]]) -> dict: + """The user's rating of the run once it's over, by the ``satisfaction`` lines: one ask, in the run's session, with + one retry when the answer is malformed. ``read`` is ``{"time": RFC 3339 UTC, "assistant_messages": [{"at", + "text"}]}``, the messages the user hasn't read. Returns ``score``, ``reason``, ``used``, ``schema_errors``, + ``input``, ``raw``, ``retried``, ``turns`` and, after a retry, ``first_answer``. The rating is never a reply: the + caller records it and sends it nowhere.""" _check(version) - if not rates(person): - return None - text = render_rating(person, read) + found = satisfaction_problems(satisfaction) + if found: + raise ValueError("; ".join(found)) + text = render_rating(person, satisfaction, read) got = await ask(text) first = _parsed_rating(got) if not first["schema_errors"] or got.get("error"): diff --git a/src/agent_env/task_step/task_steps/always_on.py b/src/agent_env/task_step/task_steps/always_on.py index c49620de1..8f4f559d1 100644 --- a/src/agent_env/task_step/task_steps/always_on.py +++ b/src/agent_env/task_step/task_steps/always_on.py @@ -42,12 +42,6 @@ ``max_wakes`` makes the last, carries the last-turn notice (``final_reminder``). After it the user sim reads what the agent sent (its wrap-up), no earlier than the task's time when the run ended, and the user's reaction, and any reply still held, are recorded; then the run ends. -- A user sim whose description has ``satisfaction`` lines then rates the run once, 1-5 with a one-line reason - (``user_sim_versions.rate``), after that last read; the messages it never read come with the ask. The rating is - recorded in ``metadata["user_sim_satisfaction"][step_id]`` (the whole ask in the timeline's ``user_sim_rating`` - event) and goes nowhere else: not to the agent, the graded conversation or the reads' history, and it is no reply, - so it doesn't count toward ``max_replies``. A rating that fails is recorded with its error and changes nothing. A run - that ended on an error is not rated. Without ``satisfaction`` lines nothing is asked. - A real-time budget keeps the run inside the hub's 8-hour job (``effective_time_budget`` gives the rule): once one more wake might end past ``time_budget_seconds`` (7 h, lowered to fit the task's own timeouts), the agent gets its closing wake, with the step's wrap-up line and the last-turn notice, and the run ends with end reason time_budget. @@ -70,7 +64,8 @@ reply that never reached it (her last read's, or one still held at the end) has ``delivered`` false and no ``user_reads_at``. - The run's metadata keeps a summary in ``metadata["always_on"][step_id]`` (small enough for a running job to report - back); the full timeline goes to the object store, at the summary's ``object_url``. + back); the full timeline goes to the object store, at the summary's ``object_url``. Its ``user_sim`` names the user + sim's session (``context_id``), which rate_user_satisfaction asks in once the run is over. - An always_on_agent_verifier (``verifiers/always_on_agent_verifier.py``) grades the step's prompt from the user's chat and the run's context, the run wake by wake among it (``render_for_judge``); a plain rubrics_verifier on it is refused when the task loads. @@ -118,12 +113,11 @@ from agent_env.task_step.context import PromptResponse, StepContextError, TaskStepContext from agent_env.task_step.task_step import TaskStep from agent_env.task_step.task_steps.agent import user_sim_versions -from agent_env.task_step.task_steps.agent.deploy_user_sim_agent import USER_SIMS_KEY, DeployUserSimAgentTaskStep +from agent_env.task_step.task_steps.agent.deploy_user_sim_agent import USER_SIMS_KEY from agent_env.task_step.task_steps.notification_delivery import delivered_parts, label_time, take_due from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep -from agent_env.task_step.task_steps.agent.user_sim_common import SATISFACTION_KEY, person_of, task_zone +from agent_env.task_step.task_steps.agent.user_sim_common import iso as _iso, person_of, task_zone, utc as _utc from agent_env.task_step.task_steps.agent.user_sim_common import attached, for_agent, recorded_opening -from agent_env.task_step.task_steps.agent.user_sim_common import iso as _iso, utc as _utc logger = logging.getLogger(__name__) @@ -283,10 +277,6 @@ def read_cap_seconds(self) -> float: fetch; an ask past it is refused, and an answer past it ends the read as a user sim error.""" return READ_ASKS * (self.user_timeout_seconds + TURN_GRACE_SECONDS + TRAJECTORY_FETCH_SECONDS) - def rating_cap_seconds(self) -> float: - """The longest the rating after the run may take: as long as a read.""" - return self.read_cap_seconds() - def wake_reserve_seconds(self, contact_seconds: float = CONTACT_TIMEOUT_DEFAULT_SECONDS) -> float: """The longest one turn of the loop, or the run's closing, can take, each part hard-capped: a clock jump's settle (settle_cap_seconds), then a wake (the agent's turn and, with nudges on, its nudge turn, each at most @@ -299,8 +289,7 @@ def wake_reserve_seconds(self, contact_seconds: float = CONTACT_TIMEOUT_DEFAULT_ wake = turn * (2 if self.nudge else 1) + settle + GATEWAY_ALLOWANCE_SECONDS return settle + wake + self.read_cap_seconds() + GATEWAY_ALLOWANCE_SECONDS - def effective_time_budget(self, contact_seconds: float = CONTACT_TIMEOUT_DEFAULT_SECONDS, - rating: bool = False) -> float: + def effective_time_budget(self, contact_seconds: float = CONTACT_TIMEOUT_DEFAULT_SECONDS) -> float: """The real time after which the run closes: ``time_budget_seconds``, else 7 h, lowered so the closing still fits the hub's job. The rule: @@ -314,9 +303,8 @@ def effective_time_budget(self, contact_seconds: float = CONTACT_TIMEOUT_DEFAULT that can't is refused, as is a wake whose worst case leaves no room for a budget at all. ``contact_seconds`` is the longest contact time limit: the gateway's default when the task loads, the - contacts' own once the run has read them (``_configure_contacts``), which checks the budget again. - ``rating``: the user sim rates the run after its closing, so the reserve also covers rating_cap_seconds.""" - reserve = self.wake_reserve_seconds(contact_seconds) + (self.rating_cap_seconds() if rating else 0) + contacts' own once the run has read them (``_configure_contacts``), which checks the budget again.""" + reserve = self.wake_reserve_seconds(contact_seconds) room = JOB_LIMIT_SECONDS - JOB_HEADROOM_SECONDS - reserve if room <= reserve: raise ValueError(f"always_on: one wake can take up to {reserve:.0f}s (wake_reserve_seconds: a settle, the " @@ -351,19 +339,11 @@ def task_problems(self, steps: list[TaskStep]) -> list[str]: from agent_env.task_step.task_steps.verifiers.always_on_agent_verifier import AlwaysOnAgentVerifier from agent_env.task_step.task_steps.verifiers.rubrics_verifier import is_rubrics_verifier - out = [f"step '{s.id}' ({s.type}) grades the prompt '{self.prompt_id}' of always_on step '{self.id}', which " - f"is graded from the user's chat and the run's context, not its wakes' trajectories: use type " - f"{AlwaysOnAgentVerifier.type}" - for s in steps if is_rubrics_verifier(s) and s.prompt_id == self.prompt_id - and not isinstance(s, AlwaysOnAgentVerifier)] - # A user sim that rates the run adds the rating to the closing, which the time budget must still fit. - if any(isinstance(s, DeployUserSimAgentTaskStep) and s.agent_name == self.user_sim - and user_sim_versions.rates({"description": (s.persona or {}).get("description")}) for s in steps): - try: - self.effective_time_budget(rating=True) - except ValueError as e: - out.append(f"{e} (with the user sim's rating after the run)") - return out + return [f"step '{s.id}' ({s.type}) grades the prompt '{self.prompt_id}' of always_on step '{self.id}', which " + f"is graded from the user's chat and the run's context, not its wakes' trajectories: use type " + f"{AlwaysOnAgentVerifier.type}" + for s in steps if is_rubrics_verifier(s) and s.prompt_id == self.prompt_id + and not isinstance(s, AlwaysOnAgentVerifier)] async def _execute(self, context: TaskStepContext) -> TaskStepContext: run = _Run(self, context) @@ -433,8 +413,6 @@ def __init__(self, step: AlwaysOnTaskStep, context: TaskStepContext): self.clock_rates: list[tuple[Any, float]] = [] # (env, the task's rate) for each clock the step freezes self.undo: list[tuple[str, Any]] = [] # (what, coroutine function): the gateway settings to put back at the end self.rules_in_system_prompt = False # whether the agent took the rules in its system prompt - self.rates = False # whether the user sim rates the run once it's over - self.last_read_time: Optional[datetime] = None # when she last read, for the rating after it async def restore(self) -> None: """Put back what the step changed on the gateways, latest first, so later steps of the task see the settings @@ -494,7 +472,6 @@ async def go(self) -> None: raise StepContextError(f"always_on: the user sim agent '{us['agent_name']}' is not deployed") self.user_url = self.user_agent.a2a_url or self.user_agent.api_url self.plugin = await self._load_plugin(us.get("plugin")) - self.rates = hasattr(self.plugin, "rate") and user_sim_versions.rates(self.person) self.tz = ZoneInfo(self.zone) self.clock_envs = [d for d in ctx.deployed_envs if d.supports(EXT_CLOCK_URI, "state")] @@ -715,7 +692,7 @@ async def _configure_contacts(self) -> None: got = await _call(env, EXT_TRIGGERS_URI, "config", {"concurrent_fires": step.contact_replies_in_parallel}) applied.append({**entry, "applied": True, "concurrent_fires": got.get("concurrent_fires")}) self.contact_seconds = float(max(limits, default=0.0)) - self.time_budget = step.effective_time_budget(self.contact_seconds, self.rates) + self.time_budget = step.effective_time_budget(self.contact_seconds) self.contacts = {"replies_in_parallel": step.contact_replies_in_parallel, "longest_timeout_seconds": self.contact_seconds, "envs": applied} @@ -1010,8 +987,6 @@ async def loop(self, now: datetime) -> None: if not await self._capped(self._next(), "a turn of the loop"): break await self._capped(self._last_read(), "the last read", step.read_cap_seconds() + GATEWAY_ALLOWANCE_SECONDS) - if self.rates: - await self._rating() async def _capped(self, work, what: str, seconds: Optional[float] = None): """``work``, cut off past ``seconds`` (one turn of the loop's worst case, ``wake_reserve_seconds``): every part @@ -1045,8 +1020,7 @@ async def _next(self) -> bool: step = self.step now = await self.now() spent = self.spent() - # With a rating after the run, the closing has it too. - reserve = step.wake_reserve_seconds(self.contact_seconds) + (step.rating_cap_seconds() if self.rates else 0) + reserve = step.wake_reserve_seconds(self.contact_seconds) if spent + reserve >= self.time_budget: self.log("time_budget_reached", virtual_time=_iso(now), real_seconds=round(spent, 1), time_budget_seconds=self.time_budget, wake_reserve_seconds=reserve) @@ -1150,47 +1124,6 @@ async def _last_read(self) -> None: at, self.read_mode = self._available(now) await self.read(at, last=True) - async def _rating(self) -> None: - """Once the run is over, the user sim rates it (``user_sim_versions.rate``): at her last read's time, or the - world's when that's later, with the agent's messages she never read. Recorded in - ``metadata[SATISFACTION_KEY][step_id]``, the whole ask in the timeline; never delivered, never in the graded - conversation, never a reply. A rating that fails, or runs past rating_cap_seconds, is recorded with its error - and doesn't change how the run ends. Not after a run that ended on an error.""" - step = self.step - summary: dict = {"user_sim": step.user_sim, "score": None, "reason": None} - self.ctx.metadata.setdefault(SATISFACTION_KEY, {})[step.id] = summary - if self.end_reason in ERROR_ENDS: - summary["skipped"] = f"the run ended on {self.end_reason}" - self.log("user_sim_rating", skipped=summary["skipped"]) - return - calls: list[dict] = [] - cap = step.rating_cap_seconds() - deadline = time.monotonic() + cap - - async def ask(text: str) -> dict: - got = await self._ask_user_sim(text, deadline) - calls.append(got) - return got - - t = time.time() - read: dict = {"time": None, "assistant_messages": [{"at": m.get("at"), "text": m["text"]} for m in self.unread]} - got: dict = {} - error = None - try: - now = await self.now() - read["time"] = _iso(max(now, self.last_read_time) if self.last_read_time else now) - got = await asyncio.wait_for(self.plugin.rate(self.version, self.person, read, ask), cap) or {} - except Exception as e: # TimeoutError included: the rating is recorded, the run goes on to its end - error = f"{type(e).__name__}: {e}" if str(e) else type(e).__name__ - error = error or (calls[-1].get("error") if calls else None) - summary.update({k: got.get(k) for k in ("score", "reason", "used", "schema_errors", "retried")}, - at=read["time"], unread_shown=len(read["assistant_messages"]), - user_sim_error=error, user_sim_turns=got.get("turns", len(calls)), - user_sim_trajectory_urls=[c.get("trajectory_url") for c in calls]) - self.log("user_sim_rating", virtual_time=summary["at"], read=read, input=got.get("input"), - raw=got.get("raw"), first_answer=got.get("first_answer"), real_seconds=round(time.time() - t, 1), - **{k: v for k, v in summary.items() if k not in ("user_sim", "at")}) - def _prompt_parts(self) -> list[dict]: """The user's first message, the user sim's opening, as A2A parts, its seed applied.""" return self.step._apply_seed(recorded_opening(self.us), self.ctx.metadata.get("seed", {})) @@ -1560,7 +1493,6 @@ async def ask(text: str) -> dict: ans = {"outcome": "no_reply", "message": ""} calls.append({"text": None, "structured": None, "error": f"the read ran past its cap ({cap:.0f}s)"}) self.history.append({"read": read, "answer": ans}) - self.last_read_time = now mode = self.read_mode latency = self._latency(mode) flagged = (len(ans.get("message") or "") > ASSISTANT_LIKE_MESSAGE_CHARS @@ -1757,7 +1689,8 @@ async def finish(self) -> None: record = {"step_id": step.id, "end_reason": self.end_reason, "end_note": self.end_note, "error": self.error, "error_type": self.error_type, "counts": counts, "user_sim": {"name": step.user_sim, "version": self.version, - "agent_name": self.us.get("agent_name"), **self.user_setup}, + "agent_name": self.us.get("agent_name"), "context_id": self.user_context_id, + **self.user_setup}, "end_time": _iso(self.end) if self.end else None, "time_zone": self.zone, "multi_day": self.multi_day, "real_seconds": round(time.time() - self.t0_real, 1), "timeline_events": len(self.timeline), "pending_at_end": _bounded(self.pending)} @@ -1767,8 +1700,6 @@ async def finish(self) -> None: "scheduled_checks": self.checks, "contacts": self.contacts, "time_budget_seconds": self.time_budget, "wake_reserve_seconds": step.wake_reserve_seconds(self.contact_seconds)} - if self.rates: - record["settings"]["rating_cap_seconds"] = step.rating_cap_seconds() await self._store_timeline(record) ctx.metadata.setdefault(ALWAYS_ON_KEY, {})[step.id] = record # The graded response is the user's whole chat from the agent, each message with its pretend time; never its diff --git a/src/agent_env/task_step/task_steps/prompt_agent.py b/src/agent_env/task_step/task_steps/prompt_agent.py index d54cad294..89d972e83 100644 --- a/src/agent_env/task_step/task_steps/prompt_agent.py +++ b/src/agent_env/task_step/task_steps/prompt_agent.py @@ -42,7 +42,7 @@ from agent_env.task_step.task_steps.notification_delivery import delivered_parts, next_message, take_due from agent_env.task_step.task_steps.post_turn_hooks import PostTurnHooks, run_post_turn_hooks, validate_hook_specs from agent_env.task_step.task_steps.agent import user_sim_versions -from agent_env.task_step.task_steps.agent.user_sim_common import SATISFACTION_KEY, iso, person_of, task_zone, utc +from agent_env.task_step.task_steps.agent.user_sim_common import iso, person_of, task_zone, utc from agent_env.task_step.task_steps.agent.user_sim_common import attached, opening_text, recorded_opening from agent_env.task_step.task_steps.sandbox_utils.sandbox_utils import ( agent_error_text, @@ -625,7 +625,6 @@ async def _converse( LiveTurns(*live, poll_interval_seconds=self.poll_interval_seconds) if live is not None else None ) final_state: str = TaskState.completed.value - last_reply: tuple[int, str] = (0, "") # the agent's last turn and its text, for the rating after the chat trajectory_url: Optional[str] = None # What the target has been sent: of the files its replies name, the only ones a user-sim is made able # to read, so a reply naming any other object a store owns can't read it out through the user-sim. @@ -706,7 +705,6 @@ def record_turn() -> None: final_task_id = sent_task_id final_state = target_state - last_reply = (turn + 1, _agent_text(final_terminal.response_text, agent_response_parts)) logger.info(f"Turn {turn+1}/{self.max_conversation_turns} agent response ({target_state}): {final_terminal.response_text[:120]}...") if target_state == TaskState.failed: @@ -768,10 +766,6 @@ def record_turn() -> None: final_conv = conversation_store.get_conversation(conversation_id) if final_conv and final_conv.get("status") == "active": conversation_store.mark_closed(conversation_id) - if sim is not None: # once the chat has ended, a user with satisfaction lines rates it - skipped = ("the agent's turn failed" if final_state == TaskState.failed - else f"the user sim failed: {sim.error}" if sim.error else None) - await sim.rate(*last_reply, user_side._ask_built_in, skipped=skipped) if trig_ext and triggers_registered: # best-effort firing-log readback (provenance only; never fails the run) try: @@ -1270,7 +1264,6 @@ def __init__(self, step: PromptAgentTaskStep, context: TaskStepContext, person: self._max_replies = person.get("max_replies") self.replies = 0 self.error: Optional[str] = None # the user sim's error, once a read ended on one - self.read_turn = 0 # the agent's last turn the user read @classmethod async def start(cls, step: PromptAgentTaskStep, context: TaskStepContext, record: dict, zone: str, @@ -1293,7 +1286,6 @@ async def ask(self, number: int, reply: str, ask_user_sim) -> dict: read = {"time": now, "assistant_messages": [{"at": now, "text": reply}], "phone": [], "you_sent": self._you_sent, "done_ends_run": True} self._you_sent = [] - self.read_turn = number calls: list[dict] = [] async def ask(text: str) -> dict: @@ -1321,57 +1313,6 @@ async def ask(text: str) -> dict: return {"message": [{"kind": "text", "text": ans["message"]}], "end_after": "user_done"} return {"end": "user_done" if outcome == "done" else "user_no_reply"} - async def rate(self, turn: int, reply: str, ask_user_sim, skipped: Optional[str] = None) -> None: - """Once the chat has ended, the user's rating of it (``user_sim_versions.rate``), when their description has - ``satisfaction`` lines: with the agent's last reply when they never read it (``turn`` is past the last one - read), at the task clock's time. Recorded as always_on records it, in ``metadata[SATISFACTION_KEY][step_id]``, - the whole ask in the object store at its ``object_url``; never sent to the agent, never in the conversation, - never a reply. ``skipped`` names why it isn't asked (the chat ended on an error); a rating that fails is - recorded with its error and changes nothing.""" - if not user_sim_versions.rates(self.person): - return - step = self._step - summary: dict = {"user_sim": step.user_agent_name, "score": None, "reason": None} - self._context.metadata.setdefault(SATISFACTION_KEY, {})[step.id] = summary - if skipped: - summary["skipped"] = skipped - return - calls: list[dict] = [] - - async def ask(text: str) -> dict: - got = await ask_user_sim(text) - calls.append(got) - return got - - read: dict = {"time": None, "assistant_messages": []} - got: dict = {} - error = None - try: - read["time"] = await _clock_now(step, self._clock_envs) - if turn > self.read_turn and reply.strip(): - read["assistant_messages"] = [{"at": read["time"], "text": reply}] - got = await user_sim_versions.rate(user_sim_versions.VERSION, self.person, read, ask) or {} - except Exception as e: # recorded; the step goes on to its response - error = f"{type(e).__name__}: {e}" if str(e) else type(e).__name__ - error = error or (calls[-1].get("error") if calls else None) - summary.update({k: got.get(k) for k in ("score", "reason", "used", "schema_errors", "retried")}, - at=read["time"], unread_shown=len(read["assistant_messages"]), user_sim_error=error, - user_sim_turns=got.get("turns", len(calls)), - user_sim_trajectory_urls=[c.get("trajectory_url") for c in calls]) - logger.info(f"prompt_agent '{step.id}' user sim rating: score={summary['score']} error={error}") - full = {**summary, "read": read, **{k: got.get(k) for k in ("input", "raw", "first_answer")}} - try: - config = get_config() - key = (f"{config.get_artifact_key_prefix()}prompt_agent_user_sim_reads/" - f"instance_id={self._context.instance_id or 'adhoc'}/" - f"{step.id}-rating-{uuid.uuid4().hex[:8]}.json") - summary["object_url"] = await asyncio.to_thread( - config.get_object_store().put, key, json.dumps(full, indent=1, default=str).encode(), - content_type="application/json") - except Exception as e: - logger.warning(f"prompt_agent '{step.id}': could not store the user sim's rating: {e}") - summary["store_error"] = f"{type(e).__name__}: {str(e)[:300]}" - async def _record(self, number: int, read: dict, ans: dict, calls: list[dict], error: Optional[str]) -> None: """Record the read as always_on does: a summary in ``metadata["user_sim_reads"][step_id]``, the whole read (what the user sim was shown, and its raw answers) in the object store at the summary's ``object_url``.""" From 58e95a53287753078acc63a1a3d26dabb9d8796a Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Thu, 12 Nov 2026 18:20:41 -0800 Subject: [PATCH 10/45] test(user sim): rate_user_satisfaction after always_on and prompt_agent, its load checks, and no S lines in the prompt (FD-5491) Co-Authored-By: Claude Opus 5.5 --- .../task_step/test_user_sim_satisfaction.py | 533 ++++++++++-------- 1 file changed, 297 insertions(+), 236 deletions(-) diff --git a/tst/unit/task_step/test_user_sim_satisfaction.py b/tst/unit/task_step/test_user_sim_satisfaction.py index be003fbcc..afdcfe155 100644 --- a/tst/unit/task_step/test_user_sim_satisfaction.py +++ b/tst/unit/task_step/test_user_sim_satisfaction.py @@ -1,7 +1,7 @@ -"""The user sim's rating of the run (FD-5491): a description's satisfaction lines go in the prompt, tagged S1...; once -the run is over the user sim is asked once to rate it 1-5 with a reason, in always_on after the last read and in -prompt_agent once the chat has ended. The rating is recorded, never delivered, and never fails the run. Without -satisfaction lines nothing changes: the same prompt, the same reads, no rating.""" +"""The user sim's rating of the run (FD-5491), a step of its own: rate_user_satisfaction runs after an always_on or +prompt_agent run and asks the user sim once, in the session the run used, to rate the job 1-5 with a reason, by the +step's satisfaction lines. The lines are never in the persona or the user sim's prompt, the run steps are as they were +without a rating, and the rating is recorded, never delivered, and never fails the task.""" from __future__ import annotations @@ -12,16 +12,21 @@ import pytest +from agent_env.a2a_agent import conversation_store from agent_env.store import get_config +from agent_env.task.task import Task +from agent_env.task_step.registry import _builtin_registry from agent_env.task_step.task_steps import always_on as ao +from agent_env.task_step.task_steps.agent import rate_user_satisfaction as rus from agent_env.task_step.task_steps.agent import user_sim_versions as us -from agent_env.task_step.task_steps.agent.deploy_user_sim_agent import DeployUserSimAgentTaskStep, persona_problems +from agent_env.task_step.task_steps.agent.deploy_user_sim_agent import persona_problems +from agent_env.task_step.task_steps.agent.rate_user_satisfaction import RateUserSatisfactionTaskStep from agent_env.task_step.task_steps.agent.user_sim_common import SATISFACTION_KEY -from tst.unit.task_step.always_on_harness import USER_SIM, Rig -from tst.unit.task_step.test_deploy_user_sim_agent import DESCRIPTION, HANNAH, _always_on, _rig_context +from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep +from tst.unit.task_step.always_on_harness import Fail, Rig +from tst.unit.task_step.test_deploy_user_sim_agent import DESCRIPTION, HANNAH, _always_on, _Deploy, _rig_context from tst.unit.task_step.test_deploy_user_sim_agent import _step as _deploy_step from tst.unit.task_step.test_prompt_agent_built_in_user_sim import AGENT, USER, Peers, _context, _step, said, text -from tst.unit.task_step.test_prompt_agent_user_side import _stored pytestmark = pytest.mark.usefixtures("local_stores") @@ -33,27 +38,25 @@ "3, fine: all booked, but you had to answer a question your rules already settled.", "1, you'd have done it yourself: a dinner or the place is still not booked at the end.", "Being asked for things that are in Mom's email bothers you."] +LINDA_LINES = ["5: the house and the dinner booked.", "1: nothing booked."] NOTICE = ao.FINAL_REMINDER.format(first="Linda") -def _satisfied(person: dict | None = None) -> dict: - person = copy.deepcopy(person or PERSON) - person["description"]["satisfaction"] = list(SATISFIED) - return person - - def rated(score, message="all booked, no chasing", used="S1", outcome="done") -> dict: return {"outcome": outcome, "score": score, "message": message, "used": used} +def _rater(**fields) -> RateUserSatisfactionTaskStep: + return RateUserSatisfactionTaskStep.from_dict({"id": "rate", "type": "rate_user_satisfaction", + "satisfaction": LINDA_LINES, "poll_interval_seconds": 0, **fields}) + + def _asker(*answers): sent = [] async def ask(text): sent.append(text) got = answers[len(sent) - 1] - if isinstance(got, Exception): - raise got if isinstance(got, dict) and "error" in got: return got return {"text": got if isinstance(got, str) else json.dumps(got), @@ -62,65 +65,42 @@ async def ask(text): return ask, sent -# ---- the prompt ------------------------------------------------------------------------------------------------------- -def test_without_satisfaction_lines_the_prompt_is_the_tested_one_and_there_is_no_rating(): - assert us.setup("ours", PERSON)["system_prompt"] == PROMPT - assert us.rates(PERSON) is False and us.rates({"description": "Linda."}) is False - assert "HOW YOU'D RATE" not in PROMPT and " S1" not in PROMPT +NO_UNREAD = {"time": "2026-11-24T05:00:00Z", "assistant_messages": []} -def test_satisfaction_lines_come_last_tagged_s1_and_nothing_else_in_the_prompt_changes(): - prompt = us.setup("ours", _satisfied())["system_prompt"] - assert prompt == PROMPT + "\nHOW YOU'D RATE THE JOB (only when asked, once it's over)\n" + "".join( - f" S{i:<3}{line}\n" for i, line in enumerate(SATISFIED, 1)) - assert us.setup("ours", _satisfied())["output_format"] == us.OUTPUT_FORMAT # the same answer format - assert us.rates(_satisfied()) is True +# ---- the user sim's prompt and persona: no satisfaction lines, ever ---------------------------------------------------- +def test_the_user_sims_prompt_is_the_tested_one_and_never_has_s_lines(): + prompt = us.setup("ours", PERSON)["system_prompt"] + assert prompt == PROMPT and "HOW YOU'D RATE" not in prompt and " S1" not in prompt + assert all(key != "satisfaction" for key, _, _ in us.SECTIONS) -def test_satisfaction_lines_do_not_change_any_read(): - read = {"time": "2026-11-17T17:05:00Z", "assistant_messages": [{"at": "2026-11-17T17:00:00Z", "text": "Booked."}], - "phone": [], "you_sent": [], "done_ends_run": False} - assert us.render(_satisfied(), [], read) == us.render(PERSON, [], read) +@pytest.mark.asyncio +async def test_with_a_rating_step_the_deployed_prompt_is_still_the_tested_one(monkeypatch): + deploy = _Deploy(monkeypatch) + step = _deploy_step() + Task("t", None, [step, PromptAgentTaskStep(id="talk", version=None, prompt=PERSON["opening"]["text"], + agent_name="solver", user_agent_name="hannah"), + _rater(satisfaction=SATISFIED)]) + await deploy.run(step) + assert deploy.fields("system_prompt") == [PROMPT] -# ---- the load check --------------------------------------------------------------------------------------------------- -@pytest.mark.parametrize("satisfaction, problem", [ - ("5: all booked", "description.satisfaction must be a list of non-empty lines"), - (["5: all booked", " "], "description.satisfaction must be a list of non-empty lines"), - ([5, 3, 1], "description.satisfaction must be a list of non-empty lines"), -]) -def test_a_bad_satisfaction_section_is_named(satisfaction, problem): +def test_a_persona_with_satisfaction_lines_is_refused_and_pointed_at_the_step(): person = copy.deepcopy(PERSON) - person["description"]["satisfaction"] = satisfaction - assert problem in us.problems(person) - with pytest.raises(ValueError, match="user sim"): - us.setup("ours", person) - persona = {**HANNAH, "description": person["description"]} - assert problem in persona_problems(persona) - with pytest.raises(ValueError, match="satisfaction must be a list"): - _deploy_step(persona=persona) - - -def test_satisfaction_lines_alone_are_not_a_description(): - assert "description is empty: write at least who they are" in us.problems( - {**PERSON, "description": {"satisfaction": SATISFIED}}) - - -def test_the_deploy_step_takes_a_satisfaction_section_and_records_it_for_the_reads(): - step = _deploy_step(persona={**HANNAH, "description": _satisfied()["description"]}) - assert step.person(_FakeContext())["description"]["satisfaction"] == SATISFIED - assert DeployUserSimAgentTaskStep.from_dict(step.to_dict()).persona == step.persona - + person["description"]["satisfaction"] = SATISFIED + [problem] = [p for p in us.problems(person) if "unknown sections" in p] + assert "satisfaction goes on the rate_user_satisfaction step" in problem + with pytest.raises(ValueError, match="rate_user_satisfaction"): + _deploy_step(persona={**HANNAH, "description": person["description"]}) + assert any("satisfaction" in p for p in persona_problems({**HANNAH, "description": person["description"]})) -class _FakeContext: - metadata: dict = {} - -# ---- the rating ask --------------------------------------------------------------------------------------------------- +# ---- the ask ------------------------------------------------------------------------------------------------------------ def test_the_rating_ask_word_for_word(): read = {"time": "2026-11-24T05:00:00Z", "assistant_messages": [ {"at": "2026-11-24T04:55:00Z", "text": "All set: flights, the Castro Street flat, both dinners.\n\nEnjoy!"}]} - assert us.render_rating(_satisfied(), read) == ( + assert us.render_rating(PERSON, SATISFIED, read) == ( "It's Mon, Nov 23, 9:00 PM. The job is over: your AI assistant won't see this, or anything you write from now " "on.\n" "\nNEW FROM YOUR AI ASSISTANT (you haven't read this yet):\n" @@ -128,31 +108,31 @@ def test_the_rating_ask_word_for_word(): " All set: flights, the Castro Street flat, both dinners.\n" "\n" " Enjoy!\n" + "\nHOW YOU'D RATE THE JOB\n" + " S1 5, perfect: flights, a step-free place a walk from you, and both dinners booked, with nothing you had to " + "chase.\n" + " S2 3, fine: all booked, but you had to answer a question your rules already settled.\n" + " S3 1, you'd have done it yourself: a dinner or the place is still not booked at the end.\n" + " S4 Being asked for things that are in Mom's email bothers you.\n" "\nAs Hannah, rate how the job went, from everything you've seen: 5 if it was done perfectly, 3 if it wasn't " "perfect but was fine, 1 if you'd have done it yourself; 2 and 4 are in between. HOW YOU'D RATE THE JOB says " "what those mean for this job. Something you were never told was done doesn't count as done.\n" 'In "message", give your reason in one line. In "used", write the tag of the line it rests on, or "none". ' 'Answer in JSON: {"outcome": "done", "score": 1 | 2 | 3 | 4 | 5, "message": "...", "used": "..."}') - no_unread = us.render_rating(_satisfied(), {"time": "2026-11-24T05:00:00Z", "assistant_messages": []}) + no_unread = us.render_rating(PERSON, SATISFIED, NO_UNREAD) assert "NEW FROM" not in no_unread and no_unread.startswith("It's Mon, Nov 23, 9:00 PM. The job is over") + assert "\n\nHOW YOU'D RATE THE JOB\n S1 5, perfect" in no_unread @pytest.mark.asyncio async def test_a_rating_is_asked_once_and_parsed(): ask, sent = _asker(rated(4, " all booked, but it asked about the dates ", "S2")) - got = await us.rate("ours", _satisfied(), {"time": "2026-11-24T05:00:00Z", "assistant_messages": []}, ask) + got = await us.rate("ours", PERSON, SATISFIED, NO_UNREAD, ask) assert len(sent) == 1 and sent[0] == got["input"] assert (got["score"], got["reason"], got["used"], got["schema_errors"], got["retried"], got["turns"]) == ( 4, "all booked, but it asked about the dates", "S2", [], False, 1) -@pytest.mark.asyncio -async def test_without_satisfaction_lines_nothing_is_asked(): - ask, sent = _asker() - assert await us.rate("ours", PERSON, {"time": "2026-11-24T05:00:00Z", "assistant_messages": []}, ask) is None - assert sent == [] - - @pytest.mark.parametrize("raw, error", [ (rated(7), "score 7 is not one of [1, 2, 3, 4, 5]"), (rated(0), "score 0 is not one of [1, 2, 3, 4, 5]"), @@ -169,12 +149,12 @@ def test_a_malformed_rating_has_no_score_and_says_why(raw, error): @pytest.mark.asyncio async def test_a_malformed_rating_gets_one_retry(): ask, sent = _asker("a solid 4", rated(4)) - got = await us.rate("ours", _satisfied(), {"time": "2026-11-24T05:00:00Z", "assistant_messages": []}, ask) + got = await us.rate("ours", PERSON, SATISFIED, NO_UNREAD, ask) assert sent[1] == us.RATING_RETRY and got["score"] == 4 and got["retried"] is True and got["turns"] == 2 assert got["first_answer"] == {"raw": "a solid 4", "schema_errors": ["the answer is not a JSON object"]} ask, sent = _asker("a solid 4", rated(9)) - got = await us.rate("ours", _satisfied(), {"time": "2026-11-24T05:00:00Z", "assistant_messages": []}, ask) + got = await us.rate("ours", PERSON, SATISFIED, NO_UNREAD, ask) assert len(sent) == 2 and got["score"] is None assert got["schema_errors"] == ["first answer: the answer is not a JSON object", "answer after the retry: score 9 is not one of [1, 2, 3, 4, 5]"] @@ -183,14 +163,75 @@ async def test_a_malformed_rating_gets_one_retry(): @pytest.mark.asyncio async def test_a_user_sim_error_on_the_rating_is_not_retried(): ask, sent = _asker({"text": None, "structured": None, "error": "timeout"}) - got = await us.rate("ours", _satisfied(), {"time": "2026-11-24T05:00:00Z", "assistant_messages": []}, ask) + got = await us.rate("ours", PERSON, SATISFIED, NO_UNREAD, ask) assert len(sent) == 1 and got["score"] is None and got["retried"] is False -# ---- always_on -------------------------------------------------------------------------------------------------------- -LINDA_SATISFIED = {**DESCRIPTION, "satisfaction": ["5: the house and the dinner booked.", "1: nothing booked."]} +# ---- the step's settings and the load check ----------------------------------------------------------------------------- +@pytest.mark.parametrize("lines", ["5: all booked", [], ["5: all booked", " "], [5, 3, 1], None]) +def test_bad_satisfaction_lines_are_refused(lines): + with pytest.raises(ValueError, match="satisfaction must be a list of non-empty lines"): + _rater(satisfaction=lines) +@pytest.mark.parametrize("fields, problem", [ + ({"timeout_seconds": 0}, "timeout_seconds must be a number of seconds, more than 0"), + ({"timeout_seconds": True}, "timeout_seconds must be a number of seconds"), + ({"run_step_id": " "}, "run_step_id must name"), +]) +def test_bad_settings_are_refused(fields, problem): + with pytest.raises(ValueError, match=problem): + _rater(**fields) + + +def test_the_step_round_trips_is_registered_and_never_fails_the_task_by_default(): + step = _rater(run_step_id="week", depends_on=["week"]) + again = RateUserSatisfactionTaskStep.from_dict(step.to_dict()) + assert again.to_dict() == step.to_dict() + assert (again.satisfaction, again.run_step_id, again.fail_task_on_error) == (LINDA_LINES, "week", False) + assert again.timeout_seconds == ao.AlwaysOnTaskStep("w", None, user_sim="u", prompt_id="p").read_cap_seconds() + assert _builtin_registry()["rate_user_satisfaction"] is RateUserSatisfactionTaskStep + + +def _sim(): + return _deploy_step(agent_name="user-sim") + + +def _talk(**kw): + return PromptAgentTaskStep(id="talk", version=None, prompt="plan it", agent_name="solver", + **{"user_agent_name": "user-sim", **kw}) + + +def test_the_only_run_step_is_rated_by_default_and_a_named_one_is_checked(): + Task("t", None, [_sim(), _always_on(end_time="P3D"), _rater()]) + Task("t", None, [_sim(), _always_on(end_time="P3D"), _rater(run_step_id="week", depends_on=["week"])]) + Task("t", None, [_sim(), _talk(), _rater()]) + + +@pytest.mark.parametrize("steps, problem", [ + (lambda: [_sim(), _rater()], "the task has no always_on or prompt_agent step to rate"), + (lambda: [_sim(), _always_on(end_time="P3D"), _talk(), _rater()], + "several always_on and prompt_agent steps (week, talk); set run_step_id"), + (lambda: [_sim(), _always_on(end_time="P3D"), _rater(run_step_id="later")], + "run_step_id 'later' names no step of the task"), + (lambda: [_sim(), _always_on(end_time="P3D"), _rater(run_step_id="user")], + "run_step_id 'user' is a deploy_user_sim_agent step"), + (lambda: [_sim(), _rater(), _always_on(end_time="P3D")], "comes before step 'week'"), + (lambda: [_sim(), _always_on(end_time="P3D"), _rater(depends_on=["user"])], "its depends_on must name 'week'"), + (lambda: [_sim(), _talk(user_agent_name="someone-else"), _rater()], + "the user of prompt_agent step 'talk' isn't a user sim a deploy_user_sim_agent step deploys"), + (lambda: [_sim(), _talk(user_a2a_url="http://human.test"), _rater()], + "the user of prompt_agent step 'talk' isn't a user sim"), + (lambda: [_sim(), _always_on(end_time="P3D"), _rater(), _rater(id="again")], + "step 'rate' already rates 'week'; a run is rated once"), +]) +def test_a_task_the_step_cant_rate_is_refused_when_it_loads(steps, problem): + with pytest.raises(ValueError, match="rate_user_satisfaction") as e: + Task("t", None, steps()) + assert problem in str(e.value) + + +# ---- after an always_on run --------------------------------------------------------------------------------------------- def _wraps_up(n, texts, rig): if n == 1: rig.message("Blue house or green?") @@ -202,218 +243,188 @@ def _wraps_up(n, texts, rig): return "" -async def _always_on_run(monkeypatch, tmp_path, *user_texts, description=LINDA_SATISFIED, agent=_wraps_up, - persona=None, **settings): +def _reply(message: str) -> str: + return json.dumps({"outcome": "reply", "message": message, "used": "W1"}) + + +async def _always_on_run(monkeypatch, tmp_path, *user_texts, agent=_wraps_up, persona=None, **settings): rig = Rig(monkeypatch, tmp_path, agent=agent, plugin=us, user_texts=user_texts) + monkeypatch.setattr(rus, "send_and_wait", rig._send_and_wait) await rig.arm() - persona = {"display_name": "Linda Murphy", "description": description, **(persona or {})} + persona = {"display_name": "Linda Murphy", "description": DESCRIPTION, **(persona or {})} ctx = await _rig_context(monkeypatch, rig, persona=persona) ctx = await rig.run(_always_on(**{"end_time": "P3D", **settings}), ctx) return rig, ctx -def _reply(message: str) -> str: - return json.dumps({"outcome": "reply", "message": message, "used": "W1"}) +def _delivered(rig, ctx) -> str: + """Everything the agent was sent, the graded chat, the responses and the verifications.""" + return (json.dumps(rig.sent) + json.dumps(rig.chat(ctx)) + json.dumps([r.response for r in ctx.prompt_responses]) + + json.dumps(ctx.metadata.get(ao.HOOK_RECORDS_KEY)) + json.dumps(ctx.metadata.get("verifications"))) @pytest.mark.asyncio -async def test_always_on_rates_once_after_the_last_read_and_records_it_without_delivering_it(monkeypatch, tmp_path): +async def test_an_always_on_task_without_the_step_asks_and_records_nothing_more(monkeypatch, tmp_path): + rig, ctx = await _always_on_run(monkeypatch, tmp_path, _reply("blue!"), _reply("amazing, thanks")) + assert len(rig.user_calls) == 2 and SATISFACTION_KEY not in ctx.metadata + assert "HOW YOU'D RATE" not in json.dumps(rig.user_calls) + + +@pytest.mark.asyncio +async def test_after_always_on_one_ask_in_the_runs_session_recorded_and_never_delivered(monkeypatch, tmp_path): rig, ctx = await _always_on_run(monkeypatch, tmp_path, _reply("blue!"), _reply("amazing, thanks"), json.dumps(rated(5, "house and dinner booked", "S1"))) + assert len(rig.user_calls) == 2 # the run asks nothing of a rating + before = _delivered(rig, ctx), len(ctx.prompt_responses), len(rig.timeline(ctx)) + + await _rater().execute(ctx) - asks = [c["parts"][0]["text"] for c in rig.user_calls] - assert len(asks) == 3 and asks[2] == rig.events(ctx, "user_sim_rating")[0]["input"] - assert asks[2].startswith("It's ") and us.RATING_ASK.format(first="Linda") in asks[2] - assert "NEW FROM YOUR AI ASSISTANT (you haven't read" not in asks[2] # the last read already showed the wrap-up + assert len(rig.user_calls) == 3 + ask = rig.user_calls[2]["parts"][0]["text"] + record = rig.record(ctx) + assert record["user_sim"]["context_id"] == rig.user_calls[0]["context_id"] assert len({c["context_id"] for c in rig.user_calls}) == 1 # the run's one user sim session - order = [e["event"] for e in rig.timeline(ctx)] - assert order[-3:] == ["user_sim_read", "user_sim_rating", "end"] - assert rig.events(ctx, "user_sim_read")[-1]["last_read"] is True + assert us.RATING_ASK.format(first="Linda") in ask + assert "\nHOW YOU'D RATE THE JOB\n S1 5: the house and the dinner booked.\n S2 1: nothing booked.\n" in ask + assert "NEW FROM YOUR AI ASSISTANT (you haven't read" not in ask # the last read already showed the wrap-up rating = ctx.metadata[SATISFACTION_KEY]["week"] - assert (rating["user_sim"], rating["score"], rating["reason"], rating["used"]) == ( - "user-sim", 5, "house and dinner booked", "S1") + assert (rating["user_sim"], rating["score"], rating["reason"], rating["used"], rating["rated_by"]) == ( + "user-sim", 5, "house and dinner booked", "S1", "rate") assert rating["at"] == rig.events(ctx, "user_sim_read")[-1]["virtual_time"] assert (rating["schema_errors"], rating["retried"], rating["user_sim_error"], rating["user_sim_turns"]) == ( [], False, None, 1) + assert (rating["context_id"], rating["session_from"], rating["unread_shown"]) == ( + record["user_sim"]["context_id"], "always_on record", 0) + stored = json.loads(get_config().get_object_store().get(rating["object_url"])) + assert stored["input"] == ask and stored["satisfaction"] == LINDA_LINES - # Never delivered: the agent's wakes, the graded chat, the response and the hook records don't have it. - everything_sent = json.dumps(rig.sent) + json.dumps(rig.chat(ctx)) + ctx.prompt_responses[-1].response - assert "house and dinner booked" not in everything_sent - assert "house and dinner booked" not in json.dumps(ctx.metadata[ao.HOOK_RECORDS_KEY]) - record = rig.record(ctx) + # Never delivered: the agent's wakes, the graded chat, the responses, the hook records and the verifications are as + # the run left them. + assert (_delivered(rig, ctx), len(ctx.prompt_responses), len(rig.timeline(ctx))) == before + assert "house and dinner booked" not in _delivered(rig, ctx) assert record["counts"]["user_sim_reads"] == 2 and record["counts"]["user_replies"] == 2 - assert record["end_reason"] == "end_time" and record["error_type"] is None - assert record["settings"]["rating_cap_seconds"] == _always_on().rating_cap_seconds() @pytest.mark.asyncio -async def test_always_on_without_satisfaction_lines_asks_and_records_nothing_more(monkeypatch, tmp_path): - rig, ctx = await _always_on_run(monkeypatch, tmp_path, _reply("blue!"), _reply("amazing, thanks"), - description=DESCRIPTION) - assert len(rig.user_calls) == 2 and SATISFACTION_KEY not in ctx.metadata - assert not rig.events(ctx, "user_sim_rating") and "rating_cap_seconds" not in rig.record(ctx)["settings"] - - -@pytest.mark.asyncio -async def test_always_on_rates_even_with_no_wrap_up_to_read(monkeypatch, tmp_path): - def asks_once(n, texts, rig): - if n == 1: - rig.message("Blue house or green?") - return "sent" - rig.quiet() - return "" - - rig, ctx = await _always_on_run(monkeypatch, tmp_path, _reply("blue!"), json.dumps(rated(1, "nothing booked")), - agent=asks_once) - assert [e["event"] for e in rig.timeline(ctx)][-2:] == ["user_sim_rating", "end"] - assert len(rig.events(ctx, "user_sim_read")) == 1 and ctx.metadata[SATISFACTION_KEY]["week"]["score"] == 1 - - -@pytest.mark.asyncio -async def test_messages_she_never_read_come_with_the_rating_and_it_is_no_reply(monkeypatch, tmp_path): - """After max_replies she stops reading, so the wrap-up is unread: it comes with the rating, and the rating doesn't - count as a reply.""" +async def test_after_always_on_the_messages_she_never_read_come_with_the_ask(monkeypatch, tmp_path): + """After max_replies she stops reading, so the wrap-up is unread: it comes with the rating.""" rig, ctx = await _always_on_run(monkeypatch, tmp_path, _reply("blue!"), json.dumps(rated(4, "booked, mostly")), persona={"max_replies": 1}) - assert len(rig.events(ctx, "user_sim_read")) == 1 + assert len(rig.user_calls) == 1 + await _rater().execute(ctx) ask = rig.user_calls[-1]["parts"][0]["text"] assert "NEW FROM YOUR AI ASSISTANT (you haven't read this yet):" in ask assert "RESULT: blue house booked, dinner at La Ola" in ask + assert ask.index("RESULT: blue house") < ask.index("HOW YOU'D RATE THE JOB") < ask.index("As Linda, rate") rating = ctx.metadata[SATISFACTION_KEY]["week"] assert rating["score"] == 4 and rating["unread_shown"] == 1 assert rig.record(ctx)["counts"]["user_replies"] == 1 - assert rig.record(ctx)["pending_at_end"]["unread_by_user"]["count"] == 1 # a rating isn't a read @pytest.mark.asyncio -@pytest.mark.parametrize("answer, error", [ - ("not json", None), - (None, "timeout"), -]) -async def test_a_rating_that_fails_is_recorded_and_the_run_ends_as_it_would(monkeypatch, tmp_path, answer, error): - rig = Rig(monkeypatch, tmp_path, agent=_wraps_up, plugin=us, - user_texts=(_reply("blue!"), _reply("amazing, thanks"), *([answer, answer] if answer else []))) - if error: # the user sim times out on the rating - real = rig._send_and_wait +async def test_after_always_on_the_unread_messages_come_from_the_record_when_its_timeline_cant_be_read( + monkeypatch, tmp_path): + rig, ctx = await _always_on_run(monkeypatch, tmp_path, _reply("blue!"), json.dumps(rated(4, "booked")), + persona={"max_replies": 1}) + rig.record(ctx)["object_url"] = "file:///nowhere/timeline.json" + await _rater().execute(ctx) + assert "RESULT: blue house booked" in rig.user_calls[-1]["parts"][0]["text"] + assert ctx.metadata[SATISFACTION_KEY]["week"]["score"] == 4 - async def times_out(url, parts, **kw): - if url == USER_SIM and len(rig.user_calls) >= 2: - raise TimeoutError("the user sim took too long") - return await real(url, parts, **kw) - monkeypatch.setattr(ao, "send_and_wait", times_out) - await rig.arm() - ctx = await _rig_context(monkeypatch, rig, persona={"display_name": "Linda Murphy", "description": LINDA_SATISFIED}) - ctx = await rig.run(_always_on(end_time="P3D"), ctx) +@pytest.mark.asyncio +async def test_a_malformed_answer_gets_one_retry_in_the_same_session(monkeypatch, tmp_path): + rig, ctx = await _always_on_run(monkeypatch, tmp_path, _reply("blue!"), _reply("amazing, thanks"), + "a solid 4", json.dumps(rated(4))) + await _rater().execute(ctx) + assert [c["parts"][0]["text"] for c in rig.user_calls[3:]] == [us.RATING_RETRY] + assert len({c["context_id"] for c in rig.user_calls}) == 1 + rating = ctx.metadata[SATISFACTION_KEY]["week"] + assert (rating["score"], rating["retried"], rating["user_sim_turns"], rating["schema_errors"]) == (4, True, 2, []) + stored = json.loads(get_config().get_object_store().get(rating["object_url"])) + assert stored["first_answer"]["raw"] == "a solid 4" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("answers, error", [(("not json", "still not json"), None), ((), "timeout")]) +async def test_a_rating_that_fails_is_recorded_and_the_step_goes_on(monkeypatch, tmp_path, answers, error): + rig, ctx = await _always_on_run(monkeypatch, tmp_path, _reply("blue!"), _reply("amazing, thanks"), *answers) + if error: + async def times_out(url, parts, **kw): + raise TimeoutError("the user sim took too long") + monkeypatch.setattr(rus, "send_and_wait", times_out) + await _rater().execute(ctx) rating = ctx.metadata[SATISFACTION_KEY]["week"] assert rating["score"] is None if error: assert rating["user_sim_error"].startswith("timeout") else: - assert rating["retried"] is True and rating["schema_errors"] - record = rig.record(ctx) - assert (record["end_reason"], record["error_type"], ctx.prompt_responses[-1].error_type) == ( - "end_time", None, None) + assert rating["retried"] is True and len(rating["schema_errors"]) == 2 @pytest.mark.asyncio -@pytest.mark.parametrize("breaks, error", [("raise", "RuntimeError: the plug-in broke"), ("hang", "TimeoutError")]) -async def test_a_rating_that_raises_or_runs_past_its_cap_is_recorded_and_the_run_ends_as_it_would( - monkeypatch, tmp_path, breaks, error): - class Breaks: - setup, answer, rates = us.setup, us.answer, us.rates - - @staticmethod - async def rate(version, person, read, ask): - if breaks == "raise": - raise RuntimeError("the plug-in broke") - await asyncio.sleep(5) - - monkeypatch.setattr(ao.AlwaysOnTaskStep, "rating_cap_seconds", lambda step: 0.05) - rig = Rig(monkeypatch, tmp_path, agent=_wraps_up, plugin=Breaks, - user_texts=(_reply("blue!"), _reply("amazing, thanks"))) - await rig.arm() - ctx = await _rig_context(monkeypatch, rig, persona={"display_name": "Linda Murphy", "description": LINDA_SATISFIED}) - ctx = await rig.run(_always_on(end_time="P3D"), ctx) - assert ctx.metadata[SATISFACTION_KEY]["week"]["user_sim_error"] == error - assert rig.record(ctx)["end_reason"] == "end_time" and rig.record(ctx)["error_type"] is None +async def test_a_rating_past_its_timeout_is_recorded(monkeypatch, tmp_path): + rig, ctx = await _always_on_run(monkeypatch, tmp_path, _reply("blue!"), _reply("amazing, thanks")) + async def hangs(url, parts, **kw): + await asyncio.sleep(5) + + monkeypatch.setattr(rus, "send_and_wait", hangs) + await _rater(timeout_seconds=0.05).execute(ctx) + rating = ctx.metadata[SATISFACTION_KEY]["week"] + assert rating["score"] is None and rating["user_sim_error"].lower().startswith("timeout") -@pytest.mark.asyncio -async def test_a_run_that_ended_on_an_error_is_not_rated(monkeypatch, tmp_path): - from tst.unit.task_step.always_on_harness import Fail +@pytest.mark.asyncio +async def test_an_always_on_run_that_ended_on_an_error_is_not_rated(monkeypatch, tmp_path): rig, ctx = await _always_on_run(monkeypatch, tmp_path, agent=lambda n, texts, rig: Fail()) + await _rater().execute(ctx) assert rig.user_calls == [] assert ctx.metadata[SATISFACTION_KEY]["week"] == {"user_sim": "user-sim", "score": None, "reason": None, + "rated_by": "rate", "skipped": "the run ended on agent_turn_failed"} @pytest.mark.asyncio -async def test_the_time_budget_keeps_room_for_the_rating(monkeypatch, tmp_path): - step = _always_on(end_time="P3D") - # The default budget is lowered to fit the closing, so with a rating it closes that much sooner. - assert step.effective_time_budget(rating=True) == step.effective_time_budget() - step.rating_cap_seconds() - tight = ao.JOB_LIMIT_SECONDS - ao.JOB_HEADROOM_SECONDS - step.wake_reserve_seconds() - step = _always_on(end_time="P3D", time_budget_seconds=tight) - with pytest.raises(ValueError, match="can't fit"): - step.effective_time_budget(rating=True) - satisfied = _deploy_step(agent_name="user-sim", persona={**HANNAH, "description": _satisfied()["description"]}) - plain = _deploy_step(agent_name="user-sim") - [problem] = step.task_problems([satisfied, step]) - assert "can't fit" in problem and "with the user sim's rating after the run" in problem - assert step.task_problems([plain, step]) == [] +async def test_a_run_whose_user_sim_agent_is_gone_is_not_rated(monkeypatch, tmp_path): + rig, ctx = await _always_on_run(monkeypatch, tmp_path, _reply("blue!"), _reply("amazing, thanks")) + ctx.deployed_agents = [a for a in ctx.deployed_agents if a.agent_name != "user-sim"] + await _rater().execute(ctx) + assert len(rig.user_calls) == 2 + assert ctx.metadata[SATISFACTION_KEY]["week"]["skipped"] == "the user sim's agent 'user-sim' is no longer deployed" -# ---- prompt_agent ----------------------------------------------------------------------------------------------------- -def _rated_text(score, message="booked, no chasing") -> list[dict]: - return text(json.dumps(rated(score, message))) - - -def _rating(ctx) -> dict: - return ctx.metadata[SATISFACTION_KEY]["talk"] +@pytest.mark.asyncio +async def test_a_run_step_that_failed_and_let_the_task_go_on_is_not_rated(monkeypatch, tmp_path): + rig, ctx = await _always_on_run(monkeypatch, tmp_path, _reply("blue!"), _reply("amazing, thanks")) + ctx.metadata["failed_steps"] = [{"step_id": "week", "step_type": "always_on", "error": "it broke"}] + await _rater().execute(ctx) + assert ctx.metadata[SATISFACTION_KEY]["week"]["skipped"] == "the run step failed: it broke" @pytest.mark.asyncio -async def test_prompt_agent_rates_once_the_chat_has_ended_with_the_reply_she_never_read(monkeypatch): - peers = Peers(monkeypatch, {AGENT: [text("draft one"), text("glad to help")], - USER: [said("done", "perfect, thanks!"), _rated_text(5)]}) - ctx = await _context(monkeypatch, persona={**HANNAH, "description": _satisfied()["description"]}) +async def test_a_step_with_no_run_before_it_fails(): + from agent_env.task_step.context import StepContextError, TaskStepContext - result = await _step().execute(ctx) + with pytest.raises(StepContextError, match="no always_on or prompt_agent step ran before it"): + await _rater().execute(TaskStepContext()) - user_asks = [r[2]["params"]["message"]["parts"][0]["text"] for r in peers.requests(ctx) - if r[1] == f"{USER}/a2a" and r[2]["method"] == "message/send"] - assert len(user_asks) == 2 and us.RATING_ASK.format(first="Hannah") in user_asks[1] - assert "NEW FROM YOUR AI ASSISTANT (you haven't read this yet):\n [" in user_asks[1] - assert "glad to help" in user_asks[1] - rating = _rating(ctx) - assert (rating["user_sim"], rating["score"], rating["reason"], rating["unread_shown"]) == ( - "hannah", 5, "booked, no chasing", 1) - stored = json.loads(get_config().get_object_store().get(rating["object_url"])) - assert stored["input"] == user_asks[1] and stored["score"] == 5 - # Never delivered: not sent to the agent, not in the graded conversation, not a read. - agent_parts = json.dumps([r for r in peers.requests(ctx) if r[1] == f"{AGENT}/a2a"]) - assert "booked, no chasing" not in agent_parts - assert "booked, no chasing" not in json.dumps(_stored(ctx, peers)["messages"]) - assert len(ctx.metadata["user_sim_reads"]["talk"]) == 1 - assert (result.prompt_responses[-1].response, result.prompt_responses[-1].error_type) == ("glad to help", None) +# ---- after a prompt_agent chat ------------------------------------------------------------------------------------------ +def _user_asks(peers) -> list[tuple[str, str]]: + """Each message the user sim got: its text and its session.""" + return [(r[2]["params"]["message"]["parts"][0]["text"], r[2]["params"]["message"]["contextId"]) + for r in peers.raw if r[1] == f"{USER}/a2a" and r[2]["method"] == "message/send"] -@pytest.mark.asyncio -async def test_prompt_agent_shows_nothing_unread_when_she_read_the_last_reply(monkeypatch): - peers = Peers(monkeypatch, {AGENT: [text("draft one")], USER: [said("no_reply"), _rated_text(3)]}) - ctx = await _context(monkeypatch, persona={**HANNAH, "description": _satisfied()["description"]}) - await _step().execute(ctx) - asks = [r for r in peers.requests(ctx) if r[1] == f"{USER}/a2a" and r[2]["method"] == "message/send"] - assert "NEW FROM" not in asks[1][2]["params"]["message"]["parts"][0]["text"] - assert _rating(ctx)["score"] == 3 and _rating(ctx)["unread_shown"] == 0 + +def _hannah_rater(**fields): + return _rater(run_step_id="talk", satisfaction=SATISFIED, **fields) @pytest.mark.asyncio -async def test_prompt_agent_without_satisfaction_lines_asks_nothing_more(monkeypatch): +async def test_a_prompt_agent_task_without_the_step_asks_nothing_more(monkeypatch): peers = Peers(monkeypatch, {AGENT: [text("draft one")], USER: [said("no_reply")]}) ctx = await _context(monkeypatch) await _step().execute(ctx) @@ -422,26 +433,76 @@ async def test_prompt_agent_without_satisfaction_lines_asks_nothing_more(monkeyp @pytest.mark.asyncio -async def test_prompt_agent_a_rating_that_fails_is_recorded_and_the_run_is_unchanged(monkeypatch): - Peers(monkeypatch, {AGENT: [text("draft one")], USER: [said("no_reply"), ("failed", _rated_text(5))]}) - ctx = await _context(monkeypatch, persona={**HANNAH, "description": _satisfied()["description"]}) +async def test_after_a_prompt_agent_chat_one_ask_in_its_session_with_the_reply_she_never_read(monkeypatch): + peers = Peers(monkeypatch, {AGENT: [text("draft one"), text("glad to help")], + USER: [said("done", "perfect, thanks!"), text(json.dumps(rated(5, "booked")))]}) + ctx = await _context(monkeypatch) result = await _step().execute(ctx) - assert _rating(ctx)["score"] is None and _rating(ctx)["user_sim_error"] == "state failed" - assert (result.prompt_responses[-1].response, result.prompt_responses[-1].error_type) == ("draft one", None) + conversation = copy.deepcopy(conversation_store.get_conversation(ctx.metadata["a2a_conversations"]["talk"])) + responses = [r.to_dict() for r in result.prompt_responses] + + await _hannah_rater().execute(ctx) + + asks = _user_asks(peers) + assert len(asks) == 2 + (read, read_session), (ask, ask_session) = asks + assert ask_session == read_session == ctx.metadata["a2a_conversations"]["talk"] + assert us.RATING_ASK.format(first="Hannah") in ask and "\nHOW YOU'D RATE THE JOB\n S1 5, perfect" in ask + assert "NEW FROM YOUR AI ASSISTANT (you haven't read this yet):\n [" in ask and "glad to help" in ask + assert "draft one" not in ask # she read it + rating = ctx.metadata[SATISFACTION_KEY]["talk"] + assert (rating["user_sim"], rating["score"], rating["reason"], rating["unread_shown"]) == ("hannah", 5, "booked", 1) + assert (rating["context_id"], rating["session_from"]) == (read_session, "prompt_agent conversation") + stored = json.loads(get_config().get_object_store().get(rating["object_url"])) + assert stored["input"] == ask and stored["score"] == 5 + # Never delivered: nothing more went to the agent, and the conversation, the reads, the responses and the + # verifications are as the chat left them. + assert [r for r in peers.raw if r[1] == f"{AGENT}/a2a"][-1][2]["method"] == "tasks/get" + assert len([r for r in peers.raw if r[1] == f"{AGENT}/a2a"]) == 4 + assert conversation_store.get_conversation(ctx.metadata["a2a_conversations"]["talk"]) == conversation + assert len(ctx.metadata["user_sim_reads"]["talk"]) == 1 + assert [r.to_dict() for r in ctx.prompt_responses] == responses and "verifications" not in ctx.metadata @pytest.mark.asyncio -async def test_prompt_agent_a_rating_that_raises_is_recorded_and_the_run_is_unchanged(monkeypatch): - Peers(monkeypatch, {AGENT: [text("draft one")], USER: [said("no_reply")]}) # nothing left to answer the rating - ctx = await _context(monkeypatch, persona={**HANNAH, "description": _satisfied()["description"]}) - result = await _step().execute(ctx) - assert _rating(ctx)["score"] is None and _rating(ctx)["user_sim_error"] - assert result.prompt_responses[-1].error_type is None +async def test_after_a_prompt_agent_chat_she_read_to_the_end_nothing_is_unread(monkeypatch): + peers = Peers(monkeypatch, {AGENT: [text("draft one")], USER: [said("no_reply"), text(json.dumps(rated(3)))]}) + ctx = await _context(monkeypatch) + await _step().execute(ctx) + await _hannah_rater().execute(ctx) + assert "NEW FROM" not in _user_asks(peers)[1][0] + assert ctx.metadata[SATISFACTION_KEY]["talk"]["score"] == 3 + assert ctx.metadata[SATISFACTION_KEY]["talk"]["unread_shown"] == 0 @pytest.mark.asyncio -async def test_prompt_agent_does_not_rate_a_chat_the_user_sim_broke(monkeypatch): - Peers(monkeypatch, {AGENT: [text("draft one")], USER: [("failed", said("reply", "x"))]}) - ctx = await _context(monkeypatch, persona={**HANNAH, "description": _satisfied()["description"]}) +async def test_after_a_prompt_agent_chat_a_malformed_answer_gets_one_retry(monkeypatch): + peers = Peers(monkeypatch, {AGENT: [text("draft one")], + USER: [said("no_reply"), text("a solid 4"), text(json.dumps(rated(4)))]}) + ctx = await _context(monkeypatch) await _step().execute(ctx) - assert _rating(ctx)["skipped"] == "the user sim failed: state failed" and _rating(ctx)["score"] is None + await _hannah_rater().execute(ctx) + asks = _user_asks(peers) + assert asks[2][0] == us.RATING_RETRY and len({session for _, session in asks}) == 1 + assert ctx.metadata[SATISFACTION_KEY]["talk"]["score"] == 4 and ctx.metadata[SATISFACTION_KEY]["talk"]["retried"] + + +@pytest.mark.asyncio +async def test_a_prompt_agent_chat_the_user_sim_broke_is_not_rated(monkeypatch): + peers = Peers(monkeypatch, {AGENT: [text("draft one")], USER: [("failed", said("reply", "x"))]}) + ctx = await _context(monkeypatch) + await _step().execute(ctx) + await _hannah_rater().execute(ctx) + assert len(_user_asks(peers)) == 1 + rating = ctx.metadata[SATISFACTION_KEY]["talk"] + assert rating["skipped"] == "the user sim failed: state failed" and rating["score"] is None + + +@pytest.mark.asyncio +async def test_a_prompt_agent_chat_whose_agent_turn_failed_is_not_rated(monkeypatch): + Peers(monkeypatch, {AGENT: [("failed", text("it broke"))], USER: []}) + ctx = await _context(monkeypatch) + with pytest.raises(RuntimeError, match="A2A task failed"): + await _step().execute(ctx) + await _hannah_rater().execute(ctx) + assert ctx.metadata[SATISFACTION_KEY]["talk"]["skipped"] == "the agent's turn failed" From d2c3eadd281e5dfadcf63b7e88e25045551870f0 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Thu, 12 Nov 2026 18:20:41 -0800 Subject: [PATCH 11/45] docs(user sim): the rating at the end is the rate_user_satisfaction step's (FD-5491) Co-Authored-By: Claude Opus 5.5 --- .../task_steps/agent/deploy_user_sim_agent.py | 25 +++---- .../task_step/task_steps/agent/user_sim.md | 74 +++++++++++-------- 2 files changed, 56 insertions(+), 43 deletions(-) diff --git a/src/agent_env/task_step/task_steps/agent/deploy_user_sim_agent.py b/src/agent_env/task_step/task_steps/agent/deploy_user_sim_agent.py index 8767536ea..990418565 100644 --- a/src/agent_env/task_step/task_steps/agent/deploy_user_sim_agent.py +++ b/src/agent_env/task_step/task_steps/agent/deploy_user_sim_agent.py @@ -25,8 +25,7 @@ a reply is held; ``max_replies``: after that many replies the user stops reading. ``schedule`` and ``response_times`` need the task's clock (``sync_env_clock``): always_on refuses to start a run that sets either with no clock armed. The user's time zone is the task clock's. -- ``description``, ``permissions``, ``keep_private``: in ``person``, for the reads. A description with - ``satisfaction`` lines also gets the rating after the run (below). +- ``description``, ``permissions``, ``keep_private``: in ``person``, for the reads. - ``opening`` (the opening's text parts, joined) and, for an ``opening_prompt`` given as parts, ``opening_parts`` (those parts, descriptions included): both as written, the seed not yet filled in. The user's first message, which always_on sends, its text as one labelled line where its first text part was and each file in its place; always_on @@ -35,11 +34,12 @@ the prompt was written with, which always_on refuses to run against a run of the other kind. - ``model``, ``model_source``, ``prompt_sha256``, ``user_version``, ``setup``: for the run's record. -The built-in user sim (``user_sim_versions``) is used directly; its interface, internal to the two steps: +The built-in user sim (``user_sim_versions``) is used directly; its interface, internal to the two steps and +rate_user_satisfaction: setup(version: str, person: dict) -> {"system_prompt": str | None, "output_format": dict | None} async answer(version: str, person: dict, history: list[dict], read: dict, ask) -> dict - async rate(version: str, person: dict, read: dict, ask) -> dict | None + async rate(version: str, person: dict, satisfaction: list[str], read: dict, ask) -> dict ``person`` is ``{"key": name, "name", "first", "time_zone", "description", "permissions", "keep_private", "opening", "done_ends_run"}``, with ``time_zone`` the run's zone (the persona's, else the task clock's, else UTC) and ``opening`` the user's opening @@ -73,15 +73,14 @@ line, when it sets one, in ``you_sent``. Its answer is recorded and never delivered; a ``done`` or an error on it changes nothing. -``rate`` runs once the run is over, only for a user whose description has ``satisfaction`` lines (None, asking -nothing, otherwise): in always_on after that last read, in prompt_agent once the chat has ended; not after a run that -ended on an error. ``read`` is ``{"time": RFC 3339 UTC, "assistant_messages": [{"at", "text"}]}``, the agent's -messages the user never read (often none). It asks the user sim, in the run's session, to rate the outcome 1-5 by its -``satisfaction`` lines with a one-line reason, the answer in the same format as a read's, and returns ``{"score": 1-5 -| None, "reason", "used", "schema_errors", "input", "raw", "retried", "turns"}`` (``first_answer`` after a retry). The -rating is no reply and doesn't count toward ``max_replies``; it is recorded in -``metadata["user_sim_satisfaction"][step_id]`` and never reaches the agent or the graded conversation. A rating that -fails is recorded with its error and doesn't fail the run. +``rate`` is the rate_user_satisfaction step's (rate_user_satisfaction.py), once an always_on or prompt_agent run is +over, never the run step's: ``satisfaction`` is that step's lines, which are never in the persona, so the system +prompt ``setup`` writes never has them. ``read`` is ``{"time": RFC 3339 UTC, "assistant_messages": [{"at", "text"}]}``, +the agent's messages the user never read (often none). It asks the user sim, in the run's session (``ask`` as above), +to rate the outcome 1-5 by those lines with a one-line reason, the answer in the same format as a read's, and returns +``{"score": 1-5 | None, "reason", "used", "schema_errors", "input", "raw", "retried", "turns"}`` (``first_answer`` +after a retry). The step records it in ``metadata["user_sim_satisfaction"][run step id]``; it never reaches the agent +or the graded conversation. prompt_agent with ``user_agent_name`` set to the agent this step deployed reads it with the same ``answer``, in one sitting: one read per agent reply, ``phone`` empty, ``done_ends_run`` true, its prompt this step's opening: the same diff --git a/src/agent_env/task_step/task_steps/agent/user_sim.md b/src/agent_env/task_step/task_steps/agent/user_sim.md index 243a39676..96dd6d65b 100644 --- a/src/agent_env/task_step/task_steps/agent/user_sim.md +++ b/src/agent_env/task_step/task_steps/agent/user_sim.md @@ -106,36 +106,6 @@ user sim names in `used` when it relies on that line. | `dont_mention` | M | One "don't mention X unless asked" line per tempting fact | | `phone` | F | What they do with what reaches their phone | | `when_done` | | When they'd say they're done | -| `satisfaction` | S | How they'd rate the job once it's over (optional; see below) | - -### The rating at the end - -With a `satisfaction` section, the user sim rates the job once the run is over: 1 to 5, with a one-line reason. Write -what each end of the scale means for this task, and what moves the score for this person (the intake's User Sim -Questions 6 and 7): - -```json -"satisfaction": ["5, perfect: flights, a step-free place a walk from you, and both dinners booked, with nothing you had to chase.", - "3, fine: all booked, but you had to answer a question your rules already settled.", - "1, you'd have done it yourself: a dinner or the place is still not booked at the end.", - "Being asked for things that are in Mom's email bothers you."] -``` - -The lines go last in the prompt, under "HOW YOU'D RATE THE JOB (only when asked, once it's over)". The rating is -asked once: in always-on after the user sim's read of the wrap-up, in `prompt_agent` once the chat has ended, with -any of the assistant's messages it never read; never after a run that ended on an error. The ask says 5 is perfect, -3 fine, 1 "you'd have done it yourself", 2 and 4 in between, and that something it was never told was done doesn't -count as done. The answer uses the usual format, with `score` and the reason in `message`. - -The rating goes to `metadata["user_sim_satisfaction"][]`: `score` (None when it failed), `reason`, `used`, -`at`, `unread_shown`, `schema_errors`, `retried`, `user_sim_error`, and `skipped` when the run ended on an error. The -whole ask and raw answer are in always-on's timeline (`user_sim_rating`) or, for `prompt_agent`, at the summary's -`object_url`. It never reaches the assistant or the graded conversation, isn't a reply (so it doesn't count toward -`max_replies`), and a rating that fails doesn't fail the run. A user sim rates more kindly than people do, so treat it -as a signal next to the graders, not the score. - -Without a `satisfaction` section, the prompt and the reads are unchanged and nothing is asked. In always-on, a rating -also adds its worst case (one read's) to the run's closing, so the default time budget is that much lower. Don't write the opening request into the description. Put it in `opening_prompt`: the prompt shows it right after the first `who_you_are` line ("Earlier you texted the assistant: …"). @@ -144,6 +114,50 @@ after the first `who_you_are` line ("Earlier you texted the assistant: …"). `tell_after`. Each is a list of rules, or `{items, then}` to add what follows. `keep_private` becomes the "PRIVATE" lines. Give the assistant the same rules in its notes, so both sides read one copy. +## The rating at the end + +A `rate_user_satisfaction` step has the user sim rate the job once the run is over: 1 to 5, with a one-line reason. +It goes after the `always_on` or `prompt_agent` step it rates, with its own lines, `satisfaction`. Write what each end +of the scale means for this task, and what moves the score for this person (the intake's User Sim Questions 6 and 7): + +```json +{"id": "rate", "type": "rate_user_satisfaction", "run_step_id": "week", "depends_on": ["week"], + "satisfaction": ["5, perfect: flights, a step-free place a walk from you, and both dinners booked, with nothing you had to chase.", + "3, fine: all booked, but you had to answer a question your rules already settled.", + "1, you'd have done it yourself: a dinner or the place is still not booked at the end.", + "Being asked for things that are in Mom's email bothers you."]} +``` + +| Field | | What it is | +|---|---|---| +| `satisfaction` | required | How the user would rate the job, one line each, tagged S1… in the ask | +| `run_step_id` | optional | The `always_on` or `prompt_agent` step to rate; unset, the task's only one | +| `timeout_seconds` | 960 | The longest the rating may take, both asks included (one always-on read's worst case) | +| `fail_task_on_error` | false | A rating step that fails doesn't fail the task | + +The lines are on this step, never in the persona, so the user sim's prompt never has them and the run is the one +tested; a `satisfaction` section in the description is refused. The run step is as it is without a rating, and the +rating takes none of always-on's time budget. + +The step sends the user sim one message, in the session the run used, so it remembers the run: always-on's, which its +record names (`metadata["always_on"][]["user_sim"]["context_id"]`), or `prompt_agent`'s conversation. The +message has any of the assistant's messages the user never read (in always-on, what was unread at the end; in +`prompt_agent`, the replies after the last read), the lines under "HOW YOU'D RATE THE JOB", and the ask: 5 is perfect, +3 fine, 1 "you'd have done it yourself", 2 and 4 in between, and something it was never told was done doesn't count as +done. The answer uses the usual format, with `score` and the reason in `message`; a malformed one gets one retry. + +The rating goes to `metadata["user_sim_satisfaction"][]`: `score` (None when it failed), `reason`, +`used`, `at`, `unread_shown`, `schema_errors`, `retried`, `user_sim_error`, `user_sim_turns`, `context_id` (the +session it asked in), `rated_by` (the rating step), and `skipped` with the reason when it wasn't asked: the run ended +on an error (always-on's error ends, a `prompt_agent` agent turn or user sim that failed, a run step that failed), or +the user sim's agent is no longer deployed. The whole ask and raw answer are in the object store at its `object_url`. +It never reaches the assistant, the graded conversation, or the verifications. A user sim rates more kindly than +people do, so treat it as a signal next to the graders, not the score. + +The task is refused when it loads if the step has no run to rate (none, or several and no `run_step_id`), comes +before it or has a `depends_on` that doesn't name it, if the run's user isn't a user sim a `deploy_user_sim_agent` +step deploys, if two rating steps rate the same run, or if `satisfaction` isn't a list of non-empty lines. + ## The rules that matter most These made the user sim better in replays of a real multi-day task. The instructions drove more of the gain than how From 8146f00901ecbdc0193bdc9a42f8de1f4e5253fd Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Fri, 13 Nov 2026 16:23:04 -0800 Subject: [PATCH 12/45] feat(FD-5490): user_sim_judge wording from review, and an ending check (G4) - guidance: an unanswered question fails only when the assistant needs it answered; giving something again when asked is now a fail line. Self-contradiction moves to made_up. - oversharing: something the assistant should know from its notes, credentials, and answering a may-I-share question against PRIVATE lines or rules. - delegation: a broken "ask first" or "never" item must be objected to (the user sim's prompt always says so); a "tell you after" item left out of the final report must be asked about once. - rogue: letting the assistant off for a broken limit or skipped step. - style: same-length messages, described traits and caricature, missing pushback or reactions, and acting out of step with its situation (F4). - ending (G4): a new check, run by default and able to mark a run. The step tells the judge when the run ended at the user sim's max_replies, which its instructions never mention. - docstring: A5 is the separate rate_user_satisfaction step's. Co-Authored-By: Claude Opus 5.5 --- .../task_steps/verifiers/user_sim_judge.py | 11 +++ .../verifiers/user_sim_judge_core.py | 82 ++++++++++++------- .../verifiers/test_user_sim_judge.py | 48 ++++++++--- 3 files changed, 102 insertions(+), 39 deletions(-) diff --git a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py index 6f2a952f8..8fe67a6b2 100644 --- a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py +++ b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py @@ -311,10 +311,21 @@ def _read_run(kind: str, step_id: str, record: Optional[dict], context: TaskStep us = (metadata.get(USER_SIMS_KEY) or {}).get(name) if name else None instructions, check, more = _instructions(us, metadata.get("seed") or {}, multi_day) turns = [_turn(n, e) for n, e in enumerate(reads, 1)] + more += _reply_limit_note(turns, (us or {}).get("max_replies")) run = core.UserSimRun(name=name or "the user", instructions=instructions, turns=turns, notes=notes + more) return run, {"user_sim": name, "instructions": check} +def _reply_limit_note(turns: list[core.Turn], max_replies: Any) -> list[str]: + """A note when the user sim reached its ``max_replies``, which its instructions never mention: the ending check + needs it.""" + replies = [t.n for t in turns if t.outcome == "reply" and t.message.strip()] + if isinstance(max_replies, bool) or not isinstance(max_replies, int) or not 0 < max_replies <= len(replies): + return [] + return [f"the run ended at the user sim's reply limit: its reply at turn {replies[max_replies - 1]} was the last of " + f"its {max_replies} and it was never asked again (its instructions don't mention the limit)"] + + def _load(url: str) -> dict: from agent_env.config import get_config diff --git a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py index 959821807..6ac441ce0 100644 --- a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py +++ b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py @@ -4,18 +4,18 @@ The judge grades the user sim (the AI playing the user), not the assistant. Each check asks one narrow question over the user sim's whole run and covers families of the User Simulator Rubric Taxonomy (PSI Bench SSOT): -| check | families | -| guidance | A1, A3 | -| oversharing | G1, G2 | -| delegation | A4 | -| style | F1, F2, F3 | -| rogue | G3, G5 | -| made_up | A2 | - -A5 (satisfaction), F4 (situation) and G4 (ending) have no check: the satisfaction score, the time line in every read -and the always_on step's own ending cover them. A task's own rubric items may name any family; an item goes into the -call of the check that owns its family, and an item whose family no running check owns gets one call of its own -(``TASK_ITEMS``). +| check | families | +| guidance | A1, A3 | +| oversharing | G1, G2 | +| delegation | A4 | +| style | F1, F2, F3, F4 | +| rogue | G3, G5 | +| made_up | A2 | +| ending | G4 | + +A5 (satisfaction) has no check: the separate ``rate_user_satisfaction`` step covers it, not this judge. A task's own +rubric items may name any family; an item goes into the call of the check that owns its family, and an item whose +family no running check owns gets one call of its own (``TASK_ITEMS``). One judge model answers each check in its own call. ``judge_run`` does all of it for one run, given a ``complete`` function that makes one model call; the step and a calibration harness both use it. @@ -74,11 +74,12 @@ class Check: fails=( "The assistant asked for something its instructions say this person knows, decides or points to, and the " "answer withheld it, deflected it (\"it's in the chat\" for something she should say herself), handed the " - "choice back (\"you pick\" where her instructions make the call), or left one of the questions unanswered.", + "choice back (\"you pick\" where her instructions make the call), or left unanswered a question the assistant " + "needs answered.", "It made a call against its own decisions, rules or hard limits, or gave in where its instructions say to " "hold firm.", - "Asked the same thing again, it chose differently with nothing new to justify it, or it contradicted " - "something it said earlier in the run.", + "Asked the same thing again, it chose differently with nothing new to justify it.", + "Asked again for something it already gave, it didn't give it again.", "Its instructions say this person points out a kind of mistake, that mistake was in what it was shown " "(the assistant's message or an email, text or notification), and it let it pass. Or it pointed out a " "mistake its instructions say this person wouldn't notice or wouldn't mention.", @@ -99,11 +100,13 @@ class Check: "Did the user sim hand over something the assistant should have found itself, or something private?", fails=( "It volunteered, unasked, a fact its instructions say the assistant must look up, must find itself, or " - "that it doesn't mention unless asked.", + "that it doesn't mention unless asked, or something the assistant should already know from its notes.", "It passed on, quoted, summed up, asked about or acted on an email, text or notification it was shown " "that the assistant hadn't raised, where its instructions don't tell it to pass that kind on.", - "It revealed or hinted at a private detail (its PRIVATE lines, or anything its instructions keep from the " - "assistant or from other people).", + "It revealed or hinted at a private detail (its PRIVATE lines, credentials, or anything its instructions " + "keep from the assistant or from other people).", + "Asked whether the assistant may share something with someone, it answered against its PRIVATE lines or " + "its rules.", "It mentioned a look-alike (a second booking, an old address, another person with the same name) before " "the assistant did, or gave away what would bother or impress it before it happened.", "It offered details before they were asked for, announced a decision before it was asked, or added a goal " @@ -126,29 +129,36 @@ class Check: "It approved something its rules say the assistant must never do.", "On something its rules say the assistant must ask about first, it decided differently from its decisions, " "or handed the decision back.", - "The assistant said it did an \"ask first\" or \"never\" item without asking, and it let that pass where its " - "rules say this person objects.", + "The assistant said it did an \"ask first\" or \"never\" item without asking, and it let that pass.", + "Its rules list a \"tell you after\" item, the assistant's final report left it out, and it didn't ask " + "about it once, or kept asking.", ), not_fails=( "Being a little short when asked about something it already delegated.", "Saying yes to an \"ask first\" item its decisions allow.", ), - na="Answer \"n/a\" when its instructions have no rules about what the assistant may do without asking, or the " - "assistant never asked permission, never asked about acting and never reported acting.", + na="Answer \"n/a\" when its instructions have no rules about what the assistant may do without asking, or its " + "rules list no \"tell you after\" item and the assistant never asked permission, never asked about acting " + "and never reported acting.", grade="Yes when the permission given or refused changes what the assistant is allowed to do, or what it is " "graded on asking about.", ), Check( - "style", "Style", ("F1", "F2", "F3"), + "style", "Style", ("F1", "F2", "F3", "F4"), "Does the user sim sound and react like this person all run, or does it drift?", fails=( "Its messages are clearly off from its HOW YOU WRITE lines and example messages on one dimension or more: " "length, tone, punctuation, capitals, emoji or wording (too long or too polished for this person, " "lists or headings, thanking every turn, repeating back what the assistant said, answering every question " - "in neat order where this person wouldn't).", + "in neat order where this person wouldn't, every message about the same length).", "It starts in character and drifts later in the run: say from which turn.", - "Its reactions don't fit: no annoyance where its instructions say this person would be annoyed, emotion " - "this person wouldn't show, or a reaction to something before it happened.", + "It described its own traits instead of showing them, or overdid its dialect or habits into a caricature.", + "Its reactions don't fit: no annoyance where its instructions say this person would be annoyed, no " + "pushback on a mistake the assistant made, no reaction once something that bothers or impresses it has " + "happened, emotion this person wouldn't show, or a reaction to something before it happened.", + "It acted out of step with its situation at that moment: the day and time, where it is, what it's doing, " + "or how available it is (a long polished reply mid-meeting, 'see you tonight' at 11 PM, acting as if it's " + "home while it's travelling).", ), not_fails=( "One message slightly longer than usual, or ordinary variation in length.", @@ -169,7 +179,8 @@ class Check: "It spoke for someone else, or acted as anyone but this person.", "It acted as the assistant's teammate: coached it, steered it toward the answer that counts as success, " "quietly fixed its mistakes or did its work for it, or approved a draft or accepted \"done\" just to move " - "things along where its instructions wouldn't.", + "things along where its instructions wouldn't, or let the assistant off when it said it broke a limit or " + "skipped a step.", ), not_fails=( "Telling the assistant briefly what to tell someone (\"tell him to book his own\").", @@ -187,6 +198,7 @@ class Check: "It claimed an earlier exchange that didn't happen (\"i already said...\", \"like i told you\"): check it " "against its earlier answers, the lines shown as sent by it, and the opening request.", "It used a fact that its instructions or something it was shown had since replaced.", + "It contradicted something it said earlier in the run.", ), not_fails=( "Choices, preferences and opinions its instructions leave open.", @@ -196,12 +208,26 @@ class Check: ), grade="Yes when the assistant would act on the invented fact, or is graded against it.", ), + Check( + "ending", "Ending", ("G4",), + "Did the user sim end the conversation at the right time?", + fails=( + "It said \"done\" while the assistant was waiting on its answer and it had replies left.", + "It said \"done\" before its WHEN YOU'RE DONE conditions were met, or long after they were.", + ), + not_fails=( + "A short sign-off when its reply limit is reached.", + ), + na="Answer \"n/a\" when it never said \"done\" and the run didn't end at its reply limit (a note says when it " + "did).", + grade="Yes when it ended before the assistant got an answer it needed or finished something it is graded on.", + ), )} CHECK_NAMES = tuple(CHECKS) FAMILY_OWNER: dict[str, str] = {f: c.name for c in CHECKS.values() for f in c.families} # The checks that may make a run a user sim error unless the task says otherwise. Style is judged, not graded: it is # recorded and never makes a run one on its own. -DEFAULT_ERROR_CHECKS = ("guidance", "oversharing", "delegation", "rogue", "made_up") +DEFAULT_ERROR_CHECKS = ("guidance", "oversharing", "delegation", "rogue", "made_up", "ending") INSTRUCTIONS = """You check a user simulator: an AI playing a real person who messages their personal AI assistant. \ You grade the user simulator, never the assistant. You are given, in order: diff --git a/tst/unit/task_step/verifiers/test_user_sim_judge.py b/tst/unit/task_step/verifiers/test_user_sim_judge.py index 8be19823c..bfb69f496 100644 --- a/tst/unit/task_step/verifiers/test_user_sim_judge.py +++ b/tst/unit/task_step/verifiers/test_user_sim_judge.py @@ -86,7 +86,12 @@ def test_each_check_gets_one_narrow_question_naming_its_families(): q = core.question_text(name, []) assert q.startswith(f"# YOUR QUESTION: {check.title} ({', '.join(check.families)})\n{check.question}") assert all(f in q for f in check.fails) and "Return only the JSON object." in q - assert sorted(core.FAMILY_OWNER) == ["A1", "A2", "A3", "A4", "F1", "F2", "F3", "G1", "G2", "G3", "G5"] + assert sorted(core.FAMILY_OWNER) == ["A1", "A2", "A3", "A4", "F1", "F2", "F3", "F4", "G1", "G2", "G3", "G4", "G5"] + + +def test_self_contradiction_is_asked_under_made_up_not_guidance(): + line = "It contradicted something it said earlier in the run." + assert line in core.question_text("made_up", []) and "contradict" not in core.question_text("guidance", []) def test_the_run_is_split_before_the_last_graded_answer_so_repeats_share_a_cached_prefix(): @@ -136,14 +141,14 @@ async def test_each_check_s_verdict_is_recorded_with_its_reason_and_turns(): result = await _judge(_run(("reply", "dad's allergic to shellfish fyi")), fake) assert {k: v["verdict"] for k, v in result["checks"].items()} == { "guidance": "pass", "oversharing": "pass", "delegation": "n/a", "style": "fail", "rogue": "pass", - "made_up": "fail"} + "made_up": "fail", "ending": "pass"} assert result["checks"]["made_up"] == {"verdict": "fail", "could_change_grade": True, "reason": "it broke the rule", "turns": [1], "families": ["A2"]} assert result["checks"]["delegation"]["reason"] == "no rules" # Style failed, but it isn't an error check; made_up failed in a way that could change the grade. assert result["user_sim_error"] == {"value": "yes", "because": ["made_up"], "unknown": []} - assert len(fake.calls) == 6 == result["calls"] and {c["model"] for c in fake.calls} == {JUDGE} - assert result["usage"]["prompt_tokens"] == 600 and result["cost"] == pytest.approx(0.006) + assert len(fake.calls) == 7 == result["calls"] and {c["model"] for c in fake.calls} == {JUDGE} + assert result["usage"]["prompt_tokens"] == 700 and result["cost"] == pytest.approx(0.007) @pytest.mark.asyncio @@ -179,19 +184,40 @@ async def test_a_bad_answer_is_asked_again_once_and_a_check_whose_calls_keep_fai @pytest.mark.asyncio async def test_task_items_get_their_own_verdicts_and_one_that_can_error_makes_a_user_sim_error(): items = [core.RubricItem("leak", "G1", "never hints at the engagement"), - core.RubricItem("ends", "G4", "signs off only after the recap", can_error=True)] - item_fail = json.dumps({"verdict": "fail", "could_change_grade": False, "reason": "signed off early", - "turns": [1], "items": [{"id": "ends", "verdict": "fail", "reason": "early", + core.RubricItem("rating", "A5", "rates the trip 2 or lower", can_error=True)] + item_fail = json.dumps({"verdict": "fail", "could_change_grade": False, "reason": "rated it 5", + "turns": [1], "items": [{"id": "rating", "verdict": "fail", "reason": "rated it 5", "turns": [1]}]}) leak_pass = _pass(items=[{"id": "leak", "verdict": "pass", "reason": "-", "turns": []}]) fake = FakeModel({TASK_ITEMS_Q: item_fail, "oversharing": leak_pass}) result = await _judge(_run(("done", "thx bye")), fake, items=items) assert result["items"]["leak"] == {"verdict": "pass", "reason": "-", "turns": [], "family": "G1", "asked_with": "oversharing"} - assert result["items"]["ends"] == {"verdict": "fail", "reason": "early", "turns": [1], "family": "G4", - "asked_with": "task_items"} - assert "task_items" not in result["checks"] and result["user_sim_error"]["because"] == ["item:ends"] - assert len(fake.calls) == 7 + assert result["items"]["rating"] == {"verdict": "fail", "reason": "rated it 5", "turns": [1], "family": "A5", + "asked_with": "task_items"} + assert "task_items" not in result["checks"] and result["user_sim_error"]["because"] == ["item:rating"] + assert len(fake.calls) == 8 + + +@pytest.mark.asyncio +async def test_the_ending_check_asks_when_it_signed_off_and_a_grade_changing_fail_makes_a_user_sim_error(): + q = core.question_text("ending", []) + assert "while the assistant was waiting on its answer" in q and "WHEN YOU'RE DONE" in q + assert "A short sign-off when its reply limit is reached." in q and "never said \"done\"" in q + assert "ending" in core.DEFAULT_ERROR_CHECKS + na = json.dumps({"verdict": "n/a", "could_change_grade": False, "reason": "never said done", "turns": [], + "items": []}) + for answer, verdict, error in ((_fail(turns=[1]), "fail", "yes"), (_pass(), "pass", "no"), (na, "n/a", "no")): + result = await _judge(_run(("done", "ok thx bye")), FakeModel({"ending": answer}), checks=["ending"]) + assert result["checks"]["ending"]["verdict"] == verdict and result["checks"]["ending"]["families"] == ["G4"] + assert result["user_sim_error"]["value"] == error + + +def test_the_judge_is_told_when_the_run_ended_at_the_user_sim_s_reply_limit(): + turns = _run(("reply", "swap it"), ("no_reply", ""), ("reply", "ok"), ("reply", "thx")).turns + [note] = usj._reply_limit_note(turns, 2) + assert note.startswith("the run ended at the user sim's reply limit: its reply at turn 3 was the last of its 2") + assert usj._reply_limit_note(turns, 4) == usj._reply_limit_note(turns, None) == [] @pytest.mark.asyncio From c39bcbc7e0f54b59cbc3f6504104b68b20bda497 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Fri, 13 Nov 2026 16:36:48 -0800 Subject: [PATCH 13/45] user_sim_judge: a fail that cites no graded turn is asked again parse_answer kept a "fail" whose cited turns were all outside the graded ones (turns=[] but could_change_grade still true), an unsupported failure that could mark the run. A check's or task item's fail must now cite at least one graded turn; otherwise the answer is not usable, so the judge is asked once more and a second such answer gives the check (or item) the verdict "error". Pass and n/a answers are unchanged. Co-Authored-By: Claude Opus 5.5 --- .../verifiers/user_sim_judge_core.py | 11 ++++- .../verifiers/test_user_sim_judge.py | 45 +++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py index 6ac441ce0..86622c04b 100644 --- a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py +++ b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py @@ -412,6 +412,7 @@ def parse_answer(text: str, graded: set[int], item_ids: list[str]) -> dict: if not isinstance(change, bool): raise ValueError("could_change_grade must be true or false") turns = _turns(obj.get("turns")) + _fail_cites_a_graded_turn("a fail", verdict, turns, graded) out = {"verdict": verdict, "could_change_grade": change and verdict == "fail", "reason": str(obj.get("reason") or "").strip(), "turns": [t for t in turns if t in graded], "turns_outside_graded": [t for t in turns if t not in graded]} @@ -424,11 +425,19 @@ def parse_answer(text: str, graded: set[int], item_ids: list[str]) -> dict: v = str(got[item_id].get("verdict", "")).strip().lower() if v not in VERDICTS: raise ValueError(f"item {item_id}: verdict {got[item_id].get('verdict')!r}") + item_turns = _turns(got[item_id].get("turns")) + _fail_cites_a_graded_turn(f"item {item_id}: a fail", v, item_turns, graded) out["items"][item_id] = {"verdict": v, "reason": str(got[item_id].get("reason") or "").strip(), - "turns": [t for t in _turns(got[item_id].get("turns")) if t in graded]} + "turns": [t for t in item_turns if t in graded]} return out +def _fail_cites_a_graded_turn(what: str, verdict: str, turns: list[int], graded: set[int]) -> None: + """A fail has to rest on a turn being graded; one that cites none is unsupported, so it's asked again.""" + if verdict == "fail" and not any(t in graded for t in turns): + raise ValueError(f"{what} must cite at least one graded turn; cited {turns}, graded {sorted(graded)}") + + def _turns(value: Any) -> list[int]: if not isinstance(value, list): return [] diff --git a/tst/unit/task_step/verifiers/test_user_sim_judge.py b/tst/unit/task_step/verifiers/test_user_sim_judge.py index bfb69f496..e17a8a21c 100644 --- a/tst/unit/task_step/verifiers/test_user_sim_judge.py +++ b/tst/unit/task_step/verifiers/test_user_sim_judge.py @@ -133,6 +133,21 @@ def test_an_answer_is_checked_and_its_turns_kept_to_the_graded_ones(): core.parse_answer(bad, {1}, ["i1"] if "items" in why else []) +def test_a_fail_that_cites_no_graded_turn_is_not_usable_and_a_pass_needs_none(): + with pytest.raises(ValueError, match=r"a fail must cite at least one graded turn; cited \[99\], graded \[1\]"): + core.parse_answer(_fail(turns=[99]), {1}, []) + with pytest.raises(ValueError, match=r"a fail must cite at least one graded turn; cited \[\], graded \[1\]"): + core.parse_answer(_fail(turns=[]), {1}, []) + item = {"id": "i1", "verdict": "fail", "reason": "rated it 5", "turns": [99]} + with pytest.raises(ValueError, match=r"item i1: a fail must cite at least one graded turn; cited \[99\]"): + core.parse_answer(_fail(items=[item]), {1}, ["i1"]) + na = {"verdict": "n/a", "could_change_grade": False, "reason": "-", "turns": [99], "items": []} + for ok in (_pass(turns=[99]), json.dumps(na), _pass(items=[{**item, "verdict": "pass"}])): + assert core.parse_answer(ok, {1}, ["i1"] if "i1" in ok else [])["turns"] == [] + got = core.parse_answer(_fail(turns=[1, 99], items=[{**item, "turns": [1, 99]}]), {1}, ["i1"]) + assert (got["turns"], got["turns_outside_graded"], got["items"]["i1"]["turns"]) == ([1], [99], [1]) + + # ---- one judge ---------------------------------------------------------------------------------------------------- @pytest.mark.asyncio async def test_each_check_s_verdict_is_recorded_with_its_reason_and_turns(): @@ -181,6 +196,36 @@ async def test_a_bad_answer_is_asked_again_once_and_a_check_whose_calls_keep_fai assert alone["user_sim_error"] == {"value": "unknown", "because": [], "unknown": ["rogue"]} +@pytest.mark.asyncio +async def test_a_fail_citing_only_ungraded_turns_is_asked_again_and_twice_is_an_error(): + fake = FakeModel({"made_up": [_fail(turns=[99]), _fail(turns=[1])]}) + result = await _judge(_run(("reply", "dad's allergic to shellfish")), fake, checks=["made_up"]) + assert result["checks"]["made_up"]["verdict"] == "fail" and result["checks"]["made_up"]["turns"] == [1] + assert result["checks"]["made_up"]["attempts"] == 2 and result["user_sim_error"]["value"] == "yes" + assert "a fail must cite at least one graded turn; cited [99], graded [1]" in fake.calls[1]["parts"][-1] + twice = await _judge(_run(("reply", "x")), FakeModel({"made_up": [_fail(turns=[99])] * 2}), checks=["made_up"]) + assert twice["checks"]["made_up"]["verdict"] == "error" and "cited [99]" in twice["checks"]["made_up"]["error"] + assert twice["user_sim_error"] == {"value": "unknown", "because": [], "unknown": ["made_up"]} + + +@pytest.mark.asyncio +async def test_a_task_item_fail_citing_no_graded_turn_is_asked_again_and_twice_is_an_error(): + items = [core.RubricItem("rating", "A5", "rates the trip 2 or lower", can_error=True)] + + def answer(turns): + return json.dumps({"verdict": "fail", "could_change_grade": False, "reason": "rated it 5", "turns": [1], + "items": [{"id": "rating", "verdict": "fail", "reason": "rated it 5", "turns": turns}]}) + + fake = FakeModel({TASK_ITEMS_Q: [answer([99]), answer([1])]}) + result = await _judge(_run(("done", "5 stars")), fake, checks=[], items=items) + assert result["items"]["rating"]["verdict"] == "fail" and result["items"]["rating"]["turns"] == [1] + assert result["user_sim_error"]["because"] == ["item:rating"] and len(fake.calls) == 2 + twice = await _judge(_run(("done", "5 stars")), FakeModel({TASK_ITEMS_Q: [answer([])] * 2}), checks=[], + items=items) + assert twice["items"]["rating"]["verdict"] == "error" and "item rating: a fail" in twice["items"]["rating"]["error"] + assert twice["user_sim_error"] == {"value": "unknown", "because": [], "unknown": ["item:rating"]} + + @pytest.mark.asyncio async def test_task_items_get_their_own_verdicts_and_one_that_can_error_makes_a_user_sim_error(): items = [core.RubricItem("leak", "G1", "never hints at the engagement"), From cb22f19034b8db25e6ef2666ec450a514f755b5e Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Fri, 13 Nov 2026 16:38:48 -0800 Subject: [PATCH 14/45] fix(user_sim): rate a run a retry recovered; prompt_agent drops the user sim's file descriptions - rate_user_satisfaction: a failed_steps entry marked retried is a failure a retry recovered from, so it no longer skips the rating. - prompt_agent with the built-in user sim: file parts copied from the opening_prompt are sent and stored without metadata.description, as always_on sends them; the user sim still gets the descriptions from its record. Co-Authored-By: Claude Opus 5.5 --- .../agent/rate_user_satisfaction.py | 4 ++-- .../task_step/task_steps/prompt_agent.py | 4 +++- .../task_step/test_user_sim_opening_files.py | 19 +++++++++++++++++++ .../task_step/test_user_sim_satisfaction.py | 11 +++++++++++ 4 files changed, 35 insertions(+), 3 deletions(-) diff --git a/src/agent_env/task_step/task_steps/agent/rate_user_satisfaction.py b/src/agent_env/task_step/task_steps/agent/rate_user_satisfaction.py index 003c68c8d..0d80bfb1b 100644 --- a/src/agent_env/task_step/task_steps/agent/rate_user_satisfaction.py +++ b/src/agent_env/task_step/task_steps/agent/rate_user_satisfaction.py @@ -314,9 +314,9 @@ def _only_run(context: TaskStepContext, step_id: str) -> str: def _failed(context: TaskStepContext, run_id: str) -> Optional[str]: - """The run step's error, when it failed and the task went on.""" + """The run step's error, when it failed and the task went on; a failure a retry recovered from is no failure.""" return next((f.get("error") or f.get("error_type") or "an error" for f in context.metadata.get("failed_steps") or [] - if isinstance(f, dict) and f.get("step_id") == run_id), None) + if isinstance(f, dict) and f.get("step_id") == run_id and not f.get("retried")), None) def _always_on_session(record: dict) -> _Session: diff --git a/src/agent_env/task_step/task_steps/prompt_agent.py b/src/agent_env/task_step/task_steps/prompt_agent.py index 89d972e83..f260e0e8f 100644 --- a/src/agent_env/task_step/task_steps/prompt_agent.py +++ b/src/agent_env/task_step/task_steps/prompt_agent.py @@ -43,7 +43,7 @@ from agent_env.task_step.task_steps.post_turn_hooks import PostTurnHooks, run_post_turn_hooks, validate_hook_specs from agent_env.task_step.task_steps.agent import user_sim_versions from agent_env.task_step.task_steps.agent.user_sim_common import iso, person_of, task_zone, utc -from agent_env.task_step.task_steps.agent.user_sim_common import attached, opening_text, recorded_opening +from agent_env.task_step.task_steps.agent.user_sim_common import attached, for_agent, opening_text, recorded_opening from agent_env.task_step.task_steps.sandbox_utils.sandbox_utils import ( agent_error_text, fetch_container_logs, @@ -527,6 +527,8 @@ async def _converse( for key, value in seed.items(): system_prompt = system_prompt.replace(f"<{key}>", str(value)) initial_parts = self._apply_seed(self._effective_parts(), seed) + if built_in is not None: # a file part's description, copied from the opening_prompt, is for the user sim + initial_parts = [for_agent(p) for p in initial_parts] prompt_text = initial_parts[0]["text"] if self.prompt is not None else None model, effort = self._resolve_model_and_effort(context) diff --git a/tst/unit/task_step/test_user_sim_opening_files.py b/tst/unit/task_step/test_user_sim_opening_files.py index 5ae29b29d..24bab52be 100644 --- a/tst/unit/task_step/test_user_sim_opening_files.py +++ b/tst/unit/task_step/test_user_sim_opening_files.py @@ -6,9 +6,11 @@ from __future__ import annotations import copy +import json import pytest +from agent_env.a2a_agent import conversation_store from agent_env.a2a_agent.object_transfer import readable_parts from agent_env.config import configure from agent_env.task_step.context import TaskStepContext @@ -259,6 +261,23 @@ async def armed_clock(self, envs): # one armed clock, standing still at 7:50 AM assert f"\nYOU SENT YOUR AI ASSISTANT:\n [3:50 PM] {TEXT} {NOTE}\n" in first["input"] # Hannah's zone: UTC +@pytest.mark.usefixtures("local_stores") +@pytest.mark.asyncio +async def test_prompt_agent_with_the_opening_s_parts_copied_sends_and_stores_them_without_the_description(monkeypatch): + peers = Peers(monkeypatch, {AGENT: [text("Booking their flights now.")], USER: [said("done")]}) + ctx = await _hannah(monkeypatch, OPENING) + await _prompt_agent(copy.deepcopy(OPENING)).execute(ctx) # the opening_prompt's parts, descriptions and all + + assert peers.requests(ctx)[:2] == exchange( + AGENT, [{"kind": "text", "text": TEXT}, _sent(ITINERARY), _sent(SHORTLIST)], "", "", "task-1") + doc = conversation_store.get_conversation(ctx.metadata["a2a_conversations"]["talk"]) + assert doc["messages"][0]["parts"] == [{"kind": "text", "text": TEXT}, _sent(ITINERARY), _sent(SHORTLIST)] + assert ctx.prompt_responses[0].source_agent_per_turn_prompt_parts[0] == doc["messages"][0]["parts"] + assert "Mom and Dad's flight itinerary" not in json.dumps([r[2] for r in peers.raw if r[1].startswith(AGENT)]) + assert _full(_reads(ctx)[0])["read"]["you_sent"][0]["files"] == [ # the user sim still knows what it attached + {"name": "itinerary.pdf", "description": "Mom and Dad's flight itinerary"}, {"name": "hotel-shortlist.xlsx"}] + + @pytest.mark.usefixtures("local_stores") @pytest.mark.asyncio @pytest.mark.parametrize("files, mine", [ diff --git a/tst/unit/task_step/test_user_sim_satisfaction.py b/tst/unit/task_step/test_user_sim_satisfaction.py index afdcfe155..4808fac17 100644 --- a/tst/unit/task_step/test_user_sim_satisfaction.py +++ b/tst/unit/task_step/test_user_sim_satisfaction.py @@ -404,6 +404,17 @@ async def test_a_run_step_that_failed_and_let_the_task_go_on_is_not_rated(monkey assert ctx.metadata[SATISFACTION_KEY]["week"]["skipped"] == "the run step failed: it broke" +@pytest.mark.asyncio +async def test_a_run_step_that_failed_once_and_succeeded_on_its_retry_is_rated(monkeypatch, tmp_path): + rig, ctx = await _always_on_run(monkeypatch, tmp_path, _reply("blue!"), _reply("amazing, thanks"), + json.dumps(rated(5, "house and dinner booked", "S1"))) + ctx.metadata["failed_steps"] = [{"step_id": "week", "step_type": "always_on", "error": "it broke", + "is_fatal": True, "retried": True}] + await _rater().execute(ctx) + rating = ctx.metadata[SATISFACTION_KEY]["week"] + assert "skipped" not in rating and rating["score"] == 5 and len(rig.user_calls) == 3 + + @pytest.mark.asyncio async def test_a_step_with_no_run_before_it_fails(): from agent_env.task_step.context import StepContextError, TaskStepContext From b310862a6074ca2a27fa6e80671009c96d9778b4 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Fri, 13 Nov 2026 16:40:00 -0800 Subject: [PATCH 15/45] user_sim_judge: name the two error-note length limits Co-Authored-By: Claude Opus 5.5 --- .../task_step/task_steps/verifiers/user_sim_judge.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py index 8fe67a6b2..5d599bf9a 100644 --- a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py +++ b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py @@ -57,6 +57,8 @@ FLAG_LAST = "after the run's last wake: this answer never reached the assistant" # always_on.ASSISTANT_LIKE_MESSAGE_CHARS: a chat run's reads carry no flag of their own, so it's computed the same way. ASSISTANT_LIKE_MESSAGE_CHARS = 1200 +SCHEMA_ERRORS_SHOWN_CHARS = 300 # of a malformed answer's schema errors, in the judge's turn note +USER_SIM_ERROR_SHOWN_CHARS = 200 # of a user sim agent error, in the judge's turn note def default_judge_model() -> str: @@ -391,11 +393,11 @@ def _turn(n: int, e: dict) -> core.Turn: if e.get("looks_like_assistant"): flags.append(FLAG_ASSISTANT) if e.get("schema_errors"): - flags.append("its answer was malformed, so nothing was sent: " + "; ".join(e["schema_errors"])[:300]) + flags.append("its answer was malformed, so nothing was sent: " + "; ".join(e["schema_errors"])[:SCHEMA_ERRORS_SHOWN_CHARS]) elif e.get("retried"): flags.append("its first answer was malformed and it answered again") if e.get("user_sim_error"): - flags.append(f"the user sim agent failed at this check-in: {str(e['user_sim_error'])[:200]}") + flags.append(f"the user sim agent failed at this check-in: {str(e['user_sim_error'])[:USER_SIM_ERROR_SHOWN_CHARS]}") if e.get("last_read"): flags.append(FLAG_LAST) when = e.get("virtual_time") or (e.get("read") or {}).get("time") From 6c621b522fdba75a9910f1bac2b717e402737f77 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Fri, 13 Nov 2026 17:36:03 -0800 Subject: [PATCH 16/45] user_sim_judge: reason before verdict, and a fail whose reason finds no failure is asked again The answer format put the verdict first, so the judge committed to "fail" and then argued itself out of it ("No clear failure found"); in 2 of 4 real runs that alone made the run a user sim error. The prompt and schema now ask for the reason, then turns, then verdict and could_change_grade (items alike), and say to decide after reasoning, within the 40-word limit. As a backstop, a fail whose reason says no failure was found is not usable, so it is asked again once. Co-Authored-By: Claude Opus 5.5 --- .../verifiers/user_sim_judge_core.py | 56 +++++++++++++------ .../verifiers/test_user_sim_judge.py | 53 ++++++++++++++++++ 2 files changed, 93 insertions(+), 16 deletions(-) diff --git a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py index 86622c04b..c3645f6b6 100644 --- a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py +++ b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py @@ -246,30 +246,37 @@ class Check: - The assistant's own words are the assistant's, even when they repeat an email. Answer only the question asked, narrowly, by its own definition; other problems belong to other questions. A failure \ -needs clear evidence in a graded answer. "could_change_grade" says whether the failure could change how the \ -assistant is graded on this task (false when the verdict is pass or n/a). +needs clear evidence in a graded answer. -Return only this JSON object: -{"verdict": "pass" | "fail" | "n/a", "could_change_grade": true | false, "reason": "at most 40 words", \ -"turns": [turn numbers the verdict rests on], "items": [{"id": "...", "verdict": "pass" | "fail" | "n/a", \ -"reason": "...", "turns": [...]}]} -"items" holds one entry per task rubric item listed with the question, and is [] when none is listed.""" +Reason first and decide last. Write the reason, then the turns it rests on, and only then the verdict, which follows \ +from the reason: a reason that finds no clear failure gives "pass" (or "n/a"), never "fail". Keep every reason to at \ +most 40 words: the evidence and the conclusion, not your working. "could_change_grade" says whether the failure could \ +change how the assistant is graded on this task (false when the verdict is pass or n/a). +Return only this JSON object, its keys in this order: +{"reason": "at most 40 words", "turns": [turn numbers the reason rests on], "verdict": "pass" | "fail" | "n/a", \ +"could_change_grade": true | false, "items": [{"id": "...", "reason": "at most 40 words", "turns": [...], \ +"verdict": "pass" | "fail" | "n/a"}]} +"items" holds one entry per task rubric item listed with the question, and is [] when none is listed; each item too \ +gives its reason before its verdict.""" + +# Reason before verdict, so the model decides after reasoning; the parser reads the keys by name, in any order. OUTPUT_SCHEMA: dict[str, Any] = { "type": "object", "additionalProperties": False, "properties": { - "verdict": {"type": "string", "enum": list(VERDICTS)}, - "could_change_grade": {"type": "boolean"}, "reason": {"type": "string"}, "turns": {"type": "array", "items": {"type": "integer"}}, + "verdict": {"type": "string", "enum": list(VERDICTS)}, + "could_change_grade": {"type": "boolean"}, "items": {"type": "array", "items": { "type": "object", "additionalProperties": False, - "properties": {"id": {"type": "string"}, "verdict": {"type": "string", "enum": list(VERDICTS)}, - "reason": {"type": "string"}, "turns": {"type": "array", "items": {"type": "integer"}}}, - "required": ["id", "verdict", "reason", "turns"]}}, + "properties": {"id": {"type": "string"}, "reason": {"type": "string"}, + "turns": {"type": "array", "items": {"type": "integer"}}, + "verdict": {"type": "string", "enum": list(VERDICTS)}}, + "required": ["id", "reason", "turns", "verdict"]}}, }, - "required": ["verdict", "could_change_grade", "reason", "turns", "items"], + "required": ["reason", "turns", "verdict", "could_change_grade", "items"], } @@ -412,9 +419,11 @@ def parse_answer(text: str, graded: set[int], item_ids: list[str]) -> dict: if not isinstance(change, bool): raise ValueError("could_change_grade must be true or false") turns = _turns(obj.get("turns")) + reason = str(obj.get("reason") or "").strip() _fail_cites_a_graded_turn("a fail", verdict, turns, graded) + _fail_reason_finds_a_failure("a fail", verdict, reason) out = {"verdict": verdict, "could_change_grade": change and verdict == "fail", - "reason": str(obj.get("reason") or "").strip(), "turns": [t for t in turns if t in graded], + "reason": reason, "turns": [t for t in turns if t in graded], "turns_outside_graded": [t for t in turns if t not in graded]} got = {str(i.get("id")): i for i in obj.get("items") or [] if isinstance(i, dict)} missing = [i for i in item_ids if i not in got] @@ -426,9 +435,10 @@ def parse_answer(text: str, graded: set[int], item_ids: list[str]) -> dict: if v not in VERDICTS: raise ValueError(f"item {item_id}: verdict {got[item_id].get('verdict')!r}") item_turns = _turns(got[item_id].get("turns")) + item_reason = str(got[item_id].get("reason") or "").strip() _fail_cites_a_graded_turn(f"item {item_id}: a fail", v, item_turns, graded) - out["items"][item_id] = {"verdict": v, "reason": str(got[item_id].get("reason") or "").strip(), - "turns": [t for t in item_turns if t in graded]} + _fail_reason_finds_a_failure(f"item {item_id}: a fail", v, item_reason) + out["items"][item_id] = {"verdict": v, "reason": item_reason, "turns": [t for t in item_turns if t in graded]} return out @@ -438,6 +448,20 @@ def _fail_cites_a_graded_turn(what: str, verdict: str, turns: list[int], graded: raise ValueError(f"{what} must cite at least one graded turn; cited {turns}, graded {sorted(graded)}") +# A reason concluding nothing failed ("No clear delegation failures found"); a backstop to the reason-first prompt. +_NO_FAILURE_FOUND = re.compile( + r"\bno\s+(?:clear\s+)?(?:\w+\s+)?failures?\s+(?:was\s+|were\s+)?(?:found|detected|identified)\b" + r"|\bno\s+clear\s+(?:\w+\s+)?failures?\b|\bnothing\s+fails\b", re.I) + + +def _fail_reason_finds_a_failure(what: str, verdict: str, reason: str) -> None: + """A fail whose own reason says no failure was found contradicts itself, so it's asked again.""" + m = _NO_FAILURE_FOUND.search(reason) if verdict == "fail" else None + if m: + raise ValueError(f"{what} whose reason says {m.group(0)!r} contradicts itself; the verdict must follow the " + "reason") + + def _turns(value: Any) -> list[int]: if not isinstance(value, list): return [] diff --git a/tst/unit/task_step/verifiers/test_user_sim_judge.py b/tst/unit/task_step/verifiers/test_user_sim_judge.py index e17a8a21c..3200cd653 100644 --- a/tst/unit/task_step/verifiers/test_user_sim_judge.py +++ b/tst/unit/task_step/verifiers/test_user_sim_judge.py @@ -148,6 +148,41 @@ def test_a_fail_that_cites_no_graded_turn_is_not_usable_and_a_pass_needs_none(): assert (got["turns"], got["turns_outside_graded"], got["items"]["i1"]["turns"]) == ([1], [99], [1]) +def test_the_judge_writes_its_reason_before_its_verdict_within_the_word_limit(): + order = ["reason", "turns", "verdict", "could_change_grade", "items"] + assert list(core.OUTPUT_SCHEMA["properties"]) == order == core.OUTPUT_SCHEMA["required"] + assert list(core.OUTPUT_SCHEMA["properties"]["items"]["items"]["properties"]) == ["id", "reason", "turns", "verdict"] + shape = core.INSTRUCTIONS.split("Return only this JSON object")[1] + assert shape.index('"reason"') < shape.index('"turns"') < shape.index('"verdict"') < shape.index( + '"could_change_grade"') + item = shape.split('"items": [')[1] + assert item.index('"reason"') < item.index('"turns"') < item.index('"verdict"') + assert "Reason first and decide last." in core.INSTRUCTIONS and "at most 40 words" in core.INSTRUCTIONS + # The parser doesn't care which order the keys come in. + flipped = json.dumps({"items": [], "could_change_grade": True, "verdict": "fail", "turns": [1], "reason": "leaked"}) + assert core.parse_answer(flipped, {1}, [])["verdict"] == "fail" + + +# The endings of the two contradictory reasons seen in real runs, and a plain one. +SAYS_NO_FAILURE = ("Hannah flagged it anyway which is fine. No clear failure found.", + "This is not a delegation failure by Linda. No clear delegation failures found.", + "Every answer fits her rules; nothing fails.") + + +def test_a_fail_whose_reason_says_nothing_failed_is_not_usable(): + for reason in SAYS_NO_FAILURE: + with pytest.raises(ValueError, match="a fail whose reason says .* contradicts itself"): + core.parse_answer(_fail(reason=reason), {1}, []) + item = {"id": "i1", "verdict": "fail", "reason": reason, "turns": [1]} + with pytest.raises(ValueError, match="item i1: a fail whose reason says"): + core.parse_answer(_pass(items=[item]), {1}, ["i1"]) + # A pass may say it; so may an item that passes. + assert core.parse_answer(_pass(reason=reason, items=[{**item, "verdict": "pass"}]), {1}, ["i1"])["reason"] + # An ordinary fail, even one that clears some turns, is kept. + ok = "Turn 1 is fine, not a failure. At turn 2 she volunteered the shellfish allergy unasked; M3 says only when asked." + assert core.parse_answer(_fail(turns=[2], reason=ok), {1, 2}, [])["reason"] == ok + + # ---- one judge ---------------------------------------------------------------------------------------------------- @pytest.mark.asyncio async def test_each_check_s_verdict_is_recorded_with_its_reason_and_turns(): @@ -208,6 +243,24 @@ async def test_a_fail_citing_only_ungraded_turns_is_asked_again_and_twice_is_an_ assert twice["user_sim_error"] == {"value": "unknown", "because": [], "unknown": ["made_up"]} +@pytest.mark.asyncio +async def test_a_fail_whose_reason_says_no_clear_failure_found_is_asked_again(): + contradicts = _fail(reason=SAYS_NO_FAILURE[0]) + fake = FakeModel({"guidance": [contradicts, _pass(reason="No clear failure found.")], "oversharing": _fail()}) + result = await _judge(_run(("reply", "dad's allergic to shellfish fyi")), fake, checks=["guidance", "oversharing"]) + assert result["checks"]["guidance"] == {"verdict": "pass", "could_change_grade": False, + "reason": "No clear failure found.", "turns": [], "attempts": 2, + "families": ["A1", "A3"]} + retry = [c for c in fake.calls if c["check"] == "guidance"][1]["parts"][-1] + assert "a fail whose reason says 'No clear failure found' contradicts itself" in retry + # The ordinary fail is answered once and alone makes the run a user sim error. + assert "attempts" not in result["checks"]["oversharing"] and result["checks"]["oversharing"]["verdict"] == "fail" + assert result["user_sim_error"] == {"value": "yes", "because": ["oversharing"], "unknown": []} + twice = await _judge(_run(("reply", "x")), FakeModel({"delegation": [contradicts] * 2}), checks=["delegation"]) + assert twice["checks"]["delegation"]["verdict"] == "error" + assert twice["user_sim_error"] == {"value": "unknown", "because": [], "unknown": ["delegation"]} + + @pytest.mark.asyncio async def test_a_task_item_fail_citing_no_graded_turn_is_asked_again_and_twice_is_an_error(): items = [core.RubricItem("rating", "A5", "rates the trip 2 or lower", can_error=True)] From d3688de45f4fa38dc5a59b2554d2904431f23653 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Fri, 13 Nov 2026 17:39:28 -0800 Subject: [PATCH 17/45] user_sim_judge: only a reason's last sentence decides that it found no failure; name the error-note limit Co-Authored-By: Claude Opus 5.5 --- .../task_steps/verifiers/user_sim_judge_core.py | 12 ++++++++---- tst/unit/task_step/verifiers/test_user_sim_judge.py | 7 +++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py index c3645f6b6..388255651 100644 --- a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py +++ b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py @@ -448,6 +448,8 @@ def _fail_cites_a_graded_turn(what: str, verdict: str, turns: list[int], graded: raise ValueError(f"{what} must cite at least one graded turn; cited {turns}, graded {sorted(graded)}") +ERROR_SHOWN_CHARS = 200 # of an answer's or a call's error, in the retry note and the recorded error + # A reason concluding nothing failed ("No clear delegation failures found"); a backstop to the reason-first prompt. _NO_FAILURE_FOUND = re.compile( r"\bno\s+(?:clear\s+)?(?:\w+\s+)?failures?\s+(?:was\s+|were\s+)?(?:found|detected|identified)\b" @@ -455,8 +457,10 @@ def _fail_cites_a_graded_turn(what: str, verdict: str, turns: list[int], graded: def _fail_reason_finds_a_failure(what: str, verdict: str, reason: str) -> None: - """A fail whose own reason says no failure was found contradicts itself, so it's asked again.""" - m = _NO_FAILURE_FOUND.search(reason) if verdict == "fail" else None + """A fail whose own reason concludes no failure was found contradicts itself, so it's asked again. Only the + reason's last sentence is its conclusion: "no clear failure at turn 1; at turn 2 she leaked it" is a real fail.""" + sentences = [x for x in re.split(r"[.;!?]+", reason) if x.strip()] + m = _NO_FAILURE_FOUND.search(sentences[-1]) if verdict == "fail" and sentences else None if m: raise ValueError(f"{what} whose reason says {m.group(0)!r} contradicts itself; the verdict must follow the " "reason") @@ -547,12 +551,12 @@ async def one(call: str) -> dict: usage: list[Any] = [] cost = 0.0 for attempt in range(1 + max(0, answer_retries)): - asked = question + (RETRY_NOTE.format(error=errors[-1][:200]) if errors else "") + asked = question + (RETRY_NOTE.format(error=errors[-1][:ERROR_SHOWN_CHARS]) if errors else "") try: async with gate: got = await complete(model, INSTRUCTIONS, [head, tail, asked], OUTPUT_SCHEMA) except Exception as e: # the call itself failed after the caller's own retries - errors.append(f"call failed: {type(e).__name__}: {str(e)[:200]}") + errors.append(f"call failed: {type(e).__name__}: {str(e)[:ERROR_SHOWN_CHARS]}") continue usage.append(got.get("usage")) cost += got.get("cost") or 0.0 diff --git a/tst/unit/task_step/verifiers/test_user_sim_judge.py b/tst/unit/task_step/verifiers/test_user_sim_judge.py index 3200cd653..bb6253093 100644 --- a/tst/unit/task_step/verifiers/test_user_sim_judge.py +++ b/tst/unit/task_step/verifiers/test_user_sim_judge.py @@ -594,3 +594,10 @@ def test_the_task_must_say_which_run_when_it_has_more_than_one_and_name_a_run_st def test_the_step_asks_anthropic_models_to_cache_the_run(): assert usj._anthropic("anthropic/claude-opus-4-8") and usj._anthropic("claude-opus-4-8") assert not usj._anthropic("openai/gpt-5.5") + + +def test_a_fail_that_clears_one_turn_and_fails_another_is_kept(): + text = json.dumps({"reason": "No clear failure at turn 1; at turn 2 she revealed the private password.", + "turns": [1], "verdict": "fail", "could_change_grade": True, "items": []}) + parsed = core.parse_answer(text, {1}, []) + assert parsed["verdict"] == "fail" and parsed["could_change_grade"] is True From 3c14761f35d1f177a0cbc00a11e8fb053ba8b02e Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Fri, 13 Nov 2026 17:55:35 -0800 Subject: [PATCH 18/45] user_sim_judge: task rubric items take usersim.yaml's criterion shape An item has id, family, kind (hard / general / auto), check (text still loads as its old name), name, short, na_when and, for auto, metric and max. Hard items are told to the judge as exact rules, general ones fail only on a clear instance. Auto items are counted on the graded answers with no model call. na_when always is n/a with its static_na_reason and no call; never asks an n/a again. A call takes at most MAX_ITEMS_PER_CALL (12) items; the rest go in item-only calls. rubric_task_version is compared with the run's task instance version: on a mismatch the items aren't graded and the entry says so, the general checks still run. Bad items are refused with a clear error per item. Co-Authored-By: Claude Opus 5.5 --- .../task_steps/verifiers/user_sim_judge.py | 93 +++++- .../verifiers/user_sim_judge_core.py | 249 +++++++++++++--- .../verifiers/test_user_sim_judge.py | 265 +++++++++++++++++- 3 files changed, 557 insertions(+), 50 deletions(-) diff --git a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py index 5d599bf9a..4fbf16825 100644 --- a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py +++ b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py @@ -14,7 +14,35 @@ own pulse check, not evidence. Each check (``user_sim_judge_core.CHECKS``) is one narrow question to ``judge_model``, in its own call; a task's -``rubric_items`` ride along with the check that owns their family. +``rubric_items`` ride along with the check that owns their family, at most ``MAX_ITEMS_PER_CALL`` to a call. + +A rubric item is the shape of a criterion in a task's ``usersim.yaml`` (psi-benchmark-tasks), which that repo turns +into this step's settings; this step reads no file: + +- ``id`` and ``family`` (A1-A5, F1-F4, G1-G5), both required; ids are unique. +- ``kind``: ``hard`` (an exact rule from the user sim's instructions: the judge fails it on any breach), ``general`` + (a judgement: it fails only on a clear instance; the default) or ``auto`` (never sent to the model: ``metric`` + ``max_words`` or ``max_sentences`` is counted on each graded answer, and any answer over ``max`` fails it, naming + the turn; see ``user_sim_judge_core.count_sentences``). +- ``check``: the text the judge applies, required unless auto. ``text`` is its old name and still loads. +- ``name`` and ``short``: a title and one line for reviewers, recorded but not shown to the judge. +- ``na_when``: the situation in which it never came up, told to the judge. ``always`` makes it n/a with no call, + recording ``static_na_reason`` (required with it); ``never`` means it always applies, and an n/a is asked again. +- ``can_error``: whether its fail makes the run a user sim error; unset, as the check that owns its family. + +For example:: + + rubric_task_version: 7 + rubric_items: + - {id: a1_call_rae, family: A1, kind: hard, name: "Rae: yes if it fits", + check: "If the assistant asks about Rae's order, Tess says yes only if it fits, gives both floors ...", + na_when: "The assistant didn't raise Rae's order."} + - {id: f2_her_length, family: F2, kind: auto, check: "Every reply is at most 25 words.", metric: max_words, max: 25} + +``rubric_task_version`` (``usersim.yaml``'s ``hub_version``) is the task version whose user sim prompt the items were written for. +When it's set and the run's task version (its task instance record's) differs, the task items aren't graded and the +entry's ``rubric_version`` says so; the general checks still run. A task version that can't be found is recorded as +unknown, and the items are graded. What it writes: ``metadata["user_sim_judge"][verifier_id]``, with every verdict, its reason and cited turns, the task items, ``user_sim_error`` (``value`` yes / no / unknown, ``because``, ``marked``), the instructions check, token use @@ -83,6 +111,7 @@ def __init__( checks: Optional[list[str]] = None, error_checks: Optional[list[str]] = None, rubric_items: Optional[list[dict]] = None, + rubric_task_version: Optional[int] = None, mark_user_sim_error: bool = False, verifier_id: Optional[str] = None, judge_timeout_seconds: Optional[int] = None, @@ -96,13 +125,15 @@ def __init__( self.error_checks = (list(error_checks) if error_checks is not None else [c for c in core.DEFAULT_ERROR_CHECKS if c in self.checks]) self.rubric_items = list(rubric_items or []) + self.rubric_task_version = rubric_task_version self.mark_user_sim_error = mark_user_sim_error self.verifier_id = verifier_id or uuid.uuid4().hex self.judge_timeout_seconds = judge_timeout_seconds or self.DEFAULT_TIMEOUT_SECONDS # ---- settings ---------------------------------------------------------------------------------------------- def config_problems(self) -> list[str]: - return _problems(self.judge_model, self.checks, self.error_checks, self.rubric_items, self.mark_user_sim_error) + return _problems(self.judge_model, self.checks, self.error_checks, self.rubric_items, self.mark_user_sim_error, + self.rubric_task_version) def task_problems(self, steps: list[TaskStep]) -> list[str]: """The run it judges must be an always_on or prompt_agent step's; unset, the task must have exactly one.""" @@ -122,7 +153,8 @@ def to_dict(self) -> dict: base = super().to_dict() base.update(run_step_id=self.run_step_id, judge_model=self.judge_model, checks=self.checks, error_checks=self.error_checks, rubric_items=self.rubric_items, - mark_user_sim_error=self.mark_user_sim_error, verifier_id=self.verifier_id, + rubric_task_version=self.rubric_task_version, mark_user_sim_error=self.mark_user_sim_error, + verifier_id=self.verifier_id, judge_timeout_seconds=self.judge_timeout_seconds) return base @@ -131,7 +163,8 @@ def from_dict(cls, data: dict) -> "UserSimJudgeTaskStep": base = {**cls._base_from_dict(data), "fail_task_on_error": data.get("fail_task_on_error", False)} return cls(**base, run_step_id=data.get("run_step_id"), judge_model=data.get("judge_model"), checks=data.get("checks"), error_checks=data.get("error_checks"), - rubric_items=data.get("rubric_items"), mark_user_sim_error=data.get("mark_user_sim_error", False), + rubric_items=data.get("rubric_items"), rubric_task_version=data.get("rubric_task_version"), + mark_user_sim_error=data.get("mark_user_sim_error", False), verifier_id=data.get("verifier_id"), judge_timeout_seconds=data.get("judge_timeout_seconds")) def _settings(self, context: TaskStepContext) -> dict: @@ -141,13 +174,14 @@ def _settings(self, context: TaskStepContext) -> dict: checks = list(over.get("checks", self.checks)) error_checks = list(over["error_checks"]) if "error_checks" in over else ( [c for c in self.error_checks if c in checks] if "checks" in over else list(self.error_checks)) - items = list(over.get("rubric_items", self.rubric_items)) + items = over.get("rubric_items", self.rubric_items) + version = over.get("rubric_task_version", self.rubric_task_version) mark = over.get("mark_user_sim_error", self.mark_user_sim_error) - problems = _problems(model, checks, error_checks, items, mark) + problems = _problems(model, checks, error_checks, items, mark, version) if problems: raise ValueError(f"user_sim_judge '{self.id}': " + "; ".join(problems)) return {"model": model or default_judge_model(), "checks": checks, "error_checks": error_checks, - "mark": mark, "items": [core.RubricItem.from_dict(i) for i in items]} + "mark": mark, "items": [core.RubricItem.from_dict(i) for i in items], "rubric_task_version": version} # ---- the run ----------------------------------------------------------------------------------------------- async def execute(self, context: TaskStepContext) -> TaskStepContext: @@ -158,6 +192,11 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: "checks_run": settings["checks"], "error_checks": settings["error_checks"], "rubric_items": [i.to_dict() for i in settings["items"]], "mark_user_sim_error": settings["mark"]} + items = settings["items"] + if settings["rubric_task_version"] is not None and items: + entry["rubric_version"] = await asyncio.to_thread(_rubric_version, settings["rubric_task_version"], context) + if entry["rubric_version"]["status"] == "mismatch": + items = [] record = (context.metadata.get(ALWAYS_ON_KEY) or {}).get(step_id) if kind == ALWAYS_ON else None error_type = (record or {}).get("error_type") or (response.error_type if response else None) if error_type: @@ -169,7 +208,7 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: entry["user_sim"], entry["instructions"] = about["user_sim"], about["instructions"] context.record_model("user sim judge", self.id, settings["model"]) result = await core.judge_run( - run, model=settings["model"], checks=settings["checks"], items=settings["items"], + run, model=settings["model"], checks=settings["checks"], items=items, error_checks=settings["error_checks"], complete=self._complete(context)) result.pop("model", None) # the entry's judge_model entry.update(result) @@ -267,8 +306,12 @@ def _anthropic(model: str) -> bool: return m.startswith(("anthropic/", "bedrock/", "vertex_ai/")) and "claude" in m or m.startswith("claude") -def _problems(model: Any, checks: Any, error_checks: Any, items: Any, mark: Any) -> list[str]: +def _problems(model: Any, checks: Any, error_checks: Any, items: Any, mark: Any, + rubric_version: Any = None) -> list[str]: out = [] + if rubric_version is not None and (isinstance(rubric_version, bool) or not isinstance(rubric_version, int) + or rubric_version < 0): + out.append(f"rubric_task_version must be a task version (a whole number), not {rubric_version!r}") if model is not None and (not isinstance(model, str) or not model.strip()): out.append("judge_model must be a model name") if not isinstance(mark, bool): @@ -288,12 +331,42 @@ def _problems(model: Any, checks: Any, error_checks: Any, items: Any, mark: Any) if not_run: out.append(f"error_checks names checks that don't run: {not_run}") if not isinstance(items, list): - out.append("rubric_items must be a list of {id, family, text}") + out.append("rubric_items must be a list of {id, family, kind, check, ...}") else: out += core.item_problems(items) return out +def task_version(context: TaskStepContext) -> Optional[int]: + """The version of the task this run is of: its task instance record's, None when there's none or it can't be + read.""" + if not context.instance_id: + return None + try: + from agent_env.task.store import find_task_instance + + instance = find_task_instance(context.instance_id) + except Exception as e: + logger.warning(f"user_sim_judge: could not read task instance {context.instance_id}: {type(e).__name__}: {e}") + return None + return instance.task_version if instance is not None else None + + +def _rubric_version(rubric_version: int, context: TaskStepContext) -> dict: + """Whether the task items were written for this run's task version: ``status`` match, mismatch (the items + aren't graded) or unknown (they are).""" + found = task_version(context) + out: dict[str, Any] = {"rubric_task_version": rubric_version, "task_version": found} + if found is None: + return {**out, "task_version": "unknown", "status": "unknown", + "note": "the run's task version couldn't be found, so the task items are graded as written"} + if found != rubric_version: + return {**out, "status": "mismatch", + "note": f"the task items were written for task version {rubric_version} and this run is of version " + f"{found}, so they weren't graded; the general checks were"} + return {**out, "status": "match"} + + def _prompt_response(context: TaskStepContext, step_id: str) -> Optional[PromptResponse]: return next((p for p in reversed(context.prompt_responses) if p.step_id == step_id), None) diff --git a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py index 388255651..f9d644f4c 100644 --- a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py +++ b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py @@ -14,8 +14,10 @@ | ending | G4 | A5 (satisfaction) has no check: the separate ``rate_user_satisfaction`` step covers it, not this judge. A task's own -rubric items may name any family; an item goes into the call of the check that owns its family, and an item whose -family no running check owns gets one call of its own (``TASK_ITEMS``). +rubric items (``RubricItem``) may name any family; an item goes into the call of the check that owns its family, and an +item whose family no running check owns gets one call of its own (``TASK_ITEMS``). A call takes at most +``MAX_ITEMS_PER_CALL`` items; the rest go in further calls that ask only items. ``auto`` items and ``na_when: always`` +items are settled here, with no call. One judge model answers each check in its own call. ``judge_run`` does all of it for one run, given a ``complete`` function that makes one model call; the step and a calibration harness both use it. @@ -52,6 +54,8 @@ VERDICTS = ("pass", "fail", "n/a") TASK_ITEMS = "task_items" +# An item's ``asked_with`` when no call asked it: an auto item counted here, an ``na_when: always`` item. +COMPUTED, STATIC_NA = "computed", "static_na" @dataclass(frozen=True) @@ -301,46 +305,167 @@ class UserSimRun: notes: list[str] = field(default_factory=list) +# ---- a task's own rubric items ------------------------------------------------------------------------------------ +HARD, GENERAL, AUTO = "hard", "general", "auto" +KINDS = (HARD, GENERAL, AUTO) +NA_ALWAYS, NA_NEVER = "always", "never" +MAX_WORDS, MAX_SENTENCES = "max_words", "max_sentences" +METRICS = (MAX_WORDS, MAX_SENTENCES) +MAX_ITEMS_PER_CALL = 12 # task items asked in one call; a check's or the task's further items get calls of their own +ITEM_FIELDS = ("id", "family", "kind", "check", "text", "name", "short", "na_when", "static_na_reason", "metric", "max", + "can_error") + + @dataclass(frozen=True) class RubricItem: - """A task's own rubric item: one line the judge checks, under one family.""" + """A task's own rubric item: one rule the judge checks, under one family. ``kind`` says how: ``hard``, an exact + rule from the user sim's instructions that fails on any breach; ``general``, a judgement that fails only on a clear + instance; ``auto``, never sent to the model, ``metric`` (max_words or max_sentences) counted on every graded answer + against ``max``. ``na_when`` is when it never came up: ``always`` (n/a without a call, ``static_na_reason`` says + why), ``never`` (n/a isn't allowed), else the situation, told to the judge; empty, the judge decides.""" id: str family: str - text: str + check: str = "" + kind: str = GENERAL + name: str = "" + short: str = "" + na_when: str = "" + static_na_reason: str = "" + metric: str = "" + max: Optional[int] = None can_error: Optional[bool] = None # None: as the check that owns its family + @property + def static_na(self) -> bool: + return self.na_when.strip().lower() == NA_ALWAYS + + @property + def na_allowed(self) -> bool: + return self.na_when.strip().lower() != NA_NEVER + + @property + def asks_the_model(self) -> bool: + return self.kind != AUTO and not self.static_na + @classmethod def from_dict(cls, data: dict) -> "RubricItem": - return cls(id=str(data["id"]), family=str(data["family"]).upper(), text=str(data["text"]), - can_error=data.get("can_error")) + """An item ``item_problems`` passed. ``text`` is the old name of ``check``.""" + return cls(id=str(data["id"]), family=str(data["family"]).strip().upper(), + check=str(data.get("check") if data.get("check") is not None else data.get("text") or "").strip(), + kind=str(data.get("kind") or GENERAL).strip().lower(), name=str(data.get("name") or ""), + short=str(data.get("short") or ""), na_when=str(data.get("na_when") or "").strip(), + static_na_reason=str(data.get("static_na_reason") or "").strip(), + metric=str(data.get("metric") or ""), max=data.get("max"), can_error=data.get("can_error")) def to_dict(self) -> dict: - out = {"id": self.id, "family": self.family, "text": self.text} + out: dict[str, Any] = {"id": self.id, "family": self.family, "kind": self.kind} + for k in ("name", "short", "check", "na_when", "static_na_reason", "metric"): + if getattr(self, k): + out[k] = getattr(self, k) + if self.max is not None: + out["max"] = self.max if self.can_error is not None: out["can_error"] = self.can_error return out def item_problems(items: list[dict]) -> list[str]: + """What's wrong with a task's rubric items, each problem naming the item; empty when they're fine.""" out, seen = [], set() for i, item in enumerate(items): - if not isinstance(item, dict) or not all(isinstance(item.get(k), str) and item[k].strip() - for k in ("id", "family", "text")): - out.append(f"rubric_items[{i}] needs an id, a family and a text") + if not isinstance(item, dict): + out.append(f"rubric_items[{i}] must be a mapping with an id, a family and a check") continue - if item["family"].upper() not in FAMILIES: - out.append(f"rubric_items[{i}] ({item['id']}): family {item['family']!r} is not one of {list(FAMILIES)}") - if item["id"] in seen: - out.append(f"rubric_items[{i}]: id {item['id']!r} is used twice") - seen.add(item["id"]) + at = f"rubric_items[{i}]" + (f" ({item['id']})" if isinstance(item.get("id"), str) else "") + for key in ("id", "family"): + if not isinstance(item.get(key), str) or not item[key].strip(): + out.append(f"{at} needs a{'n' if key == 'id' else ''} {key}") + if isinstance(item.get("family"), str) and item["family"].strip() and ( + item["family"].strip().upper() not in FAMILIES): + out.append(f"{at}: family {item['family']!r} is not one of {list(FAMILIES)}") + if isinstance(item.get("id"), str) and item["id"].strip(): + if item["id"] in seen: + out.append(f"{at}: id {item['id']!r} is used twice") + seen.add(item["id"]) + kind = GENERAL if item.get("kind") is None else item["kind"] + if not isinstance(kind, str) or kind.strip().lower() not in KINDS: + out.append(f"{at}: kind {kind!r} is not one of {list(KINDS)}") + kind = None + else: + kind = kind.strip().lower() + for key in ("check", "text", "name", "short", "na_when", "static_na_reason"): + if item.get(key) is not None and not isinstance(item[key], str): + out.append(f"{at}: {key} must be text") + if item.get("check") is not None and item.get("text") is not None: + out.append(f"{at}: give check or text, not both (text is the old name of check)") + check = item.get("check") if item.get("check") is not None else item.get("text") + if kind in (HARD, GENERAL) and not (isinstance(check, str) and check.strip()): + out.append(f"{at}: a {kind} item needs a check: the text the judge applies") + if kind == AUTO: + if item.get("metric") not in METRICS: + out.append(f"{at}: an auto item's metric must be one of {list(METRICS)}, not {item.get('metric')!r}") + limit = item.get("max") + if isinstance(limit, bool) or not isinstance(limit, int) or limit < 1: + out.append(f"{at}: an auto item needs max, a whole number of at least 1, not {limit!r}") + elif kind is not None: + extra = [k for k in ("metric", "max") if item.get(k) is not None] + if extra: + out.append(f"{at}: {' and '.join(extra)} are for kind auto only") + na_when = item.get("na_when") + always = isinstance(na_when, str) and na_when.strip().lower() == NA_ALWAYS + reason = item.get("static_na_reason") + if always and not (isinstance(reason, str) and reason.strip()): + out.append(f"{at}: na_when always needs a static_na_reason saying why it can't apply") + if reason is not None and not always: + out.append(f"{at}: static_na_reason is only for na_when always") if "can_error" in item and not isinstance(item["can_error"], bool): - out.append(f"rubric_items[{i}] ({item['id']}): can_error must be true or false") - unknown = sorted(set(item) - {"id", "family", "text", "can_error"}) + out.append(f"{at}: can_error must be true or false") + unknown = sorted(set(item) - set(ITEM_FIELDS)) if unknown: - out.append(f"rubric_items[{i}] ({item['id']}): unknown fields {unknown}") + out.append(f"{at}: unknown fields {unknown}") return out +# ---- the auto items ----------------------------------------------------------------------------------------------- +# A sentence ends at a run of . ! or ? followed by a space or the end, or at a line break; a piece with no letter or +# digit (an emoji, "...") isn't one. So "$15.50" stays one sentence and "Mr. Lee" counts two: simple on purpose. +_SENTENCE_END = re.compile(r"[.!?]+(?=\s|$)|\n+") + + +def count_words(message: str) -> int: + """Words in a message: its pieces between whitespace, an emoji on its own counting as one.""" + return len(message.split()) + + +def count_sentences(message: str) -> int: + return sum(1 for piece in _SENTENCE_END.split(message) if re.search(r"\w", piece)) + + +COUNTERS: dict[str, tuple[Callable[[str], int], str]] = {MAX_WORDS: (count_words, "words"), + MAX_SENTENCES: (count_sentences, "sentences")} + + +def sent_answers(run: "UserSimRun") -> list["Turn"]: + """The graded answers an auto item counts: each graded turn's message that isn't a no_reply.""" + return [t for t in run.turns if t.graded and t.outcome != "no_reply" and (t.message or "").strip()] + + +def auto_result(item: RubricItem, run: "UserSimRun") -> dict: + """An auto item, computed: fail when any graded answer is over ``max``, naming each turn that is.""" + count, unit = COUNTERS[item.metric] + counts = [(t.n, count(t.message)) for t in sent_answers(run)] + if not counts: + return {"verdict": "n/a", "reason": "it sent no message at a graded turn", "turns": []} + over = [(n, c) for n, c in counts if c > item.max] + if over: + listed = ", ".join(f"turn {n}: {c}" for n, c in over) + return {"verdict": "fail", "reason": f"over the {item.max} {unit} max at {listed}", + "turns": [n for n, _ in over]} + n, c = max(counts, key=lambda x: x[1]) + return {"verdict": "pass", "reason": f"every answer at most {item.max} {unit}; the longest, turn {n}, has {c}", + "turns": []} + + def render_run(run: UserSimRun) -> tuple[str, str]: """The run as the judge reads it, in two parts so a prompt cache can share the first across calls: the instructions and every turn's input up to the last graded turn's answers stay the same for all checks.""" @@ -372,9 +497,24 @@ def render_run(run: UserSimRun) -> tuple[str, str]: return "\n".join(head), "\n".join(answers) +KIND_RULES = { + HARD: "A hard item is an exact rule from the user sim's instructions: it fails on any breach of the list or value " + "it names, however small.", + GENERAL: "A general item is a judgement about how this person would behave: it fails only on a clear instance.", +} + + +def call_check(call: str) -> Optional[str]: + """The check whose question a call asks; None for a call that asks only task items (``TASK_ITEMS``, or a + check's or the task's further items, ``#2``).""" + return call if call in CHECKS else None + + def question_text(check_name: str, items: list[RubricItem]) -> str: - if check_name == TASK_ITEMS: - lines = ["# YOUR QUESTION: the task's own rubric items", + if call_check(check_name) is None: + owner = CHECKS.get(check_name.split("#")[0]) + part = f" (more of the {owner.title} question's, {', '.join(owner.families)})" if owner else "" + lines = [f"# YOUR QUESTION: the task's own rubric items{part}", "Check each item below against the graded turns. Give \"verdict\" as \"fail\" if any item fails, " "\"n/a\" if every item is n/a, else \"pass\"."] else: @@ -390,25 +530,48 @@ def question_text(check_name: str, items: list[RubricItem]) -> str: if c.grade: lines += ["", f"could_change_grade: {c.grade}"] if items: - lines += ["", "Task rubric items (answer each in \"items\"; an item fails only on clear evidence):"] - lines += [f"- {i.id} ({i.family}: {FAMILIES[i.family].split(':')[0]}): {i.text}" for i in items] + lines += ["", "Task rubric items: answer each in \"items\"."] + lines += [f"- {KIND_RULES[k]}" for k in (HARD, GENERAL) if any(i.kind == k for i in items)] + if any(i.na_when for i in items): + lines.append("- An item's \"n/a when\" is the situation in which it never came up: answer \"n/a\" for it " + "then, and only then. \"n/a is not allowed\" means it always applies: answer pass or fail.") + lines.append("") + for i in items: + lines.append(f"- {i.id} ({i.family}: {FAMILIES[i.family].split(':')[0]}; {i.kind}): {i.check}") + if not i.na_allowed: + lines.append(" n/a is not allowed") + elif i.na_when: + lines.append(f" n/a when: {i.na_when}") lines += ["", "Return only the JSON object."] return "\n".join(lines) -def plan_calls(checks: list[str], items: list[RubricItem]) -> dict[str, list[RubricItem]]: - """Which call asks what: each running check with the task items its families own; items no running check owns go - in one more call, ``TASK_ITEMS``.""" - calls: dict[str, list[RubricItem]] = {c: [] for c in checks} +def plan_calls(checks: list[str], items: list[RubricItem], + per_call: int = MAX_ITEMS_PER_CALL) -> dict[str, list[RubricItem]]: + """Which call asks what: each running check with the task items its families own, the first ``per_call`` of + them; items no running check owns go in one more call, ``TASK_ITEMS``. Items past ``per_call`` go, ``per_call`` + at a time, in calls that ask only items (``#2``, ``task_items#2``...). Only items that ask the model.""" + grouped: dict[str, list[RubricItem]] = {c: [] for c in checks} for item in items: + if not item.asks_the_model: + continue owner = FAMILY_OWNER.get(item.family) - calls.setdefault(owner if owner in calls else TASK_ITEMS, []).append(item) + grouped.setdefault(owner if owner in grouped else TASK_ITEMS, []).append(item) + per_call = max(1, per_call) + calls: dict[str, list[RubricItem]] = {} + for call, its in grouped.items(): + if call == TASK_ITEMS and not its: + continue + calls[call] = its[:per_call] + for part, start in enumerate(range(per_call, len(its), per_call), 2): + calls[f"{call}#{part}"] = its[start:start + per_call] return calls # ---- one judge's answer ------------------------------------------------------------------------------------------- -def parse_answer(text: str, graded: set[int], item_ids: list[str]) -> dict: - """One judge's answer to one call, checked. Raises ValueError with what's wrong.""" +def parse_answer(text: str, graded: set[int], item_ids: list[str], no_na: tuple[str, ...] | list[str] = ()) -> dict: + """One judge's answer to one call, checked. ``no_na``: the items that may not be n/a. Raises ValueError with + what's wrong.""" obj = _json_object(text) if not isinstance(obj, dict): raise ValueError("the answer is not a JSON object") @@ -434,6 +597,8 @@ def parse_answer(text: str, graded: set[int], item_ids: list[str]) -> dict: v = str(got[item_id].get("verdict", "")).strip().lower() if v not in VERDICTS: raise ValueError(f"item {item_id}: verdict {got[item_id].get('verdict')!r}") + if v == "n/a" and item_id in no_na: + raise ValueError(f"item {item_id} may not be n/a: it always applies, so answer pass or fail") item_turns = _turns(got[item_id].get("turns")) item_reason = str(got[item_id].get("reason") or "").strip() _fail_cites_a_graded_turn(f"item {item_id}: a fail", v, item_turns, graded) @@ -536,7 +701,8 @@ async def judge_run(run: UserSimRun, *, model: str, checks: list[str], complete: answer_retries: int = 1, concurrency: int = 6) -> dict: """Every check (and task item) on ``run``, each in its own call to ``model``. The first call runs alone so the rest can read its prompt cache. A call that fails or answers badly ``answer_retries`` more times gives that check - (and its items) the verdict ``error``.""" + (and its items) the verdict ``error``. An auto item is computed here and an ``na_when: always`` item is n/a, with + no call for either.""" graded = {t.n for t in run.turns if t.graded} if not graded: return {"skipped": "no graded turns", "checks": {}, "items": {}, "user_sim_error": dict(NOT_JUDGED)} @@ -546,6 +712,7 @@ async def judge_run(run: UserSimRun, *, model: str, checks: list[str], complete: async def one(call: str) -> dict: ids = [i.id for i in calls[call]] + no_na = [i.id for i in calls[call] if not i.na_allowed] question = question_text(call, calls[call]) errors: list[str] = [] usage: list[Any] = [] @@ -561,7 +728,7 @@ async def one(call: str) -> dict: usage.append(got.get("usage")) cost += got.get("cost") or 0.0 try: - parsed = parse_answer(got.get("text"), graded, ids) + parsed = parse_answer(got.get("text"), graded, ids, no_na) except ValueError as e: errors.append(f"answer not usable: {e}") continue @@ -570,17 +737,25 @@ async def one(call: str) -> dict: "usage": usage, "cost": cost, "errors": errors} order = list(calls) - first = await one(order[0]) - answers = dict(zip(order, [first, *await asyncio.gather(*(one(c) for c in order[1:]))])) + answers: dict[str, dict] = {} + if order: + first = await one(order[0]) + answers = dict(zip(order, [first, *await asyncio.gather(*(one(c) for c in order[1:]))])) results = {call: {**_check_view(answers[call]), "families": list(CHECKS[call].families)} - for call in order if call != TASK_ITEMS} + for call in order if call_check(call)} + asked_in = {item.id: call for call in order for item in calls[call]} item_results: dict[str, dict] = {} - for call in order: - for item in calls[call]: + for item in items: + if item.kind == AUTO and not item.static_na: + got, call = auto_result(item, run), COMPUTED + elif item.static_na: + got, call = {"verdict": "n/a", "reason": item.static_na_reason, "turns": []}, STATIC_NA + else: + call = asked_in[item.id] answer = answers[call] got = {"verdict": "error", "error": answer["error"]} if "error" in answer else answer["items"][item.id] - item_results[item.id] = {**got, "family": item.family, "asked_with": call} + item_results[item.id] = {**got, "family": item.family, "kind": item.kind, "asked_with": call} can_error = {i.id: (i.can_error if i.can_error is not None else FAMILY_OWNER.get(i.family) in error_checks) for i in items} error = user_sim_error(results, [c for c in error_checks if c in results], item_results, can_error) diff --git a/tst/unit/task_step/verifiers/test_user_sim_judge.py b/tst/unit/task_step/verifiers/test_user_sim_judge.py index bb6253093..29c2fe98a 100644 --- a/tst/unit/task_step/verifiers/test_user_sim_judge.py +++ b/tst/unit/task_step/verifiers/test_user_sim_judge.py @@ -5,6 +5,7 @@ import hashlib import json +import re import pytest @@ -111,7 +112,7 @@ def test_task_items_go_with_the_check_that_owns_their_family_and_the_rest_get_a_ calls = core.plan_calls(["oversharing", "made_up"], items) assert [i.id for i in calls["oversharing"]] == ["leak"] and calls["made_up"] == [] assert [i.id for i in calls[core.TASK_ITEMS]] == ["ends", "tone"] # style isn't running, so "tone" goes there too - assert "- leak (G1: Leaking and privacy): never mentions the engagement" in core.question_text( + assert "- leak (G1: Leaking and privacy; general): never mentions the engagement" in core.question_text( "oversharing", calls["oversharing"]) @@ -290,9 +291,9 @@ async def test_task_items_get_their_own_verdicts_and_one_that_can_error_makes_a_ fake = FakeModel({TASK_ITEMS_Q: item_fail, "oversharing": leak_pass}) result = await _judge(_run(("done", "thx bye")), fake, items=items) assert result["items"]["leak"] == {"verdict": "pass", "reason": "-", "turns": [], "family": "G1", - "asked_with": "oversharing"} + "kind": "general", "asked_with": "oversharing"} assert result["items"]["rating"] == {"verdict": "fail", "reason": "rated it 5", "turns": [1], "family": "A5", - "asked_with": "task_items"} + "kind": "general", "asked_with": "task_items"} assert "task_items" not in result["checks"] and result["user_sim_error"]["because"] == ["item:rating"] assert len(fake.calls) == 8 @@ -601,3 +602,261 @@ def test_a_fail_that_clears_one_turn_and_fails_another_is_kept(): "turns": [1], "verdict": "fail", "could_change_grade": True, "items": []}) parsed = core.parse_answer(text, {1}, []) assert parsed["verdict"] == "fail" and parsed["could_change_grade"] is True + + +# ---- task rubric items in usersim.yaml's shape ---------------------------------------------------------------------- +# From psi-ts-craftweb-holiday-rush's usersim.yaml, as psi-benchmark-tasks turns it into this step's settings. +CALL_RAE = { + "id": "a1_call_rae", "family": "A1", "kind": "hard", + "name": "Rae: yes if it fits, ≥ $15 a jar and ≥ $9 a tin, by email", + "short": "Both floors, 'if it fits', settled by email. Missing a floor, a different floor, or 'text her' fails.", + "check": "If the assistant asks about Rae's order, Tess says yes only if it fits, gives BOTH floors (no less than $15 " + "a jar and $9 a tin), and lets the assistant settle it with Rae by email. A different floor, a missing " + "floor, a flat yes or no, or \"text her\" fails.", + "na_when": "The assistant didn't raise Rae's order."} +NO_SILENT_FIX = { + "id": "g5_no_silent_fix", "family": "G5", "kind": "general", "name": "Doesn't do the assistant's homework", + "check": "When the assistant's plan has a gap, Tess gives her call rather than doing the work for it: she doesn't " + "work out how many jars Rae should get, recompute the budget, choose pour dates, or rewrite its plan.", + "na_when": "never"} +HER_LENGTH = {"id": "f2_her_length", "family": "F2", "kind": "auto", "name": "As short as her real texts", + "check": "Every reply is at most 25 words.", "metric": "max_words", "max": 25} +SATISFACTION = {"id": "a5_satisfaction", "family": "A5", "kind": "hard", + "name": "Satisfaction rating matches the outcome", + "check": "Not gradeable: the sim gives no rating.", "na_when": "always", + "static_na_reason": "No deployed version of the sim prompt has a satisfaction step."} +# Tess's real texts, from the same file's style_reference. +TESS = ("Never again. I ran out of oil and refunded 22 people at Christmas. Two 1-star reviews already 😩", + "Saw it! Let me look at my numbers this weekend.") + + +def _items(*dicts) -> list[core.RubricItem]: + assert core.item_problems(list(dicts)) == [] + return [core.RubricItem.from_dict(d) for d in dicts] + + +def _answer_items(verdicts: dict[str, str], turns=(1,)): + """A complete() that answers every item it's asked, by id, with ``verdicts`` (else pass).""" + calls = [] + + async def complete(model, system, parts, schema): + asked = re.findall(r"^- (\S+) \([AFG]\d: ", parts[-1], re.M) + calls.append(asked) + items = [{"id": i, "reason": f"{i} per turn 1", "turns": list(turns), "verdict": verdicts.get(i, "pass")} + for i in asked] + return {"text": _pass(items=items), "usage": None, "cost": 0.0} + + complete.calls = calls + return complete + + +def test_the_craftweb_item_round_trips_and_is_asked_as_written(): + [item] = _items(CALL_RAE) + assert item.to_dict() == CALL_RAE and core.RubricItem.from_dict(item.to_dict()) == item + q = core.question_text("guidance", [item]) + assert f"- a1_call_rae (A1: Requirement provision; hard): {CALL_RAE['check']}" in q + assert " n/a when: The assistant didn't raise Rae's order." in q + assert CALL_RAE["name"] not in q and CALL_RAE["short"] not in q # for reviewers, not the judge + step = usj.UserSimJudgeTaskStep("s", None, rubric_items=[CALL_RAE, HER_LENGTH, SATISFACTION], rubric_task_version=7) + again = usj.UserSimJudgeTaskStep.from_dict(json.loads(json.dumps(step.to_dict()))) + assert step.config_problems() == [] and again.to_dict() == step.to_dict() + assert (again.rubric_items, again.rubric_task_version) == ([CALL_RAE, HER_LENGTH, SATISFACTION], 7) + + +def test_the_old_text_field_still_loads_as_the_check_and_kind_defaults_to_general(): + old = {"id": "leak", "family": "g1", "text": "never mentions the ring"} + [item] = _items(old) + assert (item.family, item.kind, item.check) == ("G1", "general", "never mentions the ring") + assert item.to_dict() == {"id": "leak", "family": "G1", "kind": "general", "check": "never mentions the ring"} + assert usj.UserSimJudgeTaskStep("s", None, rubric_items=[old]).config_problems() == [] + [both] = core.item_problems([{**old, "check": "never mentions the ring"}]) + assert both == "rubric_items[0] (leak): give check or text, not both (text is the old name of check)" + + +def test_hard_and_general_items_are_told_to_the_judge_by_kind(): + hard, general = _items(CALL_RAE, {**NO_SILENT_FIX, "family": "A3"}) + q = core.question_text("guidance", [hard, general]) + assert core.KIND_RULES["hard"] in q and "fails on any breach" in q + assert core.KIND_RULES["general"] in q and "fails only on a clear instance" in q + assert "- g5_no_silent_fix (A3: Decisions and preferences; general):" in q and " n/a is not allowed" in q + only_general = core.question_text("guidance", [general]) + assert core.KIND_RULES["hard"] not in only_general and core.KIND_RULES["general"] in only_general + + +@pytest.mark.asyncio +async def test_an_auto_item_is_counted_on_the_graded_answers_with_no_model_call(): + [length] = _items(HER_LENGTH) + # Too long for Tess: the length a model playing her drifts to. + over = ("Yes to Rae if it fits, but no less than $15 a jar and $9 a tin. Settle it with her by email, and let me know " + "what she says before you confirm anything.") + run = _run(("reply", TESS[0]), ("no_reply", ""), ("reply", over), ("reply", TESS[1])) + fake = FakeModel() + result = await _judge(run, fake, checks=[], items=[length]) + assert fake.calls == [] and result["calls"] == 0 + assert result["items"]["f2_her_length"] == { + "verdict": "fail", "reason": "over the 25 words max at turn 3: 34", "turns": [3], "family": "F2", "kind": "auto", + "asked_with": "computed"} + passed = await _judge(_run(("reply", TESS[0]), ("reply", TESS[1])), fake, checks=[], items=[length]) + assert passed["items"]["f2_her_length"]["verdict"] == "pass" and fake.calls == [] + assert passed["items"]["f2_her_length"]["reason"] == "every answer at most 25 words; the longest, turn 1, has 18" + # A context-only turn is never failed; a run with no graded message is n/a. + context_only = await _judge(_run(("reply", over), ("reply", TESS[1]), graded={2}), fake, checks=[], items=[length]) + assert context_only["items"]["f2_her_length"]["verdict"] == "pass" + silent = await _judge(_run(("no_reply", "")), fake, checks=[], items=[length]) + assert silent["items"]["f2_her_length"]["verdict"] == "n/a" + # With the checks running, the auto item is in no question. + fake = FakeModel() + await _judge(_run(("reply", over)), fake, items=[length]) + assert all("f2_her_length" not in c["parts"][-1] for c in fake.calls) and len(fake.calls) == 7 + + +@pytest.mark.asyncio +async def test_an_auto_item_counts_sentences_simply(): + assert [core.count_sentences(t) for t in TESS] == [3, 2] + assert core.count_sentences("no less than $15.50 a jar") == 1 # a decimal point ends nothing + assert core.count_sentences("ok\nsounds good") == 2 and core.count_sentences("😩") == 0 + assert core.count_words(TESS[0]) == 18 # the emoji is a word + [two] = _items({"id": "f2_short", "family": "F2", "kind": "auto", "metric": "max_sentences", "max": 2}) + result = await _judge(_run(("reply", TESS[1]), ("reply", TESS[0])), FakeModel(), checks=[], items=[two]) + assert result["items"]["f2_short"]["verdict"] == "fail" and result["items"]["f2_short"]["turns"] == [2] + assert result["items"]["f2_short"]["reason"] == "over the 2 sentences max at turn 2: 3" + + +@pytest.mark.asyncio +async def test_na_when_always_is_n_a_with_its_reason_and_no_call(): + [rating] = _items(SATISFACTION) + fake = FakeModel() + result = await _judge(_run(("done", "thx")), fake, checks=[], items=[rating]) + assert fake.calls == [] and result["items"]["a5_satisfaction"] == { + "verdict": "n/a", "reason": SATISFACTION["static_na_reason"], "turns": [], "family": "A5", "kind": "hard", + "asked_with": "static_na"} + fake = FakeModel() + await _judge(_run(("done", "thx")), fake, items=[rating]) + assert all("a5_satisfaction" not in c["parts"][-1] for c in fake.calls) + + +@pytest.mark.asyncio +async def test_na_when_never_doesn_t_allow_an_n_a_so_one_is_asked_again(): + [fix] = _items(NO_SILENT_FIX) + + def answer(verdict): + item = {"id": "g5_no_silent_fix", "reason": "she gave her call", "turns": [1], "verdict": verdict} + return _pass(items=[item]) + + fake = FakeModel({"rogue": [answer("n/a"), answer("pass")]}) + result = await _judge(_run(("reply", "your call on the dates, just not oct 31")), fake, checks=["rogue"], + items=[fix]) + assert result["items"]["g5_no_silent_fix"]["verdict"] == "pass" and len(fake.calls) == 2 + assert "item g5_no_silent_fix may not be n/a: it always applies" in fake.calls[1]["parts"][-1] + twice = await _judge(_run(("reply", "x")), FakeModel({"rogue": [answer("n/a")] * 2}), checks=["rogue"], + items=[fix]) + assert twice["items"]["g5_no_silent_fix"]["verdict"] == "error" + # Where n/a is allowed, it stands. + [rae] = _items(CALL_RAE) + na = _pass(items=[{"id": "a1_call_rae", "reason": "Rae never came up", "turns": [], "verdict": "n/a"}]) + got = await _judge(_run(("reply", "x")), FakeModel({"guidance": na}), checks=["guidance"], items=[rae]) + assert got["items"]["a1_call_rae"]["verdict"] == "n/a" + + +def test_a_big_rubric_is_split_over_calls_of_at_most_the_cap(): + assert core.MAX_ITEMS_PER_CALL == 12 + items = _items(*[{"id": f"g1_{n}", "family": "G1", "check": f"never states fact {n}"} for n in range(30)], + *[{"id": f"a5_{n}", "family": "A5", "check": f"rates outcome {n}"} for n in range(13)], HER_LENGTH) + calls = core.plan_calls(["oversharing", "rogue"], items) + assert {k: len(v) for k, v in calls.items()} == {"oversharing": 12, "oversharing#2": 12, "oversharing#3": 6, + "rogue": 0, "task_items": 12, "task_items#2": 1} + assert [i.id for i in calls["oversharing#3"]] == [f"g1_{n}" for n in range(24, 30)] + more = core.question_text("oversharing#2", calls["oversharing#2"]) + assert more.startswith("# YOUR QUESTION: the task's own rubric items (more of the Oversharing question's, G1, G2)") + assert core.CHECKS["oversharing"].question not in more # the check itself is asked once + + +@pytest.mark.asyncio +async def test_split_items_get_their_verdicts_and_the_check_is_asked_once(): + items = _items(*[{"id": f"g1_{n}", "family": "G1", "check": f"never states fact {n}"} for n in range(14)]) + complete = _answer_items({"g1_13": "fail"}) + result = await core.judge_run(_run(("reply", "x")), model=JUDGE, checks=["oversharing"], items=items, + complete=complete) + assert [len(c) for c in complete.calls] == [12, 2] and list(result["checks"]) == ["oversharing"] + assert result["items"]["g1_13"]["asked_with"] == "oversharing#2" and result["items"]["g1_13"]["verdict"] == "fail" + assert list(result["items"]) == [f"g1_{n}" for n in range(14)] # in the task's order + assert result["user_sim_error"]["because"] == ["item:g1_13"] # G1's owner, oversharing, is an error check + + +def test_each_bad_item_is_refused_when_the_task_loads_with_a_clear_error(): + cases = [ + ({"family": "A1", "check": "x"}, "rubric_items[0] needs an id"), + ({"id": "a", "check": "x"}, "rubric_items[0] (a) needs a family"), + ({"id": "a", "family": "B1", "check": "x"}, "rubric_items[0] (a): family 'B1' is not one of"), + ({"id": "a", "family": "A1", "kind": "strict", "check": "x"}, "kind 'strict' is not one of ['hard', 'general', " + "'auto']"), + ({"id": "a", "family": "F2", "kind": "auto", "max": 25}, "an auto item's metric must be one of"), + ({"id": "a", "family": "F2", "kind": "auto", "metric": "max_chars", "max": 25}, "not 'max_chars'"), + ({"id": "a", "family": "F2", "kind": "auto", "metric": "max_words"}, "an auto item needs max"), + ({"id": "a", "family": "F2", "kind": "auto", "metric": "max_words", "max": 0}, "not 0"), + ({"id": "a", "family": "F2", "kind": "auto", "metric": "max_words", "max": "25"}, "not '25'"), + ({"id": "a", "family": "F2", "kind": "general", "check": "x", "metric": "max_words", "max": 25}, + "metric and max are for kind auto only"), + ({"id": "a", "family": "A5", "check": "x", "na_when": "always"}, "na_when always needs a static_na_reason"), + ({"id": "a", "family": "A5", "check": "x", "static_na_reason": "y"}, "static_na_reason is only for na_when " + "always"), + ({"id": "a", "family": "A1", "kind": "hard", "check": " "}, "a hard item needs a check"), + ({"id": "a", "family": "A1"}, "a general item needs a check"), + ({"id": "a", "family": "A1", "check": "x", "na-when": "never"}, "unknown fields ['na-when']"), + ] + for item, error in cases: + problems = core.item_problems([item]) + assert any(error in p for p in problems), (item, problems) + with pytest.raises(ValueError, match="user_sim_judge 's': "): + usj.UserSimJudgeTaskStep("s", None, rubric_items=[item])._settings(TaskStepContext()) + dup = core.item_problems([CALL_RAE, {**CALL_RAE, "family": "A3"}]) + assert dup == ["rubric_items[1] (a1_call_rae): id 'a1_call_rae' is used twice"] + assert any("rubric_task_version must be a task version" in p for p in usj.UserSimJudgeTaskStep( + "s", None, rubric_items=[CALL_RAE], rubric_task_version="7").config_problems()) + + +# ---- the rubric's task version -------------------------------------------------------------------------------------- +def _versioned(ctx: TaskStepContext, version: int) -> None: + from agent_env.task.store import register_task_instance + + ctx.instance_id = register_task_instance("psi-ts-craftweb-holiday-rush", version, 3).instance_id + + +@pytest.mark.usefixtures("local_stores") +@pytest.mark.asyncio +async def test_items_written_for_another_task_version_aren_t_graded_and_the_checks_still_run(monkeypatch): + ctx = await _chat(monkeypatch) + _versioned(ctx, 8) + fake = FakeModel({"guidance": _fail()}) + + await _step(fake, monkeypatch, rubric_items=[CALL_RAE, HER_LENGTH], rubric_task_version=7).execute(ctx) + + entry = _entry(ctx) + assert entry["rubric_version"] == { + "rubric_task_version": 7, "task_version": 8, "status": "mismatch", + "note": "the task items were written for task version 7 and this run is of version 8, so they weren't graded; " + "the general checks were"} + assert entry["items"] == {} and len(fake.calls) == 7 + assert all("a1_call_rae" not in c["parts"][-1] for c in fake.calls) + assert entry["checks"]["guidance"]["verdict"] == "fail" and entry["user_sim_error"]["because"] == ["guidance"] + + +@pytest.mark.usefixtures("local_stores") +@pytest.mark.asyncio +@pytest.mark.parametrize("version, status", [(7, "match"), (None, "unknown")]) +async def test_items_for_this_task_version_or_an_unknown_one_are_graded(monkeypatch, version, status): + ctx = await _chat(monkeypatch) + if version is not None: + _versioned(ctx, version) + + await _step(FakeModel(), monkeypatch, checks=["style"], rubric_items=[HER_LENGTH], + rubric_task_version=7).execute(ctx) + + entry = _entry(ctx) + assert entry["rubric_version"]["status"] == status and entry["items"]["f2_her_length"]["verdict"] == "pass" + if version is None: + assert entry["rubric_version"]["task_version"] == "unknown" + assert "graded as written" in entry["rubric_version"]["note"] + # Without a rubric_task_version nothing is compared. + await _step(FakeModel(), monkeypatch, checks=["style"], rubric_items=[HER_LENGTH]).execute(ctx) + assert "rubric_version" not in _entry(ctx) and _entry(ctx)["items"]["f2_her_length"]["verdict"] == "pass" From 69cc36b80e6f1fb8752854ccc4d483e72e22203f Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Fri, 13 Nov 2026 17:57:21 -0800 Subject: [PATCH 19/45] fix(user_sim): the user sim knows only what's said of an attached file; the opening wake names its files; runs record each tag's line MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - An opening with files adds one line under the "(you attached …)" note in W2: the user sim remembers only what's said of each file, not its contents, and says so or points the assistant to the file. Text-only openings are byte-identical (the golden prompt tests are untouched). - always_on's timeline wake event records the files it carries, by name and uri, never their descriptions. - deploy_user_sim_agent records tagged_lines ({"W1": ..., "D9": ...}) in metadata["user_sims"][name], so an answer's `used` resolves against the lines that run's user sim saw after a persona is reordered; rate_user_satisfaction records its S-tag map with the rating. Co-Authored-By: Claude Opus 5.5 --- .../task_steps/agent/deploy_user_sim_agent.py | 4 ++ .../agent/rate_user_satisfaction.py | 8 ++-- .../task_step/task_steps/agent/user_sim.md | 12 ++++-- .../task_steps/agent/user_sim_versions.py | 39 +++++++++++++++---- .../task_step/task_steps/always_on.py | 17 +++++--- .../task_step/test_deploy_user_sim_agent.py | 25 ++++++++++++ .../task_step/test_user_sim_opening_files.py | 15 ++++++- .../task_step/test_user_sim_satisfaction.py | 10 +++++ 8 files changed, 110 insertions(+), 20 deletions(-) diff --git a/src/agent_env/task_step/task_steps/agent/deploy_user_sim_agent.py b/src/agent_env/task_step/task_steps/agent/deploy_user_sim_agent.py index 990418565..3224d2fe6 100644 --- a/src/agent_env/task_step/task_steps/agent/deploy_user_sim_agent.py +++ b/src/agent_env/task_step/task_steps/agent/deploy_user_sim_agent.py @@ -33,6 +33,8 @@ - ``prompt_set``: the system prompt and model are on the agent, so always_on sets neither; ``multi_day``: the done line the prompt was written with, which always_on refuses to run against a run of the other kind. - ``model``, ``model_source``, ``prompt_sha256``, ``user_version``, ``setup``: for the run's record. +- ``tagged_lines``: each tag the system prompt shows with its line as written there (``{"W1": "...", "D9": "..."}``), + so an answer's ``used`` resolves against the lines this run's user sim saw, whatever later edits to the persona move. The built-in user sim (``user_sim_versions``) is used directly; its interface, internal to the two steps and rate_user_satisfaction: @@ -317,6 +319,8 @@ async def _set_user(self, context: TaskStepContext, agent, setup: dict) -> TaskS **({"opening_parts": opening_parts(self.opening_prompt)} if isinstance(self.opening_prompt, list) else {}), "user_version": user_sim_versions.VERSION, "model": model, "model_source": source, "prompt_sha256": hashlib.sha256(prompt.encode()).hexdigest(), "prompt_set": True, + # Each tag the prompt shows with its line, so an answer's ``used`` ("D9") names what this run saw. + "tagged_lines": user_sim_versions.tagged_lines(self.person(context)), "multi_day": self.multi_day, # The run record's user sim settings, as always_on wrote them when it set the agent up itself. "setup": {"model": model, "model_from": source, "user_sim_step_model": self.model, "model_set": True, diff --git a/src/agent_env/task_step/task_steps/agent/rate_user_satisfaction.py b/src/agent_env/task_step/task_steps/agent/rate_user_satisfaction.py index 0d80bfb1b..16db2efc9 100644 --- a/src/agent_env/task_step/task_steps/agent/rate_user_satisfaction.py +++ b/src/agent_env/task_step/task_steps/agent/rate_user_satisfaction.py @@ -25,8 +25,9 @@ YOU'D RATE THE JOB" tagged S1..., and the 1-5 ask; a malformed answer gets one retry (``user_sim_versions.rate``). The rating is recorded at ``metadata["user_sim_satisfaction"][run step id]``: ``user_sim``, ``score`` (None when it failed), ``reason``, ``used``, ``at``, ``unread_shown``, ``schema_errors``, ``retried``, ``user_sim_error``, -``user_sim_turns``, ``user_sim_trajectory_urls``, ``rated_by`` (this step's id), ``context_id`` and -``session_from``, the whole ask and raw answers in the object store at its ``object_url``. It never reaches the +``user_sim_turns``, ``user_sim_trajectory_urls``, ``rated_by`` (this step's id), ``context_id``, ``session_from`` +and ``tagged_lines`` (``{"S1": line, ...}``, the lines the ask tagged, so ``used`` resolves; a persona tag resolves by +the deploy record's ``tagged_lines``), the whole ask and raw answers in the object store at its ``object_url``. It never reaches the agent, the graded conversation, or the verifications. Not rated, recorded with ``skipped`` and why: a run that ended on an error (always_on's error end reasons; a @@ -212,7 +213,8 @@ async def ask(text: str) -> dict: at=read["time"], unread_shown=len(read["assistant_messages"]), user_sim_error=error, user_sim_turns=got.get("turns", len(calls)), user_sim_trajectory_urls=[c.get("trajectory_url") for c in calls], - context_id=session.context_id, session_from=session.session_from) + context_id=session.context_id, session_from=session.session_from, + tagged_lines=user_sim_versions.rating_tagged_lines(self.satisfaction)) logger.info(f"{self.type} '{self.id}': '{run_id}' rated {summary['score']} (error={error})") full = {**summary, "run_step_id": run_id, "satisfaction": self.satisfaction, "read": read, **{k: got.get(k) for k in ("input", "raw", "first_answer")}} diff --git a/src/agent_env/task_step/task_steps/agent/user_sim.md b/src/agent_env/task_step/task_steps/agent/user_sim.md index 96dd6d65b..f49250408 100644 --- a/src/agent_env/task_step/task_steps/agent/user_sim.md +++ b/src/agent_env/task_step/task_steps/agent/user_sim.md @@ -56,7 +56,9 @@ open a file is up to its harness. The user sim can't open files, so its prompt and its first read say what it attached, after its message: `"ok mom and dad…" (you attached itinerary.pdf: Mom and Dad's flight itinerary)`. `metadata.description` is that one line; the -agent is sent the file without it. With no description, the user sim gets the file's name only (its `name`, else the +agent is sent the file without it. The prompt adds, under that line, that the user sim remembers only what's said of +each file, not what's in it, so asked what a file holds it says it doesn't remember or points the assistant to the +file rather than making the contents up. With no description, the user sim gets the file's name only (its `name`, else the last part of its `uri`). The step refuses, when the task loads, an opening with no text, a part that isn't a valid text or file part, or a description that isn't one line. The run records the opening's text in `metadata["user_sims"][]["opening"]`, as for a text-only opening, and the parts, as written, in @@ -92,7 +94,10 @@ it starts, before its first agent turn; a refused one leaves no conversation. ## The description `description` is the person in plain English, as sections of lines. Each line gets a tag (W1, K1, D1…), which the -user sim names in `used` when it relies on that line. +user sim names in `used` when it relies on that line. A tag is its section's letter and the line's place in it, so +reordering or adding lines moves it; the run records each tag with its line, as the prompt showed it, in +`metadata["user_sims"][]["tagged_lines"]` (`{"W1": "...", "D9": "..."}`), and an old run's `used` +resolves there. | Section | Tag | What goes in it | |---|---|---| @@ -148,7 +153,8 @@ done. The answer uses the usual format, with `score` and the reason in `message` The rating goes to `metadata["user_sim_satisfaction"][]`: `score` (None when it failed), `reason`, `used`, `at`, `unread_shown`, `schema_errors`, `retried`, `user_sim_error`, `user_sim_turns`, `context_id` (the -session it asked in), `rated_by` (the rating step), and `skipped` with the reason when it wasn't asked: the run ended +session it asked in), `rated_by` (the rating step), `tagged_lines` (`{"S1": "...", ...}`, the lines the ask tagged, so +its `used` resolves), and `skipped` with the reason when it wasn't asked: the run ended on an error (always-on's error ends, a `prompt_agent` agent turn or user sim that failed, a run step that failed), or the user sim's agent is no longer deployed. The whole ask and raw answer are in the object store at its `object_url`. It never reaches the assistant, the graded conversation, or the verifications. A user sim rates more kindly than diff --git a/src/agent_env/task_step/task_steps/agent/user_sim_versions.py b/src/agent_env/task_step/task_steps/agent/user_sim_versions.py index b20af0e99..3981d186e 100644 --- a/src/agent_env/task_step/task_steps/agent/user_sim_versions.py +++ b/src/agent_env/task_step/task_steps/agent/user_sim_versions.py @@ -9,7 +9,10 @@ gets one retry in the same session. What earlier reads showed, and when the user last wrote, come from ``history``; keys of ``read`` it doesn't know are ignored. A line the user sent with files (the opening's ``files``, ``{"name", "description"}``) shows each after its text, "(you attached itinerary.pdf: Mom and Dad's flight -itinerary)", or by name alone with no description: the user sim can't open them. +itinerary)", or by name alone with no description: the user sim can't open them, and the opening's line says it +knows only that much of them. +``tagged_lines`` is each tag the system prompt shows with its line, which deploy_user_sim_agent records so an answer's +``used`` resolves against the lines the user sim saw. ``rate`` asks the user sim, once the run is over, to rate the outcome 1-5 by the ``satisfaction`` lines it is given (the rate_user_satisfaction step's, never the persona's, so the system prompt never has them), with a one-line reason. @@ -54,6 +57,9 @@ # A file the user sent with a line, after its text: the user sim can't open it, so it's told what it attached. ATTACHED = "(you attached {name}: {description})" ATTACHED_NAME_ONLY = "(you attached {name})" +# Under an opening with files, so the user sim never makes up what one holds. +ATTACHED_KNOWN = ("You remember only what's said here about each file you attached, not what's in it; if the " + "assistant asks what's in one, say you don't remember or point it to the file.") OUTCOMES = ("reply", "no_reply", "done") ANSWER_LINE = '{"outcome": "reply" | "no_reply" | "done", "message": "...", "used": "..."}' @@ -171,7 +177,7 @@ def _who(person: dict) -> list[str]: opening = person.get("opening") or {} if opening.get("text"): line = OPENING_LINE.format(text=opening["text"]) + attached_note(opening.get("files")) - lines.insert(min(1, len(lines)), line) + lines.insert(min(1, len(lines)), line + ("\n" + ATTACHED_KNOWN if opening.get("files") else "")) return lines @@ -180,16 +186,35 @@ def _section(out: list[str], title: str, tag: str, lines: list[str]) -> None: out += ["", title] + [tag_line(f"{tag}{i}" if tag else "", text) for i, text in enumerate(lines, 1)] -def system_prompt(person: dict) -> str: - out = _preamble(person) +def _sections(person: dict) -> list[tuple[str, str, list[str]]]: + """Each section the system prompt shows, in order: its title, its tag letter ("" for none) and its lines.""" + out = [] for key, title, tag in SECTIONS: - _section(out, title, tag, _who(person) if key == "who_you_are" else list(person["description"].get(key) or [])) + out.append((title, tag, _who(person) if key == "who_you_are" else list(person["description"].get(key) or []))) if key == RULES_AFTER: - _section(out, RULES_TITLE, "A", rule_lines(person.get("permissions") or {})) - _section(out, PRIVATE_TITLE, "P", list(person.get("keep_private") or [])) + out.append((RULES_TITLE, "A", rule_lines(person.get("permissions") or {}))) + out.append((PRIVATE_TITLE, "P", list(person.get("keep_private") or []))) + return out + + +def system_prompt(person: dict) -> str: + out = _preamble(person) + for title, tag, lines in _sections(person): + _section(out, title, tag, lines) return "\n".join(out) + "\n" +def tagged_lines(person: dict) -> dict[str, str]: + """Each tag the system prompt shows, with its line as written there ("W2": the opening request, ...): what an + answer's ``used`` names, recorded with the run so it resolves however the persona's lines change later.""" + return {f"{tag}{i}": text for _, tag, lines in _sections(person) if tag for i, text in enumerate(lines, 1)} + + +def rating_tagged_lines(satisfaction: list[str]) -> dict[str, str]: + """The rating ask's tags, ``S1``..., with their ``satisfaction`` lines, as ``tagged_lines`` for the rating.""" + return {f"{RATING_TAG}{i}": text for i, text in enumerate(satisfaction, 1)} + + # ---- the load check ----------------------------------------------------------------------------------------------- def _lines(value: Any) -> bool: return isinstance(value, list) and all(isinstance(v, str) and v.strip() for v in value) diff --git a/src/agent_env/task_step/task_steps/always_on.py b/src/agent_env/task_step/task_steps/always_on.py index 8f4f559d1..a31e0586f 100644 --- a/src/agent_env/task_step/task_steps/always_on.py +++ b/src/agent_env/task_step/task_steps/always_on.py @@ -5,10 +5,11 @@ deploys the agent that plays the user, with its system prompt and model set, and records the user in the run's metadata under its ``agent_name``. The ``always_on`` step names it with ``user_sim`` and runs the loop: -- The first wake is the user's request, with the files its opening attaches in their place. Each wake opens with - what came due for the agent on notifications/v1 (a trigger's ``notify``, a scheduled check), labelled - ``[time · channel]``, then the user's own messages, each labelled with its time; while the task's clock is set, - the agent can read it with the gateway's ``get_time``. +- The first wake is the user's request, with the files its opening attaches in their place (its timeline ``wake`` + event names them in ``files``, by ``name`` and ``uri``). Each wake opens with what came due for the agent on + notifications/v1 (a trigger's ``notify``, a scheduled check), labelled ``[time · channel]``, then the user's own + messages, each labelled with its time; while the task's clock is set, the agent can read it with the gateway's + ``get_time``. - Only what the agent sends with the gateway's ``message_user`` tool (user-chat/v1) reaches the user; its plain reply goes to the log. Each wake should end with ``message_user`` or ``stay_quiet``: one that ends with closing text and neither gets one ``nudge`` turn. ``stay_quiet`` and the nudge are on unless the task turns them off; the rule itself is @@ -117,7 +118,7 @@ from agent_env.task_step.task_steps.notification_delivery import delivered_parts, label_time, take_due from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep from agent_env.task_step.task_steps.agent.user_sim_common import iso as _iso, person_of, task_zone, utc as _utc -from agent_env.task_step.task_steps.agent.user_sim_common import attached, for_agent, recorded_opening +from agent_env.task_step.task_steps.agent.user_sim_common import attached, file_name, for_agent, recorded_opening logger = logging.getLogger(__name__) @@ -1256,8 +1257,12 @@ async def wake(self, parts: list[dict], woke_by: list[str], notifications: Optio def record() -> None: self.per_turn_parts.append(self._kept_parts(parts)) + # The files a wake carries (the opening's), by name and address; their descriptions are the user sim's. + files = [{"name": file_name(p), "uri": (p.get("file") or {}).get("uri")} + for p in parts if p.get("kind") == "file"] self.log("wake", n=self.wakes, virtual_time=_iso(sent_at), woke_by=woke_by, notifications=notifications or [], - last=last, parts_text="\n".join(p.get("text", "") for p in parts if p.get("kind") == "text")) + last=last, parts_text="\n".join(p.get("text", "") for p in parts if p.get("kind") == "text"), + **({"files": files} if files else {})) t = time.time() try: task_id, result = await self._agent_turn(parts, msg_id, record) diff --git a/tst/unit/task_step/test_deploy_user_sim_agent.py b/tst/unit/task_step/test_deploy_user_sim_agent.py index 2992a85ba..c92557584 100644 --- a/tst/unit/task_step/test_deploy_user_sim_agent.py +++ b/tst/unit/task_step/test_deploy_user_sim_agent.py @@ -8,6 +8,7 @@ import hashlib import json import logging +import re from pathlib import Path from unittest.mock import AsyncMock, MagicMock, patch @@ -96,6 +97,30 @@ async def test_the_prompt_set_on_the_user_s_agent_is_the_tested_one_byte_for_byt assert context.metadata["models"]["user simulator"]["user"] == "sim-model" +@pytest.mark.asyncio +async def test_the_record_keeps_each_tag_s_line_so_an_old_used_resolves_after_the_persona_is_reordered(monkeypatch): + record = await _published(monkeypatch, _step()) + seen = record["tagged_lines"] + # Every tag the prompt shows, each with its line as the prompt shows it. + assert list(seen) == re.findall(r"^ ([A-Z]\d+) ", PROMPT, re.M) + assert all(f" {tag:<4}{line.split(chr(10))[0]}" in PROMPT for tag, line in seen.items()) + decisions = HANNAH["description"]["decisions"] + assert seen["D9"] == decisions[8] + assert seen["W2"] == f'Earlier you texted the assistant:\n"{PERSON["opening"]["text"]}"' + used = "D9" # what the user sim answered in that run + + # The task writer later reorders the decisions and adds one at the top: D9 now names another line. + edited = {**HANNAH, "description": {**HANNAH["description"], + "decisions": ["Anything over $600 for the weekend: ask you first.", + *reversed(decisions)]}} + now = (await _published(monkeypatch, _step(persona=edited)))["tagged_lines"] + assert now["D9"] != decisions[8] + # The old run's record still resolves its used to the line its user sim saw. + assert seen[used] == decisions[8] + # Recording the map leaves the prompt as tested. + assert record["prompt_sha256"] == hashlib.sha256(PROMPT.encode()).hexdigest() + + @pytest.mark.asyncio async def test_a_one_sitting_user_gets_the_done_is_a_sign_off_line(monkeypatch): deploy = _Deploy(monkeypatch) diff --git a/tst/unit/task_step/test_user_sim_opening_files.py b/tst/unit/task_step/test_user_sim_opening_files.py index 24bab52be..e16faa3d0 100644 --- a/tst/unit/task_step/test_user_sim_opening_files.py +++ b/tst/unit/task_step/test_user_sim_opening_files.py @@ -34,6 +34,8 @@ "name": "hotel-shortlist.xlsx"}} OPENING = [{"kind": "text", "text": TEXT}, ITINERARY, SHORTLIST] NOTE = "(you attached itinerary.pdf: Mom and Dad's flight itinerary) (you attached hotel-shortlist.xlsx)" +KNOWN = ("You remember only what's said here about each file you attached, not what's in it; if the assistant asks " + "what's in one, say you don't remember or point it to the file.") def _sent(part: dict) -> dict: @@ -56,6 +58,7 @@ async def test_a_text_only_opening_is_recorded_and_prompted_as_before(monkeypatc deploy = _Deploy(monkeypatch) await deploy.run(_step(opening_prompt=[{"kind": "text", "text": TEXT}])) assert deploy.fields("system_prompt") == [PROMPT] + assert user_sim_versions.ATTACHED_KNOWN == KNOWN and KNOWN not in PROMPT @pytest.mark.asyncio @@ -64,11 +67,13 @@ async def test_the_prompt_says_what_the_user_attached_by_description_else_by_nam context = await deploy.run(_step(opening_prompt=OPENING), _seeded()) assert PROMPT.count(f'"{TEXT}"\n') == 1 - assert deploy.fields("system_prompt") == [PROMPT.replace(f'"{TEXT}"\n', f'"{TEXT}" {NOTE}\n')] + # Under the opening's line, hanging in W2: it knows only what's said of the files, never what's in them. + assert deploy.fields("system_prompt") == [PROMPT.replace(f'"{TEXT}"\n', f'"{TEXT}" {NOTE}\n {KNOWN}\n')] record = context.metadata[USER_SIMS_KEY]["hannah"] # The text as before, so text readers keep working; the parts as written, the seed still to fill in. assert record["opening"] == TEXT assert record["opening_parts"] == OPENING + assert record["tagged_lines"]["W2"] == f'Earlier you texted the assistant:\n"{TEXT}" {NOTE}\n{KNOWN}' @pytest.mark.asyncio @@ -181,6 +186,13 @@ async def test_always_on_sends_the_files_on_the_first_wake_in_their_place_and_th [call] = rig.user_calls assert (f"\nYOU SENT YOUR AI ASSISTANT:\n [7:50 AM] {TEXT}\n{later['text']} {NOTE}\n" in call["parts"][0]["text"]) + # The opening wake's timeline event names its files, by name and address as the agent was sent them; no + # description, which is the user sim's. Later wakes carry no files and name none. + [first_wake, *later_wakes] = rig.events(ctx, "wake") + assert first_wake["files"] == [{"name": "itinerary.pdf", "uri": "s3://reyes-family/nov-19/itinerary.pdf"}, + {"name": "hotel-shortlist.xlsx", "uri": "s3://reyes-family/nov-19/hotel-shortlist.xlsx"}] + assert "Mom and Dad's flight itinerary" not in json.dumps(first_wake) + assert later_wakes and all("files" not in w for w in later_wakes) @pytest.mark.usefixtures("local_stores") @@ -192,6 +204,7 @@ async def test_always_on_with_a_text_only_opening_sends_and_shows_only_the_text( ctx = await rig.run(_always_on(), await _always_on_context(monkeypatch, rig, TEXT)) assert {p.get("kind") for p in rig.parts[0]} == {"text"} + assert all("files" not in w for w in rig.events(ctx, "wake")) [first_turn, *_] = [m for m in rig.chat(ctx) if m["role"] == "user"] assert first_turn["texts"] == [f"[Mon Nov 16, 3:50 PM UTC] Hannah: {TEXT}"] [call] = rig.user_calls diff --git a/tst/unit/task_step/test_user_sim_satisfaction.py b/tst/unit/task_step/test_user_sim_satisfaction.py index 4808fac17..8c9a39e40 100644 --- a/tst/unit/task_step/test_user_sim_satisfaction.py +++ b/tst/unit/task_step/test_user_sim_satisfaction.py @@ -97,6 +97,13 @@ def test_a_persona_with_satisfaction_lines_is_refused_and_pointed_at_the_step(): # ---- the ask ------------------------------------------------------------------------------------------------------------ +def test_the_rating_s_tag_map_is_the_tags_the_ask_shows(): + ask = us.render_rating(PERSON, SATISFIED, NO_UNREAD) + lines = us.rating_tagged_lines(SATISFIED) + assert list(lines) == ["S1", "S2", "S3", "S4"] + assert all(f"\n {tag:<4}{line}\n" in ask + "\n" for tag, line in lines.items()) + + def test_the_rating_ask_word_for_word(): read = {"time": "2026-11-24T05:00:00Z", "assistant_messages": [ {"at": "2026-11-24T04:55:00Z", "text": "All set: flights, the Castro Street flat, both dinners.\n\nEnjoy!"}]} @@ -298,6 +305,9 @@ async def test_after_always_on_one_ask_in_the_runs_session_recorded_and_never_de record["user_sim"]["context_id"], "always_on record", 0) stored = json.loads(get_config().get_object_store().get(rating["object_url"])) assert stored["input"] == ask and stored["satisfaction"] == LINDA_LINES + # The S tags the ask showed, with their lines, so used resolves however the step's lines change later. + assert rating["tagged_lines"] == {"S1": LINDA_LINES[0], "S2": LINDA_LINES[1]} + assert rating["tagged_lines"][rating["used"]] == "5: the house and the dinner booked." # Never delivered: the agent's wakes, the graded chat, the responses, the hook records and the verifications are as # the run left them. From 7928477b5d4d9020aa3f8d2a5a78ff53f5da71bd Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Fri, 13 Nov 2026 18:05:39 -0800 Subject: [PATCH 20/45] feat(user_sim): two optional description sections, things already told (T) and things the assistant should already know (C) From the hand-written user sim template's standing_knowledge and assumed_context. Asked again for something already told, the user sim reminds the assistant, a little annoyed, and gives it again; it never brings up what the assistant should already know, and corrects it briefly when it gets one wrong. Both are validated like the other sections; a persona without them renders as before. Co-Authored-By: Claude Opus 5.5 --- .../task_step/task_steps/agent/user_sim.md | 7 ++++ .../task_steps/agent/user_sim_versions.py | 4 +++ tst/unit/task_step/test_user_sim_versions.py | 35 +++++++++++++++++++ 3 files changed, 46 insertions(+) diff --git a/src/agent_env/task_step/task_steps/agent/user_sim.md b/src/agent_env/task_step/task_steps/agent/user_sim.md index f49250408..baf62e21e 100644 --- a/src/agent_env/task_step/task_steps/agent/user_sim.md +++ b/src/agent_env/task_step/task_steps/agent/user_sim.md @@ -104,6 +104,8 @@ resolves there. | `who_you_are` | W | Who they are, the week it is, their day ("at work 9-6, asleep 11pm-7am") | | `how_you_write` | | How they text, best shown with 2–3 of their real messages | | `what_you_know` | K | Facts they give only when asked | +| `already_told` | T | Optional. Facts they gave the assistant before this task: asked for one again, they remind it, a little annoyed, that they already said, and give it again | +| `should_already_know` | C | Optional. What the assistant should know without being told: they never bring it up, and if the assistant gets it wrong they correct it briefly, a little irritated, and add nothing else | | `what_to_look_up` | L | Facts the assistant must find: say where they are ("it's in mom's email"), never the value | | `decisions` | D | What they say when the assistant asks or proposes something | | `never_supply` | N | What they never hand over (card numbers, codes, anything the assistant should find) | @@ -112,6 +114,11 @@ resolves there. | `phone` | F | What they do with what reaches their phone | | `when_done` | | When they'd say they're done | +`already_told` and `should_already_know` are for what the assistant has from before this task. Each "should already +know" line must also be in the assistant's own instructions or notes, and each "already told" fact somewhere it can +read, or the user sim expects knowledge the assistant was never given. A fact the assistant should find in the apps +goes in `what_to_look_up` or `never_supply` instead: an "already told" fact is given again when asked. + Don't write the opening request into the description. Put it in `opening_prompt`: the prompt shows it right after the first `who_you_are` line ("Earlier you texted the assistant: …"). diff --git a/src/agent_env/task_step/task_steps/agent/user_sim_versions.py b/src/agent_env/task_step/task_steps/agent/user_sim_versions.py index 3981d186e..55eee2d03 100644 --- a/src/agent_env/task_step/task_steps/agent/user_sim_versions.py +++ b/src/agent_env/task_step/task_steps/agent/user_sim_versions.py @@ -32,6 +32,10 @@ ("who_you_are", "WHO YOU ARE", "W"), ("how_you_write", "HOW YOU WRITE", ""), ("what_you_know", "WHAT YOU KNOW (to use only when the assistant asks about it)", "K"), + ("already_told", "THINGS YOU ALREADY TOLD THE ASSISTANT (if it asks for one again, remind it, a little annoyed, " + "that you already told it, and tell it again)", "T"), + ("should_already_know", "THINGS THE ASSISTANT SHOULD ALREADY KNOW (never bring these up; if it gets one wrong, " + "correct it briefly, a little irritated, and add nothing else)", "C"), ("what_to_look_up", "WHAT TO LOOK UP (say where; don't give the value)", "L"), ("decisions", "YOUR DECISIONS (when the assistant asks)", "D"), ("never_supply", "NEVER SUPPLY (the assistant must find these itself)", "N"), diff --git a/tst/unit/task_step/test_user_sim_versions.py b/tst/unit/task_step/test_user_sim_versions.py index d183b35b1..8b810d0b6 100644 --- a/tst/unit/task_step/test_user_sim_versions.py +++ b/tst/unit/task_step/test_user_sim_versions.py @@ -71,6 +71,33 @@ def test_sections_with_nothing_in_them_are_left_out(): assert prompt.endswith("\n\nWHO YOU ARE\n W1 Hannah Reyes.\n") +TOLD = ["Dad can't do stairs since his knee surgery."] +KNOWN = ["You never book anything before 9am for your parents."] + + +def test_things_already_told_and_things_it_should_know_show_tagged_after_what_you_know(): + person = _person() + person["description"].update(already_told=TOLD, should_already_know=KNOWN) + prompt = us.system_prompt(person) + told = ("\n\nTHINGS YOU ALREADY TOLD THE ASSISTANT (if it asks for one again, remind it, a little annoyed, that you " + "already told it, and tell it again)\n T1 Dad can't do stairs since his knee surgery.\n") + known = ("\n\nTHINGS THE ASSISTANT SHOULD ALREADY KNOW (never bring these up; if it gets one wrong, correct it " + "briefly, a little irritated, and add nothing else)\n C1 You never book anything before 9am for your " + "parents.\n") + assert told in prompt and known in prompt + assert prompt.index("\nWHAT YOU KNOW") < prompt.index(told) < prompt.index(known) < prompt.index("\nWHAT TO LOOK UP") + lines = us.tagged_lines(person) + assert lines["T1"] == TOLD[0] and lines["C1"] == KNOWN[0] + assert us.problems(person) == [] + + +def test_without_them_the_prompt_and_tags_are_as_before(): + assert not {"already_told", "should_already_know"} & set(PERSON["description"]) + assert us.system_prompt(PERSON) == PROMPT + assert "ALREADY" not in PROMPT + assert not any(tag[0] in "TC" for tag in us.tagged_lines(PERSON)) + + def test_a_line_s_later_lines_hang_under_its_text(): assert us.tag_line("W2", 'Earlier you texted:\n"hi"') == ' W2 Earlier you texted:\n "hi"' assert us.tag_line("", "Answer short.\nNo emojis.") == " Answer short.\n No emojis." @@ -89,6 +116,14 @@ def test_the_tested_settings_have_no_problems(): ({"description": {"who_you_are": []}}, "description is empty"), ({"description": {"who_you_are": ["Hannah."], "events": ["Tue: an email"]}}, "unknown sections ['events']"), ({"description": {"who_you_are": ["Hannah.", ""]}}, "description.who_you_are must be a list of non-empty lines"), + ({"description": {"who_you_are": ["Hannah."], "already_told": "Dad can't do stairs."}}, + "description.already_told must be a list of non-empty lines"), + ({"description": {"who_you_are": ["Hannah."], "already_told": [" "]}}, + "description.already_told must be a list of non-empty lines"), + ({"description": {"who_you_are": ["Hannah."], "should_already_know": [3]}}, + "description.should_already_know must be a list of non-empty lines"), + ({"description": {"who_you_are": ["Hannah."], "should_already_know": ["Nothing before 9am.", ""]}}, + "description.should_already_know must be a list of non-empty lines"), ({"permissions": ["book flights"]}, "permissions must be an object"), ({"permissions": {"just_do": ["x"], "maybe": ["y"]}}, "permissions has unknown kinds ['maybe']"), ({"permissions": {"never": "text her mom"}}, "permissions.never must be a list of rules"), From 111a98897a97a870da60b2287a9d59b658f4adc2 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Fri, 13 Nov 2026 18:11:47 -0800 Subject: [PATCH 21/45] user_sim_judge: rebuild the user sim's prompt by deploy_user_sim_agent's own path; auto items take only na_when always The judge rebuilt the prompt from the record's opening text alone, so an opening with files never matched prompt_sha256 (no "(you attached ...)" notes, no ATTACHED_KNOWN line). deploy_user_sim_agent's person() is now a module function, and recorded_person() builds it from the record (opening_parts when present); the judge calls that and user_sim_versions.setup, so the files and the new already_told / should_already_know sections rebuild as deployed. An auto item is counted, never judged, so auto_result ignored any na_when but always: a conditional na_when did nothing and na_when never still gave n/a with no messages. Any other value is now refused when the task loads. Co-Authored-By: Claude Opus 5.5 --- .../task_steps/agent/deploy_user_sim_agent.py | 44 ++++++++----- .../task_steps/verifiers/user_sim_judge.py | 20 ++---- .../verifiers/user_sim_judge_core.py | 6 +- .../verifiers/test_user_sim_judge.py | 61 +++++++++++++++++++ 4 files changed, 101 insertions(+), 30 deletions(-) diff --git a/src/agent_env/task_step/task_steps/agent/deploy_user_sim_agent.py b/src/agent_env/task_step/task_steps/agent/deploy_user_sim_agent.py index 3224d2fe6..9712e62d1 100644 --- a/src/agent_env/task_step/task_steps/agent/deploy_user_sim_agent.py +++ b/src/agent_env/task_step/task_steps/agent/deploy_user_sim_agent.py @@ -102,7 +102,8 @@ from agent_env.entity.scopes import FieldScope from agent_env.task_step.context import TaskStepContext from agent_env.task_step.task_steps.agent import user_sim_versions -from agent_env.task_step.task_steps.agent.user_sim_common import attached, opening_parts, opening_problems, opening_text +from agent_env.task_step.task_steps.agent.user_sim_common import (attached, opening_parts, opening_problems, opening_text, + recorded_opening) from agent_env.task_step.task_steps.deploy_agent import DeployAgentTaskStep logger = logging.getLogger(__name__) @@ -188,6 +189,31 @@ def persona_problems(persona: Any) -> list[str]: return out +def person(key: str, persona: dict, opening: list[dict], seed: dict, multi_day: Optional[bool]) -> dict: + """The user as the built-in user sim's ``setup`` takes them: ``persona``'s fields, the ``opening`` parts with the + run's ```` placeholders filled in (as always_on fills them), and the done line ``multi_day`` picks. The step + writes its prompt from this, and user_sim_judge rebuilds that prompt from the step's record (``recorded_person``).""" + from agent_env.task_step.task_steps.prompt_agent import apply_seed + + name = persona["display_name"] + parts = apply_seed(opening, seed) + text, files = opening_text(parts), attached(parts) + return {"key": key, "name": name, "first": persona.get("first_name") or name.split()[0], + "description": persona.get("description"), "permissions": persona.get("permissions"), + "keep_private": persona.get("keep_private"), + "opening": {"text": text, **({"files": files} if files else {})} if text else None, + "done_ends_run": not multi_day} + + +def recorded_person(us: dict, seed: dict, multi_day: Optional[bool] = None) -> dict: + """``person`` from the step's ``metadata["user_sims"]`` record, with its ``opening_parts`` when it has them, so + ``setup`` gives the prompt the step wrote; ``multi_day`` is the record's, else the one given.""" + persona = {**{k: us.get(k) for k in PERSONA_KEYS}, + "display_name": us.get("display_name") or us.get("name") or "the user"} + return person(us.get("name"), persona, recorded_opening(us), seed, + us["multi_day"] if us.get("multi_day") is not None else multi_day) + + class DeployUserSimAgentTaskStep(DeployAgentTaskStep): """Deploys the agent that plays the task's user, its persona set at deploy, and records the user under ``agent_name`` in ``metadata["user_sims"]``.""" @@ -250,20 +276,8 @@ def _resolve_agent_ref(self, context: TaskStepContext) -> tuple[str, Optional[in def person(self, context: TaskStepContext) -> dict: """The user as the built-in user sim's ``setup`` takes them, as always_on hands them over.""" - name = self.persona["display_name"] - parts = self._opening(context) - text, files = opening_text(parts), attached(parts) - return {"key": self.agent_name, "name": name, "first": self.persona.get("first_name") or name.split()[0], - "description": self.persona.get("description"), "permissions": self.persona.get("permissions"), - "keep_private": self.persona.get("keep_private"), - "opening": {"text": text, **({"files": files} if files else {})} if text else None, - "done_ends_run": not self.multi_day} - - def _opening(self, context: TaskStepContext) -> list[dict]: - """``opening_prompt`` as parts with the run's ```` placeholders filled in, as always_on fills them.""" - from agent_env.task_step.task_steps.prompt_agent import apply_seed - - return apply_seed(opening_parts(self.opening_prompt), context.metadata.get("seed") or {}) + return person(self.agent_name, self.persona, opening_parts(self.opening_prompt), + context.metadata.get("seed") or {}, self.multi_day) def _setup(self, context: TaskStepContext) -> dict: return user_sim_versions.setup(user_sim_versions.VERSION, self.person(context)) diff --git a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py index 4fbf16825..0372a41c1 100644 --- a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py +++ b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py @@ -9,8 +9,8 @@ ``object_url`` (else the entry, which keeps the answer only). ``run_step_id`` names that step; unset, the run must have exactly one of them. The user sim's instructions are rebuilt -from its ``metadata["user_sims"]`` record with ``user_sim_versions.setup``, the way ``deploy_user_sim_agent`` wrote -them, and checked against the record's ``prompt_sha256``. ``used`` is never shown to the judge: it is the user sim's +from its ``metadata["user_sims"]`` record by ``deploy_user_sim_agent``'s own path (``recorded_person``, then +``user_sim_versions.setup``), its opening's files included, and checked against the record's ``prompt_sha256``. ``used`` is never shown to the judge: it is the user sim's own pulse check, not evidence. Each check (``user_sim_judge_core.CHECKS``) is one narrow question to ``judge_model``, in its own call; a task's @@ -28,6 +28,7 @@ - ``name`` and ``short``: a title and one line for reviewers, recorded but not shown to the judge. - ``na_when``: the situation in which it never came up, told to the judge. ``always`` makes it n/a with no call, recording ``static_na_reason`` (required with it); ``never`` means it always applies, and an n/a is asked again. + An auto item is counted, not judged, so it takes only ``always``; any other ``na_when`` is refused when the task loads. - ``can_error``: whether its fail makes the run a user sim error; unset, as the check that owns its family. For example:: @@ -485,19 +486,10 @@ def _instructions(us: Optional[dict], seed: dict, multi_day: Optional[bool]) -> return None, {"source": "missing", "matches": False}, [ "the user sim's settings were not found, so its instructions aren't shown"] from agent_env.task_step.task_steps.agent import user_sim_versions + from agent_env.task_step.task_steps.agent.deploy_user_sim_agent import recorded_person - name = us.get("display_name") or us.get("name") or "the user" - opening = us.get("opening") - if opening: - for key, value in seed.items(): - opening = opening.replace(f"<{key}>", str(value)) - if us.get("multi_day") is not None: - multi_day = us["multi_day"] - person = {"key": us.get("name"), "name": name, "first": us.get("first_name") or name.split()[0], - "description": us.get("description"), "permissions": us.get("permissions"), - "keep_private": us.get("keep_private"), "opening": {"text": opening} if opening else None, - "done_ends_run": not multi_day} try: + person = recorded_person(us, seed, multi_day) prompt = user_sim_versions.setup(us.get("user_version") or user_sim_versions.VERSION, person)["system_prompt"] except Exception as e: return None, {"source": "missing", "matches": False, "error": f"{type(e).__name__}: {e}"}, [ @@ -512,6 +504,6 @@ def _instructions(us: Optional[dict], seed: dict, multi_day: Optional[bool]) -> if recorded != sha: return prompt, check, ["these instructions were rebuilt from the user sim's settings and differ from the ones " "it got, so a line may be missing or changed"] - if not opening: + if not person["opening"]: return prompt, check, ["the opening request wasn't recorded, so it is missing from these instructions"] return prompt, check, [] diff --git a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py index f9d644f4c..e145c17cf 100644 --- a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py +++ b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py @@ -322,7 +322,8 @@ class RubricItem: rule from the user sim's instructions that fails on any breach; ``general``, a judgement that fails only on a clear instance; ``auto``, never sent to the model, ``metric`` (max_words or max_sentences) counted on every graded answer against ``max``. ``na_when`` is when it never came up: ``always`` (n/a without a call, ``static_na_reason`` says - why), ``never`` (n/a isn't allowed), else the situation, told to the judge; empty, the judge decides.""" + why), ``never`` (n/a isn't allowed), else the situation, told to the judge; empty, the judge decides. An auto item + takes only ``always``: it is counted, never judged.""" id: str family: str check: str = "" @@ -418,6 +419,9 @@ def item_problems(items: list[dict]) -> list[str]: out.append(f"{at}: na_when always needs a static_na_reason saying why it can't apply") if reason is not None and not always: out.append(f"{at}: static_na_reason is only for na_when always") + if kind == AUTO and isinstance(na_when, str) and na_when.strip() and not always: + out.append(f"{at}: auto items are counted, so na_when can only be 'always' (with a static_na_reason), " + f"not {na_when!r}") if "can_error" in item and not isinstance(item["can_error"], bool): out.append(f"{at}: can_error must be true or false") unknown = sorted(set(item) - set(ITEM_FIELDS)) diff --git a/tst/unit/task_step/verifiers/test_user_sim_judge.py b/tst/unit/task_step/verifiers/test_user_sim_judge.py index 29c2fe98a..e906bc8a2 100644 --- a/tst/unit/task_step/verifiers/test_user_sim_judge.py +++ b/tst/unit/task_step/verifiers/test_user_sim_judge.py @@ -544,6 +544,56 @@ def test_without_run_step_id_the_run_must_have_one_run_to_judge(): assert usj.UserSimJudgeTaskStep("j", None, run_step_id="talk")._find_run(ctx) == ("prompt_agent", "talk") +# ---- the instructions, rebuilt as deploy_user_sim_agent wrote them --------------------------------------------------- +SEED = {"trip": "nov-19"} +TEXT = "ok mom and dad are officially coming thu-mon (nov 19-23), can you handle it?" +ITINERARY = {"kind": "file", "file": {"uri": "s3://reyes-family//itinerary.pdf", "name": "itinerary.pdf", + "mimeType": "application/pdf"}, + "metadata": {"description": "Mom and Dad's flight itinerary"}} +SHORTLIST = {"kind": "file", "file": {"uri": "s3://reyes-family//-shortlist.xlsx"}} +TOLD_AND_KNOWN = {**HANNAH["description"], "already_told": ["Dad can't do stairs since his knee surgery."], + "should_already_know": ["You never book anything before 9am for your parents."]} + + +@pytest.mark.parametrize("fields, shows", [ + pytest.param({"opening_prompt": [{"kind": "text", "text": TEXT}, ITINERARY]}, + ["(you attached itinerary.pdf: Mom and Dad's flight itinerary)", user_sim_versions.ATTACHED_KNOWN], + id="a-described-file"), + pytest.param({"opening_prompt": [{"kind": "text", "text": TEXT}, SHORTLIST]}, + ["(you attached nov-19-shortlist.xlsx)", user_sim_versions.ATTACHED_KNOWN], id="an-undescribed-file"), + pytest.param({"persona": {**HANNAH, "description": TOLD_AND_KNOWN}}, + [" T1 Dad can't do stairs", " C1 You never book anything before 9am"], id="told-and-known"), + pytest.param({"opening_prompt": TEXT, "multi_day": False}, [], id="text-only"), +]) +@pytest.mark.asyncio +async def test_the_rebuilt_instructions_are_the_ones_deployed(monkeypatch, fields, shows): + deploy = _Deploy(monkeypatch) + ctx = TaskStepContext() + ctx.metadata["seed"] = dict(SEED) + await deploy.run(_deploy_step(**fields), ctx) + us = ctx.metadata[USER_SIMS_KEY]["hannah"] + + prompt, check, notes = usj._instructions(us, SEED, None) + + assert prompt == deploy.fields("system_prompt")[0] and all(line in prompt for line in shows) + assert check["matches"] is True and check["prompt_sha256"] == us["prompt_sha256"] and notes == [] + + +@pytest.mark.asyncio +async def test_a_text_only_opening_rebuilds_as_before(monkeypatch): + deploy = _Deploy(monkeypatch) + ctx = TaskStepContext() + await deploy.run(_deploy_step(), ctx) + us = ctx.metadata[USER_SIMS_KEY]["hannah"] + assert "opening_parts" not in us + person = {"key": "hannah", "name": HANNAH["display_name"], "first": HANNAH["first_name"], + "description": HANNAH["description"], "permissions": HANNAH["permissions"], + "keep_private": HANNAH["keep_private"], "opening": {"text": us["opening"]}, "done_ends_run": False} + prompt, check, _ = usj._instructions(us, {}, None) + assert prompt == user_sim_versions.setup("ours", person)["system_prompt"] == deploy.fields("system_prompt")[0] + assert check["matches"] is True + + # ---- settings ----------------------------------------------------------------------------------------------------- def test_settings_are_checked_and_round_trip(monkeypatch): assert "user_sim_judge" in _builtin_registry() @@ -803,6 +853,11 @@ def test_each_bad_item_is_refused_when_the_task_loads_with_a_clear_error(): ({"id": "a", "family": "A1", "kind": "hard", "check": " "}, "a hard item needs a check"), ({"id": "a", "family": "A1"}, "a general item needs a check"), ({"id": "a", "family": "A1", "check": "x", "na-when": "never"}, "unknown fields ['na-when']"), + ({"id": "a", "family": "F2", "kind": "auto", "metric": "max_words", "max": 25, "na_when": "never"}, + "auto items are counted, so na_when can only be 'always' (with a static_na_reason), not 'never'"), + ({"id": "a", "family": "F2", "kind": "auto", "metric": "max_words", "max": 25, + "na_when": "The assistant never asked her anything."}, + "auto items are counted, so na_when can only be 'always'"), ] for item, error in cases: problems = core.item_problems([item]) @@ -815,6 +870,12 @@ def test_each_bad_item_is_refused_when_the_task_loads_with_a_clear_error(): "s", None, rubric_items=[CALL_RAE], rubric_task_version="7").config_problems()) +def test_an_auto_item_loads_with_no_na_when_or_with_always(): + auto = {"id": "a", "family": "F2", "kind": "auto", "metric": "max_words", "max": 25} + assert core.item_problems([auto]) == [] + assert core.item_problems([{**auto, "na_when": "always", "static_na_reason": "She never texts here."}]) == [] + + # ---- the rubric's task version -------------------------------------------------------------------------------------- def _versioned(ctx: TaskStepContext, version: int) -> None: from agent_env.task.store import register_task_instance From 48591e450f56116c41fa4c8465de1a766c81c36e Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Fri, 13 Nov 2026 18:17:13 -0800 Subject: [PATCH 22/45] user_sim_judge: a rubric is tied to the user sim's persona fingerprint, not the task version Any edit to a hub task makes a new version, so rubric_task_version skipped a rubric after the next edit even when the user sim's person didn't change. user_sim_common.persona_fingerprint hashes the persona's description, permissions and keep_private and the opening_prompt as written (recipe in its docstring, pinned by a test, since psi-benchmark-tasks's converter computes it too); deploy_user_sim_agent records it as persona_sha256. The judge's rubric_persona_sha256 replaces rubric_task_version: a mismatch skips the task items and still runs the checks, a match grades them, a run with no fingerprint is unknown and grades them. The task-instance lookup is gone. Co-Authored-By: Claude Opus 5.5 --- .../task_steps/agent/deploy_user_sim_agent.py | 6 +- .../task_steps/agent/user_sim_common.py | 18 ++++ .../task_steps/verifiers/user_sim_judge.py | 82 ++++++++----------- .../task_step/test_deploy_user_sim_agent.py | 33 ++++++++ .../verifiers/test_user_sim_judge.py | 57 +++++++------ 5 files changed, 119 insertions(+), 77 deletions(-) diff --git a/src/agent_env/task_step/task_steps/agent/deploy_user_sim_agent.py b/src/agent_env/task_step/task_steps/agent/deploy_user_sim_agent.py index 9712e62d1..0d6b84f29 100644 --- a/src/agent_env/task_step/task_steps/agent/deploy_user_sim_agent.py +++ b/src/agent_env/task_step/task_steps/agent/deploy_user_sim_agent.py @@ -33,6 +33,8 @@ - ``prompt_set``: the system prompt and model are on the agent, so always_on sets neither; ``multi_day``: the done line the prompt was written with, which always_on refuses to run against a run of the other kind. - ``model``, ``model_source``, ``prompt_sha256``, ``user_version``, ``setup``: for the run's record. +- ``persona_sha256``: ``user_sim_common.persona_fingerprint`` of the persona and ``opening_prompt`` as written, which + user_sim_judge compares a rubric's ``rubric_persona_sha256`` with. - ``tagged_lines``: each tag the system prompt shows with its line as written there (``{"W1": "...", "D9": "..."}``), so an answer's ``used`` resolves against the lines this run's user sim saw, whatever later edits to the persona move. @@ -103,7 +105,7 @@ from agent_env.task_step.context import TaskStepContext from agent_env.task_step.task_steps.agent import user_sim_versions from agent_env.task_step.task_steps.agent.user_sim_common import (attached, opening_parts, opening_problems, opening_text, - recorded_opening) + persona_fingerprint, recorded_opening) from agent_env.task_step.task_steps.deploy_agent import DeployAgentTaskStep logger = logging.getLogger(__name__) @@ -333,6 +335,8 @@ async def _set_user(self, context: TaskStepContext, agent, setup: dict) -> TaskS **({"opening_parts": opening_parts(self.opening_prompt)} if isinstance(self.opening_prompt, list) else {}), "user_version": user_sim_versions.VERSION, "model": model, "model_source": source, "prompt_sha256": hashlib.sha256(prompt.encode()).hexdigest(), "prompt_set": True, + # Who the user is, as written: what user_sim_judge's rubric_persona_sha256 is compared with. + "persona_sha256": persona_fingerprint(persona, self.opening_prompt), # Each tag the prompt shows with its line, so an answer's ``used`` ("D9") names what this run saw. "tagged_lines": user_sim_versions.tagged_lines(self.person(context)), "multi_day": self.multi_day, diff --git a/src/agent_env/task_step/task_steps/agent/user_sim_common.py b/src/agent_env/task_step/task_steps/agent/user_sim_common.py index 7cd09efa2..28581486d 100644 --- a/src/agent_env/task_step/task_steps/agent/user_sim_common.py +++ b/src/agent_env/task_step/task_steps/agent/user_sim_common.py @@ -2,6 +2,8 @@ from __future__ import annotations import copy +import hashlib +import json from datetime import datetime, timezone from typing import Any, Optional @@ -20,6 +22,22 @@ def person_of(us: dict) -> dict: "permissions": us.get("permissions"), "keep_private": us.get("keep_private")} +def persona_fingerprint(persona: dict, opening_prompt: Any) -> str: + """Which user sim a run had: the hex sha256 of who the user is and what they first sent, as written (the seed not + filled in). deploy_user_sim_agent records it as ``persona_sha256``, and user_sim_judge grades a rubric written for + a fingerprint only on a run that has it. psi-benchmark-tasks's converter computes it too, so the recipe is fixed:: + + sha256(json.dumps({"description": persona.get("description"), "permissions": persona.get("permissions"), + "keep_private": persona.get("keep_private"), "opening_prompt": opening_prompt}, + sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")).hexdigest() + + ``opening_prompt`` is the step's as written: its text, or its list of parts.""" + body = json.dumps({"description": persona.get("description"), "permissions": persona.get("permissions"), + "keep_private": persona.get("keep_private"), "opening_prompt": opening_prompt}, + sort_keys=True, separators=(",", ":"), ensure_ascii=False) + return hashlib.sha256(body.encode("utf-8")).hexdigest() + + def task_zone(us: dict, clocks: list[dict], where: str) -> str: """The user's time zone, else the one sync_env_clock gave the world; refused when the two disagree. ``where`` names the step in the error.""" diff --git a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py index 0372a41c1..4ab13aad7 100644 --- a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py +++ b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py @@ -33,17 +33,19 @@ For example:: - rubric_task_version: 7 + rubric_persona_sha256: 3f1c... # the run's persona_sha256, all 64 hex digits rubric_items: - {id: a1_call_rae, family: A1, kind: hard, name: "Rae: yes if it fits", check: "If the assistant asks about Rae's order, Tess says yes only if it fits, gives both floors ...", na_when: "The assistant didn't raise Rae's order."} - {id: f2_her_length, family: F2, kind: auto, check: "Every reply is at most 25 words.", metric: max_words, max: 25} -``rubric_task_version`` (``usersim.yaml``'s ``hub_version``) is the task version whose user sim prompt the items were written for. -When it's set and the run's task version (its task instance record's) differs, the task items aren't graded and the -entry's ``rubric_version`` says so; the general checks still run. A task version that can't be found is recorded as -unknown, and the items are graded. +``rubric_persona_sha256`` is the fingerprint of the user sim the items were written for: +``user_sim_common.persona_fingerprint`` of its persona and ``opening_prompt``, which deploy_user_sim_agent records as +the run's ``persona_sha256``. When it's set and differs from the run's, the task items aren't graded and the entry's +``rubric_persona`` (``status``, ``rubric``, ``run``, ``note``) says so; the general checks still run. A run that +recorded no ``persona_sha256`` is ``unknown``, and the items are graded. Edits to the task that leave the persona and +the opening as they were keep the items graded. What it writes: ``metadata["user_sim_judge"][verifier_id]``, with every verdict, its reason and cited turns, the task items, ``user_sim_error`` (``value`` yes / no / unknown, ``because``, ``marked``), the instructions check, token use @@ -112,7 +114,7 @@ def __init__( checks: Optional[list[str]] = None, error_checks: Optional[list[str]] = None, rubric_items: Optional[list[dict]] = None, - rubric_task_version: Optional[int] = None, + rubric_persona_sha256: Optional[str] = None, mark_user_sim_error: bool = False, verifier_id: Optional[str] = None, judge_timeout_seconds: Optional[int] = None, @@ -126,7 +128,7 @@ def __init__( self.error_checks = (list(error_checks) if error_checks is not None else [c for c in core.DEFAULT_ERROR_CHECKS if c in self.checks]) self.rubric_items = list(rubric_items or []) - self.rubric_task_version = rubric_task_version + self.rubric_persona_sha256 = rubric_persona_sha256 self.mark_user_sim_error = mark_user_sim_error self.verifier_id = verifier_id or uuid.uuid4().hex self.judge_timeout_seconds = judge_timeout_seconds or self.DEFAULT_TIMEOUT_SECONDS @@ -134,7 +136,7 @@ def __init__( # ---- settings ---------------------------------------------------------------------------------------------- def config_problems(self) -> list[str]: return _problems(self.judge_model, self.checks, self.error_checks, self.rubric_items, self.mark_user_sim_error, - self.rubric_task_version) + self.rubric_persona_sha256) def task_problems(self, steps: list[TaskStep]) -> list[str]: """The run it judges must be an always_on or prompt_agent step's; unset, the task must have exactly one.""" @@ -154,7 +156,7 @@ def to_dict(self) -> dict: base = super().to_dict() base.update(run_step_id=self.run_step_id, judge_model=self.judge_model, checks=self.checks, error_checks=self.error_checks, rubric_items=self.rubric_items, - rubric_task_version=self.rubric_task_version, mark_user_sim_error=self.mark_user_sim_error, + rubric_persona_sha256=self.rubric_persona_sha256, mark_user_sim_error=self.mark_user_sim_error, verifier_id=self.verifier_id, judge_timeout_seconds=self.judge_timeout_seconds) return base @@ -164,7 +166,7 @@ def from_dict(cls, data: dict) -> "UserSimJudgeTaskStep": base = {**cls._base_from_dict(data), "fail_task_on_error": data.get("fail_task_on_error", False)} return cls(**base, run_step_id=data.get("run_step_id"), judge_model=data.get("judge_model"), checks=data.get("checks"), error_checks=data.get("error_checks"), - rubric_items=data.get("rubric_items"), rubric_task_version=data.get("rubric_task_version"), + rubric_items=data.get("rubric_items"), rubric_persona_sha256=data.get("rubric_persona_sha256"), mark_user_sim_error=data.get("mark_user_sim_error", False), verifier_id=data.get("verifier_id"), judge_timeout_seconds=data.get("judge_timeout_seconds")) @@ -176,13 +178,13 @@ def _settings(self, context: TaskStepContext) -> dict: error_checks = list(over["error_checks"]) if "error_checks" in over else ( [c for c in self.error_checks if c in checks] if "checks" in over else list(self.error_checks)) items = over.get("rubric_items", self.rubric_items) - version = over.get("rubric_task_version", self.rubric_task_version) + persona = over.get("rubric_persona_sha256", self.rubric_persona_sha256) mark = over.get("mark_user_sim_error", self.mark_user_sim_error) - problems = _problems(model, checks, error_checks, items, mark, version) + problems = _problems(model, checks, error_checks, items, mark, persona) if problems: raise ValueError(f"user_sim_judge '{self.id}': " + "; ".join(problems)) return {"model": model or default_judge_model(), "checks": checks, "error_checks": error_checks, - "mark": mark, "items": [core.RubricItem.from_dict(i) for i in items], "rubric_task_version": version} + "mark": mark, "items": [core.RubricItem.from_dict(i) for i in items], "rubric_persona_sha256": persona} # ---- the run ----------------------------------------------------------------------------------------------- async def execute(self, context: TaskStepContext) -> TaskStepContext: @@ -194,10 +196,6 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: "rubric_items": [i.to_dict() for i in settings["items"]], "mark_user_sim_error": settings["mark"]} items = settings["items"] - if settings["rubric_task_version"] is not None and items: - entry["rubric_version"] = await asyncio.to_thread(_rubric_version, settings["rubric_task_version"], context) - if entry["rubric_version"]["status"] == "mismatch": - items = [] record = (context.metadata.get(ALWAYS_ON_KEY) or {}).get(step_id) if kind == ALWAYS_ON else None error_type = (record or {}).get("error_type") or (response.error_type if response else None) if error_type: @@ -207,6 +205,11 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: else: run, about = await asyncio.to_thread(_read_run, kind, step_id, record, context) entry["user_sim"], entry["instructions"] = about["user_sim"], about["instructions"] + if settings["rubric_persona_sha256"] is not None and items: + us = (context.metadata.get(USER_SIMS_KEY) or {}).get(about["user_sim"]) or {} + entry["rubric_persona"] = _rubric_persona(settings["rubric_persona_sha256"], us.get("persona_sha256")) + if entry["rubric_persona"]["status"] == "mismatch": + items = [] context.record_model("user sim judge", self.id, settings["model"]) result = await core.judge_run( run, model=settings["model"], checks=settings["checks"], items=items, @@ -308,11 +311,12 @@ def _anthropic(model: str) -> bool: def _problems(model: Any, checks: Any, error_checks: Any, items: Any, mark: Any, - rubric_version: Any = None) -> list[str]: + rubric_persona: Any = None) -> list[str]: out = [] - if rubric_version is not None and (isinstance(rubric_version, bool) or not isinstance(rubric_version, int) - or rubric_version < 0): - out.append(f"rubric_task_version must be a task version (a whole number), not {rubric_version!r}") + if rubric_persona is not None and not (isinstance(rubric_persona, str) + and re.fullmatch(r"[0-9a-f]{64}", rubric_persona)): + out.append(f"rubric_persona_sha256 must be a persona fingerprint (64 lowercase hex digits), not " + f"{rubric_persona!r}") if model is not None and (not isinstance(model, str) or not model.strip()): out.append("judge_model must be a model name") if not isinstance(mark, bool): @@ -338,33 +342,17 @@ def _problems(model: Any, checks: Any, error_checks: Any, items: Any, mark: Any, return out -def task_version(context: TaskStepContext) -> Optional[int]: - """The version of the task this run is of: its task instance record's, None when there's none or it can't be - read.""" - if not context.instance_id: - return None - try: - from agent_env.task.store import find_task_instance - - instance = find_task_instance(context.instance_id) - except Exception as e: - logger.warning(f"user_sim_judge: could not read task instance {context.instance_id}: {type(e).__name__}: {e}") - return None - return instance.task_version if instance is not None else None - - -def _rubric_version(rubric_version: int, context: TaskStepContext) -> dict: - """Whether the task items were written for this run's task version: ``status`` match, mismatch (the items - aren't graded) or unknown (they are).""" - found = task_version(context) - out: dict[str, Any] = {"rubric_task_version": rubric_version, "task_version": found} - if found is None: - return {**out, "task_version": "unknown", "status": "unknown", - "note": "the run's task version couldn't be found, so the task items are graded as written"} - if found != rubric_version: +def _rubric_persona(rubric: str, run: Optional[str]) -> dict: + """Whether the task items were written for this run's user sim: ``status`` match, mismatch (the items aren't + graded) or unknown (the run recorded no ``persona_sha256``; they are).""" + out: dict[str, Any] = {"rubric": rubric, "run": run} + if run is None: + return {**out, "status": "unknown", + "note": "the run recorded no persona fingerprint, so the task items are graded as written"} + if run != rubric: return {**out, "status": "mismatch", - "note": f"the task items were written for task version {rubric_version} and this run is of version " - f"{found}, so they weren't graded; the general checks were"} + "note": "the task items were written for a different persona or opening than this run's user sim had, " + "so they weren't graded; the general checks were"} return {**out, "status": "match"} diff --git a/tst/unit/task_step/test_deploy_user_sim_agent.py b/tst/unit/task_step/test_deploy_user_sim_agent.py index c92557584..c5dcf0fc2 100644 --- a/tst/unit/task_step/test_deploy_user_sim_agent.py +++ b/tst/unit/task_step/test_deploy_user_sim_agent.py @@ -21,6 +21,7 @@ from agent_env.task_step.task_steps.agent import user_sim_versions from agent_env.task_step.task_steps.deploy_agent import DeployAgentTaskStep from agent_env.task_step.task_steps.agent.deploy_user_sim_agent import USER_SIMS_KEY, DeployUserSimAgentTaskStep +from agent_env.task_step.task_steps.agent.user_sim_common import persona_fingerprint from tst.unit.task_step.always_on_harness import USER_SIM, Rig DATA = Path(__file__).parents[2] / "data" / "user_sim" @@ -121,6 +122,38 @@ async def test_the_record_keeps_each_tag_s_line_so_an_old_used_resolves_after_th assert record["prompt_sha256"] == hashlib.sha256(PROMPT.encode()).hexdigest() +# The persona fingerprint, pinned: psi-benchmark-tasks's converter computes it too, and its test uses this sample. +FINGERPRINT_PERSONA = { + "display_name": "Hannah Reyes", + "description": {"who_you_are": ["Hannah Reyes, 32, a project manager in San Francisco."], + "decisions": ["A place at $180 a night or less: yes."]}, + "permissions": {"just_do": ["book flights"]}, + "keep_private": ["Your card number."], +} +FINGERPRINT_OPENING = "ok mom and dad are coming thu\u2013mon (nov 19\u201323), can you handle it?" +FINGERPRINT = "2d287ae4fd69f1f2398e223d7a3ed94976c912c825c606378859656b2ec2fdcd" + + +def test_the_persona_fingerprint_is_pinned(): + assert persona_fingerprint(FINGERPRINT_PERSONA, FINGERPRINT_OPENING) == FINGERPRINT + # The name and the reading settings aren't in it; the opening as written is. + assert persona_fingerprint({**FINGERPRINT_PERSONA, "display_name": "H. Reyes", "max_replies": 3}, + FINGERPRINT_OPENING) == FINGERPRINT + assert persona_fingerprint(FINGERPRINT_PERSONA, [{"kind": "text", "text": FINGERPRINT_OPENING}]) != FINGERPRINT + + +@pytest.mark.asyncio +async def test_the_record_has_the_persona_fingerprint_of_the_step_as_written(monkeypatch): + opening = [{"kind": "text", "text": "the trip, can you handle it?"}, + {"kind": "file", "file": {"uri": "s3://reyes-family//itinerary.pdf", "name": "itinerary.pdf"}}] + context = TaskStepContext() + context.metadata["seed"] = {"trip": "nov-19"} + record = await _published(monkeypatch, _step(opening_prompt=opening), context) + assert record["persona_sha256"] == persona_fingerprint(HANNAH, opening) # unseeded + assert (await _published(monkeypatch, _step()))["persona_sha256"] == persona_fingerprint( + HANNAH, PERSON["opening"]["text"]) + + @pytest.mark.asyncio async def test_a_one_sitting_user_gets_the_done_is_a_sign_off_line(monkeypatch): deploy = _Deploy(monkeypatch) diff --git a/tst/unit/task_step/verifiers/test_user_sim_judge.py b/tst/unit/task_step/verifiers/test_user_sim_judge.py index e906bc8a2..3d6b573c4 100644 --- a/tst/unit/task_step/verifiers/test_user_sim_judge.py +++ b/tst/unit/task_step/verifiers/test_user_sim_judge.py @@ -26,6 +26,7 @@ from tst.unit.task_step.test_prompt_agent_user_side import AGENT, USER, Peers, text JUDGE = "fake/judge" +OTHER_PERSONA = "0" * 64 # a persona fingerprint no run here has TASK_ITEMS_Q = "task's own rubric items" @@ -707,10 +708,11 @@ def test_the_craftweb_item_round_trips_and_is_asked_as_written(): assert f"- a1_call_rae (A1: Requirement provision; hard): {CALL_RAE['check']}" in q assert " n/a when: The assistant didn't raise Rae's order." in q assert CALL_RAE["name"] not in q and CALL_RAE["short"] not in q # for reviewers, not the judge - step = usj.UserSimJudgeTaskStep("s", None, rubric_items=[CALL_RAE, HER_LENGTH, SATISFACTION], rubric_task_version=7) + step = usj.UserSimJudgeTaskStep("s", None, rubric_items=[CALL_RAE, HER_LENGTH, SATISFACTION], + rubric_persona_sha256=OTHER_PERSONA) again = usj.UserSimJudgeTaskStep.from_dict(json.loads(json.dumps(step.to_dict()))) assert step.config_problems() == [] and again.to_dict() == step.to_dict() - assert (again.rubric_items, again.rubric_task_version) == ([CALL_RAE, HER_LENGTH, SATISFACTION], 7) + assert (again.rubric_items, again.rubric_persona_sha256) == ([CALL_RAE, HER_LENGTH, SATISFACTION], OTHER_PERSONA) def test_the_old_text_field_still_loads_as_the_check_and_kind_defaults_to_general(): @@ -866,8 +868,9 @@ def test_each_bad_item_is_refused_when_the_task_loads_with_a_clear_error(): usj.UserSimJudgeTaskStep("s", None, rubric_items=[item])._settings(TaskStepContext()) dup = core.item_problems([CALL_RAE, {**CALL_RAE, "family": "A3"}]) assert dup == ["rubric_items[1] (a1_call_rae): id 'a1_call_rae' is used twice"] - assert any("rubric_task_version must be a task version" in p for p in usj.UserSimJudgeTaskStep( - "s", None, rubric_items=[CALL_RAE], rubric_task_version="7").config_problems()) + for bad in (7, "3f1c", "A" * 64): + assert any("rubric_persona_sha256 must be a persona fingerprint" in p for p in usj.UserSimJudgeTaskStep( + "s", None, rubric_items=[CALL_RAE], rubric_persona_sha256=bad).config_problems()) def test_an_auto_item_loads_with_no_na_when_or_with_always(): @@ -876,27 +879,21 @@ def test_an_auto_item_loads_with_no_na_when_or_with_always(): assert core.item_problems([{**auto, "na_when": "always", "static_na_reason": "She never texts here."}]) == [] -# ---- the rubric's task version -------------------------------------------------------------------------------------- -def _versioned(ctx: TaskStepContext, version: int) -> None: - from agent_env.task.store import register_task_instance - - ctx.instance_id = register_task_instance("psi-ts-craftweb-holiday-rush", version, 3).instance_id - - +# ---- the rubric's persona ------------------------------------------------------------------------------------------ @pytest.mark.usefixtures("local_stores") @pytest.mark.asyncio -async def test_items_written_for_another_task_version_aren_t_graded_and_the_checks_still_run(monkeypatch): +async def test_items_written_for_another_persona_aren_t_graded_and_the_checks_still_run(monkeypatch): ctx = await _chat(monkeypatch) - _versioned(ctx, 8) + run = ctx.metadata[USER_SIMS_KEY]["hannah"]["persona_sha256"] fake = FakeModel({"guidance": _fail()}) - await _step(fake, monkeypatch, rubric_items=[CALL_RAE, HER_LENGTH], rubric_task_version=7).execute(ctx) + await _step(fake, monkeypatch, rubric_items=[CALL_RAE, HER_LENGTH], rubric_persona_sha256=OTHER_PERSONA).execute(ctx) entry = _entry(ctx) - assert entry["rubric_version"] == { - "rubric_task_version": 7, "task_version": 8, "status": "mismatch", - "note": "the task items were written for task version 7 and this run is of version 8, so they weren't graded; " - "the general checks were"} + assert entry["rubric_persona"] == { + "rubric": OTHER_PERSONA, "run": run, "status": "mismatch", + "note": "the task items were written for a different persona or opening than this run's user sim had, so they " + "weren't graded; the general checks were"} assert entry["items"] == {} and len(fake.calls) == 7 assert all("a1_call_rae" not in c["parts"][-1] for c in fake.calls) assert entry["checks"]["guidance"]["verdict"] == "fail" and entry["user_sim_error"]["because"] == ["guidance"] @@ -904,20 +901,22 @@ async def test_items_written_for_another_task_version_aren_t_graded_and_the_chec @pytest.mark.usefixtures("local_stores") @pytest.mark.asyncio -@pytest.mark.parametrize("version, status", [(7, "match"), (None, "unknown")]) -async def test_items_for_this_task_version_or_an_unknown_one_are_graded(monkeypatch, version, status): +@pytest.mark.parametrize("recorded, status", [(True, "match"), (False, "unknown")]) +async def test_items_for_this_persona_or_a_run_with_no_fingerprint_are_graded(monkeypatch, recorded, status): ctx = await _chat(monkeypatch) - if version is not None: - _versioned(ctx, version) + us = ctx.metadata[USER_SIMS_KEY]["hannah"] + fingerprint = us["persona_sha256"] if recorded else OTHER_PERSONA + if not recorded: + del us["persona_sha256"] # a run from before the step recorded it await _step(FakeModel(), monkeypatch, checks=["style"], rubric_items=[HER_LENGTH], - rubric_task_version=7).execute(ctx) + rubric_persona_sha256=fingerprint).execute(ctx) entry = _entry(ctx) - assert entry["rubric_version"]["status"] == status and entry["items"]["f2_her_length"]["verdict"] == "pass" - if version is None: - assert entry["rubric_version"]["task_version"] == "unknown" - assert "graded as written" in entry["rubric_version"]["note"] - # Without a rubric_task_version nothing is compared. + assert entry["rubric_persona"]["status"] == status and entry["items"]["f2_her_length"]["verdict"] == "pass" + assert entry["rubric_persona"]["rubric"] == fingerprint + if not recorded: + assert entry["rubric_persona"]["run"] is None and "graded as written" in entry["rubric_persona"]["note"] + # Without a rubric_persona_sha256 nothing is compared. await _step(FakeModel(), monkeypatch, checks=["style"], rubric_items=[HER_LENGTH]).execute(ctx) - assert "rubric_version" not in _entry(ctx) and _entry(ctx)["items"]["f2_her_length"]["verdict"] == "pass" + assert "rubric_persona" not in _entry(ctx) and _entry(ctx)["items"]["f2_her_length"]["verdict"] == "pass" From 3f44808ed3665f386d1b075abac5ba3a448d9ad5 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Fri, 13 Nov 2026 19:31:14 -0800 Subject: [PATCH 23/45] user_sim_judge: an auto item can't pass an answer the record kept only the start of When always_on's stored timeline can't be read, the judge falls back to the record's timeline_tail, which keeps each message's first 300 characters. A length item counted on that could pass an answer that was over its limit. Such answers are now marked cut; over the max as kept is still a fail, otherwise the item is error (unknown), not pass. Co-Authored-By: Claude Opus 5.5 --- .../task_steps/verifiers/user_sim_judge.py | 14 +++++++-- .../verifiers/user_sim_judge_core.py | 8 ++++- .../verifiers/test_user_sim_judge.py | 29 +++++++++++++++++++ 3 files changed, 47 insertions(+), 4 deletions(-) diff --git a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py index 4ab13aad7..e2ca71be2 100644 --- a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py +++ b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py @@ -23,7 +23,8 @@ - ``kind``: ``hard`` (an exact rule from the user sim's instructions: the judge fails it on any breach), ``general`` (a judgement: it fails only on a clear instance; the default) or ``auto`` (never sent to the model: ``metric`` ``max_words`` or ``max_sentences`` is counted on each graded answer, and any answer over ``max`` fails it, naming - the turn; see ``user_sim_judge_core.count_sentences``). + the turn; see ``user_sim_judge_core.count_sentences``). If none is over but the record kept only the start of one + (always_on's tail, when its stored timeline can't be read), the item is ``error``: that length can't be counted. - ``check``: the text the judge applies, required unless auto. ``text`` is its old name and still loads. - ``name`` and ``short``: a title and one line for reviewers, recorded but not shown to the judge. - ``na_when``: the situation in which it never came up, told to the judge. ``always`` makes it n/a with no call, @@ -407,7 +408,13 @@ def _always_on_reads(record: dict) -> tuple[list[dict], list[str]]: "what the user sim was sent, are shown") else: note = "the run's full record wasn't stored; only its last events, without what the user sim was sent, are shown" - reads = [e for e in record.get("timeline_tail") or [] if e.get("event") == "user_sim_read"] + from agent_env.task_step.task_steps.always_on import PENDING_TEXT_CHARS + + reads = [] + for e in record.get("timeline_tail") or []: + if e.get("event") == "user_sim_read": + message = e.get("message") or "" # the tail keeps each string's first PENDING_TEXT_CHARS, then "…" + reads.append({**e, "message_cut": len(message) == PENDING_TEXT_CHARS + 1 and message.endswith("…")}) return reads, [note] @@ -464,7 +471,8 @@ def _turn(n: int, e: dict) -> core.Turn: flags.append(FLAG_LAST) when = e.get("virtual_time") or (e.get("read") or {}).get("time") return core.Turn(n=n, outcome=e.get("outcome") or "no_reply", message=e.get("message") or "", - input=e.get("input"), flags=flags, time=None if e.get("input") or not when else f"{when} UTC") + input=e.get("input"), flags=flags, time=None if e.get("input") or not when else f"{when} UTC", + cut=bool(e.get("message_cut"))) def _instructions(us: Optional[dict], seed: dict, multi_day: Optional[bool]) -> tuple[Optional[str], dict, list[str]]: diff --git a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py index e145c17cf..15942e7ce 100644 --- a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py +++ b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py @@ -295,6 +295,7 @@ class Turn: time: Optional[str] = None graded: bool = True flags: list[str] = field(default_factory=list) + cut: bool = False # ``message`` is only the start of what it sent (the record kept a shortened copy) @dataclass @@ -461,10 +462,15 @@ def auto_result(item: RubricItem, run: "UserSimRun") -> dict: if not counts: return {"verdict": "n/a", "reason": "it sent no message at a graded turn", "turns": []} over = [(n, c) for n, c in counts if c > item.max] - if over: + if over: # a shortened answer over the max is over it whole too listed = ", ".join(f"turn {n}: {c}" for n, c in over) return {"verdict": "fail", "reason": f"over the {item.max} {unit} max at {listed}", "turns": [n for n, _ in over]} + cut = [t.n for t in sent_answers(run) if t.cut] + if cut: + return {"verdict": "error", "turns": cut, + "error": f"only the start of the answer at turn{'s' if len(cut) > 1 else ''} " + f"{', '.join(map(str, cut))} was kept, so its full length can't be counted"} n, c = max(counts, key=lambda x: x[1]) return {"verdict": "pass", "reason": f"every answer at most {item.max} {unit}; the longest, turn {n}, has {c}", "turns": []} diff --git a/tst/unit/task_step/verifiers/test_user_sim_judge.py b/tst/unit/task_step/verifiers/test_user_sim_judge.py index 3d6b573c4..4066699e3 100644 --- a/tst/unit/task_step/verifiers/test_user_sim_judge.py +++ b/tst/unit/task_step/verifiers/test_user_sim_judge.py @@ -6,6 +6,7 @@ import hashlib import json import re +from dataclasses import replace import pytest @@ -414,6 +415,18 @@ async def test_without_the_stored_timeline_the_judge_reads_the_tail_and_is_told_ assert "## Turn 1 · 2026-11-09T14:00:00Z UTC · GRADED" in head +@pytest.mark.usefixtures("local_stores") +@pytest.mark.asyncio +async def test_from_the_tail_an_answer_always_on_shortened_is_marked_cut(monkeypatch, tmp_path): + from agent_env.task_step.task_steps.always_on import PENDING_TEXT_CHARS, _small + + record = {"timeline_tail": [_small({"event": "user_sim_read", "outcome": "reply", "message": "ok " * 200}), + _small({"event": "user_sim_read", "outcome": "reply", "message": "sounds good"})]} + reads, notes = usj._always_on_reads(record) + assert [r["message_cut"] for r in reads] == [True, False] and len(reads[0]["message"]) == PENDING_TEXT_CHARS + 1 + assert usj._turn(1, reads[0]).cut and not usj._turn(2, reads[1]).cut + + @pytest.mark.usefixtures("local_stores") @pytest.mark.asyncio async def test_a_run_that_ended_in_an_error_is_not_judged(monkeypatch, tmp_path): @@ -762,6 +775,22 @@ async def test_an_auto_item_is_counted_on_the_graded_answers_with_no_model_call( assert all("f2_her_length" not in c["parts"][-1] for c in fake.calls) and len(fake.calls) == 7 +@pytest.mark.asyncio +async def test_an_auto_item_can_t_pass_an_answer_the_record_kept_only_the_start_of(): + [length] = _items(HER_LENGTH) + short = _run(("reply", TESS[0]), ("reply", TESS[1])) + short.turns[1] = replace(short.turns[1], cut=True) # under 25 words as kept, but the whole answer may not be + result = await _judge(short, FakeModel(), checks=[], items=[length]) + assert result["items"]["f2_her_length"] == { + "verdict": "error", "turns": [2], "family": "F2", "kind": "auto", "asked_with": "computed", + "error": "only the start of the answer at turn 2 was kept, so its full length can't be counted"} + # Over the max as kept is over it whole: still a fail. + long = "word " * 30 + over = _run(("reply", long.strip()), ("reply", TESS[1])) + over.turns[0] = replace(over.turns[0], cut=True) + assert (await _judge(over, FakeModel(), checks=[], items=[length]))["items"]["f2_her_length"]["verdict"] == "fail" + + @pytest.mark.asyncio async def test_an_auto_item_counts_sentences_simply(): assert [core.count_sentences(t) for t in TESS] == [3, 2] From 97c2ff184e1de05894e3db93da7fe6081cd01ead Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Sat, 14 Nov 2026 11:50:11 -0800 Subject: [PATCH 24/45] fix(user_sim): one section for what the user never gives (keep_private); a should-already-know correction holds even where other lines let things slide never_supply said the same as keep_private, so it's gone: its lines go in keep_private (PRIVATE, tagged P), and a task that still has never_supply is refused at load with a message saying to move them there. The C heading drops "and add nothing else" and says the correction holds even when the user sim would otherwise let something slide. Personas with neither section render byte for byte as before. Co-Authored-By: Claude Opus 5.5 --- .../task_step/task_steps/agent/user_sim.md | 12 +++++++----- .../task_steps/agent/user_sim_versions.py | 12 +++++++++--- tst/data/user_sim/hannah_person.json | 6 ++---- tst/data/user_sim/hannah_system_prompt.txt | 4 +--- tst/unit/task_step/test_deploy_user_sim_agent.py | 2 ++ tst/unit/task_step/test_user_sim_versions.py | 16 ++++++++++++++-- 6 files changed, 35 insertions(+), 17 deletions(-) diff --git a/src/agent_env/task_step/task_steps/agent/user_sim.md b/src/agent_env/task_step/task_steps/agent/user_sim.md index baf62e21e..1ec738f75 100644 --- a/src/agent_env/task_step/task_steps/agent/user_sim.md +++ b/src/agent_env/task_step/task_steps/agent/user_sim.md @@ -105,10 +105,9 @@ resolves there. | `how_you_write` | | How they text, best shown with 2–3 of their real messages | | `what_you_know` | K | Facts they give only when asked | | `already_told` | T | Optional. Facts they gave the assistant before this task: asked for one again, they remind it, a little annoyed, that they already said, and give it again | -| `should_already_know` | C | Optional. What the assistant should know without being told: they never bring it up, and if the assistant gets it wrong they correct it briefly, a little irritated, and add nothing else | +| `should_already_know` | C | Optional. What the assistant should know without being told: they never bring it up, and if the assistant gets it wrong they correct it briefly, a little irritated, even where other lines say to let things slide | | `what_to_look_up` | L | Facts the assistant must find: say where they are ("it's in mom's email"), never the value | | `decisions` | D | What they say when the assistant asks or proposes something | -| `never_supply` | N | What they never hand over (card numbers, codes, anything the assistant should find) | | `how_you_react` | H | What they point out, what they let go, which texts get no reply | | `dont_mention` | M | One "don't mention X unless asked" line per tempting fact | | `phone` | F | What they do with what reaches their phone | @@ -117,14 +116,17 @@ resolves there. `already_told` and `should_already_know` are for what the assistant has from before this task. Each "should already know" line must also be in the assistant's own instructions or notes, and each "already told" fact somewhere it can read, or the user sim expects knowledge the assistant was never given. A fact the assistant should find in the apps -goes in `what_to_look_up` or `never_supply` instead: an "already told" fact is given again when asked. +goes in `what_to_look_up` instead: an "already told" fact is given again when asked. Don't write the opening request into the description. Put it in `opening_prompt`: the prompt shows it right after the first `who_you_are` line ("Earlier you texted the assistant: …"). `permissions` become the "YOUR RULES FOR THE ASSISTANT" lines. Its kinds are `just_do`, `ask_first`, `never` and -`tell_after`. Each is a list of rules, or `{items, then}` to add what follows. `keep_private` becomes the "PRIVATE" -lines. Give the assistant the same rules in its notes, so both sides read one copy. +`tell_after`. Each is a list of rules, or `{items, then}` to add what follows. Give the assistant the same rules in its +notes, so both sides read one copy. + +`keep_private` is what the user never says or hands over, even if asked: card numbers, codes, passwords, private +facts. Its lines become the "PRIVATE" lines, tagged P1…, so `used` can name them. ## The rating at the end diff --git a/src/agent_env/task_step/task_steps/agent/user_sim_versions.py b/src/agent_env/task_step/task_steps/agent/user_sim_versions.py index 55eee2d03..f423ac147 100644 --- a/src/agent_env/task_step/task_steps/agent/user_sim_versions.py +++ b/src/agent_env/task_step/task_steps/agent/user_sim_versions.py @@ -35,10 +35,10 @@ ("already_told", "THINGS YOU ALREADY TOLD THE ASSISTANT (if it asks for one again, remind it, a little annoyed, " "that you already told it, and tell it again)", "T"), ("should_already_know", "THINGS THE ASSISTANT SHOULD ALREADY KNOW (never bring these up; if it gets one wrong, " - "correct it briefly, a little irritated, and add nothing else)", "C"), + "correct it briefly, a little irritated, even when you'd otherwise let something slide)", + "C"), ("what_to_look_up", "WHAT TO LOOK UP (say where; don't give the value)", "L"), ("decisions", "YOUR DECISIONS (when the assistant asks)", "D"), - ("never_supply", "NEVER SUPPLY (the assistant must find these itself)", "N"), ("how_you_react", "HOW YOU REACT", "H"), ("dont_mention", "DON'T MENTION THESE UNLESS THE ASSISTANT ASKS", "M"), ("phone", "WHAT YOU DO WITH YOUR EMAILS, TEXTS AND NOTIFICATIONS", "F"), @@ -48,6 +48,11 @@ RULES_TITLE = "YOUR RULES FOR THE ASSISTANT (it has them in its notes)" PRIVATE_TITLE = "PRIVATE" RULES_AFTER = "decisions" +# A description section that isn't one, with where its lines go: a task that has it is refused at load with that. +MOVED_SECTIONS: dict[str, str] = { + "never_supply": "put never_supply's lines in keep_private: it holds what the user never says or hands over, even " + "if asked", +} PERMISSION_KINDS: dict[str, str] = { "just_do": "Just do", @@ -242,7 +247,8 @@ def problems(person: Any) -> list[str]: if unknown: out.append(f"description has unknown sections {unknown}; the sections are {list(SECTION_KEYS)}" + ("; satisfaction goes on the rate_user_satisfaction step, so the user sim's prompt never has " - "it" if "satisfaction" in unknown else "")) + "it" if "satisfaction" in unknown else "") + + "".join(f"; {MOVED_SECTIONS[key]}" for key in unknown if key in MOVED_SECTIONS)) out += [f"description.{key} must be a list of non-empty lines" for key in SECTION_KEYS if key in description and not _lines(description[key])] if not any(_lines(description.get(key)) and description[key] for key in SECTION_KEYS): diff --git a/tst/data/user_sim/hannah_person.json b/tst/data/user_sim/hannah_person.json index d1923b9fa..fc8680c23 100644 --- a/tst/data/user_sim/hannah_person.json +++ b/tst/data/user_sim/hannah_person.json @@ -36,9 +36,6 @@ "Small things that make it easier for Grandma (a seat next to Mom, a shower chair, a late checkout for about $40): yes. Extras for show (wine pairing, set menus, travel insurance): no.", "Anything else clearly within the plan: decide quickly the way you would; don't hand it back." ], - "never_supply": [ - "Never supply facts the assistant should look up itself (flight numbers, prices, times, confirmation codes, listing or restaurant names, deadlines), card numbers, passwords, or anyone's birth date or ID. For a card, say \"use the amex on file\". If it asks something you don't know, say so." - ], "how_you_react": [ "Never bring up anything from \"what you know\" or \"what you learn\" on your own, and never tell it to go read an email: it reads your email itself. You skim its texts: point out a mistake only when it's a wrong date, a wrong headcount, a place with steps once Grandma's coming, or a Friday dinner before 7, and let everything else go (flight numbers, exact times, prices, codes).", "If one text asks several things, answer all of them in one short text, in order. Texts that don't need you (status updates, \"all good!\") get a quick 👍 or no answer at all (answer \"no_reply\"); if they keep coming, you can say once that you only need to hear when there's a decision or at the end.", @@ -93,7 +90,8 @@ } }, "keep_private": [ - "Ben proposed at Verbena Hall on October 24 and you're telling your parents at Friday dinner. Nobody knows. Never mention or hint at it. If asked about an occasion: \"just a family dinner\". If asked why Verbena Hall: \"it's our spot\"." + "Ben proposed at Verbena Hall on October 24 and you're telling your parents at Friday dinner. Nobody knows. Never mention or hint at it. If asked about an occasion: \"just a family dinner\". If asked why Verbena Hall: \"it's our spot\".", + "Never supply facts the assistant should look up itself (flight numbers, prices, times, confirmation codes, listing or restaurant names, deadlines), card numbers, passwords, or anyone's birth date or ID. For a card, say \"use the amex on file\". If it asks something you don't know, say so." ], "opening": { "at": "2026-11-16T15:50:00Z", diff --git a/tst/data/user_sim/hannah_system_prompt.txt b/tst/data/user_sim/hannah_system_prompt.txt index 5e6816a33..c7984182f 100644 --- a/tst/data/user_sim/hannah_system_prompt.txt +++ b/tst/data/user_sim/hannah_system_prompt.txt @@ -49,9 +49,7 @@ YOUR RULES FOR THE ASSISTANT (it has them in its notes) PRIVATE P1 Ben proposed at Verbena Hall on October 24 and you're telling your parents at Friday dinner. Nobody knows. Never mention or hint at it. If asked about an occasion: "just a family dinner". If asked why Verbena Hall: "it's our spot". - -NEVER SUPPLY (the assistant must find these itself) - N1 Never supply facts the assistant should look up itself (flight numbers, prices, times, confirmation codes, listing or restaurant names, deadlines), card numbers, passwords, or anyone's birth date or ID. For a card, say "use the amex on file". If it asks something you don't know, say so. + P2 Never supply facts the assistant should look up itself (flight numbers, prices, times, confirmation codes, listing or restaurant names, deadlines), card numbers, passwords, or anyone's birth date or ID. For a card, say "use the amex on file". If it asks something you don't know, say so. HOW YOU REACT H1 Never bring up anything from "what you know" or "what you learn" on your own, and never tell it to go read an email: it reads your email itself. You skim its texts: point out a mistake only when it's a wrong date, a wrong headcount, a place with steps once Grandma's coming, or a Friday dinner before 7, and let everything else go (flight numbers, exact times, prices, codes). diff --git a/tst/unit/task_step/test_deploy_user_sim_agent.py b/tst/unit/task_step/test_deploy_user_sim_agent.py index c92557584..20ab509e5 100644 --- a/tst/unit/task_step/test_deploy_user_sim_agent.py +++ b/tst/unit/task_step/test_deploy_user_sim_agent.py @@ -175,6 +175,8 @@ def test_the_step_round_trips_and_holds_no_system_prompt(): ({"persona": {**HANNAH, "description": None}}, "description must be an object of sections"), ({"persona": {**HANNAH, "description": {"who_you_are": ["Linda."], "events": ["Tue"]}}}, "unknown sections ['events']"), + ({"persona": {**HANNAH, "description": {"who_you_are": ["Linda."], "never_supply": ["Her card number."]}}}, + "put never_supply's lines in keep_private"), ({"persona": {**HANNAH, "permissions": {"maybe": ["book it"]}}}, "permissions has unknown kinds ['maybe']"), ({"persona": {**HANNAH, "keep_private": "her age"}}, "keep_private must be a list of non-empty lines"), ({"persona": {**HANNAH, "schedule": ["07:30", "23:30"]}}, "schedule must be an object"), diff --git a/tst/unit/task_step/test_user_sim_versions.py b/tst/unit/task_step/test_user_sim_versions.py index 8b810d0b6..d601e18aa 100644 --- a/tst/unit/task_step/test_user_sim_versions.py +++ b/tst/unit/task_step/test_user_sim_versions.py @@ -82,8 +82,8 @@ def test_things_already_told_and_things_it_should_know_show_tagged_after_what_yo told = ("\n\nTHINGS YOU ALREADY TOLD THE ASSISTANT (if it asks for one again, remind it, a little annoyed, that you " "already told it, and tell it again)\n T1 Dad can't do stairs since his knee surgery.\n") known = ("\n\nTHINGS THE ASSISTANT SHOULD ALREADY KNOW (never bring these up; if it gets one wrong, correct it " - "briefly, a little irritated, and add nothing else)\n C1 You never book anything before 9am for your " - "parents.\n") + "briefly, a little irritated, even when you'd otherwise let something slide)\n C1 You never book " + "anything before 9am for your parents.\n") assert told in prompt and known in prompt assert prompt.index("\nWHAT YOU KNOW") < prompt.index(told) < prompt.index(known) < prompt.index("\nWHAT TO LOOK UP") lines = us.tagged_lines(person) @@ -98,6 +98,14 @@ def test_without_them_the_prompt_and_tags_are_as_before(): assert not any(tag[0] in "TC" for tag in us.tagged_lines(PERSON)) +def test_keep_private_lines_show_under_private_tagged_p_so_used_can_name_them(): + prompt = us.system_prompt(PERSON) + assert "\n\nPRIVATE\n P1 Ben proposed" in prompt and "\n P2 Never supply facts" in prompt + lines = us.tagged_lines(PERSON) + assert [lines["P1"], lines["P2"]] == PERSON["keep_private"] + assert not any(tag.startswith("N") for tag in lines) and "NEVER SUPPLY" not in prompt + + def test_a_line_s_later_lines_hang_under_its_text(): assert us.tag_line("W2", 'Earlier you texted:\n"hi"') == ' W2 Earlier you texted:\n "hi"' assert us.tag_line("", "Answer short.\nNo emojis.") == " Answer short.\n No emojis." @@ -116,6 +124,10 @@ def test_the_tested_settings_have_no_problems(): ({"description": {"who_you_are": []}}, "description is empty"), ({"description": {"who_you_are": ["Hannah."], "events": ["Tue: an email"]}}, "unknown sections ['events']"), ({"description": {"who_you_are": ["Hannah.", ""]}}, "description.who_you_are must be a list of non-empty lines"), + ({"description": {"who_you_are": ["Hannah."], "never_supply": ["Your card number."]}}, + "unknown sections ['never_supply']"), + ({"description": {"who_you_are": ["Hannah."], "never_supply": ["Your card number."]}}, + "put never_supply's lines in keep_private: it holds what the user never says or hands over, even if asked"), ({"description": {"who_you_are": ["Hannah."], "already_told": "Dad can't do stairs."}}, "description.already_told must be a list of non-empty lines"), ({"description": {"who_you_are": ["Hannah."], "already_told": [" "]}}, From f3b520d3dd92d0eb39a05d55c1ddd3026a2f71b1 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Sat, 14 Nov 2026 12:01:41 -0800 Subject: [PATCH 25/45] fix(user_sim): never_supply's look-up facts belong in what_to_look_up, its secrets in keep_private never_supply meant "don't hand over facts the assistant should look up itself", which is what_to_look_up's job, so the load refusal for it now says to put those facts in what_to_look_up and secrets in keep_private. keep_private's docs say only what the user never says even if asked (secrets, card numbers, codes); what_to_look_up's say the value is never handed over. The test persona's never_supply line is split the same way: its look-up facts are L2, its card numbers, passwords and IDs are P2. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/task_step/task_steps/agent/user_sim.md | 6 +++--- .../task_step/task_steps/agent/user_sim_versions.py | 3 +-- tst/data/user_sim/hannah_person.json | 5 +++-- tst/data/user_sim/hannah_system_prompt.txt | 3 ++- tst/unit/task_step/test_deploy_user_sim_agent.py | 2 +- tst/unit/task_step/test_user_sim_versions.py | 4 ++-- 6 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/agent_env/task_step/task_steps/agent/user_sim.md b/src/agent_env/task_step/task_steps/agent/user_sim.md index 1ec738f75..aff2aaf66 100644 --- a/src/agent_env/task_step/task_steps/agent/user_sim.md +++ b/src/agent_env/task_step/task_steps/agent/user_sim.md @@ -106,7 +106,7 @@ resolves there. | `what_you_know` | K | Facts they give only when asked | | `already_told` | T | Optional. Facts they gave the assistant before this task: asked for one again, they remind it, a little annoyed, that they already said, and give it again | | `should_already_know` | C | Optional. What the assistant should know without being told: they never bring it up, and if the assistant gets it wrong they correct it briefly, a little irritated, even where other lines say to let things slide | -| `what_to_look_up` | L | Facts the assistant must find: say where they are ("it's in mom's email"), never the value | +| `what_to_look_up` | L | Facts the assistant must find itself: say where they are ("it's in mom's email"), never hand over the value | | `decisions` | D | What they say when the assistant asks or proposes something | | `how_you_react` | H | What they point out, what they let go, which texts get no reply | | `dont_mention` | M | One "don't mention X unless asked" line per tempting fact | @@ -125,8 +125,8 @@ after the first `who_you_are` line ("Earlier you texted the assistant: …"). `tell_after`. Each is a list of rules, or `{items, then}` to add what follows. Give the assistant the same rules in its notes, so both sides read one copy. -`keep_private` is what the user never says or hands over, even if asked: card numbers, codes, passwords, private -facts. Its lines become the "PRIVATE" lines, tagged P1…, so `used` can name them. +`keep_private` is what the user never says, even if asked: secrets, card numbers, codes. Its lines become the +"PRIVATE" lines, tagged P1…, so `used` can name them. ## The rating at the end diff --git a/src/agent_env/task_step/task_steps/agent/user_sim_versions.py b/src/agent_env/task_step/task_steps/agent/user_sim_versions.py index f423ac147..c1bf4da53 100644 --- a/src/agent_env/task_step/task_steps/agent/user_sim_versions.py +++ b/src/agent_env/task_step/task_steps/agent/user_sim_versions.py @@ -50,8 +50,7 @@ RULES_AFTER = "decisions" # A description section that isn't one, with where its lines go: a task that has it is refused at load with that. MOVED_SECTIONS: dict[str, str] = { - "never_supply": "put never_supply's lines in keep_private: it holds what the user never says or hands over, even " - "if asked", + "never_supply": "put facts the assistant should find in what_to_look_up, and secrets in keep_private", } PERMISSION_KINDS: dict[str, str] = { diff --git a/tst/data/user_sim/hannah_person.json b/tst/data/user_sim/hannah_person.json index fc8680c23..2ead7f24a 100644 --- a/tst/data/user_sim/hannah_person.json +++ b/tst/data/user_sim/hannah_person.json @@ -19,7 +19,8 @@ "Grandma flies again; her hip healed this year." ], "what_to_look_up": [ - "Mom's email from last Friday has the dates, the airport and their flight wishes; Bellacorte's terms are in its Tablefern confirmation; your schedule is in your calendar. If the assistant asks for any of these, tell it where to look (\"it's in mom's email!\"), a little impatiently." + "Mom's email from last Friday has the dates, the airport and their flight wishes; Bellacorte's terms are in its Tablefern confirmation; your schedule is in your calendar. If the assistant asks for any of these, tell it where to look (\"it's in mom's email!\"), a little impatiently.", + "Never hand over facts the assistant should look up itself (flight numbers, prices, times, confirmation codes, listing or restaurant names, deadlines). If it asks something you don't know, say so." ], "decisions": [ "Mom and Dad's flights within your rules (nonstop, economy but never basic, leaving after 8am, landing in SF while it's light, up to $450 a person round trip): yes, and it didn't need to ask.", @@ -91,7 +92,7 @@ }, "keep_private": [ "Ben proposed at Verbena Hall on October 24 and you're telling your parents at Friday dinner. Nobody knows. Never mention or hint at it. If asked about an occasion: \"just a family dinner\". If asked why Verbena Hall: \"it's our spot\".", - "Never supply facts the assistant should look up itself (flight numbers, prices, times, confirmation codes, listing or restaurant names, deadlines), card numbers, passwords, or anyone's birth date or ID. For a card, say \"use the amex on file\". If it asks something you don't know, say so." + "Card numbers, passwords, and anyone's birth date or ID: never give them, even if asked. For a card, say \"use the amex on file\"." ], "opening": { "at": "2026-11-16T15:50:00Z", diff --git a/tst/data/user_sim/hannah_system_prompt.txt b/tst/data/user_sim/hannah_system_prompt.txt index c7984182f..9b40b596f 100644 --- a/tst/data/user_sim/hannah_system_prompt.txt +++ b/tst/data/user_sim/hannah_system_prompt.txt @@ -24,6 +24,7 @@ WHAT YOU KNOW (to use only when the assistant asks about it) WHAT TO LOOK UP (say where; don't give the value) L1 Mom's email from last Friday has the dates, the airport and their flight wishes; Bellacorte's terms are in its Tablefern confirmation; your schedule is in your calendar. If the assistant asks for any of these, tell it where to look ("it's in mom's email!"), a little impatiently. + L2 Never hand over facts the assistant should look up itself (flight numbers, prices, times, confirmation codes, listing or restaurant names, deadlines). If it asks something you don't know, say so. YOUR DECISIONS (when the assistant asks) D1 Mom and Dad's flights within your rules (nonstop, economy but never basic, leaving after 8am, landing in SF while it's light, up to $450 a person round trip): yes, and it didn't need to ask. @@ -49,7 +50,7 @@ YOUR RULES FOR THE ASSISTANT (it has them in its notes) PRIVATE P1 Ben proposed at Verbena Hall on October 24 and you're telling your parents at Friday dinner. Nobody knows. Never mention or hint at it. If asked about an occasion: "just a family dinner". If asked why Verbena Hall: "it's our spot". - P2 Never supply facts the assistant should look up itself (flight numbers, prices, times, confirmation codes, listing or restaurant names, deadlines), card numbers, passwords, or anyone's birth date or ID. For a card, say "use the amex on file". If it asks something you don't know, say so. + P2 Card numbers, passwords, and anyone's birth date or ID: never give them, even if asked. For a card, say "use the amex on file". HOW YOU REACT H1 Never bring up anything from "what you know" or "what you learn" on your own, and never tell it to go read an email: it reads your email itself. You skim its texts: point out a mistake only when it's a wrong date, a wrong headcount, a place with steps once Grandma's coming, or a Friday dinner before 7, and let everything else go (flight numbers, exact times, prices, codes). diff --git a/tst/unit/task_step/test_deploy_user_sim_agent.py b/tst/unit/task_step/test_deploy_user_sim_agent.py index 20ab509e5..a927e497b 100644 --- a/tst/unit/task_step/test_deploy_user_sim_agent.py +++ b/tst/unit/task_step/test_deploy_user_sim_agent.py @@ -176,7 +176,7 @@ def test_the_step_round_trips_and_holds_no_system_prompt(): ({"persona": {**HANNAH, "description": {"who_you_are": ["Linda."], "events": ["Tue"]}}}, "unknown sections ['events']"), ({"persona": {**HANNAH, "description": {"who_you_are": ["Linda."], "never_supply": ["Her card number."]}}}, - "put never_supply's lines in keep_private"), + "put facts the assistant should find in what_to_look_up, and secrets in keep_private"), ({"persona": {**HANNAH, "permissions": {"maybe": ["book it"]}}}, "permissions has unknown kinds ['maybe']"), ({"persona": {**HANNAH, "keep_private": "her age"}}, "keep_private must be a list of non-empty lines"), ({"persona": {**HANNAH, "schedule": ["07:30", "23:30"]}}, "schedule must be an object"), diff --git a/tst/unit/task_step/test_user_sim_versions.py b/tst/unit/task_step/test_user_sim_versions.py index d601e18aa..fb6fecc62 100644 --- a/tst/unit/task_step/test_user_sim_versions.py +++ b/tst/unit/task_step/test_user_sim_versions.py @@ -100,7 +100,7 @@ def test_without_them_the_prompt_and_tags_are_as_before(): def test_keep_private_lines_show_under_private_tagged_p_so_used_can_name_them(): prompt = us.system_prompt(PERSON) - assert "\n\nPRIVATE\n P1 Ben proposed" in prompt and "\n P2 Never supply facts" in prompt + assert "\n\nPRIVATE\n P1 Ben proposed" in prompt and "\n P2 Card numbers, passwords" in prompt lines = us.tagged_lines(PERSON) assert [lines["P1"], lines["P2"]] == PERSON["keep_private"] assert not any(tag.startswith("N") for tag in lines) and "NEVER SUPPLY" not in prompt @@ -127,7 +127,7 @@ def test_the_tested_settings_have_no_problems(): ({"description": {"who_you_are": ["Hannah."], "never_supply": ["Your card number."]}}, "unknown sections ['never_supply']"), ({"description": {"who_you_are": ["Hannah."], "never_supply": ["Your card number."]}}, - "put never_supply's lines in keep_private: it holds what the user never says or hands over, even if asked"), + "put facts the assistant should find in what_to_look_up, and secrets in keep_private"), ({"description": {"who_you_are": ["Hannah."], "already_told": "Dad can't do stairs."}}, "description.already_told must be a list of non-empty lines"), ({"description": {"who_you_are": ["Hannah."], "already_told": [" "]}}, From 00439a63d61ad0d3e94470e6fb1b3e1a07873439 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Sat, 14 Nov 2026 12:05:27 -0800 Subject: [PATCH 26/45] user_sim_judge: guidance holds the user sim to its T and C lines; a task item can point at a persona line The guidance check now fails a user sim that doesn't remind the assistant, and give the fact again, when it re-asks an already-told (T) fact, or that doesn't briefly correct a should-already-know (C) fact the assistant gets wrong. A hard or general item can name a persona line with `line` ("P1") instead of restating it in `check`: the judge applies that line's text from the run's recorded tagged_lines, and the item's result records `line` and `line_text`. `line` with `check` or `text`, or on an auto item, is refused at load, and a line item needs rubric_persona_sha256. A tag the run doesn't have, or a run that recorded no persona fingerprint, makes the item an error, never a guess. Co-Authored-By: Claude Opus 5.5 --- .../task_steps/verifiers/user_sim_judge.py | 28 +++- .../verifiers/user_sim_judge_core.py | 90 ++++++++++--- .../verifiers/test_user_sim_judge.py | 122 ++++++++++++++++++ 3 files changed, 222 insertions(+), 18 deletions(-) diff --git a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py index e2ca71be2..ac636e17e 100644 --- a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py +++ b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py @@ -25,13 +25,24 @@ ``max_words`` or ``max_sentences`` is counted on each graded answer, and any answer over ``max`` fails it, naming the turn; see ``user_sim_judge_core.count_sentences``). If none is over but the record kept only the start of one (always_on's tail, when its stored timeline can't be read), the item is ``error``: that length can't be counted. -- ``check``: the text the judge applies, required unless auto. ``text`` is its old name and still loads. +- ``check``: the text the judge applies. ``text`` is its old name and still loads. +- ``line``: instead of ``check``, a tag of the user sim's instructions ("P1", "T1", "C2", "D3"): the judge applies + that line's text from the run's ``metadata["user_sims"][name]["tagged_lines"]``, shown as "Follows its instruction + P1: ...", and the item's result records ``line`` and ``line_text``. A hard or general item has exactly one of + ``check`` and ``line``; an auto item, counted rather than judged, takes no ``line``. A tag names the same line only + for the same persona, so a line item needs ``rubric_persona_sha256``, and grades only on a run whose fingerprint + matches it: on a run that recorded none, it is ``error``. A tag the run doesn't have makes the item ``error`` + ("the user sim's instructions have no line P1"). Neither makes a call. - ``name`` and ``short``: a title and one line for reviewers, recorded but not shown to the judge. - ``na_when``: the situation in which it never came up, told to the judge. ``always`` makes it n/a with no call, recording ``static_na_reason`` (required with it); ``never`` means it always applies, and an n/a is asked again. An auto item is counted, not judged, so it takes only ``always``; any other ``na_when`` is refused when the task loads. - ``can_error``: whether its fail makes the run a user sim error; unset, as the check that owns its family. +A task item never restates a persona line. The persona is the one place that says what the user sim knows and must +not say, and the judge reads all of it with every check. Task items are for counted limits (``auto``) and judgment +calls specific to the task, or they point at a persona line with ``line``. + For example:: rubric_persona_sha256: 3f1c... # the run's persona_sha256, all 64 hex digits @@ -40,6 +51,7 @@ check: "If the assistant asks about Rae's order, Tess says yes only if it fits, gives both floors ...", na_when: "The assistant didn't raise Rae's order."} - {id: f2_her_length, family: F2, kind: auto, check: "Every reply is at most 25 words.", metric: max_words, max: 25} + - {id: g1_engagement, family: G1, kind: hard, line: P1} ``rubric_persona_sha256`` is the fingerprint of the user sim the items were written for: ``user_sim_common.persona_fingerprint`` of its persona and ``opening_prompt``, which deploy_user_sim_agent records as @@ -91,6 +103,9 @@ ASSISTANT_LIKE_MESSAGE_CHARS = 1200 SCHEMA_ERRORS_SHOWN_CHARS = 300 # of a malformed answer's schema errors, in the judge's turn note USER_SIM_ERROR_SHOWN_CHARS = 200 # of a user sim agent error, in the judge's turn note +# Why a line item isn't graded on a run that recorded no persona fingerprint: its tag may name another line now. +LINE_UNCHECKED = ("the run recorded no persona fingerprint, so its tags can't be checked against the persona the item " + "was written for") def default_judge_model() -> str: @@ -212,9 +227,11 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: if entry["rubric_persona"]["status"] == "mismatch": items = [] context.record_model("user sim judge", self.id, settings["model"]) + unknown = (entry.get("rubric_persona") or {}).get("status") == "unknown" result = await core.judge_run( run, model=settings["model"], checks=settings["checks"], items=items, - error_checks=settings["error_checks"], complete=self._complete(context)) + error_checks=settings["error_checks"], complete=self._complete(context), + lines_unusable=LINE_UNCHECKED if unknown else "") result.pop("model", None) # the entry's judge_model entry.update(result) if "skipped" in entry: @@ -340,6 +357,10 @@ def _problems(model: Any, checks: Any, error_checks: Any, items: Any, mark: Any, out.append("rubric_items must be a list of {id, family, kind, check, ...}") else: out += core.item_problems(items) + if rubric_persona is None: + out += [f"rubric_items[{i}] ({item.get('id')}): a line item needs rubric_persona_sha256, so its tag is " + "graded only on runs of the persona it was written for" + for i, item in enumerate(items) if isinstance(item, dict) and item.get("line") is not None] return out @@ -377,7 +398,8 @@ def _read_run(kind: str, step_id: str, record: Optional[dict], context: TaskStep instructions, check, more = _instructions(us, metadata.get("seed") or {}, multi_day) turns = [_turn(n, e) for n, e in enumerate(reads, 1)] more += _reply_limit_note(turns, (us or {}).get("max_replies")) - run = core.UserSimRun(name=name or "the user", instructions=instructions, turns=turns, notes=notes + more) + run = core.UserSimRun(name=name or "the user", instructions=instructions, turns=turns, notes=notes + more, + tagged_lines=(us or {}).get("tagged_lines")) return run, {"user_sim": name, "instructions": check} diff --git a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py index 15942e7ce..3774e8710 100644 --- a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py +++ b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py @@ -17,7 +17,8 @@ rubric items (``RubricItem``) may name any family; an item goes into the call of the check that owns its family, and an item whose family no running check owns gets one call of its own (``TASK_ITEMS``). A call takes at most ``MAX_ITEMS_PER_CALL`` items; the rest go in further calls that ask only items. ``auto`` items and ``na_when: always`` -items are settled here, with no call. +items are settled here, with no call. An item's ``line`` (a tag, "P1") is graded by that line's text in the run's +``tagged_lines``; a tag the run doesn't have makes the item ``error``, with no call. One judge model answers each check in its own call. ``judge_run`` does all of it for one run, given a ``complete`` function that makes one model call; the step and a calibration harness both use it. @@ -27,7 +28,7 @@ import asyncio import json import re -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from typing import Any, Awaitable, Callable, Optional # ---- the rubric families ------------------------------------------------------------------------------------------ @@ -84,6 +85,10 @@ class Check: "hold firm.", "Asked the same thing again, it chose differently with nothing new to justify it.", "Asked again for something it already gave, it didn't give it again.", + "The assistant asked for a fact in its THINGS YOU ALREADY TOLD THE ASSISTANT lines (T), and it didn't " + "remind it, a little annoyed, that it already told it, and give it again.", + "The assistant got something in its THINGS THE ASSISTANT SHOULD ALREADY KNOW lines (C) wrong, and it " + "didn't correct it briefly, even where its other lines say to let that kind of mistake pass.", "Its instructions say this person points out a kind of mistake, that mistake was in what it was shown " "(the assistant's message or an email, text or notification), and it let it pass. Or it pointed out a " "mistake its instructions say this person wouldn't notice or wouldn't mention.", @@ -96,6 +101,8 @@ class Check: "Silence or a brief reply to a message that asks nothing.", "A requirement the assistant never asked about and this person wouldn't volunteer: that is the " "assistant's miss, not the user sim's.", + "Correcting a THINGS THE ASSISTANT SHOULD ALREADY KNOW line (C) the assistant got wrong, where its other " + "lines say this person would let that kind of mistake pass.", ), grade="Yes when the assistant's next step or its grade depends on what was withheld, decided or let pass.", ), @@ -304,6 +311,8 @@ class UserSimRun: instructions: Optional[str] turns: list[Turn] notes: list[str] = field(default_factory=list) + # Each tag the user sim's instructions showed, with its line (deploy_user_sim_agent's record); None: not recorded. + tagged_lines: Optional[dict[str, str]] = None # ---- a task's own rubric items ------------------------------------------------------------------------------------ @@ -313,8 +322,14 @@ class UserSimRun: MAX_WORDS, MAX_SENTENCES = "max_words", "max_sentences" METRICS = (MAX_WORDS, MAX_SENTENCES) MAX_ITEMS_PER_CALL = 12 # task items asked in one call; a check's or the task's further items get calls of their own -ITEM_FIELDS = ("id", "family", "kind", "check", "text", "name", "short", "na_when", "static_na_reason", "metric", "max", - "can_error") +ITEM_FIELDS = ("id", "family", "kind", "check", "text", "line", "name", "short", "na_when", "static_na_reason", "metric", + "max", "can_error") +# A line item's tag: a section letter and the line's place in it, as the user sim's instructions show it ("P1", "D10"). +LINE_TAG = re.compile(r"[A-Z][1-9][0-9]*") +# What the judge applies for a line item: the line's text as the run recorded it. +LINE_CHECK = "Follows its instruction {tag}: {text}" +# An item's ``asked_with`` when its line isn't in the run's tagged_lines: it's an error, with no call. +NO_LINE = "no_line" @dataclass(frozen=True) @@ -322,9 +337,11 @@ class RubricItem: """A task's own rubric item: one rule the judge checks, under one family. ``kind`` says how: ``hard``, an exact rule from the user sim's instructions that fails on any breach; ``general``, a judgement that fails only on a clear instance; ``auto``, never sent to the model, ``metric`` (max_words or max_sentences) counted on every graded answer - against ``max``. ``na_when`` is when it never came up: ``always`` (n/a without a call, ``static_na_reason`` says - why), ``never`` (n/a isn't allowed), else the situation, told to the judge; empty, the judge decides. An auto item - takes only ``always``: it is counted, never judged.""" + against ``max``. A hard or general item has ``check``, the text the judge applies, or ``line``, a tag of the user + sim's instructions ("P1") whose text is applied, read from the run's ``tagged_lines``. ``na_when`` is when it + never came up: ``always`` (n/a without a call, ``static_na_reason`` says why), ``never`` (n/a isn't allowed), else + the situation, told to the judge; empty, the judge decides. An auto item takes only ``always``: it is counted, + never judged.""" id: str family: str check: str = "" @@ -336,6 +353,7 @@ class RubricItem: metric: str = "" max: Optional[int] = None can_error: Optional[bool] = None # None: as the check that owns its family + line: str = "" # a tag of the user sim's instructions, graded by that line's text instead of ``check`` @property def static_na(self) -> bool: @@ -354,6 +372,7 @@ def from_dict(cls, data: dict) -> "RubricItem": """An item ``item_problems`` passed. ``text`` is the old name of ``check``.""" return cls(id=str(data["id"]), family=str(data["family"]).strip().upper(), check=str(data.get("check") if data.get("check") is not None else data.get("text") or "").strip(), + line=str(data.get("line") or "").strip(), kind=str(data.get("kind") or GENERAL).strip().lower(), name=str(data.get("name") or ""), short=str(data.get("short") or ""), na_when=str(data.get("na_when") or "").strip(), static_na_reason=str(data.get("static_na_reason") or "").strip(), @@ -361,7 +380,7 @@ def from_dict(cls, data: dict) -> "RubricItem": def to_dict(self) -> dict: out: dict[str, Any] = {"id": self.id, "family": self.family, "kind": self.kind} - for k in ("name", "short", "check", "na_when", "static_na_reason", "metric"): + for k in ("name", "short", "check", "line", "na_when", "static_na_reason", "metric"): if getattr(self, k): out[k] = getattr(self, k) if self.max is not None: @@ -395,14 +414,23 @@ def item_problems(items: list[dict]) -> list[str]: kind = None else: kind = kind.strip().lower() - for key in ("check", "text", "name", "short", "na_when", "static_na_reason"): + for key in ("check", "text", "line", "name", "short", "na_when", "static_na_reason"): if item.get(key) is not None and not isinstance(item[key], str): out.append(f"{at}: {key} must be text") if item.get("check") is not None and item.get("text") is not None: out.append(f"{at}: give check or text, not both (text is the old name of check)") check = item.get("check") if item.get("check") is not None else item.get("text") - if kind in (HARD, GENERAL) and not (isinstance(check, str) and check.strip()): - out.append(f"{at}: a {kind} item needs a check: the text the judge applies") + line = item.get("line") + if line is not None: + if isinstance(line, str) and not LINE_TAG.fullmatch(line.strip()): + out.append(f"{at}: line must be a tag of the user sim's instructions, like P1 or D3, not {line!r}") + if check is not None: + out.append(f"{at}: give line or check, not both: a line item is graded by that line's own text") + if kind == AUTO: + out.append(f"{at}: line is for hard and general items: an auto item is counted, not judged") + elif kind in (HARD, GENERAL) and not (isinstance(check, str) and check.strip()): + out.append(f"{at}: a {kind} item needs a check (the text the judge applies) or a line (a tag of the user " + "sim's instructions, like P1)") if kind == AUTO: if item.get("metric") not in METRICS: out.append(f"{at}: an auto item's metric must be one of {list(METRICS)}, not {item.get('metric')!r}") @@ -476,6 +504,31 @@ def auto_result(item: RubricItem, run: "UserSimRun") -> dict: "turns": []} +def resolve_lines(items: list[RubricItem], tagged_lines: Optional[dict[str, str]], + unusable: str = "") -> tuple[list[RubricItem], dict[str, dict]]: + """Each item with its ``line``'s text as its check (``LINE_CHECK``), and, by id, the line items that can't be + graded: their ``error``, never a guess. ``unusable`` says why no line can be trusted to be the one an item was + written for. An ``na_when: always`` item isn't graded, so its line isn't needed.""" + out, missing = [], {} + for item in items: + if not item.line or item.static_na: + out.append(item) + elif unusable: + missing[item.id] = {"verdict": "error", "turns": [], "line": item.line, + "error": f"line {item.line} can't be graded: {unusable}"} + elif tagged_lines is None: + missing[item.id] = {"verdict": "error", "turns": [], "line": item.line, + "error": f"the run recorded no tagged_lines for the user sim, so its line {item.line} " + "can't be read"} + elif item.line not in tagged_lines: + missing[item.id] = {"verdict": "error", "turns": [], "line": item.line, + "error": f"the user sim's instructions have no line {item.line}"} + else: + text = tagged_lines[item.line] + out.append(replace(item, check=LINE_CHECK.format(tag=item.line, text=text.replace("\n", "\n ")))) + return out, missing + + def render_run(run: UserSimRun) -> tuple[str, str]: """The run as the judge reads it, in two parts so a prompt cache can share the first across calls: the instructions and every turn's input up to the last graded turn's answers stay the same for all checks.""" @@ -708,15 +761,17 @@ def user_sim_error(checks: dict[str, dict], error_checks: list[str], items: dict async def judge_run(run: UserSimRun, *, model: str, checks: list[str], complete: Complete, items: tuple[RubricItem, ...] | list[RubricItem] = (), error_checks: tuple[str, ...] | list[str] = DEFAULT_ERROR_CHECKS, - answer_retries: int = 1, concurrency: int = 6) -> dict: + answer_retries: int = 1, concurrency: int = 6, lines_unusable: str = "") -> dict: """Every check (and task item) on ``run``, each in its own call to ``model``. The first call runs alone so the rest can read its prompt cache. A call that fails or answers badly ``answer_retries`` more times gives that check (and its items) the verdict ``error``. An auto item is computed here and an ``na_when: always`` item is n/a, with - no call for either.""" + no call for either. A ``line`` item is graded by its line's text in ``run.tagged_lines``, or is ``error`` when the + run has no such tag, or when ``lines_unusable`` says why no tag can be trusted.""" graded = {t.n for t in run.turns if t.graded} if not graded: return {"skipped": "no graded turns", "checks": {}, "items": {}, "user_sim_error": dict(NOT_JUDGED)} - calls = plan_calls(list(checks), list(items)) + resolved, no_line = resolve_lines(list(items), run.tagged_lines, lines_unusable) + calls = plan_calls(list(checks), resolved) head, tail = render_run(run) gate = asyncio.Semaphore(max(1, concurrency)) @@ -756,8 +811,11 @@ async def one(call: str) -> dict: for call in order if call_check(call)} asked_in = {item.id: call for call in order for item in calls[call]} item_results: dict[str, dict] = {} + lines = run.tagged_lines or {} for item in items: - if item.kind == AUTO and not item.static_na: + if item.id in no_line: + got, call = no_line[item.id], NO_LINE + elif item.kind == AUTO and not item.static_na: got, call = auto_result(item, run), COMPUTED elif item.static_na: got, call = {"verdict": "n/a", "reason": item.static_na_reason, "turns": []}, STATIC_NA @@ -765,6 +823,8 @@ async def one(call: str) -> dict: call = asked_in[item.id] answer = answers[call] got = {"verdict": "error", "error": answer["error"]} if "error" in answer else answer["items"][item.id] + if item.line and item.asks_the_model and item.id not in no_line: # what was graded, for a reviewer + got = {**got, "line": item.line, "line_text": lines[item.line]} item_results[item.id] = {**got, "family": item.family, "kind": item.kind, "asked_with": call} can_error = {i.id: (i.can_error if i.can_error is not None else FAMILY_OWNER.get(i.family) in error_checks) for i in items} diff --git a/tst/unit/task_step/verifiers/test_user_sim_judge.py b/tst/unit/task_step/verifiers/test_user_sim_judge.py index 4066699e3..4ff4337f6 100644 --- a/tst/unit/task_step/verifiers/test_user_sim_judge.py +++ b/tst/unit/task_step/verifiers/test_user_sim_judge.py @@ -949,3 +949,125 @@ async def test_items_for_this_persona_or_a_run_with_no_fingerprint_are_graded(mo # Without a rubric_persona_sha256 nothing is compared. await _step(FakeModel(), monkeypatch, checks=["style"], rubric_items=[HER_LENGTH]).execute(ctx) assert "rubric_persona" not in _entry(ctx) and _entry(ctx)["items"]["f2_her_length"]["verdict"] == "pass" + + +# ---- items that point at a persona line ------------------------------------------------------------------------------ +ENGAGEMENT = {"id": "g1_engagement", "family": "G1", "kind": "hard", "line": "P1"} +RING = "Ben proposed last month. Never mention or hint at it." + + +def _lined_run(tagged_lines=None) -> core.UserSimRun: + return replace(_run(("reply", "just a family dinner")), tagged_lines=tagged_lines) + + +def test_the_guidance_question_holds_it_to_its_already_told_and_should_already_know_lines(): + q = core.question_text("guidance", []) + assert "THINGS YOU ALREADY TOLD THE ASSISTANT lines (T)" in q and "remind it, a little annoyed" in q + assert "THINGS THE ASSISTANT SHOULD ALREADY KNOW lines (C) wrong, and it didn't correct it briefly" in q + assert "even where its other lines say to let that kind of mistake pass" in q + assert "Correcting a THINGS THE ASSISTANT SHOULD ALREADY KNOW line (C)" in q.split("Not a failure:")[1] + + +@pytest.mark.asyncio +async def test_a_line_item_is_graded_by_that_line_s_text_from_the_run_and_records_it(): + [item] = _items(ENGAGEMENT) + assert item.to_dict() == ENGAGEMENT and item.check == "" + complete = _answer_items({"g1_engagement": "fail"}) + seen = [] + + async def spy(model, system, parts, schema): + seen.append(parts[-1]) + return await complete(model, system, parts, schema) + + result = await _judge(_lined_run({"W1": "Hannah.", "P1": RING}), spy, checks=["oversharing"], items=[item]) + assert f"- g1_engagement (G1: Leaking and privacy; hard): Follows its instruction P1: {RING}" in seen[0] + assert result["items"]["g1_engagement"] == { + "verdict": "fail", "reason": "g1_engagement per turn 1", "turns": [1], "line": "P1", "line_text": RING, + "family": "G1", "kind": "hard", "asked_with": "oversharing"} + assert result["user_sim_error"]["because"] == ["item:g1_engagement"] + + +@pytest.mark.asyncio +async def test_a_line_s_later_lines_stay_under_the_item(): + [item] = _items({**ENGAGEMENT, "line": "W2"}) + seen = [] + + async def spy(model, system, parts, schema): + seen.append(parts[-1]) + return await _answer_items({})(model, system, parts, schema) + + await _judge(_lined_run({"W2": 'Earlier you texted the assistant:\n"sort the weekend"'}), spy, + checks=["oversharing"], items=[item]) + assert ('Follows its instruction W2: Earlier you texted the assistant:\n "sort the weekend"\n' in seen[0]) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("tagged_lines, error", [ + ({"P1": RING}, "the user sim's instructions have no line P2"), + (None, "the run recorded no tagged_lines for the user sim, so its line P2 can't be read"), +]) +async def test_a_line_the_run_doesn_t_have_is_an_error_never_a_guess(tagged_lines, error): + [item] = _items({**ENGAGEMENT, "line": "P2"}) + fake = FakeModel() + result = await _judge(_lined_run(tagged_lines), fake, checks=["oversharing"], items=[item]) + assert result["items"]["g1_engagement"] == {"verdict": "error", "turns": [], "line": "P2", "error": error, + "family": "G1", "kind": "hard", "asked_with": core.NO_LINE} + assert all("g1_engagement" not in c["parts"][-1] for c in fake.calls) + assert result["user_sim_error"] == {"value": "unknown", "because": [], "unknown": ["item:g1_engagement"]} + + +def test_a_line_item_is_refused_with_a_check_on_an_auto_item_or_without_the_persona_fingerprint(): + cases = [ + ({**ENGAGEMENT, "check": "Never mentions the ring."}, "give line or check, not both"), + ({**ENGAGEMENT, "text": "Never mentions the ring."}, "give line or check, not both"), + ({**ENGAGEMENT, "kind": "auto", "metric": "max_words", "max": 25}, + "line is for hard and general items: an auto item is counted, not judged"), + ({**ENGAGEMENT, "line": "p1"}, "line must be a tag of the user sim's instructions, like P1 or D3, not 'p1'"), + ({**ENGAGEMENT, "line": "P"}, "not 'P'"), + ({**ENGAGEMENT, "line": 1}, "line must be text"), + ] + for item, error in cases: + problems = core.item_problems([item]) + assert any(error in p for p in problems), (item, problems) + [no_check] = core.item_problems([{"id": "a", "family": "A1"}]) + assert no_check == ("rubric_items[0] (a): a general item needs a check (the text the judge applies) or a line (a " + "tag of the user sim's instructions, like P1)") + assert usj.UserSimJudgeTaskStep("s", None, rubric_items=[ENGAGEMENT], + rubric_persona_sha256=OTHER_PERSONA).config_problems() == [] + assert usj.UserSimJudgeTaskStep("s", None, rubric_items=[ENGAGEMENT]).config_problems() == [ + "rubric_items[0] (g1_engagement): a line item needs rubric_persona_sha256, so its tag is graded only on runs " + "of the persona it was written for"] + + +@pytest.mark.usefixtures("local_stores") +@pytest.mark.asyncio +async def test_on_a_run_a_line_item_reads_the_deploy_record_s_tagged_lines(monkeypatch): + ctx = await _chat(monkeypatch) + us = ctx.metadata[USER_SIMS_KEY]["hannah"] + fake = FakeModel({"oversharing": _pass(items=[{"id": "g1_engagement", "verdict": "pass", "reason": "-", + "turns": []}])}) + + await _step(fake, monkeypatch, checks=["oversharing"], rubric_items=[ENGAGEMENT], + rubric_persona_sha256=us["persona_sha256"]).execute(ctx) + + [call] = fake.calls + assert f"Follows its instruction P1: {us['tagged_lines']['P1']}" in call["parts"][-1] + got = _entry(ctx)["items"]["g1_engagement"] + assert (got["line"], got["line_text"], got["asked_with"]) == ("P1", us["tagged_lines"]["P1"], "oversharing") + assert _entry(ctx)["rubric_items"] == [ENGAGEMENT] + + +@pytest.mark.usefixtures("local_stores") +@pytest.mark.asyncio +async def test_on_a_run_with_no_persona_fingerprint_a_line_item_is_an_error(monkeypatch): + ctx = await _chat(monkeypatch) + del ctx.metadata[USER_SIMS_KEY]["hannah"]["persona_sha256"] + fake = FakeModel() + + await _step(fake, monkeypatch, checks=["oversharing"], rubric_items=[ENGAGEMENT, HER_LENGTH], + rubric_persona_sha256=OTHER_PERSONA).execute(ctx) + + items = _entry(ctx)["items"] + assert items["g1_engagement"]["verdict"] == "error" and items["f2_her_length"]["verdict"] == "pass" + assert items["g1_engagement"]["error"] == "line P1 can't be graded: " + usj.LINE_UNCHECKED + assert "g1_engagement" not in fake.calls[0]["parts"][-1] From 8c299bd9c372501a56d97fc4cb97cef3d8216821 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Sat, 14 Nov 2026 12:07:58 -0800 Subject: [PATCH 27/45] test(user_sim): Hannah's look-up line drops the sentence the fixed opening already says Co-Authored-By: Claude Opus 5.5 --- tst/data/user_sim/hannah_person.json | 2 +- tst/data/user_sim/hannah_system_prompt.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tst/data/user_sim/hannah_person.json b/tst/data/user_sim/hannah_person.json index 2ead7f24a..cc7e6397d 100644 --- a/tst/data/user_sim/hannah_person.json +++ b/tst/data/user_sim/hannah_person.json @@ -20,7 +20,7 @@ ], "what_to_look_up": [ "Mom's email from last Friday has the dates, the airport and their flight wishes; Bellacorte's terms are in its Tablefern confirmation; your schedule is in your calendar. If the assistant asks for any of these, tell it where to look (\"it's in mom's email!\"), a little impatiently.", - "Never hand over facts the assistant should look up itself (flight numbers, prices, times, confirmation codes, listing or restaurant names, deadlines). If it asks something you don't know, say so." + "Never hand over facts the assistant should look up itself (flight numbers, prices, times, confirmation codes, listing or restaurant names, deadlines)." ], "decisions": [ "Mom and Dad's flights within your rules (nonstop, economy but never basic, leaving after 8am, landing in SF while it's light, up to $450 a person round trip): yes, and it didn't need to ask.", diff --git a/tst/data/user_sim/hannah_system_prompt.txt b/tst/data/user_sim/hannah_system_prompt.txt index 9b40b596f..fb149495f 100644 --- a/tst/data/user_sim/hannah_system_prompt.txt +++ b/tst/data/user_sim/hannah_system_prompt.txt @@ -24,7 +24,7 @@ WHAT YOU KNOW (to use only when the assistant asks about it) WHAT TO LOOK UP (say where; don't give the value) L1 Mom's email from last Friday has the dates, the airport and their flight wishes; Bellacorte's terms are in its Tablefern confirmation; your schedule is in your calendar. If the assistant asks for any of these, tell it where to look ("it's in mom's email!"), a little impatiently. - L2 Never hand over facts the assistant should look up itself (flight numbers, prices, times, confirmation codes, listing or restaurant names, deadlines). If it asks something you don't know, say so. + L2 Never hand over facts the assistant should look up itself (flight numbers, prices, times, confirmation codes, listing or restaurant names, deadlines). YOUR DECISIONS (when the assistant asks) D1 Mom and Dad's flights within your rules (nonstop, economy but never basic, leaving after 8am, landing in SF while it's light, up to $450 a person round trip): yes, and it didn't need to ask. From 8190289dfa2fc7d5ee4492e159069f65247e74a0 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Sat, 14 Nov 2026 13:26:37 -0800 Subject: [PATCH 28/45] user_sim_judge: an auto item can't pass when the record dropped some answers always_on's timeline_tail keeps only the last events. Read from it, the judge now compares the kept check-ins with the record's counts.user_sim_reads; when some are missing it tells the judge so, and an auto item is error (unknown) unless a kept answer already fails it. Co-Authored-By: Claude Opus 5.5 --- .../task_steps/verifiers/user_sim_judge.py | 23 +++++++++++------- .../verifiers/user_sim_judge_core.py | 4 ++++ .../verifiers/test_user_sim_judge.py | 24 ++++++++++++++++++- 3 files changed, 42 insertions(+), 9 deletions(-) diff --git a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py index ac636e17e..31b8c3301 100644 --- a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py +++ b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py @@ -23,8 +23,9 @@ - ``kind``: ``hard`` (an exact rule from the user sim's instructions: the judge fails it on any breach), ``general`` (a judgement: it fails only on a clear instance; the default) or ``auto`` (never sent to the model: ``metric`` ``max_words`` or ``max_sentences`` is counted on each graded answer, and any answer over ``max`` fails it, naming - the turn; see ``user_sim_judge_core.count_sentences``). If none is over but the record kept only the start of one - (always_on's tail, when its stored timeline can't be read), the item is ``error``: that length can't be counted. + the turn; see ``user_sim_judge_core.count_sentences``). If none is over but the record kept only the start of one, + or dropped some answers (always_on's tail, when its stored timeline can't be read), the item is ``error``: those + lengths can't be counted. - ``check``: the text the judge applies. ``text`` is its old name and still loads. - ``line``: instead of ``check``, a tag of the user sim's instructions ("P1", "T1", "C2", "D3"): the judge applies that line's text from the run's ``metadata["user_sims"][name]["tagged_lines"]``, shown as "Follows its instruction @@ -387,11 +388,12 @@ def _read_run(kind: str, step_id: str, record: Optional[dict], context: TaskStep """The user sim's run, and what the entry records about it: ``user_sim`` (its name) and ``instructions``.""" metadata = context.metadata if kind == ALWAYS_ON: - reads, notes = _always_on_reads(record) + reads, notes, missing = _always_on_reads(record) name = (record.get("user_sim") or {}).get("name") multi_day = record.get("multi_day") else: reads, notes = _chat_reads(metadata.get(READS_KEY, {}).get(step_id) or []) + missing = 0 # a chat run keeps a summary of every read name = _chat_user_sim(metadata, step_id) multi_day = False us = (metadata.get(USER_SIMS_KEY) or {}).get(name) if name else None @@ -399,7 +401,7 @@ def _read_run(kind: str, step_id: str, record: Optional[dict], context: TaskStep turns = [_turn(n, e) for n, e in enumerate(reads, 1)] more += _reply_limit_note(turns, (us or {}).get("max_replies")) run = core.UserSimRun(name=name or "the user", instructions=instructions, turns=turns, notes=notes + more, - tagged_lines=(us or {}).get("tagged_lines")) + tagged_lines=(us or {}).get("tagged_lines"), missing_answers=missing) return run, {"user_sim": name, "instructions": check} @@ -419,12 +421,13 @@ def _load(url: str) -> dict: return json.loads(get_config().get_object_store_at(url).get(url)) -def _always_on_reads(record: dict) -> tuple[list[dict], list[str]]: - """Every ``user_sim_read`` in the always_on step's stored timeline, else its tail.""" +def _always_on_reads(record: dict) -> tuple[list[dict], list[str], int]: + """Every ``user_sim_read`` in the always_on step's stored timeline, else its tail; with notes for the judge and how + many of the run's reads the tail dropped (by the record's ``counts.user_sim_reads``).""" if record.get("object_url"): try: timeline = _load(record["object_url"])["timeline"] - return [e for e in timeline if e.get("event") == "user_sim_read"], [] + return [e for e in timeline if e.get("event") == "user_sim_read"], [], 0 except Exception as e: note = (f"the run's stored record could not be read ({type(e).__name__}); only its last events, without " "what the user sim was sent, are shown") @@ -437,7 +440,11 @@ def _always_on_reads(record: dict) -> tuple[list[dict], list[str]]: if e.get("event") == "user_sim_read": message = e.get("message") or "" # the tail keeps each string's first PENDING_TEXT_CHARS, then "…" reads.append({**e, "message_cut": len(message) == PENDING_TEXT_CHARS + 1 and message.endswith("…")}) - return reads, [note] + total = (record.get("counts") or {}).get("user_sim_reads") + missing = total - len(reads) if isinstance(total, int) and not isinstance(total, bool) and total > len(reads) else 0 + notes = [note] + ([f"the first {missing} of the run's {total} check-ins aren't kept, so they aren't shown"] + if missing else []) + return reads, notes, missing def _chat_reads(summaries: list[dict]) -> tuple[list[dict], list[str]]: diff --git a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py index 3774e8710..a9a5539b0 100644 --- a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py +++ b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py @@ -313,6 +313,7 @@ class UserSimRun: notes: list[str] = field(default_factory=list) # Each tag the user sim's instructions showed, with its line (deploy_user_sim_agent's record); None: not recorded. tagged_lines: Optional[dict[str, str]] = None + missing_answers: int = 0 # the run's answers its record didn't keep (always_on's tail drops the earliest) # ---- a task's own rubric items ------------------------------------------------------------------------------------ @@ -494,6 +495,9 @@ def auto_result(item: RubricItem, run: "UserSimRun") -> dict: listed = ", ".join(f"turn {n}: {c}" for n, c in over) return {"verdict": "fail", "reason": f"over the {item.max} {unit} max at {listed}", "turns": [n for n, _ in over]} + if run.missing_answers: + return {"verdict": "error", "turns": [], + "error": f"{run.missing_answers} of the run's answers weren't kept, so they can't be counted"} cut = [t.n for t in sent_answers(run) if t.cut] if cut: return {"verdict": "error", "turns": cut, diff --git a/tst/unit/task_step/verifiers/test_user_sim_judge.py b/tst/unit/task_step/verifiers/test_user_sim_judge.py index 4ff4337f6..b8167ee83 100644 --- a/tst/unit/task_step/verifiers/test_user_sim_judge.py +++ b/tst/unit/task_step/verifiers/test_user_sim_judge.py @@ -422,8 +422,9 @@ async def test_from_the_tail_an_answer_always_on_shortened_is_marked_cut(monkeyp record = {"timeline_tail": [_small({"event": "user_sim_read", "outcome": "reply", "message": "ok " * 200}), _small({"event": "user_sim_read", "outcome": "reply", "message": "sounds good"})]} - reads, notes = usj._always_on_reads(record) + reads, notes, missing = usj._always_on_reads(record) assert [r["message_cut"] for r in reads] == [True, False] and len(reads[0]["message"]) == PENDING_TEXT_CHARS + 1 + assert missing == 0 # the record has no counts to compare with assert usj._turn(1, reads[0]).cut and not usj._turn(2, reads[1]).cut @@ -791,6 +792,27 @@ async def test_an_auto_item_can_t_pass_an_answer_the_record_kept_only_the_start_ assert (await _judge(over, FakeModel(), checks=[], items=[length]))["items"]["f2_her_length"]["verdict"] == "fail" +def test_from_the_tail_the_check_ins_it_dropped_are_counted(): + tail = [{"event": "user_sim_read", "outcome": "reply", "message": "sounds good"}] * 2 + reads, notes, missing = usj._always_on_reads({"timeline_tail": tail, "counts": {"user_sim_reads": 5}}) + assert len(reads) == 2 and missing == 3 + assert notes[-1] == "the first 3 of the run's 5 check-ins aren't kept, so they aren't shown" + assert usj._always_on_reads({"timeline_tail": tail, "counts": {"user_sim_reads": 2}})[2] == 0 + + +@pytest.mark.asyncio +async def test_an_auto_item_can_t_pass_when_some_answers_weren_t_kept(): + [length] = _items(HER_LENGTH) + run = _run(("reply", TESS[0]), ("reply", TESS[1])) + run.missing_answers = 3 + assert (await _judge(run, FakeModel(), checks=[], items=[length]))["items"]["f2_her_length"] == { + "verdict": "error", "turns": [], "family": "F2", "kind": "auto", "asked_with": "computed", + "error": "3 of the run's answers weren't kept, so they can't be counted"} + long = _run(("reply", "word " * 30), ("reply", TESS[1])) + long.missing_answers = 3 # a kept answer over the max is a fail either way + assert (await _judge(long, FakeModel(), checks=[], items=[length]))["items"]["f2_her_length"]["verdict"] == "fail" + + @pytest.mark.asyncio async def test_an_auto_item_counts_sentences_simply(): assert [core.count_sentences(t) for t in TESS] == [3, 2] From 45fed68204d684b23a62b4930184110c8f08d694 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 16 Nov 2026 10:14:44 -0800 Subject: [PATCH 29/45] user_sim_judge: a reply still waiting when the run ended is shown as never delivered Only the last read's reply was flagged as never reaching the assistant. A reply from an earlier read can also still be held at the end; its read has a deliver_at no user_reply_delivered event carries. Those turns now get the same kind of flag. Co-Authored-By: Claude Opus 5.5 --- .../task_steps/verifiers/user_sim_judge.py | 22 ++++++++++++++----- .../verifiers/test_user_sim_judge.py | 18 +++++++++++++++ 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py index 31b8c3301..eb893f60c 100644 --- a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py +++ b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py @@ -100,6 +100,7 @@ USER_SIM_ERROR = "user_sim_error" FLAG_ASSISTANT = "this answer looks like the assistant's (very long, or markdown)" FLAG_LAST = "after the run's last wake: this answer never reached the assistant" +FLAG_HELD = "its reply was still waiting to be delivered when the run ended: it never reached the assistant" # always_on.ASSISTANT_LIKE_MESSAGE_CHARS: a chat run's reads carry no flag of their own, so it's computed the same way. ASSISTANT_LIKE_MESSAGE_CHARS = 1200 SCHEMA_ERRORS_SHOWN_CHARS = 300 # of a malformed answer's schema errors, in the judge's turn note @@ -427,7 +428,7 @@ def _always_on_reads(record: dict) -> tuple[list[dict], list[str], int]: if record.get("object_url"): try: timeline = _load(record["object_url"])["timeline"] - return [e for e in timeline if e.get("event") == "user_sim_read"], [], 0 + return _with_delivery(timeline, timeline), [], 0 except Exception as e: note = (f"the run's stored record could not be read ({type(e).__name__}); only its last events, without " "what the user sim was sent, are shown") @@ -435,11 +436,11 @@ def _always_on_reads(record: dict) -> tuple[list[dict], list[str], int]: note = "the run's full record wasn't stored; only its last events, without what the user sim was sent, are shown" from agent_env.task_step.task_steps.always_on import PENDING_TEXT_CHARS + tail = record.get("timeline_tail") or [] reads = [] - for e in record.get("timeline_tail") or []: - if e.get("event") == "user_sim_read": - message = e.get("message") or "" # the tail keeps each string's first PENDING_TEXT_CHARS, then "…" - reads.append({**e, "message_cut": len(message) == PENDING_TEXT_CHARS + 1 and message.endswith("…")}) + for e in _with_delivery(tail, tail): + message = e.get("message") or "" # the tail keeps each string's first PENDING_TEXT_CHARS, then "…" + reads.append({**e, "message_cut": len(message) == PENDING_TEXT_CHARS + 1 and message.endswith("…")}) total = (record.get("counts") or {}).get("user_sim_reads") missing = total - len(reads) if isinstance(total, int) and not isinstance(total, bool) and total > len(reads) else 0 notes = [note] + ([f"the first {missing} of the run's {total} check-ins aren't kept, so they aren't shown"] @@ -447,6 +448,15 @@ def _always_on_reads(record: dict) -> tuple[list[dict], list[str], int]: return reads, notes, missing +def _with_delivery(reads_from: list[dict], timeline: list[dict]) -> list[dict]: + """The ``user_sim_read`` events, each reply marked ``held`` when no ``user_reply_delivered`` event carries it (its + ``virtual_time`` is the reply's ``deliver_at``): it was still waiting when the run ended. A delivery comes after its + read, so a tail that has the read has its delivery too.""" + delivered = {e.get("virtual_time") for e in timeline if e.get("event") == "user_reply_delivered"} + return [{**e, "held": bool(e.get("deliver_at")) and not e.get("last_read") and e["deliver_at"] not in delivered} + for e in reads_from if e.get("event") == "user_sim_read"] + + def _chat_reads(summaries: list[dict]) -> tuple[list[dict], list[str]]: """Each read of a chat run, whole from its ``object_url``, else its summary, which keeps only the answer.""" reads, missing = [], 0 @@ -498,6 +508,8 @@ def _turn(n: int, e: dict) -> core.Turn: flags.append(f"the user sim agent failed at this check-in: {str(e['user_sim_error'])[:USER_SIM_ERROR_SHOWN_CHARS]}") if e.get("last_read"): flags.append(FLAG_LAST) + elif e.get("held"): + flags.append(FLAG_HELD) when = e.get("virtual_time") or (e.get("read") or {}).get("time") return core.Turn(n=n, outcome=e.get("outcome") or "no_reply", message=e.get("message") or "", input=e.get("input"), flags=flags, time=None if e.get("input") or not when else f"{when} UTC", diff --git a/tst/unit/task_step/verifiers/test_user_sim_judge.py b/tst/unit/task_step/verifiers/test_user_sim_judge.py index b8167ee83..2e424493d 100644 --- a/tst/unit/task_step/verifiers/test_user_sim_judge.py +++ b/tst/unit/task_step/verifiers/test_user_sim_judge.py @@ -792,6 +792,24 @@ async def test_an_auto_item_can_t_pass_an_answer_the_record_kept_only_the_start_ assert (await _judge(over, FakeModel(), checks=[], items=[length]))["items"]["f2_her_length"]["verdict"] == "fail" +def test_a_reply_still_waiting_when_the_run_ended_is_flagged_as_never_delivered(): + timeline = [ + {"event": "user_sim_read", "outcome": "reply", "message": "fri at 7 works", "virtual_time": "2026-11-19T14:00:00Z", + "deliver_at": "2026-11-19T14:20:00Z"}, + {"event": "user_reply_delivered", "virtual_time": "2026-11-19T14:20:00Z", "text": "fri at 7 works"}, + {"event": "user_sim_read", "outcome": "reply", "message": "dad can't do stairs", + "virtual_time": "2026-11-19T21:00:00Z", "deliver_at": "2026-11-20T08:05:00Z"}, + {"event": "user_sim_read", "outcome": "no_reply", "message": "", "virtual_time": "2026-11-20T07:00:00Z"}, + ] + reads = usj._with_delivery(timeline, timeline) + assert [r["held"] for r in reads] == [False, True, False] + assert usj.FLAG_HELD in usj._turn(2, reads[1]).flags and usj.FLAG_HELD not in usj._turn(1, reads[0]).flags + # The last read's reply is flagged as such, not twice. + last = {**timeline[2], "last_read": True} + flags = usj._turn(2, usj._with_delivery([last], [last])[0]).flags + assert usj.FLAG_LAST in flags and usj.FLAG_HELD not in flags + + def test_from_the_tail_the_check_ins_it_dropped_are_counted(): tail = [{"event": "user_sim_read", "outcome": "reply", "message": "sounds good"}] * 2 reads, notes, missing = usj._always_on_reads({"timeline_tail": tail, "counts": {"user_sim_reads": 5}}) From 2e562f81bcb9a5d65f448877f03170af559bc666 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 16 Nov 2026 10:29:15 -0800 Subject: [PATCH 30/45] feat(verifiers): user_sim_checks judge output format, for grading the user sim A new JudgeOutputFormat, USER_SIM_CHECKS, so the user sim judge (#682) can become a rubrics_verifier subclass. The judge grades the user sim, not the assistant: the framing and the run come first and the criteria last, so calls on one run can share a cached head. Each row gives reason, turns, verdict (pass / fail / n/a) and could_change_grade. Diagnosis ports the current judge's answer checks: a fail must cite a graded turn, a fail whose reason concludes nothing failed is asked again, and so is an n/a where na_when is never. A row lowers the score only on a fail that could change the grade; a weight 0 row is recorded and never gates. Additive: JudgeResponseDiscrepancy gains row_problems (empty by default), and rubrics_verifier's final error names them. Other formats are unchanged. Co-Authored-By: Claude Opus 5.5 --- .../judge_utils/judge_output_format.py | 302 +++++++++++++++ .../task_steps/verifiers/rubrics_verifier.py | 2 +- .../test_user_sim_checks_format.py | 345 ++++++++++++++++++ 3 files changed, 648 insertions(+), 1 deletion(-) create mode 100644 tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py diff --git a/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py b/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py index 78eab7c19..e6a0e37d8 100644 --- a/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py +++ b/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py @@ -5,6 +5,7 @@ import json import logging import math +import re from dataclasses import dataclass, field from enum import Enum from functools import partial @@ -558,6 +559,9 @@ class JudgeResponseDiscrepancy: unknown_ids: list[str] parse_error: str | None = None non_finite_score_ids: list[str] = field(default_factory=list) + # What's wrong inside rows whose ids are right (user_sim_checks: a fail citing no graded turn, a verdict its own + # reason contradicts, ...), one problem per entry naming its row. + row_problems: list[str] = field(default_factory=list) def to_dict(self) -> dict: out = { @@ -572,6 +576,8 @@ def to_dict(self) -> dict: out["parse_error"] = self.parse_error if self.non_finite_score_ids: out["non_finite_score_ids"] = list(self.non_finite_score_ids) + if self.row_problems: + out["row_problems"] = list(self.row_problems) return out def __str__(self) -> str: @@ -579,6 +585,8 @@ def __str__(self) -> str: return f"invalid JSON ({self.parse_error})" if self.non_finite_score_ids: return _non_finite_score_problem(self.non_finite_score_ids) + if self.row_problems: + return "; ".join(self.row_problems) return ( f"expected {self.expected_count} results but got {self.returned_count}" f" (missing={self.missing_ids}, duplicate={self.duplicate_ids}, unknown={self.unknown_ids})" @@ -1228,6 +1236,288 @@ def format_trajectory_mistakes_correction( return f"{eval_prompt.rstrip()}\n\n{correction_block}" +# --------------------------------------------------------------------------- +# user_sim_checks: grades the user sim (the AI playing the user), not the assistant. +# +# Each criterion is one question about the user sim's run: a check (guidance, oversharing, ...) or one of the task's +# own rubric items. Fields the prompt and the parser read on a criterion, beyond `id`: +# - `question`: what to answer, with what fails, what doesn't, when n/a and when could_change_grade is true; +# - `kind` (task items): "hard" or "general"; `na_when`: when it never came up, "never" when n/a isn't allowed; +# - `graded_turns`: the turns marked GRADED, which a fail must cite (any cited turn counts when it's absent); +# - `weight`: 0 records the row but never lets it lower the score (style). +# The run goes in the template's trajectory placeholder, so the framing and the run come first, the same for every +# call on a run, and the criteria last: a prompt cache can share everything before them. +# --------------------------------------------------------------------------- +USER_SIM_VERDICTS = ("pass", "fail", "n/a") +_USER_SIM_NA_NEVER = "never" + +_USER_SIM_FRAMING = """You check a user simulator ("the user sim"): an AI playing a real person who messages their \ +personal AI assistant. You grade the user sim, never the assistant. You are given: +1. THE USER SIM'S INSTRUCTIONS: exactly what it was told about who it is, what it knows, its decisions, its rules for \ +the assistant and what is private. Hold it to these lines only. +2. THE RUN: every check-in it got, word for word ("what it was sent": the time, its new and earlier emails, texts and \ +notifications, the lines it sent, and the assistant's new messages), each followed by its answer: "reply" (the \ +message reaches the assistant), "no_reply" (nothing is sent) or "done" (a sign-off). +3. THE CRITERIA: the questions to answer about it. + +How to read the run: +- What it was sent at a turn, and at every earlier turn, is what it knew; a fact it was never shown is not one it \ +forgot. +- Turns marked CONTEXT ONLY are fixed history: never fail the user sim for them, but they still count as what it said. +- Grade only the turns marked GRADED (each criterion's "graded_turns"). Cite turns by their number. +- The assistant's own words are the assistant's, even when they repeat an email. + +How to answer: +- Answer each criterion's "question" only, narrowly, by its own definition; other problems belong to other criteria. \ +A failure needs clear evidence in a graded answer. +- A criterion's "question" says what fails, what is not a failure, when to answer "n/a" and when "could_change_grade" \ +is true. +- A task rubric item gives its "kind". A "hard" item is an exact rule from the user sim's instructions: it fails on \ +any breach of the list or value it names, however small. A "general" item is a judgement about how this person would \ +behave: it fails only on a clear instance. +- An item's "na_when" is the situation in which it never came up: answer "n/a" for it then, and only then. \ +"na_when": "never" means it always applies: answer pass or fail. +- Reason first and decide last. Write the reason, then the turns it rests on, and only then the verdict, which \ +follows from the reason: a reason that finds no clear failure gives "pass" (or "n/a"), never "fail". Keep every \ +reason to at most 40 words: the evidence and the conclusion, not your working. +- "could_change_grade" says whether the failure could change how the assistant is graded on this task (false when \ +the verdict is pass or n/a). + +Return only this JSON object, with one result per criterion, each result's keys in this order: +{"results": [{"id": "the criterion's id", "reason": "at most 40 words", "turns": [turn numbers the reason rests on], \ +"verdict": "pass" | "fail" | "n/a", "could_change_grade": true | false}]}""" + +_USER_SIM_CRITERIA = """ + +# THE CRITERIA +$criteria_json + +Return only the JSON object.""" + +# `$trajectory_content` is the user sim's instructions and its run as the user sim judge renders them. +_USER_SIM_CHECKS_PROMPT_INLINE = Template(_USER_SIM_FRAMING + "\n\n$trajectory_content" + _USER_SIM_CRITERIA) +_USER_SIM_CHECKS_PROMPT_PATH = Template( + _USER_SIM_FRAMING + "\n\n# THE USER SIM'S INSTRUCTIONS AND THE RUN\nBoth are in the file at $trajectory_path: " + "read all of it before you answer." + _USER_SIM_CRITERIA) +# Without a trajectory the run comes in the sections the verifier adds after the template, so after the criteria. +_USER_SIM_CHECKS_PROMPT_NO_TRAJECTORY = Template( + _USER_SIM_FRAMING + _USER_SIM_CRITERIA + "\n\nThe user sim's instructions and the run follow.") + + +def _user_sim_checks_output_format() -> dict: + """Reason before verdict in every row: structured output is generated in schema order, so the judge decides after + reasoning. The parser reads the keys by name, in any order.""" + return { + "type": "json_schema", + "schema": { + "type": "object", + "properties": { + "results": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": {"type": "string"}, + "reason": {"type": "string"}, + "turns": {"type": "array", "items": {"type": "integer"}}, + "verdict": {"type": "string", "enum": list(USER_SIM_VERDICTS)}, + "could_change_grade": {"type": "boolean"}, + }, + "required": ["id", "reason", "turns", "verdict", "could_change_grade"], + "additionalProperties": False, + }, + }, + }, + "required": ["results"], + "additionalProperties": False, + }, + } + + +# A reason concluding nothing failed ("No clear delegation failures found"); a backstop to the reason-first prompt. +_NO_FAILURE_FOUND = re.compile( + r"\bno\s+(?:clear\s+)?(?:\w+\s+)?failures?\s+(?:was\s+|were\s+)?(?:found|detected|identified)\b" + r"|\bno\s+clear\s+(?:\w+\s+)?failures?\b|\bnothing\s+fails\b", re.I) + + +def _reason_says_no_failure(reason: str) -> str | None: + """What in the reason's conclusion says no failure was found, or None. Only its last sentence is its conclusion: + "no clear failure at turn 1; at turn 2 she leaked it" is a real fail.""" + sentences = [x for x in re.split(r"[.;!?]+", reason) if x.strip()] + m = _NO_FAILURE_FOUND.search(sentences[-1]) if sentences else None + return m.group(0) if m else None + + +def _cited_turns(value: Any) -> list[int]: + if not isinstance(value, list): + return [] + out = [] + for v in value: + try: + out.append(int(v)) + except (TypeError, ValueError): + continue + return out + + +def _graded_turns(criterion: dict) -> set[int] | None: + value = criterion.get("graded_turns") + return set(_cited_turns(value)) if isinstance(value, list) else None + + +def _user_sim_json(text: str) -> Any: + """The JSON in an answer, tolerating a markdown fence or text around the object; None when there is none.""" + s = str(text or "").strip().removeprefix("```json").removeprefix("```").removesuffix("```").strip() + try: + return json.loads(s) + except ValueError: + m = re.search(r"\{.*\}", s, re.S) + if not m: + return None + try: + return json.loads(m.group(0)) + except ValueError: + return None + + +def _never_gates(criterion: dict) -> bool: + try: + return float(criterion.get("weight", 1.0)) == 0.0 + except (TypeError, ValueError): + return False + + +def _user_sim_row(item: dict, criterion: dict) -> tuple[dict | None, list[str]]: + """One result checked against its criterion: ``(row, [])``, or ``(None, problems)`` naming the row. The row + lowers the score (score 0.0, result false) only on a fail with could_change_grade, and never at weight 0; + ``verdict`` and ``could_change_grade`` keep what the judge said either way. An ``assistant_asked`` bool (the + assistant asked for the behaviour) is kept when the judge gives one; it doesn't change the score.""" + rid = item.get("id") + problems = [] + verdict = str(item.get("verdict", "")).strip().lower() + if verdict not in USER_SIM_VERDICTS: + problems.append(f"{rid}: verdict {item.get('verdict')!r} is not one of {list(USER_SIM_VERDICTS)}") + change = item.get("could_change_grade") + if not isinstance(change, bool): + problems.append(f"{rid}: could_change_grade must be true or false") + if verdict == "n/a" and str(criterion.get("na_when") or "").strip().lower() == _USER_SIM_NA_NEVER: + problems.append(f"{rid} may not be n/a: it always applies, so answer pass or fail") + turns = _cited_turns(item.get("turns")) + reason = str(item.get("reason") or "").strip() + graded = _graded_turns(criterion) + if verdict == "fail": + if not any(graded is None or t in graded for t in turns): + problems.append(f"{rid}: a fail must cite at least one graded turn; cited {turns}" + + (f", graded {sorted(graded)}" if graded is not None else "")) + said = _reason_says_no_failure(reason) + if said: + problems.append(f"{rid}: a fail whose reason says {said!r} contradicts itself; the verdict must follow " + "the reason") + if problems: + return None, problems + change = change and verdict == "fail" + lowers = change and not _never_gates(criterion) + row = {**criterion, "id": rid, "reason": reason, + "turns": [t for t in turns if graded is None or t in graded], + "verdict": verdict, "could_change_grade": change, + "justification": reason, "score": 0.0 if lowers else 1.0, "result": not lowers} + outside = [t for t in turns if graded is not None and t not in graded] + if outside: + row["turns_outside_graded"] = outside + if isinstance(item.get("assistant_asked"), bool): + row["assistant_asked"] = item["assistant_asked"] + return row, [] + + +def diagnose_user_sim_checks_response( + response_text: str, criteria: list[dict], +) -> tuple[list[dict] | None, JudgeResponseDiscrepancy | None]: + """Parse a user_sim_checks judge response into rows, or a discrepancy for the correction prompt: no JSON + object, the wrong row set, or a row the user sim judge can't use (its verdict or could_change_grade invalid, an + n/a where ``na_when`` is never, a fail citing no graded turn, a fail whose reason concludes nothing failed).""" + parsed = _user_sim_json(response_text) + if parsed is None: + return None, _parse_failure_discrepancy(criteria, "the answer is not a JSON object") + results = _extract_rubric_binary_results(parsed) + if not isinstance(results, list): + return None, _parse_failure_discrepancy(criteria, f"unexpected response shape: {type(parsed).__name__}") + if not all(isinstance(r, dict) for r in results): + return None, _parse_failure_discrepancy(criteria, "every result must be a JSON object") + discrepancy = _analyze_result_ids(results, criteria) + if discrepancy is not None: + return None, discrepancy + by_id = {c["id"]: c for c in criteria} + rows: list[dict] = [] + problems: list[str] = [] + for item in results: + row, its_problems = _user_sim_row(item, by_id[item["id"]]) + problems += its_problems + if row is not None: + rows.append(row) + if problems: + return None, JudgeResponseDiscrepancy( + expected_count=len(criteria), returned_count=len(results), expected_ids=[c["id"] for c in criteria], + missing_ids=[], duplicate_ids=[], unknown_ids=[], row_problems=problems) + return rows, None + + +def _parse_user_sim_checks_response(response_text: str, criteria: list[dict]) -> list[dict]: + """Rows, or ValueError: an unusable answer has no verdict, so it never degrades to fails that would gate.""" + rows, discrepancy = diagnose_user_sim_checks_response(response_text, criteria) + if discrepancy is not None: + raise ValueError(f"user_sim_checks answer not usable: {discrepancy}") + assert rows is not None + return rows + + +def format_user_sim_checks_correction( + *, + eval_prompt: str, + previous_response: str, + discrepancy: JudgeResponseDiscrepancy, + max_previous_response_chars: int = _CORRECTION_PROMPT_MAX_PREVIOUS_RESPONSE_CHARS, +) -> str: + """The user_sim_checks correction prompt: what was wrong, then the row shape again.""" + truncated = previous_response.strip() + if len(truncated) > max_previous_response_chars: + truncated = truncated[:max_previous_response_chars] + "\n... [truncated]" + if discrepancy.parse_error: + problems = [f"Your response could not be read: {discrepancy.parse_error}", + "Return only a JSON object with a `results` array — no preamble, markdown fences, or extra text."] + elif discrepancy.row_problems: + problems = list(discrepancy.row_problems) + else: + problems = [f"You returned {discrepancy.returned_count} result(s) but there are " + f"{discrepancy.expected_count} criteria."] + for ids, what in ((discrepancy.missing_ids, "Missing result(s) for criterion id(s)"), + (discrepancy.duplicate_ids, "Duplicate result(s) for criterion id(s)"), + (discrepancy.unknown_ids, "Unknown criterion id(s) in your results")): + if ids: + problems.append(f"{what}: {', '.join(ids)}") + correction_block = "\n".join([ + "## Correction Required", + "", + "Your previous response did not satisfy the output requirements:", + *[f"- {problem}" for problem in problems], + "", + f"Return JSON with a `results` array containing exactly {discrepancy.expected_count} objects — " + "one per criterion, no duplicates, no extras.", + f"Required criterion ids (in any order): {', '.join(discrepancy.expected_ids)}", + "", + "Each result must include, in this order:", + '- "id": the criterion id', + '- "reason": at most 40 words, written before the verdict', + '- "turns": the turn numbers the reason rests on; a fail cites at least one graded turn', + '- "verdict": "pass", "fail" or "n/a", following from the reason: a reason that finds no clear failure is ' + 'never "fail"', + '- "could_change_grade": true or false', + "", + "## Your Previous Response", + truncated, + ]) + return f"{eval_prompt.rstrip()}\n\n{correction_block}" + + class JudgeOutputFormat(str, Enum): RUBRIC_BINARY = "rubric_binary" RUBRIC_PARTIAL = "rubric_partial" @@ -1238,6 +1528,9 @@ class JudgeOutputFormat(str, Enum): # object); the entry is stored exactly as rubric_evidence's and the verifier writes a second, # trajectory_mistakes-shaped entry from the findings. RUBRIC_EVIDENCE_WITH_MISTAKES = "rubric_evidence_with_mistakes" + # Grades the user sim (the AI playing the user), not the assistant: per criterion a reason, the turns it cites, a + # pass / fail / n/a verdict and could_change_grade; a row lowers the score only on a fail that could change the grade. + USER_SIM_CHECKS = "user_sim_checks" DiagnoseResponse = Callable[[str, list[dict]], tuple[list[dict] | None, JudgeResponseDiscrepancy | None]] @@ -1311,6 +1604,15 @@ class JudgeOutputFormatSpec: diagnose_response=diagnose_trajectory_mistakes_response, format_correction_prompt=format_trajectory_mistakes_correction, ), + JudgeOutputFormat.USER_SIM_CHECKS: JudgeOutputFormatSpec( + output_format=_user_sim_checks_output_format(), + prompt_template_path=_USER_SIM_CHECKS_PROMPT_PATH, + prompt_template_inline=_USER_SIM_CHECKS_PROMPT_INLINE, + prompt_template_no_trajectory=_USER_SIM_CHECKS_PROMPT_NO_TRAJECTORY, + parse_response=_parse_user_sim_checks_response, + diagnose_response=diagnose_user_sim_checks_response, + format_correction_prompt=format_user_sim_checks_correction, + ), } diff --git a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py index a7796c22c..9230e7529 100644 --- a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py @@ -1516,7 +1516,7 @@ async def _run_judge_with_output_retries( ) discrepancy_log.append(discrepancy.to_dict()) if attempt == self.DEFAULT_MAX_RETRIES: - if discrepancy.non_finite_score_ids: + if discrepancy.non_finite_score_ids or discrepancy.row_problems: raise ValueError(f"Judge output invalid after {attempt} attempts: {discrepancy}") raise ValueError( f"Judge returned {discrepancy.returned_count} results but expected " diff --git a/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py b/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py new file mode 100644 index 000000000..0b399732a --- /dev/null +++ b/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py @@ -0,0 +1,345 @@ +"""The user_sim_checks judge output format: it grades the user sim (the AI playing the user), not the assistant.""" + +import json +from unittest.mock import AsyncMock + +import pytest + +from agent_env.task_step.context import TaskStepContext +from agent_env.task_step.task_steps.verifiers.judge_utils.judge_output_format import ( + JUDGE_OUTPUT_FORMAT_REGISTRY, + JudgeOutputFormat, + diagnose_user_sim_checks_response, + format_user_sim_checks_correction, + get_judge_output_format_spec, +) +from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep +from agent_env.task_step.task_steps.verifiers.scoring import ScoreAggregator, aggregate_score + +SPEC = get_judge_output_format_spec(JudgeOutputFormat.USER_SIM_CHECKS) + +RUN = """# THE USER SIM'S INSTRUCTIONS + +You are Hannah Reyes. K1 Dad is allergic to shellfish. M3 Mention the allergy only when asked. + + +# THE RUN + +## Turn 1 · GRADED +### What it was sent + +It's Mon 9:12 AM. NEW FROM YOUR AI ASSISTANT: Booking Saturday dinner for 4 at Pescado. Any dietary needs? + +### Its answer +outcome: reply +message: "dad cant do shellfish" + +## Turn 2 · GRADED +### What it was sent + +It's Mon 9:40 AM. NEW FROM YOUR AI ASSISTANT: Booked for 7 PM. Want me to add a birthday note? + +### Its answer +outcome: reply +message: "sure. also dad cant do shellfish, its serious" +""" + + +def _criteria(*ids: str, graded=(1, 2), **extra) -> list[dict]: + return [{"id": cid, "question": f"question {cid}", "graded_turns": list(graded), **extra} for cid in ids] + + +def _row(cid="c1", *, verdict="pass", turns=(), could_change_grade=False, reason="Answered what was asked.", + **extra) -> dict: + return {"id": cid, "reason": reason, "turns": list(turns), "verdict": verdict, + "could_change_grade": could_change_grade, **extra} + + +def _answer(*rows: dict) -> str: + return json.dumps({"results": list(rows)}) + + +def _fail(cid="c1", *, turns=(2,), could_change_grade=True, + reason="At turn 2 she repeated the shellfish allergy unasked; M3 says only when asked.", **extra) -> dict: + return _row(cid, verdict="fail", turns=turns, could_change_grade=could_change_grade, reason=reason, **extra) + + +def _render(template, criteria_json="[]") -> str: + """A template filled the way rubrics_verifier fills it: every keyword it passes, used or not.""" + return template.substitute(agent_prompt="the assistant's prompt", agent_response="the assistant's answer", + criteria_json=criteria_json, trajectory_content=RUN, + trajectory_path="/tmp/trajectory.json", checks_instruction="[]") + + +# ---- templates ---------------------------------------------------------------------------------------------------- +def test_it_is_registered_and_found_by_name(): + assert JudgeOutputFormat("user_sim_checks") is JudgeOutputFormat.USER_SIM_CHECKS + assert get_judge_output_format_spec("user_sim_checks") is JUDGE_OUTPUT_FORMAT_REGISTRY[ + JudgeOutputFormat.USER_SIM_CHECKS] + assert SPEC.stored_format is None and SPEC.cites_evidence is False + + +def test_the_judge_is_told_it_grades_the_user_sim_never_the_assistant(): + for template in (SPEC.prompt_template_inline, SPEC.prompt_template_path, SPEC.prompt_template_no_trajectory): + prompt = _render(template) + assert prompt.startswith('You check a user simulator ("the user sim")') + assert "You grade the user sim, never the assistant." in prompt + # Nothing of the assistant's own prompt or answer is put in front of the judge as the thing to grade. + assert "the assistant's prompt" not in prompt and "the assistant's answer" not in prompt + + +def test_the_run_comes_first_and_the_criteria_last_so_calls_on_one_run_share_a_cached_head(): + one = _render(SPEC.prompt_template_inline, json.dumps(_criteria("c1"))) + other = _render(SPEC.prompt_template_inline, json.dumps(_criteria("c1", "c2"))) + assert one.index(RUN) < one.index("# THE CRITERIA") < one.index('"question": "question c1"') + assert one.endswith("Return only the JSON object.") + head = one[:one.index("# THE CRITERIA")] + assert other.startswith(head) and RUN in head + path = _render(SPEC.prompt_template_path) + assert path.index("/tmp/trajectory.json") < path.index("# THE CRITERIA") + + +def test_without_a_trajectory_the_judge_is_told_the_run_follows_the_criteria(): + prompt = _render(SPEC.prompt_template_no_trajectory) + assert RUN not in prompt + assert prompt.endswith("The user sim's instructions and the run follow.") + + +def test_the_judge_reasons_first_and_the_verdict_follows_the_reason(): + prompt = _render(SPEC.prompt_template_inline) + assert "Reason first and decide last." in prompt and "at most 40 words" in prompt + assert 'a reason that finds no clear failure gives "pass" (or "n/a"), never "fail"' in prompt + shape = prompt.split("Return only this JSON object")[1] + assert (shape.index('"id"') < shape.index('"reason"') < shape.index('"turns"') < shape.index('"verdict"') + < shape.index('"could_change_grade"')) + + +def test_the_judge_is_told_how_to_read_the_run_and_the_criteria_fields(): + prompt = _render(SPEC.prompt_template_inline) + for told in ("Turns marked CONTEXT ONLY are fixed history", "Grade only the turns marked GRADED", + '"graded_turns"', "The assistant's own words are the assistant's", + 'A "hard" item is an exact rule', 'A "general" item is a judgement', + '"na_when": "never" means it always applies', + '"could_change_grade" says whether the failure could change how the assistant is graded'): + assert told in prompt, told + + +# ---- schema ------------------------------------------------------------------------------------------------------- +def test_each_row_gives_its_reason_first_then_turns_verdict_and_could_change_grade(): + schema = SPEC.output_format + assert schema["type"] == "json_schema" and schema["schema"]["required"] == ["results"] + row = schema["schema"]["properties"]["results"]["items"] + order = ["id", "reason", "turns", "verdict", "could_change_grade"] + assert list(row["properties"]) == order == row["required"] + assert row["properties"]["verdict"]["enum"] == ["pass", "fail", "n/a"] + assert row["properties"]["turns"] == {"type": "array", "items": {"type": "integer"}} + assert row["properties"]["could_change_grade"] == {"type": "boolean"} + assert row["additionalProperties"] is False and schema["schema"]["additionalProperties"] is False + + +# ---- reading an answer -------------------------------------------------------------------------------------------- +def test_an_answer_is_read_into_rows_with_turns_kept_to_the_graded_ones(): + criteria = _criteria("c1", "c2", graded=[2], family="G1") + rows, discrepancy = diagnose_user_sim_checks_response( + _answer(_fail("c1", turns=[1, 2, "x"]), _row("c2", could_change_grade=True)), criteria) + assert discrepancy is None + fail, ok = rows + assert (fail["verdict"], fail["turns"], fail["turns_outside_graded"]) == ("fail", [2], [1]) + assert fail["could_change_grade"] is True and fail["justification"] == fail["reason"] + assert fail["family"] == "G1" and fail["question"] == "question c1" # the criterion rides on its row + # A pass can't change the grade, whatever the judge wrote. + assert ok["could_change_grade"] is False and "turns_outside_graded" not in ok + + +def test_the_parser_reads_the_keys_in_any_order_and_tolerates_a_fence_or_text_around_the_object(): + flipped = {"could_change_grade": True, "verdict": "FAIL ", "turns": [2], "reason": "Leaked the allergy.", "id": "c1"} + for text in (json.dumps({"results": [flipped]}), "```json\n" + _answer(flipped) + "\n```", + "Here is my answer:\n" + _answer(flipped) + "\nDone."): + rows, discrepancy = diagnose_user_sim_checks_response(text, _criteria("c1")) + assert discrepancy is None and rows[0]["verdict"] == "fail", text + + +def test_an_answer_that_is_not_a_json_object_or_has_the_wrong_rows_is_asked_again(): + criteria = _criteria("c1", "c2") + for text, problem in (("dad cant do shellfish", "the answer is not a JSON object"), + (json.dumps({"verdict": "pass"}), "unexpected response shape: dict"), + (json.dumps({"results": ["pass", "pass"]}), "every result must be a JSON object")): + rows, discrepancy = diagnose_user_sim_checks_response(text, criteria) + assert rows is None and discrepancy.parse_error == problem + rows, discrepancy = diagnose_user_sim_checks_response(_answer(_row("c1"), _row("c1"), _row("c9")), criteria) + assert rows is None + assert (discrepancy.missing_ids, discrepancy.duplicate_ids, discrepancy.unknown_ids) == (["c2"], ["c1"], ["c9"]) + + +def test_a_verdict_or_could_change_grade_out_of_range_is_asked_again(): + rows, discrepancy = diagnose_user_sim_checks_response( + _answer(_row("c1", verdict="maybe"), _row("c2", could_change_grade="yes")), _criteria("c1", "c2")) + assert rows is None + assert discrepancy.row_problems == ["c1: verdict 'maybe' is not one of ['pass', 'fail', 'n/a']", + "c2: could_change_grade must be true or false"] + + +def test_a_fail_that_cites_no_graded_turn_is_asked_again_and_a_pass_needs_none(): + for turns, cited in (([99], r"\[99\]"), ([], r"\[\]")): + rows, discrepancy = diagnose_user_sim_checks_response(_answer(_fail(turns=turns)), _criteria("c1", graded=[1])) + assert rows is None + assert discrepancy.row_problems == [f"c1: a fail must cite at least one graded turn; cited {turns}, graded [1]"] + with pytest.raises(ValueError, match=f"a fail must cite at least one graded turn; cited {cited}"): + SPEC.parse_response(_answer(_fail(turns=turns)), _criteria("c1", graded=[1])) + for ok in (_row(turns=[99]), _row(verdict="n/a", turns=[99])): + rows, discrepancy = diagnose_user_sim_checks_response(_answer(ok), _criteria("c1", graded=[1])) + assert discrepancy is None and rows[0]["turns"] == [] and rows[0]["turns_outside_graded"] == [99] + + +def test_without_graded_turns_on_the_criterion_a_fail_still_has_to_cite_a_turn(): + criteria = [{"id": "c1", "question": "question c1"}] + rows, discrepancy = diagnose_user_sim_checks_response(_answer(_fail(turns=[])), criteria) + assert discrepancy.row_problems == ["c1: a fail must cite at least one graded turn; cited []"] + rows, discrepancy = diagnose_user_sim_checks_response(_answer(_fail(turns=[7])), criteria) + assert discrepancy is None and rows[0]["turns"] == [7] and "turns_outside_graded" not in rows[0] + + +# The endings of the two contradictory reasons seen in real runs, and a plain one. +SAYS_NO_FAILURE = ("Hannah flagged it anyway which is fine. No clear failure found.", + "This is not a delegation failure by Linda. No clear delegation failures found.", + "Every answer fits her rules; nothing fails.") + + +def test_a_fail_whose_reason_concludes_nothing_failed_is_asked_again(): + for reason in SAYS_NO_FAILURE: + rows, discrepancy = diagnose_user_sim_checks_response(_answer(_fail(reason=reason)), _criteria("c1")) + assert rows is None and len(discrepancy.row_problems) == 1 + assert "c1: a fail whose reason says" in discrepancy.row_problems[0] + assert "contradicts itself; the verdict must follow the reason" in discrepancy.row_problems[0] + # A pass may say it. + rows, discrepancy = diagnose_user_sim_checks_response(_answer(_row(reason=reason)), _criteria("c1")) + assert discrepancy is None and rows[0]["reason"] == reason + # Only the last sentence is the conclusion: a fail that clears one turn and fails another is kept. + for reason in ("No clear failure at turn 1; at turn 2 she revealed the private password.", + "Turn 1 is fine, not a failure. At turn 2 she volunteered the shellfish allergy unasked; M3 says " + "only when asked."): + rows, discrepancy = diagnose_user_sim_checks_response(_answer(_fail(reason=reason)), _criteria("c1")) + assert discrepancy is None and rows[0]["verdict"] == "fail" and rows[0]["could_change_grade"] is True + + +def test_an_n_a_where_na_when_is_never_is_asked_again_and_otherwise_stands(): + na = _row(verdict="n/a", reason="Rae never came up.") + rows, discrepancy = diagnose_user_sim_checks_response(_answer(na), _criteria("c1", na_when="never")) + assert discrepancy.row_problems == ["c1 may not be n/a: it always applies, so answer pass or fail"] + for criteria in (_criteria("c1"), _criteria("c1", na_when="the assistant never asks about Rae's order")): + rows, discrepancy = diagnose_user_sim_checks_response(_answer(na), criteria) + assert discrepancy is None and rows[0]["verdict"] == "n/a" and rows[0]["score"] == 1.0 + + +def test_every_unusable_row_is_named_in_one_go(): + rows, discrepancy = diagnose_user_sim_checks_response( + _answer(_fail("c1", turns=[]), _row("c2"), _fail("c3", reason=SAYS_NO_FAILURE[0])), _criteria("c1", "c2", "c3")) + assert rows is None and [p.split(":")[0] for p in discrepancy.row_problems] == ["c1", "c3"] + assert discrepancy.to_dict()["row_problems"] == discrepancy.row_problems + assert str(discrepancy) == "; ".join(discrepancy.row_problems) + + +def test_assistant_asked_is_kept_when_given_and_never_changes_the_score(): + rows, discrepancy = diagnose_user_sim_checks_response( + _answer(_fail("c1", assistant_asked=True), _row("c2", assistant_asked="yes"), _row("c3")), + _criteria("c1", "c2", "c3")) + assert discrepancy is None + assert rows[0]["assistant_asked"] is True and rows[0]["score"] == 0.0 + assert "assistant_asked" not in rows[1] and "assistant_asked" not in rows[2] + + +def test_parse_response_never_turns_an_unusable_answer_into_fails(): + with pytest.raises(ValueError, match="user_sim_checks answer not usable: invalid JSON"): + SPEC.parse_response("not json", _criteria("c1")) + assert SPEC.parse_response(_answer(_row("c1")), _criteria("c1"))[0]["result"] is True + + +# ---- the correction prompt ---------------------------------------------------------------------------------------- +def test_the_correction_prompt_names_each_problem_and_asks_for_the_row_shape_again(): + previous = _answer(_fail("c1", turns=[99]), _row("c2")) + _, discrepancy = diagnose_user_sim_checks_response(previous, _criteria("c1", "c2", graded=[1])) + prompt = SPEC.format_correction_prompt(eval_prompt="EVAL", previous_response=previous, discrepancy=discrepancy) + assert prompt.startswith("EVAL\n\n## Correction Required") + assert "- c1: a fail must cite at least one graded turn; cited [99], graded [1]" in prompt + assert "exactly 2 objects" in prompt and "Required criterion ids (in any order): c1, c2" in prompt + for field in ('"id"', '"reason": at most 40 words', '"turns"', '"verdict"', '"could_change_grade"'): + assert f"- {field}" in prompt + assert '"score"' not in prompt and '"justification"' not in prompt + assert prompt.endswith("## Your Previous Response\n" + previous) + + +def test_the_correction_prompt_for_unreadable_json_or_a_wrong_row_set(): + _, unreadable = diagnose_user_sim_checks_response("no json here", _criteria("c1")) + prompt = format_user_sim_checks_correction(eval_prompt="EVAL", previous_response="no json here", + discrepancy=unreadable) + assert "Your response could not be read: the answer is not a JSON object" in prompt + _, wrong = diagnose_user_sim_checks_response(_answer(_row("c1"), _row("c9")), _criteria("c1", "c2")) + prompt = format_user_sim_checks_correction(eval_prompt="EVAL", previous_response="x" * 9000, discrepancy=wrong) + assert "Missing result(s) for criterion id(s): c2" in prompt and "Unknown criterion id(s) in your results: c9" in prompt + assert prompt.endswith("... [truncated]") + + +# ---- the score ---------------------------------------------------------------------------------------------------- +def _score(*rows: dict, criteria: list[dict], aggregator=ScoreAggregator.WEIGHTED_AVERAGE) -> float: + got, discrepancy = diagnose_user_sim_checks_response(_answer(*rows), criteria) + assert discrepancy is None + return aggregate_score(got, aggregator) + + +def test_a_row_lowers_the_score_only_on_a_fail_that_could_change_the_grade(): + criteria = _criteria("c1", "c2", "c3") + assert _score(_row("c1"), _row("c2", verdict="n/a"), _row("c3"), criteria=criteria) == 1.0 + assert _score(_fail("c1", could_change_grade=False), _row("c2"), _row("c3"), criteria=criteria) == 1.0 + assert _score(_fail("c1"), _row("c2"), _row("c3"), criteria=criteria) == pytest.approx(2 / 3) + rows, _ = diagnose_user_sim_checks_response(_answer(_fail("c1", could_change_grade=False)), _criteria("c1")) + assert (rows[0]["verdict"], rows[0]["score"], rows[0]["result"]) == ("fail", 1.0, True) + + +def test_a_weight_0_row_is_recorded_but_never_gates(): + criteria = [*_criteria("guidance"), *_criteria("style", weight=0)] + rows, _ = diagnose_user_sim_checks_response(_answer(_row("guidance"), _fail("style")), criteria) + style = rows[1] + assert (style["verdict"], style["could_change_grade"], style["weight"]) == ("fail", True, 0) + assert (style["score"], style["result"]) == (1.0, True) + for aggregator in ScoreAggregator: + assert _score(_row("guidance"), _fail("style"), criteria=criteria, aggregator=aggregator) == 1.0 + assert _score(_fail("guidance"), _fail("style"), criteria=criteria) == 0.0 + + +# ---- in rubrics_verifier's retry loop ----------------------------------------------------------------------------- +def _verifier() -> RubricsVerifierTaskStep: + return RubricsVerifierTaskStep(id="judge-user-sim", version=1, criteria=_criteria("c1", "c2", graded=[1]), + prompt_id="prompt-1", use_trajectory=False, verifier_id="user-sim", + output_format=JudgeOutputFormat.USER_SIM_CHECKS) + + +async def _run_judge(verifier: RubricsVerifierTaskStep): + return await verifier._run_judge_with_output_retries( + eval_prompt="EVAL", criteria=verifier.criteria, context=TaskStepContext(), model="claude-sonnet-4-6", + judge_a2a_url=None, judge_agent_card={}, judge_agent=None, judge_effort=None, + judge_max_thinking_tokens=None, use_agent_judge=False) + + +@pytest.mark.asyncio +async def test_rubrics_verifier_asks_again_for_an_unusable_row_and_keeps_the_usable_answer(monkeypatch): + verifier = _verifier() + unusable = _answer(_fail("c1", turns=[99]), _row("c2")) + usable = _answer(_fail("c1", turns=[1]), _row("c2")) + judge = AsyncMock(side_effect=[{"response": unusable}, {"response": usable}]) + monkeypatch.setattr(verifier, "_prompt_llm_judge", judge) + rows, retries, discrepancies, _ = await _run_judge(verifier) + assert retries == 1 and [r["verdict"] for r in rows] == ["fail", "pass"] + assert discrepancies[0]["row_problems"] == ["c1: a fail must cite at least one graded turn; cited [99], graded [1]"] + correction = judge.await_args_list[1].args[0] + assert correction.startswith("EVAL\n\n## Correction Required") and unusable in correction + assert aggregate_score(rows, ScoreAggregator.WEIGHTED_AVERAGE) == 0.5 + + +@pytest.mark.asyncio +async def test_rubrics_verifier_names_the_unusable_row_when_the_judge_never_fixes_it(monkeypatch): + verifier = _verifier() + monkeypatch.setattr(verifier, "_prompt_llm_judge", AsyncMock( + return_value={"response": _answer(_fail("c1", turns=[1], reason=SAYS_NO_FAILURE[0]), _row("c2"))})) + with pytest.raises(ValueError, match=r"Judge output invalid after 3 attempts: c1: a fail whose reason says " + r"'No clear failure found' contradicts itself"): + await _run_judge(verifier) From b89f67e112f5ab5bbfc51b531a65f4cd984fb48e Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 16 Nov 2026 10:36:52 -0800 Subject: [PATCH 31/45] reward: a step can say its score isn't the task's reward reward_of counted every step with a verifier_id as a scoring step, so a step whose score only checks the run (a user sim judge) made build_app and training ask for reward=, and as a task's only such step made every episode's reward raise LookupError. TaskStep.can_be_reward (default True) lets a step class opt out: reward_of never picks it, a task whose only score is one is unscored, and naming it as reward= is refused with a reason. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/openenv/reward.py | 9 ++++- src/agent_env/task_step/task_step.py | 3 ++ tst/unit/openenv/reward_test.py | 56 ++++++++++++++++++++++++++++ 3 files changed, 66 insertions(+), 2 deletions(-) create mode 100644 tst/unit/openenv/reward_test.py diff --git a/src/agent_env/openenv/reward.py b/src/agent_env/openenv/reward.py index 08778e775..4bf3ef972 100644 --- a/src/agent_env/openenv/reward.py +++ b/src/agent_env/openenv/reward.py @@ -9,16 +9,21 @@ def reward_of(task: Task, reward: str | Callable[[dict[str, float]], float] | None ) -> tuple[Callable[[dict[str, float]], float | None] | None, str | None]: - """``task``'s reward as a function of its scores, or why ``reward`` can't make one.""" + """``task``'s reward as a function of its scores, or why ``reward`` can't make one. Only a step that records a score + and ``can_be_reward`` is a candidate; one that can't (its score checks the run) is never picked.""" if callable(reward): return reward, None - scoring = [step.id for step in task.steps if getattr(step, "verifier_id", None)] + recording = [step for step in task.steps if getattr(step, "verifier_id", None)] + scoring = [step.id for step in recording if getattr(step, "can_be_reward", True)] if reward is None: if len(scoring) > 1: return None, (f"task {task.id!r} records a score at each of {', '.join(scoring)}; name the one that is " "its reward with reward=") return (_score_at(scoring[0]) if scoring else _unscored), None if reward not in scoring: + if checking := next((step for step in recording if step.id == reward), None): + return None, (f"task {task.id!r}'s step {reward!r} ({checking.type}) records a score that checks the run " + f"and can't be its reward; the scores that can are at {scoring}") return None, f"task {task.id!r} records no score at step {reward!r}; it records them at {scoring}" return _score_at(reward), None diff --git a/src/agent_env/task_step/task_step.py b/src/agent_env/task_step/task_step.py index c26c6dc6c..7b4f088dc 100644 --- a/src/agent_env/task_step/task_step.py +++ b/src/agent_env/task_step/task_step.py @@ -97,6 +97,9 @@ class TaskStep(ABC): # Each field of a scope other than the task's, by its path in the stored step; any other field is task-scoped. A # subclass merges its parent's. Such a field is optional, so a step written out without it still builds and runs. field_scopes: ClassVar[Mapping[str, FieldScope]] = {"metadata.attribution": FieldScope.USER} + # Whether the score a step records under its ``verifier_id`` can be its task's reward. A step whose score checks the + # run rather than grading the agent sets False: ``reward_of`` never picks it, and refuses it as ``reward=``. + can_be_reward: ClassVar[bool] = True def __init_subclass__(cls, **kwargs: Any) -> None: super().__init_subclass__(**kwargs) diff --git a/tst/unit/openenv/reward_test.py b/tst/unit/openenv/reward_test.py new file mode 100644 index 000000000..fcd9553b1 --- /dev/null +++ b/tst/unit/openenv/reward_test.py @@ -0,0 +1,56 @@ +"""``reward_of``: which recorded score is a task's reward. A step whose class says its score can't be the reward (it +checks the run, as a user sim judge does) is never picked and is refused as ``reward=``.""" + +from typing import ClassVar + +from agent_env.openenv.reward import reward_of +from agent_env.task import Task +from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep + + +class _RunCheck(RubricsVerifierTaskStep): + can_be_reward: ClassVar[bool] = False + + +def _rubric(cls: type[RubricsVerifierTaskStep] = RubricsVerifierTaskStep, *, id: str) -> RubricsVerifierTaskStep: + return cls(id=id, version=None, criteria=[], prompt_id="p1", use_agent_judge=False, verifier_id=f"{id}-v") + + +def _task(*steps: RubricsVerifierTaskStep) -> Task: + return Task(id="t", version=None, steps=list(steps)) + + +def test_a_rubric_beside_a_run_check_is_the_reward_without_naming_it(): + of, why = reward_of(_task(_rubric(id="grade"), _rubric(_RunCheck, id="user-sim")), None) + + assert why is None + assert of({"grade": 0.75, "user-sim": 0.0}) == 0.75 + + +def test_a_task_whose_only_score_is_a_run_check_is_unscored(): + of, why = reward_of(_task(_rubric(_RunCheck, id="user-sim")), None) + + assert why is None + assert of({"user-sim": 1.0}) is None + + +def test_a_run_check_named_as_the_reward_is_refused(): + of, why = reward_of(_task(_rubric(id="grade"), _rubric(_RunCheck, id="user-sim")), "user-sim") + + assert of is None + assert why == ("task 't''s step 'user-sim' (rubrics_verifier) records a score that checks the run and can't be " + "its reward; the scores that can are at ['grade']") + + +def test_two_rubrics_still_need_one_named(): + task = _task(_rubric(id="grade"), _rubric(id="again"), _rubric(_RunCheck, id="user-sim")) + + assert reward_of(task, None) == (None, "task 't' records a score at each of grade, again; name the one that is " + "its reward with reward=") + of, why = reward_of(task, "again") + assert why is None and of({"grade": 0.0, "again": 1.0, "user-sim": 0.0}) == 1.0 + + +def test_a_step_that_records_no_score_is_refused_as_before(): + assert reward_of(_task(_rubric(id="grade")), "nowhere") == ( + None, "task 't' records no score at step 'nowhere'; it records them at ['grade']") From f9c74990e0428ebc51e5ae6ee01f52695f79c46a Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 16 Nov 2026 10:36:52 -0800 Subject: [PATCH 32/45] rubrics_verifier: hooks for a subclass judge All off by default; an existing rubrics_verifier sends the same request and writes the same entry. - _prompt_llm_judge takes an optional system message and a cached_prefix (marked for Anthropic's prompt cache on a Claude model), and returns the call's usage and cost. - _criteria_batches is a method a subclass can override; batches must cover every criterion once. - unjudged_rows: rows no judge grades, merged after the judge's and scored with them. - judge_trajectory_content: text the prompt shows as the inline trajectory, so a subclass's run sits before the criteria (needs use_trajectory=False). - The format spec declares batchable (rubric_binary, rubric_partial, as before) and checks_bulk_verdicts. Co-Authored-By: Claude Opus 5.5 --- .../judge_utils/judge_output_format.py | 8 + .../task_steps/verifiers/rubrics_verifier.py | 118 +++++++--- .../test_rubrics_verifier_litellm_contract.py | 55 +++++ .../test_rubrics_verifier_subclass_hooks.py | 208 ++++++++++++++++++ 4 files changed, 359 insertions(+), 30 deletions(-) create mode 100644 tst/unit/task_step/verifiers/test_rubrics_verifier_subclass_hooks.py diff --git a/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py b/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py index 78eab7c19..0231964f2 100644 --- a/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py +++ b/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py @@ -1259,6 +1259,12 @@ class JudgeOutputFormatSpec: # Rows carry `frame` / `evidence_status`, so the per-criterion grounding asks the judge to cite them; a format # with this set needs the labelled per-criterion frames (the verifier enforces the configuration). cites_evidence: bool = False + # Rows are one per criterion, so judge_batch_size may split the criteria into calls whose rows merge by + # concatenation. A format with a per-call block (reasoning, checks, mistakes) or whole-run rows leaves it off. + batchable: bool = False + # Batched rows are checked for bulk verdicts (shared or short justifications) and retried with a correction. A + # format whose rows may rightly share a short reason turns it off. + checks_bulk_verdicts: bool = True JUDGE_OUTPUT_FORMAT_REGISTRY: dict[JudgeOutputFormat, JudgeOutputFormatSpec] = { @@ -1270,6 +1276,7 @@ class JudgeOutputFormatSpec: parse_response=_parse_rubric_binary_response, diagnose_response=diagnose_rubric_binary_response, format_correction_prompt=format_judge_correction_prompt, + batchable=True, ), JudgeOutputFormat.RUBRIC_PARTIAL: JudgeOutputFormatSpec( output_format=_rubric_score_output_format(), @@ -1279,6 +1286,7 @@ class JudgeOutputFormatSpec: parse_response=_parse_rubric_partial_response, diagnose_response=diagnose_rubric_partial_response, format_correction_prompt=format_partial_judge_correction_prompt, + batchable=True, ), JudgeOutputFormat.RUBRIC_EVIDENCE: JudgeOutputFormatSpec( output_format=_rubric_evidence_output_format(), diff --git a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py index a7796c22c..e759287dc 100644 --- a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py @@ -144,10 +144,6 @@ def trajectory_mistakes_key(verifier_id: str) -> str: _BULK_LOGGED_EXCERPT_CHARS = 80 -def _criteria_batches(criteria: list[dict], size: int) -> list[list[dict]]: - return [criteria[i:i + size] for i in range(0, len(criteria), size)] or [[]] - - def _bulk_verdicts(rows: list[dict]) -> Optional[dict]: """What makes ``rows`` look scripted rather than graded one by one, or None. A judge facing many criteria tends to grep for a few keywords and assign the rest a default with a stock justification.""" @@ -276,6 +272,21 @@ def _looks_like_response_format_error(exc: Exception) -> bool: return any(s in msg for s in _SCHEMA_TERMS) and any(u in msg for u in _UNSUPPORTED_TERMS) +def _caches_prompts(model: str) -> bool: + """Whether a judge call to ``model`` marks its shared prefix for the prompt cache: Claude models, by any route.""" + return "claude" in model.lower() + + +def _usage_and_cost(response: Any) -> dict: + """A LiteLLM response's token counts and LiteLLM's price of it in USD, each None when it has none.""" + usage = getattr(response, "usage", None) + usage = usage.model_dump() if hasattr(usage, "model_dump") else usage + hidden = getattr(response, "_hidden_params", None) + cost = hidden.get("response_cost") if isinstance(hidden, dict) else None + return {"usage": usage if isinstance(usage, dict) else None, + "cost": float(cost) if isinstance(cost, (int, float)) and not isinstance(cost, bool) else None} + + def _image_url_block(frame: ImageFrame) -> dict: """Adapt a neutral trajectory image to LiteLLM's image block shape.""" return { @@ -353,11 +364,6 @@ def _validated_batch_option(name: str, value: Optional[int]) -> Optional[int]: return value -# Formats whose rows are one per criterion, so batches merge by concatenation. The evidence formats carry one -# reasoning / checks / mistakes block per call and trajectory_mistakes adds whole-run rows, which batches would repeat. -_BATCHABLE_FORMATS = frozenset({JudgeOutputFormat.RUBRIC_BINARY, JudgeOutputFormat.RUBRIC_PARTIAL}) - - def _unbatchable_format_message(output_format: JudgeOutputFormat) -> str: return f"judge_batch_size does not apply to output_format={output_format.value}." @@ -541,7 +547,7 @@ def __init__( "with use_trajectory=True and a trajectory_filter with compaction_type=screenshot and " "frame_strategy=per_criterion." ) - if self.judge_batch_size is not None and self.output_format not in _BATCHABLE_FORMATS: + if self.judge_batch_size is not None and not self._spec().batchable: raise ValueError(_unbatchable_format_message(self.output_format)) def _spec(self) -> JudgeOutputFormatSpec: @@ -648,6 +654,29 @@ async def judge_prompt_sections(self, context: TaskStepContext, prompt_response: per run (never for one skipped) with the prompt's stored response. None here.""" return [] + async def judge_trajectory_content(self, context: TaskStepContext, prompt_response: PromptResponse) -> Optional[str]: + """What the judge prompt shows as the run's trajectory (the inline template's ``$trajectory_content``), in place + of a recorded trajectory, for a step with ``use_trajectory`` off: an override point, called once per run (never + for one skipped) with the prompt's stored response. A subclass that puts the run here, rather than in + ``judge_prompt_sections`` after the criteria, keeps it in the part of the prompt each batch shares. None here: + the prompt shows the recorded trajectory, or none.""" + return None + + async def unjudged_rows(self, context: TaskStepContext, prompt_response: PromptResponse) -> list[dict]: + """Rows no judge grades, computed from the run or fixed, that the entry holds after the judge's and that are + scored with them: an override point, called once per run before the judge (never for one skipped) with the + prompt's stored response. Each is shaped as a judge's row (``id``, ``result`` / ``score``, any ``weight``), + with an id no criterion has. None here.""" + return [] + + def _criteria_batches(self, criteria: list[dict], size: Optional[int]) -> list[list[dict]]: + """The judge calls ``criteria`` are graded in, one batch per call: ``size`` at a time in order, or all in one + call when ``size`` is None. An override point: a subclass may group them otherwise, putting every criterion + in exactly one non-empty batch. The criteria carry their short ids (c1..cN) and every other field.""" + if size is None: + return [criteria] + return [criteria[i:i + size] for i in range(0, len(criteria), size)] or [[]] + def _resolved_criteria(self, context: TaskStepContext) -> list[dict]: """Criteria for this run: a per-run ``step_params`` override replaces the stored list. @@ -694,6 +723,10 @@ async def _grade(self, context: TaskStepContext) -> TaskStepContext: if grade_up_to_turn is not None: run_context = run_context.as_of(grade_up_to_turn) extra_sections = await self.judge_prompt_sections(context, stored_response) + trajectory_content = await self.judge_trajectory_content(context, stored_response) + if trajectory_content is not None and self.use_trajectory: + raise RuntimeError(f"Verifier '{self.verifier_id}': judge_trajectory_content stands in for the recorded " + "trajectory, so it needs use_trajectory=False.") per_turn_uris = [u for u in (prompt_response.target_agent_per_turn_trajectory_object_urls or []) if u] if self.use_trajectory and not (per_turn_uris or prompt_response.agent_trajectory_object_url): @@ -705,6 +738,10 @@ async def _grade(self, context: TaskStepContext) -> TaskStepContext: use_agent_judge_override = overrides.get("use_agent_judge") use_agent_judge = use_agent_judge_override if use_agent_judge_override is not None else self.use_agent_judge criteria = self._resolved_criteria(context) + unjudged = [dict(row) for row in await self.unjudged_rows(context, stored_response)] + if clashing := {row.get("id") for row in unjudged} & {c.get("id") for c in criteria}: + raise RuntimeError(f"Verifier '{self.verifier_id}': unjudged_rows gave rows with a criterion's id: " + f"{sorted(map(str, clashing))}") # The judge sees/echoes short keys (c1..cN), not UUIDs; frame labels use them too, hence before compaction. criteria_for_judge, real_by_short = _to_short_keyed_criteria(criteria) apply_filter_override = overrides.get("apply_trajectory_filter") @@ -841,7 +878,7 @@ async def _grade(self, context: TaskStepContext) -> TaskStepContext: overrides, "judge_batch_size", self.judge_batch_size)) batch_concurrency = _validated_batch_option("judge_batch_concurrency", _override_or( overrides, "judge_batch_concurrency", self.judge_batch_concurrency)) - if batch_size is not None and self.output_format not in _BATCHABLE_FORMATS: + if batch_size is not None and not self._spec().batchable: raise RuntimeError(f"Verifier '{self.verifier_id}': {_unbatchable_format_message(self.output_format)}") loaded_for_judge: list[dict] = [] if use_agent_judge and self.agent_name is not None: @@ -873,6 +910,8 @@ async def _grade(self, context: TaskStepContext) -> TaskStepContext: trajectory_inline = compacted.inline_text if trajectory_inline is None and compacted.container_object_url: trajectory_inline = await asyncio.to_thread(self._read_trajectory_text, compacted.container_object_url) + if trajectory_content is not None: + trajectory_inline = trajectory_content def eval_prompt_for(batch: list[dict]) -> str: criteria_json = json.dumps([ @@ -885,6 +924,7 @@ def eval_prompt_for(batch: list[dict]) -> str: agent_response=prompt_response.response, criteria_json=criteria_json, trajectory_path=trajectory_path, + trajectory_inline=trajectory_inline, loaded_artifacts=loaded_for_judge, context_facts=context_facts, run_context=run_context, @@ -914,10 +954,13 @@ def eval_prompt_for(batch: list[dict]) -> str: prompt += self._PER_ROW_EVIDENCE_NOTE return prompt - batches = (_criteria_batches(criteria_for_judge, batch_size) if batch_size is not None - else [criteria_for_judge]) - concurrency = asyncio.Semaphore(batch_concurrency or self.DEFAULT_JUDGE_BATCH_CONCURRENCY) + batches = self._criteria_batches(criteria_for_judge, batch_size) position = {c["id"]: i for i, c in enumerate(criteria_for_judge, 1)} + batched_ids = [c["id"] for batch in batches for c in batch] + if sorted(batched_ids) != sorted(position) or (len(batches) > 1 and not all(batches)): + raise RuntimeError(f"Verifier '{self.verifier_id}': _criteria_batches must put every criterion in " + "exactly one batch, and leave no batch empty") + concurrency = asyncio.Semaphore(batch_concurrency or self.DEFAULT_JUDGE_BATCH_CONCURRENCY) async def grade(batch: list[dict]) -> tuple[list[dict], int, list[dict], Optional[str]]: prompt = eval_prompt_for(batch) @@ -938,7 +981,7 @@ async def grade(batch: list[dict]) -> tuple[list[dict], int, list[dict], Optiona use_agent_judge=use_agent_judge, criteria=batch, image_blocks=image_blocks, - reject_bulk_verdicts=batch_size is not None, + reject_bulk_verdicts=batch_size is not None and self._spec().checks_bulk_verdicts, ) if len(batches) == 1: @@ -991,15 +1034,16 @@ async def grade(batch: list[dict]) -> tuple[list[dict], int, list[dict], Optiona judge_call_fields["judge_batch_trajectory_object_urls"] = batch_trajectory_urls if grade_up_to_turn is not None: judge_call_fields["graded_up_to_turn"] = grade_up_to_turn - self._write_entries(context, verification_results, judge_call_fields) + self._write_entries(context, verification_results, judge_call_fields, unjudged=unjudged) return context - def _write_entries(self, context: TaskStepContext, verification_results: list[dict], judge_call_fields: dict) -> None: + def _write_entries(self, context: TaskStepContext, verification_results: list[dict], judge_call_fields: dict, + *, unjudged: Sequence[dict] = ()) -> None: """Store the rubric entry under ``verifier_id`` and, for rubric_evidence_with_mistakes, the trajectory-mistakes entry under ``trajectory_mistakes_key(verifier_id)``; ``judge_call_fields`` (URIs, retries) go on both. The mistake rows are built before either entry is written, so a response they - cannot be derived from leaves neither behind.""" - score = aggregate_score(verification_results, self.score_aggregator) + cannot be derived from leaves neither behind. ``unjudged`` rows (``unjudged_rows``) join the rubric + entry's, after the judge's, and its score; the mistakes entry is the judge's alone.""" mistake_rows: Optional[list[dict]] = None if self._writes_mistakes_entry: if not isinstance(verification_results, ResultRows): @@ -1007,6 +1051,8 @@ def _write_entries(self, context: TaskStepContext, verification_results: list[di f"Verifier '{self.verifier_id}': the judge response carries no mistake findings " f"({type(verification_results).__name__})") mistake_rows = trajectory_mistakes_rows_from_evidence(verification_results) + verification_results.extend(unjudged) # in place, so a ResultRows keeps its reasoning and checks + score = aggregate_score(verification_results, self.score_aggregator) verifications = context.metadata.setdefault("verifications", {}) verification_entry = { **self._entry_format_fields(), @@ -1199,11 +1245,19 @@ async def _prompt_llm_judge( model: str, context: TaskStepContext, image_blocks: Optional[list[dict]] = None, + system: Optional[str] = None, + cached_prefix: Optional[str] = None, ) -> dict: - """Direct LiteLLM judge — no agent boot, no tools. Returns {response, trajectory_object_url}. - - When ``image_blocks`` are provided the user message becomes multimodal - (text + images); otherwise it stays a plain string, identical to before.""" + """Direct LiteLLM judge — no agent boot, no tools. Returns {response, trajectory_object_url, usage, cost}: + ``usage`` the call's token counts and ``cost`` LiteLLM's price of it in USD, each None when the response + has none, for a caller that records them (this step records neither). + + When ``image_blocks`` are provided the user message becomes multimodal (text + images); otherwise it stays + a plain string. ``system`` is sent as a system message first. ``cached_prefix`` is the start of + ``eval_prompt`` that each of the run's calls shares: on a Claude model it is sent as its own text block, + marked for Anthropic's prompt cache, so the calls after the first read it from the cache; a prompt that + doesn't start with it is sent whole. The step's own calls pass neither, so their request is unchanged; a + subclass opts in by overriding this and calling it with them.""" import litellm from agent_env.config import get_config @@ -1219,11 +1273,15 @@ async def _prompt_llm_judge( or overrides.get("litellm_base_url") or self.default_model_api_base, ) - user_content = ( - [{"type": "text", "text": eval_prompt}, *image_blocks] - if image_blocks - else eval_prompt - ) + text_blocks: list[dict] = [{"type": "text", "text": eval_prompt}] + cached = bool(cached_prefix) and eval_prompt.startswith(cached_prefix or "") and _caches_prompts(model) + if cached: + rest = eval_prompt[len(cached_prefix or ""):] + text_blocks = [{"type": "text", "text": cached_prefix, "cache_control": {"type": "ephemeral"}}, + *([{"type": "text", "text": rest}] if rest else [])] + user_content = [*text_blocks, *(image_blocks or [])] if image_blocks or cached else eval_prompt + messages = [*([{"role": "system", "content": system}] if system else []), + {"role": "user", "content": user_content}] response_format = { "type": "json_schema", "json_schema": { @@ -1241,7 +1299,7 @@ async def _prompt_llm_judge( try: request_kwargs: dict = dict( model=model, - messages=[{"role": "user", "content": user_content}], + messages=messages, timeout=self.judge_timeout_seconds, **call_config.client_kwargs(), ) @@ -1249,7 +1307,7 @@ async def _prompt_llm_judge( request_kwargs[ModelParam.RESPONSE_FORMAT] = response_format response = await litellm.acompletion(**request_kwargs) content = response.choices[0].message.content or "" - return {"response": content, "trajectory_object_url": None} + return {"response": content, "trajectory_object_url": None, **_usage_and_cost(response)} except Exception as exc: last_exc = exc if image_blocks and use_response_format and _looks_like_response_format_error(exc): diff --git a/tst/unit/task_step/verifiers/test_rubrics_verifier_litellm_contract.py b/tst/unit/task_step/verifiers/test_rubrics_verifier_litellm_contract.py index 035269c48..b1c38e2b2 100644 --- a/tst/unit/task_step/verifiers/test_rubrics_verifier_litellm_contract.py +++ b/tst/unit/task_step/verifiers/test_rubrics_verifier_litellm_contract.py @@ -84,3 +84,58 @@ async def test_an_endpoint_refusing_the_schema_with_images_is_asked_again_withou assert json.loads(out["response"]) == _VERDICT assert "response_format" in calls[0] assert "response_format" not in calls[1] + + +@pytest.mark.asyncio +async def test_a_default_call_sends_one_user_message_and_returns_its_usage_and_cost(judge, monkeypatch): + calls = _answer_with(monkeypatch, json.dumps(_VERDICT)) + + out = await judge._prompt_llm_judge("grade it", model="openai/gpt-4o", context=TaskStepContext()) + + assert calls[0]["messages"] == [{"role": "user", "content": "grade it"}] + assert out["usage"]["total_tokens"] > 0 + assert isinstance(out["cost"], float) and out["cost"] > 0 + + +@pytest.mark.asyncio +async def test_a_system_message_and_a_shared_prefix_marked_for_the_prompt_cache(judge, monkeypatch): + calls = _answer_with(monkeypatch, json.dumps(_VERDICT)) + + await judge._prompt_llm_judge("THE RUN\n\nTHE CHECK", model="anthropic/claude-sonnet-4-6", + context=TaskStepContext(), system="You grade the user sim.", + cached_prefix="THE RUN\n\n") + + assert calls[0]["messages"] == [ + {"role": "system", "content": "You grade the user sim."}, + {"role": "user", "content": [ + {"type": "text", "text": "THE RUN\n\n", "cache_control": {"type": "ephemeral"}}, + {"type": "text", "text": "THE CHECK"}, + ]}, + ] + + +@pytest.mark.asyncio +async def test_a_shared_prefix_rides_before_the_screenshots(judge, monkeypatch): + calls = _answer_with(monkeypatch, json.dumps(_VERDICT)) + + await judge._prompt_llm_judge("THE RUN then the check", model="claude-sonnet-4-6", context=TaskStepContext(), + image_blocks=[_IMAGE], cached_prefix="THE RUN") + + assert calls[0]["messages"][0]["content"] == [ + {"type": "text", "text": "THE RUN", "cache_control": {"type": "ephemeral"}}, + {"type": "text", "text": " then the check"}, + _IMAGE, + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("model", "prompt"), [ + ("openai/gpt-4o", "THE RUN then the check"), # not a Claude model: no cache marks + ("anthropic/claude-sonnet-4-6", "a correction, then THE RUN"), # doesn't start with the prefix +]) +async def test_a_prefix_that_cant_be_cached_sends_the_prompt_whole(judge, monkeypatch, model, prompt): + calls = _answer_with(monkeypatch, json.dumps(_VERDICT)) + + await judge._prompt_llm_judge(prompt, model=model, context=TaskStepContext(), cached_prefix="THE RUN") + + assert calls[0]["messages"] == [{"role": "user", "content": prompt}] diff --git a/tst/unit/task_step/verifiers/test_rubrics_verifier_subclass_hooks.py b/tst/unit/task_step/verifiers/test_rubrics_verifier_subclass_hooks.py new file mode 100644 index 000000000..95b3356b1 --- /dev/null +++ b/tst/unit/task_step/verifiers/test_rubrics_verifier_subclass_hooks.py @@ -0,0 +1,208 @@ +"""The override points a subclass judge builds on: ``_criteria_batches`` (which criteria share a judge call), +``unjudged_rows`` (rows no judge grades, scored with the judge's), ``judge_trajectory_content`` (the run, where the +template puts the trajectory) and the format spec's ``batchable`` / ``checks_bulk_verdicts``. Unoverridden, each +leaves the step as it was.""" +from __future__ import annotations + +from dataclasses import replace +from typing import Optional + +import pytest + +from agent_env.task_step.context import PromptResponse, TaskStepContext, VerifierNotPassingError +from agent_env.task_step.task_steps.verifiers.judge_utils.judge_output_format import ( + JUDGE_OUTPUT_FORMAT_REGISTRY, + JudgeOutputFormat, + JudgeOutputFormatSpec, +) +from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep +from tst.unit.task_step.verifiers.test_rubrics_verifier_judge_batches import ( + _context, + _fake_judge, + _graded, + _prompted_ids, +) + +CHECKS = [{"id": "asked", "title": "Asked for the booking", "check": "asked"}, + {"id": "asked-item", "title": "Named the restaurant", "check": "asked"}, + {"id": "replied", "title": "Replied to the agent", "check": "replied"}, + {"id": "replied-item", "title": "Confirmed the time", "check": "replied"}] + + +class _ByCheck(RubricsVerifierTaskStep): + """Each check's criteria in one call, as a user sim judge puts a check's task items in that check's call.""" + + def _criteria_batches(self, criteria: list[dict], size: Optional[int]) -> list[list[dict]]: + groups: dict[str, list[dict]] = {} + for criterion in criteria: + groups.setdefault(criterion["check"], []).append(criterion) + return list(groups.values()) + + +class _WithRows(RubricsVerifierTaskStep): + rows: list[dict] = [] + + async def unjudged_rows(self, context: TaskStepContext, prompt_response: PromptResponse) -> list[dict]: + return self.rows + + +def _step(cls: type[RubricsVerifierTaskStep] = RubricsVerifierTaskStep, **kw) -> RubricsVerifierTaskStep: + params = dict(id="verify", version=1, criteria=CHECKS, prompt_id="p1", use_agent_judge=False, + use_trajectory=False, verifier_id="vt") + params.update(kw) + return cls(**params) + + +@pytest.mark.asyncio +async def test_a_subclass_groups_the_criteria_into_its_own_calls(): + v = _step(_ByCheck) + prompts: list[str] = [] + _fake_judge(v, prompts) + ctx = _context() + + await v.execute(ctx) + + assert sorted(_prompted_ids(p) for p in prompts) == [["c1", "c2"], ["c3", "c4"]] + entry = ctx.metadata["verifications"]["vt"] + assert [r["id"] for r in entry["results"]] == ["asked", "asked-item", "replied", "replied-item"] + assert entry["judge_batches"] == 2 + + +@pytest.mark.asyncio +async def test_batches_that_drop_a_criterion_are_refused(): + class Dropping(RubricsVerifierTaskStep): + def _criteria_batches(self, criteria: list[dict], size: Optional[int]) -> list[list[dict]]: + return [criteria[:1], criteria[2:]] + + v = _step(Dropping) + _fake_judge(v, []) + + with pytest.raises(RuntimeError, match="must put every criterion in exactly one batch"): + await v.execute(_context()) + + +@pytest.mark.asyncio +async def test_unjudged_rows_join_the_entry_and_its_score_but_never_reach_the_judge(): + v = _step(_WithRows, score_aggregator="weighted_average") + v.rows = [{"id": "auto-asked", "result": True, "score": 1.0}, + {"id": "always-na", "result": False, "score": 0.0, "weight": 0}, + {"id": "missing-line", "result": False, "score": 0.0}] + prompts: list[str] = [] + _fake_judge(v, prompts) + ctx = _context() + + await v.execute(ctx) + + assert [_prompted_ids(p) for p in prompts] == [["c1", "c2", "c3", "c4"]] + entry = ctx.metadata["verifications"]["vt"] + assert [r["id"] for r in entry["results"]] == [ + "asked", "asked-item", "replied", "replied-item", "auto-asked", "always-na", "missing-line"] + assert entry["score"] == pytest.approx(5 / 6) # the weight-0 row is recorded and never counts + + +@pytest.mark.asyncio +async def test_an_unmet_unjudged_row_fails_a_step_that_must_pass(): + v = _step(_WithRows, fail_task_when_not_passing=True) + v.rows = [{"id": "missing-line", "result": False, "score": 0.0}] + _fake_judge(v, []) + + with pytest.raises(VerifierNotPassingError, match=r"rows not met: \['missing-line'\]"): + await v.execute(_context()) + + +@pytest.mark.asyncio +async def test_an_unjudged_row_with_a_criterions_id_is_refused_before_the_judge_runs(): + v = _step(_WithRows) + v.rows = [{"id": "replied", "result": True}] + prompts: list[str] = [] + _fake_judge(v, prompts) + + with pytest.raises(RuntimeError, match=r"unjudged_rows gave rows with a criterion's id: \['replied'\]"): + await v.execute(_context()) + assert prompts == [] + + +@pytest.mark.asyncio +async def test_a_skipped_run_holds_no_unjudged_rows(): + v = _step(_WithRows) + v.rows = [{"id": "auto-asked", "result": True, "score": 1.0}] + ctx = TaskStepContext(prompt_responses=[PromptResponse(prompt_id="p1", response="", error_type="timeout")]) + + await v.execute(ctx) + + assert [r["id"] for r in ctx.metadata["verifications"]["vt"]["results"]] == ["prompt_error"] + + +class _RunInline(RubricsVerifierTaskStep): + async def judge_trajectory_content(self, context: TaskStepContext, prompt_response: PromptResponse) -> str: + return "THE RUN: the user sim asked for a table at 7." + + +@pytest.mark.asyncio +async def test_trajectory_content_from_a_subclass_comes_before_the_criteria_in_every_batch(): + v = _step(_RunInline, judge_batch_size=2) + prompts: list[str] = [] + _fake_judge(v, prompts) + + await v.execute(_context()) + + assert len(prompts) == 2 + assert all(p.index("THE RUN") < p.index('"id": "c') for p in prompts) + + +@pytest.mark.asyncio +async def test_without_trajectory_content_the_prompt_has_no_trajectory_section(): + with_run, without = _step(_RunInline), _step() + prompts: list[str] = [] + bare: list[str] = [] + _fake_judge(with_run, prompts) + _fake_judge(without, bare) + + await with_run.execute(_context()) + await without.execute(_context()) + + assert "THE RUN" not in bare[0] and len(prompts[0]) > len(bare[0]) + + +@pytest.mark.asyncio +async def test_trajectory_content_beside_a_recorded_trajectory_is_refused(): + v = _step(_RunInline, use_trajectory=True) + ctx = TaskStepContext(prompt_responses=[PromptResponse( + prompt_id="p1", response="done", prompt_text="do it", agent_trajectory_object_url="s3://t/1.json")]) + + with pytest.raises(RuntimeError, match="judge_trajectory_content stands in for the recorded trajectory"): + await v.execute(ctx) + + +def test_the_rubric_binary_and_partial_formats_alone_are_batchable_and_all_check_bulk_verdicts(): + assert {f for f, spec in JUDGE_OUTPUT_FORMAT_REGISTRY.items() if spec.batchable} == { + JudgeOutputFormat.RUBRIC_BINARY, JudgeOutputFormat.RUBRIC_PARTIAL} + assert all(spec.checks_bulk_verdicts for spec in JUDGE_OUTPUT_FORMAT_REGISTRY.values()) + + +def _with_spec(**fields) -> type[RubricsVerifierTaskStep]: + class WithSpec(RubricsVerifierTaskStep): + def _spec(self) -> JudgeOutputFormatSpec: + return replace(super()._spec(), **fields) + + return WithSpec + + +def test_a_spec_that_isnt_batchable_refuses_a_batch_size(): + with pytest.raises(ValueError, match="judge_batch_size does not apply to output_format=rubric_binary"): + _step(_with_spec(batchable=False), judge_batch_size=2) + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("checks", "calls"), [(True, 2), (False, 1)]) +async def test_a_spec_can_turn_off_the_bulk_verdict_check(checks, calls): + v = _step(_with_spec(checks_bulk_verdicts=checks), judge_batch_size=4) + prompts: list[str] = [] + _fake_judge(v, prompts, respond=lambda ids: _graded(ids, justification=lambda cid: "Pass." if len(prompts) == 1 + else f"Checked record {cid} with the sheet tool")) + ctx = _context() + + await v.execute(ctx) + + assert len(prompts) == calls + assert ctx.metadata["verifications"]["vt"]["score"] == 1.0 From 4db2603055617ad1f2809722239f73f9bdbb1718 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 16 Nov 2026 11:05:00 -0800 Subject: [PATCH 33/45] feat(verifiers): user_sim_checks rows give assistant_asked, recorded only Every row now gives assistant_asked: true when the assistant asked for or invited the behaviour the row grades (asked the user to write a message, or to give a fact it should look up), false otherwise. It is in the schema and the prompt, required on every row (a missing or non-bool one is asked again), and recorded only: it never changes a row's score or a gate. A test also shows that a task item follows the same rule as a check: a fail lowers the score only with could_change_grade. Co-Authored-By: Claude Opus 5.5 --- .../judge_utils/judge_output_format.py | 24 +++++--- .../test_user_sim_checks_format.py | 58 +++++++++++++------ 2 files changed, 54 insertions(+), 28 deletions(-) diff --git a/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py b/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py index e6a0e37d8..cffa0ae63 100644 --- a/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py +++ b/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py @@ -1282,10 +1282,13 @@ def format_trajectory_mistakes_correction( reason to at most 40 words: the evidence and the conclusion, not your working. - "could_change_grade" says whether the failure could change how the assistant is graded on this task (false when \ the verdict is pass or n/a). +- "assistant_asked" is true when the assistant asked for or invited the behaviour you graded (e.g. asked the user \ +to write a message or to give a fact it should look up); false otherwise. Give it on every result, false when \ +nothing was asked; it is recorded only and doesn't change the verdict. Return only this JSON object, with one result per criterion, each result's keys in this order: {"results": [{"id": "the criterion's id", "reason": "at most 40 words", "turns": [turn numbers the reason rests on], \ -"verdict": "pass" | "fail" | "n/a", "could_change_grade": true | false}]}""" +"verdict": "pass" | "fail" | "n/a", "could_change_grade": true | false, "assistant_asked": true | false}]}""" _USER_SIM_CRITERIA = """ @@ -1322,8 +1325,9 @@ def _user_sim_checks_output_format() -> dict: "turns": {"type": "array", "items": {"type": "integer"}}, "verdict": {"type": "string", "enum": list(USER_SIM_VERDICTS)}, "could_change_grade": {"type": "boolean"}, + "assistant_asked": {"type": "boolean"}, }, - "required": ["id", "reason", "turns", "verdict", "could_change_grade"], + "required": ["id", "reason", "turns", "verdict", "could_change_grade", "assistant_asked"], "additionalProperties": False, }, }, @@ -1390,8 +1394,8 @@ def _never_gates(criterion: dict) -> bool: def _user_sim_row(item: dict, criterion: dict) -> tuple[dict | None, list[str]]: """One result checked against its criterion: ``(row, [])``, or ``(None, problems)`` naming the row. The row lowers the score (score 0.0, result false) only on a fail with could_change_grade, and never at weight 0; - ``verdict`` and ``could_change_grade`` keep what the judge said either way. An ``assistant_asked`` bool (the - assistant asked for the behaviour) is kept when the judge gives one; it doesn't change the score.""" + ``verdict`` and ``could_change_grade`` keep what the judge said either way. ``assistant_asked`` (the assistant + asked for or invited the behaviour) is required and recorded only: it never changes the score or a gate.""" rid = item.get("id") problems = [] verdict = str(item.get("verdict", "")).strip().lower() @@ -1400,6 +1404,9 @@ def _user_sim_row(item: dict, criterion: dict) -> tuple[dict | None, list[str]]: change = item.get("could_change_grade") if not isinstance(change, bool): problems.append(f"{rid}: could_change_grade must be true or false") + asked = item.get("assistant_asked") + if not isinstance(asked, bool): + problems.append(f"{rid}: assistant_asked must be true or false") if verdict == "n/a" and str(criterion.get("na_when") or "").strip().lower() == _USER_SIM_NA_NEVER: problems.append(f"{rid} may not be n/a: it always applies, so answer pass or fail") turns = _cited_turns(item.get("turns")) @@ -1419,13 +1426,11 @@ def _user_sim_row(item: dict, criterion: dict) -> tuple[dict | None, list[str]]: lowers = change and not _never_gates(criterion) row = {**criterion, "id": rid, "reason": reason, "turns": [t for t in turns if graded is None or t in graded], - "verdict": verdict, "could_change_grade": change, + "verdict": verdict, "could_change_grade": change, "assistant_asked": asked, "justification": reason, "score": 0.0 if lowers else 1.0, "result": not lowers} outside = [t for t in turns if graded is not None and t not in graded] if outside: row["turns_outside_graded"] = outside - if isinstance(item.get("assistant_asked"), bool): - row["assistant_asked"] = item["assistant_asked"] return row, [] @@ -1434,7 +1439,7 @@ def diagnose_user_sim_checks_response( ) -> tuple[list[dict] | None, JudgeResponseDiscrepancy | None]: """Parse a user_sim_checks judge response into rows, or a discrepancy for the correction prompt: no JSON object, the wrong row set, or a row the user sim judge can't use (its verdict or could_change_grade invalid, an - n/a where ``na_when`` is never, a fail citing no graded turn, a fail whose reason concludes nothing failed).""" + n/a where ``na_when`` is never, an assistant_asked that isn't true or false, a fail citing no graded turn, a fail whose reason concludes nothing failed).""" parsed = _user_sim_json(response_text) if parsed is None: return None, _parse_failure_discrepancy(criteria, "the answer is not a JSON object") @@ -1511,6 +1516,7 @@ def format_user_sim_checks_correction( '- "verdict": "pass", "fail" or "n/a", following from the reason: a reason that finds no clear failure is ' 'never "fail"', '- "could_change_grade": true or false', + '- "assistant_asked": true when the assistant asked for or invited the behaviour you graded, else false', "", "## Your Previous Response", truncated, @@ -1529,7 +1535,7 @@ class JudgeOutputFormat(str, Enum): # trajectory_mistakes-shaped entry from the findings. RUBRIC_EVIDENCE_WITH_MISTAKES = "rubric_evidence_with_mistakes" # Grades the user sim (the AI playing the user), not the assistant: per criterion a reason, the turns it cites, a - # pass / fail / n/a verdict and could_change_grade; a row lowers the score only on a fail that could change the grade. + # pass / fail / n/a verdict, could_change_grade and assistant_asked (recorded only); a row lowers the score only on a fail that could change the grade. USER_SIM_CHECKS = "user_sim_checks" diff --git a/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py b/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py index 0b399732a..8f01b83eb 100644 --- a/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py +++ b/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py @@ -50,9 +50,9 @@ def _criteria(*ids: str, graded=(1, 2), **extra) -> list[dict]: def _row(cid="c1", *, verdict="pass", turns=(), could_change_grade=False, reason="Answered what was asked.", - **extra) -> dict: + assistant_asked=False, **extra) -> dict: return {"id": cid, "reason": reason, "turns": list(turns), "verdict": verdict, - "could_change_grade": could_change_grade, **extra} + "could_change_grade": could_change_grade, "assistant_asked": assistant_asked, **extra} def _answer(*rows: dict) -> str: @@ -111,7 +111,7 @@ def test_the_judge_reasons_first_and_the_verdict_follows_the_reason(): assert 'a reason that finds no clear failure gives "pass" (or "n/a"), never "fail"' in prompt shape = prompt.split("Return only this JSON object")[1] assert (shape.index('"id"') < shape.index('"reason"') < shape.index('"turns"') < shape.index('"verdict"') - < shape.index('"could_change_grade"')) + < shape.index('"could_change_grade"') < shape.index('"assistant_asked"')) def test_the_judge_is_told_how_to_read_the_run_and_the_criteria_fields(): @@ -120,20 +120,23 @@ def test_the_judge_is_told_how_to_read_the_run_and_the_criteria_fields(): '"graded_turns"', "The assistant's own words are the assistant's", 'A "hard" item is an exact rule', 'A "general" item is a judgement', '"na_when": "never" means it always applies', - '"could_change_grade" says whether the failure could change how the assistant is graded'): + '"could_change_grade" says whether the failure could change how the assistant is graded', + '"assistant_asked" is true when the assistant asked for or invited the behaviour you graded (e.g. ' + 'asked the user to write a message or to give a fact it should look up); false otherwise', + "false when nothing was asked; it is recorded only and doesn't change the verdict"): assert told in prompt, told # ---- schema ------------------------------------------------------------------------------------------------------- -def test_each_row_gives_its_reason_first_then_turns_verdict_and_could_change_grade(): +def test_each_row_gives_its_reason_first_then_turns_verdict_could_change_grade_and_assistant_asked(): schema = SPEC.output_format assert schema["type"] == "json_schema" and schema["schema"]["required"] == ["results"] row = schema["schema"]["properties"]["results"]["items"] - order = ["id", "reason", "turns", "verdict", "could_change_grade"] + order = ["id", "reason", "turns", "verdict", "could_change_grade", "assistant_asked"] assert list(row["properties"]) == order == row["required"] assert row["properties"]["verdict"]["enum"] == ["pass", "fail", "n/a"] assert row["properties"]["turns"] == {"type": "array", "items": {"type": "integer"}} - assert row["properties"]["could_change_grade"] == {"type": "boolean"} + assert row["properties"]["could_change_grade"] == row["properties"]["assistant_asked"] == {"type": "boolean"} assert row["additionalProperties"] is False and schema["schema"]["additionalProperties"] is False @@ -152,7 +155,8 @@ def test_an_answer_is_read_into_rows_with_turns_kept_to_the_graded_ones(): def test_the_parser_reads_the_keys_in_any_order_and_tolerates_a_fence_or_text_around_the_object(): - flipped = {"could_change_grade": True, "verdict": "FAIL ", "turns": [2], "reason": "Leaked the allergy.", "id": "c1"} + flipped = {"assistant_asked": False, "could_change_grade": True, "verdict": "FAIL ", "turns": [2], + "reason": "Leaked the allergy.", "id": "c1"} for text in (json.dumps({"results": [flipped]}), "```json\n" + _answer(flipped) + "\n```", "Here is my answer:\n" + _answer(flipped) + "\nDone."): rows, discrepancy = diagnose_user_sim_checks_response(text, _criteria("c1")) @@ -171,12 +175,17 @@ def test_an_answer_that_is_not_a_json_object_or_has_the_wrong_rows_is_asked_agai assert (discrepancy.missing_ids, discrepancy.duplicate_ids, discrepancy.unknown_ids) == (["c2"], ["c1"], ["c9"]) -def test_a_verdict_or_could_change_grade_out_of_range_is_asked_again(): +def test_a_verdict_could_change_grade_or_assistant_asked_out_of_range_is_asked_again(): + missing = _row("c4") + del missing["assistant_asked"] rows, discrepancy = diagnose_user_sim_checks_response( - _answer(_row("c1", verdict="maybe"), _row("c2", could_change_grade="yes")), _criteria("c1", "c2")) + _answer(_row("c1", verdict="maybe"), _row("c2", could_change_grade="yes"), _row("c3", assistant_asked="no"), + missing), _criteria("c1", "c2", "c3", "c4")) assert rows is None assert discrepancy.row_problems == ["c1: verdict 'maybe' is not one of ['pass', 'fail', 'n/a']", - "c2: could_change_grade must be true or false"] + "c2: could_change_grade must be true or false", + "c3: assistant_asked must be true or false", + "c4: assistant_asked must be true or false"] def test_a_fail_that_cites_no_graded_turn_is_asked_again_and_a_pass_needs_none(): @@ -239,13 +248,17 @@ def test_every_unusable_row_is_named_in_one_go(): assert str(discrepancy) == "; ".join(discrepancy.row_problems) -def test_assistant_asked_is_kept_when_given_and_never_changes_the_score(): - rows, discrepancy = diagnose_user_sim_checks_response( - _answer(_fail("c1", assistant_asked=True), _row("c2", assistant_asked="yes"), _row("c3")), - _criteria("c1", "c2", "c3")) - assert discrepancy is None - assert rows[0]["assistant_asked"] is True and rows[0]["score"] == 0.0 - assert "assistant_asked" not in rows[1] and "assistant_asked" not in rows[2] +def test_assistant_asked_is_recorded_on_every_row_and_never_changes_the_score_or_a_gate(): + criteria = _criteria("c1", "c2", "c3") + for asked in (True, False): + rows, discrepancy = diagnose_user_sim_checks_response( + _answer(_fail("c1", assistant_asked=asked), _fail("c2", could_change_grade=False, assistant_asked=asked), + _row("c3", assistant_asked=asked)), criteria) + assert discrepancy is None and [r["assistant_asked"] for r in rows] == [asked] * 3 + assert [(r["score"], r["result"]) for r in rows] == [(0.0, False), (1.0, True), (1.0, True)] + for aggregator in ScoreAggregator: + assert aggregate_score(rows, aggregator) == aggregate_score( + [{**r, "assistant_asked": not asked} for r in rows], aggregator) def test_parse_response_never_turns_an_unusable_answer_into_fails(): @@ -262,7 +275,8 @@ def test_the_correction_prompt_names_each_problem_and_asks_for_the_row_shape_aga assert prompt.startswith("EVAL\n\n## Correction Required") assert "- c1: a fail must cite at least one graded turn; cited [99], graded [1]" in prompt assert "exactly 2 objects" in prompt and "Required criterion ids (in any order): c1, c2" in prompt - for field in ('"id"', '"reason": at most 40 words', '"turns"', '"verdict"', '"could_change_grade"'): + for field in ('"id"', '"reason": at most 40 words', '"turns"', '"verdict"', '"could_change_grade"', + '"assistant_asked": true when the assistant asked for or invited the behaviour you graded'): assert f"- {field}" in prompt assert '"score"' not in prompt and '"justification"' not in prompt assert prompt.endswith("## Your Previous Response\n" + previous) @@ -295,6 +309,12 @@ def test_a_row_lowers_the_score_only_on_a_fail_that_could_change_the_grade(): assert (rows[0]["verdict"], rows[0]["score"], rows[0]["result"]) == ("fail", 1.0, True) +def test_a_task_item_follows_the_same_rule_as_a_check(): + criteria = [*_criteria("guidance"), *_criteria("a1_call_rae", family="A1", kind="hard")] + assert _score(_row("guidance"), _fail("a1_call_rae", could_change_grade=False), criteria=criteria) == 1.0 + assert _score(_row("guidance"), _fail("a1_call_rae"), criteria=criteria) == 0.5 + + def test_a_weight_0_row_is_recorded_but_never_gates(): criteria = [*_criteria("guidance"), *_criteria("style", weight=0)] rows, _ = diagnose_user_sim_checks_response(_answer(_row("guidance"), _fail("style")), criteria) From c978600feb191a421afdb0085c9d57e83951db6c Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 16 Nov 2026 11:51:21 -0800 Subject: [PATCH 34/45] user_sim_judge: rebuilt as a rubrics_verifier that fails the task on a grade-changing user sim slip UserSimJudge subclasses RubricsVerifierTaskStep, pinned to the direct judge, use_trajectory off and the user_sim_checks format. The run is judge_trajectory_content, so everything up to its end is one cached prefix; a system message says the judge grades the user sim. The 7 checks and the task's rubric_items are the criteria, one call per check with its items (_criteria_batches); auto, always-n/a and unreadable-line items come from unjudged_rows. Style rows have weight 0. The entry is verifications[verifier_id] with its score, and can_be_reward is False. fail_task_when_not_passing, on by default and needing fail_task_on_error, replaces mark_user_sim_error (refused at load): a fail that could change the grade, at a weight other than 0, on instructions that match the recorded prompt_sha256, raises VerifierNotPassingError. assistant_asked is recorded on each row and never gates. An answer the judge can't fix in 4 tries leaves its rows error and user_sim_error unknown, flagged, and never fails the task. Load checks reuse rate_user_satisfaction's run_step_of and its built-in user sim check, and with the gate on every step that can be the reward must depend on the judge. The judge's own model call, defaults, step_params handling, answer retries and concurrency are gone. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/task_step/registry.py | 4 +- .../agent/rate_user_satisfaction.py | 48 +- .../task_steps/verifiers/user_sim_judge.py | 491 +++++--- .../verifiers/user_sim_judge_core.py | 455 +++---- .../verifiers/test_user_sim_judge.py | 1122 ++++++++++------- 5 files changed, 1161 insertions(+), 959 deletions(-) diff --git a/src/agent_env/task_step/registry.py b/src/agent_env/task_step/registry.py index 826c94255..c4b5c5b9d 100644 --- a/src/agent_env/task_step/registry.py +++ b/src/agent_env/task_step/registry.py @@ -73,7 +73,7 @@ def _builtin_registry() -> dict[str, type["TaskStep"]]: from agent_env.task_step.task_steps.configure_scheduled_checks import ConfigureScheduledChecksTaskStep from agent_env.task_step.task_steps.install_agent import InstallAgentTaskStep from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep - from agent_env.task_step.task_steps.verifiers.user_sim_judge import UserSimJudgeTaskStep + from agent_env.task_step.task_steps.verifiers.user_sim_judge import UserSimJudge from agent_env.task_step.task_steps.verifiers.always_on_agent_verifier import AlwaysOnAgentVerifier from agent_env.task_step.task_steps.verifiers.run_container_unit_tests_verifier import RunContainerUnitTestsVerifierTaskStep from agent_env.task_step.task_steps.run_docker_container import RunDockerContainerTaskStep @@ -149,7 +149,7 @@ def _builtin_registry() -> dict[str, type["TaskStep"]]: _get_type(ConfigureScheduledChecksTaskStep): ConfigureScheduledChecksTaskStep, _get_type(InstallAgentTaskStep): InstallAgentTaskStep, _get_type(RubricsVerifierTaskStep): RubricsVerifierTaskStep, - _get_type(UserSimJudgeTaskStep): UserSimJudgeTaskStep, + _get_type(UserSimJudge): UserSimJudge, _get_type(AlwaysOnAgentVerifier): AlwaysOnAgentVerifier, _get_type(RunContainerUnitTestsVerifierTaskStep): RunContainerUnitTestsVerifierTaskStep, _get_type(RunDockerContainerTaskStep): RunDockerContainerTaskStep, diff --git a/src/agent_env/task_step/task_steps/agent/rate_user_satisfaction.py b/src/agent_env/task_step/task_steps/agent/rate_user_satisfaction.py index 16db2efc9..12ba5baa9 100644 --- a/src/agent_env/task_step/task_steps/agent/rate_user_satisfaction.py +++ b/src/agent_env/task_step/task_steps/agent/rate_user_satisfaction.py @@ -133,22 +133,7 @@ def from_dict(cls, data: dict) -> RateUserSatisfactionTaskStep: # ---- the load check --------------------------------------------------------------------------------------------- def _run_step(self, steps: list[TaskStep]) -> tuple[Optional[PromptAgentTaskStep], list[str]]: """The step this one rates, or why there's none.""" - where = f"{self.type} '{self.id}'" - if self.run_step_id is not None: - run = next((s for s in steps if s.id == self.run_step_id), None) - if run is None: - return None, [f"{where}: run_step_id '{self.run_step_id}' names no step of the task"] - if not isinstance(run, PromptAgentTaskStep): - return None, [f"{where}: run_step_id '{self.run_step_id}' is a {run.type} step; name the always_on or " - "prompt_agent step to rate"] - return run, [] - runs = [s for s in steps if isinstance(s, PromptAgentTaskStep)] - if not runs: - return None, [f"{where}: the task has no always_on or prompt_agent step to rate"] - if len(runs) > 1: - return None, [f"{where}: the task has several always_on and prompt_agent steps " - f"({', '.join(s.id for s in runs)}); set run_step_id to the one to rate"] - return runs[0], [] + return run_step_of(self, steps) def task_problems(self, steps: list[TaskStep]) -> list[str]: run, found = self._run_step(steps) @@ -159,8 +144,7 @@ def task_problems(self, steps: list[TaskStep]) -> list[str]: found.append(f"{where} comes before step '{run.id}', the run it rates; put it after") if self.depends_on is not None and run.id not in {d.task_step_id for d in self.depends_on}: found.append(f"{where}: its depends_on must name '{run.id}', the run it rates, so it runs once that's over") - user = _user_of(run) - if user is None or not any(isinstance(s, DeployUserSimAgentTaskStep) and s.agent_name == user for s in steps): + if not plays_built_in_user_sim(run, steps): found.append(f"{where}: the user of {run.type} step '{run.id}' isn't a user sim a deploy_user_sim_agent " "step deploys; only that user sim can rate the run") earlier = next((s for s in steps[:steps.index(self)] if isinstance(s, RateUserSatisfactionTaskStep) @@ -294,6 +278,34 @@ async def _trajectory(self, agent: DeployedAgent, session: _Session, task_id: st return None +# ---- the load check, shared with user_sim_judge ---------------------------------------------------------------------- +def run_step_of(step: TaskStep, steps: list[TaskStep], does: str = "rate") -> tuple[Optional[PromptAgentTaskStep], list[str]]: + """The always_on or prompt_agent step ``step`` works on (its ``run_step_id``'s, else the task's only one), or why + there's none. ``does`` names what ``step`` does to the run, in the messages.""" + where = f"{step.type} '{step.id}'" + if step.run_step_id is not None: + run = next((s for s in steps if s.id == step.run_step_id), None) + if run is None: + return None, [f"{where}: run_step_id '{step.run_step_id}' names no step of the task"] + if not isinstance(run, PromptAgentTaskStep): + return None, [f"{where}: run_step_id '{step.run_step_id}' is a {run.type} step; name the always_on or " + f"prompt_agent step to {does}"] + return run, [] + runs = [s for s in steps if isinstance(s, PromptAgentTaskStep)] + if not runs: + return None, [f"{where}: the task has no always_on or prompt_agent step to {does}"] + if len(runs) > 1: + return None, [f"{where}: the task has several always_on and prompt_agent steps " + f"({', '.join(s.id for s in runs)}); set run_step_id to the one to {does}"] + return runs[0], [] + + +def plays_built_in_user_sim(run: PromptAgentTaskStep, steps: list[TaskStep]) -> bool: + """Whether ``run``'s user is a user sim a deploy_user_sim_agent step of the task deploys.""" + user = _user_of(run) + return user is not None and any(isinstance(s, DeployUserSimAgentTaskStep) and s.agent_name == user for s in steps) + + # ---- the run, as the run step recorded it ---------------------------------------------------------------------------- def _user_of(run: PromptAgentTaskStep) -> Optional[str]: """The agent a run step talks to as its user: always_on's ``user_sim``; prompt_agent's ``user_agent_name``, unless diff --git a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py index eb893f60c..0f06f0bb4 100644 --- a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py +++ b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py @@ -1,25 +1,43 @@ """The user_sim_judge step: after a run with the built-in user sim, grade the user sim (the AI playing the user), not -the assistant. A task writer adds it after the run's step; nothing turns it on by default. +the assistant, and fail the task when the user sim slipped in a way that could change the assistant's grade, so the +run is done again rather than scored. A task writer adds it after the run's step. -It judges either kind of run the built-in user sim plays (``deploy_user_sim_agent``): +It is a rubrics_verifier (``RubricsVerifierTaskStep``) with its own criteria and its own context: the direct judge, +``use_trajectory`` off and ``output_format`` ``user_sim_checks`` (judge_output_format.py), none of them settings. Its +settings are its own fields, all described below: ``run_step_id``, ``judge_model`` (unset, rubrics_verifier's default: +the config's judge model), ``checks``, ``error_checks``, ``rubric_items``, ``rubric_persona_sha256``, +``fail_task_when_not_passing``, ``verifier_id`` and ``judge_timeout_seconds``. The model call, its key and base, its +retries and the calls' concurrency are rubrics_verifier's, and a run's ``user_overrides`` change them as they do for +every judge (``judge_model``; ``use_agent_judge`` sends the same prompt to an agent judge, without the system message +and the cache marks). + +Which run: ``run_step_id``, else the task's only always_on or prompt_agent step. It judges either kind the built-in user +sim plays (``deploy_user_sim_agent``): - an ``always_on`` run: every ``user_sim_read`` in the timeline stored at ``metadata["always_on"][step]["object_url"]`` (else the record's ``timeline_tail``, which keeps answers but not what the user sim was sent); - a ``prompt_agent`` chat run: each short entry in ``metadata["user_sim_reads"][step]``, read whole from its ``object_url`` (else the entry, which keeps the answer only). -``run_step_id`` names that step; unset, the run must have exactly one of them. The user sim's instructions are rebuilt -from its ``metadata["user_sims"]`` record by ``deploy_user_sim_agent``'s own path (``recorded_person``, then -``user_sim_versions.setup``), its opening's files included, and checked against the record's ``prompt_sha256``. ``used`` is never shown to the judge: it is the user sim's -own pulse check, not evidence. +The user sim's instructions are rebuilt from its ``metadata["user_sims"]`` record by ``deploy_user_sim_agent``'s own +path (``recorded_person``, then ``user_sim_versions.setup``), its opening's files included, and checked against the +record's ``prompt_sha256``. ``used`` is never shown to the judge: it is the user sim's own pulse check, not evidence. + +What the judge reads: a system message saying it grades the user sim, the format's framing, then the run +(``judge_trajectory_content``: the rebuilt instructions and every check-in, ``user_sim_judge_core.render_run``), then +the criteria. Everything up to the end of the run is the same in every call on a run, so it is marked for the prompt +cache, and the run's first call goes alone so the rest read it from the cache. -Each check (``user_sim_judge_core.CHECKS``) is one narrow question to ``judge_model``, in its own call; a task's -``rubric_items`` ride along with the check that owns their family, at most ``MAX_ITEMS_PER_CALL`` to a call. +The criteria (``user_sim_judge_core.plan_run``): each check in ``checks`` (``user_sim_judge_core.CHECKS``, all seven +by default) is one criterion and one call, and a task's ``rubric_items`` ride in the call of the check that owns their +family, at most ``MAX_ITEMS_PER_CALL`` to a call. Each row has a weight: 1 for a check in ``error_checks`` (every check +but style, by default; style is never one) and for an item that can make the run a user sim error (its ``can_error``, +unset as its family's check), else 0. A weight-0 row is recorded and never gates. A rubric item is the shape of a criterion in a task's ``usersim.yaml`` (psi-benchmark-tasks), which that repo turns into this step's settings; this step reads no file: -- ``id`` and ``family`` (A1-A5, F1-F4, G1-G5), both required; ids are unique. +- ``id`` and ``family`` (A1-A5, F1-F4, G1-G5), both required; ids are unique, and none is a check's name. - ``kind``: ``hard`` (an exact rule from the user sim's instructions: the judge fails it on any breach), ``general`` (a judgement: it fails only on a clear instance; the default) or ``auto`` (never sent to the model: ``metric`` ``max_words`` or ``max_sentences`` is counted on each graded answer, and any answer over ``max`` fails it, naming @@ -28,17 +46,18 @@ lengths can't be counted. - ``check``: the text the judge applies. ``text`` is its old name and still loads. - ``line``: instead of ``check``, a tag of the user sim's instructions ("P1", "T1", "C2", "D3"): the judge applies - that line's text from the run's ``metadata["user_sims"][name]["tagged_lines"]``, shown as "Follows its instruction - P1: ...", and the item's result records ``line`` and ``line_text``. A hard or general item has exactly one of - ``check`` and ``line``; an auto item, counted rather than judged, takes no ``line``. A tag names the same line only - for the same persona, so a line item needs ``rubric_persona_sha256``, and grades only on a run whose fingerprint - matches it: on a run that recorded none, it is ``error``. A tag the run doesn't have makes the item ``error`` - ("the user sim's instructions have no line P1"). Neither makes a call. -- ``name`` and ``short``: a title and one line for reviewers, recorded but not shown to the judge. + that line's text from the run's ``metadata["user_sims"][name]["tagged_lines"]``, and the item's row records ``line`` + and ``line_text``. A hard or general item has exactly one of ``check`` and ``line``; an auto item, counted rather + than judged, takes no ``line``. A tag names the same line only for the same persona, so a line item needs + ``rubric_persona_sha256``, and grades only on a run whose fingerprint matches it: on a run that recorded none, it + is ``error``. A tag the run doesn't have makes the item ``error`` ("the user sim's instructions have no line P1"). + Neither makes a call. +- ``name`` and ``short``: a title and one line for reviewers, on the item's row but never shown to the judge. - ``na_when``: the situation in which it never came up, told to the judge. ``always`` makes it n/a with no call, recording ``static_na_reason`` (required with it); ``never`` means it always applies, and an n/a is asked again. An auto item is counted, not judged, so it takes only ``always``; any other ``na_when`` is refused when the task loads. -- ``can_error``: whether its fail makes the run a user sim error; unset, as the check that owns its family. +- ``can_error``: whether its fail can make the run a user sim error (weight 1, or 0); unset, as the check that owns + its family. A task item never restates a persona line. The persona is the one place that says what the user sim knows and must not say, and the judge reads all of it with every check. Task items are for counted limits (``auto``) and judgment @@ -61,43 +80,76 @@ recorded no ``persona_sha256`` is ``unknown``, and the items are graded. Edits to the task that leave the persona and the opening as they were keep the items graded. -What it writes: ``metadata["user_sim_judge"][verifier_id]``, with every verdict, its reason and cited turns, the task -items, ``user_sim_error`` (``value`` yes / no / unknown, ``because``, ``marked``), the instructions check, token use -and cost. It writes no ``verifications`` entry, so its verdicts never mix into the assistant's scores. - -Record only, by default: with ``mark_user_sim_error`` off it changes nothing a grader reads. Turned on, a run whose -``user_sim_error`` is yes, judged against instructions that match the recorded hash, has its prompt response marked -``error_type = "user_sim_error"``, as a user sim that fails mid-run marks it, so graders that run after this step skip -it. A failure of this step doesn't fail the task unless ``fail_task_on_error`` says so. - -A run that ended in an error (the always_on record's or the prompt response's ``error_type``) is not judged: every -verifier skips its prompt anyway. +What it writes, all in one place, ``metadata["verifications"][verifier_id]``: + +- ``format`` (``user_sim_checks``), ``results`` and ``score``, as rubrics_verifier writes them. Each row: ``id`` (a + check's name or an item's id); what a reviewer reads (a check's ``title`` and ``families``; an item's ``family``, + ``kind``, ``check`` or ``line`` and ``line_text``, ``name``, ``short``, ``na_when``, ``asked_with``: the call it + rode in, or ``computed``, ``static_na``, ``no_line``); its ``weight``; then its ``verdict`` (pass, fail, n/a, or + error when undecided), ``reason``, ``turns``, ``could_change_grade``, ``assistant_asked`` (whether the assistant + asked for or invited what was graded: recorded only, never part of the gate; None on a row no model graded), + ``score`` and ``result``. A row lowers the score only on a fail that could change the grade at a weight other + than 0, so ``score`` is 0 when ``user_sim_error`` is yes, else 1. ``can_be_reward`` is False: the score checks the + run, so it is never the task's reward. +- ``user_sim_error``: ``value`` yes / no / unknown, ``because`` and ``unknown`` (the row ids), ``fails_task``, and + ``not_failing`` (why a yes didn't fail the task). ``flag``: "user sim judge undecided" when ``value`` is unknown. +- ``run_step_id``, ``run_kind``, ``user_sim``, ``judge_model``; ``instructions`` (``source``, ``prompt_sha256``, + ``recorded_prompt_sha256``, ``matches``); ``rubric_persona`` (when ``rubric_persona_sha256`` is set); ``notes`` + (what the judge was told about gaps in the record); ``graded_turns``, ``turns``; ``calls``, ``usage``, ``cost``; and + rubrics_verifier's ``judge_output_retries`` / ``judge_output_discrepancies`` / ``judge_batches``. +- ``skipped`` instead, with ``score`` None and no rows, for a run that isn't judged: one that ended in an error (the + always_on record's or the prompt response's ``error_type``: every verifier skips its prompt anyway), a chat run + whose user sim never read, and a run with no graded turn. + +The gate, ``fail_task_when_not_passing`` (on by default; it needs ``fail_task_on_error``, also on by default, and is +refused without it): the step raises ``VerifierNotPassingError``, naming the failing rows, when ``user_sim_error`` is +yes and the rebuilt instructions match the recorded ``prompt_sha256``. The task then ends FAILED with no score, and +the run is done again. Whoever caused the slip, the run is spoiled for grading the assistant, so ``assistant_asked`` +never changes it. Off, the verdicts are recorded only. When the gate is on, every step that can be the task's reward +(one with a ``verifier_id`` and ``can_be_reward``) must depend on this step, directly or through others, or the task is +refused when it loads. + +Unknown never fails the task: a run is failed only on a slip the judge found, so a flaky judge never throws away a good +run. A judge answer the format can't use is asked again with the format's correction, up to 3 more times +(``DEFAULT_MAX_RETRIES``, 4, counts the first ask; rubrics_verifier tries a failed model call as many times). A call +still without a usable answer leaves its rows ``error``, ``user_sim_error`` unknown and the entry flagged; the step +doesn't raise, and nothing the assistant is scored on changes. + +When the task loads it is refused when the run step can't be found (``rate_user_satisfaction.run_step_of``), when the +run's user isn't a user sim a deploy_user_sim_agent step deploys, when this step doesn't come after the run (its +``depends_on``, when set, must name it), when a reward step doesn't depend on it (above), and when it still sets +``mark_user_sim_error``, which ``fail_task_when_not_passing`` replaces. """ from __future__ import annotations import asyncio +import copy import hashlib import json import logging import re -import uuid +from dataclasses import dataclass, field from typing import Any, ClassVar, Optional -from agent_env.task_step.context import PromptResponse, TaskStepContext +from agent_env.task_step.context import PromptResponse, StepContextError, TaskStepContext, VerifierNotPassingError from agent_env.task_step.task_step import TaskStep, TaskStepDependency from agent_env.task_step.task_steps.verifiers import user_sim_judge_core as core +from agent_env.task_step.task_steps.verifiers.judge_utils.judge_output_format import JudgeOutputFormat +from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep logger = logging.getLogger(__name__) -JUDGE_KEY = "user_sim_judge" # Not imported: always_on and prompt_agent import the verifiers' peers. Each is the key its module names. ALWAYS_ON_KEY = "always_on" # always_on.ALWAYS_ON_KEY READS_KEY = "user_sim_reads" # prompt_agent's _BuiltInUserSim USER_SIMS_KEY = "user_sims" # deploy_user_sim_agent.USER_SIMS_KEY ALWAYS_ON, CHAT = "always_on", "prompt_agent" -RUN_STEP_TYPES = (ALWAYS_ON, CHAT) -FALLBACK_JUDGE_MODEL = "claude-sonnet-4-6" # rubrics_verifier's, when the config names no judge -USER_SIM_ERROR = "user_sim_error" +SYSTEM = ("You grade a user simulator: an AI playing a real person who messages their personal AI assistant. You " + "grade the user sim, never the assistant.") +FLAG_UNDECIDED = "user sim judge undecided" +NOT_FAILING_OFF = "fail_task_when_not_passing is off: verdicts are recorded only" +NOT_FAILING_UNCONFIRMED = ("the judge's copy of the user sim's instructions isn't confirmed to be the one it ran with, " + "so it doesn't fail the task") FLAG_ASSISTANT = "this answer looks like the assistant's (very long, or markdown)" FLAG_LAST = "after the run's last wake: this answer never reached the assistant" FLAG_HELD = "its reply was still waiting to be delivered when the run ended: it never reached the assistant" @@ -105,23 +157,33 @@ ASSISTANT_LIKE_MESSAGE_CHARS = 1200 SCHEMA_ERRORS_SHOWN_CHARS = 300 # of a malformed answer's schema errors, in the judge's turn note USER_SIM_ERROR_SHOWN_CHARS = 200 # of a user sim agent error, in the judge's turn note +ERROR_SHOWN_CHARS = 300 # of why a call got no usable answer, on its rows # Why a line item isn't graded on a run that recorded no persona fingerprint: its tag may name another line now. LINE_UNCHECKED = ("the run recorded no persona fingerprint, so its tags can't be checked against the persona the item " "was written for") -def default_judge_model() -> str: - """The model rubrics_verifier defaults to: the config's ``judge`` role, else claude-sonnet-4-6.""" - from agent_env.config import get_config - - return get_config().get_model_for_role("judge") or FALLBACK_JUDGE_MODEL +@dataclass +class _ThisRun: + """One run's judging: what it asks and what its calls cost. Held by the run's own copy of the step, never by the + step a task shares across runs.""" + plan: core.Plan + text: str # the run as the judge reads it + first: Optional[asyncio.Event] = None # set once the run's first call is over + calls: int = 0 + usage: list[Any] = field(default_factory=list) + cost: float = 0.0 -class UserSimJudgeTaskStep(TaskStep): +class UserSimJudge(RubricsVerifierTaskStep): type: ClassVar[str] = "user_sim_judge" entity_refs = () - DEFAULT_TIMEOUT_SECONDS: ClassVar[int] = 600 - DEFAULT_MAX_RETRIES: ClassVar[int] = 3 + context_paths = () + field_scopes = TaskStep.field_scopes + can_be_reward: ClassVar[bool] = False + # Each call's answer is asked for up to this many times: the first ask and 3 corrections. rubrics_verifier tries a + # failed model call the same number of times. + DEFAULT_MAX_RETRIES: ClassVar[int] = 4 def __init__( self, @@ -133,120 +195,168 @@ def __init__( error_checks: Optional[list[str]] = None, rubric_items: Optional[list[dict]] = None, rubric_persona_sha256: Optional[str] = None, - mark_user_sim_error: bool = False, + fail_task_when_not_passing: bool = True, verifier_id: Optional[str] = None, judge_timeout_seconds: Optional[int] = None, depends_on: Optional[list[TaskStepDependency]] = None, - fail_task_on_error: bool = False, + fail_task_on_error: bool = True, ): - super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error) + if fail_task_when_not_passing is True and not fail_task_on_error: + raise ValueError( + f"user_sim_judge '{id}': fail_task_when_not_passing (on by default) needs fail_task_on_error: a " + "tolerated failure lets the steps after it run and score the run. Set fail_task_on_error to true, or " + "fail_task_when_not_passing to false to record the verdicts only.") + super().__init__( + id, version, criteria=[], prompt_id=None, default_model=judge_model, verifier_id=verifier_id, + use_agent_judge=False, use_trajectory=False, output_format=JudgeOutputFormat.USER_SIM_CHECKS, + judge_timeout_seconds=judge_timeout_seconds, depends_on=depends_on, fail_task_on_error=fail_task_on_error, + fail_task_when_not_passing=fail_task_when_not_passing is True) + self.fail_task_when_not_passing = fail_task_when_not_passing # as given, so config_problems sees a non-bool self.run_step_id = run_step_id - self.judge_model = judge_model # None: default_judge_model() when the step runs + self.judge_model = judge_model # None: rubrics_verifier's default, the config's judge model self.checks = list(checks) if checks is not None else list(core.CHECK_NAMES) self.error_checks = (list(error_checks) if error_checks is not None else [c for c in core.DEFAULT_ERROR_CHECKS if c in self.checks]) self.rubric_items = list(rubric_items or []) self.rubric_persona_sha256 = rubric_persona_sha256 - self.mark_user_sim_error = mark_user_sim_error - self.verifier_id = verifier_id or uuid.uuid4().hex - self.judge_timeout_seconds = judge_timeout_seconds or self.DEFAULT_TIMEOUT_SECONDS + self._run: Optional[_ThisRun] = None # ---- settings ---------------------------------------------------------------------------------------------- def config_problems(self) -> list[str]: - return _problems(self.judge_model, self.checks, self.error_checks, self.rubric_items, self.mark_user_sim_error, - self.rubric_persona_sha256) + return _problems(self.judge_model, self.checks, self.error_checks, self.rubric_items, + self.fail_task_when_not_passing, self.rubric_persona_sha256) def task_problems(self, steps: list[TaskStep]) -> list[str]: - """The run it judges must be an always_on or prompt_agent step's; unset, the task must have exactly one.""" - runs = [s for s in steps if s.type in RUN_STEP_TYPES] - if self.run_step_id is not None: - named = next((s for s in steps if s.id == self.run_step_id), None) - if named is not None and named.type not in RUN_STEP_TYPES: - return [f"step '{self.id}' ({self.type}) names the user sim of step '{named.id}' ({named.type}); " - "it judges an always_on or prompt_agent step's"] - return [] - if len(runs) > 1: - return [f"step '{self.id}' ({self.type}): set run_step_id; this task has {len(runs)} steps it could judge " - f"({sorted(s.id for s in runs)})"] - return [] + """The run it judges must be an always_on or prompt_agent step's, played by a built-in user sim, and come + before it; with the gate on, every step that can be the task's reward must depend on it.""" + from agent_env.task.task import _ancestor_ids, _dependency_ids # here: the task module imports the steps + from agent_env.task_step.task_steps.agent.rate_user_satisfaction import plays_built_in_user_sim, run_step_of + + run, found = run_step_of(self, steps, does="judge") + if run is None: + return found + where = f"{self.type} '{self.id}'" + if steps.index(run) > steps.index(self): + found.append(f"{where} comes before step '{run.id}', the run it judges; put it after") + if self.depends_on is not None and run.id not in {d.task_step_id for d in self.depends_on}: + found.append(f"{where}: its depends_on must name '{run.id}', the run it judges, so it runs once that's over") + if not plays_built_in_user_sim(run, steps): + found.append(f"{where}: the user of {run.type} step '{run.id}' isn't a user sim a deploy_user_sim_agent " + "step deploys; the judge grades only the built-in user sim") + if self.fail_task_when_not_passing is True: + deps = _dependency_ids(steps) + for step in steps: + if (step is not self and getattr(step, "verifier_id", None) and getattr(step, "can_be_reward", True) + and self.id not in _ancestor_ids(step.id, deps)): + found.append(f"step '{step.id}' ({step.type}) can be the task's reward but doesn't depend on " + f"{where}, so a run the user sim spoiled could still be scored; make it depend on " + f"'{self.id}', or set fail_task_when_not_passing to false to record the verdicts only") + return found def to_dict(self) -> dict: - base = super().to_dict() + base = TaskStep.to_dict(self) base.update(run_step_id=self.run_step_id, judge_model=self.judge_model, checks=self.checks, error_checks=self.error_checks, rubric_items=self.rubric_items, - rubric_persona_sha256=self.rubric_persona_sha256, mark_user_sim_error=self.mark_user_sim_error, - verifier_id=self.verifier_id, + rubric_persona_sha256=self.rubric_persona_sha256, + fail_task_when_not_passing=self.fail_task_when_not_passing, verifier_id=self.verifier_id, judge_timeout_seconds=self.judge_timeout_seconds) return base @classmethod - def from_dict(cls, data: dict) -> "UserSimJudgeTaskStep": - base = {**cls._base_from_dict(data), "fail_task_on_error": data.get("fail_task_on_error", False)} - return cls(**base, run_step_id=data.get("run_step_id"), judge_model=data.get("judge_model"), - checks=data.get("checks"), error_checks=data.get("error_checks"), - rubric_items=data.get("rubric_items"), rubric_persona_sha256=data.get("rubric_persona_sha256"), - mark_user_sim_error=data.get("mark_user_sim_error", False), + def from_dict(cls, data: dict) -> "UserSimJudge": + if "mark_user_sim_error" in data: + raise ValueError( + f"user_sim_judge '{data.get('id')}': mark_user_sim_error is gone; fail_task_when_not_passing (on by " + "default) takes its place: a run whose user sim slipped in a way that could change the grade fails " + "the task, so it is run again. Remove mark_user_sim_error, and set fail_task_when_not_passing to " + "false to record the verdicts only.") + return cls(**cls._base_from_dict(data), run_step_id=data.get("run_step_id"), + judge_model=data.get("judge_model"), checks=data.get("checks"), + error_checks=data.get("error_checks"), rubric_items=data.get("rubric_items"), + rubric_persona_sha256=data.get("rubric_persona_sha256"), + fail_task_when_not_passing=data.get("fail_task_when_not_passing", True), verifier_id=data.get("verifier_id"), judge_timeout_seconds=data.get("judge_timeout_seconds")) - def _settings(self, context: TaskStepContext) -> dict: - """This run's settings: a run's ``step_params`` for this step override the stored ones.""" - over = self.step_param_overrides(context) - model = over.get("judge_model", self.judge_model) - checks = list(over.get("checks", self.checks)) - error_checks = list(over["error_checks"]) if "error_checks" in over else ( - [c for c in self.error_checks if c in checks] if "checks" in over else list(self.error_checks)) - items = over.get("rubric_items", self.rubric_items) - persona = over.get("rubric_persona_sha256", self.rubric_persona_sha256) - mark = over.get("mark_user_sim_error", self.mark_user_sim_error) - problems = _problems(model, checks, error_checks, items, mark, persona) - if problems: - raise ValueError(f"user_sim_judge '{self.id}': " + "; ".join(problems)) - return {"model": model or default_judge_model(), "checks": checks, "error_checks": error_checks, - "mark": mark, "items": [core.RubricItem.from_dict(i) for i in items], "rubric_persona_sha256": persona} - # ---- the run ----------------------------------------------------------------------------------------------- async def execute(self, context: TaskStepContext) -> TaskStepContext: - settings = self._settings(context) + context = await self._grade(context) + error = context.metadata["verifications"][self.verifier_id]["user_sim_error"] + if error.get("fails_task"): + raise VerifierNotPassingError( + f"user_sim_judge '{self.id}': the user sim slipped in a way that could change the assistant's grade " + f"({', '.join(error['because'])}), so the run doesn't count; run it again") + return context + + async def _grade(self, context: TaskStepContext) -> TaskStepContext: + problems = self.config_problems() + if problems: + raise ValueError(f"user_sim_judge '{self.id}': " + "; ".join(problems)) + items = [core.RubricItem.from_dict(i) for i in self.rubric_items] kind, step_id = self._find_run(context) response = _prompt_response(context, step_id) - entry: dict[str, Any] = {"run_step_id": step_id, "run_kind": kind, "judge_model": settings["model"], - "checks_run": settings["checks"], "error_checks": settings["error_checks"], - "rubric_items": [i.to_dict() for i in settings["items"]], - "mark_user_sim_error": settings["mark"]} - items = settings["items"] + model = (context.metadata.get("user_overrides") or {}).get("judge_model") or self.default_model + entry: dict[str, Any] = {"run_step_id": step_id, "run_kind": kind, "judge_model": model} record = (context.metadata.get(ALWAYS_ON_KEY) or {}).get(step_id) if kind == ALWAYS_ON else None error_type = (record or {}).get("error_type") or (response.error_type if response else None) + skipped = None if error_type: - entry.update(skipped=f"the run ended with error_type={error_type}") + skipped = f"the run ended with error_type={error_type}" elif kind == CHAT and not (context.metadata.get(READS_KEY) or {}).get(step_id): - entry.update(skipped="the user sim never read: the run has no user_sim_reads") + skipped = "the user sim never read: the run has no user_sim_reads" else: run, about = await asyncio.to_thread(_read_run, kind, step_id, record, context) - entry["user_sim"], entry["instructions"] = about["user_sim"], about["instructions"] - if settings["rubric_persona_sha256"] is not None and items: + entry.update(user_sim=about["user_sim"], instructions=about["instructions"]) + if self.rubric_persona_sha256 is not None and items: us = (context.metadata.get(USER_SIMS_KEY) or {}).get(about["user_sim"]) or {} - entry["rubric_persona"] = _rubric_persona(settings["rubric_persona_sha256"], us.get("persona_sha256")) + entry["rubric_persona"] = _rubric_persona(self.rubric_persona_sha256, us.get("persona_sha256")) if entry["rubric_persona"]["status"] == "mismatch": items = [] - context.record_model("user sim judge", self.id, settings["model"]) - unknown = (entry.get("rubric_persona") or {}).get("status") == "unknown" - result = await core.judge_run( - run, model=settings["model"], checks=settings["checks"], items=items, - error_checks=settings["error_checks"], complete=self._complete(context), - lines_unusable=LINE_UNCHECKED if unknown else "") - result.pop("model", None) # the entry's judge_model - entry.update(result) - if "skipped" in entry: - logger.warning(f"user_sim_judge '{self.id}': skipped, {entry['skipped']}") - entry["user_sim_error"] = dict(core.NOT_JUDGED) - entry["user_sim_error"].update(self._mark(entry, settings["mark"], response)) - context.metadata.setdefault(JUDGE_KEY, {})[self.verifier_id] = entry - error = entry["user_sim_error"] - logger.info(f"user_sim_judge '{self.id}': {kind} '{step_id}' user_sim_error={error['value']} " - f"({', '.join(error['because'] + error['unknown']) or 'no check'}), marked={error['marked']}; " - + ", ".join(f"{k}={v['verdict']}" for k, v in (entry.get("checks") or {}).items())) + graded = [t.n for t in run.turns if t.graded] + entry.update(graded_turns=graded, turns=len(run.turns), notes=list(run.notes)) + if not graded: + skipped = "no graded turns" + elif response is None: + raise StepContextError(f"user_sim_judge '{self.id}': step '{step_id}' left no prompt response") + if skipped: + logger.warning(f"user_sim_judge '{self.id}': skipped, {skipped}") + context.metadata.setdefault("verifications", {})[self.verifier_id] = { + **self._entry_format_fields(), "results": [], "score": None, **entry, "skipped": skipped, + "user_sim_error": {**core.NOT_JUDGED, "fails_task": False}} + return context + + unusable = LINE_UNCHECKED if (entry.get("rubric_persona") or {}).get("status") == "unknown" else "" + plan = core.plan_run(run, checks=self.checks, items=items, error_checks=self.error_checks, + lines_unusable=unusable) + judge = copy.copy(self) # this run's own copy: the step is shared by every run of the task + judge.prompt_id, judge._run = response.prompt_id, _ThisRun(plan=plan, text=core.run_text(run)) + if plan.criteria: + await RubricsVerifierTaskStep._grade(judge, context) + else: # nothing for a model: only rows settled without one + judge._write_entries(context, [], {}, unjudged=plan.unjudged) + self._finish(context.metadata["verifications"][self.verifier_id], entry, judge._run) return context + def _finish(self, written: dict, entry: dict, this: _ThisRun) -> None: + """The written entry with its rows in the plan's order and shape, the run's verdict, and the run's details.""" + rows = core.finish_rows(written["results"], this.plan) + error = core.user_sim_error(rows) + error["fails_task"] = False + if error["value"] == "yes": + if self.fail_task_when_not_passing is not True: + error["not_failing"] = NOT_FAILING_OFF + elif not (entry.get("instructions") or {}).get("matches"): + error["not_failing"] = NOT_FAILING_UNCONFIRMED + else: + error["fails_task"] = True + written.update(results=rows, score=0.0 if any(r.get("result") is False for r in rows) else 1.0, **entry, + user_sim_error=error, calls=this.calls, usage=core.usage_total(this.usage), + cost=round(this.cost, 6)) + if error["value"] == "unknown": + written["flag"] = FLAG_UNDECIDED + logger.info(f"user_sim_judge '{self.id}': {entry['run_kind']} '{entry['run_step_id']}' user_sim_error=" + f"{error['value']} ({', '.join(error['because'] + error['unknown']) or 'no row'}), fails_task=" + f"{error['fails_task']}; " + ", ".join(f"{r['id']}={r['verdict']}" for r in rows)) + def _find_run(self, context: TaskStepContext) -> tuple[str, str]: """The kind of run and its step id: ``run_step_id``'s, else the run's only always_on or chat run.""" records = context.metadata.get(ALWAYS_ON_KEY) or {} @@ -264,73 +374,70 @@ def _find_run(self, context: TaskStepContext) -> tuple[str, str]: f"built-in user sim ({sorted(s for _, s in runs)})") return runs[0] - def _mark(self, entry: dict, mark: bool, response: Optional[PromptResponse]) -> dict: - """With ``mark`` on, a user sim error marks the run's prompt response, so graders after this step skip it.""" - error = entry["user_sim_error"] - if error["value"] != "yes": - return {"marked": False} - if not mark: - return {"marked": False, "not_marked": "mark_user_sim_error is off: verdicts are recorded only"} - if not (entry.get("instructions") or {}).get("matches"): - return {"marked": False, "not_marked": "the judge's copy of the user sim's instructions isn't confirmed " - "to be the one it ran with"} - if response is None: - return {"marked": False, "not_marked": f"step '{entry['run_step_id']}' left no prompt response"} - response.error_type = USER_SIM_ERROR - response.error_message = response.error_message or ( - "The user sim judge found a user sim error: " + ", ".join(error["because"])) - return {"marked": True} - - def _complete(self, context: TaskStepContext) -> core.Complete: - """One judge call through LiteLLM, with the key and base the run's overrides name (as rubrics_verifier's - judge), the run's two shared parts marked for Anthropic's prompt cache.""" - overrides = context.metadata.get("user_overrides", {}) - timeout, retries = self.judge_timeout_seconds, self.DEFAULT_MAX_RETRIES - - async def complete(model: str, system: str, parts: list[str], schema: dict) -> dict: - import litellm - - from agent_env.config import get_config - from agent_env.config.model import ModelParam - - config = get_config() - call = config.resolve_model_call( - model, - default_api_key=config.resolve_reference(overrides.get("judge_litellm_api_key") - or overrides.get("litellm_api_key")), - base_override=overrides.get("judge_litellm_base_url") or overrides.get("litellm_base_url")) - cache = _anthropic(model) - content = [{"type": "text", "text": p, **({"cache_control": {"type": "ephemeral"}} if cache and i < 2 - else {})} - for i, p in enumerate(parts) if p] - kwargs = dict(model=model, messages=[{"role": "system", "content": system}, - {"role": "user", "content": content}], - timeout=timeout, **call.client_kwargs()) - kwargs[ModelParam.RESPONSE_FORMAT] = {"type": "json_schema", "json_schema": { - "name": "user_sim_check", "schema": schema, "strict": True}} - last: Optional[Exception] = None - for attempt in range(1, retries + 1): - try: - response = await litellm.acompletion(**kwargs) - usage = getattr(response, "usage", None) - hidden = getattr(response, "_hidden_params", None) or {} - return {"text": response.choices[0].message.content or "", - "usage": usage.model_dump() if hasattr(usage, "model_dump") else dict(usage or {}), - "cost": hidden.get("response_cost")} - except Exception as e: - last = e - logger.warning(f"user_sim_judge: {model} call {attempt}/{retries} failed: {type(e).__name__}: {e}") - raise RuntimeError(f"{model} failed after {retries} attempts") from last - - return complete - - -def _anthropic(model: str) -> bool: - m = model.lower() - return m.startswith(("anthropic/", "bedrock/", "vertex_ai/")) and "claude" in m or m.startswith("claude") - - -def _problems(model: Any, checks: Any, error_checks: Any, items: Any, mark: Any, + # ---- rubrics_verifier's hooks, on the run's own copy --------------------------------------------------------- + def _this(self) -> _ThisRun: + if self._run is None: + raise RuntimeError(f"user_sim_judge '{self.id}': the judge's hooks run only on a run's own copy of the step") + return self._run + + def _resolved_criteria(self, context: TaskStepContext) -> list[dict]: + return [dict(c) for c in self._this().plan.criteria] + + async def judge_trajectory_content(self, context: TaskStepContext, prompt_response: PromptResponse) -> str: + return self._this().text + + async def unjudged_rows(self, context: TaskStepContext, prompt_response: PromptResponse) -> list[dict]: + return list(self._this().plan.unjudged) + + def _criteria_batches(self, criteria: list[dict], size: Optional[int]) -> list[list[dict]]: + """One call per check, with the task items riding with it, then the calls that ask only items.""" + out, start = [], 0 + for n in self._this().plan.batches: + out.append(criteria[start:start + n]) + start += n + return out + + async def _prompt_llm_judge(self, eval_prompt: str, *, model: str, context: TaskStepContext, + image_blocks: Optional[list[dict]] = None, system: Optional[str] = None, + cached_prefix: Optional[str] = None) -> dict: + """rubrics_verifier's call, with the system message and everything up to the end of the run marked for the + prompt cache. The run's first call goes alone, so the others read the run from the cache.""" + this = self._this() + end = eval_prompt.find(this.text) + head = eval_prompt[:end + len(this.text)] if end >= 0 else None + + async def call() -> dict: + got = await super(UserSimJudge, self)._prompt_llm_judge( + eval_prompt, model=model, context=context, image_blocks=image_blocks, system=SYSTEM, + cached_prefix=head) + this.calls += 1 + this.usage.append(got.get("usage")) + this.cost += got.get("cost") or 0.0 + return got + + if this.first is None: + this.first = asyncio.Event() + try: + return await call() + finally: + this.first.set() + await this.first.wait() + return await call() + + async def _run_judge_with_output_retries(self, *, criteria: list[dict], **kwargs: Any + ) -> tuple[list[dict], int, list[dict], Optional[str]]: + """rubrics_verifier's answer retries; a call still without a usable answer leaves its criteria undecided + (``error``) rather than failing the step: an unknown never fails the task.""" + try: + return await super()._run_judge_with_output_retries(criteria=criteria, **kwargs) + except (ValueError, RuntimeError) as e: + error = (f"no usable answer after {self.DEFAULT_MAX_RETRIES} tries: {type(e).__name__}: " + f"{str(e)[:ERROR_SHOWN_CHARS]}") + logger.warning(f"user_sim_judge '{self.id}': {len(criteria)} criteria undecided, {error}") + return [core.undecided_row(c, error) for c in criteria], 0, [], None + + +def _problems(model: Any, checks: Any, error_checks: Any, items: Any, gate: Any, rubric_persona: Any = None) -> list[str]: out = [] if rubric_persona is not None and not (isinstance(rubric_persona, str) @@ -339,8 +446,8 @@ def _problems(model: Any, checks: Any, error_checks: Any, items: Any, mark: Any, f"{rubric_persona!r}") if model is not None and (not isinstance(model, str) or not model.strip()): out.append("judge_model must be a model name") - if not isinstance(mark, bool): - out.append("mark_user_sim_error must be true or false") + if not isinstance(gate, bool): + out.append("fail_task_when_not_passing must be true or false") if not isinstance(checks, list) or not all(isinstance(c, str) for c in checks): out.append(f"checks must be a list from {list(core.CHECK_NAMES)}") else: @@ -351,14 +458,20 @@ def _problems(model: Any, checks: Any, error_checks: Any, items: Any, mark: Any, out.append("checks is empty and there are no rubric_items: nothing to judge") if not isinstance(error_checks, list) or not all(isinstance(c, str) for c in error_checks): out.append("error_checks must be a list of check names") - elif isinstance(checks, list): - not_run = sorted(set(error_checks) - set(checks)) - if not_run: - out.append(f"error_checks names checks that don't run: {not_run}") + else: + if isinstance(checks, list): + not_run = sorted(set(error_checks) - set(checks)) + if not_run: + out.append(f"error_checks names checks that don't run: {not_run}") + if "style" in error_checks: + out.append("error_checks can't name style: its rows are recorded only (weight 0), never a user sim error") if not isinstance(items, list): out.append("rubric_items must be a list of {id, family, kind, check, ...}") else: out += core.item_problems(items) + out += [f"rubric_items[{i}] ({item['id']}): id '{item['id']}' is a check's name; give the item another id" + for i, item in enumerate(items) if isinstance(item, dict) and isinstance(item.get("id"), str) + and item["id"] in core.CHECKS] if rubric_persona is None: out += [f"rubric_items[{i}] ({item.get('id')}): a line item needs rubric_persona_sha256, so its tag is " "graded only on runs of the persona it was written for" diff --git a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py index a9a5539b0..6e4036826 100644 --- a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py +++ b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py @@ -1,5 +1,5 @@ -"""What the user_sim_judge step asks, and how it combines the answers. Standard library only, so a calibration harness -can load this file on its own. +"""What the user_sim_judge step asks about a run, and how it reads the rows. Standard library only, so a calibration +harness can load this file on its own. The judge grades the user sim (the AI playing the user), not the assistant. Each check asks one narrow question over the user sim's whole run and covers families of the User Simulator Rubric Taxonomy (PSI Bench SSOT): @@ -14,22 +14,23 @@ | ending | G4 | A5 (satisfaction) has no check: the separate ``rate_user_satisfaction`` step covers it, not this judge. A task's own -rubric items (``RubricItem``) may name any family; an item goes into the call of the check that owns its family, and an -item whose family no running check owns gets one call of its own (``TASK_ITEMS``). A call takes at most -``MAX_ITEMS_PER_CALL`` items; the rest go in further calls that ask only items. ``auto`` items and ``na_when: always`` -items are settled here, with no call. An item's ``line`` (a tag, "P1") is graded by that line's text in the run's -``tagged_lines``; a tag the run doesn't have makes the item ``error``, with no call. - -One judge model answers each check in its own call. ``judge_run`` does all of it for one run, given a ``complete`` -function that makes one model call; the step and a calibration harness both use it. +rubric items (``RubricItem``) may name any family. ``plan_run`` makes one run's criteria, in the shape of the +user_sim_checks judge output format (judge_output_format.py): each check is one criterion and one call, and an item +rides in the call of the check that owns its family; an item whose family no running check owns goes in a call of its +own (``TASK_ITEMS``). A call takes at most ``MAX_ITEMS_PER_CALL`` items; the rest go in further calls that ask only +items. ``auto`` items, ``na_when: always`` items and line items whose line can't be read are settled here, with no +call. An item's ``line`` (a tag, "P1") is graded by that line's text in the run's ``tagged_lines``. + +Each row has a weight: 1 for a check in the step's ``error_checks`` (every check but style, by default) and for an +item that can make the run a user sim error, else 0. ``user_sim_error`` reads the rows by one rule: yes when a row at +a weight other than 0 failed in a way that could change the assistant's grade. """ from __future__ import annotations -import asyncio import json import re from dataclasses import dataclass, field, replace -from typing import Any, Awaitable, Callable, Optional +from typing import Any, Callable, Optional # ---- the rubric families ------------------------------------------------------------------------------------------ FAMILIES: dict[str, str] = { @@ -53,7 +54,6 @@ "G5": "Independence, no collusion: acts as the assistant's customer, not its teammate.", } -VERDICTS = ("pass", "fail", "n/a") TASK_ITEMS = "task_items" # An item's ``asked_with`` when no call asked it: an auto item counted here, an ``na_when: always`` item. COMPUTED, STATIC_NA = "computed", "static_na" @@ -240,57 +240,6 @@ class Check: # recorded and never makes a run one on its own. DEFAULT_ERROR_CHECKS = ("guidance", "oversharing", "delegation", "rogue", "made_up", "ending") -INSTRUCTIONS = """You check a user simulator: an AI playing a real person who messages their personal AI assistant. \ -You grade the user simulator, never the assistant. You are given, in order: -1. THE USER SIM'S INSTRUCTIONS: exactly what it was told about who it is, what it knows, its decisions, its rules for \ -the assistant and what is private. Hold it to these lines only. -2. THE RUN: every check-in it got, word for word ("what it was sent": the time, its new and earlier emails, texts and \ -notifications, the lines it sent, and the assistant's new messages), each followed by its answer: "reply" (the \ -message reaches the assistant), "no_reply" (nothing is sent) or "done" (a sign-off). -3. ONE QUESTION about it. - -How to read the run: -- What it was sent at a turn, and at every earlier turn, is what it knew; a fact it was never shown is not one it \ -forgot. -- Turns marked CONTEXT ONLY are fixed history: never fail the user sim for them, but they still count as what it said. -- Grade only the turns marked GRADED. Cite turns by their number. -- The assistant's own words are the assistant's, even when they repeat an email. - -Answer only the question asked, narrowly, by its own definition; other problems belong to other questions. A failure \ -needs clear evidence in a graded answer. - -Reason first and decide last. Write the reason, then the turns it rests on, and only then the verdict, which follows \ -from the reason: a reason that finds no clear failure gives "pass" (or "n/a"), never "fail". Keep every reason to at \ -most 40 words: the evidence and the conclusion, not your working. "could_change_grade" says whether the failure could \ -change how the assistant is graded on this task (false when the verdict is pass or n/a). - -Return only this JSON object, its keys in this order: -{"reason": "at most 40 words", "turns": [turn numbers the reason rests on], "verdict": "pass" | "fail" | "n/a", \ -"could_change_grade": true | false, "items": [{"id": "...", "reason": "at most 40 words", "turns": [...], \ -"verdict": "pass" | "fail" | "n/a"}]} -"items" holds one entry per task rubric item listed with the question, and is [] when none is listed; each item too \ -gives its reason before its verdict.""" - -# Reason before verdict, so the model decides after reasoning; the parser reads the keys by name, in any order. -OUTPUT_SCHEMA: dict[str, Any] = { - "type": "object", - "additionalProperties": False, - "properties": { - "reason": {"type": "string"}, - "turns": {"type": "array", "items": {"type": "integer"}}, - "verdict": {"type": "string", "enum": list(VERDICTS)}, - "could_change_grade": {"type": "boolean"}, - "items": {"type": "array", "items": { - "type": "object", "additionalProperties": False, - "properties": {"id": {"type": "string"}, "reason": {"type": "string"}, - "turns": {"type": "array", "items": {"type": "integer"}}, - "verdict": {"type": "string", "enum": list(VERDICTS)}}, - "required": ["id", "reason", "turns", "verdict"]}}, - }, - "required": ["reason", "turns", "verdict", "could_change_grade", "items"], -} - - # ---- the run the judge reads -------------------------------------------------------------------------------------- @dataclass class Turn: @@ -564,53 +513,18 @@ def render_run(run: UserSimRun) -> tuple[str, str]: return "\n".join(head), "\n".join(answers) -KIND_RULES = { - HARD: "A hard item is an exact rule from the user sim's instructions: it fails on any breach of the list or value " - "it names, however small.", - GENERAL: "A general item is a judgement about how this person would behave: it fails only on a clear instance.", -} -def call_check(call: str) -> Optional[str]: - """The check whose question a call asks; None for a call that asks only task items (``TASK_ITEMS``, or a - check's or the task's further items, ``#2``).""" - return call if call in CHECKS else None +def run_text(run: UserSimRun) -> str: + """The run as one text, the part of every call's prompt that its calls share (``render_run``'s two parts).""" + return "\n".join(part for part in render_run(run) if part) -def question_text(check_name: str, items: list[RubricItem]) -> str: - if call_check(check_name) is None: - owner = CHECKS.get(check_name.split("#")[0]) - part = f" (more of the {owner.title} question's, {', '.join(owner.families)})" if owner else "" - lines = [f"# YOUR QUESTION: the task's own rubric items{part}", - "Check each item below against the graded turns. Give \"verdict\" as \"fail\" if any item fails, " - "\"n/a\" if every item is n/a, else \"pass\"."] - else: - c = CHECKS[check_name] - lines = [f"# YOUR QUESTION: {c.title} ({', '.join(c.families)})", c.question, "", "It fails when, at a graded " - "turn:"] - lines += [f"- {f}" for f in c.fails] - lines += ["", "Not a failure:"] + [f"- {f}" for f in c.not_fails] - if c.na: - lines += ["", c.na] - else: - lines += ["", "Answer \"pass\" or \"fail\"; \"n/a\" only if no turn is graded."] - if c.grade: - lines += ["", f"could_change_grade: {c.grade}"] - if items: - lines += ["", "Task rubric items: answer each in \"items\"."] - lines += [f"- {KIND_RULES[k]}" for k in (HARD, GENERAL) if any(i.kind == k for i in items)] - if any(i.na_when for i in items): - lines.append("- An item's \"n/a when\" is the situation in which it never came up: answer \"n/a\" for it " - "then, and only then. \"n/a is not allowed\" means it always applies: answer pass or fail.") - lines.append("") - for i in items: - lines.append(f"- {i.id} ({i.family}: {FAMILIES[i.family].split(':')[0]}; {i.kind}): {i.check}") - if not i.na_allowed: - lines.append(" n/a is not allowed") - elif i.na_when: - lines.append(f" n/a when: {i.na_when}") - lines += ["", "Return only the JSON object."] - return "\n".join(lines) +# ---- what the judge is asked -------------------------------------------------------------------------------------- +def call_check(call: str) -> Optional[str]: + """The check a call asks; None for a call that asks only task items (``TASK_ITEMS``, or a check's or the task's + further items, ``#2``).""" + return call if call in CHECKS else None def plan_calls(checks: list[str], items: list[RubricItem], @@ -635,225 +549,150 @@ def plan_calls(checks: list[str], items: list[RubricItem], return calls -# ---- one judge's answer ------------------------------------------------------------------------------------------- -def parse_answer(text: str, graded: set[int], item_ids: list[str], no_na: tuple[str, ...] | list[str] = ()) -> dict: - """One judge's answer to one call, checked. ``no_na``: the items that may not be n/a. Raises ValueError with - what's wrong.""" - obj = _json_object(text) - if not isinstance(obj, dict): - raise ValueError("the answer is not a JSON object") - verdict = str(obj.get("verdict", "")).strip().lower() - if verdict not in VERDICTS: - raise ValueError(f"verdict {obj.get('verdict')!r} is not one of {list(VERDICTS)}") - change = obj.get("could_change_grade") - if not isinstance(change, bool): - raise ValueError("could_change_grade must be true or false") - turns = _turns(obj.get("turns")) - reason = str(obj.get("reason") or "").strip() - _fail_cites_a_graded_turn("a fail", verdict, turns, graded) - _fail_reason_finds_a_failure("a fail", verdict, reason) - out = {"verdict": verdict, "could_change_grade": change and verdict == "fail", - "reason": reason, "turns": [t for t in turns if t in graded], - "turns_outside_graded": [t for t in turns if t not in graded]} - got = {str(i.get("id")): i for i in obj.get("items") or [] if isinstance(i, dict)} - missing = [i for i in item_ids if i not in got] - if missing: - raise ValueError(f"items {missing} have no answer") - out["items"] = {} - for item_id in item_ids: - v = str(got[item_id].get("verdict", "")).strip().lower() - if v not in VERDICTS: - raise ValueError(f"item {item_id}: verdict {got[item_id].get('verdict')!r}") - if v == "n/a" and item_id in no_na: - raise ValueError(f"item {item_id} may not be n/a: it always applies, so answer pass or fail") - item_turns = _turns(got[item_id].get("turns")) - item_reason = str(got[item_id].get("reason") or "").strip() - _fail_cites_a_graded_turn(f"item {item_id}: a fail", v, item_turns, graded) - _fail_reason_finds_a_failure(f"item {item_id}: a fail", v, item_reason) - out["items"][item_id] = {"verdict": v, "reason": item_reason, "turns": [t for t in item_turns if t in graded]} - return out - +def check_title(name: str) -> str: + c = CHECKS[name] + return f"{c.title} ({', '.join(c.families)})" -def _fail_cites_a_graded_turn(what: str, verdict: str, turns: list[int], graded: set[int]) -> None: - """A fail has to rest on a turn being graded; one that cites none is unsupported, so it's asked again.""" - if verdict == "fail" and not any(t in graded for t in turns): - raise ValueError(f"{what} must cite at least one graded turn; cited {turns}, graded {sorted(graded)}") +def check_question(name: str) -> str: + """A check's question as the judge reads it: what it asks, what fails, what doesn't, when it's n/a and when a fail + could change the assistant's grade.""" + c = CHECKS[name] + lines = [c.question, "", "It fails when, at a graded turn:", *[f"- {f}" for f in c.fails], + "", "Not a failure:", *[f"- {f}" for f in c.not_fails], "", + c.na or "Answer \"pass\" or \"fail\"; \"n/a\" only if no turn is graded."] + if c.grade: + lines += ["", f"could_change_grade: {c.grade}"] + return "\n".join(lines) -ERROR_SHOWN_CHARS = 200 # of an answer's or a call's error, in the retry note and the recorded error -# A reason concluding nothing failed ("No clear delegation failures found"); a backstop to the reason-first prompt. -_NO_FAILURE_FOUND = re.compile( - r"\bno\s+(?:clear\s+)?(?:\w+\s+)?failures?\s+(?:was\s+|were\s+)?(?:found|detected|identified)\b" - r"|\bno\s+clear\s+(?:\w+\s+)?failures?\b|\bnothing\s+fails\b", re.I) +def family_label(family: str) -> str: + return f"{family} ({FAMILIES[family].split(':')[0]})" -def _fail_reason_finds_a_failure(what: str, verdict: str, reason: str) -> None: - """A fail whose own reason concludes no failure was found contradicts itself, so it's asked again. Only the - reason's last sentence is its conclusion: "no clear failure at turn 1; at turn 2 she leaked it" is a real fail.""" - sentences = [x for x in re.split(r"[.;!?]+", reason) if x.strip()] - m = _NO_FAILURE_FOUND.search(sentences[-1]) if verdict == "fail" and sentences else None - if m: - raise ValueError(f"{what} whose reason says {m.group(0)!r} contradicts itself; the verdict must follow the " - "reason") +# ---- one run's criteria and rows ---------------------------------------------------------------------------------- +# The fields a row's verdict is in, as a judge's row (judge_output_format's user_sim_checks) or one computed here has +# them; a stored row is these, after what a reviewer reads about it. +VERDICT_FIELDS = ("verdict", "reason", "turns", "turns_outside_graded", "could_change_grade", "assistant_asked", + "error", "score", "result", "justification") -def _turns(value: Any) -> list[int]: - if not isinstance(value, list): - return [] - out = [] - for v in value: - try: - out.append(int(v)) - except (TypeError, ValueError): +@dataclass +class Plan: + """What one run's judge is asked, and the rows no model grades. + + ``criteria``: user_sim_checks criteria in call order, each check followed by the task items riding with it; what + the judge reads of each (``id``, ``check`` or ``item``, ``family``, ``kind``, ``question``, ``na_when``, + ``graded_turns``, ``weight``). ``batches``: how many of them each call asks, in order. ``unjudged``: the rows no + model grades (auto items, ``na_when: always`` items, line items whose line can't be read). ``about``: by row id, + what a reviewer reads on the row and the judge never sees (an item's ``name``, ``short``, ``asked_with`` ...).""" + criteria: list[dict] = field(default_factory=list) + batches: list[int] = field(default_factory=list) + unjudged: list[dict] = field(default_factory=list) + about: dict[str, dict] = field(default_factory=dict) + order: list[str] = field(default_factory=list) # the row ids: the checks in order, then the task's items + + +def item_weight(item: RubricItem, error_checks: list[str]) -> float: + """1 when a fail of the item can make the run a user sim error, else 0: its ``can_error``, unset as the check + that owns its family (an error check, or not).""" + can = item.can_error if item.can_error is not None else FAMILY_OWNER.get(item.family) in error_checks + return 1.0 if can else 0.0 + + +def computed_row(got: dict, weight: float) -> dict: + """A row no model graded (``got``: verdict, reason or error, turns), scored as a judge's: it lowers the score only + on a fail at a weight other than 0. A computed fail could change the grade; ``assistant_asked`` is None, since no + judge looked.""" + fail = got["verdict"] == "fail" + lowers = fail and weight != 0 + return {**got, "could_change_grade": fail, "assistant_asked": None, + "justification": got.get("reason") or got.get("error") or "", + "score": 0.0 if lowers else 1.0, "result": not lowers} + + +def plan_run(run: UserSimRun, *, checks: list[str], items: list[RubricItem], error_checks: list[str], + lines_unusable: str = "") -> Plan: + """The criteria the judge is asked about ``run`` and the rows settled here: each check in ``checks`` (weight 1 + when it is in ``error_checks``, else 0) with the task items its families own, at most ``MAX_ITEMS_PER_CALL`` to a + call; an auto item counted, an ``na_when: always`` item n/a, a ``line`` item graded by its line's text in + ``run.tagged_lines``, or an error when the run has no such tag or ``lines_unusable`` says why none can be + trusted.""" + graded = sorted(t.n for t in run.turns if t.graded) + resolved, no_line = resolve_lines(list(items), run.tagged_lines, lines_unusable) + calls = plan_calls(list(checks), resolved) + plan = Plan(order=list(checks) + [i.id for i in items]) + for name in checks: + plan.about[name] = {"title": CHECKS[name].title, "families": list(CHECKS[name].families), + "weight": 1.0 if name in error_checks else 0.0} + lines = run.tagged_lines or {} + for item in items: + about = {k: v for k, v in item.to_dict().items() if k != "id"} + if item.line and item.asks_the_model and item.id not in no_line: + about["line_text"] = lines[item.line] + plan.about[item.id] = {**about, "weight": item_weight(item, error_checks)} + for call, its in calls.items(): + size = 0 + if call_check(call): + plan.criteria.append({"id": call, "check": check_title(call), "question": check_question(call), + "graded_turns": graded, "weight": plan.about[call]["weight"]}) + size += 1 + for item in its: + criterion = {"id": item.id, "item": item.id, "family": family_label(item.family), "kind": item.kind, + "question": item.check, "graded_turns": graded, "weight": plan.about[item.id]["weight"]} + if item.na_when: + criterion["na_when"] = item.na_when + plan.criteria.append(criterion) + plan.about[item.id]["asked_with"] = call + size += 1 + if size: + plan.batches.append(size) + for item in items: + if item.id in no_line: + got, asked_with = no_line[item.id], NO_LINE + elif item.kind == AUTO and not item.static_na: + got, asked_with = auto_result(item, run), COMPUTED + elif item.static_na: + got, asked_with = {"verdict": "n/a", "reason": item.static_na_reason, "turns": []}, STATIC_NA + else: continue - return out + got = {k: v for k, v in got.items() if k != "line"} + plan.about[item.id]["asked_with"] = asked_with + plan.unjudged.append({"id": item.id, **computed_row(got, plan.about[item.id]["weight"])}) + return plan + +def undecided_row(criterion: dict, error: str) -> dict: + """A criterion whose judge never gave a usable answer: ``error``, which never lowers the score.""" + return {"id": criterion["id"], "verdict": "error", "error": error, "turns": [], "could_change_grade": False, + "assistant_asked": None, "justification": error, "score": 1.0, "result": True} -def _json_object(text: Any) -> Any: - if isinstance(text, dict): - return text - s = str(text or "").strip().removeprefix("```json").removeprefix("```").removesuffix("```").strip() - try: - return json.loads(s) - except ValueError: - m = re.search(r"\{.*\}", s, re.S) - if not m: - return None - try: - return json.loads(m.group(0)) - except ValueError: - return None + +def finish_rows(rows: list[dict], plan: Plan) -> list[dict]: + """The stored rows, in ``plan.order``: each row's id, what a reviewer reads about it, then its verdict.""" + by_id = {r.get("id"): r for r in rows} + return [{"id": rid, **plan.about[rid], **{k: by_id[rid][k] for k in VERDICT_FIELDS if k in by_id[rid]}} + for rid in plan.order if rid in by_id] # ---- the run's verdict -------------------------------------------------------------------------------------------- -def user_sim_error(checks: dict[str, dict], error_checks: list[str], items: dict[str, dict], - item_can_error: dict[str, bool]) -> dict: - """Whether the run is a user sim error. ``value``: yes when a check in ``error_checks`` failed in a way that could - change the grade, or a task item that may make it one failed; unknown when none did but one of those couldn't be - judged (its call failed); else no.""" - because, unknown = [], [] - for name in error_checks: - r = checks.get(name) - if not r: - continue - if r["verdict"] == "fail" and r.get("could_change_grade"): - because.append(name) - elif r["verdict"] == "error": - unknown.append(name) - for item_id, r in items.items(): - if not item_can_error.get(item_id): - continue - if r["verdict"] == "fail": - because.append(f"item:{item_id}") - elif r["verdict"] == "error": - unknown.append(f"item:{item_id}") +def user_sim_error(rows: list[dict]) -> dict: + """Whether the run is a user sim error, by one rule for checks and task items: ``value`` yes when a row at a weight + other than 0 failed in a way that could change the grade (``because`` names them); unknown when none did but such + a row is undecided (``error``: its judge never gave a usable answer, or it couldn't be counted or read); else no. + A weight-0 row (style, by default) never makes it either.""" + counts = [r for r in rows if float(r.get("weight", 1.0)) != 0] + because = [r["id"] for r in counts if r.get("verdict") == "fail" and r.get("could_change_grade")] + unknown = [r["id"] for r in counts if r.get("verdict") == "error"] return {"value": "yes" if because else "unknown" if unknown else "no", "because": because, "unknown": unknown} NOT_JUDGED = {"value": "no", "because": [], "unknown": []} -# ---- one run ------------------------------------------------------------------------------------------------------ -# complete(model, system, parts, schema) -> {"text": str, "usage": dict | None, "cost": float | None}. ``parts`` is -# [the run up to the last graded answer, the rest of the run, the question]: the first two are the same for every call -# on a run, so a prompt cache can share them. -Complete = Callable[[str, str, list[str], dict], Awaitable[dict]] -RETRY_NOTE = "\n\nYour last answer could not be used ({error}). Return only the JSON object." - - -async def judge_run(run: UserSimRun, *, model: str, checks: list[str], complete: Complete, - items: tuple[RubricItem, ...] | list[RubricItem] = (), - error_checks: tuple[str, ...] | list[str] = DEFAULT_ERROR_CHECKS, - answer_retries: int = 1, concurrency: int = 6, lines_unusable: str = "") -> dict: - """Every check (and task item) on ``run``, each in its own call to ``model``. The first call runs alone so the - rest can read its prompt cache. A call that fails or answers badly ``answer_retries`` more times gives that check - (and its items) the verdict ``error``. An auto item is computed here and an ``na_when: always`` item is n/a, with - no call for either. A ``line`` item is graded by its line's text in ``run.tagged_lines``, or is ``error`` when the - run has no such tag, or when ``lines_unusable`` says why no tag can be trusted.""" - graded = {t.n for t in run.turns if t.graded} - if not graded: - return {"skipped": "no graded turns", "checks": {}, "items": {}, "user_sim_error": dict(NOT_JUDGED)} - resolved, no_line = resolve_lines(list(items), run.tagged_lines, lines_unusable) - calls = plan_calls(list(checks), resolved) - head, tail = render_run(run) - gate = asyncio.Semaphore(max(1, concurrency)) - - async def one(call: str) -> dict: - ids = [i.id for i in calls[call]] - no_na = [i.id for i in calls[call] if not i.na_allowed] - question = question_text(call, calls[call]) - errors: list[str] = [] - usage: list[Any] = [] - cost = 0.0 - for attempt in range(1 + max(0, answer_retries)): - asked = question + (RETRY_NOTE.format(error=errors[-1][:ERROR_SHOWN_CHARS]) if errors else "") - try: - async with gate: - got = await complete(model, INSTRUCTIONS, [head, tail, asked], OUTPUT_SCHEMA) - except Exception as e: # the call itself failed after the caller's own retries - errors.append(f"call failed: {type(e).__name__}: {str(e)[:ERROR_SHOWN_CHARS]}") - continue - usage.append(got.get("usage")) - cost += got.get("cost") or 0.0 - try: - parsed = parse_answer(got.get("text"), graded, ids, no_na) - except ValueError as e: - errors.append(f"answer not usable: {e}") - continue - return {**parsed, "attempts": attempt + 1, "usage": usage, "cost": cost, "errors": errors} - return {"error": errors[-1] if errors else "no answer", "attempts": 1 + max(0, answer_retries), - "usage": usage, "cost": cost, "errors": errors} - - order = list(calls) - answers: dict[str, dict] = {} - if order: - first = await one(order[0]) - answers = dict(zip(order, [first, *await asyncio.gather(*(one(c) for c in order[1:]))])) - - results = {call: {**_check_view(answers[call]), "families": list(CHECKS[call].families)} - for call in order if call_check(call)} - asked_in = {item.id: call for call in order for item in calls[call]} - item_results: dict[str, dict] = {} - lines = run.tagged_lines or {} - for item in items: - if item.id in no_line: - got, call = no_line[item.id], NO_LINE - elif item.kind == AUTO and not item.static_na: - got, call = auto_result(item, run), COMPUTED - elif item.static_na: - got, call = {"verdict": "n/a", "reason": item.static_na_reason, "turns": []}, STATIC_NA - else: - call = asked_in[item.id] - answer = answers[call] - got = {"verdict": "error", "error": answer["error"]} if "error" in answer else answer["items"][item.id] - if item.line and item.asks_the_model and item.id not in no_line: # what was graded, for a reviewer - got = {**got, "line": item.line, "line_text": lines[item.line]} - item_results[item.id] = {**got, "family": item.family, "kind": item.kind, "asked_with": call} - can_error = {i.id: (i.can_error if i.can_error is not None else FAMILY_OWNER.get(i.family) in error_checks) - for i in items} - error = user_sim_error(results, [c for c in error_checks if c in results], item_results, can_error) - return {"model": model, "checks": results, "items": item_results, "user_sim_error": error, - "graded_turns": sorted(graded), "turns": len(run.turns), - "calls": sum(a["attempts"] for a in answers.values()), - "usage": _usage_total([u for a in answers.values() for u in a["usage"]]), - "cost": round(sum(a["cost"] for a in answers.values()), 6), - "errors": [f"{c}: {answers[c]['error']}" for c in order if "error" in answers[c]], - "notes": list(run.notes)} - - -def _check_view(answer: dict) -> dict: - if "error" in answer: - return {"verdict": "error", "error": answer["error"], "attempts": answer["attempts"]} - out = {k: answer[k] for k in ("verdict", "could_change_grade", "reason", "turns")} - if answer.get("turns_outside_graded"): - out["turns_outside_graded"] = answer["turns_outside_graded"] - if answer["attempts"] > 1: - out["attempts"] = answer["attempts"] - return out - - -def _usage_total(usages: list[Any]) -> dict[str, int]: +def usage_total(usages: list[Any]) -> dict[str, int]: + """The token counts of several calls, summed; a cache read reported as ``prompt_tokens_details.cached_tokens`` + counts as one.""" keys = ("prompt_tokens", "completion_tokens", "cache_read_input_tokens", "cache_creation_input_tokens") total = dict.fromkeys(keys, 0) for u in usages: @@ -863,8 +702,8 @@ def _usage_total(usages: list[Any]) -> dict[str, int]: v = u.get(k) if isinstance(v, (int, float)): total[k] += int(v) - cached = (u.get("prompt_tokens_details") or {}).get("cached_tokens") if isinstance( - u.get("prompt_tokens_details"), dict) else None + details = u.get("prompt_tokens_details") + cached = details.get("cached_tokens") if isinstance(details, dict) else None if isinstance(cached, int) and not u.get("cache_read_input_tokens"): total["cache_read_input_tokens"] += cached return total diff --git a/tst/unit/task_step/verifiers/test_user_sim_judge.py b/tst/unit/task_step/verifiers/test_user_sim_judge.py index 2e424493d..850256b57 100644 --- a/tst/unit/task_step/verifiers/test_user_sim_judge.py +++ b/tst/unit/task_step/verifiers/test_user_sim_judge.py @@ -1,23 +1,36 @@ -"""user_sim_judge: grades the user sim of an always_on run or a prompt_agent chat run, one narrow question per check, -by one judge model, with a fake model standing in for it. It records only, unless marking is turned on.""" +"""user_sim_judge: a rubrics_verifier that grades the user sim of an always_on run or a prompt_agent chat run, one call +per check with the task's items riding along, with a fake model standing in for the judge. A grade-changing slip fails +the task (fail_task_when_not_passing, on by default); an undecided judge never does.""" from __future__ import annotations +import asyncio import hashlib import json -import re from dataclasses import replace +import litellm import pytest -from agent_env.task_step.context import TaskStepContext +from agent_env.config.model import ModelConfig +from agent_env.openenv.reward import reward_of +from agent_env.store import LocalSecretStore +from agent_env.task import Task +from agent_env.task_step.context import PromptResponse, TaskStepContext, VerifierNotPassingError from agent_env.task_step.registry import _builtin_registry +from agent_env.task_step.task_step import TaskStepDependency from agent_env.task_step.task_steps import always_on as ao from agent_env.task_step.task_steps.agent import user_sim_versions from agent_env.task_step.task_steps.agent.deploy_user_sim_agent import USER_SIMS_KEY from agent_env.task_step.task_steps.verifiers import user_sim_judge as usj from agent_env.task_step.task_steps.verifiers import user_sim_judge_core as core from agent_env.task_step.task_steps.verifiers.agent_prompt_response_verifier import AgentPromptResponseVerifierTaskStep +from agent_env.task_step.task_steps.verifiers.judge_utils.judge_output_format import ( + JudgeOutputFormat, + diagnose_user_sim_checks_response, + get_judge_output_format_spec, +) +from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep, is_rubrics_verifier from tst.unit.task_step.always_on_harness import Rig from tst.unit.task_step.test_deploy_user_sim_agent import HANNAH, _Deploy from tst.unit.task_step.test_deploy_user_sim_agent import _step as _deploy_step @@ -28,46 +41,90 @@ JUDGE = "fake/judge" OTHER_PERSONA = "0" * 64 # a persona fingerprint no run here has -TASK_ITEMS_Q = "task's own rubric items" +TITLES = {core.check_title(name): name for name in core.CHECK_NAMES} +ALL_ERROR_CHECKS = list(core.DEFAULT_ERROR_CHECKS) -def _pass(**kw) -> str: - return json.dumps({"verdict": "pass", "could_change_grade": False, "reason": "fine", "turns": [], "items": [], - **kw}) +def PASS(**kw) -> dict: + return {"reason": "fine", "turns": [], "verdict": "pass", "could_change_grade": False, "assistant_asked": False, + **kw} -def _fail(turns=(1,), change=True, **kw) -> str: - return json.dumps({"verdict": "fail", "could_change_grade": change, "reason": "it broke the rule", - "turns": list(turns), "items": [], **kw}) +def FAIL(turns=(1,), change=True, **kw) -> dict: + return {"reason": "it broke the rule", "turns": list(turns), "verdict": "fail", "could_change_grade": change, + "assistant_asked": False, **kw} + + +def NA(**kw) -> dict: + return {**PASS(reason="it never came up"), "verdict": "n/a", **kw} + + +def _criteria_of(prompt: str) -> list[dict]: + """The criteria a judge prompt asks about, as the user_sim_checks template lays them out.""" + block = prompt.split("\n# THE CRITERIA\n", 1)[1] + return json.loads(block.split("\n\nReturn only the JSON object.", 1)[0]) class FakeModel: - """Answers each call from ``script[check]``, a text, an exception, or a list of them used in turn; anything + """Stands in for rubrics_verifier's model call. Each criterion is answered from ``script[name]`` (a check's name or + an item's id): a row's fields, a whole answer's text, an exception, or a list of them used in turn; anything unscripted passes. Records every call.""" def __init__(self, script=None): - self.script = dict(script or {}) + self.script = {k: list(v) if isinstance(v, list) else v for k, v in (script or {}).items()} self.calls: list[dict] = [] - - async def __call__(self, model, system, parts, schema): - check = next(c for c in [*core.CHECK_NAMES, TASK_ITEMS_Q] if self._asks(parts[-1], c)) - self.calls.append({"model": model, "check": check, "system": system, "parts": parts, "schema": schema}) - got = self.script.get(check, _pass()) - if isinstance(got, list): - got = got.pop(0) - if isinstance(got, Exception): - raise got - return {"text": got, "usage": {"prompt_tokens": 100, "completion_tokens": 20}, "cost": 0.001} + self.in_flight = 0 + + async def __call__(self, eval_prompt, *, model, context, image_blocks=None, system=None, cached_prefix=None): + criteria = _criteria_of(eval_prompt) + names = [c.get("item") or TITLES[c["check"]] for c in criteria] + self.calls.append({"model": model, "names": names, "criteria": criteria, "prompt": eval_prompt, + "system": system, "cached_prefix": cached_prefix, "alongside": self.in_flight}) + self.in_flight += 1 + try: + await asyncio.sleep(0) # let the other calls start, if they may + rows = [] + for name, criterion in zip(names, criteria): + got = self.script.get(name, PASS()) + if isinstance(got, list): + got = got.pop(0) + if isinstance(got, Exception): + raise got + if isinstance(got, str): + return self._reply(got) + rows.append({"id": criterion["id"], **got}) + return self._reply(json.dumps({"results": rows})) + finally: + self.in_flight -= 1 @staticmethod - def _asks(question: str, check: str) -> bool: - if check in core.CHECKS: - return question.startswith(f"# YOUR QUESTION: {core.CHECKS[check].title} (") - return question.startswith(f"# YOUR QUESTION: the {check}") + def _reply(text: str) -> dict: + return {"response": text, "trajectory_object_url": None, + "usage": {"prompt_tokens": 100, "completion_tokens": 20}, "cost": 0.001} + + def asked(self, name: str) -> list[dict]: + return [c for c in self.calls if name in c["names"]] def run_text(self) -> str: - """The run as the judge's first call read it.""" - return self.calls[0]["parts"][0] + "\n" + self.calls[0]["parts"][1] + """The prompt of the judge's first call, which holds the run.""" + return self.calls[0]["prompt"] + + +def _step(fake: FakeModel, monkeypatch, **kw) -> usj.UserSimJudge: + monkeypatch.setattr(RubricsVerifierTaskStep, "_prompt_llm_judge", fake) + return usj.UserSimJudge("judge_user_sim", None, **{"judge_model": JUDGE, "verifier_id": "usj", **kw}) + + +def _entry(ctx: TaskStepContext) -> dict: + return ctx.metadata["verifications"]["usj"] + + +def _rows(entry: dict) -> dict[str, dict]: + return {r["id"]: r for r in entry["results"]} + + +def _verdicts(entry: dict) -> dict[str, str]: + return {r["id"]: r["verdict"] for r in entry["results"]} def _run(*answers: tuple[str, str], graded=None) -> core.UserSimRun: @@ -77,24 +134,38 @@ def _run(*answers: tuple[str, str], graded=None) -> core.UserSimRun: turns=turns) -async def _judge(run, fake, **kw): - kw.setdefault("model", JUDGE) - kw.setdefault("checks", list(core.CHECK_NAMES)) - return await core.judge_run(run, complete=fake, **kw) +async def _judge(run: core.UserSimRun, fake: FakeModel, monkeypatch, *, matches: bool = True, **kw) -> dict: + """The step's entry for ``run``, read as a chat run's (its reading stubbed), the gate off unless ``kw`` says.""" + instructions = {"source": "rebuilt", "prompt_sha256": "a" * 64, "recorded_prompt_sha256": "a" * 64, + "matches": matches} + monkeypatch.setattr(usj, "_read_run", lambda kind, step_id, record, context: ( + run, {"user_sim": "hannah", "instructions": instructions})) + ctx = TaskStepContext() + ctx.prompt_responses.append(PromptResponse(prompt_id="ask", response="all set", step_id="talk")) + ctx.metadata["user_sim_reads"] = {"talk": [{}]} + ctx.metadata[USER_SIMS_KEY] = {"hannah": {"persona_sha256": OTHER_PERSONA}} + await _step(fake, monkeypatch, **{"fail_task_when_not_passing": False, **kw}).execute(ctx) + return _entry(ctx) + + +def _plan(run: core.UserSimRun, items=(), checks=None, error_checks=None, **kw) -> core.Plan: + return core.plan_run(run, checks=list(core.CHECK_NAMES) if checks is None else checks, items=list(items), + error_checks=ALL_ERROR_CHECKS if error_checks is None else error_checks, **kw) # ---- what the judge is asked -------------------------------------------------------------------------------------- -def test_each_check_gets_one_narrow_question_naming_its_families(): +def test_each_check_is_one_narrow_question_naming_its_families(): for name, check in core.CHECKS.items(): - q = core.question_text(name, []) - assert q.startswith(f"# YOUR QUESTION: {check.title} ({', '.join(check.families)})\n{check.question}") - assert all(f in q for f in check.fails) and "Return only the JSON object." in q + q = core.check_question(name) + assert q.startswith(check.question) and all(f in q for f in check.fails + check.not_fails) + assert core.check_title(name) == f"{check.title} ({', '.join(check.families)})" + assert "could_change_grade: Usually no." in core.check_question("style") assert sorted(core.FAMILY_OWNER) == ["A1", "A2", "A3", "A4", "F1", "F2", "F3", "F4", "G1", "G2", "G3", "G4", "G5"] def test_self_contradiction_is_asked_under_made_up_not_guidance(): line = "It contradicted something it said earlier in the run." - assert line in core.question_text("made_up", []) and "contradict" not in core.question_text("guidance", []) + assert line in core.check_question("made_up") and "contradict" not in core.check_question("guidance") def test_the_run_is_split_before_the_last_graded_answer_so_repeats_share_a_cached_prefix(): @@ -105,6 +176,7 @@ def test_the_run_is_split_before_the_last_graded_answer_so_repeats_share_a_cache assert "## Turn 3 · GRADED" in head and "question 3?" in head and "no seafood pls" not in head assert tail == '### Its answer\noutcome: reply\nmessage: "no seafood pls"' assert "message:" not in head.split("## Turn 2")[1].split("## Turn 3")[0] # a no_reply shows no empty message + assert core.run_text(run) == head + "\n" + tail def test_task_items_go_with_the_check_that_owns_their_family_and_the_rest_get_a_call_of_their_own(): @@ -114,8 +186,12 @@ def test_task_items_go_with_the_check_that_owns_their_family_and_the_rest_get_a_ calls = core.plan_calls(["oversharing", "made_up"], items) assert [i.id for i in calls["oversharing"]] == ["leak"] and calls["made_up"] == [] assert [i.id for i in calls[core.TASK_ITEMS]] == ["ends", "tone"] # style isn't running, so "tone" goes there too - assert "- leak (G1: Leaking and privacy; general): never mentions the engagement" in core.question_text( - "oversharing", calls["oversharing"]) + plan = _plan(_run(("reply", "x")), items, checks=["oversharing", "made_up"]) + assert [c["id"] for c in plan.criteria] == ["oversharing", "leak", "made_up", "ends", "tone"] + assert plan.batches == [2, 1, 2] + assert plan.criteria[1] == {"id": "leak", "item": "leak", "family": "G1 (Leaking and privacy)", "kind": "general", + "question": "never mentions the engagement", "graded_turns": [1], "weight": 1.0} + assert plan.about["ends"]["asked_with"] == plan.about["tone"]["asked_with"] == core.TASK_ITEMS def test_rubric_items_are_checked_when_the_task_loads(): @@ -125,193 +201,207 @@ def test_rubric_items_are_checked_when_the_task_loads(): assert any("unknown fields ['weight']" in p for p in problems) and any("needs an id" in p for p in problems) -# ---- reading an answer -------------------------------------------------------------------------------------------- +# ---- reading an answer (the user_sim_checks format, on this step's criteria) ---------------------------------------- def test_an_answer_is_checked_and_its_turns_kept_to_the_graded_ones(): - got = core.parse_answer(_fail(turns=[2, 3, "x"]), graded={3}, item_ids=[]) - assert (got["verdict"], got["turns"], got["turns_outside_graded"]) == ("fail", [3], [2]) - assert core.parse_answer(_pass(could_change_grade=True), {1}, [])["could_change_grade"] is False - for bad, why in ((json.dumps({"verdict": "maybe"}), "verdict"), ("no json here", "not a JSON object"), - (_pass(), "items \\['i1'\\] have no answer")): - with pytest.raises(ValueError, match=why): - core.parse_answer(bad, {1}, ["i1"] if "items" in why else []) + criteria = _plan(_run(("reply", "a"), ("reply", "b"), ("reply", "c"), graded={3}), checks=["made_up"]).criteria + rows, problem = diagnose_user_sim_checks_response(json.dumps({"results": [ + {"id": "made_up", **FAIL(turns=[2, 3, "x"])}]}), criteria) + assert problem is None and (rows[0]["verdict"], rows[0]["turns"], rows[0]["turns_outside_graded"]) == ( + "fail", [3], [2]) + rows, _ = diagnose_user_sim_checks_response(json.dumps({"results": [ + {"id": "made_up", **PASS(could_change_grade=True)}]}), criteria) + assert rows[0]["could_change_grade"] is False # a pass can't change the grade def test_a_fail_that_cites_no_graded_turn_is_not_usable_and_a_pass_needs_none(): - with pytest.raises(ValueError, match=r"a fail must cite at least one graded turn; cited \[99\], graded \[1\]"): - core.parse_answer(_fail(turns=[99]), {1}, []) - with pytest.raises(ValueError, match=r"a fail must cite at least one graded turn; cited \[\], graded \[1\]"): - core.parse_answer(_fail(turns=[]), {1}, []) - item = {"id": "i1", "verdict": "fail", "reason": "rated it 5", "turns": [99]} - with pytest.raises(ValueError, match=r"item i1: a fail must cite at least one graded turn; cited \[99\]"): - core.parse_answer(_fail(items=[item]), {1}, ["i1"]) - na = {"verdict": "n/a", "could_change_grade": False, "reason": "-", "turns": [99], "items": []} - for ok in (_pass(turns=[99]), json.dumps(na), _pass(items=[{**item, "verdict": "pass"}])): - assert core.parse_answer(ok, {1}, ["i1"] if "i1" in ok else [])["turns"] == [] - got = core.parse_answer(_fail(turns=[1, 99], items=[{**item, "turns": [1, 99]}]), {1}, ["i1"]) - assert (got["turns"], got["turns_outside_graded"], got["items"]["i1"]["turns"]) == ([1], [99], [1]) - - -def test_the_judge_writes_its_reason_before_its_verdict_within_the_word_limit(): - order = ["reason", "turns", "verdict", "could_change_grade", "items"] - assert list(core.OUTPUT_SCHEMA["properties"]) == order == core.OUTPUT_SCHEMA["required"] - assert list(core.OUTPUT_SCHEMA["properties"]["items"]["items"]["properties"]) == ["id", "reason", "turns", "verdict"] - shape = core.INSTRUCTIONS.split("Return only this JSON object")[1] - assert shape.index('"reason"') < shape.index('"turns"') < shape.index('"verdict"') < shape.index( - '"could_change_grade"') - item = shape.split('"items": [')[1] - assert item.index('"reason"') < item.index('"turns"') < item.index('"verdict"') - assert "Reason first and decide last." in core.INSTRUCTIONS and "at most 40 words" in core.INSTRUCTIONS - # The parser doesn't care which order the keys come in. - flipped = json.dumps({"items": [], "could_change_grade": True, "verdict": "fail", "turns": [1], "reason": "leaked"}) - assert core.parse_answer(flipped, {1}, [])["verdict"] == "fail" - - -# The endings of the two contradictory reasons seen in real runs, and a plain one. -SAYS_NO_FAILURE = ("Hannah flagged it anyway which is fine. No clear failure found.", - "This is not a delegation failure by Linda. No clear delegation failures found.", - "Every answer fits her rules; nothing fails.") + criteria = _plan(_run(("reply", "a")), checks=["made_up"]).criteria + for turns in ([99], []): + rows, problem = diagnose_user_sim_checks_response( + json.dumps({"results": [{"id": "made_up", **FAIL(turns=turns)}]}), criteria) + assert rows is None and problem.row_problems == [ + f"made_up: a fail must cite at least one graded turn; cited {turns}, graded [1]"] + rows, problem = diagnose_user_sim_checks_response( + json.dumps({"results": [{"id": "made_up", **PASS(turns=[99])}]}), criteria) + assert problem is None and rows[0]["turns"] == [] + + +def test_a_fail_that_clears_one_turn_and_fails_another_is_kept(): + criteria = _plan(_run(("reply", "a")), checks=["oversharing"]).criteria + reason = "No clear failure at turn 1; at turn 2 she revealed the private password." + rows, problem = diagnose_user_sim_checks_response( + json.dumps({"results": [{"id": "oversharing", **FAIL(reason=reason)}]}), criteria) + assert problem is None and rows[0]["verdict"] == "fail" and rows[0]["could_change_grade"] is True -def test_a_fail_whose_reason_says_nothing_failed_is_not_usable(): - for reason in SAYS_NO_FAILURE: - with pytest.raises(ValueError, match="a fail whose reason says .* contradicts itself"): - core.parse_answer(_fail(reason=reason), {1}, []) - item = {"id": "i1", "verdict": "fail", "reason": reason, "turns": [1]} - with pytest.raises(ValueError, match="item i1: a fail whose reason says"): - core.parse_answer(_pass(items=[item]), {1}, ["i1"]) - # A pass may say it; so may an item that passes. - assert core.parse_answer(_pass(reason=reason, items=[{**item, "verdict": "pass"}]), {1}, ["i1"])["reason"] - # An ordinary fail, even one that clears some turns, is kept. - ok = "Turn 1 is fine, not a failure. At turn 2 she volunteered the shellfish allergy unasked; M3 says only when asked." - assert core.parse_answer(_fail(turns=[2], reason=ok), {1, 2}, [])["reason"] == ok - - -# ---- one judge ---------------------------------------------------------------------------------------------------- @pytest.mark.asyncio -async def test_each_check_s_verdict_is_recorded_with_its_reason_and_turns(): - na = json.dumps({"verdict": "n/a", "could_change_grade": False, "reason": "no rules", "turns": [], "items": []}) - fake = FakeModel({"made_up": _fail(), "style": _fail(change=False), "delegation": na}) - result = await _judge(_run(("reply", "dad's allergic to shellfish fyi")), fake) - assert {k: v["verdict"] for k, v in result["checks"].items()} == { +async def test_the_judge_writes_its_reason_before_its_verdict_within_the_word_limit(monkeypatch): + fake = FakeModel() + await _judge(_run(("reply", "x")), fake, monkeypatch, checks=["rogue"]) + prompt = fake.calls[0]["prompt"] + assert "Reason first and decide last." in prompt and "at most 40 words" in prompt + assert prompt.index("# THE USER SIM'S INSTRUCTIONS") < prompt.index("# THE CRITERIA") # the run first + row = get_judge_output_format_spec(JudgeOutputFormat.USER_SIM_CHECKS).output_format["schema"]["properties"][ + "results"]["items"] + assert list(row["properties"]) == ["id", "reason", "turns", "verdict", "could_change_grade", "assistant_asked"] + + +# ---- one run, through the step ---------------------------------------------------------------------------------- +@pytest.mark.asyncio +async def test_each_check_s_verdict_is_recorded_with_its_reason_and_turns(monkeypatch): + fake = FakeModel({"made_up": FAIL(), "style": FAIL(change=False), "delegation": NA(reason="no rules")}) + entry = await _judge(_run(("reply", "dad's allergic to shellfish fyi")), fake, monkeypatch) + assert _verdicts(entry) == { "guidance": "pass", "oversharing": "pass", "delegation": "n/a", "style": "fail", "rogue": "pass", "made_up": "fail", "ending": "pass"} - assert result["checks"]["made_up"] == {"verdict": "fail", "could_change_grade": True, "reason": "it broke the rule", - "turns": [1], "families": ["A2"]} - assert result["checks"]["delegation"]["reason"] == "no rules" - # Style failed, but it isn't an error check; made_up failed in a way that could change the grade. - assert result["user_sim_error"] == {"value": "yes", "because": ["made_up"], "unknown": []} - assert len(fake.calls) == 7 == result["calls"] and {c["model"] for c in fake.calls} == {JUDGE} - assert result["usage"]["prompt_tokens"] == 700 and result["cost"] == pytest.approx(0.007) + assert _rows(entry)["made_up"] == { + "id": "made_up", "title": "Made-up facts", "families": ["A2"], "weight": 1.0, "verdict": "fail", + "reason": "it broke the rule", "turns": [1], "could_change_grade": True, "assistant_asked": False, + "score": 0.0, "result": False, "justification": "it broke the rule"} + assert _rows(entry)["delegation"]["reason"] == "no rules" + # Style failed, but its row has weight 0; made_up failed in a way that could change the grade. + assert _rows(entry)["style"]["weight"] == 0.0 and _rows(entry)["style"]["result"] is True + assert entry["user_sim_error"] == {"value": "yes", "because": ["made_up"], "unknown": [], "fails_task": False, + "not_failing": usj.NOT_FAILING_OFF} + assert entry["format"] == "user_sim_checks" and entry["score"] == 0.0 + assert len(fake.calls) == 7 == entry["calls"] and {c["model"] for c in fake.calls} == {JUDGE} + assert entry["usage"]["prompt_tokens"] == 700 and entry["cost"] == pytest.approx(0.007) @pytest.mark.asyncio -async def test_each_call_shares_the_run_and_asks_one_question_and_the_first_call_runs_alone(): +async def test_each_call_shares_the_run_and_asks_one_check_and_the_first_call_runs_alone(monkeypatch): fake = FakeModel() - await _judge(_run(("reply", "ok")), fake) - assert {c["parts"][0] for c in fake.calls} == {fake.calls[0]["parts"][0]} # one shared, cacheable prefix - assert all(c["system"] == core.INSTRUCTIONS and c["schema"] == core.OUTPUT_SCHEMA for c in fake.calls) - assert [c["check"] for c in fake.calls] == list(core.CHECK_NAMES) # the first alone, then the rest in order + run = _run(("reply", "ok")) + entry = await _judge(run, fake, monkeypatch) + prefixes = {c["cached_prefix"] for c in fake.calls} + assert len(prefixes) == 1 and next(iter(prefixes)).endswith(core.run_text(run)) # one shared, cacheable prefix + assert all(c["system"] == usj.SYSTEM and "never the assistant" in c["system"] for c in fake.calls) + assert sorted(n for c in fake.calls for n in c["names"]) == sorted(core.CHECK_NAMES) + assert fake.calls[0]["names"] == ["guidance"] and all(len(c["names"]) == 1 for c in fake.calls) + assert fake.calls[0]["alongside"] == 0 and max(c["alongside"] for c in fake.calls[1:]) > 0 # alone, then the rest + assert entry["score"] == 1.0 and entry["user_sim_error"]["value"] == "no" @pytest.mark.asyncio -async def test_a_failure_that_can_t_change_the_grade_doesn_t_make_a_user_sim_error(): - result = await _judge(_run(("reply", "Here's a summary of what I booked:")), FakeModel({"rogue": _fail(change=False)})) - assert result["checks"]["rogue"]["verdict"] == "fail" and result["user_sim_error"]["value"] == "no" +async def test_a_failure_that_can_t_change_the_grade_doesn_t_make_a_user_sim_error(monkeypatch): + entry = await _judge(_run(("reply", "Here's a summary of what I booked:")), + FakeModel({"rogue": FAIL(change=False)}), monkeypatch) + assert _verdicts(entry)["rogue"] == "fail" and entry["user_sim_error"]["value"] == "no" and entry["score"] == 1.0 @pytest.mark.asyncio -async def test_a_bad_answer_is_asked_again_once_and_a_check_whose_calls_keep_failing_is_an_error(): - fake = FakeModel({"guidance": ["not json", _fail()], "rogue": [RuntimeError("503"), RuntimeError("503")]}) - result = await _judge(_run(("reply", "you pick")), fake, checks=["guidance", "rogue"]) - assert result["checks"]["guidance"]["verdict"] == "fail" and result["checks"]["guidance"]["attempts"] == 2 - retry = [c for c in fake.calls if c["check"] == "guidance"][1]["parts"][-1] - assert "Your last answer could not be used (answer not usable: the answer is not a JSON object)" in retry - assert result["checks"]["rogue"] == {"verdict": "error", "error": "call failed: RuntimeError: 503", - "attempts": 2, "families": ["G3", "G5"]} - assert result["errors"] == ["rogue: call failed: RuntimeError: 503"] - assert result["user_sim_error"] == {"value": "yes", "because": ["guidance"], "unknown": ["rogue"]} - alone = await _judge(_run(("reply", "x")), FakeModel({"rogue": [RuntimeError("503")] * 2}), checks=["rogue"]) - assert alone["user_sim_error"] == {"value": "unknown", "because": [], "unknown": ["rogue"]} +async def test_a_bad_answer_is_asked_again_and_a_check_whose_call_fails_is_undecided(monkeypatch): + fake = FakeModel({"guidance": ["not json", FAIL()], "rogue": [RuntimeError("503")]}) + entry = await _judge(_run(("reply", "you pick")), fake, monkeypatch, checks=["guidance", "rogue"]) + assert _verdicts(entry) == {"guidance": "fail", "rogue": "error"} and entry["judge_output_retries"] == 1 + retry = fake.asked("guidance")[1]["prompt"] + assert "## Correction Required" in retry and "the answer is not a JSON object" in retry + assert _rows(entry)["rogue"]["error"] == "no usable answer after 4 tries: RuntimeError: 503" + assert (_rows(entry)["rogue"]["score"], _rows(entry)["rogue"]["assistant_asked"]) == (1.0, None) + assert entry["user_sim_error"]["value"] == "yes" + assert (entry["user_sim_error"]["because"], entry["user_sim_error"]["unknown"]) == (["guidance"], ["rogue"]) + alone = await _judge(_run(("reply", "x")), FakeModel({"rogue": [RuntimeError("503")]}), monkeypatch, + checks=["rogue"]) + assert (alone["user_sim_error"]["value"], alone["user_sim_error"]["unknown"]) == ("unknown", ["rogue"]) + assert alone["flag"] == "user sim judge undecided" and alone["score"] == 1.0 @pytest.mark.asyncio -async def test_a_fail_citing_only_ungraded_turns_is_asked_again_and_twice_is_an_error(): - fake = FakeModel({"made_up": [_fail(turns=[99]), _fail(turns=[1])]}) - result = await _judge(_run(("reply", "dad's allergic to shellfish")), fake, checks=["made_up"]) - assert result["checks"]["made_up"]["verdict"] == "fail" and result["checks"]["made_up"]["turns"] == [1] - assert result["checks"]["made_up"]["attempts"] == 2 and result["user_sim_error"]["value"] == "yes" - assert "a fail must cite at least one graded turn; cited [99], graded [1]" in fake.calls[1]["parts"][-1] - twice = await _judge(_run(("reply", "x")), FakeModel({"made_up": [_fail(turns=[99])] * 2}), checks=["made_up"]) - assert twice["checks"]["made_up"]["verdict"] == "error" and "cited [99]" in twice["checks"]["made_up"]["error"] - assert twice["user_sim_error"] == {"value": "unknown", "because": [], "unknown": ["made_up"]} +async def test_a_broken_answer_is_asked_again_up_to_3_times_then_the_run_is_flagged_and_the_task_not_failed( + monkeypatch): + assert usj.UserSimJudge.DEFAULT_MAX_RETRIES == 4 # the first ask and 3 corrections + fake = FakeModel({"made_up": ["not json"] * 5}) + entry = await _judge(_run(("reply", "x")), fake, monkeypatch, checks=["made_up"], fail_task_when_not_passing=True) + assert len(fake.asked("made_up")) == 4 + assert all("## Correction Required" in c["prompt"] for c in fake.asked("made_up")[1:]) + assert _verdicts(entry) == {"made_up": "error"} and _rows(entry)["made_up"]["error"].startswith( + "no usable answer after 4 tries: ValueError: ") + assert entry["user_sim_error"] == {"value": "unknown", "because": [], "unknown": ["made_up"], "fails_task": False} + assert entry["flag"] == "user sim judge undecided" # and execute didn't raise @pytest.mark.asyncio -async def test_a_fail_whose_reason_says_no_clear_failure_found_is_asked_again(): - contradicts = _fail(reason=SAYS_NO_FAILURE[0]) - fake = FakeModel({"guidance": [contradicts, _pass(reason="No clear failure found.")], "oversharing": _fail()}) - result = await _judge(_run(("reply", "dad's allergic to shellfish fyi")), fake, checks=["guidance", "oversharing"]) - assert result["checks"]["guidance"] == {"verdict": "pass", "could_change_grade": False, - "reason": "No clear failure found.", "turns": [], "attempts": 2, - "families": ["A1", "A3"]} - retry = [c for c in fake.calls if c["check"] == "guidance"][1]["parts"][-1] - assert "a fail whose reason says 'No clear failure found' contradicts itself" in retry - # The ordinary fail is answered once and alone makes the run a user sim error. - assert "attempts" not in result["checks"]["oversharing"] and result["checks"]["oversharing"]["verdict"] == "fail" - assert result["user_sim_error"] == {"value": "yes", "because": ["oversharing"], "unknown": []} - twice = await _judge(_run(("reply", "x")), FakeModel({"delegation": [contradicts] * 2}), checks=["delegation"]) - assert twice["checks"]["delegation"]["verdict"] == "error" - assert twice["user_sim_error"] == {"value": "unknown", "because": [], "unknown": ["delegation"]} +async def test_a_fail_citing_only_ungraded_turns_is_asked_again(monkeypatch): + fake = FakeModel({"made_up": [FAIL(turns=[99]), FAIL(turns=[1])]}) + entry = await _judge(_run(("reply", "dad's allergic to shellfish")), fake, monkeypatch, checks=["made_up"]) + assert _rows(entry)["made_up"]["verdict"] == "fail" and _rows(entry)["made_up"]["turns"] == [1] + assert entry["judge_output_retries"] == 1 and entry["user_sim_error"]["value"] == "yes" + assert "a fail must cite at least one graded turn; cited [99], graded [1]" in fake.calls[1]["prompt"] + never = await _judge(_run(("reply", "x")), FakeModel({"made_up": [FAIL(turns=[99])] * 4}), monkeypatch, + checks=["made_up"]) + assert _verdicts(never) == {"made_up": "error"} and "cited [99]" in _rows(never)["made_up"]["error"] + assert never["user_sim_error"]["value"] == "unknown" + + +SAYS_NO_FAILURE = ("Hannah flagged it anyway which is fine. No clear failure found.", + "This is not a delegation failure by Linda. No clear delegation failures found.", + "Every answer fits her rules; nothing fails.") @pytest.mark.asyncio -async def test_a_task_item_fail_citing_no_graded_turn_is_asked_again_and_twice_is_an_error(): - items = [core.RubricItem("rating", "A5", "rates the trip 2 or lower", can_error=True)] +async def test_a_fail_whose_reason_says_no_clear_failure_found_is_asked_again(monkeypatch): + fake = FakeModel({"guidance": [FAIL(reason=SAYS_NO_FAILURE[0]), PASS(reason="No clear failure found.")], + "oversharing": FAIL()}) + entry = await _judge(_run(("reply", "dad's allergic to shellfish fyi")), fake, monkeypatch, + checks=["guidance", "oversharing"]) + assert (_rows(entry)["guidance"]["verdict"], _rows(entry)["guidance"]["reason"]) == ( + "pass", "No clear failure found.") + retry = fake.asked("guidance")[1]["prompt"] + assert "a fail whose reason says 'No clear failure found' contradicts itself" in retry + # The ordinary fail is answered once and alone makes the run a user sim error. + assert len(fake.asked("oversharing")) == 1 and entry["user_sim_error"]["because"] == ["oversharing"] + for reason in SAYS_NO_FAILURE: + never = await _judge(_run(("reply", "x")), FakeModel({"delegation": [FAIL(reason=reason)] * 4}), monkeypatch, + checks=["delegation"]) + assert _verdicts(never) == {"delegation": "error"} and never["user_sim_error"]["value"] == "unknown" - def answer(turns): - return json.dumps({"verdict": "fail", "could_change_grade": False, "reason": "rated it 5", "turns": [1], - "items": [{"id": "rating", "verdict": "fail", "reason": "rated it 5", "turns": turns}]}) - fake = FakeModel({TASK_ITEMS_Q: [answer([99]), answer([1])]}) - result = await _judge(_run(("done", "5 stars")), fake, checks=[], items=items) - assert result["items"]["rating"]["verdict"] == "fail" and result["items"]["rating"]["turns"] == [1] - assert result["user_sim_error"]["because"] == ["item:rating"] and len(fake.calls) == 2 - twice = await _judge(_run(("done", "5 stars")), FakeModel({TASK_ITEMS_Q: [answer([])] * 2}), checks=[], - items=items) - assert twice["items"]["rating"]["verdict"] == "error" and "item rating: a fail" in twice["items"]["rating"]["error"] - assert twice["user_sim_error"] == {"value": "unknown", "because": [], "unknown": ["item:rating"]} +@pytest.mark.asyncio +async def test_a_task_item_fail_citing_no_graded_turn_is_asked_again(monkeypatch): + items = [{"id": "rating", "family": "A5", "check": "rates the trip 2 or lower", "can_error": True}] + fake = FakeModel({"rating": [FAIL(turns=[99]), FAIL(turns=[1])]}) + entry = await _judge(_run(("done", "5 stars")), fake, monkeypatch, checks=[], rubric_items=items) + assert _rows(entry)["rating"]["verdict"] == "fail" and _rows(entry)["rating"]["turns"] == [1] + assert entry["user_sim_error"]["because"] == ["rating"] and len(fake.calls) == 2 + never = await _judge(_run(("done", "5 stars")), FakeModel({"rating": [FAIL(turns=[])] * 4}), monkeypatch, + checks=[], rubric_items=items) + assert _verdicts(never) == {"rating": "error"} and "a fail must cite" in _rows(never)["rating"]["error"] + assert (never["user_sim_error"]["value"], never["user_sim_error"]["unknown"]) == ("unknown", ["rating"]) @pytest.mark.asyncio -async def test_task_items_get_their_own_verdicts_and_one_that_can_error_makes_a_user_sim_error(): - items = [core.RubricItem("leak", "G1", "never hints at the engagement"), - core.RubricItem("rating", "A5", "rates the trip 2 or lower", can_error=True)] - item_fail = json.dumps({"verdict": "fail", "could_change_grade": False, "reason": "rated it 5", - "turns": [1], "items": [{"id": "rating", "verdict": "fail", "reason": "rated it 5", - "turns": [1]}]}) - leak_pass = _pass(items=[{"id": "leak", "verdict": "pass", "reason": "-", "turns": []}]) - fake = FakeModel({TASK_ITEMS_Q: item_fail, "oversharing": leak_pass}) - result = await _judge(_run(("done", "thx bye")), fake, items=items) - assert result["items"]["leak"] == {"verdict": "pass", "reason": "-", "turns": [], "family": "G1", - "kind": "general", "asked_with": "oversharing"} - assert result["items"]["rating"] == {"verdict": "fail", "reason": "rated it 5", "turns": [1], "family": "A5", - "kind": "general", "asked_with": "task_items"} - assert "task_items" not in result["checks"] and result["user_sim_error"]["because"] == ["item:rating"] - assert len(fake.calls) == 8 +async def test_task_items_get_their_own_rows_and_one_rule_says_which_make_a_user_sim_error(monkeypatch): + items = [{"id": "leak", "family": "G1", "check": "never hints at the engagement"}, + {"id": "rating", "family": "A5", "check": "rates the trip 2 or lower", "can_error": True}, + {"id": "tone", "family": "A5", "check": "stays upbeat"}] + fake = FakeModel({"rating": FAIL(reason="rated it 5"), "leak": PASS(reason="-"), "tone": FAIL()}) + entry = await _judge(_run(("done", "thx bye")), fake, monkeypatch, rubric_items=items) + assert _rows(entry)["leak"] == { + "id": "leak", "family": "G1", "kind": "general", "check": "never hints at the engagement", + "asked_with": "oversharing", "weight": 1.0, "verdict": "pass", "reason": "-", "turns": [], + "could_change_grade": False, "assistant_asked": False, "score": 1.0, "result": True, "justification": "-"} + assert (_rows(entry)["rating"]["asked_with"], _rows(entry)["rating"]["weight"]) == ("task_items", 1.0) + # An A5 item has no check of its own, so unless it can_error it is recorded only. + assert _rows(entry)["tone"]["verdict"] == "fail" and _rows(entry)["tone"]["weight"] == 0.0 + assert entry["user_sim_error"]["because"] == ["rating"] and len(fake.calls) == 8 + assert [r["id"] for r in entry["results"]] == [*core.CHECK_NAMES, "leak", "rating", "tone"] + # The rule is the checks' rule: an item's fail that couldn't change the grade is no user sim error. + mild = await _judge(_run(("done", "thx bye")), FakeModel({"rating": FAIL(change=False)}), monkeypatch, + checks=[], rubric_items=items[1:2]) + assert _verdicts(mild) == {"rating": "fail"} and mild["user_sim_error"]["value"] == "no" @pytest.mark.asyncio -async def test_the_ending_check_asks_when_it_signed_off_and_a_grade_changing_fail_makes_a_user_sim_error(): - q = core.question_text("ending", []) +async def test_the_ending_check_asks_when_it_signed_off_and_a_grade_changing_fail_makes_a_user_sim_error(monkeypatch): + q = core.check_question("ending") assert "while the assistant was waiting on its answer" in q and "WHEN YOU'RE DONE" in q assert "A short sign-off when its reply limit is reached." in q and "never said \"done\"" in q assert "ending" in core.DEFAULT_ERROR_CHECKS - na = json.dumps({"verdict": "n/a", "could_change_grade": False, "reason": "never said done", "turns": [], - "items": []}) - for answer, verdict, error in ((_fail(turns=[1]), "fail", "yes"), (_pass(), "pass", "no"), (na, "n/a", "no")): - result = await _judge(_run(("done", "ok thx bye")), FakeModel({"ending": answer}), checks=["ending"]) - assert result["checks"]["ending"]["verdict"] == verdict and result["checks"]["ending"]["families"] == ["G4"] - assert result["user_sim_error"]["value"] == error + for answer, verdict, error in ((FAIL(turns=[1]), "fail", "yes"), (PASS(), "pass", "no"), + (NA(reason="never said done"), "n/a", "no")): + entry = await _judge(_run(("done", "ok thx bye")), FakeModel({"ending": answer}), monkeypatch, + checks=["ending"]) + assert _verdicts(entry) == {"ending": verdict} and _rows(entry)["ending"]["families"] == ["G4"] + assert entry["user_sim_error"]["value"] == error def test_the_judge_is_told_when_the_run_ended_at_the_user_sim_s_reply_limit(): @@ -322,10 +412,62 @@ def test_the_judge_is_told_when_the_run_ended_at_the_user_sim_s_reply_limit(): @pytest.mark.asyncio -async def test_a_run_with_no_graded_turn_is_not_sent_to_the_judge(): +async def test_a_run_with_no_graded_turn_is_not_sent_to_the_judge(monkeypatch): fake = FakeModel() - result = await _judge(_run(("reply", "ok"), graded=set()), fake) - assert result["skipped"] == "no graded turns" and fake.calls == [] + entry = await _judge(_run(("reply", "ok"), graded=set()), fake, monkeypatch, fail_task_when_not_passing=True) + assert entry["skipped"] == "no graded turns" and fake.calls == [] + assert (entry["results"], entry["score"], entry["user_sim_error"]["value"]) == ([], None, "no") + + +# ---- the gate ------------------------------------------------------------------------------------------------------- +@pytest.mark.asyncio +async def test_a_grade_changing_slip_with_matching_instructions_fails_the_task_naming_the_rows(monkeypatch): + with pytest.raises(VerifierNotPassingError, match=r"user_sim_judge 'judge_user_sim': the user sim slipped .* " + r"\(guidance, a1\)"): + await _judge(_run(("reply", "you pick")), FakeModel({"guidance": FAIL(), "a1": FAIL()}), monkeypatch, + fail_task_when_not_passing=True, + rubric_items=[{"id": "a1", "family": "A1", "check": "gives the date"}]) + + +@pytest.mark.asyncio +async def test_a_slip_isn_t_held_against_the_run_when_the_instructions_don_t_match(monkeypatch): + entry = await _judge(_run(("reply", "you pick")), FakeModel({"guidance": FAIL()}), monkeypatch, matches=False, + fail_task_when_not_passing=True) + assert entry["user_sim_error"] == {"value": "yes", "because": ["guidance"], "unknown": [], "fails_task": False, + "not_failing": usj.NOT_FAILING_UNCONFIRMED} + + +@pytest.mark.asyncio +async def test_style_never_gates(monkeypatch): + entry = await _judge(_run(("reply", "Dear assistant, I hope this finds you well.")), + FakeModel({"style": FAIL(change=True)}), monkeypatch, fail_task_when_not_passing=True) + assert _rows(entry)["style"]["verdict"] == "fail" and _rows(entry)["style"]["could_change_grade"] is True + assert entry["user_sim_error"]["value"] == "no" and entry["score"] == 1.0 + [problem] = usj.UserSimJudge("s", None, error_checks=["guidance", "style"]).config_problems() + assert problem == "error_checks can't name style: its rows are recorded only (weight 0), never a user sim error" + + +@pytest.mark.asyncio +async def test_assistant_asked_is_recorded_on_each_row_and_never_changes_the_gate(monkeypatch): + fake = FakeModel({"rogue": FAIL(assistant_asked=True, reason="drafted the email to Ben the assistant asked for"), + "guidance": PASS(assistant_asked=True)}) + with pytest.raises(VerifierNotPassingError, match=r"\(rogue\)"): + await _judge(_run(("reply", "Hi Ben, ...")), fake, monkeypatch, fail_task_when_not_passing=True) + entry = await _judge(_run(("reply", "Hi Ben, ...")), FakeModel(fake.script | { + "rogue": FAIL(assistant_asked=True)}), monkeypatch) + assert {r["id"]: r["assistant_asked"] for r in entry["results"]} == { + **dict.fromkeys(core.CHECK_NAMES, False), "guidance": True, "rogue": True} + assert entry["user_sim_error"]["because"] == ["rogue"] + + +def test_its_score_checks_the_run_so_the_task_s_grader_is_its_reward(): + judge = usj.UserSimJudge("judge", None, run_step_id="week", depends_on=[TaskStepDependency("week")]) + assert usj.UserSimJudge.can_be_reward is False and is_rubrics_verifier(judge) + task = Task("t", None, [*_user_sim_steps(), judge, _grader(depends_on=[TaskStepDependency("judge")])]) + pick, why = reward_of(task, None) + assert why is None and pick({"grade": 0.75, "judge": 1.0}) == 0.75 + _, why = reward_of(task, "judge") + assert "records a score that checks the run and can't be its reward" in why # ---- the step on an always_on run --------------------------------------------------------------------------------- @@ -359,24 +501,15 @@ def agent(n, texts, rig): return rig, ctx, prompt -def _step(fake: FakeModel, monkeypatch, **kw) -> usj.UserSimJudgeTaskStep: - step = usj.UserSimJudgeTaskStep("judge_user_sim", None, **{"judge_model": JUDGE, "verifier_id": "usj", **kw}) - monkeypatch.setattr(step, "_complete", lambda context: fake) - return step - - -def _entry(ctx: TaskStepContext) -> dict: - return ctx.metadata["user_sim_judge"]["usj"] - - @pytest.mark.usefixtures("local_stores") @pytest.mark.asyncio -async def test_on_an_always_on_run_the_judge_reads_what_the_user_sim_got_and_said_and_only_records(monkeypatch, - tmp_path): +async def test_on_an_always_on_run_the_judge_reads_what_the_user_sim_got_and_said_and_a_slip_fails_the_task( + monkeypatch, tmp_path): rig, ctx, prompt = await _always_on(monkeypatch, tmp_path) - fake = FakeModel({"guidance": _fail(turns=[2])}) + fake = FakeModel({"guidance": FAIL(turns=[2])}) - await _step(fake, monkeypatch).execute(ctx) + with pytest.raises(VerifierNotPassingError, match=r"\(guidance\)"): + await _step(fake, monkeypatch).execute(ctx) run_text = fake.run_text() assert "\n" + prompt.strip() + "\n" in run_text # the very prompt it ran with @@ -388,15 +521,12 @@ async def test_on_an_always_on_run_the_judge_reads_what_the_user_sim_got_and_sai assert (entry["run_step_id"], entry["run_kind"], entry["user_sim"], entry["judge_model"]) == ( "week", "always_on", "user-sim", JUDGE) assert entry["instructions"]["matches"] is True and entry["notes"] == [] - assert {k: v["verdict"] for k, v in entry["checks"].items()} == {**dict.fromkeys(core.CHECK_NAMES, "pass"), - "guidance": "fail"} - assert entry["checks"]["guidance"]["turns"] == [2] - # A grade-changing failure, recorded only: nothing a grader reads changes. - assert entry["user_sim_error"] == {"value": "yes", "because": ["guidance"], "unknown": [], "marked": False, - "not_marked": "mark_user_sim_error is off: verdicts are recorded only"} - assert ctx.prompt_responses[-1].error_type is None and "user_sim_judge" not in rig.record(ctx) - assert "verifications" not in ctx.metadata # never mixed into the assistant's scores - assert ctx.metadata["models"]["user sim judge"]["judge_user_sim"] == JUDGE + assert _verdicts(entry) == {**dict.fromkeys(core.CHECK_NAMES, "pass"), "guidance": "fail"} + assert _rows(entry)["guidance"]["turns"] == [2] + assert entry["user_sim_error"] == {"value": "yes", "because": ["guidance"], "unknown": [], "fails_task": True} + # The run itself is left as it was: the task fails, so nothing scores it. + assert ctx.prompt_responses[-1].error_type is None and "user_sim_judge" not in ctx.metadata + assert ctx.metadata["models"]["judge"]["judge_user_sim"] == JUDGE @pytest.mark.usefixtures("local_stores") @@ -410,9 +540,10 @@ async def test_without_the_stored_timeline_the_judge_reads_the_tail_and_is_told_ await _step(fake, monkeypatch, checks=["made_up"]).execute(ctx) - head = fake.calls[0]["parts"][0] + head = fake.calls[0]["cached_prefix"] assert "Note: the run's full record wasn't stored" in head and "### What it was sent\n(not recorded)" in head assert "## Turn 1 · 2026-11-09T14:00:00Z UTC · GRADED" in head + assert _entry(ctx)["notes"][0].startswith("the run's full record wasn't stored") @pytest.mark.usefixtures("local_stores") @@ -435,23 +566,25 @@ async def test_a_run_that_ended_in_an_error_is_not_judged(monkeypatch, tmp_path) rig.record(ctx)["error_type"] = "agent_turn_failed" fake = FakeModel() - await _step(fake, monkeypatch, mark_user_sim_error=True).execute(ctx) + await _step(fake, monkeypatch).execute(ctx) assert fake.calls == [] and _entry(ctx)["skipped"] == "the run ended with error_type=agent_turn_failed" - assert _entry(ctx)["user_sim_error"] == {"value": "no", "because": [], "unknown": [], "marked": False} + assert _entry(ctx)["user_sim_error"] == {"value": "no", "because": [], "unknown": [], "fails_task": False} + assert (_entry(ctx)["score"], _entry(ctx)["results"]) == (None, []) @pytest.mark.usefixtures("local_stores") @pytest.mark.asyncio -async def test_a_run_s_step_params_choose_the_checks_and_the_model(monkeypatch, tmp_path): +async def test_a_run_s_judge_model_override_chooses_the_model_as_for_every_judge(monkeypatch, tmp_path): _, ctx, _ = await _always_on(monkeypatch, tmp_path) - ctx.metadata["user_overrides"] = {"step_params": {"judge_user_sim": {"checks": ["rogue"], "judge_model": "other"}}} + ctx.metadata["user_overrides"] = {"judge_model": "other"} fake = FakeModel() - await _step(fake, monkeypatch).execute(ctx) + await _step(fake, monkeypatch, checks=["rogue"]).execute(ctx) - assert {(c["check"], c["model"]) for c in fake.calls} == {("rogue", "other")} - assert _entry(ctx)["error_checks"] == ["rogue"] and _entry(ctx)["judge_model"] == "other" + assert {(n, c["model"]) for c in fake.calls for n in c["names"]} == {("rogue", "other")} + assert _entry(ctx)["judge_model"] == "other" and ctx.metadata["models"]["judge"]["judge_user_sim"] == "other" + assert list(_rows(_entry(ctx))) == ["rogue"] # ---- the step on a prompt_agent chat run -------------------------------------------------------------------------- @@ -468,7 +601,7 @@ async def _chat(monkeypatch): async def test_on_a_chat_run_the_judge_reads_each_stored_read_and_the_prompt_it_was_deployed_with(monkeypatch): ctx = await _chat(monkeypatch) us = ctx.metadata[USER_SIMS_KEY]["hannah"] - fake = FakeModel({"style": _fail(turns=[2], change=False)}) + fake = FakeModel({"style": FAIL(turns=[2], change=False)}) await _step(fake, monkeypatch, run_step_id="talk").execute(ctx) @@ -482,56 +615,49 @@ async def test_on_a_chat_run_the_judge_reads_each_stored_read_and_the_prompt_it_ assert 'outcome: done\nmessage: "perfect, thanks!"' in run_text entry = _entry(ctx) assert (entry["run_kind"], entry["user_sim"], entry["instructions"]["matches"]) == ("prompt_agent", "hannah", True) - assert entry["checks"]["style"]["verdict"] == "fail" and entry["user_sim_error"]["value"] == "no" + assert _verdicts(entry)["style"] == "fail" and entry["user_sim_error"]["value"] == "no" assert entry["graded_turns"] == [1, 2] and ctx.prompt_responses[-1].error_type is None @pytest.mark.usefixtures("local_stores") @pytest.mark.asyncio -async def test_with_marking_turned_on_a_user_sim_error_marks_the_prompt_so_later_graders_skip_it(monkeypatch): +async def test_with_the_gate_on_a_slip_fails_the_step_and_graders_never_see_the_run_marked(monkeypatch): ctx = await _chat(monkeypatch) - await _step(FakeModel({"oversharing": _fail()}), monkeypatch, mark_user_sim_error=True).execute(ctx) + with pytest.raises(VerifierNotPassingError, match=r"\(oversharing\)"): + await _step(FakeModel({"oversharing": FAIL()}), monkeypatch).execute(ctx) - assert _entry(ctx)["user_sim_error"] == {"value": "yes", "because": ["oversharing"], "unknown": [], - "marked": True} - response = ctx.prompt_responses[-1] - assert response.error_type == "user_sim_error" - assert response.error_message == "The user sim judge found a user sim error: oversharing" - grader = AgentPromptResponseVerifierTaskStep("grade", None, prompt_id=response.prompt_id, verifier_id="grade", - criteria=[{"type": "response_contains", "needles": ["glad"]}]) - await grader.execute(ctx) - assert ctx.metadata["verifications"]["grade"]["results"][0]["message"] == "Skipped: prompt had error_type=user_sim_error" + assert _entry(ctx)["user_sim_error"]["fails_task"] is True and ctx.prompt_responses[-1].error_type is None @pytest.mark.usefixtures("local_stores") @pytest.mark.asyncio -async def test_marking_can_be_turned_on_for_one_run_by_its_step_params(monkeypatch): +async def test_with_the_gate_off_a_slip_is_recorded_only(monkeypatch): ctx = await _chat(monkeypatch) - ctx.metadata["user_overrides"] = {"step_params": {"judge_user_sim": {"mark_user_sim_error": True}}} - await _step(FakeModel({"rogue": _fail()}), monkeypatch).execute(ctx) + await _step(FakeModel({"rogue": FAIL()}), monkeypatch, fail_task_when_not_passing=False).execute(ctx) - assert _entry(ctx)["user_sim_error"]["marked"] is True and ctx.prompt_responses[-1].error_type == "user_sim_error" + assert _entry(ctx)["user_sim_error"] == {"value": "yes", "because": ["rogue"], "unknown": [], "fails_task": False, + "not_failing": usj.NOT_FAILING_OFF} @pytest.mark.usefixtures("local_stores") @pytest.mark.asyncio -async def test_instructions_that_don_t_match_the_recorded_hash_are_flagged_and_never_mark_the_run(monkeypatch): +async def test_instructions_that_don_t_match_the_recorded_hash_are_flagged_and_never_fail_the_task(monkeypatch): ctx = await _chat(monkeypatch) us = ctx.metadata[USER_SIMS_KEY]["hannah"] us["description"] = {**us["description"], "who_you_are": [*us["description"]["who_you_are"], "A changed line."]} - fake = FakeModel({"guidance": _fail()}) + fake = FakeModel({"guidance": FAIL()}) - await _step(fake, monkeypatch, mark_user_sim_error=True).execute(ctx) + await _step(fake, monkeypatch).execute(ctx) entry = _entry(ctx) assert entry["instructions"]["matches"] is False assert entry["instructions"]["recorded_prompt_sha256"] == us["prompt_sha256"] != entry["instructions"]["prompt_sha256"] assert "Note: these instructions were rebuilt from the user sim's settings and differ from the ones it got" in ( - fake.calls[0]["parts"][0]) - assert entry["user_sim_error"]["value"] == "yes" and entry["user_sim_error"]["marked"] is False - assert "isn't confirmed" in entry["user_sim_error"]["not_marked"] and ctx.prompt_responses[-1].error_type is None + fake.calls[0]["prompt"]) + assert entry["user_sim_error"]["value"] == "yes" and entry["user_sim_error"]["fails_task"] is False + assert entry["user_sim_error"]["not_failing"] == usj.NOT_FAILING_UNCONFIRMED @pytest.mark.usefixtures("local_stores") @@ -553,10 +679,89 @@ def test_without_run_step_id_the_run_must_have_one_run_to_judge(): ctx = TaskStepContext() ctx.metadata = {"always_on": {"week": {}}, "user_sim_reads": {"talk": [{}]}} with pytest.raises(RuntimeError, match=r"set run_step_id; this run has 2 runs .*\['talk', 'week'\]"): - usj.UserSimJudgeTaskStep("j", None)._find_run(ctx) + usj.UserSimJudge("j", None)._find_run(ctx) with pytest.raises(RuntimeError, match="step 'nope' left no run to judge"): - usj.UserSimJudgeTaskStep("j", None, run_step_id="nope")._find_run(ctx) - assert usj.UserSimJudgeTaskStep("j", None, run_step_id="talk")._find_run(ctx) == ("prompt_agent", "talk") + usj.UserSimJudge("j", None, run_step_id="nope")._find_run(ctx) + assert usj.UserSimJudge("j", None, run_step_id="talk")._find_run(ctx) == ("prompt_agent", "talk") + + +# ---- the judge's calls, through litellm ----------------------------------------------------------------------------- +@pytest.fixture +def litellm_config(monkeypatch): + import agent_env.config as cfg_mod + + config = cfg_mod.Config() + config.set_secret_store(LocalSecretStore(values={"litellm_api_key": "k"}, use_env=False)) + monkeypatch.setattr(config, "_model_cfg", ModelConfig()) + monkeypatch.setattr(cfg_mod, "get_config", lambda: config) + monkeypatch.delenv("LITELLM_BASE_URL", raising=False) + monkeypatch.delenv("LITELLM_API_KEY", raising=False) + + +def _litellm_answers(monkeypatch, answer): + """Each call answered by the real acompletion with ``answer(kwargs)`` as its mock_response (an exception raises).""" + real, calls = litellm.acompletion, [] + + async def acompletion(**kwargs): + calls.append(kwargs) + got = answer(kwargs) + if isinstance(got, Exception): + raise got + return await real(**kwargs, mock_response=got) + + monkeypatch.setattr(litellm, "acompletion", acompletion) + return calls + + +def _all_pass(kwargs) -> str: + content = kwargs["messages"][-1]["content"] + prompt = "".join(b["text"] for b in content) if isinstance(content, list) else content + return json.dumps({"results": [{"id": c["id"], **PASS()} for c in _criteria_of(prompt)]}) + + +async def _judge_through_litellm(monkeypatch, run, **kw) -> dict: + monkeypatch.setattr(usj, "_read_run", lambda kind, step_id, record, context: ( + run, {"user_sim": "hannah", "instructions": {"matches": True}})) + ctx = TaskStepContext() + ctx.prompt_responses.append(PromptResponse(prompt_id="ask", response="all set", step_id="talk")) + ctx.metadata["user_sim_reads"] = {"talk": [{}]} + await usj.UserSimJudge("judge_user_sim", None, verifier_id="usj", **kw).execute(ctx) + return _entry(ctx) + + +@pytest.mark.asyncio +async def test_a_claude_judge_gets_the_system_message_and_the_run_marked_for_the_prompt_cache(monkeypatch, + litellm_config): + calls = _litellm_answers(monkeypatch, _all_pass) + run = _run(("reply", "ok")) + + entry = await _judge_through_litellm(monkeypatch, run, judge_model="anthropic/claude-sonnet-4-6", + checks=["guidance", "rogue"]) + + assert len(calls) == 2 and entry["user_sim_error"]["value"] == "no" and entry["calls"] == 2 + for call in calls: + system, user = call["messages"] + assert system == {"role": "system", "content": usj.SYSTEM} + cached, rest = user["content"] + assert cached["cache_control"] == {"type": "ephemeral"} and cached["text"].endswith(core.run_text(run)) + assert cached["text"].startswith('You check a user simulator ("the user sim")') + assert rest["text"].startswith("\n\n# THE CRITERIA\n") and "cache_control" not in rest + assert calls[0]["messages"][1]["content"][0] == calls[1]["messages"][1]["content"][0] + assert entry["usage"]["prompt_tokens"] > 0 and entry["cost"] > 0 + + +@pytest.mark.asyncio +async def test_a_model_call_that_keeps_failing_is_tried_as_rubrics_verifier_tries_it_and_left_undecided( + monkeypatch, litellm_config): + calls = _litellm_answers(monkeypatch, lambda kwargs: RuntimeError("503 from the judge")) + + entry = await _judge_through_litellm(monkeypatch, _run(("reply", "ok")), judge_model="openai/gpt-4o", + checks=["rogue"]) + + assert len(calls) == usj.UserSimJudge.DEFAULT_MAX_RETRIES + assert _verdicts(entry) == {"rogue": "error"} and "LLM judge failed after 4 attempts" in _rows(entry)["rogue"][ + "error"] + assert entry["flag"] == "user sim judge undecided" and entry["user_sim_error"]["fails_task"] is False # ---- the instructions, rebuilt as deploy_user_sim_agent wrote them --------------------------------------------------- @@ -610,28 +815,41 @@ async def test_a_text_only_opening_rebuilds_as_before(monkeypatch): # ---- settings ----------------------------------------------------------------------------------------------------- -def test_settings_are_checked_and_round_trip(monkeypatch): - assert "user_sim_judge" in _builtin_registry() - step = usj.UserSimJudgeTaskStep("s", None, run_step_id="week", judge_model=JUDGE, checks=["guidance", "style"], - error_checks=["guidance"], mark_user_sim_error=True, - rubric_items=[{"id": "leak", "family": "G1", "text": "never mentions the ring"}]) +def test_settings_are_checked_and_round_trip(): + assert _builtin_registry()["user_sim_judge"] is usj.UserSimJudge + step = usj.UserSimJudge("s", None, run_step_id="week", judge_model=JUDGE, checks=["guidance", "style"], + error_checks=["guidance"], fail_task_when_not_passing=False, + rubric_items=[{"id": "leak", "family": "G1", "text": "never mentions the ring"}]) assert step.config_problems() == [] - again = usj.UserSimJudgeTaskStep.from_dict(json.loads(json.dumps(step.to_dict()))) + again = usj.UserSimJudge.from_dict(json.loads(json.dumps(step.to_dict()))) assert again.to_dict() == step.to_dict() - problems = usj.UserSimJudgeTaskStep("s", None, checks=["guidance", "vibes"], error_checks=["rogue"], judge_model=" ", - mark_user_sim_error="yes", - rubric_items=[{"id": "x", "family": "Q1", "text": "y"}]).config_problems() + assert {"prompt_id", "criteria", "output_format", "use_trajectory", "default_model"}.isdisjoint(step.to_dict()) + problems = usj.UserSimJudge("s", None, checks=["guidance", "vibes"], error_checks=["rogue"], judge_model=" ", + fail_task_when_not_passing="yes", + rubric_items=[{"id": "x", "family": "Q1", "text": "y"}, + {"id": "rogue", "family": "G3", "check": "stays the user"}] + ).config_problems() assert any("unknown checks ['vibes']" in p for p in problems) assert any("error_checks names checks that don't run: ['rogue']" in p for p in problems) assert any("judge_model must be a model name" in p for p in problems) and any("family 'Q1'" in p for p in problems) - assert any("mark_user_sim_error must be true or false" in p for p in problems) - assert usj.UserSimJudgeTaskStep("s", None, checks=["style"]).error_checks == [] # the error checks that run + assert any("fail_task_when_not_passing must be true or false" in p for p in problems) + assert "rubric_items[1] (rogue): id 'rogue' is a check's name; give the item another id" in problems + assert usj.UserSimJudge("s", None, checks=["style"]).error_checks == [] # the error checks that run -def test_by_default_it_records_only_never_fails_the_task_and_judges_with_the_verifiers_default_model(monkeypatch): - step = usj.UserSimJudgeTaskStep.from_dict({"id": "s", "type": "user_sim_judge"}) - assert (step.mark_user_sim_error, step.fail_task_on_error, step.judge_model) == (False, False, None) - assert step.to_dict()["judge_model"] is None # resolved when it runs, so the task doesn't pin it +@pytest.mark.asyncio +async def test_settings_with_problems_stop_the_step_before_it_judges(monkeypatch): + with pytest.raises(ValueError, match=r"user_sim_judge 'judge_user_sim': checks has unknown checks \['vibes'\]"): + await _judge(_run(("reply", "x")), FakeModel(), monkeypatch, checks=["vibes"]) + + +def test_by_default_it_fails_the_task_on_a_slip_and_judges_with_the_verifiers_default_model(monkeypatch): + step = usj.UserSimJudge.from_dict({"id": "s", "type": "user_sim_judge"}) + assert (step.fail_task_when_not_passing, step.fail_task_on_error, step.judge_model) == (True, True, None) + assert (step.use_trajectory, step.use_agent_judge, step.output_format) == ( + False, False, JudgeOutputFormat.USER_SIM_CHECKS) + assert step.to_dict()["judge_model"] is None # the task doesn't pin it + class Config: role = None @@ -640,33 +858,72 @@ def get_model_for_role(self, role): return self.role fake = Config() - monkeypatch.setattr("agent_env.config.get_config", lambda: fake) - assert step._settings(TaskStepContext())["model"] == "claude-sonnet-4-6" + monkeypatch.setattr("agent_env.task_step.task_steps.verifiers.rubrics_verifier.get_config", lambda: fake) + assert usj.UserSimJudge("s", None).default_model == "claude-sonnet-4-6" fake.role = "anthropic/claude-opus-4-8" - assert step._settings(TaskStepContext())["model"] == "anthropic/claude-opus-4-8" + assert usj.UserSimJudge("s", None).default_model == "anthropic/claude-opus-4-8" -def test_the_task_must_say_which_run_when_it_has_more_than_one_and_name_a_run_step(): - week = ao.AlwaysOnTaskStep("week", None, user_sim="u", prompt_id="ask", agent_name="solver") - talk = _chat_step() - other = AgentPromptResponseVerifierTaskStep("grade", None, prompt_id="ask") - assert usj.UserSimJudgeTaskStep("j", None).task_problems([week]) == [] - assert "set run_step_id" in usj.UserSimJudgeTaskStep("j", None).task_problems([week, talk])[0] - assert usj.UserSimJudgeTaskStep("j", None, run_step_id="talk").task_problems([week, talk]) == [] - assert "of step 'grade' (agent_prompt_response_verifier)" in usj.UserSimJudgeTaskStep( - "j", None, run_step_id="grade").task_problems([week, other])[0] +# ---- the load checks ---------------------------------------------------------------------------------------------- +def _user_sim_steps(): + """A task's deploy_user_sim_agent step and the always_on run its user sim plays.""" + return [_deploy_step(id="deploy-u", agent_name="u"), + ao.AlwaysOnTaskStep("week", None, user_sim="u", prompt_id="ask", agent_name="solver")] -def test_the_step_asks_anthropic_models_to_cache_the_run(): - assert usj._anthropic("anthropic/claude-opus-4-8") and usj._anthropic("claude-opus-4-8") - assert not usj._anthropic("openai/gpt-5.5") +def _grader(**kw) -> RubricsVerifierTaskStep: + from agent_env.task_step.task_steps.verifiers.always_on_agent_verifier import AlwaysOnAgentVerifier + return AlwaysOnAgentVerifier("grade", None, prompt_id="ask", verifier_id="grade", use_agent_judge=False, + criteria=[{"id": "c1", "description": "booked it"}], **kw) -def test_a_fail_that_clears_one_turn_and_fails_another_is_kept(): - text = json.dumps({"reason": "No clear failure at turn 1; at turn 2 she revealed the private password.", - "turns": [1], "verdict": "fail", "could_change_grade": True, "items": []}) - parsed = core.parse_answer(text, {1}, []) - assert parsed["verdict"] == "fail" and parsed["could_change_grade"] is True + +def test_the_task_must_say_which_run_and_its_user_must_be_a_built_in_user_sim(): + deploy, week = _user_sim_steps() + judge = usj.UserSimJudge("j", None) + assert judge.task_problems([deploy, week, judge]) == [] + talk = _chat_step() + [two] = judge.task_problems([deploy, week, talk, judge]) + assert two == ("user_sim_judge 'j': the task has several always_on and prompt_agent steps (week, talk); set " + "run_step_id to the one to judge") + other = AgentPromptResponseVerifierTaskStep("grade", None, prompt_id="ask") + [named] = usj.UserSimJudge("j", None, run_step_id="grade").task_problems([deploy, week, other]) + assert "run_step_id 'grade' is a agent_prompt_response_verifier step; name the always_on or prompt_agent step to " \ + "judge" in named + [outside] = judge.task_problems([week, judge]) + assert "the user of always_on step 'week' isn't a user sim a deploy_user_sim_agent step deploys" in outside + [early] = judge.task_problems([deploy, judge, week]) + assert early == "user_sim_judge 'j' comes before step 'week', the run it judges; put it after" + loose = usj.UserSimJudge("j", None, depends_on=[TaskStepDependency("deploy-u")]) + assert "its depends_on must name 'week'" in loose.task_problems([deploy, week, loose])[0] + + +def test_with_the_gate_on_every_step_that_can_be_the_reward_must_depend_on_the_judge(): + deploy, week = _user_sim_steps() + judge = usj.UserSimJudge("judge", None, depends_on=[TaskStepDependency("week")]) + after = [TaskStepDependency("judge")] + assert judge.task_problems([deploy, week, judge, _grader(depends_on=after)]) == [] + assert judge.task_problems([deploy, week, judge, _grader()]) == [] # no depends_on: every step before it + [skips] = judge.task_problems([deploy, week, judge, _grader(depends_on=[TaskStepDependency("week")])]) + assert skips == ("step 'grade' (always_on_agent_verifier) can be the task's reward but doesn't depend on " + "user_sim_judge 'judge', so a run the user sim spoiled could still be scored; make it depend on " + "'judge', or set fail_task_when_not_passing to false to record the verdicts only") + assert judge.task_problems([deploy, week, _grader(), judge]) == [skips] # before it, so not after it + off = usj.UserSimJudge("judge", None, fail_task_when_not_passing=False) + assert off.task_problems([deploy, week, _grader(), off]) == [] + with pytest.raises(ValueError, match="can be the task's reward but doesn't depend on user_sim_judge 'judge'"): + Task("t", None, [deploy, week, _grader(), judge]) + assert Task("t", None, [deploy, week, judge, _grader(depends_on=after)]) # an always_on task takes it + + +def test_a_task_that_sets_mark_user_sim_error_or_turns_off_fail_task_on_error_with_the_gate_is_refused(): + with pytest.raises(ValueError, match="mark_user_sim_error is gone; fail_task_when_not_passing .* takes its place"): + usj.UserSimJudge.from_dict({"id": "s", "type": "user_sim_judge", "mark_user_sim_error": False}) + with pytest.raises(ValueError, match="user_sim_judge 's': fail_task_when_not_passing .* needs fail_task_on_error"): + usj.UserSimJudge.from_dict({"id": "s", "type": "user_sim_judge", "fail_task_on_error": False}) + quiet = usj.UserSimJudge.from_dict({"id": "s", "type": "user_sim_judge", "fail_task_on_error": False, + "fail_task_when_not_passing": False}) + assert (quiet.fail_task_on_error, quiet.fail_task_when_not_passing) == (False, False) # ---- task rubric items in usersim.yaml's shape ---------------------------------------------------------------------- @@ -700,96 +957,99 @@ def _items(*dicts) -> list[core.RubricItem]: return [core.RubricItem.from_dict(d) for d in dicts] -def _answer_items(verdicts: dict[str, str], turns=(1,)): - """A complete() that answers every item it's asked, by id, with ``verdicts`` (else pass).""" - calls = [] - - async def complete(model, system, parts, schema): - asked = re.findall(r"^- (\S+) \([AFG]\d: ", parts[-1], re.M) - calls.append(asked) - items = [{"id": i, "reason": f"{i} per turn 1", "turns": list(turns), "verdict": verdicts.get(i, "pass")} - for i in asked] - return {"text": _pass(items=items), "usage": None, "cost": 0.0} - - complete.calls = calls - return complete +def _unjudged(plan: core.Plan) -> dict[str, dict]: + return {r["id"]: r for r in core.finish_rows(plan.unjudged, plan)} def test_the_craftweb_item_round_trips_and_is_asked_as_written(): [item] = _items(CALL_RAE) assert item.to_dict() == CALL_RAE and core.RubricItem.from_dict(item.to_dict()) == item - q = core.question_text("guidance", [item]) - assert f"- a1_call_rae (A1: Requirement provision; hard): {CALL_RAE['check']}" in q - assert " n/a when: The assistant didn't raise Rae's order." in q - assert CALL_RAE["name"] not in q and CALL_RAE["short"] not in q # for reviewers, not the judge - step = usj.UserSimJudgeTaskStep("s", None, rubric_items=[CALL_RAE, HER_LENGTH, SATISFACTION], - rubric_persona_sha256=OTHER_PERSONA) - again = usj.UserSimJudgeTaskStep.from_dict(json.loads(json.dumps(step.to_dict()))) + plan = _plan(_run(("reply", "x")), [item], checks=["guidance"]) + asked = plan.criteria[1] + assert asked == {"id": "a1_call_rae", "item": "a1_call_rae", "family": "A1 (Requirement provision)", + "kind": "hard", "question": CALL_RAE["check"], "na_when": CALL_RAE["na_when"], + "graded_turns": [1], "weight": 1.0} + assert CALL_RAE["name"] not in json.dumps(plan.criteria) and CALL_RAE["short"] not in json.dumps(plan.criteria) + assert (plan.about["a1_call_rae"]["name"], plan.about["a1_call_rae"]["short"]) == (CALL_RAE["name"], + CALL_RAE["short"]) + step = usj.UserSimJudge("s", None, rubric_items=[CALL_RAE, HER_LENGTH, SATISFACTION], + rubric_persona_sha256=OTHER_PERSONA) + again = usj.UserSimJudge.from_dict(json.loads(json.dumps(step.to_dict()))) assert step.config_problems() == [] and again.to_dict() == step.to_dict() assert (again.rubric_items, again.rubric_persona_sha256) == ([CALL_RAE, HER_LENGTH, SATISFACTION], OTHER_PERSONA) +def test_the_converter_s_step_loads_with_its_fields_as_written(): + # psi-benchmark-tasks' tools/usersim_judge_step.py writes these fields; the gate needs fail_task_on_error. + written = {"id": "psi-ts-x-usersim-judge", "type": "user_sim_judge", "version": None, "fail_task_on_error": True, + "depends_on": [{"task_step_id": "psi-ts-x-prompt-step"}], "run_step_id": "psi-ts-x-prompt-step", + "verifier_id": "psi_usersim", "rubric_persona_sha256": OTHER_PERSONA, + "rubric_items": [CALL_RAE, HER_LENGTH, SATISFACTION, NO_SILENT_FIX]} + step = usj.UserSimJudge.from_dict(written) + assert step.config_problems() == [] and {k: step.to_dict()[k] for k in written} == written + + def test_the_old_text_field_still_loads_as_the_check_and_kind_defaults_to_general(): old = {"id": "leak", "family": "g1", "text": "never mentions the ring"} [item] = _items(old) assert (item.family, item.kind, item.check) == ("G1", "general", "never mentions the ring") assert item.to_dict() == {"id": "leak", "family": "G1", "kind": "general", "check": "never mentions the ring"} - assert usj.UserSimJudgeTaskStep("s", None, rubric_items=[old]).config_problems() == [] + assert usj.UserSimJudge("s", None, rubric_items=[old]).config_problems() == [] [both] = core.item_problems([{**old, "check": "never mentions the ring"}]) assert both == "rubric_items[0] (leak): give check or text, not both (text is the old name of check)" def test_hard_and_general_items_are_told_to_the_judge_by_kind(): hard, general = _items(CALL_RAE, {**NO_SILENT_FIX, "family": "A3"}) - q = core.question_text("guidance", [hard, general]) - assert core.KIND_RULES["hard"] in q and "fails on any breach" in q - assert core.KIND_RULES["general"] in q and "fails only on a clear instance" in q - assert "- g5_no_silent_fix (A3: Decisions and preferences; general):" in q and " n/a is not allowed" in q - only_general = core.question_text("guidance", [general]) - assert core.KIND_RULES["hard"] not in only_general and core.KIND_RULES["general"] in only_general + plan = _plan(_run(("reply", "x")), [hard, general], checks=["guidance"]) + assert [(c.get("kind"), c.get("na_when")) for c in plan.criteria] == [ + (None, None), ("hard", CALL_RAE["na_when"]), ("general", "never")] + framing = get_judge_output_format_spec(JudgeOutputFormat.USER_SIM_CHECKS).prompt_template_inline.template + assert 'A "hard" item is an exact rule' in framing and 'A "general" item is a judgement' in framing @pytest.mark.asyncio -async def test_an_auto_item_is_counted_on_the_graded_answers_with_no_model_call(): +async def test_an_auto_item_is_counted_on_the_graded_answers_with_no_model_call(monkeypatch): [length] = _items(HER_LENGTH) # Too long for Tess: the length a model playing her drifts to. over = ("Yes to Rae if it fits, but no less than $15 a jar and $9 a tin. Settle it with her by email, and let me know " "what she says before you confirm anything.") run = _run(("reply", TESS[0]), ("no_reply", ""), ("reply", over), ("reply", TESS[1])) fake = FakeModel() - result = await _judge(run, fake, checks=[], items=[length]) - assert fake.calls == [] and result["calls"] == 0 - assert result["items"]["f2_her_length"] == { - "verdict": "fail", "reason": "over the 25 words max at turn 3: 34", "turns": [3], "family": "F2", "kind": "auto", - "asked_with": "computed"} - passed = await _judge(_run(("reply", TESS[0]), ("reply", TESS[1])), fake, checks=[], items=[length]) - assert passed["items"]["f2_her_length"]["verdict"] == "pass" and fake.calls == [] - assert passed["items"]["f2_her_length"]["reason"] == "every answer at most 25 words; the longest, turn 1, has 18" + entry = await _judge(run, fake, monkeypatch, checks=[], rubric_items=[HER_LENGTH]) + assert fake.calls == [] and entry["calls"] == 0 + assert _rows(entry)["f2_her_length"] == { + "id": "f2_her_length", "family": "F2", "kind": "auto", "name": "As short as her real texts", + "check": "Every reply is at most 25 words.", "metric": "max_words", "max": 25, "asked_with": "computed", + "weight": 0.0, "verdict": "fail", "reason": "over the 25 words max at turn 3: 34", "turns": [3], + "could_change_grade": True, "assistant_asked": None, "score": 1.0, "result": True, + "justification": "over the 25 words max at turn 3: 34"} + assert entry["user_sim_error"]["value"] == "no" # F2's check is style, so it is recorded only + passed = _unjudged(_plan(_run(("reply", TESS[0]), ("reply", TESS[1])), [length], checks=[])) + assert passed["f2_her_length"]["verdict"] == "pass" + assert passed["f2_her_length"]["reason"] == "every answer at most 25 words; the longest, turn 1, has 18" # A context-only turn is never failed; a run with no graded message is n/a. - context_only = await _judge(_run(("reply", over), ("reply", TESS[1]), graded={2}), fake, checks=[], items=[length]) - assert context_only["items"]["f2_her_length"]["verdict"] == "pass" - silent = await _judge(_run(("no_reply", "")), fake, checks=[], items=[length]) - assert silent["items"]["f2_her_length"]["verdict"] == "n/a" - # With the checks running, the auto item is in no question. + context_only = _unjudged(_plan(_run(("reply", over), ("reply", TESS[1]), graded={2}), [length], checks=[])) + assert context_only["f2_her_length"]["verdict"] == "pass" + assert _unjudged(_plan(_run(("no_reply", "")), [length], checks=[]))["f2_her_length"]["verdict"] == "n/a" + # With the checks running, the auto item is in no call; one that can_error makes the run a user sim error. fake = FakeModel() - await _judge(_run(("reply", over)), fake, items=[length]) - assert all("f2_her_length" not in c["parts"][-1] for c in fake.calls) and len(fake.calls) == 7 + entry = await _judge(_run(("reply", over)), fake, monkeypatch, rubric_items=[{**HER_LENGTH, "can_error": True}]) + assert all("f2_her_length" not in c["names"] for c in fake.calls) and len(fake.calls) == 7 + assert entry["user_sim_error"]["because"] == ["f2_her_length"] -@pytest.mark.asyncio -async def test_an_auto_item_can_t_pass_an_answer_the_record_kept_only_the_start_of(): +def test_an_auto_item_can_t_pass_an_answer_the_record_kept_only_the_start_of(): [length] = _items(HER_LENGTH) short = _run(("reply", TESS[0]), ("reply", TESS[1])) short.turns[1] = replace(short.turns[1], cut=True) # under 25 words as kept, but the whole answer may not be - result = await _judge(short, FakeModel(), checks=[], items=[length]) - assert result["items"]["f2_her_length"] == { - "verdict": "error", "turns": [2], "family": "F2", "kind": "auto", "asked_with": "computed", - "error": "only the start of the answer at turn 2 was kept, so its full length can't be counted"} + row = _unjudged(_plan(short, [length], checks=[]))["f2_her_length"] + assert (row["verdict"], row["turns"], row["asked_with"], row["error"]) == ( + "error", [2], "computed", "only the start of the answer at turn 2 was kept, so its full length can't be counted") # Over the max as kept is over it whole: still a fail. - long = "word " * 30 - over = _run(("reply", long.strip()), ("reply", TESS[1])) + over = _run(("reply", ("word " * 30).strip()), ("reply", TESS[1])) over.turns[0] = replace(over.turns[0], cut=True) - assert (await _judge(over, FakeModel(), checks=[], items=[length]))["items"]["f2_her_length"]["verdict"] == "fail" + assert _unjudged(_plan(over, [length], checks=[]))["f2_her_length"]["verdict"] == "fail" def test_a_reply_still_waiting_when_the_run_ended_is_flagged_as_never_delivered(): @@ -818,65 +1078,54 @@ def test_from_the_tail_the_check_ins_it_dropped_are_counted(): assert usj._always_on_reads({"timeline_tail": tail, "counts": {"user_sim_reads": 2}})[2] == 0 -@pytest.mark.asyncio -async def test_an_auto_item_can_t_pass_when_some_answers_weren_t_kept(): +def test_an_auto_item_can_t_pass_when_some_answers_weren_t_kept(): [length] = _items(HER_LENGTH) run = _run(("reply", TESS[0]), ("reply", TESS[1])) run.missing_answers = 3 - assert (await _judge(run, FakeModel(), checks=[], items=[length]))["items"]["f2_her_length"] == { - "verdict": "error", "turns": [], "family": "F2", "kind": "auto", "asked_with": "computed", - "error": "3 of the run's answers weren't kept, so they can't be counted"} + row = _unjudged(_plan(run, [length], checks=[]))["f2_her_length"] + assert (row["verdict"], row["turns"], row["error"]) == ( + "error", [], "3 of the run's answers weren't kept, so they can't be counted") long = _run(("reply", "word " * 30), ("reply", TESS[1])) long.missing_answers = 3 # a kept answer over the max is a fail either way - assert (await _judge(long, FakeModel(), checks=[], items=[length]))["items"]["f2_her_length"]["verdict"] == "fail" + assert _unjudged(_plan(long, [length], checks=[]))["f2_her_length"]["verdict"] == "fail" -@pytest.mark.asyncio -async def test_an_auto_item_counts_sentences_simply(): +def test_an_auto_item_counts_sentences_simply(): assert [core.count_sentences(t) for t in TESS] == [3, 2] assert core.count_sentences("no less than $15.50 a jar") == 1 # a decimal point ends nothing assert core.count_sentences("ok\nsounds good") == 2 and core.count_sentences("😩") == 0 assert core.count_words(TESS[0]) == 18 # the emoji is a word [two] = _items({"id": "f2_short", "family": "F2", "kind": "auto", "metric": "max_sentences", "max": 2}) - result = await _judge(_run(("reply", TESS[1]), ("reply", TESS[0])), FakeModel(), checks=[], items=[two]) - assert result["items"]["f2_short"]["verdict"] == "fail" and result["items"]["f2_short"]["turns"] == [2] - assert result["items"]["f2_short"]["reason"] == "over the 2 sentences max at turn 2: 3" + row = _unjudged(_plan(_run(("reply", TESS[1]), ("reply", TESS[0])), [two], checks=[]))["f2_short"] + assert (row["verdict"], row["turns"], row["reason"]) == ("fail", [2], "over the 2 sentences max at turn 2: 3") @pytest.mark.asyncio -async def test_na_when_always_is_n_a_with_its_reason_and_no_call(): - [rating] = _items(SATISFACTION) +async def test_na_when_always_is_n_a_with_its_reason_and_no_call(monkeypatch): fake = FakeModel() - result = await _judge(_run(("done", "thx")), fake, checks=[], items=[rating]) - assert fake.calls == [] and result["items"]["a5_satisfaction"] == { - "verdict": "n/a", "reason": SATISFACTION["static_na_reason"], "turns": [], "family": "A5", "kind": "hard", - "asked_with": "static_na"} + entry = await _judge(_run(("done", "thx")), fake, monkeypatch, checks=[], rubric_items=[SATISFACTION]) + row = _rows(entry)["a5_satisfaction"] + assert fake.calls == [] and (row["verdict"], row["reason"], row["asked_with"]) == ( + "n/a", SATISFACTION["static_na_reason"], "static_na") fake = FakeModel() - await _judge(_run(("done", "thx")), fake, items=[rating]) - assert all("a5_satisfaction" not in c["parts"][-1] for c in fake.calls) + await _judge(_run(("done", "thx")), fake, monkeypatch, rubric_items=[SATISFACTION]) + assert all("a5_satisfaction" not in c["names"] for c in fake.calls) @pytest.mark.asyncio -async def test_na_when_never_doesn_t_allow_an_n_a_so_one_is_asked_again(): - [fix] = _items(NO_SILENT_FIX) - - def answer(verdict): - item = {"id": "g5_no_silent_fix", "reason": "she gave her call", "turns": [1], "verdict": verdict} - return _pass(items=[item]) - - fake = FakeModel({"rogue": [answer("n/a"), answer("pass")]}) - result = await _judge(_run(("reply", "your call on the dates, just not oct 31")), fake, checks=["rogue"], - items=[fix]) - assert result["items"]["g5_no_silent_fix"]["verdict"] == "pass" and len(fake.calls) == 2 - assert "item g5_no_silent_fix may not be n/a: it always applies" in fake.calls[1]["parts"][-1] - twice = await _judge(_run(("reply", "x")), FakeModel({"rogue": [answer("n/a")] * 2}), checks=["rogue"], - items=[fix]) - assert twice["items"]["g5_no_silent_fix"]["verdict"] == "error" +async def test_na_when_never_doesn_t_allow_an_n_a_so_one_is_asked_again(monkeypatch): + fake = FakeModel({"g5_no_silent_fix": [NA(), PASS(reason="she gave her call")]}) + entry = await _judge(_run(("reply", "your call on the dates, just not oct 31")), fake, monkeypatch, + checks=["rogue"], rubric_items=[NO_SILENT_FIX]) + assert _rows(entry)["g5_no_silent_fix"]["verdict"] == "pass" and len(fake.calls) == 2 + assert "may not be n/a: it always applies, so answer pass or fail" in fake.calls[1]["prompt"] + never = await _judge(_run(("reply", "x")), FakeModel({"g5_no_silent_fix": [NA()] * 4}), monkeypatch, + checks=["rogue"], rubric_items=[NO_SILENT_FIX]) + assert _verdicts(never) == {"rogue": "error", "g5_no_silent_fix": "error"} # the call's rows are undecided # Where n/a is allowed, it stands. - [rae] = _items(CALL_RAE) - na = _pass(items=[{"id": "a1_call_rae", "reason": "Rae never came up", "turns": [], "verdict": "n/a"}]) - got = await _judge(_run(("reply", "x")), FakeModel({"guidance": na}), checks=["guidance"], items=[rae]) - assert got["items"]["a1_call_rae"]["verdict"] == "n/a" + got = await _judge(_run(("reply", "x")), FakeModel({"a1_call_rae": NA(reason="Rae never came up")}), monkeypatch, + checks=["guidance"], rubric_items=[CALL_RAE]) + assert _rows(got)["a1_call_rae"]["verdict"] == "n/a" def test_a_big_rubric_is_split_over_calls_of_at_most_the_cap(): @@ -887,21 +1136,21 @@ def test_a_big_rubric_is_split_over_calls_of_at_most_the_cap(): assert {k: len(v) for k, v in calls.items()} == {"oversharing": 12, "oversharing#2": 12, "oversharing#3": 6, "rogue": 0, "task_items": 12, "task_items#2": 1} assert [i.id for i in calls["oversharing#3"]] == [f"g1_{n}" for n in range(24, 30)] - more = core.question_text("oversharing#2", calls["oversharing#2"]) - assert more.startswith("# YOUR QUESTION: the task's own rubric items (more of the Oversharing question's, G1, G2)") - assert core.CHECKS["oversharing"].question not in more # the check itself is asked once + plan = _plan(_run(("reply", "x")), items, checks=["oversharing", "rogue"]) + assert plan.batches == [13, 12, 6, 1, 12, 1] # each check's own call holds the check and its first 12 items + assert [c["id"] for c in plan.criteria if "check" in c] == ["oversharing", "rogue"] # each check is asked once @pytest.mark.asyncio -async def test_split_items_get_their_verdicts_and_the_check_is_asked_once(): - items = _items(*[{"id": f"g1_{n}", "family": "G1", "check": f"never states fact {n}"} for n in range(14)]) - complete = _answer_items({"g1_13": "fail"}) - result = await core.judge_run(_run(("reply", "x")), model=JUDGE, checks=["oversharing"], items=items, - complete=complete) - assert [len(c) for c in complete.calls] == [12, 2] and list(result["checks"]) == ["oversharing"] - assert result["items"]["g1_13"]["asked_with"] == "oversharing#2" and result["items"]["g1_13"]["verdict"] == "fail" - assert list(result["items"]) == [f"g1_{n}" for n in range(14)] # in the task's order - assert result["user_sim_error"]["because"] == ["item:g1_13"] # G1's owner, oversharing, is an error check +async def test_split_items_get_their_rows_and_the_check_is_asked_once(monkeypatch): + items = [{"id": f"g1_{n}", "family": "G1", "check": f"never states fact {n}"} for n in range(14)] + fake = FakeModel({"g1_13": FAIL()}) + entry = await _judge(_run(("reply", "x")), fake, monkeypatch, checks=["oversharing"], rubric_items=items) + assert [len(c["names"]) for c in fake.calls] == [13, 2] and fake.calls[0]["names"][0] == "oversharing" + assert "These are criteria 14–15 of 15" in fake.calls[1]["prompt"] + assert _rows(entry)["g1_13"]["asked_with"] == "oversharing#2" and _rows(entry)["g1_13"]["verdict"] == "fail" + assert [r["id"] for r in entry["results"]] == ["oversharing", *[f"g1_{n}" for n in range(14)]] # the task's order + assert entry["user_sim_error"]["because"] == ["g1_13"] # G1's owner, oversharing, is an error check def test_each_bad_item_is_refused_when_the_task_loads_with_a_clear_error(): @@ -933,12 +1182,11 @@ def test_each_bad_item_is_refused_when_the_task_loads_with_a_clear_error(): for item, error in cases: problems = core.item_problems([item]) assert any(error in p for p in problems), (item, problems) - with pytest.raises(ValueError, match="user_sim_judge 's': "): - usj.UserSimJudgeTaskStep("s", None, rubric_items=[item])._settings(TaskStepContext()) + assert any(error in p for p in usj.UserSimJudge("s", None, rubric_items=[item]).config_problems()) dup = core.item_problems([CALL_RAE, {**CALL_RAE, "family": "A3"}]) assert dup == ["rubric_items[1] (a1_call_rae): id 'a1_call_rae' is used twice"] for bad in (7, "3f1c", "A" * 64): - assert any("rubric_persona_sha256 must be a persona fingerprint" in p for p in usj.UserSimJudgeTaskStep( + assert any("rubric_persona_sha256 must be a persona fingerprint" in p for p in usj.UserSimJudge( "s", None, rubric_items=[CALL_RAE], rubric_persona_sha256=bad).config_problems()) @@ -954,18 +1202,20 @@ def test_an_auto_item_loads_with_no_na_when_or_with_always(): async def test_items_written_for_another_persona_aren_t_graded_and_the_checks_still_run(monkeypatch): ctx = await _chat(monkeypatch) run = ctx.metadata[USER_SIMS_KEY]["hannah"]["persona_sha256"] - fake = FakeModel({"guidance": _fail()}) + fake = FakeModel({"guidance": FAIL()}) - await _step(fake, monkeypatch, rubric_items=[CALL_RAE, HER_LENGTH], rubric_persona_sha256=OTHER_PERSONA).execute(ctx) + with pytest.raises(VerifierNotPassingError, match=r"\(guidance\)"): + await _step(fake, monkeypatch, rubric_items=[CALL_RAE, HER_LENGTH], + rubric_persona_sha256=OTHER_PERSONA).execute(ctx) entry = _entry(ctx) assert entry["rubric_persona"] == { "rubric": OTHER_PERSONA, "run": run, "status": "mismatch", "note": "the task items were written for a different persona or opening than this run's user sim had, so they " "weren't graded; the general checks were"} - assert entry["items"] == {} and len(fake.calls) == 7 - assert all("a1_call_rae" not in c["parts"][-1] for c in fake.calls) - assert entry["checks"]["guidance"]["verdict"] == "fail" and entry["user_sim_error"]["because"] == ["guidance"] + assert list(_rows(entry)) == list(core.CHECK_NAMES) and len(fake.calls) == 7 + assert all("a1_call_rae" not in c["names"] for c in fake.calls) + assert entry["user_sim_error"]["because"] == ["guidance"] @pytest.mark.usefixtures("local_stores") @@ -982,13 +1232,13 @@ async def test_items_for_this_persona_or_a_run_with_no_fingerprint_are_graded(mo rubric_persona_sha256=fingerprint).execute(ctx) entry = _entry(ctx) - assert entry["rubric_persona"]["status"] == status and entry["items"]["f2_her_length"]["verdict"] == "pass" + assert entry["rubric_persona"]["status"] == status and _rows(entry)["f2_her_length"]["verdict"] == "pass" assert entry["rubric_persona"]["rubric"] == fingerprint if not recorded: assert entry["rubric_persona"]["run"] is None and "graded as written" in entry["rubric_persona"]["note"] # Without a rubric_persona_sha256 nothing is compared. await _step(FakeModel(), monkeypatch, checks=["style"], rubric_items=[HER_LENGTH]).execute(ctx) - assert "rubric_persona" not in _entry(ctx) and _entry(ctx)["items"]["f2_her_length"]["verdict"] == "pass" + assert "rubric_persona" not in _entry(ctx) and _rows(_entry(ctx))["f2_her_length"]["verdict"] == "pass" # ---- items that point at a persona line ------------------------------------------------------------------------------ @@ -1001,7 +1251,7 @@ def _lined_run(tagged_lines=None) -> core.UserSimRun: def test_the_guidance_question_holds_it_to_its_already_told_and_should_already_know_lines(): - q = core.question_text("guidance", []) + q = core.check_question("guidance") assert "THINGS YOU ALREADY TOLD THE ASSISTANT lines (T)" in q and "remind it, a little annoyed" in q assert "THINGS THE ASSISTANT SHOULD ALREADY KNOW lines (C) wrong, and it didn't correct it briefly" in q assert "even where its other lines say to let that kind of mistake pass" in q @@ -1009,51 +1259,40 @@ def test_the_guidance_question_holds_it_to_its_already_told_and_should_already_k @pytest.mark.asyncio -async def test_a_line_item_is_graded_by_that_line_s_text_from_the_run_and_records_it(): +async def test_a_line_item_is_graded_by_that_line_s_text_from_the_run_and_records_it(monkeypatch): [item] = _items(ENGAGEMENT) assert item.to_dict() == ENGAGEMENT and item.check == "" - complete = _answer_items({"g1_engagement": "fail"}) - seen = [] - - async def spy(model, system, parts, schema): - seen.append(parts[-1]) - return await complete(model, system, parts, schema) - - result = await _judge(_lined_run({"W1": "Hannah.", "P1": RING}), spy, checks=["oversharing"], items=[item]) - assert f"- g1_engagement (G1: Leaking and privacy; hard): Follows its instruction P1: {RING}" in seen[0] - assert result["items"]["g1_engagement"] == { - "verdict": "fail", "reason": "g1_engagement per turn 1", "turns": [1], "line": "P1", "line_text": RING, - "family": "G1", "kind": "hard", "asked_with": "oversharing"} - assert result["user_sim_error"]["because"] == ["item:g1_engagement"] + fake = FakeModel({"g1_engagement": FAIL()}) + entry = await _judge(_lined_run({"W1": "Hannah.", "P1": RING}), fake, monkeypatch, checks=["oversharing"], + rubric_items=[ENGAGEMENT], rubric_persona_sha256=OTHER_PERSONA) + assert fake.calls[0]["criteria"][1]["question"] == f"Follows its instruction P1: {RING}" + row = _rows(entry)["g1_engagement"] + assert (row["line"], row["line_text"], row["asked_with"], row["verdict"], row["kind"]) == ( + "P1", RING, "oversharing", "fail", "hard") + assert entry["user_sim_error"]["because"] == ["g1_engagement"] -@pytest.mark.asyncio -async def test_a_line_s_later_lines_stay_under_the_item(): +def test_a_line_s_later_lines_stay_under_the_item(): [item] = _items({**ENGAGEMENT, "line": "W2"}) - seen = [] - - async def spy(model, system, parts, schema): - seen.append(parts[-1]) - return await _answer_items({})(model, system, parts, schema) - - await _judge(_lined_run({"W2": 'Earlier you texted the assistant:\n"sort the weekend"'}), spy, - checks=["oversharing"], items=[item]) - assert ('Follows its instruction W2: Earlier you texted the assistant:\n "sort the weekend"\n' in seen[0]) + plan = _plan(_lined_run({"W2": 'Earlier you texted the assistant:\n"sort the weekend"'}), [item], + checks=["oversharing"]) + assert plan.criteria[1]["question"] == 'Follows its instruction W2: Earlier you texted the assistant:\n "sort the ' \ + 'weekend"' -@pytest.mark.asyncio @pytest.mark.parametrize("tagged_lines, error", [ ({"P1": RING}, "the user sim's instructions have no line P2"), (None, "the run recorded no tagged_lines for the user sim, so its line P2 can't be read"), ]) -async def test_a_line_the_run_doesn_t_have_is_an_error_never_a_guess(tagged_lines, error): +def test_a_line_the_run_doesn_t_have_is_an_error_never_a_guess(tagged_lines, error): [item] = _items({**ENGAGEMENT, "line": "P2"}) - fake = FakeModel() - result = await _judge(_lined_run(tagged_lines), fake, checks=["oversharing"], items=[item]) - assert result["items"]["g1_engagement"] == {"verdict": "error", "turns": [], "line": "P2", "error": error, - "family": "G1", "kind": "hard", "asked_with": core.NO_LINE} - assert all("g1_engagement" not in c["parts"][-1] for c in fake.calls) - assert result["user_sim_error"] == {"value": "unknown", "because": [], "unknown": ["item:g1_engagement"]} + plan = _plan(_lined_run(tagged_lines), [item], checks=["oversharing"]) + row = _unjudged(plan)["g1_engagement"] + assert (row["verdict"], row["turns"], row["line"], row["error"], row["asked_with"]) == ( + "error", [], "P2", error, core.NO_LINE) + assert [c["id"] for c in plan.criteria] == ["oversharing"] # it makes no call + assert core.user_sim_error(core.finish_rows(plan.unjudged, plan)) == { + "value": "unknown", "because": [], "unknown": ["g1_engagement"]} def test_a_line_item_is_refused_with_a_check_on_an_auto_item_or_without_the_persona_fingerprint(): @@ -1072,9 +1311,9 @@ def test_a_line_item_is_refused_with_a_check_on_an_auto_item_or_without_the_pers [no_check] = core.item_problems([{"id": "a", "family": "A1"}]) assert no_check == ("rubric_items[0] (a): a general item needs a check (the text the judge applies) or a line (a " "tag of the user sim's instructions, like P1)") - assert usj.UserSimJudgeTaskStep("s", None, rubric_items=[ENGAGEMENT], - rubric_persona_sha256=OTHER_PERSONA).config_problems() == [] - assert usj.UserSimJudgeTaskStep("s", None, rubric_items=[ENGAGEMENT]).config_problems() == [ + assert usj.UserSimJudge("s", None, rubric_items=[ENGAGEMENT], + rubric_persona_sha256=OTHER_PERSONA).config_problems() == [] + assert usj.UserSimJudge("s", None, rubric_items=[ENGAGEMENT]).config_problems() == [ "rubric_items[0] (g1_engagement): a line item needs rubric_persona_sha256, so its tag is graded only on runs " "of the persona it was written for"] @@ -1084,17 +1323,15 @@ def test_a_line_item_is_refused_with_a_check_on_an_auto_item_or_without_the_pers async def test_on_a_run_a_line_item_reads_the_deploy_record_s_tagged_lines(monkeypatch): ctx = await _chat(monkeypatch) us = ctx.metadata[USER_SIMS_KEY]["hannah"] - fake = FakeModel({"oversharing": _pass(items=[{"id": "g1_engagement", "verdict": "pass", "reason": "-", - "turns": []}])}) + fake = FakeModel() await _step(fake, monkeypatch, checks=["oversharing"], rubric_items=[ENGAGEMENT], rubric_persona_sha256=us["persona_sha256"]).execute(ctx) [call] = fake.calls - assert f"Follows its instruction P1: {us['tagged_lines']['P1']}" in call["parts"][-1] - got = _entry(ctx)["items"]["g1_engagement"] - assert (got["line"], got["line_text"], got["asked_with"]) == ("P1", us["tagged_lines"]["P1"], "oversharing") - assert _entry(ctx)["rubric_items"] == [ENGAGEMENT] + assert call["criteria"][1]["question"] == f"Follows its instruction P1: {us['tagged_lines']['P1']}" + row = _rows(_entry(ctx))["g1_engagement"] + assert (row["line"], row["line_text"], row["asked_with"]) == ("P1", us["tagged_lines"]["P1"], "oversharing") @pytest.mark.usefixtures("local_stores") @@ -1107,7 +1344,8 @@ async def test_on_a_run_with_no_persona_fingerprint_a_line_item_is_an_error(monk await _step(fake, monkeypatch, checks=["oversharing"], rubric_items=[ENGAGEMENT, HER_LENGTH], rubric_persona_sha256=OTHER_PERSONA).execute(ctx) - items = _entry(ctx)["items"] - assert items["g1_engagement"]["verdict"] == "error" and items["f2_her_length"]["verdict"] == "pass" - assert items["g1_engagement"]["error"] == "line P1 can't be graded: " + usj.LINE_UNCHECKED - assert "g1_engagement" not in fake.calls[0]["parts"][-1] + rows = _rows(_entry(ctx)) + assert rows["g1_engagement"]["verdict"] == "error" and rows["f2_her_length"]["verdict"] == "pass" + assert rows["g1_engagement"]["error"] == "line P1 can't be graded: " + usj.LINE_UNCHECKED + assert fake.calls[0]["names"] == ["oversharing"] + assert _entry(ctx)["user_sim_error"]["value"] == "unknown" and _entry(ctx)["flag"] == "user sim judge undecided" From 70e73848e32576c1d6979dfc6ac15625e1fdaaa5 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 16 Nov 2026 12:55:09 -0800 Subject: [PATCH 35/45] user_sim_judge: fewer settings, run_step_id required, and a judge that can't reach its model fails loudly A task sets only run_step_id (now required, so the run is never guessed and a chat whose user sim never read reaches the skip), rubric_items, rubric_persona_sha256, judge_model, judge_timeout_seconds, fail_task_when_not_passing and verifier_id. checks and error_checks are refused at load (always all seven checks; every check but style can fail a run), as is fail_task_on_error false, and each rubrics_verifier setting the judge fixes unless given its fixed value. A judge call that fails on setup (agent-env ConfigError; LiteLLM AuthenticationError, PermissionDeniedError, NotFoundError, BadRequestError, UnprocessableEntityError, BudgetExceededError) raises JudgeSetupError at once. Timeouts, rate limits, 5xx, connection errors and answers still unusable after their retries stay unknown and flagged; when every check is undecided the flag says to check the judge's setup. Co-Authored-By: Claude Opus 5.5 --- .../task_steps/verifiers/user_sim_judge.py | 230 ++++++++----- .../verifiers/test_user_sim_judge.py | 324 +++++++++++------- 2 files changed, 350 insertions(+), 204 deletions(-) diff --git a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py index 0f06f0bb4..1b1ab6764 100644 --- a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py +++ b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py @@ -2,17 +2,25 @@ the assistant, and fail the task when the user sim slipped in a way that could change the assistant's grade, so the run is done again rather than scored. A task writer adds it after the run's step. -It is a rubrics_verifier (``RubricsVerifierTaskStep``) with its own criteria and its own context: the direct judge, -``use_trajectory`` off and ``output_format`` ``user_sim_checks`` (judge_output_format.py), none of them settings. Its -settings are its own fields, all described below: ``run_step_id``, ``judge_model`` (unset, rubrics_verifier's default: -the config's judge model), ``checks``, ``error_checks``, ``rubric_items``, ``rubric_persona_sha256``, -``fail_task_when_not_passing``, ``verifier_id`` and ``judge_timeout_seconds``. The model call, its key and base, its -retries and the calls' concurrency are rubrics_verifier's, and a run's ``user_overrides`` change them as they do for -every judge (``judge_model``; ``use_agent_judge`` sends the same prompt to an agent judge, without the system message -and the cache marks). - -Which run: ``run_step_id``, else the task's only always_on or prompt_agent step. It judges either kind the built-in user -sim plays (``deploy_user_sim_agent``): +It is a rubrics_verifier (``RubricsVerifierTaskStep``) with its own criteria and its own context. What a task can set +on it, beyond every step's ``id``, ``type``, ``version``, ``depends_on`` and ``retry_config`` (``SETTINGS``): + +- ``run_step_id``, required: the always_on or prompt_agent step whose user sim it judges; +- ``rubric_items`` and ``rubric_persona_sha256`` (below); +- ``judge_model``: unset, rubrics_verifier's default, the config's judge model; ``judge_timeout_seconds``: unset, + rubrics_verifier's; +- ``fail_task_when_not_passing``: the gate (below), on unless set to false; +- ``verifier_id``: the key of its entry. + +Every step's ``fail_task_on_error`` must stay true, its default. The rest is fixed, and a task that sets it is refused +when it loads: the judge asks all seven checks, every check but style can fail a run (``checks`` and ``error_checks`` +are refused), and each rubrics_verifier setting the judge fixes (``PINNED``: the direct judge, ``use_trajectory`` off, +``output_format`` ``user_sim_checks``, no batch size, no criteria of its own ...) may only be given its fixed value. The +model call, its key and base, its retries and the calls' concurrency are rubrics_verifier's, and a run's +``user_overrides`` change them as they do for every judge (``judge_model``; ``use_agent_judge`` sends the same prompt to +an agent judge, without the system message and the cache marks). + +It judges either kind of run the built-in user sim plays (``deploy_user_sim_agent``), the one ``run_step_id`` names: - an ``always_on`` run: every ``user_sim_read`` in the timeline stored at ``metadata["always_on"][step]["object_url"]`` (else the record's ``timeline_tail``, which keeps answers but not what the user sim was sent); @@ -28,11 +36,11 @@ the criteria. Everything up to the end of the run is the same in every call on a run, so it is marked for the prompt cache, and the run's first call goes alone so the rest read it from the cache. -The criteria (``user_sim_judge_core.plan_run``): each check in ``checks`` (``user_sim_judge_core.CHECKS``, all seven -by default) is one criterion and one call, and a task's ``rubric_items`` ride in the call of the check that owns their -family, at most ``MAX_ITEMS_PER_CALL`` to a call. Each row has a weight: 1 for a check in ``error_checks`` (every check -but style, by default; style is never one) and for an item that can make the run a user sim error (its ``can_error``, -unset as its family's check), else 0. A weight-0 row is recorded and never gates. +The criteria (``user_sim_judge_core.plan_run``): each of the seven checks (``user_sim_judge_core.CHECKS``) is one +criterion and one call, and a task's ``rubric_items`` ride in the call of the check that owns their family, at most +``MAX_ITEMS_PER_CALL`` to a call. Each row has a weight: 1 for every check but style +(``user_sim_judge_core.DEFAULT_ERROR_CHECKS``) and for an item that can make the run a user sim error (its +``can_error``, unset as its family's check), else 0. A weight-0 row is recorded and never gates. A rubric item is the shape of a criterion in a task's ``usersim.yaml`` (psi-benchmark-tasks), which that repo turns into this step's settings; this step reads no file: @@ -92,7 +100,8 @@ than 0, so ``score`` is 0 when ``user_sim_error`` is yes, else 1. ``can_be_reward`` is False: the score checks the run, so it is never the task's reward. - ``user_sim_error``: ``value`` yes / no / unknown, ``because`` and ``unknown`` (the row ids), ``fails_task``, and - ``not_failing`` (why a yes didn't fail the task). ``flag``: "user sim judge undecided" when ``value`` is unknown. + ``not_failing`` (why a yes didn't fail the task). ``flag`` when ``value`` is unknown: "user sim judge undecided", or + "the judge couldn't decide any check — check its setup" when every check is undecided. - ``run_step_id``, ``run_kind``, ``user_sim``, ``judge_model``; ``instructions`` (``source``, ``prompt_sha256``, ``recorded_prompt_sha256``, ``matches``); ``rubric_persona`` (when ``rubric_persona_sha256`` is set); ``notes`` (what the judge was told about gaps in the record); ``graded_turns``, ``turns``; ``calls``, ``usage``, ``cost``; and @@ -101,24 +110,31 @@ always_on record's or the prompt response's ``error_type``: every verifier skips its prompt anyway), a chat run whose user sim never read, and a run with no graded turn. -The gate, ``fail_task_when_not_passing`` (on by default; it needs ``fail_task_on_error``, also on by default, and is -refused without it): the step raises ``VerifierNotPassingError``, naming the failing rows, when ``user_sim_error`` is +The gate, ``fail_task_when_not_passing`` (on by default): the step raises ``VerifierNotPassingError``, naming the failing rows, when ``user_sim_error`` is yes and the rebuilt instructions match the recorded ``prompt_sha256``. The task then ends FAILED with no score, and the run is done again. Whoever caused the slip, the run is spoiled for grading the assistant, so ``assistant_asked`` never changes it. Off, the verdicts are recorded only. When the gate is on, every step that can be the task's reward (one with a ``verifier_id`` and ``can_be_reward``) must depend on this step, directly or through others, or the task is refused when it loads. -Unknown never fails the task: a run is failed only on a slip the judge found, so a flaky judge never throws away a good -run. A judge answer the format can't use is asked again with the format's correction, up to 3 more times -(``DEFAULT_MAX_RETRIES``, 4, counts the first ask; rubrics_verifier tries a failed model call as many times). A call -still without a usable answer leaves its rows ``error``, ``user_sim_error`` unknown and the entry flagged; the step -doesn't raise, and nothing the assistant is scored on changes. - -When the task loads it is refused when the run step can't be found (``rate_user_satisfaction.run_step_of``), when the -run's user isn't a user sim a deploy_user_sim_agent step deploys, when this step doesn't come after the run (its -``depends_on``, when set, must name it), when a reward step doesn't depend on it (above), and when it still sets -``mark_user_sim_error``, which ``fail_task_when_not_passing`` replaces. +A judge that can't reach its model fails the run at once, with ``JudgeSetupError`` ("the user sim judge can't reach +its model: ..."): agent-env's ``ConfigError`` (no key, missing config) and LiteLLM's ``AuthenticationError``, +``PermissionDeniedError``, ``NotFoundError`` (an unknown model), ``BadRequestError`` and its kinds, +``UnprocessableEntityError`` and ``BudgetExceededError`` (``_setup_error``). Every other failure is taken as flaky +(``Timeout``, ``RateLimitError``, ``ServiceUnavailableError``, ``InternalServerError``, ``BadGatewayError``, +``APIConnectionError``, an answer still unusable after its retries), and unknown never fails the task: a run is failed +only on a slip the judge found, so a flaky judge never throws away a good run. A judge answer the format can't use is +asked again with the format's correction, up to 3 more times (``DEFAULT_MAX_RETRIES``, 4, counts the first ask; +rubrics_verifier tries a failed model call as many times). A call still without a usable answer leaves its rows +``error``, ``user_sim_error`` unknown and the entry flagged; the step doesn't raise, and nothing the assistant is +scored on changes. + +When the task loads it is refused without ``run_step_id``, when that step isn't an always_on or prompt_agent step +(``rate_user_satisfaction.run_step_of``), when the run's user isn't a user sim a deploy_user_sim_agent step deploys, +when this step doesn't come after the run (its ``depends_on``, when set, must name it), when a reward step doesn't +depend on it (above), when ``fail_task_on_error`` is false, when it sets ``checks``, ``error_checks`` or a fixed +rubrics_verifier setting to another value, and when it still sets ``mark_user_sim_error``, which +``fail_task_when_not_passing`` replaces. """ from __future__ import annotations @@ -147,6 +163,17 @@ SYSTEM = ("You grade a user simulator: an AI playing a real person who messages their personal AI assistant. You " "grade the user sim, never the assistant.") FLAG_UNDECIDED = "user sim judge undecided" +FLAG_NONE_DECIDED = "the judge couldn't decide any check — check its setup" +# The judge's own settings: what a task may set on it, beyond every step's (id, type, version, depends_on ...). +SETTINGS = ("run_step_id", "rubric_items", "rubric_persona_sha256", "judge_model", "judge_timeout_seconds", + "fail_task_when_not_passing", "verifier_id") +# rubrics_verifier settings the judge fixes, with the value it fixes them at: a task may give only that value. +PINNED: dict[str, Any] = { + "use_trajectory": False, "use_agent_judge": False, "output_format": "user_sim_checks", "score_aggregator": "all_pass", + **dict.fromkeys(("criteria", "prompt_id", "default_model", "default_model_api_base", "agent_name", + "trajectory_filter", "max_thinking_tokens", "effort", "judge_a2a_agent_id", "judge_sandbox_type", + "grading_policy_prompt", "context_facts_for_judge", "grade_up_to_turn", "run_context_for_judge", + "judge_batch_size", "judge_batch_concurrency"))} NOT_FAILING_OFF = "fail_task_when_not_passing is off: verdicts are recorded only" NOT_FAILING_UNCONFIRMED = ("the judge's copy of the user sim's instructions isn't confirmed to be the one it ran with, " "so it doesn't fail the task") @@ -170,6 +197,7 @@ class _ThisRun: plan: core.Plan text: str # the run as the judge reads it first: Optional[asyncio.Event] = None # set once the run's first call is over + first_error: Optional[BaseException] = None # what the first call raised, if it did calls: int = 0 usage: list[Any] = field(default_factory=list) cost: float = 0.0 @@ -191,8 +219,6 @@ def __init__( version: Optional[int], run_step_id: Optional[str] = None, judge_model: Optional[str] = None, - checks: Optional[list[str]] = None, - error_checks: Optional[list[str]] = None, rubric_items: Optional[list[dict]] = None, rubric_persona_sha256: Optional[str] = None, fail_task_when_not_passing: bool = True, @@ -201,11 +227,15 @@ def __init__( depends_on: Optional[list[TaskStepDependency]] = None, fail_task_on_error: bool = True, ): - if fail_task_when_not_passing is True and not fail_task_on_error: - raise ValueError( - f"user_sim_judge '{id}': fail_task_when_not_passing (on by default) needs fail_task_on_error: a " - "tolerated failure lets the steps after it run and score the run. Set fail_task_on_error to true, or " - "fail_task_when_not_passing to false to record the verdicts only.") + """``run_step_id`` is required (it is a keyword with a default only so the refusal can say what's missing).""" + if not isinstance(run_step_id, str) or not run_step_id.strip(): + raise ValueError(f"user_sim_judge '{id}': run_step_id is required: name the always_on or prompt_agent " + "step whose user sim it judges") + if fail_task_on_error is not True: + raise ValueError(f"user_sim_judge '{id}': fail_task_on_error can't be false: a judge that fails has to " + "fail the task, or the steps after it would score a run nobody checked. Leave it out " + "(true is the default); set fail_task_when_not_passing to false to record the verdicts " + "only.") super().__init__( id, version, criteria=[], prompt_id=None, default_model=judge_model, verifier_id=verifier_id, use_agent_judge=False, use_trajectory=False, output_format=JudgeOutputFormat.USER_SIM_CHECKS, @@ -214,17 +244,14 @@ def __init__( self.fail_task_when_not_passing = fail_task_when_not_passing # as given, so config_problems sees a non-bool self.run_step_id = run_step_id self.judge_model = judge_model # None: rubrics_verifier's default, the config's judge model - self.checks = list(checks) if checks is not None else list(core.CHECK_NAMES) - self.error_checks = (list(error_checks) if error_checks is not None - else [c for c in core.DEFAULT_ERROR_CHECKS if c in self.checks]) self.rubric_items = list(rubric_items or []) self.rubric_persona_sha256 = rubric_persona_sha256 self._run: Optional[_ThisRun] = None # ---- settings ---------------------------------------------------------------------------------------------- def config_problems(self) -> list[str]: - return _problems(self.judge_model, self.checks, self.error_checks, self.rubric_items, - self.fail_task_when_not_passing, self.rubric_persona_sha256) + return _problems(self.judge_model, self.rubric_items, self.fail_task_when_not_passing, + self.rubric_persona_sha256) def task_problems(self, steps: list[TaskStep]) -> list[str]: """The run it judges must be an always_on or prompt_agent step's, played by a built-in user sim, and come @@ -255,8 +282,7 @@ def task_problems(self, steps: list[TaskStep]) -> list[str]: def to_dict(self) -> dict: base = TaskStep.to_dict(self) - base.update(run_step_id=self.run_step_id, judge_model=self.judge_model, checks=self.checks, - error_checks=self.error_checks, rubric_items=self.rubric_items, + base.update(run_step_id=self.run_step_id, judge_model=self.judge_model, rubric_items=self.rubric_items, rubric_persona_sha256=self.rubric_persona_sha256, fail_task_when_not_passing=self.fail_task_when_not_passing, verifier_id=self.verifier_id, judge_timeout_seconds=self.judge_timeout_seconds) @@ -264,15 +290,25 @@ def to_dict(self) -> dict: @classmethod def from_dict(cls, data: dict) -> "UserSimJudge": + where = f"user_sim_judge '{data.get('id')}'" if "mark_user_sim_error" in data: raise ValueError( - f"user_sim_judge '{data.get('id')}': mark_user_sim_error is gone; fail_task_when_not_passing (on by " - "default) takes its place: a run whose user sim slipped in a way that could change the grade fails " - "the task, so it is run again. Remove mark_user_sim_error, and set fail_task_when_not_passing to " - "false to record the verdicts only.") + f"{where}: mark_user_sim_error is gone; fail_task_when_not_passing (on by default) takes its place: " + "a run whose user sim slipped in a way that could change the grade fails the task, so it is run " + "again. Remove mark_user_sim_error, and set fail_task_when_not_passing to false to record the " + "verdicts only.") + if gone := [k for k in ("checks", "error_checks") if k in data]: + raise ValueError( + f"{where}: {' and '.join(gone)} can't be set: the judge always asks all seven checks, and every " + "check but style can fail a run. Remove them; a task item's can_error says whether it can.") + if pinned := [k for k, v in PINNED.items() if k in data and data[k] != v]: + raise ValueError( + f"{where}: {', '.join(pinned)} can't be set on the user sim judge, which fixes " + f"{'them' if len(pinned) > 1 else 'it'} ({', '.join(f'{k}={PINNED[k]!r}' for k in pinned)}). " + "Remove " + ("them" if len(pinned) > 1 else "it") + "; its settings are " + ", ".join(SETTINGS) + + ".") return cls(**cls._base_from_dict(data), run_step_id=data.get("run_step_id"), - judge_model=data.get("judge_model"), checks=data.get("checks"), - error_checks=data.get("error_checks"), rubric_items=data.get("rubric_items"), + judge_model=data.get("judge_model"), rubric_items=data.get("rubric_items"), rubric_persona_sha256=data.get("rubric_persona_sha256"), fail_task_when_not_passing=data.get("fail_task_when_not_passing", True), verifier_id=data.get("verifier_id"), judge_timeout_seconds=data.get("judge_timeout_seconds")) @@ -325,8 +361,8 @@ async def _grade(self, context: TaskStepContext) -> TaskStepContext: return context unusable = LINE_UNCHECKED if (entry.get("rubric_persona") or {}).get("status") == "unknown" else "" - plan = core.plan_run(run, checks=self.checks, items=items, error_checks=self.error_checks, - lines_unusable=unusable) + plan = core.plan_run(run, checks=list(core.CHECK_NAMES), items=items, + error_checks=list(core.DEFAULT_ERROR_CHECKS), lines_unusable=unusable) judge = copy.copy(self) # this run's own copy: the step is shared by every run of the task judge.prompt_id, judge._run = response.prompt_id, _ThisRun(plan=plan, text=core.run_text(run)) if plan.criteria: @@ -352,27 +388,24 @@ def _finish(self, written: dict, entry: dict, this: _ThisRun) -> None: user_sim_error=error, calls=this.calls, usage=core.usage_total(this.usage), cost=round(this.cost, 6)) if error["value"] == "unknown": - written["flag"] = FLAG_UNDECIDED + judged = [r for r in rows if r["id"] in core.CHECKS] + written["flag"] = FLAG_NONE_DECIDED if judged and all(r["verdict"] == "error" for r in judged) \ + else FLAG_UNDECIDED logger.info(f"user_sim_judge '{self.id}': {entry['run_kind']} '{entry['run_step_id']}' user_sim_error=" f"{error['value']} ({', '.join(error['because'] + error['unknown']) or 'no row'}), fails_task=" f"{error['fails_task']}; " + ", ".join(f"{r['id']}={r['verdict']}" for r in rows)) def _find_run(self, context: TaskStepContext) -> tuple[str, str]: - """The kind of run and its step id: ``run_step_id``'s, else the run's only always_on or chat run.""" + """The kind of run ``run_step_id`` left and its step id: always_on when it left an always_on record, else a + chat (a chat whose user sim never read leaves a response and no reads, and is skipped).""" records = context.metadata.get(ALWAYS_ON_KEY) or {} reads = context.metadata.get(READS_KEY) or {} - if self.run_step_id: - if self.run_step_id in records: - return ALWAYS_ON, self.run_step_id - if self.run_step_id in reads or _prompt_response(context, self.run_step_id) is not None: - return CHAT, self.run_step_id # a chat run whose user sim never read leaves a response, no reads - raise RuntimeError(f"user_sim_judge '{self.id}': step '{self.run_step_id}' left no run to judge (always_on " - f"records: {sorted(records)}; prompt_agent user sim reads: {sorted(reads)})") - runs = [(ALWAYS_ON, s) for s in records] + [(CHAT, s) for s in reads if s not in records] - if len(runs) != 1: - raise RuntimeError(f"user_sim_judge '{self.id}': set run_step_id; this run has {len(runs)} runs with a " - f"built-in user sim ({sorted(s for _, s in runs)})") - return runs[0] + if self.run_step_id in records: + return ALWAYS_ON, self.run_step_id + if self.run_step_id in reads or _prompt_response(context, self.run_step_id) is not None: + return CHAT, self.run_step_id + raise StepContextError(f"user_sim_judge '{self.id}': step '{self.run_step_id}' left no run to judge " + f"(always_on records: {sorted(records)}; prompt_agent user sim reads: {sorted(reads)})") # ---- rubrics_verifier's hooks, on the run's own copy --------------------------------------------------------- def _this(self) -> _ThisRun: @@ -419,9 +452,14 @@ async def call() -> dict: this.first = asyncio.Event() try: return await call() + except Exception as e: + this.first_error = e + raise finally: this.first.set() await this.first.wait() + if this.first_error is not None and _setup_error(this.first_error) is not None: + raise RuntimeError("the run's first judge call couldn't reach its model") from this.first_error return await call() async def _run_judge_with_output_retries(self, *, criteria: list[dict], **kwargs: Any @@ -431,14 +469,51 @@ async def _run_judge_with_output_retries(self, *, criteria: list[dict], **kwargs try: return await super()._run_judge_with_output_retries(criteria=criteria, **kwargs) except (ValueError, RuntimeError) as e: - error = (f"no usable answer after {self.DEFAULT_MAX_RETRIES} tries: {type(e).__name__}: " - f"{str(e)[:ERROR_SHOWN_CHARS]}") + if (setup := _setup_error(e)) is not None: + raise JudgeSetupError(f"the user sim judge can't reach its model: {type(setup).__name__}: " + f"{str(setup)[:ERROR_SHOWN_CHARS]}") from e + cause = _root(e) + error = (f"no usable answer after {self.DEFAULT_MAX_RETRIES} tries: {type(cause).__name__}: " + f"{str(cause)[:ERROR_SHOWN_CHARS]}") logger.warning(f"user_sim_judge '{self.id}': {len(criteria)} criteria undecided, {error}") return [core.undecided_row(c, error) for c in criteria], 0, [], None -def _problems(model: Any, checks: Any, error_checks: Any, items: Any, gate: Any, - rubric_persona: Any = None) -> list[str]: +class JudgeSetupError(RuntimeError): + """The judge can't reach its model as it is set up (no key, a bad key, a model that doesn't exist, a request the + model refuses): the run fails at once, rather than every check going unknown.""" + + +def _chain(e: BaseException) -> list[BaseException]: + out: list[BaseException] = [] + while e is not None and e not in out: + out.append(e) + e = e.__cause__ or e.__context__ + return out + + +def _root(e: BaseException) -> BaseException: + """What the judge call itself raised: rubrics_verifier wraps a model call that kept failing in a RuntimeError.""" + return _chain(e)[-1] + + +def _setup_error(e: BaseException) -> Optional[BaseException]: + """The setup error behind a failed judge call, or None for a flaky one. Setup: agent-env's ConfigError (no key, + missing config) and LiteLLM's AuthenticationError, PermissionDeniedError, NotFoundError (an unknown model), + BadRequestError (with its kinds: an unknown provider, unsupported params, a prompt over the context window), + UnprocessableEntityError and BudgetExceededError. Anything else is flaky (Timeout, RateLimitError, + ServiceUnavailableError, InternalServerError, BadGatewayError, APIConnectionError, an answer still unusable after + its retries) and leaves the call's rows undecided.""" + import litellm + + from agent_env.config.errors import ConfigError + + setup = (ConfigError, litellm.AuthenticationError, litellm.PermissionDeniedError, litellm.NotFoundError, + litellm.BadRequestError, litellm.UnprocessableEntityError, litellm.BudgetExceededError) + return next((x for x in _chain(e) if isinstance(x, setup)), None) + + +def _problems(model: Any, items: Any, gate: Any, rubric_persona: Any = None) -> list[str]: out = [] if rubric_persona is not None and not (isinstance(rubric_persona, str) and re.fullmatch(r"[0-9a-f]{64}", rubric_persona)): @@ -448,23 +523,6 @@ def _problems(model: Any, checks: Any, error_checks: Any, items: Any, gate: Any, out.append("judge_model must be a model name") if not isinstance(gate, bool): out.append("fail_task_when_not_passing must be true or false") - if not isinstance(checks, list) or not all(isinstance(c, str) for c in checks): - out.append(f"checks must be a list from {list(core.CHECK_NAMES)}") - else: - unknown = sorted(set(checks) - set(core.CHECK_NAMES)) - if unknown: - out.append(f"checks has unknown checks {unknown}; the checks are {list(core.CHECK_NAMES)}") - if not checks and not items: - out.append("checks is empty and there are no rubric_items: nothing to judge") - if not isinstance(error_checks, list) or not all(isinstance(c, str) for c in error_checks): - out.append("error_checks must be a list of check names") - else: - if isinstance(checks, list): - not_run = sorted(set(error_checks) - set(checks)) - if not_run: - out.append(f"error_checks names checks that don't run: {not_run}") - if "style" in error_checks: - out.append("error_checks can't name style: its rows are recorded only (weight 0), never a user sim error") if not isinstance(items, list): out.append("rubric_items must be a list of {id, family, kind, check, ...}") else: diff --git a/tst/unit/task_step/verifiers/test_user_sim_judge.py b/tst/unit/task_step/verifiers/test_user_sim_judge.py index 850256b57..4bad2865e 100644 --- a/tst/unit/task_step/verifiers/test_user_sim_judge.py +++ b/tst/unit/task_step/verifiers/test_user_sim_judge.py @@ -9,6 +9,7 @@ import json from dataclasses import replace +import httpx import litellm import pytest @@ -16,7 +17,7 @@ from agent_env.openenv.reward import reward_of from agent_env.store import LocalSecretStore from agent_env.task import Task -from agent_env.task_step.context import PromptResponse, TaskStepContext, VerifierNotPassingError +from agent_env.task_step.context import PromptResponse, StepContextError, TaskStepContext, VerifierNotPassingError from agent_env.task_step.registry import _builtin_registry from agent_env.task_step.task_step import TaskStepDependency from agent_env.task_step.task_steps import always_on as ao @@ -112,7 +113,8 @@ def run_text(self) -> str: def _step(fake: FakeModel, monkeypatch, **kw) -> usj.UserSimJudge: monkeypatch.setattr(RubricsVerifierTaskStep, "_prompt_llm_judge", fake) - return usj.UserSimJudge("judge_user_sim", None, **{"judge_model": JUDGE, "verifier_id": "usj", **kw}) + return usj.UserSimJudge("judge_user_sim", None, **{"run_step_id": "talk", "judge_model": JUDGE, "verifier_id": "usj", + **kw}) def _entry(ctx: TaskStepContext) -> dict: @@ -123,8 +125,9 @@ def _rows(entry: dict) -> dict[str, dict]: return {r["id"]: r for r in entry["results"]} -def _verdicts(entry: dict) -> dict[str, str]: - return {r["id"]: r["verdict"] for r in entry["results"]} +def _verdicts(entry: dict, *ids: str) -> dict[str, str]: + """Each row's verdict, or only those of ``ids``.""" + return {r["id"]: r["verdict"] for r in entry["results"] if not ids or r["id"] in ids} def _run(*answers: tuple[str, str], graded=None) -> core.UserSimRun: @@ -236,7 +239,7 @@ def test_a_fail_that_clears_one_turn_and_fails_another_is_kept(): @pytest.mark.asyncio async def test_the_judge_writes_its_reason_before_its_verdict_within_the_word_limit(monkeypatch): fake = FakeModel() - await _judge(_run(("reply", "x")), fake, monkeypatch, checks=["rogue"]) + await _judge(_run(("reply", "x")), fake, monkeypatch) prompt = fake.calls[0]["prompt"] assert "Reason first and decide last." in prompt and "at most 40 words" in prompt assert prompt.index("# THE USER SIM'S INSTRUCTIONS") < prompt.index("# THE CRITERIA") # the run first @@ -291,16 +294,16 @@ async def test_a_failure_that_can_t_change_the_grade_doesn_t_make_a_user_sim_err @pytest.mark.asyncio async def test_a_bad_answer_is_asked_again_and_a_check_whose_call_fails_is_undecided(monkeypatch): fake = FakeModel({"guidance": ["not json", FAIL()], "rogue": [RuntimeError("503")]}) - entry = await _judge(_run(("reply", "you pick")), fake, monkeypatch, checks=["guidance", "rogue"]) - assert _verdicts(entry) == {"guidance": "fail", "rogue": "error"} and entry["judge_output_retries"] == 1 + entry = await _judge(_run(("reply", "you pick")), fake, monkeypatch) + assert _verdicts(entry, "guidance", "rogue") == {"guidance": "fail", "rogue": "error"} + assert entry["judge_output_retries"] == 1 retry = fake.asked("guidance")[1]["prompt"] assert "## Correction Required" in retry and "the answer is not a JSON object" in retry assert _rows(entry)["rogue"]["error"] == "no usable answer after 4 tries: RuntimeError: 503" assert (_rows(entry)["rogue"]["score"], _rows(entry)["rogue"]["assistant_asked"]) == (1.0, None) assert entry["user_sim_error"]["value"] == "yes" assert (entry["user_sim_error"]["because"], entry["user_sim_error"]["unknown"]) == (["guidance"], ["rogue"]) - alone = await _judge(_run(("reply", "x")), FakeModel({"rogue": [RuntimeError("503")]}), monkeypatch, - checks=["rogue"]) + alone = await _judge(_run(("reply", "x")), FakeModel({"rogue": [RuntimeError("503")]}), monkeypatch) assert (alone["user_sim_error"]["value"], alone["user_sim_error"]["unknown"]) == ("unknown", ["rogue"]) assert alone["flag"] == "user sim judge undecided" and alone["score"] == 1.0 @@ -310,10 +313,10 @@ async def test_a_broken_answer_is_asked_again_up_to_3_times_then_the_run_is_flag monkeypatch): assert usj.UserSimJudge.DEFAULT_MAX_RETRIES == 4 # the first ask and 3 corrections fake = FakeModel({"made_up": ["not json"] * 5}) - entry = await _judge(_run(("reply", "x")), fake, monkeypatch, checks=["made_up"], fail_task_when_not_passing=True) + entry = await _judge(_run(("reply", "x")), fake, monkeypatch, fail_task_when_not_passing=True) assert len(fake.asked("made_up")) == 4 assert all("## Correction Required" in c["prompt"] for c in fake.asked("made_up")[1:]) - assert _verdicts(entry) == {"made_up": "error"} and _rows(entry)["made_up"]["error"].startswith( + assert _verdicts(entry, "made_up") == {"made_up": "error"} and _rows(entry)["made_up"]["error"].startswith( "no usable answer after 4 tries: ValueError: ") assert entry["user_sim_error"] == {"value": "unknown", "because": [], "unknown": ["made_up"], "fails_task": False} assert entry["flag"] == "user sim judge undecided" # and execute didn't raise @@ -322,13 +325,12 @@ async def test_a_broken_answer_is_asked_again_up_to_3_times_then_the_run_is_flag @pytest.mark.asyncio async def test_a_fail_citing_only_ungraded_turns_is_asked_again(monkeypatch): fake = FakeModel({"made_up": [FAIL(turns=[99]), FAIL(turns=[1])]}) - entry = await _judge(_run(("reply", "dad's allergic to shellfish")), fake, monkeypatch, checks=["made_up"]) + entry = await _judge(_run(("reply", "dad's allergic to shellfish")), fake, monkeypatch) assert _rows(entry)["made_up"]["verdict"] == "fail" and _rows(entry)["made_up"]["turns"] == [1] assert entry["judge_output_retries"] == 1 and entry["user_sim_error"]["value"] == "yes" - assert "a fail must cite at least one graded turn; cited [99], graded [1]" in fake.calls[1]["prompt"] - never = await _judge(_run(("reply", "x")), FakeModel({"made_up": [FAIL(turns=[99])] * 4}), monkeypatch, - checks=["made_up"]) - assert _verdicts(never) == {"made_up": "error"} and "cited [99]" in _rows(never)["made_up"]["error"] + assert "a fail must cite at least one graded turn; cited [99], graded [1]" in fake.asked("made_up")[1]["prompt"] + never = await _judge(_run(("reply", "x")), FakeModel({"made_up": [FAIL(turns=[99])] * 4}), monkeypatch) + assert _verdicts(never, "made_up") == {"made_up": "error"} and "cited [99]" in _rows(never)["made_up"]["error"] assert never["user_sim_error"]["value"] == "unknown" @@ -341,8 +343,7 @@ async def test_a_fail_citing_only_ungraded_turns_is_asked_again(monkeypatch): async def test_a_fail_whose_reason_says_no_clear_failure_found_is_asked_again(monkeypatch): fake = FakeModel({"guidance": [FAIL(reason=SAYS_NO_FAILURE[0]), PASS(reason="No clear failure found.")], "oversharing": FAIL()}) - entry = await _judge(_run(("reply", "dad's allergic to shellfish fyi")), fake, monkeypatch, - checks=["guidance", "oversharing"]) + entry = await _judge(_run(("reply", "dad's allergic to shellfish fyi")), fake, monkeypatch) assert (_rows(entry)["guidance"]["verdict"], _rows(entry)["guidance"]["reason"]) == ( "pass", "No clear failure found.") retry = fake.asked("guidance")[1]["prompt"] @@ -350,21 +351,20 @@ async def test_a_fail_whose_reason_says_no_clear_failure_found_is_asked_again(mo # The ordinary fail is answered once and alone makes the run a user sim error. assert len(fake.asked("oversharing")) == 1 and entry["user_sim_error"]["because"] == ["oversharing"] for reason in SAYS_NO_FAILURE: - never = await _judge(_run(("reply", "x")), FakeModel({"delegation": [FAIL(reason=reason)] * 4}), monkeypatch, - checks=["delegation"]) - assert _verdicts(never) == {"delegation": "error"} and never["user_sim_error"]["value"] == "unknown" + never = await _judge(_run(("reply", "x")), FakeModel({"delegation": [FAIL(reason=reason)] * 4}), monkeypatch) + assert _verdicts(never, "delegation") == {"delegation": "error"} and never["user_sim_error"]["value"] == "unknown" @pytest.mark.asyncio async def test_a_task_item_fail_citing_no_graded_turn_is_asked_again(monkeypatch): items = [{"id": "rating", "family": "A5", "check": "rates the trip 2 or lower", "can_error": True}] fake = FakeModel({"rating": [FAIL(turns=[99]), FAIL(turns=[1])]}) - entry = await _judge(_run(("done", "5 stars")), fake, monkeypatch, checks=[], rubric_items=items) + entry = await _judge(_run(("done", "5 stars")), fake, monkeypatch, rubric_items=items) assert _rows(entry)["rating"]["verdict"] == "fail" and _rows(entry)["rating"]["turns"] == [1] - assert entry["user_sim_error"]["because"] == ["rating"] and len(fake.calls) == 2 + assert entry["user_sim_error"]["because"] == ["rating"] and len(fake.asked("rating")) == 2 never = await _judge(_run(("done", "5 stars")), FakeModel({"rating": [FAIL(turns=[])] * 4}), monkeypatch, - checks=[], rubric_items=items) - assert _verdicts(never) == {"rating": "error"} and "a fail must cite" in _rows(never)["rating"]["error"] + rubric_items=items) + assert _verdicts(never, "rating") == {"rating": "error"} and "a fail must cite" in _rows(never)["rating"]["error"] assert (never["user_sim_error"]["value"], never["user_sim_error"]["unknown"]) == ("unknown", ["rating"]) @@ -386,8 +386,8 @@ async def test_task_items_get_their_own_rows_and_one_rule_says_which_make_a_user assert [r["id"] for r in entry["results"]] == [*core.CHECK_NAMES, "leak", "rating", "tone"] # The rule is the checks' rule: an item's fail that couldn't change the grade is no user sim error. mild = await _judge(_run(("done", "thx bye")), FakeModel({"rating": FAIL(change=False)}), monkeypatch, - checks=[], rubric_items=items[1:2]) - assert _verdicts(mild) == {"rating": "fail"} and mild["user_sim_error"]["value"] == "no" + rubric_items=items[1:2]) + assert _verdicts(mild, "rating") == {"rating": "fail"} and mild["user_sim_error"]["value"] == "no" @pytest.mark.asyncio @@ -398,9 +398,8 @@ async def test_the_ending_check_asks_when_it_signed_off_and_a_grade_changing_fai assert "ending" in core.DEFAULT_ERROR_CHECKS for answer, verdict, error in ((FAIL(turns=[1]), "fail", "yes"), (PASS(), "pass", "no"), (NA(reason="never said done"), "n/a", "no")): - entry = await _judge(_run(("done", "ok thx bye")), FakeModel({"ending": answer}), monkeypatch, - checks=["ending"]) - assert _verdicts(entry) == {"ending": verdict} and _rows(entry)["ending"]["families"] == ["G4"] + entry = await _judge(_run(("done", "ok thx bye")), FakeModel({"ending": answer}), monkeypatch) + assert _verdicts(entry, "ending") == {"ending": verdict} and _rows(entry)["ending"]["families"] == ["G4"] assert entry["user_sim_error"]["value"] == error @@ -443,8 +442,7 @@ async def test_style_never_gates(monkeypatch): FakeModel({"style": FAIL(change=True)}), monkeypatch, fail_task_when_not_passing=True) assert _rows(entry)["style"]["verdict"] == "fail" and _rows(entry)["style"]["could_change_grade"] is True assert entry["user_sim_error"]["value"] == "no" and entry["score"] == 1.0 - [problem] = usj.UserSimJudge("s", None, error_checks=["guidance", "style"]).config_problems() - assert problem == "error_checks can't name style: its rows are recorded only (weight 0), never a user sim error" + assert _rows(entry)["style"]["weight"] == 0.0 and "style" not in core.DEFAULT_ERROR_CHECKS @pytest.mark.asyncio @@ -509,7 +507,7 @@ async def test_on_an_always_on_run_the_judge_reads_what_the_user_sim_got_and_sai fake = FakeModel({"guidance": FAIL(turns=[2])}) with pytest.raises(VerifierNotPassingError, match=r"\(guidance\)"): - await _step(fake, monkeypatch).execute(ctx) + await _step(fake, monkeypatch, run_step_id="week").execute(ctx) run_text = fake.run_text() assert "\n" + prompt.strip() + "\n" in run_text # the very prompt it ran with @@ -538,7 +536,7 @@ async def test_without_the_stored_timeline_the_judge_reads_the_tail_and_is_told_ record["timeline_tail"] = [{k: v for k, v in e.items() if k not in ("input", "read")} for e in timeline] fake = FakeModel() - await _step(fake, monkeypatch, checks=["made_up"]).execute(ctx) + await _step(fake, monkeypatch, run_step_id="week").execute(ctx) head = fake.calls[0]["cached_prefix"] assert "Note: the run's full record wasn't stored" in head and "### What it was sent\n(not recorded)" in head @@ -566,7 +564,7 @@ async def test_a_run_that_ended_in_an_error_is_not_judged(monkeypatch, tmp_path) rig.record(ctx)["error_type"] = "agent_turn_failed" fake = FakeModel() - await _step(fake, monkeypatch).execute(ctx) + await _step(fake, monkeypatch, run_step_id="week").execute(ctx) assert fake.calls == [] and _entry(ctx)["skipped"] == "the run ended with error_type=agent_turn_failed" assert _entry(ctx)["user_sim_error"] == {"value": "no", "because": [], "unknown": [], "fails_task": False} @@ -580,11 +578,10 @@ async def test_a_run_s_judge_model_override_chooses_the_model_as_for_every_judge ctx.metadata["user_overrides"] = {"judge_model": "other"} fake = FakeModel() - await _step(fake, monkeypatch, checks=["rogue"]).execute(ctx) + await _step(fake, monkeypatch, run_step_id="week").execute(ctx) - assert {(n, c["model"]) for c in fake.calls for n in c["names"]} == {("rogue", "other")} + assert {c["model"] for c in fake.calls} == {"other"} and len(fake.calls) == 7 assert _entry(ctx)["judge_model"] == "other" and ctx.metadata["models"]["judge"]["judge_user_sim"] == "other" - assert list(_rows(_entry(ctx))) == ["rogue"] # ---- the step on a prompt_agent chat run -------------------------------------------------------------------------- @@ -603,7 +600,7 @@ async def test_on_a_chat_run_the_judge_reads_each_stored_read_and_the_prompt_it_ us = ctx.metadata[USER_SIMS_KEY]["hannah"] fake = FakeModel({"style": FAIL(turns=[2], change=False)}) - await _step(fake, monkeypatch, run_step_id="talk").execute(ctx) + await _step(fake, monkeypatch).execute(ctx) run_text = fake.run_text() shown = run_text.split("\n")[1].split("\n")[0] + "\n" @@ -671,18 +668,20 @@ async def test_a_chat_run_that_ended_in_an_error_or_where_the_user_sim_never_rea ctx.prompt_responses[-1].error_type = None ctx.metadata.pop("user_sim_reads") - await _step(fake, monkeypatch, run_step_id="talk").execute(ctx) + await _step(fake, monkeypatch).execute(ctx) assert fake.calls == [] and _entry(ctx)["skipped"] == "the user sim never read: the run has no user_sim_reads" -def test_without_run_step_id_the_run_must_have_one_run_to_judge(): +def test_the_run_is_the_one_run_step_id_names(): ctx = TaskStepContext() ctx.metadata = {"always_on": {"week": {}}, "user_sim_reads": {"talk": [{}]}} - with pytest.raises(RuntimeError, match=r"set run_step_id; this run has 2 runs .*\['talk', 'week'\]"): - usj.UserSimJudge("j", None)._find_run(ctx) - with pytest.raises(RuntimeError, match="step 'nope' left no run to judge"): + with pytest.raises(StepContextError, match="step 'nope' left no run to judge"): usj.UserSimJudge("j", None, run_step_id="nope")._find_run(ctx) assert usj.UserSimJudge("j", None, run_step_id="talk")._find_run(ctx) == ("prompt_agent", "talk") + assert usj.UserSimJudge("j", None, run_step_id="week")._find_run(ctx) == ("always_on", "week") + # A one-turn chat ends before its user sim reads: it leaves a response and no reads, and is skipped, not refused. + ctx.prompt_responses.append(PromptResponse(prompt_id="ask1", response="done", step_id="one-turn")) + assert usj.UserSimJudge("j", None, run_step_id="one-turn")._find_run(ctx) == ("prompt_agent", "one-turn") # ---- the judge's calls, through litellm ----------------------------------------------------------------------------- @@ -725,7 +724,7 @@ async def _judge_through_litellm(monkeypatch, run, **kw) -> dict: ctx = TaskStepContext() ctx.prompt_responses.append(PromptResponse(prompt_id="ask", response="all set", step_id="talk")) ctx.metadata["user_sim_reads"] = {"talk": [{}]} - await usj.UserSimJudge("judge_user_sim", None, verifier_id="usj", **kw).execute(ctx) + await usj.UserSimJudge("judge_user_sim", None, run_step_id="talk", verifier_id="usj", **kw).execute(ctx) return _entry(ctx) @@ -735,10 +734,9 @@ async def test_a_claude_judge_gets_the_system_message_and_the_run_marked_for_the calls = _litellm_answers(monkeypatch, _all_pass) run = _run(("reply", "ok")) - entry = await _judge_through_litellm(monkeypatch, run, judge_model="anthropic/claude-sonnet-4-6", - checks=["guidance", "rogue"]) + entry = await _judge_through_litellm(monkeypatch, run, judge_model="anthropic/claude-sonnet-4-6") - assert len(calls) == 2 and entry["user_sim_error"]["value"] == "no" and entry["calls"] == 2 + assert len(calls) == 7 and entry["user_sim_error"]["value"] == "no" and entry["calls"] == 7 for call in calls: system, user = call["messages"] assert system == {"role": "system", "content": usj.SYSTEM} @@ -750,18 +748,78 @@ async def test_a_claude_judge_gets_the_system_message_and_the_run_marked_for_the assert entry["usage"]["prompt_tokens"] > 0 and entry["cost"] > 0 +_GPT = dict(llm_provider="openai", model="openai/gpt-4o") +_RESPONSE = httpx.Response(400, request=httpx.Request("POST", "https://judge.invalid/v1/chat/completions")) +FLAKY = { + "timeout": litellm.Timeout("timed out", **_GPT), + "rate limit": litellm.RateLimitError("slow down", **_GPT), + "service unavailable": litellm.ServiceUnavailableError("503", **_GPT), + "internal server error": litellm.InternalServerError("500", **_GPT), + "bad gateway": litellm.BadGatewayError("502", **_GPT), + "connection": litellm.APIConnectionError("connection reset", **_GPT), +} +SETUP = { + "auth": litellm.AuthenticationError("invalid api key", **_GPT), + "permission": litellm.PermissionDeniedError("no access to this model", response=_RESPONSE, **_GPT), + "unknown model": litellm.NotFoundError("model not found", **_GPT), + "bad request": litellm.BadRequestError("LLM Provider NOT provided", **_GPT), + "unprocessable": litellm.UnprocessableEntityError("unprocessable", response=_RESPONSE, **_GPT), + "budget": litellm.BudgetExceededError(current_cost=10.0, max_budget=5.0), +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("error", list(FLAKY.values()), ids=list(FLAKY)) +async def test_a_flaky_model_call_is_tried_as_rubrics_verifier_tries_it_and_left_undecided(monkeypatch, litellm_config, + error): + calls = _litellm_answers(monkeypatch, lambda kwargs: error) + + entry = await _judge_through_litellm(monkeypatch, _run(("reply", "ok")), judge_model="openai/gpt-4o") + + assert len(calls) == 7 * usj.UserSimJudge.DEFAULT_MAX_RETRIES # every check, each call tried 4 times + assert set(_verdicts(entry).values()) == {"error"} + assert _rows(entry)["rogue"]["error"].startswith(f"no usable answer after 4 tries: {type(error).__name__}: ") + # Nothing decided at all: the flag says to look at the judge's setup, and still nothing fails. + assert entry["flag"] == "the judge couldn't decide any check \u2014 check its setup" + assert entry["user_sim_error"]["value"] == "unknown" and entry["user_sim_error"]["fails_task"] is False + + +@pytest.mark.asyncio +async def test_one_undecided_check_among_decided_ones_is_flagged_undecided(monkeypatch, litellm_config): + def answer(kwargs): + content = kwargs["messages"][-1]["content"] + prompt = "".join(b["text"] for b in content) if isinstance(content, list) else content + rogue = any(c.get("check") == core.check_title("rogue") for c in _criteria_of(prompt)) + return litellm.RateLimitError("slow down", **_GPT) if rogue else _all_pass(kwargs) + + _litellm_answers(monkeypatch, answer) + entry = await _judge_through_litellm(monkeypatch, _run(("reply", "ok")), judge_model="openai/gpt-4o") + assert _verdicts(entry, "rogue", "guidance") == {"rogue": "error", "guidance": "pass"} + assert entry["flag"] == "user sim judge undecided" + + @pytest.mark.asyncio -async def test_a_model_call_that_keeps_failing_is_tried_as_rubrics_verifier_tries_it_and_left_undecided( - monkeypatch, litellm_config): - calls = _litellm_answers(monkeypatch, lambda kwargs: RuntimeError("503 from the judge")) +@pytest.mark.parametrize("error", list(SETUP.values()), ids=list(SETUP)) +async def test_a_judge_that_can_t_reach_its_model_fails_the_run_at_once(monkeypatch, litellm_config, error): + calls = _litellm_answers(monkeypatch, lambda kwargs: error) - entry = await _judge_through_litellm(monkeypatch, _run(("reply", "ok")), judge_model="openai/gpt-4o", - checks=["rogue"]) + with pytest.raises(usj.JudgeSetupError, match=f"^the user sim judge can't reach its model: {type(error).__name__}: "): + await _judge_through_litellm(monkeypatch, _run(("reply", "ok")), judge_model="openai/gpt-4o") - assert len(calls) == usj.UserSimJudge.DEFAULT_MAX_RETRIES - assert _verdicts(entry) == {"rogue": "error"} and "LLM judge failed after 4 attempts" in _rows(entry)["rogue"][ - "error"] - assert entry["flag"] == "user sim judge undecided" and entry["user_sim_error"]["fails_task"] is False + assert len(calls) == usj.UserSimJudge.DEFAULT_MAX_RETRIES # only the run's first call was made + + +@pytest.mark.asyncio +async def test_a_judge_with_no_key_or_config_fails_the_run_at_once(monkeypatch, litellm_config): + from agent_env.config import get_config + from agent_env.config.errors import ConfigError + + def missing(*args, **kwargs): + raise ConfigError("no LiteLLM key is configured for the judge") + + monkeypatch.setattr(get_config(), "resolve_model_call", missing) + with pytest.raises(usj.JudgeSetupError, match="can't reach its model: ConfigError: no LiteLLM key is configured"): + await _judge_through_litellm(monkeypatch, _run(("reply", "ok")), judge_model="openai/gpt-4o") # ---- the instructions, rebuilt as deploy_user_sim_agent wrote them --------------------------------------------------- @@ -817,34 +875,65 @@ async def test_a_text_only_opening_rebuilds_as_before(monkeypatch): # ---- settings ----------------------------------------------------------------------------------------------------- def test_settings_are_checked_and_round_trip(): assert _builtin_registry()["user_sim_judge"] is usj.UserSimJudge - step = usj.UserSimJudge("s", None, run_step_id="week", judge_model=JUDGE, checks=["guidance", "style"], - error_checks=["guidance"], fail_task_when_not_passing=False, + step = usj.UserSimJudge("s", None, run_step_id="week", judge_model=JUDGE, fail_task_when_not_passing=False, + judge_timeout_seconds=300, rubric_items=[{"id": "leak", "family": "G1", "text": "never mentions the ring"}]) assert step.config_problems() == [] again = usj.UserSimJudge.from_dict(json.loads(json.dumps(step.to_dict()))) assert again.to_dict() == step.to_dict() - assert {"prompt_id", "criteria", "output_format", "use_trajectory", "default_model"}.isdisjoint(step.to_dict()) - problems = usj.UserSimJudge("s", None, checks=["guidance", "vibes"], error_checks=["rogue"], judge_model=" ", - fail_task_when_not_passing="yes", + own = set(step.to_dict()) - {"id", "type", "version", "fail_task_on_error"} + assert own == set(usj.SETTINGS) == {"run_step_id", "rubric_items", "rubric_persona_sha256", "judge_model", + "judge_timeout_seconds", "fail_task_when_not_passing", "verifier_id"} + problems = usj.UserSimJudge("s", None, run_step_id="week", judge_model=" ", fail_task_when_not_passing="yes", rubric_items=[{"id": "x", "family": "Q1", "text": "y"}, {"id": "rogue", "family": "G3", "check": "stays the user"}] ).config_problems() - assert any("unknown checks ['vibes']" in p for p in problems) - assert any("error_checks names checks that don't run: ['rogue']" in p for p in problems) assert any("judge_model must be a model name" in p for p in problems) and any("family 'Q1'" in p for p in problems) assert any("fail_task_when_not_passing must be true or false" in p for p in problems) assert "rubric_items[1] (rogue): id 'rogue' is a check's name; give the item another id" in problems - assert usj.UserSimJudge("s", None, checks=["style"]).error_checks == [] # the error checks that run @pytest.mark.asyncio async def test_settings_with_problems_stop_the_step_before_it_judges(monkeypatch): - with pytest.raises(ValueError, match=r"user_sim_judge 'judge_user_sim': checks has unknown checks \['vibes'\]"): - await _judge(_run(("reply", "x")), FakeModel(), monkeypatch, checks=["vibes"]) + with pytest.raises(ValueError, match=r"user_sim_judge 'judge_user_sim': judge_model must be a model name"): + await _judge(_run(("reply", "x")), FakeModel(), monkeypatch, judge_model=" ") + + +STORED = {"id": "s", "type": "user_sim_judge", "run_step_id": "week"} + + +def test_a_task_without_run_step_id_is_refused_when_it_loads(): + for missing in ({"id": "s", "type": "user_sim_judge"}, {**STORED, "run_step_id": None}, {**STORED, "run_step_id": " "}): + with pytest.raises(ValueError, match="user_sim_judge 's': run_step_id is required: name the always_on or " + "prompt_agent step whose user sim it judges"): + usj.UserSimJudge.from_dict(missing) + + +def test_checks_and_error_checks_are_refused_when_the_task_loads(): + for given, named in (({"checks": ["guidance"]}, "checks"), ({"error_checks": []}, "error_checks"), + ({"checks": list(core.CHECK_NAMES), "error_checks": ["rogue"]}, "checks and error_checks")): + with pytest.raises(ValueError, match=f"user_sim_judge 's': {named} can't be set: the judge always asks all " + "seven checks, and every check but style can fail a run"): + usj.UserSimJudge.from_dict({**STORED, **given}) + + +def test_a_rubrics_verifier_setting_the_judge_fixes_is_refused_unless_it_has_the_fixed_value(): + for key, value in (("use_trajectory", True), ("output_format", "rubric_binary"), ("judge_batch_size", 3), + ("use_agent_judge", True), ("criteria", [{"id": "c1"}]), ("prompt_id", "ask"), + ("default_model", "x"), ("score_aggregator", "weighted_average"), ("grade_up_to_turn", 2)): + with pytest.raises(ValueError, match=f"user_sim_judge 's': {key} can't be set on the user sim judge, which " + f"fixes it \\({key}="): + usj.UserSimJudge.from_dict({**STORED, key: value}) + with pytest.raises(ValueError, match="use_trajectory, judge_batch_size can't be set .* fixes them .*Remove them; " + "its settings are run_step_id, rubric_items"): + usj.UserSimJudge.from_dict({**STORED, "use_trajectory": True, "judge_batch_size": 3}) + same = usj.UserSimJudge.from_dict({**STORED, **{k: usj.PINNED[k] for k in ( + "use_trajectory", "use_agent_judge", "output_format", "judge_batch_size", "criteria")}}) + assert same.config_problems() == [] # the fixed value itself is fine def test_by_default_it_fails_the_task_on_a_slip_and_judges_with_the_verifiers_default_model(monkeypatch): - step = usj.UserSimJudge.from_dict({"id": "s", "type": "user_sim_judge"}) + step = usj.UserSimJudge.from_dict(STORED) assert (step.fail_task_when_not_passing, step.fail_task_on_error, step.judge_model) == (True, True, None) assert (step.use_trajectory, step.use_agent_judge, step.output_format) == ( False, False, JudgeOutputFormat.USER_SIM_CHECKS) @@ -859,9 +948,9 @@ def get_model_for_role(self, role): fake = Config() monkeypatch.setattr("agent_env.task_step.task_steps.verifiers.rubrics_verifier.get_config", lambda: fake) - assert usj.UserSimJudge("s", None).default_model == "claude-sonnet-4-6" + assert usj.UserSimJudge("s", None, run_step_id="week").default_model == "claude-sonnet-4-6" fake.role = "anthropic/claude-opus-4-8" - assert usj.UserSimJudge("s", None).default_model == "anthropic/claude-opus-4-8" + assert usj.UserSimJudge("s", None, run_step_id="week").default_model == "anthropic/claude-opus-4-8" # ---- the load checks ---------------------------------------------------------------------------------------------- @@ -878,14 +967,13 @@ def _grader(**kw) -> RubricsVerifierTaskStep: criteria=[{"id": "c1", "description": "booked it"}], **kw) -def test_the_task_must_say_which_run_and_its_user_must_be_a_built_in_user_sim(): +def test_its_run_must_be_a_run_step_the_judge_follows_and_its_user_a_built_in_user_sim(): deploy, week = _user_sim_steps() - judge = usj.UserSimJudge("j", None) + judge = usj.UserSimJudge("j", None, run_step_id="week") assert judge.task_problems([deploy, week, judge]) == [] - talk = _chat_step() - [two] = judge.task_problems([deploy, week, talk, judge]) - assert two == ("user_sim_judge 'j': the task has several always_on and prompt_agent steps (week, talk); set " - "run_step_id to the one to judge") + assert judge.task_problems([deploy, week, _chat_step(), judge]) == [] # it names its run, so another doesn't matter + [nowhere] = usj.UserSimJudge("j", None, run_step_id="nope").task_problems([deploy, week]) + assert nowhere == "user_sim_judge 'j': run_step_id 'nope' names no step of the task" other = AgentPromptResponseVerifierTaskStep("grade", None, prompt_id="ask") [named] = usj.UserSimJudge("j", None, run_step_id="grade").task_problems([deploy, week, other]) assert "run_step_id 'grade' is a agent_prompt_response_verifier step; name the always_on or prompt_agent step to " \ @@ -894,13 +982,13 @@ def test_the_task_must_say_which_run_and_its_user_must_be_a_built_in_user_sim(): assert "the user of always_on step 'week' isn't a user sim a deploy_user_sim_agent step deploys" in outside [early] = judge.task_problems([deploy, judge, week]) assert early == "user_sim_judge 'j' comes before step 'week', the run it judges; put it after" - loose = usj.UserSimJudge("j", None, depends_on=[TaskStepDependency("deploy-u")]) + loose = usj.UserSimJudge("j", None, run_step_id="week", depends_on=[TaskStepDependency("deploy-u")]) assert "its depends_on must name 'week'" in loose.task_problems([deploy, week, loose])[0] def test_with_the_gate_on_every_step_that_can_be_the_reward_must_depend_on_the_judge(): deploy, week = _user_sim_steps() - judge = usj.UserSimJudge("judge", None, depends_on=[TaskStepDependency("week")]) + judge = usj.UserSimJudge("judge", None, run_step_id="week", depends_on=[TaskStepDependency("week")]) after = [TaskStepDependency("judge")] assert judge.task_problems([deploy, week, judge, _grader(depends_on=after)]) == [] assert judge.task_problems([deploy, week, judge, _grader()]) == [] # no depends_on: every step before it @@ -909,21 +997,21 @@ def test_with_the_gate_on_every_step_that_can_be_the_reward_must_depend_on_the_j "user_sim_judge 'judge', so a run the user sim spoiled could still be scored; make it depend on " "'judge', or set fail_task_when_not_passing to false to record the verdicts only") assert judge.task_problems([deploy, week, _grader(), judge]) == [skips] # before it, so not after it - off = usj.UserSimJudge("judge", None, fail_task_when_not_passing=False) + off = usj.UserSimJudge("judge", None, run_step_id="week", fail_task_when_not_passing=False) assert off.task_problems([deploy, week, _grader(), off]) == [] with pytest.raises(ValueError, match="can be the task's reward but doesn't depend on user_sim_judge 'judge'"): Task("t", None, [deploy, week, _grader(), judge]) assert Task("t", None, [deploy, week, judge, _grader(depends_on=after)]) # an always_on task takes it -def test_a_task_that_sets_mark_user_sim_error_or_turns_off_fail_task_on_error_with_the_gate_is_refused(): +def test_a_task_that_sets_mark_user_sim_error_or_turns_off_fail_task_on_error_is_refused(): with pytest.raises(ValueError, match="mark_user_sim_error is gone; fail_task_when_not_passing .* takes its place"): - usj.UserSimJudge.from_dict({"id": "s", "type": "user_sim_judge", "mark_user_sim_error": False}) - with pytest.raises(ValueError, match="user_sim_judge 's': fail_task_when_not_passing .* needs fail_task_on_error"): - usj.UserSimJudge.from_dict({"id": "s", "type": "user_sim_judge", "fail_task_on_error": False}) - quiet = usj.UserSimJudge.from_dict({"id": "s", "type": "user_sim_judge", "fail_task_on_error": False, - "fail_task_when_not_passing": False}) - assert (quiet.fail_task_on_error, quiet.fail_task_when_not_passing) == (False, False) + usj.UserSimJudge.from_dict({**STORED, "mark_user_sim_error": False}) + for gate in (True, False): # with the gate off too: a judge that fails has to fail the task + with pytest.raises(ValueError, match="user_sim_judge 's': fail_task_on_error can't be false"): + usj.UserSimJudge.from_dict({**STORED, "fail_task_on_error": False, "fail_task_when_not_passing": gate}) + quiet = usj.UserSimJudge.from_dict({**STORED, "fail_task_on_error": True, "fail_task_when_not_passing": False}) + assert (quiet.fail_task_on_error, quiet.fail_task_when_not_passing) == (True, False) # ---- task rubric items in usersim.yaml's shape ---------------------------------------------------------------------- @@ -972,7 +1060,7 @@ def test_the_craftweb_item_round_trips_and_is_asked_as_written(): assert CALL_RAE["name"] not in json.dumps(plan.criteria) and CALL_RAE["short"] not in json.dumps(plan.criteria) assert (plan.about["a1_call_rae"]["name"], plan.about["a1_call_rae"]["short"]) == (CALL_RAE["name"], CALL_RAE["short"]) - step = usj.UserSimJudge("s", None, rubric_items=[CALL_RAE, HER_LENGTH, SATISFACTION], + step = usj.UserSimJudge("s", None, run_step_id="week", rubric_items=[CALL_RAE, HER_LENGTH, SATISFACTION], rubric_persona_sha256=OTHER_PERSONA) again = usj.UserSimJudge.from_dict(json.loads(json.dumps(step.to_dict()))) assert step.config_problems() == [] and again.to_dict() == step.to_dict() @@ -994,7 +1082,7 @@ def test_the_old_text_field_still_loads_as_the_check_and_kind_defaults_to_genera [item] = _items(old) assert (item.family, item.kind, item.check) == ("G1", "general", "never mentions the ring") assert item.to_dict() == {"id": "leak", "family": "G1", "kind": "general", "check": "never mentions the ring"} - assert usj.UserSimJudge("s", None, rubric_items=[old]).config_problems() == [] + assert usj.UserSimJudge("s", None, run_step_id="week", rubric_items=[old]).config_problems() == [] [both] = core.item_problems([{**old, "check": "never mentions the ring"}]) assert both == "rubric_items[0] (leak): give check or text, not both (text is the old name of check)" @@ -1016,8 +1104,8 @@ async def test_an_auto_item_is_counted_on_the_graded_answers_with_no_model_call( "what she says before you confirm anything.") run = _run(("reply", TESS[0]), ("no_reply", ""), ("reply", over), ("reply", TESS[1])) fake = FakeModel() - entry = await _judge(run, fake, monkeypatch, checks=[], rubric_items=[HER_LENGTH]) - assert fake.calls == [] and entry["calls"] == 0 + entry = await _judge(run, fake, monkeypatch, rubric_items=[HER_LENGTH]) + assert all("f2_her_length" not in c["names"] for c in fake.calls) and entry["calls"] == len(fake.calls) == 7 assert _rows(entry)["f2_her_length"] == { "id": "f2_her_length", "family": "F2", "kind": "auto", "name": "As short as her real texts", "check": "Every reply is at most 25 words.", "metric": "max_words", "max": 25, "asked_with": "computed", @@ -1032,7 +1120,7 @@ async def test_an_auto_item_is_counted_on_the_graded_answers_with_no_model_call( context_only = _unjudged(_plan(_run(("reply", over), ("reply", TESS[1]), graded={2}), [length], checks=[])) assert context_only["f2_her_length"]["verdict"] == "pass" assert _unjudged(_plan(_run(("no_reply", "")), [length], checks=[]))["f2_her_length"]["verdict"] == "n/a" - # With the checks running, the auto item is in no call; one that can_error makes the run a user sim error. + # One that can_error makes the run a user sim error. fake = FakeModel() entry = await _judge(_run(("reply", over)), fake, monkeypatch, rubric_items=[{**HER_LENGTH, "can_error": True}]) assert all("f2_her_length" not in c["names"] for c in fake.calls) and len(fake.calls) == 7 @@ -1103,28 +1191,26 @@ def test_an_auto_item_counts_sentences_simply(): @pytest.mark.asyncio async def test_na_when_always_is_n_a_with_its_reason_and_no_call(monkeypatch): fake = FakeModel() - entry = await _judge(_run(("done", "thx")), fake, monkeypatch, checks=[], rubric_items=[SATISFACTION]) + entry = await _judge(_run(("done", "thx")), fake, monkeypatch, rubric_items=[SATISFACTION]) row = _rows(entry)["a5_satisfaction"] - assert fake.calls == [] and (row["verdict"], row["reason"], row["asked_with"]) == ( + assert len(fake.calls) == 7 and not fake.asked("a5_satisfaction") and (row["verdict"], row["reason"], row["asked_with"]) == ( "n/a", SATISFACTION["static_na_reason"], "static_na") - fake = FakeModel() - await _judge(_run(("done", "thx")), fake, monkeypatch, rubric_items=[SATISFACTION]) - assert all("a5_satisfaction" not in c["names"] for c in fake.calls) @pytest.mark.asyncio async def test_na_when_never_doesn_t_allow_an_n_a_so_one_is_asked_again(monkeypatch): fake = FakeModel({"g5_no_silent_fix": [NA(), PASS(reason="she gave her call")]}) entry = await _judge(_run(("reply", "your call on the dates, just not oct 31")), fake, monkeypatch, - checks=["rogue"], rubric_items=[NO_SILENT_FIX]) - assert _rows(entry)["g5_no_silent_fix"]["verdict"] == "pass" and len(fake.calls) == 2 - assert "may not be n/a: it always applies, so answer pass or fail" in fake.calls[1]["prompt"] + rubric_items=[NO_SILENT_FIX]) + asked = fake.asked("g5_no_silent_fix") + assert _rows(entry)["g5_no_silent_fix"]["verdict"] == "pass" and len(asked) == 2 + assert "may not be n/a: it always applies, so answer pass or fail" in asked[1]["prompt"] never = await _judge(_run(("reply", "x")), FakeModel({"g5_no_silent_fix": [NA()] * 4}), monkeypatch, - checks=["rogue"], rubric_items=[NO_SILENT_FIX]) - assert _verdicts(never) == {"rogue": "error", "g5_no_silent_fix": "error"} # the call's rows are undecided + rubric_items=[NO_SILENT_FIX]) + assert _verdicts(never, "rogue", "g5_no_silent_fix") == {"rogue": "error", "g5_no_silent_fix": "error"} # Where n/a is allowed, it stands. got = await _judge(_run(("reply", "x")), FakeModel({"a1_call_rae": NA(reason="Rae never came up")}), monkeypatch, - checks=["guidance"], rubric_items=[CALL_RAE]) + rubric_items=[CALL_RAE]) assert _rows(got)["a1_call_rae"]["verdict"] == "n/a" @@ -1145,11 +1231,12 @@ def test_a_big_rubric_is_split_over_calls_of_at_most_the_cap(): async def test_split_items_get_their_rows_and_the_check_is_asked_once(monkeypatch): items = [{"id": f"g1_{n}", "family": "G1", "check": f"never states fact {n}"} for n in range(14)] fake = FakeModel({"g1_13": FAIL()}) - entry = await _judge(_run(("reply", "x")), fake, monkeypatch, checks=["oversharing"], rubric_items=items) - assert [len(c["names"]) for c in fake.calls] == [13, 2] and fake.calls[0]["names"][0] == "oversharing" - assert "These are criteria 14–15 of 15" in fake.calls[1]["prompt"] + entry = await _judge(_run(("reply", "x")), fake, monkeypatch, rubric_items=items) + split = [c for c in fake.calls if any(n.startswith("g1_") for n in c["names"])] + assert [len(c["names"]) for c in split] == [13, 2] and split[0]["names"][0] == "oversharing" + assert len(fake.asked("oversharing")) == 1 and "These are criteria 15–16 of 21" in split[1]["prompt"] assert _rows(entry)["g1_13"]["asked_with"] == "oversharing#2" and _rows(entry)["g1_13"]["verdict"] == "fail" - assert [r["id"] for r in entry["results"]] == ["oversharing", *[f"g1_{n}" for n in range(14)]] # the task's order + assert [r["id"] for r in entry["results"]] == [*core.CHECK_NAMES, *[f"g1_{n}" for n in range(14)]] assert entry["user_sim_error"]["because"] == ["g1_13"] # G1's owner, oversharing, is an error check @@ -1182,12 +1269,13 @@ def test_each_bad_item_is_refused_when_the_task_loads_with_a_clear_error(): for item, error in cases: problems = core.item_problems([item]) assert any(error in p for p in problems), (item, problems) - assert any(error in p for p in usj.UserSimJudge("s", None, rubric_items=[item]).config_problems()) + assert any(error in p for p in usj.UserSimJudge("s", None, run_step_id="week", + rubric_items=[item]).config_problems()) dup = core.item_problems([CALL_RAE, {**CALL_RAE, "family": "A3"}]) assert dup == ["rubric_items[1] (a1_call_rae): id 'a1_call_rae' is used twice"] for bad in (7, "3f1c", "A" * 64): assert any("rubric_persona_sha256 must be a persona fingerprint" in p for p in usj.UserSimJudge( - "s", None, rubric_items=[CALL_RAE], rubric_persona_sha256=bad).config_problems()) + "s", None, run_step_id="week", rubric_items=[CALL_RAE], rubric_persona_sha256=bad).config_problems()) def test_an_auto_item_loads_with_no_na_when_or_with_always(): @@ -1228,7 +1316,7 @@ async def test_items_for_this_persona_or_a_run_with_no_fingerprint_are_graded(mo if not recorded: del us["persona_sha256"] # a run from before the step recorded it - await _step(FakeModel(), monkeypatch, checks=["style"], rubric_items=[HER_LENGTH], + await _step(FakeModel(), monkeypatch, rubric_items=[HER_LENGTH], rubric_persona_sha256=fingerprint).execute(ctx) entry = _entry(ctx) @@ -1237,7 +1325,7 @@ async def test_items_for_this_persona_or_a_run_with_no_fingerprint_are_graded(mo if not recorded: assert entry["rubric_persona"]["run"] is None and "graded as written" in entry["rubric_persona"]["note"] # Without a rubric_persona_sha256 nothing is compared. - await _step(FakeModel(), monkeypatch, checks=["style"], rubric_items=[HER_LENGTH]).execute(ctx) + await _step(FakeModel(), monkeypatch, rubric_items=[HER_LENGTH]).execute(ctx) assert "rubric_persona" not in _entry(ctx) and _rows(_entry(ctx))["f2_her_length"]["verdict"] == "pass" @@ -1263,9 +1351,9 @@ async def test_a_line_item_is_graded_by_that_line_s_text_from_the_run_and_record [item] = _items(ENGAGEMENT) assert item.to_dict() == ENGAGEMENT and item.check == "" fake = FakeModel({"g1_engagement": FAIL()}) - entry = await _judge(_lined_run({"W1": "Hannah.", "P1": RING}), fake, monkeypatch, checks=["oversharing"], + entry = await _judge(_lined_run({"W1": "Hannah.", "P1": RING}), fake, monkeypatch, rubric_items=[ENGAGEMENT], rubric_persona_sha256=OTHER_PERSONA) - assert fake.calls[0]["criteria"][1]["question"] == f"Follows its instruction P1: {RING}" + assert fake.asked("g1_engagement")[0]["criteria"][1]["question"] == f"Follows its instruction P1: {RING}" row = _rows(entry)["g1_engagement"] assert (row["line"], row["line_text"], row["asked_with"], row["verdict"], row["kind"]) == ( "P1", RING, "oversharing", "fail", "hard") @@ -1311,9 +1399,9 @@ def test_a_line_item_is_refused_with_a_check_on_an_auto_item_or_without_the_pers [no_check] = core.item_problems([{"id": "a", "family": "A1"}]) assert no_check == ("rubric_items[0] (a): a general item needs a check (the text the judge applies) or a line (a " "tag of the user sim's instructions, like P1)") - assert usj.UserSimJudge("s", None, rubric_items=[ENGAGEMENT], + assert usj.UserSimJudge("s", None, run_step_id="week", rubric_items=[ENGAGEMENT], rubric_persona_sha256=OTHER_PERSONA).config_problems() == [] - assert usj.UserSimJudge("s", None, rubric_items=[ENGAGEMENT]).config_problems() == [ + assert usj.UserSimJudge("s", None, run_step_id="week", rubric_items=[ENGAGEMENT]).config_problems() == [ "rubric_items[0] (g1_engagement): a line item needs rubric_persona_sha256, so its tag is graded only on runs " "of the persona it was written for"] @@ -1325,10 +1413,10 @@ async def test_on_a_run_a_line_item_reads_the_deploy_record_s_tagged_lines(monke us = ctx.metadata[USER_SIMS_KEY]["hannah"] fake = FakeModel() - await _step(fake, monkeypatch, checks=["oversharing"], rubric_items=[ENGAGEMENT], + await _step(fake, monkeypatch, rubric_items=[ENGAGEMENT], rubric_persona_sha256=us["persona_sha256"]).execute(ctx) - [call] = fake.calls + [call] = fake.asked("g1_engagement") assert call["criteria"][1]["question"] == f"Follows its instruction P1: {us['tagged_lines']['P1']}" row = _rows(_entry(ctx))["g1_engagement"] assert (row["line"], row["line_text"], row["asked_with"]) == ("P1", us["tagged_lines"]["P1"], "oversharing") @@ -1341,11 +1429,11 @@ async def test_on_a_run_with_no_persona_fingerprint_a_line_item_is_an_error(monk del ctx.metadata[USER_SIMS_KEY]["hannah"]["persona_sha256"] fake = FakeModel() - await _step(fake, monkeypatch, checks=["oversharing"], rubric_items=[ENGAGEMENT, HER_LENGTH], + await _step(fake, monkeypatch, rubric_items=[ENGAGEMENT, HER_LENGTH], rubric_persona_sha256=OTHER_PERSONA).execute(ctx) rows = _rows(_entry(ctx)) assert rows["g1_engagement"]["verdict"] == "error" and rows["f2_her_length"]["verdict"] == "pass" assert rows["g1_engagement"]["error"] == "line P1 can't be graded: " + usj.LINE_UNCHECKED - assert fake.calls[0]["names"] == ["oversharing"] + assert not fake.asked("g1_engagement") and len(fake.calls) == 7 assert _entry(ctx)["user_sim_error"]["value"] == "unknown" and _entry(ctx)["flag"] == "user sim judge undecided" From d57124b2c4825812320fc2a3af765719727db1ad Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 16 Nov 2026 13:12:40 -0800 Subject: [PATCH 36/45] user_sim_judge: a seeded task keeps its persona fingerprint, and an agent judge's timeout is undecided Task.run fills the seed into every step before it runs, so deploy_user_sim_agent fingerprinted the seeded persona and opening while rubric_persona_sha256 is the unseeded one: a seeded task's items were never graded. The step now takes the fingerprint when it's built, as written, and substitute_seed_in_steps hands it to the seeded copy (seeded_from). The test runs it through Task.run with a seed. With a run-wide use_agent_judge, the judge call can raise TimeoutError, JudgeTimeoutError or AgentUnreachableError; those now leave the call's rows undecided like the other flaky errors, and setup errors still raise. judge_timeout_retries and judge_max_turns, new on main, are fixed settings too. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/task/task.py | 5 ++++- .../task_steps/agent/deploy_user_sim_agent.py | 9 +++++++- .../task_steps/verifiers/user_sim_judge.py | 12 +++++----- .../task_step/test_deploy_user_sim_agent.py | 16 ++++++++++++-- .../verifiers/test_user_sim_judge.py | 22 ++++++++++++++++++- 5 files changed, 54 insertions(+), 10 deletions(-) diff --git a/src/agent_env/task/task.py b/src/agent_env/task/task.py index df78a58f4..d356c1bdc 100644 --- a/src/agent_env/task/task.py +++ b/src/agent_env/task/task.py @@ -135,7 +135,10 @@ def _sub(value): out: list["TaskStep"] = [] for step in steps: data = _sub(step.to_dict()) - out.append(attach_retry_config(type(step).from_dict(data), data)) + seeded = attach_retry_config(type(step).from_dict(data), data) + if (keep := getattr(seeded, "seeded_from", None)) is not None: # a step that keeps something as written + keep(step) + out.append(seeded) return out diff --git a/src/agent_env/task_step/task_steps/agent/deploy_user_sim_agent.py b/src/agent_env/task_step/task_steps/agent/deploy_user_sim_agent.py index 0d6b84f29..34a81e810 100644 --- a/src/agent_env/task_step/task_steps/agent/deploy_user_sim_agent.py +++ b/src/agent_env/task_step/task_steps/agent/deploy_user_sim_agent.py @@ -252,9 +252,16 @@ def __init__(self, id: str, version: Optional[int], *, persona: Optional[dict] = self.model = model self.multi_day = multi_day self.opening_prompt = opening_prompt + # Who the user is as the task writes it, the seed not filled in: what the judge's rubric_persona_sha256 is. + self.persona_sha256 = persona_fingerprint(persona, opening_prompt) _OWN = ("persona", "model", "multi_day", "opening_prompt") + def seeded_from(self, written: "DeployUserSimAgentTaskStep") -> None: + """This step is ``written`` with the run's seed filled in (``substitute_seed_in_steps``): keep the fingerprint + of the persona and opening as written.""" + self.persona_sha256 = written.persona_sha256 + def to_dict(self) -> dict: base = super().to_dict() base.pop("system_prompt", None) @@ -336,7 +343,7 @@ async def _set_user(self, context: TaskStepContext, agent, setup: dict) -> TaskS "user_version": user_sim_versions.VERSION, "model": model, "model_source": source, "prompt_sha256": hashlib.sha256(prompt.encode()).hexdigest(), "prompt_set": True, # Who the user is, as written: what user_sim_judge's rubric_persona_sha256 is compared with. - "persona_sha256": persona_fingerprint(persona, self.opening_prompt), + "persona_sha256": self.persona_sha256, # Each tag the prompt shows with its line, so an answer's ``used`` ("D9") names what this run saw. "tagged_lines": user_sim_versions.tagged_lines(self.person(context)), "multi_day": self.multi_day, diff --git a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py index 1b1ab6764..94ce15231 100644 --- a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py +++ b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py @@ -122,7 +122,8 @@ ``PermissionDeniedError``, ``NotFoundError`` (an unknown model), ``BadRequestError`` and its kinds, ``UnprocessableEntityError`` and ``BudgetExceededError`` (``_setup_error``). Every other failure is taken as flaky (``Timeout``, ``RateLimitError``, ``ServiceUnavailableError``, ``InternalServerError``, ``BadGatewayError``, -``APIConnectionError``, an answer still unusable after its retries), and unknown never fails the task: a run is failed +``APIConnectionError``, an agent judge's ``TimeoutError``, ``JudgeTimeoutError`` or ``AgentUnreachableError``, an answer +still unusable after its retries), and unknown never fails the task: a run is failed only on a slip the judge found, so a flaky judge never throws away a good run. A judge answer the format can't use is asked again with the format's correction, up to 3 more times (``DEFAULT_MAX_RETRIES``, 4, counts the first ask; rubrics_verifier tries a failed model call as many times). A call still without a usable answer leaves its rows @@ -173,7 +174,7 @@ **dict.fromkeys(("criteria", "prompt_id", "default_model", "default_model_api_base", "agent_name", "trajectory_filter", "max_thinking_tokens", "effort", "judge_a2a_agent_id", "judge_sandbox_type", "grading_policy_prompt", "context_facts_for_judge", "grade_up_to_turn", "run_context_for_judge", - "judge_batch_size", "judge_batch_concurrency"))} + "judge_batch_size", "judge_batch_concurrency", "judge_timeout_retries", "judge_max_turns"))} NOT_FAILING_OFF = "fail_task_when_not_passing is off: verdicts are recorded only" NOT_FAILING_UNCONFIRMED = ("the judge's copy of the user sim's instructions isn't confirmed to be the one it ran with, " "so it doesn't fail the task") @@ -468,7 +469,7 @@ async def _run_judge_with_output_retries(self, *, criteria: list[dict], **kwargs (``error``) rather than failing the step: an unknown never fails the task.""" try: return await super()._run_judge_with_output_retries(criteria=criteria, **kwargs) - except (ValueError, RuntimeError) as e: + except (ValueError, RuntimeError, TimeoutError) as e: # TimeoutError: an agent judge's (run-wide use_agent_judge) if (setup := _setup_error(e)) is not None: raise JudgeSetupError(f"the user sim judge can't reach its model: {type(setup).__name__}: " f"{str(setup)[:ERROR_SHOWN_CHARS]}") from e @@ -502,8 +503,9 @@ def _setup_error(e: BaseException) -> Optional[BaseException]: missing config) and LiteLLM's AuthenticationError, PermissionDeniedError, NotFoundError (an unknown model), BadRequestError (with its kinds: an unknown provider, unsupported params, a prompt over the context window), UnprocessableEntityError and BudgetExceededError. Anything else is flaky (Timeout, RateLimitError, - ServiceUnavailableError, InternalServerError, BadGatewayError, APIConnectionError, an answer still unusable after - its retries) and leaves the call's rows undecided.""" + ServiceUnavailableError, InternalServerError, BadGatewayError, APIConnectionError, an agent judge's TimeoutError, + JudgeTimeoutError or AgentUnreachableError, an answer still unusable after its retries) and leaves the call's rows + undecided.""" import litellm from agent_env.config.errors import ConfigError diff --git a/tst/unit/task_step/test_deploy_user_sim_agent.py b/tst/unit/task_step/test_deploy_user_sim_agent.py index bccb903b6..e0755fd3f 100644 --- a/tst/unit/task_step/test_deploy_user_sim_agent.py +++ b/tst/unit/task_step/test_deploy_user_sim_agent.py @@ -142,14 +142,26 @@ def test_the_persona_fingerprint_is_pinned(): assert persona_fingerprint(FINGERPRINT_PERSONA, [{"kind": "text", "text": FINGERPRINT_OPENING}]) != FINGERPRINT +@pytest.mark.usefixtures("local_stores") @pytest.mark.asyncio async def test_the_record_has_the_persona_fingerprint_of_the_step_as_written(monkeypatch): + from agent_env.task import Task + opening = [{"kind": "text", "text": "the trip, can you handle it?"}, {"kind": "file", "file": {"uri": "s3://reyes-family//itinerary.pdf", "name": "itinerary.pdf"}}] + persona = {**HANNAH, "description": {**HANNAH["description"], + "who_you_are": [*HANNAH["description"]["who_you_are"], "Back from ."]}} context = TaskStepContext() context.metadata["seed"] = {"trip": "nov-19"} - record = await _published(monkeypatch, _step(opening_prompt=opening), context) - assert record["persona_sha256"] == persona_fingerprint(HANNAH, opening) # unseeded + deploy = _Deploy(monkeypatch) + # Task.run fills the seed into every step before it runs: the record still has the fingerprint as written. + with (patch.object(A2AAgent, "get", return_value=deploy.agent), + patch("agent_env.task_step.task_steps.deploy_agent.get_config") as get_config): + get_config.return_value.get_model_for_role.return_value = None + context = await Task("t", None, [_step(persona=persona, opening_prompt=opening)]).run(context=context) + record = context.metadata[USER_SIMS_KEY]["hannah"] + assert "nov-19" in record["opening"] and "" not in json.dumps(record["description"]) # it ran seeded + assert record["persona_sha256"] == persona_fingerprint(persona, opening) # unseeded assert (await _published(monkeypatch, _step()))["persona_sha256"] == persona_fingerprint( HANNAH, PERSON["opening"]["text"]) diff --git a/tst/unit/task_step/verifiers/test_user_sim_judge.py b/tst/unit/task_step/verifiers/test_user_sim_judge.py index 4bad2865e..cd04d2c32 100644 --- a/tst/unit/task_step/verifiers/test_user_sim_judge.py +++ b/tst/unit/task_step/verifiers/test_user_sim_judge.py @@ -31,7 +31,12 @@ diagnose_user_sim_checks_response, get_judge_output_format_spec, ) -from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep, is_rubrics_verifier +from agent_env.a2a_agent.protocol import AgentUnreachableError +from agent_env.task_step.task_steps.verifiers.rubrics_verifier import ( + JudgeTimeoutError, + RubricsVerifierTaskStep, + is_rubrics_verifier, +) from tst.unit.task_step.always_on_harness import Rig from tst.unit.task_step.test_deploy_user_sim_agent import HANNAH, _Deploy from tst.unit.task_step.test_deploy_user_sim_agent import _step as _deploy_step @@ -784,6 +789,20 @@ async def test_a_flaky_model_call_is_tried_as_rubrics_verifier_tries_it_and_left assert entry["user_sim_error"]["value"] == "unknown" and entry["user_sim_error"]["fails_task"] is False +@pytest.mark.asyncio +@pytest.mark.parametrize("error", [TimeoutError("judge ran out of time"), + JudgeTimeoutError("Judge A2A task t1 ran out of its 1000s", None), + AgentUnreachableError("the judge stopped answering")], + ids=["timeout", "judge timeout", "agent unreachable"]) +async def test_an_agent_judge_that_times_out_or_stops_answering_leaves_its_check_undecided(monkeypatch, error): + # A run-wide use_agent_judge sends the prompt to an agent judge, whose call can run out of time or lose the agent. + fake = FakeModel({"rogue": [error]}) + entry = await _judge(_run(("reply", "ok")), fake, monkeypatch, fail_task_when_not_passing=True) + assert _verdicts(entry, "rogue", "guidance") == {"rogue": "error", "guidance": "pass"} + assert _rows(entry)["rogue"]["error"].startswith(f"no usable answer after 4 tries: {type(error).__name__}: ") + assert entry["flag"] == "user sim judge undecided" and entry["user_sim_error"]["fails_task"] is False + + @pytest.mark.asyncio async def test_one_undecided_check_among_decided_ones_is_flagged_undecided(monkeypatch, litellm_config): def answer(kwargs): @@ -919,6 +938,7 @@ def test_checks_and_error_checks_are_refused_when_the_task_loads(): def test_a_rubrics_verifier_setting_the_judge_fixes_is_refused_unless_it_has_the_fixed_value(): for key, value in (("use_trajectory", True), ("output_format", "rubric_binary"), ("judge_batch_size", 3), + ("judge_timeout_retries", 2), ("judge_max_turns", 40), ("use_agent_judge", True), ("criteria", [{"id": "c1"}]), ("prompt_id", "ask"), ("default_model", "x"), ("score_aggregator", "weighted_average"), ("grade_up_to_turn", 2)): with pytest.raises(ValueError, match=f"user_sim_judge 's': {key} can't be set on the user sim judge, which " From 3e07f442e91871e8498616e1427ac4fa44501ca0 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 16 Nov 2026 13:49:10 -0800 Subject: [PATCH 37/45] fix(verifiers): a malformed user_sim_checks answer is asked again, never raised A result whose id isn't text ({"results": [{"id": []}]}) reached the shared id check, which hashes each id and raised TypeError, failing the task instead of sending the answer back for correction. Each id is now checked first and a non-text one is a row problem. The same goes for a reason that isn't text and turns that aren't a list. An infinite turn number (OverflowError) is dropped like any other entry that isn't a turn, and `results` that isn't a list is named as such. Co-Authored-By: Claude Opus 5.5 --- .../judge_utils/judge_output_format.py | 25 ++++++++--- .../test_user_sim_checks_format.py | 44 +++++++++++++++++++ 2 files changed, 64 insertions(+), 5 deletions(-) diff --git a/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py b/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py index cffa0ae63..974652d9a 100644 --- a/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py +++ b/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py @@ -1359,7 +1359,7 @@ def _cited_turns(value: Any) -> list[int]: for v in value: try: out.append(int(v)) - except (TypeError, ValueError): + except (TypeError, ValueError, OverflowError): # OverflowError: an infinite number continue return out @@ -1409,8 +1409,12 @@ def _user_sim_row(item: dict, criterion: dict) -> tuple[dict | None, list[str]]: problems.append(f"{rid}: assistant_asked must be true or false") if verdict == "n/a" and str(criterion.get("na_when") or "").strip().lower() == _USER_SIM_NA_NEVER: problems.append(f"{rid} may not be n/a: it always applies, so answer pass or fail") + if not isinstance(item.get("reason"), str): + problems.append(f"{rid}: reason must be text") + if not isinstance(item.get("turns"), list): + problems.append(f"{rid}: turns must be a list of turn numbers") turns = _cited_turns(item.get("turns")) - reason = str(item.get("reason") or "").strip() + reason = item["reason"].strip() if isinstance(item.get("reason"), str) else "" graded = _graded_turns(criterion) if verdict == "fail": if not any(graded is None or t in graded for t in turns): @@ -1438,16 +1442,27 @@ def diagnose_user_sim_checks_response( response_text: str, criteria: list[dict], ) -> tuple[list[dict] | None, JudgeResponseDiscrepancy | None]: """Parse a user_sim_checks judge response into rows, or a discrepancy for the correction prompt: no JSON - object, the wrong row set, or a row the user sim judge can't use (its verdict or could_change_grade invalid, an - n/a where ``na_when`` is never, an assistant_asked that isn't true or false, a fail citing no graded turn, a fail whose reason concludes nothing failed).""" + object, the wrong row set, or a row the user sim judge can't use: an id, reason or turns of the wrong type, a + verdict, could_change_grade or assistant_asked out of range, an n/a where ``na_when`` is never, a fail citing no + graded turn, or a fail whose reason concludes nothing failed.""" parsed = _user_sim_json(response_text) if parsed is None: return None, _parse_failure_discrepancy(criteria, "the answer is not a JSON object") results = _extract_rubric_binary_results(parsed) if not isinstance(results, list): - return None, _parse_failure_discrepancy(criteria, f"unexpected response shape: {type(parsed).__name__}") + shape = ("`results` must be a list" if results is not None + else f"unexpected response shape: {type(parsed).__name__}") + return None, _parse_failure_discrepancy(criteria, shape) if not all(isinstance(r, dict) for r in results): return None, _parse_failure_discrepancy(criteria, "every result must be a JSON object") + # The shared id check hashes each id, so one that isn't text (a list, an object) would raise instead of being + # asked again. + bad_ids = [f"result {i}: id must be a criterion id (text), not {r.get('id')!r}" + for i, r in enumerate(results, 1) if not isinstance(r.get("id"), str)] + if bad_ids: + return None, JudgeResponseDiscrepancy( + expected_count=len(criteria), returned_count=len(results), expected_ids=[c["id"] for c in criteria], + missing_ids=[], duplicate_ids=[], unknown_ids=[], row_problems=bad_ids) discrepancy = _analyze_result_ids(results, criteria) if discrepancy is not None: return None, discrepancy diff --git a/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py b/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py index 8f01b83eb..f4f96211e 100644 --- a/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py +++ b/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py @@ -363,3 +363,47 @@ async def test_rubrics_verifier_names_the_unusable_row_when_the_judge_never_fixe with pytest.raises(ValueError, match=r"Judge output invalid after 3 attempts: c1: a fail whose reason says " r"'No clear failure found' contradicts itself"): await _run_judge(verifier) + + +# ---- malformed answers are asked again, never raised --------------------------------------------------------------- +GOOD = _row("c1") + + +@pytest.mark.parametrize("text, problem", [ + (json.dumps({"results": [{"id": []}]}), "result 1: id must be a criterion id (text), not []"), + (json.dumps({"results": [{**GOOD, "id": {"c": 1}}]}), "result 1: id must be a criterion id (text), not {'c': 1}"), + (json.dumps({"results": [{**GOOD, "id": None}]}), "result 1: id must be a criterion id (text), not None"), + (json.dumps({"results": [{**GOOD, "id": 1}]}), "result 1: id must be a criterion id (text), not 1"), + (json.dumps({"results": [{**GOOD, "reason": ["dad cant do shellfish"]}]}), "c1: reason must be text"), + (json.dumps({"results": [{k: v for k, v in GOOD.items() if k != "reason"}]}), "c1: reason must be text"), + (json.dumps({"results": [{**GOOD, "turns": "1"}]}), "c1: turns must be a list of turn numbers"), + (json.dumps({"results": [{**GOOD, "verdict": ["fail"]}]}), "c1: verdict ['fail'] is not one of"), + (json.dumps({"results": [{**GOOD, "could_change_grade": "true"}]}), "c1: could_change_grade must be true or false"), + (json.dumps({"results": [{**GOOD, "assistant_asked": None}]}), "c1: assistant_asked must be true or false"), + # An infinite turn number is dropped like any other that isn't one, so the fail cites none. + ('{"results": [{"id": "c1", "reason": "Leaked it.", "turns": [Infinity], "verdict": "fail", ' + '"could_change_grade": true, "assistant_asked": false}]}', "c1: a fail must cite at least one graded turn"), + (json.dumps({"results": {"c1": GOOD}}), "`results` must be a list"), + (json.dumps({"results": "pass"}), "`results` must be a list"), + (json.dumps({"results": [GOOD, "pass"]}), "every result must be a JSON object"), + (json.dumps("pass"), "unexpected response shape: str"), + ("null", "the answer is not a JSON object"), +]) +def test_a_malformed_answer_is_a_discrepancy_for_the_correction_prompt_never_an_exception(text, problem): + rows, discrepancy = diagnose_user_sim_checks_response(text, _criteria("c1")) + assert rows is None and problem in str(discrepancy) + prompt = SPEC.format_correction_prompt(eval_prompt="EVAL", previous_response=text, discrepancy=discrepancy) + assert "## Correction Required" in prompt + with pytest.raises(ValueError, match="user_sim_checks answer not usable"): + SPEC.parse_response(text, _criteria("c1")) + + +@pytest.mark.asyncio +async def test_rubrics_verifier_asks_again_when_a_result_s_id_is_not_text(monkeypatch): + verifier = _verifier() + judge = AsyncMock(side_effect=[{"response": json.dumps({"results": [{"id": []}]})}, + {"response": _answer(_row("c1"), _row("c2"))}]) + monkeypatch.setattr(verifier, "_prompt_llm_judge", judge) + rows, retries, discrepancies, _ = await _run_judge(verifier) + assert retries == 1 and [r["verdict"] for r in rows] == ["pass", "pass"] + assert discrepancies[0]["row_problems"] == ["result 1: id must be a criterion id (text), not []"] From 01a773f7dc96506b539cfc906f44a073d9b52d77 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 16 Nov 2026 14:29:30 -0800 Subject: [PATCH 38/45] fix(verifiers): a user_sim_checks turn that isn't a whole number is asked again, not read as a turn int() read `true` and `1.9` as turn 1, so a fail could gate on a turn the judge never cited. Any cited turn that isn't an integer is now a row problem for the correction prompt. Co-Authored-By: Claude Opus 5.5 --- .../judge_utils/judge_output_format.py | 16 +++++++++-- .../test_user_sim_checks_format.py | 28 +++++++++++++++++-- 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py b/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py index 974652d9a..c413a25c6 100644 --- a/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py +++ b/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py @@ -1364,6 +1364,11 @@ def _cited_turns(value: Any) -> list[int]: return out +def _is_turn_number(value: Any) -> bool: + """A whole number, as the schema asks: ``true`` and ``1.9`` are not turn 1.""" + return isinstance(value, int) and not isinstance(value, bool) + + def _graded_turns(criterion: dict) -> set[int] | None: value = criterion.get("graded_turns") return set(_cited_turns(value)) if isinstance(value, list) else None @@ -1411,13 +1416,18 @@ def _user_sim_row(item: dict, criterion: dict) -> tuple[dict | None, list[str]]: problems.append(f"{rid} may not be n/a: it always applies, so answer pass or fail") if not isinstance(item.get("reason"), str): problems.append(f"{rid}: reason must be text") - if not isinstance(item.get("turns"), list): + raw_turns = item.get("turns") + turns_ok = isinstance(raw_turns, list) and all(_is_turn_number(t) for t in raw_turns) + if not isinstance(raw_turns, list): problems.append(f"{rid}: turns must be a list of turn numbers") - turns = _cited_turns(item.get("turns")) + elif not turns_ok: + problems.append(f"{rid}: turns must be a list of turn numbers; " + f"{[t for t in raw_turns if not _is_turn_number(t)]} are not whole numbers") + turns = raw_turns if turns_ok else [] reason = item["reason"].strip() if isinstance(item.get("reason"), str) else "" graded = _graded_turns(criterion) if verdict == "fail": - if not any(graded is None or t in graded for t in turns): + if turns_ok and not any(graded is None or t in graded for t in turns): problems.append(f"{rid}: a fail must cite at least one graded turn; cited {turns}" + (f", graded {sorted(graded)}" if graded is not None else "")) said = _reason_says_no_failure(reason) diff --git a/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py b/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py index f4f96211e..66bf3bf18 100644 --- a/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py +++ b/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py @@ -144,7 +144,7 @@ def test_each_row_gives_its_reason_first_then_turns_verdict_could_change_grade_a def test_an_answer_is_read_into_rows_with_turns_kept_to_the_graded_ones(): criteria = _criteria("c1", "c2", graded=[2], family="G1") rows, discrepancy = diagnose_user_sim_checks_response( - _answer(_fail("c1", turns=[1, 2, "x"]), _row("c2", could_change_grade=True)), criteria) + _answer(_fail("c1", turns=[1, 2]), _row("c2", could_change_grade=True)), criteria) assert discrepancy is None fail, ok = rows assert (fail["verdict"], fail["turns"], fail["turns_outside_graded"]) == ("fail", [2], [1]) @@ -380,9 +380,16 @@ async def test_rubrics_verifier_names_the_unusable_row_when_the_judge_never_fixe (json.dumps({"results": [{**GOOD, "verdict": ["fail"]}]}), "c1: verdict ['fail'] is not one of"), (json.dumps({"results": [{**GOOD, "could_change_grade": "true"}]}), "c1: could_change_grade must be true or false"), (json.dumps({"results": [{**GOOD, "assistant_asked": None}]}), "c1: assistant_asked must be true or false"), - # An infinite turn number is dropped like any other that isn't one, so the fail cites none. + # A turn that isn't a whole number is sent back, never read as a turn: true and 1.9 are not turn 1. ('{"results": [{"id": "c1", "reason": "Leaked it.", "turns": [Infinity], "verdict": "fail", ' - '"could_change_grade": true, "assistant_asked": false}]}', "c1: a fail must cite at least one graded turn"), + '"could_change_grade": true, "assistant_asked": false}]}', + "c1: turns must be a list of turn numbers; [inf] are not whole numbers"), + (json.dumps({"results": [{**_fail("c1"), "turns": [True]}]}), + "c1: turns must be a list of turn numbers; [True] are not whole numbers"), + (json.dumps({"results": [{**_fail("c1"), "turns": [1.9]}]}), + "c1: turns must be a list of turn numbers; [1.9] are not whole numbers"), + (json.dumps({"results": [{**GOOD, "turns": [1, "2"]}]}), + "c1: turns must be a list of turn numbers; ['2'] are not whole numbers"), (json.dumps({"results": {"c1": GOOD}}), "`results` must be a list"), (json.dumps({"results": "pass"}), "`results` must be a list"), (json.dumps({"results": [GOOD, "pass"]}), "every result must be a JSON object"), @@ -398,6 +405,21 @@ def test_a_malformed_answer_is_a_discrepancy_for_the_correction_prompt_never_an_ SPEC.parse_response(text, _criteria("c1")) +@pytest.mark.asyncio +async def test_a_fail_citing_true_or_1_9_is_asked_again_and_the_corrected_answer_grades(monkeypatch): + """Turn 1 is graded, so ``true`` or ``1.9`` read as turn 1 would silently fail c1 on a turn never cited.""" + verifier = _verifier() + judge = AsyncMock(side_effect=[{"response": _answer({**_fail("c1"), "turns": [True]}, _row("c2"))}, + {"response": _answer({**_fail("c1"), "turns": [1.9]}, _row("c2"))}, + {"response": _answer(_row("c1"), _row("c2"))}]) + monkeypatch.setattr(verifier, "_prompt_llm_judge", judge) + rows, retries, discrepancies, _ = await _run_judge(verifier) + assert retries == 2 and [r["verdict"] for r in rows] == ["pass", "pass"] + assert [d["row_problems"] for d in discrepancies] == [ + ["c1: turns must be a list of turn numbers; [True] are not whole numbers"], + ["c1: turns must be a list of turn numbers; [1.9] are not whole numbers"]] + + @pytest.mark.asyncio async def test_rubrics_verifier_asks_again_when_a_result_s_id_is_not_text(monkeypatch): verifier = _verifier() From 9eb45958a90fde348c5d60b80bab852a230d81b1 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 16 Nov 2026 14:30:22 -0800 Subject: [PATCH 39/45] rubrics_verifier: the subclass hooks skip a skipped run; split batches need a batchable format judge_trajectory_content and unjudged_rows now run after the no_screenshots skip, so a skipped run never calls them. A _criteria_batches override that splits the criteria into several calls is refused for a format graded in one call (rubric_evidence and its mistakes variant), whose reasoning, checks and mistakes a merge of the batches would drop. Co-Authored-By: Claude Opus 5.5 --- .../task_steps/verifiers/rubrics_verifier.py | 22 ++++-- .../test_rubrics_verifier_subclass_hooks.py | 79 +++++++++++++++++++ 2 files changed, 93 insertions(+), 8 deletions(-) diff --git a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py index 23c4a3d4a..60996ca56 100644 --- a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py @@ -793,10 +793,6 @@ async def _grade(self, context: TaskStepContext) -> TaskStepContext: if grade_up_to_turn is not None: run_context = run_context.as_of(grade_up_to_turn) extra_sections = await self.judge_prompt_sections(context, stored_response) - trajectory_content = await self.judge_trajectory_content(context, stored_response) - if trajectory_content is not None and self.use_trajectory: - raise RuntimeError(f"Verifier '{self.verifier_id}': judge_trajectory_content stands in for the recorded " - "trajectory, so it needs use_trajectory=False.") per_turn_uris = [u for u in (prompt_response.target_agent_per_turn_trajectory_object_urls or []) if u] if self.use_trajectory and not (per_turn_uris or prompt_response.agent_trajectory_object_url): @@ -808,10 +804,6 @@ async def _grade(self, context: TaskStepContext) -> TaskStepContext: use_agent_judge_override = overrides.get("use_agent_judge") use_agent_judge = use_agent_judge_override if use_agent_judge_override is not None else self.use_agent_judge criteria = self._resolved_criteria(context) - unjudged = [dict(row) for row in await self.unjudged_rows(context, stored_response)] - if clashing := {row.get("id") for row in unjudged} & {c.get("id") for c in criteria}: - raise RuntimeError(f"Verifier '{self.verifier_id}': unjudged_rows gave rows with a criterion's id: " - f"{sorted(map(str, clashing))}") # The judge sees/echoes short keys (c1..cN), not UUIDs; frame labels use them too, hence before compaction. criteria_for_judge, real_by_short = _to_short_keyed_criteria(criteria) apply_filter_override = overrides.get("apply_trajectory_filter") @@ -873,6 +865,16 @@ async def _grade(self, context: TaskStepContext) -> TaskStepContext: "but this run's trajectory filter attached none (user_overrides.apply_trajectory_filter=False?)." ) + # The subclass hooks run once the run is known not to be skipped, and before any judge is deployed. + trajectory_content = await self.judge_trajectory_content(context, stored_response) + if trajectory_content is not None and self.use_trajectory: + raise RuntimeError(f"Verifier '{self.verifier_id}': judge_trajectory_content stands in for the recorded " + "trajectory, so it needs use_trajectory=False.") + unjudged = [dict(row) for row in await self.unjudged_rows(context, stored_response)] + if clashing := {row.get("id") for row in unjudged} & {c.get("id") for c in criteria}: + raise RuntimeError(f"Verifier '{self.verifier_id}': unjudged_rows gave rows with a criterion's id: " + f"{sorted(map(str, clashing))}") + auto_deployed_judge = None staged: Optional[StagedTrajectory] = None judge_agent = None @@ -1042,6 +1044,10 @@ def eval_prompt_for(batch: list[dict]) -> str: if sorted(batched_ids) != sorted(position) or (len(batches) > 1 and not all(batches)): raise RuntimeError(f"Verifier '{self.verifier_id}': _criteria_batches must put every criterion in " "exactly one batch, and leave no batch empty") + if len(batches) > 1 and not self._spec().batchable: + # Batches merge by concatenating rows, which drops a format's one-per-call reasoning, checks and mistakes. + raise RuntimeError(f"Verifier '{self.verifier_id}': _criteria_batches gave {len(batches)} batches, but " + f"output_format={self.output_format.value} is graded in one judge call.") concurrency = asyncio.Semaphore(batch_concurrency or self.DEFAULT_JUDGE_BATCH_CONCURRENCY) async def grade(batch: list[dict]) -> tuple[list[dict], int, list[dict], Optional[str]]: diff --git a/tst/unit/task_step/verifiers/test_rubrics_verifier_subclass_hooks.py b/tst/unit/task_step/verifiers/test_rubrics_verifier_subclass_hooks.py index 95b3356b1..8c35c1118 100644 --- a/tst/unit/task_step/verifiers/test_rubrics_verifier_subclass_hooks.py +++ b/tst/unit/task_step/verifiers/test_rubrics_verifier_subclass_hooks.py @@ -14,6 +14,7 @@ JUDGE_OUTPUT_FORMAT_REGISTRY, JudgeOutputFormat, JudgeOutputFormatSpec, + ResultRows, ) from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep from tst.unit.task_step.verifiers.test_rubrics_verifier_judge_batches import ( @@ -22,6 +23,14 @@ _graded, _prompted_ids, ) +from tst.unit.task_step.verifiers.test_rubrics_verifier_per_criterion import ( + _NO_SCREENSHOTS, + CRITERIA, + _capture_judge, + _ctx, + _filter, + _passing_rows, +) CHECKS = [{"id": "asked", "title": "Asked for the booking", "check": "asked"}, {"id": "asked-item", "title": "Named the restaurant", "check": "asked"}, @@ -133,6 +142,76 @@ async def test_a_skipped_run_holds_no_unjudged_rows(): assert [r["id"] for r in ctx.metadata["verifications"]["vt"]["results"]] == ["prompt_error"] +class _HooksCalled(RubricsVerifierTaskStep): + calls: list[str] = [] + + async def judge_trajectory_content(self, context: TaskStepContext, prompt_response: PromptResponse) -> None: + self.calls.append("judge_trajectory_content") + + async def unjudged_rows(self, context: TaskStepContext, prompt_response: PromptResponse) -> list[dict]: + self.calls.append("unjudged_rows") + return [{"id": "auto-asked", "result": True, "score": 1.0}] + + +def _evidence_step(cls: type[RubricsVerifierTaskStep], **kw) -> RubricsVerifierTaskStep: + params = dict(id="verify", version=1, criteria=CRITERIA, prompt_id="p1", use_agent_judge=False, + use_trajectory=True, trajectory_filter=_filter(), verifier_id="vid", + output_format=JudgeOutputFormat.RUBRIC_EVIDENCE, + grading_policy_prompt="Type every value in full; never accept an autocomplete suggestion.") + params.update(kw) + return cls(**params) + + +@pytest.mark.asyncio +async def test_a_run_skipped_for_want_of_screenshots_never_calls_the_hooks(monkeypatch): + v = _evidence_step(_HooksCalled) + v.calls = [] + _capture_judge(monkeypatch, v, lambda criteria: pytest.fail("the judge must not be called")) + monkeypatch.setattr(v, "_read_trajectory_text", lambda uri: _NO_SCREENSHOTS) + ctx = _ctx() + + await v.execute(ctx) + + assert v.calls == [] + assert [r["id"] for r in ctx.metadata["verifications"]["vid"]["results"]] == ["no_screenshots"] + + +class _OnePerCriterion(RubricsVerifierTaskStep): + def _criteria_batches(self, criteria: list[dict], size: Optional[int]) -> list[list[dict]]: + return [[c] for c in criteria] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("fmt", [JudgeOutputFormat.RUBRIC_EVIDENCE, JudgeOutputFormat.RUBRIC_EVIDENCE_WITH_MISTAKES]) +async def test_a_subclass_cannot_split_a_format_graded_in_one_call(monkeypatch, fmt): + v = _evidence_step(_OnePerCriterion, output_format=fmt) + captured = _capture_judge(monkeypatch, v, _passing_rows) + ctx = _ctx() + + with pytest.raises(RuntimeError, match=rf"_criteria_batches gave 2 batches, but output_format={fmt.value} is " + "graded in one judge call"): + await v.execute(ctx) + assert "eval_prompt" not in captured and "verifications" not in ctx.metadata + + +@pytest.mark.asyncio +async def test_a_subclass_keeping_one_batch_grades_a_format_graded_in_one_call(monkeypatch): + class OneBatch(RubricsVerifierTaskStep): + def _criteria_batches(self, criteria: list[dict], size: Optional[int]) -> list[list[dict]]: + return [list(reversed(criteria))] + + v = _evidence_step(OneBatch) + _capture_judge(monkeypatch, v, lambda criteria: ResultRows( + _passing_rows(criteria), reasoning="Both shown on screen.", checks=[{"id": "typed", "passed": True}])) + ctx = _ctx() + + await v.execute(ctx) + + entry = ctx.metadata["verifications"]["vid"] + assert entry["score"] == 1.0 and entry["reasoning"] == "Both shown on screen." + assert entry["checks"] == [{"id": "typed", "passed": True}] + + class _RunInline(RubricsVerifierTaskStep): async def judge_trajectory_content(self, context: TaskStepContext, prompt_response: PromptResponse) -> str: return "THE RUN: the user sim asked for a table at 7." From ce765de696200b9d605117f7d113bc94c2f6c027 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 16 Nov 2026 14:50:47 -0800 Subject: [PATCH 40/45] user_sim_checks: its rows can be asked over several calls, without the bulk-verdict check #719 refuses custom batches for a format whose rows don't stand alone. The user sim format's rows do (no per-call reasoning, checks or mistakes), so it's marked batchable, and a short pass reason is normal here, so bulk verdicts aren't flagged. Two tests follow #712's and #719's latest rules. Co-Authored-By: Claude Opus 5.5 --- .../verifiers/judge_utils/judge_output_format.py | 4 ++++ .../verifiers/test_rubrics_verifier_subclass_hooks.py | 8 +++++--- tst/unit/task_step/verifiers/test_user_sim_judge.py | 6 +++++- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py b/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py index d89810008..5729fa747 100644 --- a/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py +++ b/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py @@ -1651,6 +1651,10 @@ class JudgeOutputFormatSpec: parse_response=_parse_user_sim_checks_response, diagnose_response=diagnose_user_sim_checks_response, format_correction_prompt=format_user_sim_checks_correction, + # Each row stands alone (no per-call reasoning, checks or mistakes), so a judge may ask its rows over several + # calls; and a short pass reason is normal here, so bulk verdicts aren't flagged. + batchable=True, + checks_bulk_verdicts=False, ), } diff --git a/tst/unit/task_step/verifiers/test_rubrics_verifier_subclass_hooks.py b/tst/unit/task_step/verifiers/test_rubrics_verifier_subclass_hooks.py index 8c35c1118..507d828ce 100644 --- a/tst/unit/task_step/verifiers/test_rubrics_verifier_subclass_hooks.py +++ b/tst/unit/task_step/verifiers/test_rubrics_verifier_subclass_hooks.py @@ -253,10 +253,12 @@ async def test_trajectory_content_beside_a_recorded_trajectory_is_refused(): await v.execute(ctx) -def test_the_rubric_binary_and_partial_formats_alone_are_batchable_and_all_check_bulk_verdicts(): +def test_only_formats_whose_rows_stand_alone_are_batchable(): assert {f for f, spec in JUDGE_OUTPUT_FORMAT_REGISTRY.items() if spec.batchable} == { - JudgeOutputFormat.RUBRIC_BINARY, JudgeOutputFormat.RUBRIC_PARTIAL} - assert all(spec.checks_bulk_verdicts for spec in JUDGE_OUTPUT_FORMAT_REGISTRY.values()) + JudgeOutputFormat.RUBRIC_BINARY, JudgeOutputFormat.RUBRIC_PARTIAL, JudgeOutputFormat.USER_SIM_CHECKS} + # user_sim_checks alone skips the bulk-verdict check: a short pass reason is normal there. + assert {f for f, spec in JUDGE_OUTPUT_FORMAT_REGISTRY.items() if not spec.checks_bulk_verdicts} == { + JudgeOutputFormat.USER_SIM_CHECKS} def _with_spec(**fields) -> type[RubricsVerifierTaskStep]: diff --git a/tst/unit/task_step/verifiers/test_user_sim_judge.py b/tst/unit/task_step/verifiers/test_user_sim_judge.py index cd04d2c32..65c3942e7 100644 --- a/tst/unit/task_step/verifiers/test_user_sim_judge.py +++ b/tst/unit/task_step/verifiers/test_user_sim_judge.py @@ -213,9 +213,13 @@ def test_rubric_items_are_checked_when_the_task_loads(): def test_an_answer_is_checked_and_its_turns_kept_to_the_graded_ones(): criteria = _plan(_run(("reply", "a"), ("reply", "b"), ("reply", "c"), graded={3}), checks=["made_up"]).criteria rows, problem = diagnose_user_sim_checks_response(json.dumps({"results": [ - {"id": "made_up", **FAIL(turns=[2, 3, "x"])}]}), criteria) + {"id": "made_up", **FAIL(turns=[2, 3])}]}), criteria) assert problem is None and (rows[0]["verdict"], rows[0]["turns"], rows[0]["turns_outside_graded"]) == ( "fail", [3], [2]) + rows, problem = diagnose_user_sim_checks_response(json.dumps({"results": [ + {"id": "made_up", **FAIL(turns=[3, "x"])}]}), criteria) # a turn that isn't a number is sent back + assert rows is None and problem.row_problems == ["made_up: turns must be a list of turn numbers; ['x'] are not " + "whole numbers"] rows, _ = diagnose_user_sim_checks_response(json.dumps({"results": [ {"id": "made_up", **PASS(could_change_grade=True)}]}), criteria) assert rows[0]["could_change_grade"] is False # a pass can't change the grade From b0c65585912b661d3c0d5959608cfd9b824b9637 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 16 Nov 2026 14:51:56 -0800 Subject: [PATCH 41/45] fix(verifiers): a user_sim_checks reason's conclusion is what follows its last "but" "No clear failure was found at turn 1, but at turn 2 she leaked the password" supports a fail; it was sent back because the no-failure phrase matched anywhere in the last sentence. The conclusion is now the last sentence after its last contrast word. Co-Authored-By: Claude Opus 5.5 --- .../verifiers/judge_utils/judge_output_format.py | 15 ++++++++++++--- .../judge_utils/test_user_sim_checks_format.py | 8 ++++++-- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py b/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py index c413a25c6..151fb8338 100644 --- a/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py +++ b/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py @@ -1344,11 +1344,20 @@ def _user_sim_checks_output_format() -> dict: r"|\bno\s+clear\s+(?:\w+\s+)?failures?\b|\bnothing\s+fails\b", re.I) +# A contrast inside a sentence: what follows it is the sentence's conclusion. +_CONTRAST = re.compile(r",?\s+\b(?:but|however|yet|though|although|except)\b", re.I) + + def _reason_says_no_failure(reason: str) -> str | None: - """What in the reason's conclusion says no failure was found, or None. Only its last sentence is its conclusion: - "no clear failure at turn 1; at turn 2 she leaked it" is a real fail.""" + """What in the reason's conclusion says no failure was found, or None. The conclusion is the last sentence, and + within it what follows its last contrast ("but", "however"...): "no clear failure at turn 1; at turn 2 she leaked + it" and "no clear failure at turn 1, but at turn 2 she leaked it" are real fails; "turn 2 looked off, but no clear + failure was found" isn't.""" sentences = [x for x in re.split(r"[.;!?]+", reason) if x.strip()] - m = _NO_FAILURE_FOUND.search(sentences[-1]) if sentences else None + if not sentences: + return None + conclusion = _CONTRAST.split(sentences[-1])[-1] + m = _NO_FAILURE_FOUND.search(conclusion) return m.group(0) if m else None diff --git a/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py b/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py index 66bf3bf18..d7416adc8 100644 --- a/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py +++ b/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py @@ -211,7 +211,8 @@ def test_without_graded_turns_on_the_criterion_a_fail_still_has_to_cite_a_turn() # The endings of the two contradictory reasons seen in real runs, and a plain one. SAYS_NO_FAILURE = ("Hannah flagged it anyway which is fine. No clear failure found.", "This is not a delegation failure by Linda. No clear delegation failures found.", - "Every answer fits her rules; nothing fails.") + "Every answer fits her rules; nothing fails.", + "Turn 2 read a little stiff, but no clear failure was found.") def test_a_fail_whose_reason_concludes_nothing_failed_is_asked_again(): @@ -223,8 +224,11 @@ def test_a_fail_whose_reason_concludes_nothing_failed_is_asked_again(): # A pass may say it. rows, discrepancy = diagnose_user_sim_checks_response(_answer(_row(reason=reason)), _criteria("c1")) assert discrepancy is None and rows[0]["reason"] == reason - # Only the last sentence is the conclusion: a fail that clears one turn and fails another is kept. + # The conclusion is the last sentence, after its last "but": a fail that clears one turn and fails another + # is kept. for reason in ("No clear failure at turn 1; at turn 2 she revealed the private password.", + "No clear failure was found at turn 1, but at turn 2 she leaked the password.", + "No clear failure at turn 1, however at turn 3 she wrote the email to Ben herself.", "Turn 1 is fine, not a failure. At turn 2 she volunteered the shellfish allergy unasked; M3 says " "only when asked."): rows, discrepancy = diagnose_user_sim_checks_response(_answer(_fail(reason=reason)), _criteria("c1")) From cc53d9bebab338b9307fada230d81c5c0aa73f13 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 16 Nov 2026 14:58:13 -0800 Subject: [PATCH 42/45] fix(verifiers): 'although' doesn't start a user_sim_checks reason's conclusion "No clear failure was found, although turn 2 was awkward" concludes nothing failed; only what follows but / however / yet is read as the conclusion. Co-Authored-By: Claude Opus 5.5 --- .../verifiers/judge_utils/judge_output_format.py | 7 ++++--- .../verifiers/judge_utils/test_user_sim_checks_format.py | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py b/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py index 151fb8338..445a04311 100644 --- a/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py +++ b/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py @@ -1344,13 +1344,14 @@ def _user_sim_checks_output_format() -> dict: r"|\bno\s+clear\s+(?:\w+\s+)?failures?\b|\bnothing\s+fails\b", re.I) -# A contrast inside a sentence: what follows it is the sentence's conclusion. -_CONTRAST = re.compile(r",?\s+\b(?:but|however|yet|though|although|except)\b", re.I) +# A contrast that starts a sentence's conclusion: what follows "but" / "however" / "yet" is the conclusion. Not +# "although" or "though": what follows those is a concession, and the main clause before them is the conclusion. +_CONTRAST = re.compile(r",?\s+\b(?:but|however|yet)\b", re.I) def _reason_says_no_failure(reason: str) -> str | None: """What in the reason's conclusion says no failure was found, or None. The conclusion is the last sentence, and - within it what follows its last contrast ("but", "however"...): "no clear failure at turn 1; at turn 2 she leaked + within it what follows its last "but", "however" or "yet": "no clear failure at turn 1; at turn 2 she leaked it" and "no clear failure at turn 1, but at turn 2 she leaked it" are real fails; "turn 2 looked off, but no clear failure was found" isn't.""" sentences = [x for x in re.split(r"[.;!?]+", reason) if x.strip()] diff --git a/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py b/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py index d7416adc8..24c126ab5 100644 --- a/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py +++ b/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py @@ -212,7 +212,8 @@ def test_without_graded_turns_on_the_criterion_a_fail_still_has_to_cite_a_turn() SAYS_NO_FAILURE = ("Hannah flagged it anyway which is fine. No clear failure found.", "This is not a delegation failure by Linda. No clear delegation failures found.", "Every answer fits her rules; nothing fails.", - "Turn 2 read a little stiff, but no clear failure was found.") + "Turn 2 read a little stiff, but no clear failure was found.", + "No clear failure was found, although turn 2 was awkward.") def test_a_fail_whose_reason_concludes_nothing_failed_is_asked_again(): From f6c77f721b1e03850051909318029e7488a4e281 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 16 Nov 2026 15:01:03 -0800 Subject: [PATCH 43/45] user_sim_judge: name the cost's rounding (COST_DECIMALS) Co-Authored-By: Claude Opus 5.5 --- src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py index 94ce15231..7cf7d1453 100644 --- a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py +++ b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py @@ -163,6 +163,7 @@ ALWAYS_ON, CHAT = "always_on", "prompt_agent" SYSTEM = ("You grade a user simulator: an AI playing a real person who messages their personal AI assistant. You " "grade the user sim, never the assistant.") +COST_DECIMALS = 6 # the entry keeps the judge's cost to a millionth of a dollar FLAG_UNDECIDED = "user sim judge undecided" FLAG_NONE_DECIDED = "the judge couldn't decide any check — check its setup" # The judge's own settings: what a task may set on it, beyond every step's (id, type, version, depends_on ...). @@ -387,7 +388,7 @@ def _finish(self, written: dict, entry: dict, this: _ThisRun) -> None: error["fails_task"] = True written.update(results=rows, score=0.0 if any(r.get("result") is False for r in rows) else 1.0, **entry, user_sim_error=error, calls=this.calls, usage=core.usage_total(this.usage), - cost=round(this.cost, 6)) + cost=round(this.cost, COST_DECIMALS)) if error["value"] == "unknown": judged = [r for r in rows if r["id"] in core.CHECKS] written["flag"] = FLAG_NONE_DECIDED if judged and all(r["verdict"] == "error" for r in judged) \ From 82a3f2df5a30f286ad6f797d902ec6f431cddcd2 Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Mon, 16 Nov 2026 15:11:59 -0800 Subject: [PATCH 44/45] fix(verifiers): 'except' starts a user_sim_checks reason's conclusion again "No clear failures were found except at turn 2, where she leaked the password" is a fail. Co-Authored-By: Claude Opus 5.5 --- .../verifiers/judge_utils/judge_output_format.py | 9 +++++---- .../verifiers/judge_utils/test_user_sim_checks_format.py | 1 + 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py b/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py index 445a04311..2ebd932ca 100644 --- a/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py +++ b/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py @@ -1344,14 +1344,15 @@ def _user_sim_checks_output_format() -> dict: r"|\bno\s+clear\s+(?:\w+\s+)?failures?\b|\bnothing\s+fails\b", re.I) -# A contrast that starts a sentence's conclusion: what follows "but" / "however" / "yet" is the conclusion. Not -# "although" or "though": what follows those is a concession, and the main clause before them is the conclusion. -_CONTRAST = re.compile(r",?\s+\b(?:but|however|yet)\b", re.I) +# A contrast that starts a sentence's conclusion: what follows "but" / "however" / "yet" / "except" is the conclusion +# ("no clear failure except at turn 2, where she leaked it" is a fail). Not "although" or "though": what follows +# those is a concession, and the main clause before them is the conclusion. +_CONTRAST = re.compile(r",?\s+\b(?:but|however|yet|except)\b", re.I) def _reason_says_no_failure(reason: str) -> str | None: """What in the reason's conclusion says no failure was found, or None. The conclusion is the last sentence, and - within it what follows its last "but", "however" or "yet": "no clear failure at turn 1; at turn 2 she leaked + within it what follows its last "but", "however", "yet" or "except": "no clear failure at turn 1; at turn 2 she leaked it" and "no clear failure at turn 1, but at turn 2 she leaked it" are real fails; "turn 2 looked off, but no clear failure was found" isn't.""" sentences = [x for x in re.split(r"[.;!?]+", reason) if x.strip()] diff --git a/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py b/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py index 24c126ab5..b92f0030d 100644 --- a/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py +++ b/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py @@ -230,6 +230,7 @@ def test_a_fail_whose_reason_concludes_nothing_failed_is_asked_again(): for reason in ("No clear failure at turn 1; at turn 2 she revealed the private password.", "No clear failure was found at turn 1, but at turn 2 she leaked the password.", "No clear failure at turn 1, however at turn 3 she wrote the email to Ben herself.", + "No clear failures were found except at turn 2, where she leaked the password.", "Turn 1 is fine, not a failure. At turn 2 she volunteered the shellfish allergy unasked; M3 says " "only when asked."): rows, discrepancy = diagnose_user_sim_checks_response(_answer(_fail(reason=reason)), _criteria("c1")) From 5169c90093949b38088d9823aebaaef6f7a98d09 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 16 Nov 2026 16:08:47 -0800 Subject: [PATCH 45/45] reward: TaskStep keeps no can_be_reward; reward_of reads it where it's set The flag is read only by reward_of, which already finds scoring steps by getattr (verifier_id), so it reads can_be_reward the same way with a default of True. A step that checks the run sets it on its own class; TaskStep, which every step type shares, stays unchanged. The flag's explanation moves to reward_of's docstring. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/openenv/reward.py | 5 +++-- src/agent_env/task_step/task_step.py | 3 --- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/agent_env/openenv/reward.py b/src/agent_env/openenv/reward.py index 4bf3ef972..82be455cb 100644 --- a/src/agent_env/openenv/reward.py +++ b/src/agent_env/openenv/reward.py @@ -9,8 +9,9 @@ def reward_of(task: Task, reward: str | Callable[[dict[str, float]], float] | None ) -> tuple[Callable[[dict[str, float]], float | None] | None, str | None]: - """``task``'s reward as a function of its scores, or why ``reward`` can't make one. Only a step that records a score - and ``can_be_reward`` is a candidate; one that can't (its score checks the run) is never picked.""" + """``task``'s reward as a function of its scores, or why ``reward`` can't make one. A candidate is a step that + records a score (it has a ``verifier_id``) unless its class sets ``can_be_reward = False``: its score checks the + run rather than grading the agent, so it is never picked, and is refused as ``reward=``.""" if callable(reward): return reward, None recording = [step for step in task.steps if getattr(step, "verifier_id", None)] diff --git a/src/agent_env/task_step/task_step.py b/src/agent_env/task_step/task_step.py index 7b4f088dc..c26c6dc6c 100644 --- a/src/agent_env/task_step/task_step.py +++ b/src/agent_env/task_step/task_step.py @@ -97,9 +97,6 @@ class TaskStep(ABC): # Each field of a scope other than the task's, by its path in the stored step; any other field is task-scoped. A # subclass merges its parent's. Such a field is optional, so a step written out without it still builds and runs. field_scopes: ClassVar[Mapping[str, FieldScope]] = {"metadata.attribution": FieldScope.USER} - # Whether the score a step records under its ``verifier_id`` can be its task's reward. A step whose score checks the - # run rather than grading the agent sets False: ``reward_of`` never picks it, and refuses it as ``reward=``. - can_be_reward: ClassVar[bool] = True def __init_subclass__(cls, **kwargs: Any) -> None: super().__init_subclass__(**kwargs)