diff --git a/src/agent_env/openenv/reward.py b/src/agent_env/openenv/reward.py index 08778e775..82be455cb 100644 --- a/src/agent_env/openenv/reward.py +++ b/src/agent_env/openenv/reward.py @@ -9,16 +9,22 @@ def reward_of(task: Task, reward: str | Callable[[dict[str, float]], float] | None ) -> tuple[Callable[[dict[str, float]], float | None] | None, str | None]: - """``task``'s reward as a function of its scores, or why ``reward`` can't make one.""" + """``task``'s reward as a function of its scores, or why ``reward`` can't make one. A candidate is a step that + records a score (it has a ``verifier_id``) unless its class sets ``can_be_reward = False``: its score checks the + run rather than grading the agent, so it is never picked, and is refused as ``reward=``.""" if callable(reward): return reward, None - scoring = [step.id for step in task.steps if getattr(step, "verifier_id", None)] + recording = [step for step in task.steps if getattr(step, "verifier_id", None)] + scoring = [step.id for step in recording if getattr(step, "can_be_reward", True)] if reward is None: if len(scoring) > 1: return None, (f"task {task.id!r} records a score at each of {', '.join(scoring)}; name the one that is " "its reward with reward=") return (_score_at(scoring[0]) if scoring else _unscored), None if reward not in scoring: + if checking := next((step for step in recording if step.id == reward), None): + return None, (f"task {task.id!r}'s step {reward!r} ({checking.type}) records a score that checks the run " + f"and can't be its reward; the scores that can are at {scoring}") return None, f"task {task.id!r} records no score at step {reward!r}; it records them at {scoring}" return _score_at(reward), None diff --git a/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py b/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py index 78eab7c19..0231964f2 100644 --- a/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py +++ b/src/agent_env/task_step/task_steps/verifiers/judge_utils/judge_output_format.py @@ -1259,6 +1259,12 @@ class JudgeOutputFormatSpec: # Rows carry `frame` / `evidence_status`, so the per-criterion grounding asks the judge to cite them; a format # with this set needs the labelled per-criterion frames (the verifier enforces the configuration). cites_evidence: bool = False + # Rows are one per criterion, so judge_batch_size may split the criteria into calls whose rows merge by + # concatenation. A format with a per-call block (reasoning, checks, mistakes) or whole-run rows leaves it off. + batchable: bool = False + # Batched rows are checked for bulk verdicts (shared or short justifications) and retried with a correction. A + # format whose rows may rightly share a short reason turns it off. + checks_bulk_verdicts: bool = True JUDGE_OUTPUT_FORMAT_REGISTRY: dict[JudgeOutputFormat, JudgeOutputFormatSpec] = { @@ -1270,6 +1276,7 @@ class JudgeOutputFormatSpec: parse_response=_parse_rubric_binary_response, diagnose_response=diagnose_rubric_binary_response, format_correction_prompt=format_judge_correction_prompt, + batchable=True, ), JudgeOutputFormat.RUBRIC_PARTIAL: JudgeOutputFormatSpec( output_format=_rubric_score_output_format(), @@ -1279,6 +1286,7 @@ class JudgeOutputFormatSpec: parse_response=_parse_rubric_partial_response, diagnose_response=diagnose_rubric_partial_response, format_correction_prompt=format_partial_judge_correction_prompt, + batchable=True, ), JudgeOutputFormat.RUBRIC_EVIDENCE: JudgeOutputFormatSpec( output_format=_rubric_evidence_output_format(), diff --git a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py index 24d600fe8..60996ca56 100644 --- a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py @@ -144,10 +144,6 @@ def trajectory_mistakes_key(verifier_id: str) -> str: _BULK_LOGGED_EXCERPT_CHARS = 80 -def _criteria_batches(criteria: list[dict], size: int) -> list[list[dict]]: - return [criteria[i:i + size] for i in range(0, len(criteria), size)] or [[]] - - def _bulk_verdicts(rows: list[dict]) -> Optional[dict]: """What makes ``rows`` look scripted rather than graded one by one, or None. A judge facing many criteria tends to grep for a few keywords and assign the rest a default with a stock justification.""" @@ -276,6 +272,21 @@ def _looks_like_response_format_error(exc: Exception) -> bool: return any(s in msg for s in _SCHEMA_TERMS) and any(u in msg for u in _UNSUPPORTED_TERMS) +def _caches_prompts(model: str) -> bool: + """Whether a judge call to ``model`` marks its shared prefix for the prompt cache: Claude models, by any route.""" + return "claude" in model.lower() + + +def _usage_and_cost(response: Any) -> dict: + """A LiteLLM response's token counts and LiteLLM's price of it in USD, each None when it has none.""" + usage = getattr(response, "usage", None) + usage = usage.model_dump() if hasattr(usage, "model_dump") else usage + hidden = getattr(response, "_hidden_params", None) + cost = hidden.get("response_cost") if isinstance(hidden, dict) else None + return {"usage": usage if isinstance(usage, dict) else None, + "cost": float(cost) if isinstance(cost, (int, float)) and not isinstance(cost, bool) else None} + + def _image_url_block(frame: ImageFrame) -> dict: """Adapt a neutral trajectory image to LiteLLM's image block shape.""" return { @@ -393,11 +404,6 @@ def _validated_batch_option(name: str, value: Optional[int]) -> Optional[int]: return value -# Formats whose rows are one per criterion, so batches merge by concatenation. The evidence formats carry one -# reasoning / checks / mistakes block per call and trajectory_mistakes adds whole-run rows, which batches would repeat. -_BATCHABLE_FORMATS = frozenset({JudgeOutputFormat.RUBRIC_BINARY, JudgeOutputFormat.RUBRIC_PARTIAL}) - - def _unbatchable_format_message(output_format: JudgeOutputFormat) -> str: return f"judge_batch_size does not apply to output_format={output_format.value}." @@ -592,7 +598,7 @@ def __init__( "with use_trajectory=True and a trajectory_filter with compaction_type=screenshot and " "frame_strategy=per_criterion." ) - if self.judge_batch_size is not None and self.output_format not in _BATCHABLE_FORMATS: + if self.judge_batch_size is not None and not self._spec().batchable: raise ValueError(_unbatchable_format_message(self.output_format)) def _spec(self) -> JudgeOutputFormatSpec: @@ -705,6 +711,29 @@ async def judge_prompt_sections(self, context: TaskStepContext, prompt_response: per run (never for one skipped) with the prompt's stored response. None here.""" return [] + async def judge_trajectory_content(self, context: TaskStepContext, prompt_response: PromptResponse) -> Optional[str]: + """What the judge prompt shows as the run's trajectory (the inline template's ``$trajectory_content``), in place + of a recorded trajectory, for a step with ``use_trajectory`` off: an override point, called once per run (never + for one skipped) with the prompt's stored response. A subclass that puts the run here, rather than in + ``judge_prompt_sections`` after the criteria, keeps it in the part of the prompt each batch shares. None here: + the prompt shows the recorded trajectory, or none.""" + return None + + async def unjudged_rows(self, context: TaskStepContext, prompt_response: PromptResponse) -> list[dict]: + """Rows no judge grades, computed from the run or fixed, that the entry holds after the judge's and that are + scored with them: an override point, called once per run before the judge (never for one skipped) with the + prompt's stored response. Each is shaped as a judge's row (``id``, ``result`` / ``score``, any ``weight``), + with an id no criterion has. None here.""" + return [] + + def _criteria_batches(self, criteria: list[dict], size: Optional[int]) -> list[list[dict]]: + """The judge calls ``criteria`` are graded in, one batch per call: ``size`` at a time in order, or all in one + call when ``size`` is None. An override point: a subclass may group them otherwise, putting every criterion + in exactly one non-empty batch. The criteria carry their short ids (c1..cN) and every other field.""" + if size is None: + return [criteria] + return [criteria[i:i + size] for i in range(0, len(criteria), size)] or [[]] + async def stage_trajectory(self, context: TaskStepContext, prompt_response: PromptResponse, run_context: Optional[Run], sandbox: Any, container: Optional[str]) -> Optional[StagedTrajectory]: @@ -836,6 +865,16 @@ async def _grade(self, context: TaskStepContext) -> TaskStepContext: "but this run's trajectory filter attached none (user_overrides.apply_trajectory_filter=False?)." ) + # The subclass hooks run once the run is known not to be skipped, and before any judge is deployed. + trajectory_content = await self.judge_trajectory_content(context, stored_response) + if trajectory_content is not None and self.use_trajectory: + raise RuntimeError(f"Verifier '{self.verifier_id}': judge_trajectory_content stands in for the recorded " + "trajectory, so it needs use_trajectory=False.") + unjudged = [dict(row) for row in await self.unjudged_rows(context, stored_response)] + if clashing := {row.get("id") for row in unjudged} & {c.get("id") for c in criteria}: + raise RuntimeError(f"Verifier '{self.verifier_id}': unjudged_rows gave rows with a criterion's id: " + f"{sorted(map(str, clashing))}") + auto_deployed_judge = None staged: Optional[StagedTrajectory] = None judge_agent = None @@ -919,7 +958,7 @@ async def _grade(self, context: TaskStepContext) -> TaskStepContext: overrides, "judge_batch_size", self.judge_batch_size)) batch_concurrency = _validated_batch_option("judge_batch_concurrency", _override_or( overrides, "judge_batch_concurrency", self.judge_batch_concurrency)) - if batch_size is not None and self.output_format not in _BATCHABLE_FORMATS: + if batch_size is not None and not self._spec().batchable: raise RuntimeError(f"Verifier '{self.verifier_id}': {_unbatchable_format_message(self.output_format)}") loaded_for_judge: list[dict] = [] if use_agent_judge and self.agent_name is not None: @@ -953,6 +992,8 @@ async def _grade(self, context: TaskStepContext) -> TaskStepContext: trajectory_inline = compacted.inline_text if trajectory_inline is None and compacted.container_object_url: trajectory_inline = await asyncio.to_thread(self._read_trajectory_text, compacted.container_object_url) + if trajectory_content is not None: + trajectory_inline = trajectory_content def eval_prompt_for(batch: list[dict]) -> str: criteria_json = json.dumps([ @@ -965,6 +1006,7 @@ def eval_prompt_for(batch: list[dict]) -> str: agent_response=prompt_response.response, criteria_json=criteria_json, trajectory_path=trajectory_path, + trajectory_inline=trajectory_inline, loaded_artifacts=loaded_for_judge, context_facts=context_facts, run_context=run_context, @@ -996,10 +1038,17 @@ def eval_prompt_for(batch: list[dict]) -> str: prompt += self._PER_ROW_EVIDENCE_NOTE return prompt - batches = (_criteria_batches(criteria_for_judge, batch_size) if batch_size is not None - else [criteria_for_judge]) - concurrency = asyncio.Semaphore(batch_concurrency or self.DEFAULT_JUDGE_BATCH_CONCURRENCY) + batches = self._criteria_batches(criteria_for_judge, batch_size) position = {c["id"]: i for i, c in enumerate(criteria_for_judge, 1)} + batched_ids = [c["id"] for batch in batches for c in batch] + if sorted(batched_ids) != sorted(position) or (len(batches) > 1 and not all(batches)): + raise RuntimeError(f"Verifier '{self.verifier_id}': _criteria_batches must put every criterion in " + "exactly one batch, and leave no batch empty") + if len(batches) > 1 and not self._spec().batchable: + # Batches merge by concatenating rows, which drops a format's one-per-call reasoning, checks and mistakes. + raise RuntimeError(f"Verifier '{self.verifier_id}': _criteria_batches gave {len(batches)} batches, but " + f"output_format={self.output_format.value} is graded in one judge call.") + concurrency = asyncio.Semaphore(batch_concurrency or self.DEFAULT_JUDGE_BATCH_CONCURRENCY) async def grade(batch: list[dict]) -> tuple[list[dict], int, list[dict], Optional[str]]: prompt = eval_prompt_for(batch) @@ -1020,7 +1069,7 @@ async def grade(batch: list[dict]) -> tuple[list[dict], int, list[dict], Optiona use_agent_judge=use_agent_judge, criteria=batch, image_blocks=image_blocks, - reject_bulk_verdicts=batch_size is not None, + reject_bulk_verdicts=batch_size is not None and self._spec().checks_bulk_verdicts, ) if len(batches) == 1: @@ -1066,15 +1115,16 @@ async def grade(batch: list[dict]) -> tuple[list[dict], int, list[dict], Optiona judge_call_fields["graded_up_to_turn"] = grade_up_to_turn if staged is not None: judge_call_fields["trajectory_staging"] = staged.record - self._write_entries(context, verification_results, judge_call_fields) + self._write_entries(context, verification_results, judge_call_fields, unjudged=unjudged) return context - def _write_entries(self, context: TaskStepContext, verification_results: list[dict], judge_call_fields: dict) -> None: + def _write_entries(self, context: TaskStepContext, verification_results: list[dict], judge_call_fields: dict, + *, unjudged: Sequence[dict] = ()) -> None: """Store the rubric entry under ``verifier_id`` and, for rubric_evidence_with_mistakes, the trajectory-mistakes entry under ``trajectory_mistakes_key(verifier_id)``; ``judge_call_fields`` (URIs, retries) go on both. The mistake rows are built before either entry is written, so a response they - cannot be derived from leaves neither behind.""" - score = aggregate_score(verification_results, self.score_aggregator) + cannot be derived from leaves neither behind. ``unjudged`` rows (``unjudged_rows``) join the rubric + entry's, after the judge's, and its score; the mistakes entry is the judge's alone.""" mistake_rows: Optional[list[dict]] = None if self._writes_mistakes_entry: if not isinstance(verification_results, ResultRows): @@ -1082,6 +1132,8 @@ def _write_entries(self, context: TaskStepContext, verification_results: list[di f"Verifier '{self.verifier_id}': the judge response carries no mistake findings " f"({type(verification_results).__name__})") mistake_rows = trajectory_mistakes_rows_from_evidence(verification_results) + verification_results.extend(unjudged) # in place, so a ResultRows keeps its reasoning and checks + score = aggregate_score(verification_results, self.score_aggregator) verifications = context.metadata.setdefault("verifications", {}) verification_entry = { **self._entry_format_fields(), @@ -1274,11 +1326,19 @@ async def _prompt_llm_judge( model: str, context: TaskStepContext, image_blocks: Optional[list[dict]] = None, + system: Optional[str] = None, + cached_prefix: Optional[str] = None, ) -> dict: - """Direct LiteLLM judge — no agent boot, no tools. Returns {response, trajectory_object_url}. - - When ``image_blocks`` are provided the user message becomes multimodal - (text + images); otherwise it stays a plain string, identical to before.""" + """Direct LiteLLM judge — no agent boot, no tools. Returns {response, trajectory_object_url, usage, cost}: + ``usage`` the call's token counts and ``cost`` LiteLLM's price of it in USD, each None when the response + has none, for a caller that records them (this step records neither). + + When ``image_blocks`` are provided the user message becomes multimodal (text + images); otherwise it stays + a plain string. ``system`` is sent as a system message first. ``cached_prefix`` is the start of + ``eval_prompt`` that each of the run's calls shares: on a Claude model it is sent as its own text block, + marked for Anthropic's prompt cache, so the calls after the first read it from the cache; a prompt that + doesn't start with it is sent whole. The step's own calls pass neither, so their request is unchanged; a + subclass opts in by overriding this and calling it with them.""" import litellm from agent_env.config import get_config @@ -1294,11 +1354,15 @@ async def _prompt_llm_judge( or overrides.get("litellm_base_url") or self.default_model_api_base, ) - user_content = ( - [{"type": "text", "text": eval_prompt}, *image_blocks] - if image_blocks - else eval_prompt - ) + text_blocks: list[dict] = [{"type": "text", "text": eval_prompt}] + cached = bool(cached_prefix) and eval_prompt.startswith(cached_prefix or "") and _caches_prompts(model) + if cached: + rest = eval_prompt[len(cached_prefix or ""):] + text_blocks = [{"type": "text", "text": cached_prefix, "cache_control": {"type": "ephemeral"}}, + *([{"type": "text", "text": rest}] if rest else [])] + user_content = [*text_blocks, *(image_blocks or [])] if image_blocks or cached else eval_prompt + messages = [*([{"role": "system", "content": system}] if system else []), + {"role": "user", "content": user_content}] response_format = { "type": "json_schema", "json_schema": { @@ -1316,7 +1380,7 @@ async def _prompt_llm_judge( try: request_kwargs: dict = dict( model=model, - messages=[{"role": "user", "content": user_content}], + messages=messages, timeout=self.judge_timeout_seconds, **call_config.client_kwargs(), ) @@ -1324,7 +1388,7 @@ async def _prompt_llm_judge( request_kwargs[ModelParam.RESPONSE_FORMAT] = response_format response = await litellm.acompletion(**request_kwargs) content = response.choices[0].message.content or "" - return {"response": content, "trajectory_object_url": None} + return {"response": content, "trajectory_object_url": None, **_usage_and_cost(response)} except Exception as exc: last_exc = exc if image_blocks and use_response_format and _looks_like_response_format_error(exc): diff --git a/tst/unit/openenv/reward_test.py b/tst/unit/openenv/reward_test.py new file mode 100644 index 000000000..fcd9553b1 --- /dev/null +++ b/tst/unit/openenv/reward_test.py @@ -0,0 +1,56 @@ +"""``reward_of``: which recorded score is a task's reward. A step whose class says its score can't be the reward (it +checks the run, as a user sim judge does) is never picked and is refused as ``reward=``.""" + +from typing import ClassVar + +from agent_env.openenv.reward import reward_of +from agent_env.task import Task +from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep + + +class _RunCheck(RubricsVerifierTaskStep): + can_be_reward: ClassVar[bool] = False + + +def _rubric(cls: type[RubricsVerifierTaskStep] = RubricsVerifierTaskStep, *, id: str) -> RubricsVerifierTaskStep: + return cls(id=id, version=None, criteria=[], prompt_id="p1", use_agent_judge=False, verifier_id=f"{id}-v") + + +def _task(*steps: RubricsVerifierTaskStep) -> Task: + return Task(id="t", version=None, steps=list(steps)) + + +def test_a_rubric_beside_a_run_check_is_the_reward_without_naming_it(): + of, why = reward_of(_task(_rubric(id="grade"), _rubric(_RunCheck, id="user-sim")), None) + + assert why is None + assert of({"grade": 0.75, "user-sim": 0.0}) == 0.75 + + +def test_a_task_whose_only_score_is_a_run_check_is_unscored(): + of, why = reward_of(_task(_rubric(_RunCheck, id="user-sim")), None) + + assert why is None + assert of({"user-sim": 1.0}) is None + + +def test_a_run_check_named_as_the_reward_is_refused(): + of, why = reward_of(_task(_rubric(id="grade"), _rubric(_RunCheck, id="user-sim")), "user-sim") + + assert of is None + assert why == ("task 't''s step 'user-sim' (rubrics_verifier) records a score that checks the run and can't be " + "its reward; the scores that can are at ['grade']") + + +def test_two_rubrics_still_need_one_named(): + task = _task(_rubric(id="grade"), _rubric(id="again"), _rubric(_RunCheck, id="user-sim")) + + assert reward_of(task, None) == (None, "task 't' records a score at each of grade, again; name the one that is " + "its reward with reward=") + of, why = reward_of(task, "again") + assert why is None and of({"grade": 0.0, "again": 1.0, "user-sim": 0.0}) == 1.0 + + +def test_a_step_that_records_no_score_is_refused_as_before(): + assert reward_of(_task(_rubric(id="grade")), "nowhere") == ( + None, "task 't' records no score at step 'nowhere'; it records them at ['grade']") diff --git a/tst/unit/task_step/verifiers/test_rubrics_verifier_litellm_contract.py b/tst/unit/task_step/verifiers/test_rubrics_verifier_litellm_contract.py index 035269c48..b1c38e2b2 100644 --- a/tst/unit/task_step/verifiers/test_rubrics_verifier_litellm_contract.py +++ b/tst/unit/task_step/verifiers/test_rubrics_verifier_litellm_contract.py @@ -84,3 +84,58 @@ async def test_an_endpoint_refusing_the_schema_with_images_is_asked_again_withou assert json.loads(out["response"]) == _VERDICT assert "response_format" in calls[0] assert "response_format" not in calls[1] + + +@pytest.mark.asyncio +async def test_a_default_call_sends_one_user_message_and_returns_its_usage_and_cost(judge, monkeypatch): + calls = _answer_with(monkeypatch, json.dumps(_VERDICT)) + + out = await judge._prompt_llm_judge("grade it", model="openai/gpt-4o", context=TaskStepContext()) + + assert calls[0]["messages"] == [{"role": "user", "content": "grade it"}] + assert out["usage"]["total_tokens"] > 0 + assert isinstance(out["cost"], float) and out["cost"] > 0 + + +@pytest.mark.asyncio +async def test_a_system_message_and_a_shared_prefix_marked_for_the_prompt_cache(judge, monkeypatch): + calls = _answer_with(monkeypatch, json.dumps(_VERDICT)) + + await judge._prompt_llm_judge("THE RUN\n\nTHE CHECK", model="anthropic/claude-sonnet-4-6", + context=TaskStepContext(), system="You grade the user sim.", + cached_prefix="THE RUN\n\n") + + assert calls[0]["messages"] == [ + {"role": "system", "content": "You grade the user sim."}, + {"role": "user", "content": [ + {"type": "text", "text": "THE RUN\n\n", "cache_control": {"type": "ephemeral"}}, + {"type": "text", "text": "THE CHECK"}, + ]}, + ] + + +@pytest.mark.asyncio +async def test_a_shared_prefix_rides_before_the_screenshots(judge, monkeypatch): + calls = _answer_with(monkeypatch, json.dumps(_VERDICT)) + + await judge._prompt_llm_judge("THE RUN then the check", model="claude-sonnet-4-6", context=TaskStepContext(), + image_blocks=[_IMAGE], cached_prefix="THE RUN") + + assert calls[0]["messages"][0]["content"] == [ + {"type": "text", "text": "THE RUN", "cache_control": {"type": "ephemeral"}}, + {"type": "text", "text": " then the check"}, + _IMAGE, + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("model", "prompt"), [ + ("openai/gpt-4o", "THE RUN then the check"), # not a Claude model: no cache marks + ("anthropic/claude-sonnet-4-6", "a correction, then THE RUN"), # doesn't start with the prefix +]) +async def test_a_prefix_that_cant_be_cached_sends_the_prompt_whole(judge, monkeypatch, model, prompt): + calls = _answer_with(monkeypatch, json.dumps(_VERDICT)) + + await judge._prompt_llm_judge(prompt, model=model, context=TaskStepContext(), cached_prefix="THE RUN") + + assert calls[0]["messages"] == [{"role": "user", "content": prompt}] diff --git a/tst/unit/task_step/verifiers/test_rubrics_verifier_subclass_hooks.py b/tst/unit/task_step/verifiers/test_rubrics_verifier_subclass_hooks.py new file mode 100644 index 000000000..8c35c1118 --- /dev/null +++ b/tst/unit/task_step/verifiers/test_rubrics_verifier_subclass_hooks.py @@ -0,0 +1,287 @@ +"""The override points a subclass judge builds on: ``_criteria_batches`` (which criteria share a judge call), +``unjudged_rows`` (rows no judge grades, scored with the judge's), ``judge_trajectory_content`` (the run, where the +template puts the trajectory) and the format spec's ``batchable`` / ``checks_bulk_verdicts``. Unoverridden, each +leaves the step as it was.""" +from __future__ import annotations + +from dataclasses import replace +from typing import Optional + +import pytest + +from agent_env.task_step.context import PromptResponse, TaskStepContext, VerifierNotPassingError +from agent_env.task_step.task_steps.verifiers.judge_utils.judge_output_format import ( + JUDGE_OUTPUT_FORMAT_REGISTRY, + JudgeOutputFormat, + JudgeOutputFormatSpec, + ResultRows, +) +from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep +from tst.unit.task_step.verifiers.test_rubrics_verifier_judge_batches import ( + _context, + _fake_judge, + _graded, + _prompted_ids, +) +from tst.unit.task_step.verifiers.test_rubrics_verifier_per_criterion import ( + _NO_SCREENSHOTS, + CRITERIA, + _capture_judge, + _ctx, + _filter, + _passing_rows, +) + +CHECKS = [{"id": "asked", "title": "Asked for the booking", "check": "asked"}, + {"id": "asked-item", "title": "Named the restaurant", "check": "asked"}, + {"id": "replied", "title": "Replied to the agent", "check": "replied"}, + {"id": "replied-item", "title": "Confirmed the time", "check": "replied"}] + + +class _ByCheck(RubricsVerifierTaskStep): + """Each check's criteria in one call, as a user sim judge puts a check's task items in that check's call.""" + + def _criteria_batches(self, criteria: list[dict], size: Optional[int]) -> list[list[dict]]: + groups: dict[str, list[dict]] = {} + for criterion in criteria: + groups.setdefault(criterion["check"], []).append(criterion) + return list(groups.values()) + + +class _WithRows(RubricsVerifierTaskStep): + rows: list[dict] = [] + + async def unjudged_rows(self, context: TaskStepContext, prompt_response: PromptResponse) -> list[dict]: + return self.rows + + +def _step(cls: type[RubricsVerifierTaskStep] = RubricsVerifierTaskStep, **kw) -> RubricsVerifierTaskStep: + params = dict(id="verify", version=1, criteria=CHECKS, prompt_id="p1", use_agent_judge=False, + use_trajectory=False, verifier_id="vt") + params.update(kw) + return cls(**params) + + +@pytest.mark.asyncio +async def test_a_subclass_groups_the_criteria_into_its_own_calls(): + v = _step(_ByCheck) + prompts: list[str] = [] + _fake_judge(v, prompts) + ctx = _context() + + await v.execute(ctx) + + assert sorted(_prompted_ids(p) for p in prompts) == [["c1", "c2"], ["c3", "c4"]] + entry = ctx.metadata["verifications"]["vt"] + assert [r["id"] for r in entry["results"]] == ["asked", "asked-item", "replied", "replied-item"] + assert entry["judge_batches"] == 2 + + +@pytest.mark.asyncio +async def test_batches_that_drop_a_criterion_are_refused(): + class Dropping(RubricsVerifierTaskStep): + def _criteria_batches(self, criteria: list[dict], size: Optional[int]) -> list[list[dict]]: + return [criteria[:1], criteria[2:]] + + v = _step(Dropping) + _fake_judge(v, []) + + with pytest.raises(RuntimeError, match="must put every criterion in exactly one batch"): + await v.execute(_context()) + + +@pytest.mark.asyncio +async def test_unjudged_rows_join_the_entry_and_its_score_but_never_reach_the_judge(): + v = _step(_WithRows, score_aggregator="weighted_average") + v.rows = [{"id": "auto-asked", "result": True, "score": 1.0}, + {"id": "always-na", "result": False, "score": 0.0, "weight": 0}, + {"id": "missing-line", "result": False, "score": 0.0}] + prompts: list[str] = [] + _fake_judge(v, prompts) + ctx = _context() + + await v.execute(ctx) + + assert [_prompted_ids(p) for p in prompts] == [["c1", "c2", "c3", "c4"]] + entry = ctx.metadata["verifications"]["vt"] + assert [r["id"] for r in entry["results"]] == [ + "asked", "asked-item", "replied", "replied-item", "auto-asked", "always-na", "missing-line"] + assert entry["score"] == pytest.approx(5 / 6) # the weight-0 row is recorded and never counts + + +@pytest.mark.asyncio +async def test_an_unmet_unjudged_row_fails_a_step_that_must_pass(): + v = _step(_WithRows, fail_task_when_not_passing=True) + v.rows = [{"id": "missing-line", "result": False, "score": 0.0}] + _fake_judge(v, []) + + with pytest.raises(VerifierNotPassingError, match=r"rows not met: \['missing-line'\]"): + await v.execute(_context()) + + +@pytest.mark.asyncio +async def test_an_unjudged_row_with_a_criterions_id_is_refused_before_the_judge_runs(): + v = _step(_WithRows) + v.rows = [{"id": "replied", "result": True}] + prompts: list[str] = [] + _fake_judge(v, prompts) + + with pytest.raises(RuntimeError, match=r"unjudged_rows gave rows with a criterion's id: \['replied'\]"): + await v.execute(_context()) + assert prompts == [] + + +@pytest.mark.asyncio +async def test_a_skipped_run_holds_no_unjudged_rows(): + v = _step(_WithRows) + v.rows = [{"id": "auto-asked", "result": True, "score": 1.0}] + ctx = TaskStepContext(prompt_responses=[PromptResponse(prompt_id="p1", response="", error_type="timeout")]) + + await v.execute(ctx) + + assert [r["id"] for r in ctx.metadata["verifications"]["vt"]["results"]] == ["prompt_error"] + + +class _HooksCalled(RubricsVerifierTaskStep): + calls: list[str] = [] + + async def judge_trajectory_content(self, context: TaskStepContext, prompt_response: PromptResponse) -> None: + self.calls.append("judge_trajectory_content") + + async def unjudged_rows(self, context: TaskStepContext, prompt_response: PromptResponse) -> list[dict]: + self.calls.append("unjudged_rows") + return [{"id": "auto-asked", "result": True, "score": 1.0}] + + +def _evidence_step(cls: type[RubricsVerifierTaskStep], **kw) -> RubricsVerifierTaskStep: + params = dict(id="verify", version=1, criteria=CRITERIA, prompt_id="p1", use_agent_judge=False, + use_trajectory=True, trajectory_filter=_filter(), verifier_id="vid", + output_format=JudgeOutputFormat.RUBRIC_EVIDENCE, + grading_policy_prompt="Type every value in full; never accept an autocomplete suggestion.") + params.update(kw) + return cls(**params) + + +@pytest.mark.asyncio +async def test_a_run_skipped_for_want_of_screenshots_never_calls_the_hooks(monkeypatch): + v = _evidence_step(_HooksCalled) + v.calls = [] + _capture_judge(monkeypatch, v, lambda criteria: pytest.fail("the judge must not be called")) + monkeypatch.setattr(v, "_read_trajectory_text", lambda uri: _NO_SCREENSHOTS) + ctx = _ctx() + + await v.execute(ctx) + + assert v.calls == [] + assert [r["id"] for r in ctx.metadata["verifications"]["vid"]["results"]] == ["no_screenshots"] + + +class _OnePerCriterion(RubricsVerifierTaskStep): + def _criteria_batches(self, criteria: list[dict], size: Optional[int]) -> list[list[dict]]: + return [[c] for c in criteria] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("fmt", [JudgeOutputFormat.RUBRIC_EVIDENCE, JudgeOutputFormat.RUBRIC_EVIDENCE_WITH_MISTAKES]) +async def test_a_subclass_cannot_split_a_format_graded_in_one_call(monkeypatch, fmt): + v = _evidence_step(_OnePerCriterion, output_format=fmt) + captured = _capture_judge(monkeypatch, v, _passing_rows) + ctx = _ctx() + + with pytest.raises(RuntimeError, match=rf"_criteria_batches gave 2 batches, but output_format={fmt.value} is " + "graded in one judge call"): + await v.execute(ctx) + assert "eval_prompt" not in captured and "verifications" not in ctx.metadata + + +@pytest.mark.asyncio +async def test_a_subclass_keeping_one_batch_grades_a_format_graded_in_one_call(monkeypatch): + class OneBatch(RubricsVerifierTaskStep): + def _criteria_batches(self, criteria: list[dict], size: Optional[int]) -> list[list[dict]]: + return [list(reversed(criteria))] + + v = _evidence_step(OneBatch) + _capture_judge(monkeypatch, v, lambda criteria: ResultRows( + _passing_rows(criteria), reasoning="Both shown on screen.", checks=[{"id": "typed", "passed": True}])) + ctx = _ctx() + + await v.execute(ctx) + + entry = ctx.metadata["verifications"]["vid"] + assert entry["score"] == 1.0 and entry["reasoning"] == "Both shown on screen." + assert entry["checks"] == [{"id": "typed", "passed": True}] + + +class _RunInline(RubricsVerifierTaskStep): + async def judge_trajectory_content(self, context: TaskStepContext, prompt_response: PromptResponse) -> str: + return "THE RUN: the user sim asked for a table at 7." + + +@pytest.mark.asyncio +async def test_trajectory_content_from_a_subclass_comes_before_the_criteria_in_every_batch(): + v = _step(_RunInline, judge_batch_size=2) + prompts: list[str] = [] + _fake_judge(v, prompts) + + await v.execute(_context()) + + assert len(prompts) == 2 + assert all(p.index("THE RUN") < p.index('"id": "c') for p in prompts) + + +@pytest.mark.asyncio +async def test_without_trajectory_content_the_prompt_has_no_trajectory_section(): + with_run, without = _step(_RunInline), _step() + prompts: list[str] = [] + bare: list[str] = [] + _fake_judge(with_run, prompts) + _fake_judge(without, bare) + + await with_run.execute(_context()) + await without.execute(_context()) + + assert "THE RUN" not in bare[0] and len(prompts[0]) > len(bare[0]) + + +@pytest.mark.asyncio +async def test_trajectory_content_beside_a_recorded_trajectory_is_refused(): + v = _step(_RunInline, use_trajectory=True) + ctx = TaskStepContext(prompt_responses=[PromptResponse( + prompt_id="p1", response="done", prompt_text="do it", agent_trajectory_object_url="s3://t/1.json")]) + + with pytest.raises(RuntimeError, match="judge_trajectory_content stands in for the recorded trajectory"): + await v.execute(ctx) + + +def test_the_rubric_binary_and_partial_formats_alone_are_batchable_and_all_check_bulk_verdicts(): + assert {f for f, spec in JUDGE_OUTPUT_FORMAT_REGISTRY.items() if spec.batchable} == { + JudgeOutputFormat.RUBRIC_BINARY, JudgeOutputFormat.RUBRIC_PARTIAL} + assert all(spec.checks_bulk_verdicts for spec in JUDGE_OUTPUT_FORMAT_REGISTRY.values()) + + +def _with_spec(**fields) -> type[RubricsVerifierTaskStep]: + class WithSpec(RubricsVerifierTaskStep): + def _spec(self) -> JudgeOutputFormatSpec: + return replace(super()._spec(), **fields) + + return WithSpec + + +def test_a_spec_that_isnt_batchable_refuses_a_batch_size(): + with pytest.raises(ValueError, match="judge_batch_size does not apply to output_format=rubric_binary"): + _step(_with_spec(batchable=False), judge_batch_size=2) + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("checks", "calls"), [(True, 2), (False, 1)]) +async def test_a_spec_can_turn_off_the_bulk_verdict_check(checks, calls): + v = _step(_with_spec(checks_bulk_verdicts=checks), judge_batch_size=4) + prompts: list[str] = [] + _fake_judge(v, prompts, respond=lambda ids: _graded(ids, justification=lambda cid: "Pass." if len(prompts) == 1 + else f"Checked record {cid} with the sheet tool")) + ctx = _context() + + await v.execute(ctx) + + assert len(prompts) == calls + assert ctx.metadata["verifications"]["vt"]["score"] == 1.0