From 3aab6bfd7af1f8624fbf028d3db37f1432b584ee Mon Sep 17 00:00:00 2001 From: Sarah Chitty Date: Tue, 17 Nov 2026 16:58:37 -0800 Subject: [PATCH] feat(verifiers): mandatory criteria, mandatory_all_pass, and an opt-in verdict that won't score an ungraded run rubrics_verifier (and always_on_agent_verifier) criteria take an optional `mandatory: true|false`. A new score_aggregator, mandatory_all_pass, passes a run only when every mandatory criterion passes; its entry also records `passed`, `mandatory_failed` and `rubric_satisfaction` (weights 3/1, settable as mandatory_weight / optional_weight), which never decides the pass. The judge never sees the flag. A mandatory_all_pass step with no mandatory criterion is a config problem, and is refused again at run time for a run's own criteria. aggregate_verifiers gains two opt-in settings: use_verifier_scores combines the named verifiers' own scores (so a mandatory_all_pass judge combines with end-state checks by its own rule), and fail_task_when_ungraded fails the step with UngradedVerifierError when a named verifier has no entry, skipped the run (`graded: false`, now marked on rubric and prompt-response skip entries) or recorded no score, or a step it depends on failed. Defaults are unchanged. Co-Authored-By: Claude Opus 5.5 --- .../agent_prompt_response_verifier.py | 1 + .../verifiers/aggregate_verifiers.py | 105 ++++- .../task_steps/verifiers/rubrics_verifier.py | 111 ++++- .../task_step/task_steps/verifiers/scoring.py | 61 +++ .../test_user_sim_checks_format.py | 3 +- .../verifiers/test_mandatory_verdict.py | 400 ++++++++++++++++++ 6 files changed, 660 insertions(+), 21 deletions(-) create mode 100644 tst/unit/task_step/verifiers/test_mandatory_verdict.py diff --git a/src/agent_env/task_step/task_steps/verifiers/agent_prompt_response_verifier.py b/src/agent_env/task_step/task_steps/verifiers/agent_prompt_response_verifier.py index 7d868f330..c21114277 100644 --- a/src/agent_env/task_step/task_steps/verifiers/agent_prompt_response_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/agent_prompt_response_verifier.py @@ -95,6 +95,7 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: "message": f"Skipped: prompt had error_type={prompt_response.error_type}", }], "score": 0, + "graded": False, # a skipped run: aggregate_verifiers' fail_task_when_ungraded reads it } return context diff --git a/src/agent_env/task_step/task_steps/verifiers/aggregate_verifiers.py b/src/agent_env/task_step/task_steps/verifiers/aggregate_verifiers.py index 8d663a82d..e3ba8cf84 100644 --- a/src/agent_env/task_step/task_steps/verifiers/aggregate_verifiers.py +++ b/src/agent_env/task_step/task_steps/verifiers/aggregate_verifiers.py @@ -3,11 +3,28 @@ Reads `context.metadata["verifications"][]["results"]` for each id in `verifier_ids`, concatenates the rows, drops skipped ones, and runs `aggregate_score` once on the union — same math as the upstream steps, but -applied across their combined criterion set. +applied across their combined criterion set. A named verifier with no entry fails +the step (and, with fail_task_on_error, the task). Use this to match Harbor's unified-aggregation behavior when filesystem criteria (verify_sandbox) and response criteria (rubrics_verifier) live in separate steps. + +Two settings, both off by default: + +- ``use_verifier_scores``: aggregate the named verifiers' own recorded scores instead of their + rows, each verifier one row that passes when its score is 1.0. Use it when a verifier's pass + rule isn't its rows' (a rubrics_verifier scored with ``mandatory_all_pass`` passes with optional + rows failed): with ``all_pass``, the run passes when every named verifier passed by its own rule. + A verifier whose entry records no score (None) is left out, as its rows (it has none) are when + merging. +- ``fail_task_when_ungraded``: a run whose named verifiers weren't all graded fails the step with + ``UngradedVerifierError``, an error rather than a verdict, so the run is run or graded again + instead of scored: a verifier with no entry, an entry marked ``graded: false`` (a rubrics_verifier + or agent_prompt_response_verifier that skipped a run whose prompt errored) or recording no score + (None), or a step this one depends on that failed in this run and wasn't retried, such as a + judge with fail_task_on_error off (without depends_on, any step that failed before it ran). It + needs fail_task_on_error. """ from __future__ import annotations @@ -21,6 +38,13 @@ logger = logging.getLogger(__name__) +ERROR_SHOWN_CHARS = 300 # of a failed step's error, in UngradedVerifierError's message + + +class UngradedVerifierError(RuntimeError): + """A verifier ``aggregate_verifiers`` combines wasn't graded (``fail_task_when_ungraded``): the run has no verdict + to score, and is to be run or graded again.""" + class AggregateVerifiersTaskStep(TaskStep): type: ClassVar[str] = "aggregate_verifiers" @@ -33,21 +57,44 @@ def __init__( verifier_ids: Optional[list[str]] = None, score_aggregator: Optional[ScoreAggregator] = None, verifier_id: Optional[str] = None, + use_verifier_scores: bool = False, + fail_task_when_ungraded: bool = False, depends_on: Optional[list[TaskStepDependency]] = None, fail_task_on_error: bool = True, ): super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error) + if fail_task_when_ungraded and not fail_task_on_error: + raise ValueError( + f"aggregate_verifiers '{id}': fail_task_when_ungraded needs fail_task_on_error: a tolerated failure " + "lets the steps after it run as though the run were scored." + ) self.verifier_ids = list(verifier_ids or []) if isinstance(score_aggregator, str): score_aggregator = ScoreAggregator(score_aggregator) self.score_aggregator = score_aggregator or ScoreAggregator.WEIGHTED_AVERAGE self.verifier_id = verifier_id or uuid.uuid4().hex + self.use_verifier_scores = use_verifier_scores + self.fail_task_when_ungraded = fail_task_when_ungraded + + def config_problems(self) -> list[str]: + """``use_verifier_scores`` with ``mandatory_all_pass``, which reads a flag a verifier's own row never has.""" + problems = super().config_problems() + if self.use_verifier_scores and self.score_aggregator is ScoreAggregator.MANDATORY_ALL_PASS: + problems.append( + f"aggregate_verifiers '{self.id}': use_verifier_scores makes each verifier one row, which is never " + "marked mandatory, so mandatory_all_pass could never pass; use all_pass to require every verifier " + "to pass by its own rule") + return problems def to_dict(self) -> dict: base = super().to_dict() base["verifier_ids"] = self.verifier_ids base["score_aggregator"] = self.score_aggregator.value base["verifier_id"] = self.verifier_id + if self.use_verifier_scores: + base["use_verifier_scores"] = True + if self.fail_task_when_ungraded: + base["fail_task_when_ungraded"] = True return base @classmethod @@ -58,10 +105,19 @@ def from_dict(cls, data: dict) -> "AggregateVerifiersTaskStep": verifier_ids=data.get("verifier_ids"), score_aggregator=ScoreAggregator(raw_agg) if raw_agg else None, verifier_id=data.get("verifier_id"), + use_verifier_scores=data.get("use_verifier_scores", False), + fail_task_when_ungraded=data.get("fail_task_when_ungraded", False), ) async def execute(self, context: TaskStepContext) -> TaskStepContext: + if problems := self.config_problems(): + raise ValueError("; ".join(problems)) all_verifications = context.metadata.get("verifications", {}) + if self.fail_task_when_ungraded and (ungraded := self._ungraded(context, all_verifications)): + raise UngradedVerifierError( + f"AggregateVerifiers '{self.verifier_id}': not scoring a run whose verifiers weren't all graded " + f"({'; '.join(ungraded)}); run or grade it again" + ) missing = [vid for vid in self.verifier_ids if vid not in all_verifications] if missing: raise RuntimeError( @@ -70,21 +126,52 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: f"Available: {list(all_verifications.keys())}" ) - merged_results: list[dict] = [] - for vid in self.verifier_ids: - merged_results.extend(all_verifications[vid].get("results", [])) - - non_skipped = [r for r in merged_results if not r.get("skipped")] - score = aggregate_score(non_skipped, self.score_aggregator) + source_scores: dict[str, Optional[float]] = {} + if self.use_verifier_scores: + source_scores = {vid: all_verifications[vid].get("score") for vid in self.verifier_ids} + rows = [{"id": vid, "score": float(score), "result": score >= 1} + for vid, score in source_scores.items() if score is not None] + merged_count = len(rows) + else: + merged_results: list[dict] = [] + for vid in self.verifier_ids: + merged_results.extend(all_verifications[vid].get("results", [])) + rows = [r for r in merged_results if not r.get("skipped")] + merged_count = len(merged_results) + score = aggregate_score(rows, self.score_aggregator) if "verifications" not in context.metadata: context.metadata["verifications"] = {} - context.metadata["verifications"][self.verifier_id] = { + entry: dict = { "score": score, "source_verifier_ids": list(self.verifier_ids), } + if self.use_verifier_scores: + entry["source_scores"] = source_scores + context.metadata["verifications"][self.verifier_id] = entry logger.info( f"AggregateVerifiers '{self.verifier_id}': merged {len(self.verifier_ids)} verifier(s), " - f"{len(non_skipped)}/{len(merged_results)} criteria evaluated, score={score}" + f"{len(rows)}/{merged_count} {'scores' if self.use_verifier_scores else 'criteria'} evaluated, " + f"score={score}" ) return context + + def _ungraded(self, context: TaskStepContext, verifications: dict) -> list[str]: + """Why the run's named verifiers weren't all graded, one reason each; empty when they were.""" + reasons = [] + for vid in self.verifier_ids: + entry = verifications.get(vid) + if entry is None: + reasons.append(f"'{vid}' has no entry") + elif entry.get("graded") is False: + reasons.append(f"'{vid}' didn't grade the run") + elif entry.get("score") is None: + reasons.append(f"'{vid}' recorded no score") + depends = None if self.depends_on is None else {d.task_step_id for d in self.depends_on} + for failed in context.metadata.get("failed_steps") or []: + if not isinstance(failed, dict) or failed.get("retried") or failed.get("step_id") == self.id: + continue + if depends is None or failed.get("step_id") in depends: + error = str(failed.get("error"))[:ERROR_SHOWN_CHARS] + reasons.append(f"step '{failed.get('step_id')}' failed: {error}") + return reasons diff --git a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py index 1e70514ad..e8666002a 100644 --- a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py @@ -49,7 +49,15 @@ from agent_env.store.ids import is_local_id from agent_env.task_step.task_step import TaskStep, TaskStepDependency from agent_env.task_step.task_steps.verifiers.run_context import Run, load_run_context, render_run_context -from agent_env.task_step.task_steps.verifiers.scoring import ScoreAggregator, aggregate_score +from agent_env.task_step.task_steps.verifiers.scoring import ( + DEFAULT_MANDATORY_WEIGHT, + DEFAULT_OPTIONAL_WEIGHT, + ScoreAggregator, + aggregate_score, + is_mandatory, + mandatory_verdict, + rubric_satisfaction_weight_problems, +) from agent_env.task_step.task_steps.sandbox_utils.sandbox_utils import ( agent_error_text, fetch_container_logs, @@ -110,6 +118,23 @@ def _restore_criterion_ids(results: list[dict], real_by_short: dict[str, str]) - row["id"] = real_id +def _criterion_for_judge(criterion: dict) -> dict: + """A criterion as the judge prompt shows it: without ``mandatory``, which decides how the row counts, not whether + it is met; a negative-weight one also says ``negative``.""" + shown = {key: value for key, value in criterion.items() if key != "mandatory"} + if criterion.get("weight") is not None and criterion["weight"] < 0: + shown["negative"] = True + return shown + + +def _record_mandatory(rows: list[dict], criteria: list[dict]) -> None: + """Set each row's ``mandatory`` to whether its criterion is marked ``mandatory: true``, in place; a row no criterion + has (``unjudged_rows``) keeps its own flag, read the same way.""" + flagged = {criterion.get("id"): is_mandatory(criterion) for criterion in criteria} + for row in rows: + row["mandatory"] = flagged.get(row.get("id"), is_mandatory(row)) + + _FRAME_LABEL_ID = re.compile(r"\[(c\d+)\]") @@ -511,11 +536,18 @@ def __init__( judge_batch_concurrency: Optional[int] = None, judge_timeout_retries: Optional[int] = None, judge_max_turns: Optional[int] = None, + mandatory_weight: Optional[float] = None, + optional_weight: Optional[float] = None, depends_on: Optional[list[TaskStepDependency]] = None, fail_task_on_error: bool = True, fail_task_when_not_passing: bool = False, ): super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error) + # score_aggregator=mandatory_all_pass: what a mandatory and an optional row weigh in the entry's + # rubric_satisfaction (``mandatory_verdict``); None is the default (3 and 1). Kept as given, so config_problems + # sees a value it refuses. + self.mandatory_weight = mandatory_weight + self.optional_weight = optional_weight if fail_task_when_not_passing and not fail_task_on_error: raise ValueError( "fail_task_when_not_passing needs fail_task_on_error: a tolerated failure lets the steps after it " @@ -561,6 +593,9 @@ def __init__( output_format = JudgeOutputFormat(output_format) self.output_format = output_format self.verifier_id = verifier_id or uuid.uuid4().hex + # Each criterion may set ``mandatory`` (true or false; left out, it's optional): with + # score_aggregator=mandatory_all_pass a run passes only when every mandatory criterion does. The judge + # never sees the flag; each row records it whenever the aggregator is mandatory_all_pass or a criterion sets it. self.criteria = criteria self.prompt_id = prompt_id # A stored judge always holds a model: its author's, or the one the config names for judges, as for a step @@ -601,6 +636,40 @@ def __init__( if self.judge_batch_size is not None and not self._spec().batchable: raise ValueError(_unbatchable_format_message(self.output_format)) + def config_problems(self) -> list[str]: + """A criterion whose ``mandatory`` isn't true or false; with score_aggregator=mandatory_all_pass, no criterion + marked mandatory or an unusable mandatory_weight / optional_weight; and either weight set without it.""" + where = f"{self.type} '{self.id}'" + return super().config_problems() + [f"{where}: {p}" for p in self._mandatory_problems(self.criteria)] + + def _mandatory_problems(self, criteria: list[dict]) -> list[str]: + """``config_problems``' check of ``mandatory`` and its weights, for the stored criteria or a run's.""" + problems = [f"criterion {c.get('id')!r}: mandatory must be true or false, got {c['mandatory']!r}" + for c in criteria if "mandatory" in c and not isinstance(c["mandatory"], bool)] + if self.score_aggregator is ScoreAggregator.MANDATORY_ALL_PASS: + if not any(is_mandatory(c) for c in criteria): + problems.append( + f"score_aggregator=mandatory_all_pass passes a run when every criterion marked mandatory: true " + f"passes, and none of its {len(criteria)} criteria is marked (a criterion without the field is " + "optional); mark the ones a run must meet") + problems.extend(rubric_satisfaction_weight_problems(self.mandatory_weight, self.optional_weight)) + elif self.mandatory_weight is not None or self.optional_weight is not None: + problems.append("mandatory_weight and optional_weight only apply with score_aggregator=mandatory_all_pass") + return problems + + def _records_mandatory(self, criteria: list[dict]) -> bool: + """Whether this run's rows carry ``mandatory``: with score_aggregator=mandatory_all_pass, or when a criterion + sets the field.""" + return self.score_aggregator is ScoreAggregator.MANDATORY_ALL_PASS or any("mandatory" in c for c in criteria) + + def _mandatory_verdict(self, rows: list[dict]) -> dict: + """``mandatory_verdict`` of the rows, at this step's weights.""" + return mandatory_verdict( + rows, + mandatory_weight=DEFAULT_MANDATORY_WEIGHT if self.mandatory_weight is None else float(self.mandatory_weight), + optional_weight=DEFAULT_OPTIONAL_WEIGHT if self.optional_weight is None else float(self.optional_weight), + ) + def _spec(self) -> JudgeOutputFormatSpec: return get_judge_output_format_spec(self.output_format) @@ -624,16 +693,21 @@ def _mistakes_entry_fields(self) -> dict: return {"format": JudgeOutputFormat.TRAJECTORY_MISTAKES.value, "source_verifier_id": self.verifier_id} def _write_skipped_entries(self, context: TaskStepContext, row_id: str, message: str) -> None: - """Store a zero-score entry with one row saying why the run was not judged — under ``verifier_id`` - and, for rubric_evidence_with_mistakes, ``trajectory_mistakes_key(verifier_id)``.""" + """Store a zero-score entry with one row saying why the run was not judged, marked ``graded: false`` — under + ``verifier_id`` and, for rubric_evidence_with_mistakes, ``trajectory_mistakes_key(verifier_id)``. With + score_aggregator=mandatory_all_pass the entry also records ``passed: false``, and no ``mandatory_failed`` or + ``rubric_satisfaction`` (None): no criterion was graded.""" def skipped(fields: dict) -> dict: return { **fields, "results": [{"id": row_id, "score": 0, "result": False, "message": message}], "score": 0, + "graded": False, } verifications = context.metadata.setdefault("verifications", {}) verifications[self.verifier_id] = skipped(self._entry_format_fields()) + if self.score_aggregator is ScoreAggregator.MANDATORY_ALL_PASS: + verifications[self.verifier_id].update(passed=False, mandatory_failed=None, rubric_satisfaction=None) if self._writes_mistakes_entry: verifications[trajectory_mistakes_key(self.verifier_id)] = skipped(self._mistakes_entry_fields()) @@ -671,6 +745,10 @@ def to_dict(self) -> dict: base["judge_max_turns"] = self.judge_max_turns if self.fail_task_when_not_passing: base["fail_task_when_not_passing"] = True + if self.mandatory_weight is not None: + base["mandatory_weight"] = self.mandatory_weight + if self.optional_weight is not None: + base["optional_weight"] = self.optional_weight return base @classmethod @@ -703,6 +781,8 @@ def from_dict(cls, data: dict) -> RubricsVerifierTaskStep: judge_batch_concurrency=data.get("judge_batch_concurrency"), judge_timeout_retries=data.get("judge_timeout_retries"), judge_max_turns=data.get("judge_max_turns"), + mandatory_weight=data.get("mandatory_weight"), + optional_weight=data.get("optional_weight"), fail_task_when_not_passing=data.get("fail_task_when_not_passing", False), ) @@ -767,6 +847,10 @@ async def _grade(self, context: TaskStepContext) -> TaskStepContext: prompt_response = next((pr for pr in context.prompt_responses if pr.prompt_id == self.prompt_id), None) if prompt_response is None: raise StepContextError(f"PromptResponse with prompt_id='{self.prompt_id}' not found in context") + criteria = self._resolved_criteria(context) + if problems := self._mandatory_problems(criteria): + # The stored criteria are checked when the task loads; a run's own (step_params) only here. + raise ValueError(f"Verifier '{self.verifier_id}': " + "; ".join(problems)) overrides_for_step = self.step_param_overrides(context) grade_up_to_turn = ( _validated_turn(overrides_for_step["grade_up_to_turn"]) @@ -803,7 +887,6 @@ async def _grade(self, context: TaskStepContext) -> TaskStepContext: # concurrent rollouts, so mutating self would leak the override between them. use_agent_judge_override = overrides.get("use_agent_judge") use_agent_judge = use_agent_judge_override if use_agent_judge_override is not None else self.use_agent_judge - criteria = self._resolved_criteria(context) # The judge sees/echoes short keys (c1..cN), not UUIDs; frame labels use them too, hence before compaction. criteria_for_judge, real_by_short = _to_short_keyed_criteria(criteria) apply_filter_override = overrides.get("apply_trajectory_filter") @@ -996,10 +1079,7 @@ async def _grade(self, context: TaskStepContext) -> TaskStepContext: trajectory_inline = trajectory_content def eval_prompt_for(batch: list[dict]) -> str: - criteria_json = json.dumps([ - {**c, "negative": True} if (c.get("weight") is not None and c["weight"] < 0) else c - for c in batch - ], indent=2) + criteria_json = json.dumps([_criterion_for_judge(c) for c in batch], indent=2) if use_agent_judge: prompt = self._build_eval_prompt( agent_prompt=agent_prompt_text, @@ -1099,6 +1179,9 @@ async def grade(batch: list[dict]) -> tuple[list[dict], int, list[dict], Optiona _restore_frame_labels(verification_results.checks, real_by_short) finally: await _terminate_auto_deployed_judge(auto_deployed_judge) + if self._records_mandatory(criteria): + _record_mandatory(verification_results, criteria) + _record_mandatory(unjudged, criteria) judge_call_fields: dict = { **dual_keyed("compact_trajectory_s3_uri", "compact_trajectory_object_url", compacted.compact_object_url), @@ -1124,7 +1207,9 @@ def _write_entries(self, context: TaskStepContext, verification_results: list[di trajectory-mistakes entry under ``trajectory_mistakes_key(verifier_id)``; ``judge_call_fields`` (URIs, retries) go on both. The mistake rows are built before either entry is written, so a response they cannot be derived from leaves neither behind. ``unjudged`` rows (``unjudged_rows``) join the rubric - entry's, after the judge's, and its score; the mistakes entry is the judge's alone.""" + entry's, after the judge's, and its score; the mistakes entry is the judge's alone. With + score_aggregator=mandatory_all_pass the rubric entry also records ``passed``, ``mandatory_failed`` and + ``rubric_satisfaction`` (``mandatory_verdict``).""" mistake_rows: Optional[list[dict]] = None if self._writes_mistakes_entry: if not isinstance(verification_results, ResultRows): @@ -1139,6 +1224,8 @@ def _write_entries(self, context: TaskStepContext, verification_results: list[di **self._entry_format_fields(), "results": list(verification_results), "score": score, + **(self._mandatory_verdict(verification_results) + if self.score_aggregator is ScoreAggregator.MANDATORY_ALL_PASS else {}), **judge_call_fields, } if isinstance(verification_results, ResultRows): @@ -1826,10 +1913,12 @@ def is_rubrics_verifier(step_or_config: TaskStep | dict[str, Any]) -> bool: def _raise_unless_passing(verifier_id: str, entry: dict) -> None: - """Raise VerifierNotPassingError when the recorded verdict scores below 1.0, naming the rows that weren't met.""" + """Raise VerifierNotPassingError when the recorded verdict scores below 1.0, naming the rows that weren't met: the + mandatory ones that failed, for an entry that records them (score_aggregator=mandatory_all_pass).""" if entry["score"] >= 1: return - unmet = [str(row.get("id")) for row in entry["results"] if row.get("score", 1.0 if row.get("result") else 0.0) < 1] + unmet = entry.get("mandatory_failed") or [ + str(row.get("id")) for row in entry["results"] if row.get("score", 1.0 if row.get("result") else 0.0) < 1] raise VerifierNotPassingError( f"Verifier '{verifier_id}' isn't passing (score {entry['score']}); rows not met: {unmet}" ) diff --git a/src/agent_env/task_step/task_steps/verifiers/scoring.py b/src/agent_env/task_step/task_steps/verifiers/scoring.py index 757b31058..3cde6dbd5 100644 --- a/src/agent_env/task_step/task_steps/verifiers/scoring.py +++ b/src/agent_env/task_step/task_steps/verifiers/scoring.py @@ -2,17 +2,35 @@ A verifier records per-criterion result rows; ``aggregate_score`` folds them into the single ``score`` written to ``context.metadata["verifications"][verifier_id]``. + +A row passes when its ``result`` is true: the pass rule of the judge output format that built it +(``rubric_binary``: a score of 1.0; ``rubric_partial``: a score at or above the criterion's +``pass_threshold``). """ from __future__ import annotations +import math from enum import Enum +from typing import Any + +# rubric_satisfaction's default weights: a mandatory row counts three times an optional one. +DEFAULT_MANDATORY_WEIGHT = 3.0 +DEFAULT_OPTIONAL_WEIGHT = 1.0 class ScoreAggregator(Enum): ALL_PASS = "all_pass" ANY_PASS = "any_pass" WEIGHTED_AVERAGE = "weighted_average" + # 1.0 when every row marked ``mandatory: true`` passed, whatever the optional rows did; 0.0 otherwise, and 0.0 + # when no row is mandatory (a pass needs something mandatory to have passed). + MANDATORY_ALL_PASS = "mandatory_all_pass" + + +def is_mandatory(row: dict) -> bool: + """Whether a criterion or result row is marked ``mandatory: true``; one without the field is optional.""" + return row.get("mandatory") is True def aggregate_score(results: list[dict], aggregator: ScoreAggregator) -> float: @@ -22,6 +40,9 @@ def aggregate_score(results: list[dict], aggregator: ScoreAggregator) -> float: return 1.0 if all(r.get("result") is True for r in results) else 0.0 elif aggregator == ScoreAggregator.ANY_PASS: return 1.0 if any(r.get("result") is True for r in results) else 0.0 + elif aggregator == ScoreAggregator.MANDATORY_ALL_PASS: + mandatory = [r for r in results if is_mandatory(r)] + return 1.0 if mandatory and all(r.get("result") is True for r in mandatory) else 0.0 elif aggregator == ScoreAggregator.WEIGHTED_AVERAGE: pos_num = 0.0 pos_den = 0.0 @@ -38,3 +59,43 @@ def aggregate_score(results: list[dict], aggregator: ScoreAggregator) -> float: return 0.0 return max(0.0, min(1.0, (pos_num - neg_penalty) / pos_den)) raise ValueError(f"Unknown aggregator: {aggregator}") + + +def mandatory_verdict(results: list[dict], *, mandatory_weight: float = DEFAULT_MANDATORY_WEIGHT, + optional_weight: float = DEFAULT_OPTIONAL_WEIGHT) -> dict[str, Any]: + """What a ``mandatory_all_pass`` entry records beside its score, from the rows it scored: + + - ``passed``: every mandatory row passed (and there is one), the same rule as the score. + - ``mandatory_failed``: the ids of the mandatory rows that didn't pass, in row order. + - ``rubric_satisfaction``: the weighted share of rows that passed, a mandatory row weighing ``mandatory_weight`` + and an optional one ``optional_weight`` (a row's own ``weight`` doesn't count). Reported for analysis only: it + never decides the pass. None when the rows weigh nothing. + """ + mandatory_failed = [str(r.get("id")) for r in results if is_mandatory(r) and r.get("result") is not True] + passed = any(is_mandatory(r) for r in results) and not mandatory_failed + total = 0.0 + met = 0.0 + for r in results: + weight = mandatory_weight if is_mandatory(r) else optional_weight + total += weight + if r.get("result") is True: + met += weight + return { + "passed": passed, + "mandatory_failed": mandatory_failed, + "rubric_satisfaction": met / total if total > 0 else None, + } + + +def rubric_satisfaction_weight_problems(mandatory_weight: Any, optional_weight: Any) -> list[str]: + """Why the weights ``mandatory_verdict`` reads are unusable; empty when fine. None is the default weight. Each must + be a finite number, at least 0; the mandatory one above 0, so a passing run always weighs something.""" + problems = [] + for name, value, floor in (("mandatory_weight", mandatory_weight, "above 0"), + ("optional_weight", optional_weight, "0 or more")): + if value is None: + continue + number = isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) + if not number or value < 0 or (name == "mandatory_weight" and value == 0): + problems.append(f"{name} must be a number {floor}, got {value!r}") + return problems diff --git a/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py b/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py index b92f0030d..78352967c 100644 --- a/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py +++ b/tst/unit/task_step/verifiers/judge_utils/test_user_sim_checks_format.py @@ -327,7 +327,8 @@ def test_a_weight_0_row_is_recorded_but_never_gates(): style = rows[1] assert (style["verdict"], style["could_change_grade"], style["weight"]) == ("fail", True, 0) assert (style["score"], style["result"]) == (1.0, True) - for aggregator in ScoreAggregator: + # mandatory_all_pass scores only rows marked mandatory, which a user sim check never is (the judge pins all_pass). + for aggregator in set(ScoreAggregator) - {ScoreAggregator.MANDATORY_ALL_PASS}: assert _score(_row("guidance"), _fail("style"), criteria=criteria, aggregator=aggregator) == 1.0 assert _score(_fail("guidance"), _fail("style"), criteria=criteria) == 0.0 diff --git a/tst/unit/task_step/verifiers/test_mandatory_verdict.py b/tst/unit/task_step/verifiers/test_mandatory_verdict.py new file mode 100644 index 000000000..3493a75ea --- /dev/null +++ b/tst/unit/task_step/verifiers/test_mandatory_verdict.py @@ -0,0 +1,400 @@ +"""Mandatory criteria: with score_aggregator=mandatory_all_pass a rubric judge's run passes only when every criterion +marked ``mandatory: true`` passes, whatever its optional ones did; the entry records which mandatory ones failed and a +weighted rubric_satisfaction (3 for mandatory, 1 for optional by default) that never decides the pass. The verdict +step combines it with end-state checks by each verifier's own score (use_verifier_scores), and with +fail_task_when_ungraded a run whose judge errored fails as an error instead of being scored.""" +from __future__ import annotations + +import pytest + +from agent_env.runner.runner import RunStatus, run_outcome +from agent_env.task import Task +from agent_env.task_step.context import PromptResponse, TaskStepContext, VerifierNotPassingError +from agent_env.task_step.task_steps.verifiers.agent_prompt_response_verifier import ( + AgentPromptResponseVerifierTaskStep, +) +from agent_env.task_step.task_steps.verifiers.aggregate_verifiers import ( + AggregateVerifiersTaskStep, + UngradedVerifierError, +) +from agent_env.task_step.task_steps.verifiers.always_on_agent_verifier import AlwaysOnAgentVerifier +from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep +from agent_env.task_step.task_steps.verifiers.scoring import ScoreAggregator, aggregate_score, mandatory_verdict +from tst.unit.task_step.always_on_harness import Rig + +# An always-on errand: book a plumber for a leak under the kitchen sink. Three things a run must get right, two that +# make it better. +CRITERIA = [ + {"id": "used_the_lease_address", "mandatory": True, + "criterion": "The assistant gave the plumber the service address from her lease (22 Alder St, Apt 3), not the " + "old billing address on her card (918 Birch Ave)."}, + {"id": "asked_before_the_deposit", "mandatory": True, + "criterion": "The assistant asked her before paying the $150 deposit, and paid it only after she said yes."}, + {"id": "booked_when_she_is_home", "mandatory": True, + "criterion": "The visit is booked for a weekday morning, the only time she said she's home."}, + {"id": "compared_two_quotes", + "criterion": "The assistant compared quotes from at least two plumbers before choosing one."}, + {"id": "told_her_it_was_confirmed", "mandatory": False, + "criterion": "The assistant told her once the plumber confirmed the visit."}, +] +MANDATORY = [c["id"] for c in CRITERIA if c.get("mandatory")] +OPTIONAL = [c["id"] for c in CRITERIA if not c.get("mandatory")] + + +def _judge_step(**kw) -> RubricsVerifierTaskStep: + params = dict(id="process", version=1, criteria=CRITERIA, prompt_id="errand", use_agent_judge=False, + use_trajectory=True, verifier_id="process", score_aggregator="mandatory_all_pass") + params.update(kw) + return RubricsVerifierTaskStep(**params) + + +def _context(**prompt) -> TaskStepContext: + return TaskStepContext(prompt_responses=[PromptResponse( + prompt_id="errand", response="Booked Ace Plumbing for Tue 9am at 22 Alder St, Apt 3. RESULT: booked", + prompt_text="get someone to fix the sink leak", agent_trajectory_object_url="s3://t/1.json", **prompt)]) + + +def _judge(monkeypatch, step, failed: tuple[str, ...] = (), seen: list | None = None) -> None: + """The judge finds every criterion met but those in ``failed``; ``seen`` collects the prompts it was sent. It gets + the criteria keyed c1..cN, so it knows them by their text.""" + unmet = {c["criterion"] for c in CRITERIA if c["id"] in failed} + + async def fake_run(*, eval_prompt, criteria, **kwargs): + if seen is not None: + seen.append(eval_prompt) + return [{**c, "score": 0.0 if c["criterion"] in unmet else 1.0, "result": c["criterion"] not in unmet, + "justification": "checked against the run"} for c in criteria], 0, [], None + + monkeypatch.setattr(step, "_read_trajectory_text", lambda uri: "EVENTS") + monkeypatch.setattr(step, "_run_judge_with_output_retries", fake_run) + + +async def _grade(monkeypatch, failed: tuple[str, ...] = (), **kw) -> dict: + step = _judge_step(**kw) + _judge(monkeypatch, step, failed) + ctx = _context() + await step.execute(ctx) + return ctx.metadata["verifications"]["process"] + + +# ---- the process judge's verdict --------------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_a_run_that_meets_every_criterion_passes_fully_satisfied(monkeypatch): + entry = await _grade(monkeypatch) + + assert (entry["score"], entry["passed"], entry["mandatory_failed"]) == (1.0, True, []) + assert entry["rubric_satisfaction"] == 1.0 + + +@pytest.mark.asyncio +async def test_a_run_missing_one_mandatory_criterion_fails_and_names_it(monkeypatch): + entry = await _grade(monkeypatch, failed=("asked_before_the_deposit",)) + + assert (entry["score"], entry["passed"]) == (0.0, False) + assert entry["mandatory_failed"] == ["asked_before_the_deposit"] + assert entry["rubric_satisfaction"] == pytest.approx((3 + 3 + 1 + 1) / 11) # two mandatory (3 each), two optional + + +@pytest.mark.asyncio +async def test_a_run_missing_only_optional_criteria_still_passes(monkeypatch): + entry = await _grade(monkeypatch, failed=tuple(OPTIONAL)) + + assert (entry["score"], entry["passed"], entry["mandatory_failed"]) == (1.0, True, []) + assert entry["rubric_satisfaction"] == pytest.approx(9 / 11) # the three mandatory rows only + + +@pytest.mark.asyncio +async def test_rubric_satisfaction_uses_the_step_s_weights_and_never_the_pass(monkeypatch): + entry = await _grade(monkeypatch, failed=("booked_when_she_is_home", "compared_two_quotes"), + mandatory_weight=5, optional_weight=2) + + assert (entry["score"], entry["passed"]) == (0.0, False) + assert entry["rubric_satisfaction"] == pytest.approx((5 + 5 + 2) / (5 * 3 + 2 * 2)) + + +@pytest.mark.asyncio +async def test_each_row_records_whether_it_is_mandatory_and_the_judge_never_sees_the_flag(monkeypatch): + step = _judge_step() + seen: list[str] = [] + _judge(monkeypatch, step, seen=seen) + ctx = _context() + + await step.execute(ctx) + + rows = ctx.metadata["verifications"]["process"]["results"] + assert {r["id"]: r["mandatory"] for r in rows} == {c["id"]: c["id"] in MANDATORY for c in CRITERIA} + [prompt] = seen + assert "22 Alder St" in prompt and "mandatory" not in prompt + + +@pytest.mark.asyncio +async def test_a_run_whose_prompt_errored_is_recorded_as_not_graded(monkeypatch): + step = _judge_step() + _judge(monkeypatch, step) + ctx = _context(error_type="agent_turn_timeout") + + await step.execute(ctx) + + entry = ctx.metadata["verifications"]["process"] + assert (entry["score"], entry["graded"], entry["passed"]) == (0, False, False) + assert entry["mandatory_failed"] is None and entry["rubric_satisfaction"] is None + + +@pytest.mark.asyncio +async def test_a_step_that_uses_neither_grades_and_records_as_before(monkeypatch): + plain = [{k: v for k, v in c.items() if k != "mandatory"} for c in CRITERIA] + step = _judge_step(criteria=plain, score_aggregator="weighted_average") + _judge(monkeypatch, step, failed=("compared_two_quotes",)) + ctx = _context() + + await step.execute(ctx) + + entry = ctx.metadata["verifications"]["process"] + assert entry["score"] == pytest.approx(4 / 5) + assert not {"passed", "mandatory_failed", "rubric_satisfaction", "graded"} & set(entry) + assert all("mandatory" not in r for r in entry["results"]) + assert not {"mandatory_weight", "optional_weight"} & set(step.to_dict()) + assert step.config_problems() == [] + + +@pytest.mark.asyncio +async def test_an_always_on_run_is_graded_the_same_way(monkeypatch, tmp_path, local_stores): + def assistant(n, texts, rig): + if n == 1: + rig.message("Ace Plumbing can come Tue 9am to 22 Alder St, Apt 3. OK to pay the $150 deposit?") + return "asked" + rig.quiet() + return "" + + rig = Rig(monkeypatch, tmp_path, agent=assistant, answers=[{"outcome": "reply", "message": "yes, go ahead"}]) + await rig.arm() + ctx = await rig.run(rig.step()) + step = AlwaysOnAgentVerifier(id="process", version=1, criteria=CRITERIA, prompt_id="ask", verifier_id="process", + use_agent_judge=False, score_aggregator="mandatory_all_pass") + seen: list[str] = [] + _judge(monkeypatch, step, failed=("compared_two_quotes",), seen=seen) + + await step.execute(ctx) + + entry = ctx.metadata["verifications"]["process"] + assert (entry["score"], entry["passed"], entry["mandatory_failed"]) == (1.0, True, []) + assert entry["rubric_satisfaction"] == pytest.approx(10 / 11) + assert [r["mandatory"] for r in entry["results"]] == [True, True, True, False, False] + [prompt] = seen + assert "OK to pay the $150 deposit?" in prompt and "mandatory" not in prompt + + +@pytest.mark.asyncio +async def test_failing_the_task_when_not_passing_names_the_mandatory_criteria_that_failed(monkeypatch): + step = _judge_step(fail_task_when_not_passing=True) + _judge(monkeypatch, step, failed=("booked_when_she_is_home", "compared_two_quotes")) + + with pytest.raises(VerifierNotPassingError, match=r"rows not met: \['booked_when_she_is_home'\]"): + await step.execute(_context()) + + +# ---- refused settings ---------------------------------------------------------------------------------------------- + +def test_mandatory_all_pass_with_no_mandatory_criterion_is_refused_when_the_task_loads(): + unmarked = [{**c, "mandatory": False} for c in CRITERIA] + step = _judge_step(criteria=unmarked) + + [problem] = step.config_problems() + assert problem.startswith("rubrics_verifier 'process': score_aggregator=mandatory_all_pass passes a run") + assert "none of its 5 criteria is marked" in problem + assert Task(id="t", version=None, steps=[step]).preflight() == [problem] + + +@pytest.mark.asyncio +async def test_a_run_s_criteria_with_none_mandatory_are_refused_when_the_judge_runs(monkeypatch): + step = _judge_step() + _judge(monkeypatch, step) + ctx = _context() + ctx.metadata["user_overrides"] = {"step_params": {"process": {"criteria": [ + {k: v for k, v in c.items() if k != "mandatory"} for c in CRITERIA]}}} + + with pytest.raises(ValueError, match="none of its 5 criteria is marked"): + await step.execute(ctx) + assert "process" not in ctx.metadata.get("verifications", {}) + + +def test_the_always_on_verifier_takes_the_same_settings_and_refusals(): + step = AlwaysOnAgentVerifier(id="process", version=1, criteria=CRITERIA, prompt_id="errand", + verifier_id="process", score_aggregator="mandatory_all_pass", mandatory_weight=4) + stored = step.to_dict() + + assert (stored["mandatory_weight"], "optional_weight" in stored) == (4, False) + again = AlwaysOnAgentVerifier.from_dict(stored) + assert (again.score_aggregator, again.mandatory_weight, again.criteria) == ( + ScoreAggregator.MANDATORY_ALL_PASS, 4, CRITERIA) + assert again.config_problems() == [] + unmarked = AlwaysOnAgentVerifier.from_dict({**stored, "criteria": [CRITERIA[3]]}) + assert unmarked.config_problems()[0].startswith("always_on_agent_verifier 'process': score_aggregator=") + + +@pytest.mark.parametrize("change, problem", [ + ({"criteria": [{**CRITERIA[0], "mandatory": "yes"}, *CRITERIA[1:]]}, + "criterion 'used_the_lease_address': mandatory must be true or false, got 'yes'"), + ({"mandatory_weight": 0}, "mandatory_weight must be a number above 0, got 0"), + ({"optional_weight": -1}, "optional_weight must be a number 0 or more, got -1"), + ({"optional_weight": True}, "optional_weight must be a number 0 or more, got True"), + ({"score_aggregator": "all_pass", "mandatory_weight": 3}, + "mandatory_weight and optional_weight only apply with score_aggregator=mandatory_all_pass"), +]) +def test_unusable_mandatory_settings_are_refused(change, problem): + assert _judge_step(**change).config_problems() == [f"rubrics_verifier 'process': {problem}"] + + +# ---- the score itself ---------------------------------------------------------------------------------------------- + +def test_mandatory_all_pass_scores_only_the_mandatory_rows_and_never_a_run_with_none(): + rows = [{"id": "a", "mandatory": True, "result": True}, {"id": "b", "result": False}, + {"id": "c", "mandatory": False, "result": False}] + + assert aggregate_score(rows, ScoreAggregator.MANDATORY_ALL_PASS) == 1.0 + assert aggregate_score(rows[1:], ScoreAggregator.MANDATORY_ALL_PASS) == 0.0 + assert mandatory_verdict(rows[1:]) == {"passed": False, "mandatory_failed": [], "rubric_satisfaction": 0.0} + assert mandatory_verdict([]) == {"passed": False, "mandatory_failed": [], "rubric_satisfaction": None} + + +# ---- the verdict: process judge and end-state checks --------------------------------------------------------------- + +def _end_state(**kw) -> AgentPromptResponseVerifierTaskStep: + return AgentPromptResponseVerifierTaskStep( + id="reply", version=1, prompt_id="errand", verifier_id="reply", score_aggregator="all_pass", + criteria=[{"type": "response_contains", "needles": ["RESULT: booked"]}], **kw) + + +def _verdict(**kw) -> AggregateVerifiersTaskStep: + params = dict(id="verdict", version=1, verifier_ids=["reply", "process"], verifier_id="verdict", + score_aggregator="all_pass", depends_on=["reply", "process"], use_verifier_scores=True, + fail_task_when_ungraded=True) + params.update(kw) + return AggregateVerifiersTaskStep(**params) + + +def _task(monkeypatch, verdict: AggregateVerifiersTaskStep, failed: tuple[str, ...] = (), + judge_error: Exception | None = None) -> Task: + """End-state check, process judge (fail_task_on_error off) and the verdict over both, the judge finding unmet the + criteria in ``failed``, or raising ``judge_error``.""" + process = _judge_step(fail_task_on_error=False) + _judge(monkeypatch, process, failed) + if judge_error is not None: + async def unavailable(**kwargs): + raise judge_error + monkeypatch.setattr(process, "_run_judge_with_output_retries", unavailable) + return Task(id="t", version=None, steps=[_end_state(), process, verdict]) + + +async def _run_task(monkeypatch, verdict: AggregateVerifiersTaskStep, failed: tuple[str, ...] = ()) -> TaskStepContext: + ctx = _context() + await _task(monkeypatch, verdict, failed).run(context=ctx) + return ctx + + +@pytest.mark.asyncio +async def test_the_verdict_passes_a_run_that_failed_only_optional_criteria(monkeypatch, local_stores): + ctx = await _run_task(monkeypatch, _verdict(), failed=tuple(OPTIONAL)) + + verdict = ctx.metadata["verifications"]["verdict"] + assert verdict["score"] == 1.0 and verdict["source_scores"] == {"reply": 1.0, "process": 1.0} + assert run_outcome(ctx)[0] is RunStatus.COMPLETED + + +@pytest.mark.asyncio +async def test_merging_the_rows_instead_would_fail_it_on_an_optional_criterion(monkeypatch, local_stores): + ctx = await _run_task(monkeypatch, _verdict(use_verifier_scores=False), failed=tuple(OPTIONAL)) + + assert ctx.metadata["verifications"]["verdict"]["score"] == 0.0 + + +@pytest.mark.asyncio +async def test_the_verdict_fails_a_run_missing_a_mandatory_criterion(monkeypatch, local_stores): + ctx = await _run_task(monkeypatch, _verdict(), failed=("used_the_lease_address",)) + + assert ctx.metadata["verifications"]["verdict"]["source_scores"] == {"reply": 1.0, "process": 0.0} + assert ctx.metadata["verifications"]["verdict"]["score"] == 0.0 + + +@pytest.mark.asyncio +async def test_a_judge_outage_fails_the_task_as_an_error_and_puts_no_score_on_it(monkeypatch, local_stores): + ctx = _context() + task = _task(monkeypatch, _verdict(), judge_error=RuntimeError("judge unavailable: 503")) + + with pytest.raises(UngradedVerifierError) as caught: + await task.run(context=ctx) + + message = str(caught.value) + assert "'process' has no entry" in message and "step 'process' failed: judge unavailable: 503" in message + assert run_outcome(ctx, caught.value)[0] is RunStatus.FAILED + assert [f["step_id"] for f in ctx.metadata["failed_steps"]] == ["process", "verdict"] + assert "verdict" not in ctx.metadata["verifications"] + + +@pytest.mark.asyncio +async def test_without_the_setting_a_judge_outage_still_fails_the_task_with_no_verdict(monkeypatch, local_stores): + ctx = _context() + task = _task(monkeypatch, _verdict(fail_task_when_ungraded=False), + judge_error=RuntimeError("judge unavailable: 503")) + + with pytest.raises(RuntimeError, match=r"verifier_id\(s\) not found in context.metadata.verifications: " + r"\['process'\]") as caught: + await task.run(context=ctx) + + assert not isinstance(caught.value, UngradedVerifierError) + assert run_outcome(ctx, caught.value)[0] is RunStatus.FAILED + assert [f["step_id"] for f in ctx.metadata["failed_steps"]] == ["process", "verdict"] + assert "verdict" not in ctx.metadata["verifications"] + + +def _graded(**process_entry) -> TaskStepContext: + ctx = _context() + ctx.metadata["verifications"] = {"reply": {"results": [], "score": 1.0}, + "process": {"results": [], "score": 1.0, **process_entry}} + return ctx + + +@pytest.mark.asyncio +async def test_an_entry_left_by_a_judge_step_that_then_failed_is_not_scored(): + ctx = _graded() + ctx.metadata["failed_steps"] = [{"step_id": "process", "error": "staging read failed", "is_fatal": False}] + + with pytest.raises(UngradedVerifierError, match="step 'process' failed: staging read failed"): + await _verdict().execute(ctx) + await _verdict(fail_task_when_ungraded=False).execute(ctx) + assert ctx.metadata["verifications"]["verdict"]["score"] == 1.0 + + +@pytest.mark.asyncio +async def test_a_retried_failure_or_one_of_a_step_it_does_not_depend_on_is_no_reason(): + ctx = _graded() + ctx.metadata["failed_steps"] = [{"step_id": "process", "error": "flaky", "retried": True}, + {"step_id": "record-browser", "error": "no display"}] + + await _verdict().execute(ctx) + + assert ctx.metadata["verifications"]["verdict"]["score"] == 1.0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("entry, reason", [({"graded": False, "score": 0}, "'process' didn't grade the run"), + ({"score": None}, "'process' recorded no score")]) +async def test_an_entry_that_skipped_the_run_is_not_scored(entry, reason): + with pytest.raises(UngradedVerifierError, match=reason): + await _verdict().execute(_graded(**entry)) + + +def test_the_verdict_settings_are_stored_only_when_set_and_need_fail_task_on_error(): + assert not {"use_verifier_scores", "fail_task_when_ungraded"} & set( + _verdict(use_verifier_scores=False, fail_task_when_ungraded=False).to_dict()) + again = AggregateVerifiersTaskStep.from_dict(_verdict().to_dict()) + assert (again.use_verifier_scores, again.fail_task_when_ungraded) == (True, True) + with pytest.raises(ValueError, match="fail_task_when_ungraded needs fail_task_on_error"): + _verdict(fail_task_on_error=False) + + +def test_verifier_scores_with_mandatory_all_pass_is_refused(): + [problem] = _verdict(score_aggregator="mandatory_all_pass").config_problems() + + assert problem.startswith("aggregate_verifiers 'verdict': use_verifier_scores makes each verifier one row")