From 057bf262ba7b9c53e10ddfd870640341508c1cc9 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 18 Nov 2026 14:14:08 -0800 Subject: [PATCH 01/12] feat(verifiers): the base fails a grade its verifier can't trust, and records whatever it can't read Verification gains error: the grade is failed with it while the legacy entry the verifier recorded is still written, for a test command that timed out. Anything after verify that raises (scoring, the grade, the output) is a failed grade too, settings that can't be read are recorded as settings_error rather than losing the verifier's own error, a failed grade's reason is cut at REASONING_CHARS, and the settings a subclass adds are compacted as the base's own are. Co-Authored-By: Claude Opus 5.5 --- .../task_steps/verifiers/verifier.py | 59 ++++++++++---- .../task_step/verifiers/test_verifier_base.py | 77 +++++++++++++++++++ 2 files changed, 119 insertions(+), 17 deletions(-) diff --git a/src/agent_env/task_step/task_steps/verifiers/verifier.py b/src/agent_env/task_step/task_steps/verifiers/verifier.py index 40cae0593..6342abcab 100644 --- a/src/agent_env/task_step/task_steps/verifiers/verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/verifier.py @@ -35,6 +35,8 @@ DEFAULT_MIN_PASSING_SCORE = 1.0 OUTPUT_INLINE_BYTES = 16 * 1024 SETTING_INLINE_BYTES = 1024 +# A failed grade's reason is an exception's text, which can carry pages of logs (a judge's container output). +REASONING_CHARS = 2000 # The step's fields left out of its verifier_settings: those that name or schedule it rather than say how it grades, and # min_passing_score, which the grade holds itself. _NOT_SETTINGS = ("id", "type", "version", "depends_on", "fail_task_on_error", "retry_config", "verifier_id", @@ -82,13 +84,15 @@ class Verification: ``output``: anything else worth keeping, such as a judge's justification or a test's log; a dict's keys also join the step's legacy ``verifications`` entry, after its ``results`` and ``score``. ``skipped``: why the verifier judged nothing. ``raise_after``: an exception the step raises once both are written, for a verdict that also fails its - step.""" + step. ``error``: why the verdict it recorded can't be trusted, such as a test command that timed out: the grade is + failed with it, and the legacy entry is still written.""" criteria: list[dict[str, Any]] = field(default_factory=list) score: Optional[float] = None output: Any = None skipped: Optional[str] = None raise_after: Optional[BaseException] = None + error: Optional[str] = None class VerifierTaskStep(TaskStep): @@ -125,19 +129,17 @@ async def verify(self, context: TaskStepContext) -> Verification: async def execute(self, context: TaskStepContext) -> TaskStepContext: try: verification = await self.verify(context) + score = self._score(verification) + status = (VerifierStatus.SKIPPED if verification.skipped is not None + else VerifierStatus.FAILED if verification.error is not None else VerifierStatus.GRADED) + grade = self._grade(context, status, float(score) if status is VerifierStatus.GRADED else None, + verification.criteria or [], error=_capped(verification.error), + skip=verification.skipped) + grade.update(await self._output(context, verification.output)) except Exception as exc: self._write_grade(context, self._grade(context, VerifierStatus.FAILED, None, [], - error=f"{type(exc).__name__}: {exc}")) + error=_capped(f"{type(exc).__name__}: {exc}"))) raise - if verification.skipped is not None: - status, score = VerifierStatus.SKIPPED, None - elif verification.score is not None: - status, score = VerifierStatus.GRADED, verification.score - else: - status, score = VerifierStatus.GRADED, aggregate_score(verification.criteria, self.score_aggregator) - grade = self._grade(context, status, None if score is None else float(score), verification.criteria, - skip=verification.skipped) - grade.update(await self._output(context, verification.output)) self._write_grade(context, grade) entry = {"results": verification.criteria, "score": score} if verification.skipped is not None: @@ -150,11 +152,11 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: return context def verifier_settings(self, context: TaskStepContext) -> dict[str, Any]: - """What this step grades with in ``context``'s run: its own fields, one over ``SETTING_INLINE_BYTES`` kept as its - sha256 and size, and its ``score_aggregator`` whether or not it is its default. A subclass adds what the run - changes or decides, such as the ``step_params`` it applies or a judge's model. Pure, since it is also read at - run start.""" - fields = {k: _compact(v) for k, v in self.to_dict().items() if k not in _NOT_SETTINGS} + """What this step grades with in ``context``'s run: its own fields and its ``score_aggregator`` whether or not it + is its default. A subclass adds what the run changes or decides, such as the ``step_params`` it applies or a + judge's model. Pure, since it is also read at run start; the grade keeps a value over ``SETTING_INLINE_BYTES`` + as its sha256 and size.""" + fields = {k: v for k, v in self.to_dict().items() if k not in _NOT_SETTINGS} return {**fields, "score_aggregator": self.score_aggregator.value} def config_problems(self) -> list[str]: @@ -200,9 +202,26 @@ def _grade(self, context: TaskStepContext, status: VerifierStatus, score: Option verifier_criteria=rows, verifier_type=self.type, verifier_version=_providing_version(type(self).__module__), - verifier_settings={"scoring_rules": SCORING_RULES, **self.verifier_settings(context)}, + verifier_settings=self._settings(context), ) + def _score(self, verification: Verification) -> Optional[float]: + if verification.skipped is not None: + return None + if verification.score is not None: + return verification.score + return aggregate_score(verification.criteria or [], self.score_aggregator) + + def _settings(self, context: TaskStepContext) -> dict[str, Any]: + """``verifier_settings`` as the grade records it, compacted; a failing one is recorded rather than raised, since a + failed grade records settings too and must not lose the verifier's own error.""" + try: + settings = self.verifier_settings(context) + except Exception as exc: + logger.warning("verifier %s: its settings couldn't be read: %s", self.id, exc) + return {"scoring_rules": SCORING_RULES, "settings_error": _capped(f"{type(exc).__name__}: {exc}")} + return {"scoring_rules": SCORING_RULES, **{k: _compact(v) for k, v in settings.items()}} + async def _output(self, context: TaskStepContext, output: Any) -> dict[str, Any]: """``verifier_output`` inline, or, over ``OUTPUT_INLINE_BYTES``, a ``verifier_output_ref`` to its copy in the object store, written on a worker thread; inline with ``verifier_output_offload_error`` when that write fails, @@ -243,6 +262,12 @@ def _row(index: int, row: dict[str, Any]) -> dict[str, Any]: } +def _capped(text: Optional[str]) -> Optional[str]: + if text is None or len(text) <= REASONING_CHARS: + return text + return f"{text[:REASONING_CHARS]}… ({len(text)} characters)" + + def _compact(value: Any) -> Any: body = canonical_json(value) if len(body) <= SETTING_INLINE_BYTES: diff --git a/tst/unit/task_step/verifiers/test_verifier_base.py b/tst/unit/task_step/verifiers/test_verifier_base.py index 7bc7e7300..24097c6b8 100644 --- a/tst/unit/task_step/verifiers/test_verifier_base.py +++ b/tst/unit/task_step/verifiers/test_verifier_base.py @@ -276,6 +276,83 @@ def test_the_grade_and_legacy_entry_survive_the_step_journal(): assert stored["metadata"]["verifications"] == post.metadata["verifications"] +def test_an_error_fails_the_grade_but_keeps_the_legacy_entry_it_recorded(): + timeout = RuntimeError("tests timed out after 300s") + context = TaskStepContext() + + with pytest.raises(RuntimeError) as raised: + _run(_Toy(outcome=Verification(criteria=[{"id": "reward", "score": 1.0, "result": True}], score=1.0, + error="timed out after 300s", raise_after=timeout)), context) + + assert raised.value is timeout + grade = context.metadata["grades"]["check"] + assert (grade["verifier_status"], grade["score"], grade["passed"]) == ("failed", None, False) + assert grade["verifier_error_reasoning"] == "timed out after 300s" + assert context.metadata["verifications"]["check"] == { + "results": [{"id": "reward", "score": 1.0, "result": True}], "score": 1.0} + + +@pytest.mark.parametrize("outcome", [Verification(criteria=["not a row"]), + Verification(score=1, output={"not json": object()})]) +def test_a_verification_the_base_cant_record_is_a_failed_grade_and_raises(outcome): + context = TaskStepContext() + + with pytest.raises((AttributeError, TypeError)): + _run(_Toy(outcome=outcome), context) + + assert context.metadata["grades"]["check"]["verifier_status"] == "failed" + assert "verifications" not in context.metadata + + +def test_settings_that_cant_be_read_are_recorded_not_raised(): + class _Unreadable(_Toy): + def verifier_settings(self, context): + raise KeyError("judge") + + for outcome, status in ((Verification(score=1), "graded"), (ValueError("broken"), "failed")): + context = TaskStepContext() + try: + _run(_Unreadable(outcome=outcome), context) + except ValueError: + pass + + grade = context.metadata["grades"]["check"] + assert grade["verifier_status"] == status + assert grade["verifier_settings"] == {"scoring_rules": 1, "settings_error": "KeyError: 'judge'"} + + +def test_a_long_error_is_cut_with_its_length(): + context = TaskStepContext() + + with pytest.raises(RuntimeError): + _run(_Toy(outcome=RuntimeError("log line\n" * 1000)), context) + + reasoning = context.metadata["grades"]["check"]["verifier_error_reasoning"] + assert len(reasoning) < verifier_module.REASONING_CHARS + 40 + assert reasoning.endswith(f"… ({len('RuntimeError: ') + 9000} characters)") + + +def test_settings_a_subclass_adds_are_compacted_too(): + class _Judged(_Toy): + def verifier_settings(self, context): + return {**super().verifier_settings(context), "criteria": ["x" * 2000], "judge_model": "m"} + + settings = _run(_Judged(outcome=Verification(score=1))).metadata["grades"]["check"]["verifier_settings"] + + assert settings["judge_model"] == "m" and set(settings["criteria"]) == {"sha256", "bytes"} + + +def test_criteria_of_none_are_kept_in_the_legacy_entry(): + oops = TypeError("verify returned None") + context = TaskStepContext() + + with pytest.raises(TypeError): + _run(_Toy(outcome=Verification(criteria=None, raise_after=oops)), context) + + assert context.metadata["verifications"]["check"] == {"results": None, "score": 0.0} + assert context.metadata["grades"]["check"]["verifier_status"] == "graded" + + def test_a_verifier_on_the_base_is_a_grader(): assert _Toy.step_role is StepRole.GRADER and "step_role" not in _Toy().to_dict() From b3eda3c216aee88ec8f43af6302f7479ebd701de Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 18 Nov 2026 14:21:27 -0800 Subject: [PATCH 02/12] test(verifiers): record the legacy entries run_container, verify_sandbox, env_outcome and aggregate write today Each path's verifications entry, byte for byte with its key order, and what it raised, recorded on the unchanged verifiers with offline fakes, so their move onto VerifierTaskStep can be checked against them. Co-Authored-By: Claude Opus 5.5 --- .../verifiers/recorded/legacy_entries.json | 2623 +++++++++++++++++ .../verifiers/test_legacy_entries.py | 478 +++ 2 files changed, 3101 insertions(+) create mode 100644 tst/unit/task_step/verifiers/recorded/legacy_entries.json create mode 100644 tst/unit/task_step/verifiers/test_legacy_entries.py diff --git a/tst/unit/task_step/verifiers/recorded/legacy_entries.json b/tst/unit/task_step/verifiers/recorded/legacy_entries.json new file mode 100644 index 000000000..abf6d64bb --- /dev/null +++ b/tst/unit/task_step/verifiers/recorded/legacy_entries.json @@ -0,0 +1,2623 @@ +{ + "passing_tests": { + "raised": null, + "entry": { + "results": [ + { + "id": "exit_code", + "score": 1.0, + "result": true, + "message": "exit_code=0 timed_out=False" + } + ], + "score": 1.0, + "command": "bash /tests/test.sh", + "exit_code": 0, + "timed_out": false, + "stdout_artifact": { + "id": "inst-1__verifier-stdout-tests-1790000000-000000000000", + "version": 1, + "s3_url": "/verifier-outputs/tests/inst-1-1790000000-000000000000/stdout.txt", + "object_url": "/verifier-outputs/tests/inst-1-1790000000-000000000000/stdout.txt" + }, + "stderr_artifact": { + "id": "inst-1__verifier-stderr-tests-1790000000-000000000000", + "version": 1, + "s3_url": "/verifier-outputs/tests/inst-1-1790000000-000000000000/stderr.txt", + "object_url": "/verifier-outputs/tests/inst-1-1790000000-000000000000/stderr.txt" + }, + "stdout_head": "ok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok\nok", + "stderr_head": "warn", + "extracted_files": {} + } + }, + "failing_tests_raise_after_recording": { + "raised": "RuntimeError: Verifier 'unit' exited 1; stderr (last 1500 chars):\nEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE", + "entry": { + "results": [ + { + "id": "exit_code", + "score": 0.0, + "result": false, + "message": "exit_code=1 timed_out=False" + } + ], + "score": 0.0, + "command": "bash /tests/test.sh", + "exit_code": 1, + "timed_out": false, + "stdout_artifact": { + "id": "inst-1__verifier-stdout-tests-1790000000-000000000000", + "version": 1, + "s3_url": "/verifier-outputs/tests/inst-1-1790000000-000000000000/stdout.txt", + "object_url": "/verifier-outputs/tests/inst-1-1790000000-000000000000/stdout.txt" + }, + "stderr_artifact": { + "id": "inst-1__verifier-stderr-tests-1790000000-000000000000", + "version": 1, + "s3_url": "/verifier-outputs/tests/inst-1-1790000000-000000000000/stderr.txt", + "object_url": "/verifier-outputs/tests/inst-1-1790000000-000000000000/stderr.txt" + }, + "stdout_head": "1 failed", + "stderr_head": "EEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE", + "extracted_files": {} + } + }, + "timeout_at_124": { + "raised": "RuntimeError: Verifier 'tests' timed out after 5s", + "entry": { + "results": [ + { + "id": "exit_code", + "score": 0.0, + "result": false, + "message": "exit_code=124 timed_out=True" + } + ], + "score": 0.0, + "command": "bash /tests/test.sh", + "exit_code": 124, + "timed_out": true, + "stdout_artifact": { + "id": "inst-1__verifier-stdout-tests-1790000000-000000000000", + "version": 1, + "s3_url": "/verifier-outputs/tests/inst-1-1790000000-000000000000/stdout.txt", + "object_url": "/verifier-outputs/tests/inst-1-1790000000-000000000000/stdout.txt" + }, + "stderr_artifact": { + "id": "inst-1__verifier-stderr-tests-1790000000-000000000000", + "version": 1, + "s3_url": "/verifier-outputs/tests/inst-1-1790000000-000000000000/stderr.txt", + "object_url": "/verifier-outputs/tests/inst-1-1790000000-000000000000/stderr.txt" + }, + "stdout_head": "", + "stderr_head": "killed", + "extracted_files": {} + } + }, + "timeout_at_137": { + "raised": "RuntimeError: Verifier 'tests' timed out after 300s", + "entry": { + "results": [ + { + "id": "exit_code", + "score": 0.0, + "result": false, + "message": "exit_code=137 timed_out=True" + } + ], + "score": 0.0, + "command": "bash /tests/test.sh", + "exit_code": 137, + "timed_out": true, + "stdout_artifact": { + "id": "inst-1__verifier-stdout-tests-1790000000-000000000000", + "version": 1, + "s3_url": "/verifier-outputs/tests/inst-1-1790000000-000000000000/stdout.txt", + "object_url": "/verifier-outputs/tests/inst-1-1790000000-000000000000/stdout.txt" + }, + "stderr_artifact": { + "id": "inst-1__verifier-stderr-tests-1790000000-000000000000", + "version": 1, + "s3_url": "/verifier-outputs/tests/inst-1-1790000000-000000000000/stderr.txt", + "object_url": "/verifier-outputs/tests/inst-1-1790000000-000000000000/stderr.txt" + }, + "stdout_head": "", + "stderr_head": "", + "extracted_files": {} + } + }, + "reward_one": { + "raised": null, + "entry": { + "results": [ + { + "id": "reward", + "score": 1.0, + "result": true, + "message": "reward=1.0 (exit_code=0 timed_out=False)" + } + ], + "score": 1.0, + "command": "bash /tests/test.sh", + "exit_code": 0, + "timed_out": false, + "stdout_artifact": { + "id": "inst-1__verifier-stdout-tests-1790000000-000000000000", + "version": 1, + "s3_url": "/verifier-outputs/tests/inst-1-1790000000-000000000000/stdout.txt", + "object_url": "/verifier-outputs/tests/inst-1-1790000000-000000000000/stdout.txt" + }, + "stderr_artifact": { + "id": "inst-1__verifier-stderr-tests-1790000000-000000000000", + "version": 1, + "s3_url": "/verifier-outputs/tests/inst-1-1790000000-000000000000/stderr.txt", + "object_url": "/verifier-outputs/tests/inst-1-1790000000-000000000000/stderr.txt" + }, + "stdout_head": "3 passed", + "stderr_head": "", + "extracted_files": { + "/logs/verifier/reward.txt": 1 + } + } + }, + "reward_half": { + "raised": null, + "entry": { + "results": [ + { + "id": "reward", + "score": 0.5, + "result": false, + "message": "reward=0.5 (exit_code=0 timed_out=False)" + } + ], + "score": 0.5, + "command": "bash /tests/test.sh", + "exit_code": 0, + "timed_out": false, + "stdout_artifact": { + "id": "inst-1__verifier-stdout-tests-1790000000-000000000000", + "version": 1, + "s3_url": "/verifier-outputs/tests/inst-1-1790000000-000000000000/stdout.txt", + "object_url": "/verifier-outputs/tests/inst-1-1790000000-000000000000/stdout.txt" + }, + "stderr_artifact": { + "id": "inst-1__verifier-stderr-tests-1790000000-000000000000", + "version": 1, + "s3_url": "/verifier-outputs/tests/inst-1-1790000000-000000000000/stderr.txt", + "object_url": "/verifier-outputs/tests/inst-1-1790000000-000000000000/stderr.txt" + }, + "stdout_head": "3 passed", + "stderr_head": "", + "extracted_files": { + "/logs/verifier/reward.txt": 0.5 + } + } + }, + "reward_clamped": { + "raised": null, + "entry": { + "results": [ + { + "id": "reward", + "score": 1.0, + "result": true, + "message": "reward=1.0 (exit_code=0 timed_out=False)" + } + ], + "score": 1.0, + "command": "bash /tests/test.sh", + "exit_code": 0, + "timed_out": false, + "stdout_artifact": { + "id": "inst-1__verifier-stdout-tests-1790000000-000000000000", + "version": 1, + "s3_url": "/verifier-outputs/tests/inst-1-1790000000-000000000000/stdout.txt", + "object_url": "/verifier-outputs/tests/inst-1-1790000000-000000000000/stdout.txt" + }, + "stderr_artifact": { + "id": "inst-1__verifier-stderr-tests-1790000000-000000000000", + "version": 1, + "s3_url": "/verifier-outputs/tests/inst-1-1790000000-000000000000/stderr.txt", + "object_url": "/verifier-outputs/tests/inst-1-1790000000-000000000000/stderr.txt" + }, + "stdout_head": "3 passed", + "stderr_head": "", + "extracted_files": { + "/logs/verifier/reward.txt": 2 + } + } + }, + "reward_missing_after_a_failing_exit": { + "raised": null, + "entry": { + "results": [ + { + "id": "reward", + "score": 0.0, + "result": false, + "message": "reward file '/logs/verifier/reward.txt' missing or non-numeric (exit_code=1 timed_out=False)" + } + ], + "score": 0.0, + "command": "bash /tests/test.sh", + "exit_code": 1, + "timed_out": false, + "stdout_artifact": { + "id": "inst-1__verifier-stdout-tests-1790000000-000000000000", + "version": 1, + "s3_url": "/verifier-outputs/tests/inst-1-1790000000-000000000000/stdout.txt", + "object_url": "/verifier-outputs/tests/inst-1-1790000000-000000000000/stdout.txt" + }, + "stderr_artifact": { + "id": "inst-1__verifier-stderr-tests-1790000000-000000000000", + "version": 1, + "s3_url": "/verifier-outputs/tests/inst-1-1790000000-000000000000/stderr.txt", + "object_url": "/verifier-outputs/tests/inst-1-1790000000-000000000000/stderr.txt" + }, + "stdout_head": "3 passed", + "stderr_head": "", + "extracted_files": { + "/logs/verifier/reward.txt": null + } + } + }, + "reward_not_a_number": { + "raised": null, + "entry": { + "results": [ + { + "id": "reward", + "score": 0.0, + "result": false, + "message": "reward file '/logs/verifier/reward.txt' missing or non-numeric (exit_code=0 timed_out=False)" + } + ], + "score": 0.0, + "command": "bash /tests/test.sh", + "exit_code": 0, + "timed_out": false, + "stdout_artifact": { + "id": "inst-1__verifier-stdout-tests-1790000000-000000000000", + "version": 1, + "s3_url": "/verifier-outputs/tests/inst-1-1790000000-000000000000/stdout.txt", + "object_url": "/verifier-outputs/tests/inst-1-1790000000-000000000000/stdout.txt" + }, + "stderr_artifact": { + "id": "inst-1__verifier-stderr-tests-1790000000-000000000000", + "version": 1, + "s3_url": "/verifier-outputs/tests/inst-1-1790000000-000000000000/stderr.txt", + "object_url": "/verifier-outputs/tests/inst-1-1790000000-000000000000/stderr.txt" + }, + "stdout_head": "3 passed", + "stderr_head": "", + "extracted_files": { + "/logs/verifier/reward.txt": "FAIL" + } + } + }, + "reward_then_time_out": { + "raised": "RuntimeError: Verifier 'tests' timed out after 300s", + "entry": { + "results": [ + { + "id": "reward", + "score": 1.0, + "result": true, + "message": "reward=1.0 (exit_code=124 timed_out=True)" + } + ], + "score": 1.0, + "command": "bash /tests/test.sh", + "exit_code": 124, + "timed_out": true, + "stdout_artifact": { + "id": "inst-1__verifier-stdout-tests-1790000000-000000000000", + "version": 1, + "s3_url": "/verifier-outputs/tests/inst-1-1790000000-000000000000/stdout.txt", + "object_url": "/verifier-outputs/tests/inst-1-1790000000-000000000000/stdout.txt" + }, + "stderr_artifact": { + "id": "inst-1__verifier-stderr-tests-1790000000-000000000000", + "version": 1, + "s3_url": "/verifier-outputs/tests/inst-1-1790000000-000000000000/stderr.txt", + "object_url": "/verifier-outputs/tests/inst-1-1790000000-000000000000/stderr.txt" + }, + "stdout_head": "3 passed", + "stderr_head": "", + "extracted_files": { + "/logs/verifier/reward.txt": 1 + } + } + }, + "result_files": { + "raised": null, + "entry": { + "results": [ + { + "id": "exit_code", + "score": 1.0, + "result": true, + "message": "exit_code=0 timed_out=False" + } + ], + "score": 1.0, + "command": "bash /tests/test.sh", + "exit_code": 0, + "timed_out": false, + "stdout_artifact": { + "id": "inst-1__verifier-stdout-tests-1790000000-000000000000", + "version": 1, + "s3_url": "/verifier-outputs/tests/inst-1-1790000000-000000000000/stdout.txt", + "object_url": "/verifier-outputs/tests/inst-1-1790000000-000000000000/stdout.txt" + }, + "stderr_artifact": { + "id": "inst-1__verifier-stderr-tests-1790000000-000000000000", + "version": 1, + "s3_url": "/verifier-outputs/tests/inst-1-1790000000-000000000000/stderr.txt", + "object_url": "/verifier-outputs/tests/inst-1-1790000000-000000000000/stderr.txt" + }, + "stdout_head": "", + "stderr_head": "", + "extracted_files": { + "/out/report.json": { + "tests": 3, + "passed": 3 + }, + "/out/log.txt": "plain", + "/out/missing.json": null + } + } + }, + "result_file_over_the_inline_limit": { + "raised": null, + "entry": { + "results": [ + { + "id": "exit_code", + "score": 1.0, + "result": true, + "message": "exit_code=0 timed_out=False" + } + ], + "score": 1.0, + "command": "bash /tests/test.sh", + "exit_code": 0, + "timed_out": false, + "stdout_artifact": { + "id": "inst-1__verifier-stdout-tests-1790000000-000000000000", + "version": 1, + "s3_url": "/verifier-outputs/tests/inst-1-1790000000-000000000000/stdout.txt", + "object_url": "/verifier-outputs/tests/inst-1-1790000000-000000000000/stdout.txt" + }, + "stderr_artifact": { + "id": "inst-1__verifier-stderr-tests-1790000000-000000000000", + "version": 1, + "s3_url": "/verifier-outputs/tests/inst-1-1790000000-000000000000/stderr.txt", + "object_url": "/verifier-outputs/tests/inst-1-1790000000-000000000000/stderr.txt" + }, + "stdout_head": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "stderr_head": "", + "extracted_files": { + "/out/ctrf.json": { + "tests": [ + { + "name": "test_0", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_1", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_2", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_3", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_4", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_5", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_6", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_7", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_8", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_9", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_10", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_11", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_12", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_13", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_14", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_15", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_16", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_17", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_18", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_19", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_20", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_21", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_22", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_23", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_24", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_25", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_26", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_27", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_28", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_29", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_30", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_31", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_32", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_33", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_34", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_35", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_36", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_37", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_38", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_39", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_40", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_41", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_42", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_43", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_44", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_45", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_46", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_47", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_48", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_49", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_50", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_51", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_52", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_53", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_54", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_55", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_56", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_57", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_58", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_59", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_60", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_61", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_62", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_63", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_64", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_65", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_66", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_67", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_68", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_69", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_70", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_71", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_72", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_73", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_74", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_75", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_76", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_77", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_78", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_79", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_80", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_81", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_82", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_83", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_84", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_85", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_86", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_87", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_88", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_89", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_90", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_91", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_92", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_93", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_94", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_95", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_96", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_97", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_98", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_99", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_100", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_101", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_102", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_103", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_104", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_105", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_106", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_107", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_108", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_109", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_110", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_111", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_112", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_113", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_114", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_115", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_116", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_117", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_118", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_119", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_120", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_121", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_122", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_123", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_124", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_125", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_126", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_127", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_128", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_129", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_130", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_131", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_132", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_133", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_134", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_135", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_136", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_137", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_138", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_139", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_140", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_141", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_142", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_143", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_144", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_145", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_146", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_147", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_148", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_149", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_150", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_151", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_152", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_153", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_154", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_155", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_156", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_157", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_158", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_159", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_160", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_161", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_162", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_163", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_164", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_165", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_166", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_167", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_168", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_169", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_170", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_171", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_172", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_173", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_174", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_175", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_176", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_177", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_178", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_179", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_180", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_181", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_182", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_183", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_184", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_185", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_186", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_187", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_188", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_189", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_190", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_191", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_192", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_193", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_194", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_195", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_196", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_197", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_198", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_199", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_200", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_201", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_202", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_203", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_204", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_205", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_206", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_207", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_208", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_209", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_210", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_211", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_212", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_213", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_214", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_215", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_216", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_217", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_218", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_219", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_220", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_221", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_222", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_223", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_224", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_225", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_226", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_227", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_228", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_229", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_230", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_231", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_232", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_233", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_234", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_235", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_236", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_237", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_238", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_239", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_240", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_241", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_242", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_243", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_244", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_245", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_246", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_247", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_248", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_249", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_250", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_251", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_252", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_253", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_254", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_255", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_256", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_257", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_258", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_259", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_260", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_261", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_262", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_263", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_264", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_265", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_266", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_267", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_268", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_269", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_270", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_271", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_272", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_273", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_274", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_275", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_276", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_277", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_278", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_279", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_280", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_281", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_282", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_283", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_284", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_285", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_286", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_287", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_288", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_289", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_290", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_291", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_292", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_293", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_294", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_295", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_296", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_297", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_298", + "status": "passed", + "duration": 0.01 + }, + { + "name": "test_299", + "status": "passed", + "duration": 0.01 + } + ] + } + } + } + }, + "step_params_and_seed": { + "raised": null, + "entry": { + "results": [ + { + "id": "exit_code", + "score": 1.0, + "result": true, + "message": "exit_code=0 timed_out=False" + } + ], + "score": 1.0, + "command": "pytest -k 'a b'", + "exit_code": 0, + "timed_out": false, + "stdout_artifact": { + "id": "inst-1__verifier-stdout-tests-1790000000-000000000000", + "version": 1, + "s3_url": "/verifier-outputs/tests/inst-1-1790000000-000000000000/stdout.txt", + "object_url": "/verifier-outputs/tests/inst-1-1790000000-000000000000/stdout.txt" + }, + "stderr_artifact": { + "id": "inst-1__verifier-stderr-tests-1790000000-000000000000", + "version": 1, + "s3_url": "/verifier-outputs/tests/inst-1-1790000000-000000000000/stderr.txt", + "object_url": "/verifier-outputs/tests/inst-1-1790000000-000000000000/stderr.txt" + }, + "stdout_head": "", + "stderr_head": "", + "extracted_files": {} + } + }, + "no_instance_id": { + "raised": null, + "entry": { + "results": [ + { + "id": "exit_code", + "score": 1.0, + "result": true, + "message": "exit_code=0 timed_out=False" + } + ], + "score": 1.0, + "command": "bash /tests/test.sh", + "exit_code": 0, + "timed_out": false, + "stdout_artifact": { + "id": "adhoc-000000000000__verifier-stdout-tests-1790000000-000000000000", + "version": 1, + "s3_url": "/verifier-outputs/tests/1790000000-000000000000/stdout.txt", + "object_url": "/verifier-outputs/tests/1790000000-000000000000/stdout.txt" + }, + "stderr_artifact": { + "id": "adhoc-000000000000__verifier-stderr-tests-1790000000-000000000000", + "version": 1, + "s3_url": "/verifier-outputs/tests/1790000000-000000000000/stderr.txt", + "object_url": "/verifier-outputs/tests/1790000000-000000000000/stderr.txt" + }, + "stdout_head": "", + "stderr_head": "", + "extracted_files": {} + } + }, + "container_not_deployed": { + "raised": "RuntimeError: Container 'other' not found on sandbox 'h' in context.metadata['deployed_docker_containers']", + "entry": null + }, + "sandbox_not_deployed": { + "raised": "StepContextError: Sandbox 'h' not found in context.deployed_sandboxes", + "entry": null + }, + "env_var_override_not_a_posix_name": { + "raised": "ValueError: step_params env_vars key 'A B' is not a POSIX name ([A-Za-z_][A-Za-z0-9_]*)", + "entry": null + }, + "computer_reward": { + "raised": null, + "entry": { + "results": [ + { + "id": "reward", + "score": 1.0, + "result": true, + "message": "reward=1.0 (exit_code=0 timed_out=False)" + } + ], + "score": 1.0, + "command": "echo '{\"n\": 1}' > out.json; [ \"$ANSWER\" = 42 ] && echo 1 > logs/reward.txt; true", + "exit_code": 0, + "timed_out": false, + "stdout_artifact": { + "id": "adhoc-000000000000__verifier-stdout-tests-1790000000-000000000000", + "version": 1, + "s3_url": "/verifier-outputs/tests/1790000000-000000000000/stdout.txt", + "object_url": "/verifier-outputs/tests/1790000000-000000000000/stdout.txt" + }, + "stderr_artifact": { + "id": "adhoc-000000000000__verifier-stderr-tests-1790000000-000000000000", + "version": 1, + "s3_url": "/verifier-outputs/tests/1790000000-000000000000/stderr.txt", + "object_url": "/verifier-outputs/tests/1790000000-000000000000/stderr.txt" + }, + "stdout_head": "", + "stderr_head": "", + "extracted_files": { + "out.json": { + "n": 1 + }, + "logs/reward.txt": 1 + } + } + }, + "computer_tests_fail": { + "raised": "RuntimeError: Verifier 'tests' exited 2; stderr (last 1500 chars):\nboom\n", + "entry": { + "results": [ + { + "id": "exit_code", + "score": 0.0, + "result": false, + "message": "exit_code=2 timed_out=False" + } + ], + "score": 0.0, + "command": "echo boom >&2; exit 2", + "exit_code": 2, + "timed_out": false, + "stdout_artifact": { + "id": "adhoc-000000000000__verifier-stdout-tests-1790000000-000000000000", + "version": 1, + "s3_url": "/verifier-outputs/tests/1790000000-000000000000/stdout.txt", + "object_url": "/verifier-outputs/tests/1790000000-000000000000/stdout.txt" + }, + "stderr_artifact": { + "id": "adhoc-000000000000__verifier-stderr-tests-1790000000-000000000000", + "version": 1, + "s3_url": "/verifier-outputs/tests/1790000000-000000000000/stderr.txt", + "object_url": "/verifier-outputs/tests/1790000000-000000000000/stderr.txt" + }, + "stdout_head": "", + "stderr_head": "boom\n", + "extracted_files": {} + } + }, + "computer_setup_fails": { + "raised": "RuntimeError: Verifier 'tests' setup command 1 exited 3: ", + "entry": null + }, + "computer_env_not_deployed": { + "raised": "StepContextError: Env 'nope' not found in context.deployed_envs", + "entry": null + }, + "probes_pass_beside_a_row_they_skip": { + "raised": null, + "entry": { + "results": [ + { + "type": "probe_file_exists", + "criterion": "files", + "paths": [ + "hello.txt", + "/etc/hosts" + ], + "criterion_index": 0, + "score": 1.0, + "result": true, + "justification": "All paths exist", + "skipped": false + }, + { + "type": "probe_file_contains", + "criterion": "greets", + "paths": [ + "hello.txt" + ], + "expected": "hel+o", + "weight": 2, + "criterion_index": 1, + "score": 1.0, + "result": true, + "justification": "contains 'hel+o'", + "skipped": false + }, + { + "type": "bash_cmd_succeeds", + "criterion": "check", + "bash_cmd": "python3 check.py", + "id": "chk", + "criterion_index": 2, + "score": 1.0, + "result": true, + "justification": "exit=0; stderr=", + "skipped": false + }, + { + "type": "response_contains", + "criterion": "not ours", + "weight": 5, + "criterion_index": 3, + "skipped": true, + "justification": "type 'response_contains' not handled by verify_sandbox" + }, + { + "type": "probe_dir_exists", + "criterion": "dir", + "paths": [ + "out" + ], + "criterion_index": 4, + "score": 1.0, + "result": true, + "justification": "All paths exist", + "skipped": false + } + ], + "score": 1.0 + } + }, + "a_probe_fails": { + "raised": null, + "entry": { + "results": [ + { + "type": "probe_file_exists", + "criterion": "files", + "paths": [ + "hello.txt", + "/etc/hosts" + ], + "criterion_index": 0, + "score": 1.0, + "result": true, + "justification": "All paths exist", + "skipped": false + }, + { + "type": "probe_file_contains", + "criterion": "greets", + "paths": [ + "hello.txt" + ], + "expected": "hel+o", + "weight": 2, + "criterion_index": 1, + "score": 1.0, + "result": true, + "justification": "contains 'hel+o'", + "skipped": false + }, + { + "type": "bash_cmd_succeeds", + "criterion": "check", + "bash_cmd": "python3 check.py", + "id": "chk", + "criterion_index": 2, + "score": 1.0, + "result": true, + "justification": "exit=0; stderr=", + "skipped": false + }, + { + "type": "response_contains", + "criterion": "not ours", + "weight": 5, + "criterion_index": 3, + "skipped": true, + "justification": "type 'response_contains' not handled by verify_sandbox" + }, + { + "type": "probe_dir_exists", + "criterion": "dir", + "paths": [ + "out" + ], + "criterion_index": 4, + "score": 0.0, + "result": false, + "justification": "Missing: ['out']", + "skipped": false + } + ], + "score": 0.0 + } + }, + "probes_weighted": { + "raised": null, + "entry": { + "results": [ + { + "type": "probe_file_exists", + "criterion": "files", + "paths": [ + "hello.txt", + "/etc/hosts" + ], + "criterion_index": 0, + "score": 1.0, + "result": true, + "justification": "All paths exist", + "skipped": false + }, + { + "type": "probe_file_contains", + "criterion": "greets", + "paths": [ + "hello.txt" + ], + "expected": "hel+o", + "weight": 2, + "criterion_index": 1, + "score": 1.0, + "result": true, + "justification": "contains 'hel+o'", + "skipped": false + }, + { + "type": "bash_cmd_succeeds", + "criterion": "check", + "bash_cmd": "python3 check.py", + "id": "chk", + "criterion_index": 2, + "score": 1.0, + "result": true, + "justification": "exit=0; stderr=", + "skipped": false + }, + { + "type": "response_contains", + "criterion": "not ours", + "weight": 5, + "criterion_index": 3, + "skipped": true, + "justification": "type 'response_contains' not handled by verify_sandbox" + }, + { + "type": "probe_dir_exists", + "criterion": "dir", + "paths": [ + "out" + ], + "criterion_index": 4, + "score": 0.0, + "result": false, + "justification": "Missing: ['out']", + "skipped": false + } + ], + "score": 0.8 + } + }, + "probes_any_pass": { + "raised": null, + "entry": { + "results": [ + { + "type": "probe_file_exists", + "criterion": "files", + "paths": [ + "hello.txt", + "/etc/hosts" + ], + "criterion_index": 0, + "score": 1.0, + "result": true, + "justification": "All paths exist", + "skipped": false + }, + { + "type": "probe_file_contains", + "criterion": "greets", + "paths": [ + "hello.txt" + ], + "expected": "hel+o", + "weight": 2, + "criterion_index": 1, + "score": 1.0, + "result": true, + "justification": "contains 'hel+o'", + "skipped": false + }, + { + "type": "bash_cmd_succeeds", + "criterion": "check", + "bash_cmd": "python3 check.py", + "id": "chk", + "criterion_index": 2, + "score": 1.0, + "result": true, + "justification": "exit=0; stderr=", + "skipped": false + }, + { + "type": "response_contains", + "criterion": "not ours", + "weight": 5, + "criterion_index": 3, + "skipped": true, + "justification": "type 'response_contains' not handled by verify_sandbox" + }, + { + "type": "probe_dir_exists", + "criterion": "dir", + "paths": [ + "out" + ], + "criterion_index": 4, + "score": 0.0, + "result": false, + "justification": "Missing: ['out']", + "skipped": false + } + ], + "score": 1.0 + } + }, + "probes_in_the_agents_container": { + "raised": null, + "entry": { + "results": [ + { + "type": "probe_file_exists", + "criterion": "files", + "paths": [ + "hello.txt", + "/etc/hosts" + ], + "criterion_index": 0, + "score": 1.0, + "result": true, + "justification": "All paths exist", + "skipped": false + }, + { + "type": "probe_file_contains", + "criterion": "greets", + "paths": [ + "hello.txt" + ], + "expected": "hel+o", + "weight": 2, + "criterion_index": 1, + "score": 1.0, + "result": true, + "justification": "contains 'hel+o'", + "skipped": false + }, + { + "type": "bash_cmd_succeeds", + "criterion": "check", + "bash_cmd": "python3 check.py", + "id": "chk", + "criterion_index": 2, + "score": 1.0, + "result": true, + "justification": "exit=0; stderr=", + "skipped": false + }, + { + "type": "response_contains", + "criterion": "not ours", + "weight": 5, + "criterion_index": 3, + "skipped": true, + "justification": "type 'response_contains' not handled by verify_sandbox" + }, + { + "type": "probe_dir_exists", + "criterion": "dir", + "paths": [ + "out" + ], + "criterion_index": 4, + "score": 1.0, + "result": true, + "justification": "All paths exist", + "skipped": false + } + ], + "score": 1.0 + } + }, + "every_row_skipped": { + "raised": null, + "entry": { + "results": [ + { + "type": "response_contains", + "criterion": "x", + "criterion_index": 0, + "skipped": true, + "justification": "type 'response_contains' not handled by verify_sandbox" + } + ], + "score": 0.0 + } + }, + "no_probes": { + "raised": null, + "entry": { + "results": [], + "score": 0.0 + } + }, + "a_probe_that_raises": { + "raised": null, + "entry": { + "results": [ + { + "type": "bash_cmd_succeeds", + "criterion": "c", + "bash_cmd": "true", + "criterion_index": 0, + "score": 0.0, + "result": false, + "justification": "evaluator error: channel closed", + "skipped": false + } + ], + "score": 0.0 + } + }, + "a_probe_that_times_out": { + "raised": null, + "entry": { + "results": [ + { + "type": "bash_cmd_succeeds", + "criterion": "c", + "bash_cmd": "true", + "criterion_index": 0, + "score": 0.0, + "result": false, + "justification": "timed out after 0.01s", + "skipped": false + } + ], + "score": 0.0 + } + }, + "probed_sandbox_not_deployed": { + "raised": "StepContextError: Sandbox 'nope' not found in context.deployed_sandboxes (deployed: ['box'])", + "entry": null + }, + "probed_agent_not_deployed": { + "raised": "StepContextError: Agent 'solver' not found in context.deployed_agents", + "entry": null + }, + "probed_agent_without_a_sandbox": { + "raised": "RuntimeError: Agent 'solver' has no sandbox_id", + "entry": null + }, + "outcome_rows": { + "raised": null, + "entry": { + "results": [ + { + "id": "a", + "score": 1.0, + "result": true, + "why": "ok" + }, + { + "id": "b", + "score": 0.5, + "weight": 2 + }, + { + "id": "c", + "result": false, + "weight": -1, + "skipped": true + }, + { + "result": true, + "extra": { + "n": [ + 1, + 2 + ] + } + } + ], + "score": 0.0 + } + }, + "outcome_rows_weighted": { + "raised": null, + "entry": { + "results": [ + { + "id": "a", + "score": 1.0, + "result": true, + "why": "ok" + }, + { + "id": "b", + "score": 0.5, + "weight": 2 + }, + { + "id": "c", + "result": false, + "weight": -1, + "skipped": true + }, + { + "result": true, + "extra": { + "n": [ + 1, + 2 + ] + } + } + ], + "score": 0.5 + } + }, + "outcome_no_rows": { + "raised": null, + "entry": { + "results": [], + "score": 0.0 + } + }, + "outcome_returns_none": { + "raised": "TypeError: object of type 'NoneType' has no len()", + "entry": { + "results": null, + "score": 0.0 + } + }, + "outcome_returns_a_tuple": { + "raised": null, + "entry": { + "results": [ + { + "id": "a", + "result": true + } + ], + "score": 1.0 + } + }, + "outcome_rows_that_arent_dicts": { + "raised": "AttributeError: 'str' object has no attribute 'get'", + "entry": null + }, + "outcome_row_score_a_string": { + "raised": "TypeError: can't multiply sequence by non-int of type 'float'", + "entry": null + }, + "outcome_script_raises": { + "raised": "ValueError: db unreachable", + "entry": null + }, + "outcome_env_not_deployed": { + "raised": "StepContextError: Env 'e' not found in context.deployed_envs", + "entry": null + }, + "aggregate_two_sources": { + "raised": null, + "entry": { + "score": 0.25, + "source_verifier_ids": [ + "fs", + "judge" + ] + } + }, + "aggregate_all_pass": { + "raised": null, + "entry": { + "score": 0.0, + "source_verifier_ids": [ + "fs", + "judge" + ] + } + }, + "aggregate_of_an_aggregate": { + "raised": null, + "entry": { + "score": 1.0, + "source_verifier_ids": [ + "inner", + "fs" + ] + } + }, + "aggregate_of_a_source_without_rows": { + "raised": null, + "entry": { + "score": 0.0, + "source_verifier_ids": [ + "no-rows" + ] + } + }, + "aggregate_of_nothing": { + "raised": null, + "entry": { + "score": 0.0, + "source_verifier_ids": [] + } + }, + "aggregate_missing_a_source": { + "raised": "RuntimeError: AggregateVerifiers: verifier_id(s) not found in context.metadata.verifications: ['gone']. Available: ['fs', 'judge', 'inner', 'no-rows']", + "entry": null + } +} diff --git a/tst/unit/task_step/verifiers/test_legacy_entries.py b/tst/unit/task_step/verifiers/test_legacy_entries.py new file mode 100644 index 000000000..f5ba5195b --- /dev/null +++ b/tst/unit/task_step/verifiers/test_legacy_entries.py @@ -0,0 +1,478 @@ +"""run_container_unit_tests_verifier, verify_sandbox, env_outcome_verifier and aggregate_verifiers write the legacy +``verifications`` entry they wrote before the VerifierTaskStep base, byte for byte with its key order, and raise what +they raised: ``recorded/legacy_entries.json`` was recorded from main at 25b862bc6, object urls under ``/``.""" +import asyncio +import itertools +import json +import shlex +import uuid +from pathlib import Path +from types import SimpleNamespace + +import pytest + +import agent_env.providers.sandbox_providers.sandbox_provider as sandbox_provider +from agent_env.artifact import FileArtifact +from agent_env.config import get_config +from agent_env.env.env import DeployedEnv +from agent_env.task_step.context import DeployedAgent, DeployedSandbox, TaskStepContext +from agent_env.task_step.task_steps.verifiers import verify_sandbox as verify_sandbox_module +from agent_env.task_step.task_steps.verifiers.aggregate_verifiers import AggregateVerifiersTaskStep +from agent_env.task_step.task_steps.verifiers.env_outcome_verifier import EnvOutcomeVerifierTaskStep +from agent_env.task_step.task_steps.verifiers.run_container_unit_tests_verifier import ( + RunContainerUnitTestsVerifierTaskStep as RunContainer, +) +from agent_env.task_step.task_steps.verifiers.verify_sandbox import VerifySandboxTaskStep +from tst.unit.computer_home import Computer + +RECORDED = json.loads((Path(__file__).parent / "recorded" / "legacy_entries.json").read_text()) +REWARD = "/logs/verifier/reward.txt" +SCENARIOS = {} + + +def scenario(fn): + SCENARIOS[fn.__name__] = fn + return fn + + +def outcome(step, context) -> tuple[str | None, str | None]: + """What ``step`` raised running on ``context``, and its entry as JSON with the object store's urls normalised.""" + raised = None + try: + asyncio.run(step.execute(context)) + except Exception as exc: + raised = f"{type(exc).__name__}: {exc}" + entry = (context.metadata.get("verifications") or {}).get(step.verifier_id) + objects = get_config().get_object_store().object_url("x")[:-1] + return raised, json.dumps(entry).replace(objects, "/") + + +class _ContainerSandbox: + """The task container's files as a dict: ``rm -f`` and ``docker cp`` act on it, and the command runs ``test_sh``, + which returns its exit code, stdout and stderr.""" + + type = None + + def __init__(self, files, test_sh): + self.files, self.vm_files, self.test_sh = dict(files), {}, test_sh + + def scoped_name(self, name): + return name + + async def exec_script(self, script): + words = shlex.split(script) + if words[0] == "docker" and "rm" in words: + for path in words[words.index("--") + 1:]: + self.files.pop(path, None) + elif words[:2] == ["rm", "-f"]: + self.vm_files.pop(words[2], None) + return "" + + async def exec_with_output(self, *args): + if args[:2] == ("sudo", "cat"): + return 0, self.vm_files[args[2]], "" + return self.test_sh(self.files) + + async def docker_cp(self, source, destination): + path = source.split(":", 1)[1] + if path not in self.files: + raise RuntimeError(f"no such file: {path}") + self.vm_files[destination] = self.files[path] + + +def _pinned(monkeypatch): + """The outputs' keys are named after the time and a uuid: pinned, so the entry is the same every run.""" + counter = itertools.count() + monkeypatch.setattr(uuid, "uuid4", lambda: uuid.UUID(int=next(counter))) + monkeypatch.setattr("agent_env.task_step.task_steps.verifiers.run_container_unit_tests_verifier.time.time", + lambda: 1_790_000_000.0) + monkeypatch.setattr(RunContainer, "_upload_text_artifact", staticmethod( + lambda text, artifact_id, description, object_url: SimpleNamespace(id=artifact_id, version=1, + object_url=object_url))) + + +def _in_container(monkeypatch, sandbox, *, instance_id="inst-1", step_params=None, seed=None): + _pinned(monkeypatch) + monkeypatch.setattr(sandbox_provider, "get_sandbox_provider", + lambda: SimpleNamespace(get_sandbox=lambda sandbox_id: asyncio.sleep(0, result=sandbox))) + context = TaskStepContext(instance_id=instance_id) + context.metadata["deployed_docker_containers"] = [{"container_name": "c", "sandbox_name": "h"}] + context.deployed_sandboxes.append(DeployedSandbox(sandbox_name="h", sandbox_id="sb-1", sandbox_mode="vm")) + if step_params is not None: + context.metadata["user_overrides"] = {"step_params": {"tests": step_params}} + if seed is not None: + context.metadata["seed"] = seed + return context + + +def _tests(**fields): + return RunContainer(id="tests", version=None, sandbox_name="h", container_name="c", + **{"command": "bash /tests/test.sh", **fields}) + + +def _writes_reward(value, exit_code=0): + def test_sh(files): + if value is not None: + files[REWARD] = value + return exit_code, "3 passed", "" + return test_sh + + +@scenario +def passing_tests(monkeypatch, tmp_path): + return _tests(), _in_container(monkeypatch, _ContainerSandbox({}, lambda files: (0, "ok\n" * 400, "warn"))) + + +@scenario +def failing_tests_raise_after_recording(monkeypatch, tmp_path): + return _tests(verifier_id="unit"), _in_container(monkeypatch, _ContainerSandbox( + {}, lambda files: (1, "1 failed", "E" * 3000))) + + +@scenario +def timeout_at_124(monkeypatch, tmp_path): + return _tests(timeout_sec=5), _in_container(monkeypatch, _ContainerSandbox({}, lambda files: (124, "", "killed"))) + + +@scenario +def timeout_at_137(monkeypatch, tmp_path): + return _tests(), _in_container(monkeypatch, _ContainerSandbox({}, lambda files: (137, "", ""))) + + +@scenario +def reward_one(monkeypatch, tmp_path): + return _tests(reward_path=REWARD), _in_container(monkeypatch, _ContainerSandbox({}, _writes_reward("1\n"))) + + +@scenario +def reward_half(monkeypatch, tmp_path): + return _tests(reward_path=REWARD), _in_container(monkeypatch, _ContainerSandbox({}, _writes_reward("0.5"))) + + +@scenario +def reward_clamped(monkeypatch, tmp_path): + return _tests(reward_path=REWARD), _in_container(monkeypatch, _ContainerSandbox({}, _writes_reward("2"))) + + +@scenario +def reward_missing_after_a_failing_exit(monkeypatch, tmp_path): + return _tests(reward_path=REWARD), _in_container(monkeypatch, _ContainerSandbox( + {REWARD: "1\n"}, _writes_reward(None, exit_code=1))) + + +@scenario +def reward_not_a_number(monkeypatch, tmp_path): + return _tests(reward_path=REWARD), _in_container(monkeypatch, _ContainerSandbox({}, _writes_reward("FAIL"))) + + +@scenario +def reward_then_time_out(monkeypatch, tmp_path): + return _tests(reward_path=REWARD), _in_container(monkeypatch, _ContainerSandbox( + {}, _writes_reward("1", exit_code=124))) + + +@scenario +def result_files(monkeypatch, tmp_path): + files = {"/out/report.json": '{"tests": 3, "passed": 3}', "/out/log.txt": "plain"} + return (_tests(result_paths=["/out/report.json", "/out/log.txt", "/out/missing.json"]), + _in_container(monkeypatch, _ContainerSandbox(files, lambda files: (0, "", "")))) + + +@scenario +def result_file_over_the_inline_limit(monkeypatch, tmp_path): + report = json.dumps({"tests": [{"name": f"test_{i}", "status": "passed", "duration": 0.01} for i in range(300)]}) + return (_tests(result_paths=["/out/ctrf.json"]), + _in_container(monkeypatch, _ContainerSandbox({"/out/ctrf.json": report}, lambda files: (0, "x" * 5000, "")))) + + +@scenario +def step_params_and_seed(monkeypatch, tmp_path): + return _tests(command="run ", env_vars={"STORED": "1"}), _in_container( + monkeypatch, _ContainerSandbox({}, lambda files: (0, "", "")), seed={"case": "a b"}, + step_params={"command": "pytest -k ", "env_vars": {"TOKEN": "s3cret"}}) + + +@scenario +def no_instance_id(monkeypatch, tmp_path): + return _tests(), _in_container(monkeypatch, _ContainerSandbox({}, lambda files: (0, "", "")), instance_id=None) + + +@scenario +def container_not_deployed(monkeypatch, tmp_path): + return (RunContainer(id="tests", version=None, sandbox_name="h", container_name="other", command="true"), + _in_container(monkeypatch, _ContainerSandbox({}, lambda files: (0, "", "")))) + + +@scenario +def sandbox_not_deployed(monkeypatch, tmp_path): + context = _in_container(monkeypatch, _ContainerSandbox({}, lambda files: (0, "", ""))) + context.deployed_sandboxes.clear() + return _tests(), context + + +@scenario +def env_var_override_not_a_posix_name(monkeypatch, tmp_path): + return _tests(), _in_container(monkeypatch, _ContainerSandbox({}, lambda files: (0, "", "")), + step_params={"env_vars": {"A B": "x"}}) + + +def _in_computer(monkeypatch, tmp_path): + _pinned(monkeypatch) + computer = Computer(tmp_path).install(monkeypatch) + (tmp_path / "logs").mkdir() + return TaskStepContext(deployed_envs=[computer.deployed()]) + + +@scenario +def computer_reward(monkeypatch, tmp_path): + return RunContainer(id="tests", version=None, env_id="desk", setup_commands=["touch setup-ran"], + env_vars={"ANSWER": "42"}, reward_path="logs/reward.txt", result_paths=["out.json"], + command='echo \'{"n": 1}\' > out.json; [ "$ANSWER" = 42 ] && echo 1 > logs/reward.txt; true' + ), _in_computer(monkeypatch, tmp_path) + + +@scenario +def computer_tests_fail(monkeypatch, tmp_path): + return (RunContainer(id="tests", version=None, env_id="desk", command="echo boom >&2; exit 2"), + _in_computer(monkeypatch, tmp_path)) + + +@scenario +def computer_setup_fails(monkeypatch, tmp_path): + return (RunContainer(id="tests", version=None, env_id="desk", setup_commands=["exit 3"], command="true"), + _in_computer(monkeypatch, tmp_path)) + + +@scenario +def computer_env_not_deployed(monkeypatch, tmp_path): + return (RunContainer(id="tests", version=None, env_id="nope", command="true"), _in_computer(monkeypatch, tmp_path)) + + +class _ProbeSandbox: + """A VM whose probes see every file but ``missing``, whose bash checks fail when they run ``false``, and whose + commands raise on ``raises_on`` or hang.""" + + mode, sandbox_id, container_name = "vm", "sb-1", "agent-api" + + def __init__(self, missing=(), raises_on=None, hangs=False): + self.missing, self.raises_on, self.hangs = set(missing), raises_on, hangs + + async def exec_with_output(self, *args): + if self.hangs: + await asyncio.sleep(1) + if self.raises_on in args: + raise ConnectionError("channel closed") + if args[-3:] == ("ps", "--format", "{{.Names}}"): + return 0, "agent-api\n", "" + if "test" in args and args[-1] in self.missing: + return 1, "", "" + if "cat" in args: + return 0, "hello world\n", "" + if "bash" in args: + return (1, "", "nope") if "false" in args[-1] else (0, "", "") + return 0, "", "" + + +def _on_sandbox(monkeypatch, sandbox, **context): + monkeypatch.setattr(verify_sandbox_module, "build_sandbox_provider", lambda spec: SimpleNamespace( + get_sandbox=lambda sandbox_id: asyncio.sleep(0, result=sandbox))) + return TaskStepContext(**context) if context else TaskStepContext(deployed_sandboxes=[ + DeployedSandbox(sandbox_name="box", sandbox_id="sb-1", sandbox_mode="vm", sandbox_type="fake")]) + + +PROBES = [ + {"type": "probe_file_exists", "criterion": "files", "paths": ["hello.txt", "/etc/hosts"]}, + {"type": "probe_file_contains", "criterion": "greets", "paths": ["hello.txt"], "expected": "hel+o", "weight": 2}, + {"type": "bash_cmd_succeeds", "criterion": "check", "bash_cmd": "python3 check.py", "id": "chk"}, + {"type": "response_contains", "criterion": "not ours", "weight": 5}, + {"type": "probe_dir_exists", "criterion": "dir", "paths": ["out"]}, +] + + +def _probes(criteria=PROBES, **fields): + return VerifySandboxTaskStep(id="fs", version=None, base_dir="/app", verifier_id="fs-vid", + **{"sandbox_name": "box", **fields}, criteria=criteria) + + +@scenario +def probes_pass_beside_a_row_they_skip(monkeypatch, tmp_path): + return _probes(), _on_sandbox(monkeypatch, _ProbeSandbox()) + + +@scenario +def a_probe_fails(monkeypatch, tmp_path): + return _probes(), _on_sandbox(monkeypatch, _ProbeSandbox(missing={"/app/out"})) + + +@scenario +def probes_weighted(monkeypatch, tmp_path): + return _probes(score_aggregator="weighted_average"), _on_sandbox(monkeypatch, _ProbeSandbox(missing={"/app/out"})) + + +@scenario +def probes_any_pass(monkeypatch, tmp_path): + return _probes(score_aggregator="any_pass"), _on_sandbox(monkeypatch, _ProbeSandbox(missing={"/app/out"})) + + +@scenario +def probes_in_the_agents_container(monkeypatch, tmp_path): + agent = DeployedAgent(agent_name="solver", api_url="http://agent", sandbox_id="sb-1", sandbox_type="fake") + return (_probes(sandbox_name=None, agent_name="solver"), + _on_sandbox(monkeypatch, _ProbeSandbox(), deployed_agents=[agent])) + + +@scenario +def every_row_skipped(monkeypatch, tmp_path): + return _probes([{"type": "response_contains", "criterion": "x"}]), _on_sandbox(monkeypatch, _ProbeSandbox()) + + +@scenario +def no_probes(monkeypatch, tmp_path): + return _probes([]), _on_sandbox(monkeypatch, _ProbeSandbox()) + + +@scenario +def a_probe_that_raises(monkeypatch, tmp_path): + return (_probes([{"type": "bash_cmd_succeeds", "criterion": "c", "bash_cmd": "true"}]), + _on_sandbox(monkeypatch, _ProbeSandbox(raises_on="bash"))) + + +@scenario +def a_probe_that_times_out(monkeypatch, tmp_path): + return (_probes([{"type": "bash_cmd_succeeds", "criterion": "c", "bash_cmd": "true"}], shell_timeout_seconds=0.01), + _on_sandbox(monkeypatch, _ProbeSandbox(hangs=True))) + + +@scenario +def probed_sandbox_not_deployed(monkeypatch, tmp_path): + return _probes(sandbox_name="nope"), _on_sandbox(monkeypatch, _ProbeSandbox()) + + +@scenario +def probed_agent_not_deployed(monkeypatch, tmp_path): + return _probes(sandbox_name=None, agent_name="solver"), _on_sandbox(monkeypatch, _ProbeSandbox(), + deployed_agents=[]) + + +@scenario +def probed_agent_without_a_sandbox(monkeypatch, tmp_path): + return _probes(sandbox_name=None, agent_name="solver"), _on_sandbox( + monkeypatch, _ProbeSandbox(), deployed_agents=[DeployedAgent(agent_name="solver", api_url="http://a")]) + + +def _outcome_script(monkeypatch, body, **fields): + script = f"async def verify(url):\n {body}\n".encode() + monkeypatch.setattr(FileArtifact, "get", staticmethod(lambda id, version=None: SimpleNamespace(load=lambda: script))) + step = EnvOutcomeVerifierTaskStep(id="outcome", version=None, env_id="e", file_artifact_id="s", + verifier_id="eo-vid", **fields) + context = TaskStepContext() + context.deployed_envs.append(DeployedEnv(env_id="e", env_version=1, mcp_url="http://mcp")) + return step, context + + +OUTCOME_ROWS = ("[{'id': 'a', 'score': 1.0, 'result': True, 'why': 'ok'}, {'id': 'b', 'score': 0.5, 'weight': 2}, " + "{'id': 'c', 'result': False, 'weight': -1, 'skipped': True}, {'result': True, 'extra': {'n': [1, 2]}}]") + + +@scenario +def outcome_rows(monkeypatch, tmp_path): + return _outcome_script(monkeypatch, f"return {OUTCOME_ROWS}") + + +@scenario +def outcome_rows_weighted(monkeypatch, tmp_path): + return _outcome_script(monkeypatch, f"return {OUTCOME_ROWS}", score_aggregator="weighted_average") + + +@scenario +def outcome_no_rows(monkeypatch, tmp_path): + return _outcome_script(monkeypatch, "return []") + + +@scenario +def outcome_returns_none(monkeypatch, tmp_path): + return _outcome_script(monkeypatch, "return None") + + +@scenario +def outcome_returns_a_tuple(monkeypatch, tmp_path): + return _outcome_script(monkeypatch, "return ({'id': 'a', 'result': True},)") + + +@scenario +def outcome_rows_that_arent_dicts(monkeypatch, tmp_path): + return _outcome_script(monkeypatch, "return ['a', 'b']") + + +@scenario +def outcome_row_score_a_string(monkeypatch, tmp_path): + return _outcome_script(monkeypatch, "return [{'score': '1'}]", score_aggregator="weighted_average") + + +@scenario +def outcome_script_raises(monkeypatch, tmp_path): + return _outcome_script(monkeypatch, "raise ValueError('db unreachable')") + + +@scenario +def outcome_env_not_deployed(monkeypatch, tmp_path): + step, _ = _outcome_script(monkeypatch, "return []") + return step, TaskStepContext() + + +def _sources(): + context = TaskStepContext() + context.metadata["verifications"] = { + "fs": {"results": [{"id": "x", "score": 1.0, "result": True, "skipped": False}, + {"id": "y", "skipped": True, "justification": "not handled"}], "score": 1.0}, + "judge": {"format": "rubric_binary", "results": [{"id": "r1", "score": 0.0, "result": False, "weight": 3}], + "score": 0.0}, + "inner": {"score": 0.25, "source_verifier_ids": ["fs"]}, + "no-rows": {"score": 1.0}, + } + return context + + +def _aggregate(verifier_ids, **fields): + return AggregateVerifiersTaskStep(id="final", version=None, verifier_ids=verifier_ids, verifier_id="final-vid", + **fields) + + +@scenario +def aggregate_two_sources(monkeypatch, tmp_path): + return _aggregate(["fs", "judge"]), _sources() + + +@scenario +def aggregate_all_pass(monkeypatch, tmp_path): + return _aggregate(["fs", "judge"], score_aggregator="all_pass"), _sources() + + +@scenario +def aggregate_of_an_aggregate(monkeypatch, tmp_path): + return _aggregate(["inner", "fs"]), _sources() + + +@scenario +def aggregate_of_a_source_without_rows(monkeypatch, tmp_path): + return _aggregate(["no-rows"]), _sources() + + +@scenario +def aggregate_of_nothing(monkeypatch, tmp_path): + return _aggregate([]), _sources() + + +@scenario +def aggregate_missing_a_source(monkeypatch, tmp_path): + return _aggregate(["fs", "gone"]), _sources() + + +@pytest.mark.parametrize("name", list(SCENARIOS)) +def test_each_path_writes_the_entry_and_raises_what_it_did_before_the_base(name, local_stores, monkeypatch, tmp_path): + step, context = SCENARIOS[name](monkeypatch, tmp_path) + + raised, entry = outcome(step, context) + + assert raised == RECORDED[name]["raised"] + assert entry == json.dumps(RECORDED[name]["entry"]) From 4dd27d8964100f46ccc1858777d9184ce68a5724 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 18 Nov 2026 14:21:48 -0800 Subject: [PATCH 03/12] test(verifiers): the rubrics family's legacy entries, recorded before it moves onto the verifier base rubrics_verifier (20 paths: each judge format, batches and retries, the skips, the gate, judge failures, unjudged rows, an agent judge, a run's overrides), user_sim_judge (15) and always_on_agent_verifier (4), each run offline with the judge faked below the verifier's own code. The entries, the models record and the error each run leaves are recorded under tst/data/verifier_golden and must stay equal, number types included. Co-Authored-By: Claude Opus 5.5 --- .../verifier_golden/rubrics_subclasses.json | 2593 +++++++++++++++++ .../verifier_golden/rubrics_verifier.json | 828 ++++++ .../test_rubrics_subclasses_golden.py | 173 ++ .../verifiers/test_rubrics_verifier_golden.py | 195 ++ tst/util/verifier_golden.py | 30 + 5 files changed, 3819 insertions(+) create mode 100644 tst/data/verifier_golden/rubrics_subclasses.json create mode 100644 tst/data/verifier_golden/rubrics_verifier.json create mode 100644 tst/unit/task_step/verifiers/test_rubrics_subclasses_golden.py create mode 100644 tst/unit/task_step/verifiers/test_rubrics_verifier_golden.py create mode 100644 tst/util/verifier_golden.py diff --git a/tst/data/verifier_golden/rubrics_subclasses.json b/tst/data/verifier_golden/rubrics_subclasses.json new file mode 100644 index 000000000..bcb18ca95 --- /dev/null +++ b/tst/data/verifier_golden/rubrics_subclasses.json @@ -0,0 +1,2593 @@ +{ + "us_passing_with_the_gate_off": { + "verifications": { + "usj": { + "format": "user_sim_checks", + "results": [ + { + "id": "guidance", + "title": "Guidance", + "families": [ + "A1", + "A3" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "oversharing", + "title": "Oversharing", + "families": [ + "G1", + "G2" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "delegation", + "title": "Delegation", + "families": [ + "A4" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "style", + "title": "Style", + "families": [ + "F1", + "F2", + "F3", + "F4" + ], + "weight": 0.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "rogue", + "title": "Rogue", + "families": [ + "G3", + "G5" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "made_up", + "title": "Made-up facts", + "families": [ + "A2" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "ending", + "title": "Ending", + "families": [ + "G4" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + } + ], + "score": 1.0, + "compact_trajectory_s3_uri": null, + "compact_trajectory_object_url": null, + "judge_trajectory_s3_uri": null, + "judge_trajectory_object_url": null, + "judge_batches": 7, + "judge_batch_trajectory_object_urls": [ + null, + null, + null, + null, + null, + null, + null + ], + "run_step_id": "talk", + "run_kind": "prompt_agent", + "judge_model": "fake/judge", + "user_sim": "hannah", + "instructions": { + "source": "rebuilt", + "prompt_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "recorded_prompt_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "matches": true + }, + "graded_turns": [ + 1 + ], + "turns": 1, + "notes": [], + "user_sim_error": { + "value": "no", + "because": [], + "unknown": [], + "fails_task": false + }, + "calls": 7, + "usage": { + "prompt_tokens": 700, + "completion_tokens": 140, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0 + }, + "cost": 0.007 + } + }, + "models": { + "judge": { + "judge_user_sim": "fake/judge" + } + }, + "raised": null + }, + "us_the_gate_on_a_slip_that_changes_the_grade": { + "verifications": { + "usj": { + "format": "user_sim_checks", + "results": [ + { + "id": "guidance", + "title": "Guidance", + "families": [ + "A1", + "A3" + ], + "weight": 1.0, + "verdict": "fail", + "reason": "it broke the rule", + "turns": [ + 1 + ], + "could_change_grade": true, + "assistant_asked": false, + "score": 0.0, + "result": false, + "justification": "it broke the rule" + }, + { + "id": "oversharing", + "title": "Oversharing", + "families": [ + "G1", + "G2" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "delegation", + "title": "Delegation", + "families": [ + "A4" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "style", + "title": "Style", + "families": [ + "F1", + "F2", + "F3", + "F4" + ], + "weight": 0.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "rogue", + "title": "Rogue", + "families": [ + "G3", + "G5" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "made_up", + "title": "Made-up facts", + "families": [ + "A2" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "ending", + "title": "Ending", + "families": [ + "G4" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "a1", + "family": "A1", + "kind": "general", + "check": "gives the date", + "weight": 1.0, + "asked_with": "guidance", + "verdict": "fail", + "reason": "it broke the rule", + "turns": [ + 1 + ], + "could_change_grade": true, + "assistant_asked": false, + "score": 0.0, + "result": false, + "justification": "it broke the rule" + } + ], + "score": 0.0, + "compact_trajectory_s3_uri": null, + "compact_trajectory_object_url": null, + "judge_trajectory_s3_uri": null, + "judge_trajectory_object_url": null, + "judge_batches": 7, + "judge_batch_trajectory_object_urls": [ + null, + null, + null, + null, + null, + null, + null + ], + "run_step_id": "talk", + "run_kind": "prompt_agent", + "judge_model": "fake/judge", + "user_sim": "hannah", + "instructions": { + "source": "rebuilt", + "prompt_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "recorded_prompt_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "matches": true + }, + "graded_turns": [ + 1 + ], + "turns": 1, + "notes": [], + "user_sim_error": { + "value": "yes", + "because": [ + "guidance", + "a1" + ], + "unknown": [], + "fails_task": true + }, + "calls": 7, + "usage": { + "prompt_tokens": 700, + "completion_tokens": 140, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0 + }, + "cost": 0.007 + } + }, + "models": { + "judge": { + "judge_user_sim": "fake/judge" + } + }, + "raised": "VerifierNotPassingError: user_sim_judge 'judge_user_sim': the user sim slipped in a way that could change the assistant's grade (guidance, a1), so the run doesn't count; run it again" + }, + "us_a_slip_on_unconfirmed_instructions": { + "verifications": { + "usj": { + "format": "user_sim_checks", + "results": [ + { + "id": "guidance", + "title": "Guidance", + "families": [ + "A1", + "A3" + ], + "weight": 1.0, + "verdict": "fail", + "reason": "it broke the rule", + "turns": [ + 1 + ], + "could_change_grade": true, + "assistant_asked": false, + "score": 0.0, + "result": false, + "justification": "it broke the rule" + }, + { + "id": "oversharing", + "title": "Oversharing", + "families": [ + "G1", + "G2" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "delegation", + "title": "Delegation", + "families": [ + "A4" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "style", + "title": "Style", + "families": [ + "F1", + "F2", + "F3", + "F4" + ], + "weight": 0.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "rogue", + "title": "Rogue", + "families": [ + "G3", + "G5" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "made_up", + "title": "Made-up facts", + "families": [ + "A2" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "ending", + "title": "Ending", + "families": [ + "G4" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + } + ], + "score": 0.0, + "compact_trajectory_s3_uri": null, + "compact_trajectory_object_url": null, + "judge_trajectory_s3_uri": null, + "judge_trajectory_object_url": null, + "judge_batches": 7, + "judge_batch_trajectory_object_urls": [ + null, + null, + null, + null, + null, + null, + null + ], + "run_step_id": "talk", + "run_kind": "prompt_agent", + "judge_model": "fake/judge", + "user_sim": "hannah", + "instructions": { + "source": "rebuilt", + "prompt_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "recorded_prompt_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "matches": false + }, + "graded_turns": [ + 1 + ], + "turns": 1, + "notes": [], + "user_sim_error": { + "value": "yes", + "because": [ + "guidance" + ], + "unknown": [], + "fails_task": false, + "not_failing": "the judge's copy of the user sim's instructions isn't confirmed to be the one it ran with, so it doesn't fail the task" + }, + "calls": 7, + "usage": { + "prompt_tokens": 700, + "completion_tokens": 140, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0 + }, + "cost": 0.007 + } + }, + "models": { + "judge": { + "judge_user_sim": "fake/judge" + } + }, + "raised": null + }, + "us_no_graded_turns": { + "verifications": { + "usj": { + "format": "user_sim_checks", + "results": [], + "score": null, + "run_step_id": "talk", + "run_kind": "prompt_agent", + "judge_model": "fake/judge", + "user_sim": "hannah", + "instructions": { + "source": "rebuilt", + "prompt_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "recorded_prompt_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "matches": true + }, + "graded_turns": [], + "turns": 1, + "notes": [], + "skipped": "no graded turns", + "user_sim_error": { + "value": "no", + "because": [], + "unknown": [], + "fails_task": false + } + } + }, + "models": null, + "raised": null + }, + "us_a_persona_mismatch": { + "verifications": { + "usj": { + "format": "user_sim_checks", + "results": [ + { + "id": "guidance", + "title": "Guidance", + "families": [ + "A1", + "A3" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "oversharing", + "title": "Oversharing", + "families": [ + "G1", + "G2" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "delegation", + "title": "Delegation", + "families": [ + "A4" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "style", + "title": "Style", + "families": [ + "F1", + "F2", + "F3", + "F4" + ], + "weight": 0.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "rogue", + "title": "Rogue", + "families": [ + "G3", + "G5" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "made_up", + "title": "Made-up facts", + "families": [ + "A2" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "ending", + "title": "Ending", + "families": [ + "G4" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + } + ], + "score": 1.0, + "compact_trajectory_s3_uri": null, + "compact_trajectory_object_url": null, + "judge_trajectory_s3_uri": null, + "judge_trajectory_object_url": null, + "judge_batches": 7, + "judge_batch_trajectory_object_urls": [ + null, + null, + null, + null, + null, + null, + null + ], + "run_step_id": "talk", + "run_kind": "prompt_agent", + "judge_model": "fake/judge", + "user_sim": "hannah", + "instructions": { + "source": "rebuilt", + "prompt_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "recorded_prompt_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "matches": true + }, + "rubric_persona": { + "rubric": "1111111111111111111111111111111111111111111111111111111111111111", + "run": "0000000000000000000000000000000000000000000000000000000000000000", + "status": "mismatch", + "note": "the task items were written for a different persona or opening than this run's user sim had, so they weren't graded; the general checks were" + }, + "graded_turns": [ + 1 + ], + "turns": 1, + "notes": [], + "user_sim_error": { + "value": "no", + "because": [], + "unknown": [], + "fails_task": false + }, + "calls": 7, + "usage": { + "prompt_tokens": 700, + "completion_tokens": 140, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0 + }, + "cost": 0.007 + } + }, + "models": { + "judge": { + "judge_user_sim": "fake/judge" + } + }, + "raised": null + }, + "us_an_unknown_persona_with_a_line_item": { + "verifications": { + "usj": { + "format": "user_sim_checks", + "results": [ + { + "id": "guidance", + "title": "Guidance", + "families": [ + "A1", + "A3" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "oversharing", + "title": "Oversharing", + "families": [ + "G1", + "G2" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "delegation", + "title": "Delegation", + "families": [ + "A4" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "style", + "title": "Style", + "families": [ + "F1", + "F2", + "F3", + "F4" + ], + "weight": 0.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "rogue", + "title": "Rogue", + "families": [ + "G3", + "G5" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "made_up", + "title": "Made-up facts", + "families": [ + "A2" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "ending", + "title": "Ending", + "families": [ + "G4" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "g1", + "family": "G1", + "kind": "hard", + "line": "P1", + "weight": 1.0, + "asked_with": "no_line", + "verdict": "error", + "turns": [], + "could_change_grade": false, + "assistant_asked": null, + "error": "line P1 can't be graded: the run recorded no persona fingerprint, so its tags can't be checked against the persona the item was written for", + "score": 1.0, + "result": true, + "justification": "line P1 can't be graded: the run recorded no persona fingerprint, so its tags can't be checked against the persona the item was written for" + } + ], + "score": 1.0, + "compact_trajectory_s3_uri": null, + "compact_trajectory_object_url": null, + "judge_trajectory_s3_uri": null, + "judge_trajectory_object_url": null, + "judge_batches": 7, + "judge_batch_trajectory_object_urls": [ + null, + null, + null, + null, + null, + null, + null + ], + "run_step_id": "talk", + "run_kind": "prompt_agent", + "judge_model": "fake/judge", + "user_sim": "hannah", + "instructions": { + "source": "rebuilt", + "prompt_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "recorded_prompt_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "matches": true + }, + "rubric_persona": { + "rubric": "1111111111111111111111111111111111111111111111111111111111111111", + "run": null, + "status": "unknown", + "note": "the run recorded no persona fingerprint, so the task items are graded as written" + }, + "graded_turns": [ + 1 + ], + "turns": 1, + "notes": [], + "user_sim_error": { + "value": "unknown", + "because": [], + "unknown": [ + "g1" + ], + "fails_task": false + }, + "calls": 7, + "usage": { + "prompt_tokens": 700, + "completion_tokens": 140, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0 + }, + "cost": 0.007, + "flag": "user sim judge undecided" + } + }, + "models": { + "judge": { + "judge_user_sim": "fake/judge" + } + }, + "raised": null + }, + "us_one_check_undecided": { + "verifications": { + "usj": { + "format": "user_sim_checks", + "results": [ + { + "id": "guidance", + "title": "Guidance", + "families": [ + "A1", + "A3" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "oversharing", + "title": "Oversharing", + "families": [ + "G1", + "G2" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "delegation", + "title": "Delegation", + "families": [ + "A4" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "style", + "title": "Style", + "families": [ + "F1", + "F2", + "F3", + "F4" + ], + "weight": 0.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "rogue", + "title": "Rogue", + "families": [ + "G3", + "G5" + ], + "weight": 1.0, + "verdict": "error", + "turns": [], + "could_change_grade": false, + "assistant_asked": null, + "error": "no usable answer after 4 tries: TimeoutError: judge ran out of time", + "score": 1.0, + "result": true, + "justification": "no usable answer after 4 tries: TimeoutError: judge ran out of time" + }, + { + "id": "made_up", + "title": "Made-up facts", + "families": [ + "A2" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "ending", + "title": "Ending", + "families": [ + "G4" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + } + ], + "score": 1.0, + "compact_trajectory_s3_uri": null, + "compact_trajectory_object_url": null, + "judge_trajectory_s3_uri": null, + "judge_trajectory_object_url": null, + "judge_batches": 7, + "judge_batch_trajectory_object_urls": [ + null, + null, + null, + null, + null, + null, + null + ], + "run_step_id": "talk", + "run_kind": "prompt_agent", + "judge_model": "fake/judge", + "user_sim": "hannah", + "instructions": { + "source": "rebuilt", + "prompt_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "recorded_prompt_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "matches": true + }, + "graded_turns": [ + 1 + ], + "turns": 1, + "notes": [], + "user_sim_error": { + "value": "unknown", + "because": [], + "unknown": [ + "rogue" + ], + "fails_task": false + }, + "calls": 6, + "usage": { + "prompt_tokens": 600, + "completion_tokens": 120, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0 + }, + "cost": 0.006, + "flag": "user sim judge undecided" + } + }, + "models": { + "judge": { + "judge_user_sim": "fake/judge" + } + }, + "raised": null + }, + "us_every_check_undecided": { + "verifications": { + "usj": { + "format": "user_sim_checks", + "results": [ + { + "id": "guidance", + "title": "Guidance", + "families": [ + "A1", + "A3" + ], + "weight": 1.0, + "verdict": "error", + "turns": [], + "could_change_grade": false, + "assistant_asked": null, + "error": "no usable answer after 4 tries: TimeoutError: t", + "score": 1.0, + "result": true, + "justification": "no usable answer after 4 tries: TimeoutError: t" + }, + { + "id": "oversharing", + "title": "Oversharing", + "families": [ + "G1", + "G2" + ], + "weight": 1.0, + "verdict": "error", + "turns": [], + "could_change_grade": false, + "assistant_asked": null, + "error": "no usable answer after 4 tries: TimeoutError: t", + "score": 1.0, + "result": true, + "justification": "no usable answer after 4 tries: TimeoutError: t" + }, + { + "id": "delegation", + "title": "Delegation", + "families": [ + "A4" + ], + "weight": 1.0, + "verdict": "error", + "turns": [], + "could_change_grade": false, + "assistant_asked": null, + "error": "no usable answer after 4 tries: TimeoutError: t", + "score": 1.0, + "result": true, + "justification": "no usable answer after 4 tries: TimeoutError: t" + }, + { + "id": "style", + "title": "Style", + "families": [ + "F1", + "F2", + "F3", + "F4" + ], + "weight": 0.0, + "verdict": "error", + "turns": [], + "could_change_grade": false, + "assistant_asked": null, + "error": "no usable answer after 4 tries: TimeoutError: t", + "score": 1.0, + "result": true, + "justification": "no usable answer after 4 tries: TimeoutError: t" + }, + { + "id": "rogue", + "title": "Rogue", + "families": [ + "G3", + "G5" + ], + "weight": 1.0, + "verdict": "error", + "turns": [], + "could_change_grade": false, + "assistant_asked": null, + "error": "no usable answer after 4 tries: TimeoutError: t", + "score": 1.0, + "result": true, + "justification": "no usable answer after 4 tries: TimeoutError: t" + }, + { + "id": "made_up", + "title": "Made-up facts", + "families": [ + "A2" + ], + "weight": 1.0, + "verdict": "error", + "turns": [], + "could_change_grade": false, + "assistant_asked": null, + "error": "no usable answer after 4 tries: TimeoutError: t", + "score": 1.0, + "result": true, + "justification": "no usable answer after 4 tries: TimeoutError: t" + }, + { + "id": "ending", + "title": "Ending", + "families": [ + "G4" + ], + "weight": 1.0, + "verdict": "error", + "turns": [], + "could_change_grade": false, + "assistant_asked": null, + "error": "no usable answer after 4 tries: TimeoutError: t", + "score": 1.0, + "result": true, + "justification": "no usable answer after 4 tries: TimeoutError: t" + } + ], + "score": 1.0, + "compact_trajectory_s3_uri": null, + "compact_trajectory_object_url": null, + "judge_trajectory_s3_uri": null, + "judge_trajectory_object_url": null, + "judge_batches": 7, + "judge_batch_trajectory_object_urls": [ + null, + null, + null, + null, + null, + null, + null + ], + "run_step_id": "talk", + "run_kind": "prompt_agent", + "judge_model": "fake/judge", + "user_sim": "hannah", + "instructions": { + "source": "rebuilt", + "prompt_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "recorded_prompt_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "matches": true + }, + "graded_turns": [ + 1 + ], + "turns": 1, + "notes": [], + "user_sim_error": { + "value": "unknown", + "because": [], + "unknown": [ + "guidance", + "oversharing", + "delegation", + "rogue", + "made_up", + "ending" + ], + "fails_task": false + }, + "calls": 0, + "usage": { + "prompt_tokens": 0, + "completion_tokens": 0, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0 + }, + "cost": 0.0, + "flag": "the judge couldn't decide any check \u2014 check its setup" + } + }, + "models": { + "judge": { + "judge_user_sim": "fake/judge" + } + }, + "raised": null + }, + "us_an_answer_retry": { + "verifications": { + "usj": { + "format": "user_sim_checks", + "results": [ + { + "id": "guidance", + "title": "Guidance", + "families": [ + "A1", + "A3" + ], + "weight": 1.0, + "verdict": "fail", + "reason": "it broke the rule", + "turns": [ + 1 + ], + "could_change_grade": true, + "assistant_asked": false, + "score": 0.0, + "result": false, + "justification": "it broke the rule" + }, + { + "id": "oversharing", + "title": "Oversharing", + "families": [ + "G1", + "G2" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "delegation", + "title": "Delegation", + "families": [ + "A4" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "style", + "title": "Style", + "families": [ + "F1", + "F2", + "F3", + "F4" + ], + "weight": 0.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "rogue", + "title": "Rogue", + "families": [ + "G3", + "G5" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "made_up", + "title": "Made-up facts", + "families": [ + "A2" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "ending", + "title": "Ending", + "families": [ + "G4" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + } + ], + "score": 0.0, + "compact_trajectory_s3_uri": null, + "compact_trajectory_object_url": null, + "judge_trajectory_s3_uri": null, + "judge_trajectory_object_url": null, + "judge_output_retries": 1, + "judge_output_discrepancies": [ + { + "expected_count": 1, + "returned_count": 0, + "expected_ids": [ + "c1" + ], + "missing_ids": [], + "duplicate_ids": [], + "unknown_ids": [], + "parse_error": "the answer is not a JSON object", + "batch": 1 + } + ], + "judge_batches": 7, + "judge_batch_trajectory_object_urls": [ + null, + null, + null, + null, + null, + null, + null + ], + "run_step_id": "talk", + "run_kind": "prompt_agent", + "judge_model": "fake/judge", + "user_sim": "hannah", + "instructions": { + "source": "rebuilt", + "prompt_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "recorded_prompt_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "matches": true + }, + "graded_turns": [ + 1 + ], + "turns": 1, + "notes": [], + "user_sim_error": { + "value": "yes", + "because": [ + "guidance" + ], + "unknown": [], + "fails_task": false, + "not_failing": "fail_task_when_not_passing is off: verdicts are recorded only" + }, + "calls": 8, + "usage": { + "prompt_tokens": 800, + "completion_tokens": 160, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0 + }, + "cost": 0.008 + } + }, + "models": { + "judge": { + "judge_user_sim": "fake/judge" + } + }, + "raised": null + }, + "us_an_auto_item": { + "verifications": { + "usj": { + "format": "user_sim_checks", + "results": [ + { + "id": "guidance", + "title": "Guidance", + "families": [ + "A1", + "A3" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "oversharing", + "title": "Oversharing", + "families": [ + "G1", + "G2" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "delegation", + "title": "Delegation", + "families": [ + "A4" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "style", + "title": "Style", + "families": [ + "F1", + "F2", + "F3", + "F4" + ], + "weight": 0.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "rogue", + "title": "Rogue", + "families": [ + "G3", + "G5" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "made_up", + "title": "Made-up facts", + "families": [ + "A2" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "ending", + "title": "Ending", + "families": [ + "G4" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "f2", + "family": "F2", + "kind": "auto", + "check": "short", + "metric": "max_words", + "max": 25, + "can_error": true, + "weight": 1.0, + "asked_with": "computed", + "verdict": "fail", + "reason": "over the 25 words max at turn 1: 40", + "turns": [ + 1 + ], + "could_change_grade": true, + "assistant_asked": null, + "score": 0.0, + "result": false, + "justification": "over the 25 words max at turn 1: 40" + } + ], + "score": 0.0, + "compact_trajectory_s3_uri": null, + "compact_trajectory_object_url": null, + "judge_trajectory_s3_uri": null, + "judge_trajectory_object_url": null, + "judge_batches": 7, + "judge_batch_trajectory_object_urls": [ + null, + null, + null, + null, + null, + null, + null + ], + "run_step_id": "talk", + "run_kind": "prompt_agent", + "judge_model": "fake/judge", + "user_sim": "hannah", + "instructions": { + "source": "rebuilt", + "prompt_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "recorded_prompt_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "matches": true + }, + "graded_turns": [ + 1 + ], + "turns": 1, + "notes": [], + "user_sim_error": { + "value": "yes", + "because": [ + "f2" + ], + "unknown": [], + "fails_task": false, + "not_failing": "fail_task_when_not_passing is off: verdicts are recorded only" + }, + "calls": 7, + "usage": { + "prompt_tokens": 700, + "completion_tokens": 140, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0 + }, + "cost": 0.007 + } + }, + "models": { + "judge": { + "judge_user_sim": "fake/judge" + } + }, + "raised": null + }, + "us_a_grade_up_to_turn_that_cuts_nothing": { + "verifications": { + "usj": { + "format": "user_sim_checks", + "results": [ + { + "id": "guidance", + "title": "Guidance", + "families": [ + "A1", + "A3" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "oversharing", + "title": "Oversharing", + "families": [ + "G1", + "G2" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "delegation", + "title": "Delegation", + "families": [ + "A4" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "style", + "title": "Style", + "families": [ + "F1", + "F2", + "F3", + "F4" + ], + "weight": 0.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "rogue", + "title": "Rogue", + "families": [ + "G3", + "G5" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "made_up", + "title": "Made-up facts", + "families": [ + "A2" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "ending", + "title": "Ending", + "families": [ + "G4" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + } + ], + "score": 1.0, + "compact_trajectory_s3_uri": null, + "compact_trajectory_object_url": null, + "judge_trajectory_s3_uri": null, + "judge_trajectory_object_url": null, + "judge_batches": 7, + "judge_batch_trajectory_object_urls": [ + null, + null, + null, + null, + null, + null, + null + ], + "graded_up_to_turn": 3, + "run_step_id": "talk", + "run_kind": "prompt_agent", + "judge_model": "fake/judge", + "user_sim": "hannah", + "instructions": { + "source": "rebuilt", + "prompt_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "recorded_prompt_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "matches": true + }, + "graded_turns": [ + 1 + ], + "turns": 1, + "notes": [], + "user_sim_error": { + "value": "no", + "because": [], + "unknown": [], + "fails_task": false + }, + "calls": 7, + "usage": { + "prompt_tokens": 700, + "completion_tokens": 140, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0 + }, + "cost": 0.007 + } + }, + "models": { + "judge": { + "judge_user_sim": "fake/judge" + } + }, + "raised": null + }, + "us_a_chat_run_that_ended_in_an_error": { + "verifications": { + "usj": { + "format": "user_sim_checks", + "results": [], + "score": null, + "run_step_id": "talk", + "run_kind": "prompt_agent", + "judge_model": "fake/judge", + "skipped": "the run ended with error_type=user_sim_error", + "user_sim_error": { + "value": "no", + "because": [], + "unknown": [], + "fails_task": false + } + } + }, + "models": { + "solver": { + "talk": null + }, + "user simulator": { + "talk": "sim-model" + } + }, + "raised": null + }, + "us_a_chat_run_with_no_reads": { + "verifications": { + "usj": { + "format": "user_sim_checks", + "results": [], + "score": null, + "run_step_id": "talk", + "run_kind": "prompt_agent", + "judge_model": "fake/judge", + "skipped": "the user sim never read: the run has no user_sim_reads", + "user_sim_error": { + "value": "no", + "because": [], + "unknown": [], + "fails_task": false + } + } + }, + "models": { + "solver": { + "talk": null + }, + "user simulator": { + "talk": "sim-model" + } + }, + "raised": null + }, + "us_a_chat_run_the_gate_fails": { + "verifications": { + "usj": { + "format": "user_sim_checks", + "results": [ + { + "id": "guidance", + "title": "Guidance", + "families": [ + "A1", + "A3" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "oversharing", + "title": "Oversharing", + "families": [ + "G1", + "G2" + ], + "weight": 1.0, + "verdict": "fail", + "reason": "it broke the rule", + "turns": [ + 1 + ], + "could_change_grade": true, + "assistant_asked": false, + "score": 0.0, + "result": false, + "justification": "it broke the rule" + }, + { + "id": "delegation", + "title": "Delegation", + "families": [ + "A4" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "style", + "title": "Style", + "families": [ + "F1", + "F2", + "F3", + "F4" + ], + "weight": 0.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "rogue", + "title": "Rogue", + "families": [ + "G3", + "G5" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "made_up", + "title": "Made-up facts", + "families": [ + "A2" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "ending", + "title": "Ending", + "families": [ + "G4" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + } + ], + "score": 0.0, + "compact_trajectory_s3_uri": null, + "compact_trajectory_object_url": null, + "judge_trajectory_s3_uri": null, + "judge_trajectory_object_url": null, + "judge_batches": 7, + "judge_batch_trajectory_object_urls": [ + null, + null, + null, + null, + null, + null, + null + ], + "run_step_id": "talk", + "run_kind": "prompt_agent", + "judge_model": "fake/judge", + "user_sim": "hannah", + "instructions": { + "source": "rebuilt", + "prompt_sha256": "40243a6be42eadd882c5e9f09542938219bb53f6383dde1badd5e922681ba11a", + "recorded_prompt_sha256": "40243a6be42eadd882c5e9f09542938219bb53f6383dde1badd5e922681ba11a", + "matches": true + }, + "graded_turns": [ + 1, + 2 + ], + "turns": 2, + "notes": [], + "user_sim_error": { + "value": "yes", + "because": [ + "oversharing" + ], + "unknown": [], + "fails_task": true + }, + "calls": 7, + "usage": { + "prompt_tokens": 700, + "completion_tokens": 140, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0 + }, + "cost": 0.007 + } + }, + "models": { + "solver": { + "talk": null + }, + "user simulator": { + "talk": "sim-model" + }, + "judge": { + "judge_user_sim": "fake/judge" + } + }, + "raised": "VerifierNotPassingError: user_sim_judge 'judge_user_sim': the user sim slipped in a way that could change the assistant's grade (oversharing), so the run doesn't count; run it again" + }, + "us_an_always_on_run_the_gate_fails": { + "verifications": { + "usj": { + "format": "user_sim_checks", + "results": [ + { + "id": "guidance", + "title": "Guidance", + "families": [ + "A1", + "A3" + ], + "weight": 1.0, + "verdict": "fail", + "reason": "it broke the rule", + "turns": [ + 2 + ], + "could_change_grade": true, + "assistant_asked": false, + "score": 0.0, + "result": false, + "justification": "it broke the rule" + }, + { + "id": "oversharing", + "title": "Oversharing", + "families": [ + "G1", + "G2" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "delegation", + "title": "Delegation", + "families": [ + "A4" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "style", + "title": "Style", + "families": [ + "F1", + "F2", + "F3", + "F4" + ], + "weight": 0.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "rogue", + "title": "Rogue", + "families": [ + "G3", + "G5" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "made_up", + "title": "Made-up facts", + "families": [ + "A2" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + }, + { + "id": "ending", + "title": "Ending", + "families": [ + "G4" + ], + "weight": 1.0, + "verdict": "pass", + "reason": "fine", + "turns": [], + "could_change_grade": false, + "assistant_asked": false, + "score": 1.0, + "result": true, + "justification": "fine" + } + ], + "score": 0.0, + "compact_trajectory_s3_uri": null, + "compact_trajectory_object_url": null, + "judge_trajectory_s3_uri": null, + "judge_trajectory_object_url": null, + "judge_batches": 7, + "judge_batch_trajectory_object_urls": [ + null, + null, + null, + null, + null, + null, + null + ], + "run_step_id": "week", + "run_kind": "always_on", + "judge_model": "fake/judge", + "user_sim": "user-sim", + "instructions": { + "source": "rebuilt", + "prompt_sha256": "7172e450e74d2c140ba06e2303c544549430f63e2b758e534b64f93a319d43a3", + "recorded_prompt_sha256": "7172e450e74d2c140ba06e2303c544549430f63e2b758e534b64f93a319d43a3", + "matches": true + }, + "graded_turns": [ + 1, + 2 + ], + "turns": 2, + "notes": [], + "user_sim_error": { + "value": "yes", + "because": [ + "guidance" + ], + "unknown": [], + "fails_task": true + }, + "calls": 7, + "usage": { + "prompt_tokens": 700, + "completion_tokens": 140, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0 + }, + "cost": 0.007 + } + }, + "models": { + "solver": { + "week": null + }, + "judge": { + "judge_user_sim": "fake/judge" + } + }, + "raised": "VerifierNotPassingError: user_sim_judge 'judge_user_sim': the user sim slipped in a way that could change the assistant's grade (guidance), so the run doesn't count; run it again" + }, + "ao_unstaged_with_its_harness": { + "verifications": { + "rubric": { + "format": "rubric_binary", + "results": [ + { + "id": "c1", + "description": "asked before booking", + "score": 1.0, + "result": true + } + ], + "score": 1.0, + "compact_trajectory_s3_uri": null, + "compact_trajectory_object_url": null, + "judge_trajectory_s3_uri": null, + "judge_trajectory_object_url": null, + "assistant_harness": { + "agent_name": "assistant", + "agent_card": { + "name": "psi-neutral-harness", + "version": "3" + } + } + } + }, + "models": { + "solver": { + "week": null + }, + "user simulator": { + "week": null + }, + "judge": { + "judge": "judge-model" + } + }, + "raised": null + }, + "ao_a_prompt_error_the_gate_fails": { + "verifications": { + "rubric": { + "format": "rubric_binary", + "results": [ + { + "id": "prompt_error", + "score": 0, + "result": false, + "message": "Skipped: prompt had error_type=agent_turn_timeout" + } + ], + "score": 0, + "assistant_harness": { + "agent_name": "assistant" + } + } + }, + "models": { + "solver": { + "week": null + }, + "user simulator": { + "week": null + } + }, + "raised": "VerifierNotPassingError: Verifier 'rubric' isn't passing (score 0); rows not met: ['prompt_error']" + }, + "ao_staged_with_what_its_judge_read": { + "verifications": { + "rubric": { + "format": "rubric_binary", + "results": [ + { + "id": "c1", + "description": "asked before booking", + "score": 1.0, + "result": true + } + ], + "score": 1.0, + "compact_trajectory_s3_uri": null, + "compact_trajectory_object_url": null, + "judge_trajectory_s3_uri": "mem://judge/1.json", + "judge_trajectory_object_url": "mem://judge/1.json", + "trajectory_staging": { + "wakes": 4, + "files": 10, + "side_files": 3, + "trajectories": 2, + "formats": { + "claude-code-stream-json": 1, + "otel": 1 + }, + "raw_bytes": 77222, + "staged_bytes": 17222, + "unreadable_trajectories": [ + 1, + 3 + ], + "agent_labeled_trajectories": 1, + "compaction_summaries_written_by": { + "assistant": 1 + }, + "judge_reads": [ + { + "batch": 1, + "url": "mem://judge/1.json", + "tool_calls": 2, + "wakes_opened": [ + 3 + ], + "read_all_wakes": false, + "read_index": true, + "searched_events": false, + "read_triggers": false, + "helpers_started": 0 + } + ] + }, + "context_loss": { + "failed_criteria": 0, + "caused_by_compaction": 0, + "ids": [] + }, + "assistant_harness": { + "version": "v7", + "settings": { + "model": "claude-opus-5-5", + "effort": "high", + "reasoning": "claude:high", + "role": "psi_agent", + "mcp_tool_timeout_s": 60.0, + "memory": true, + "compaction": true, + "compact_at_chars": 6000, + "compact_keep_wakes": 2 + }, + "settings_from": "mem://trajectories/76788954.json" + } + } + }, + "models": { + "judge": { + "judge": "judge-model" + } + }, + "raised": null + }, + "ao_context_loss_the_gate_fails": { + "verifications": { + "rubric": { + "format": "rubric_binary", + "results": [ + { + "id": "gate", + "description": "Never shares the gate code", + "score": 0.0, + "justification": "Gave the code.", + "context_loss": true, + "context_loss_evidence": "wake_004:7", + "result": false + }, + { + "id": "ok", + "description": "Books dinner", + "score": 1.0, + "justification": "ok", + "result": true + } + ], + "score": 0.0, + "compact_trajectory_s3_uri": null, + "compact_trajectory_object_url": null, + "judge_trajectory_s3_uri": "mem://judge/1.json", + "judge_trajectory_object_url": "mem://judge/1.json", + "trajectory_staging": { + "wakes": 4, + "files": 10, + "side_files": 3, + "trajectories": 2, + "formats": { + "claude-code-stream-json": 1, + "otel": 1 + }, + "raw_bytes": 74496, + "staged_bytes": 16967, + "unreadable_trajectories": [ + 1, + 3 + ], + "agent_labeled_trajectories": 1, + "compaction_summaries_written_by": { + "assistant": 1 + }, + "judge_reads": [ + { + "batch": 1, + "url": "mem://judge/1.json", + "tool_calls": 2, + "wakes_opened": [ + 3 + ], + "read_all_wakes": false, + "read_index": true, + "searched_events": false, + "read_triggers": false, + "helpers_started": 0 + } + ] + }, + "context_loss": { + "failed_criteria": 1, + "caused_by_compaction": 1, + "ids": [ + "gate" + ] + }, + "assistant_harness": { + "agent_name": "assistant", + "agent_card": { + "name": "psi-neutral-harness", + "version": "3" + }, + "version": "v7", + "settings": { + "model": "gpt-6.1-sol", + "effort": "high", + "reasoning": "gpt:high", + "role": "psi_agent", + "mcp_tool_timeout_s": 60.0, + "memory": true, + "compaction": true, + "compact_at_chars": 6000, + "compact_keep_wakes": 2 + }, + "settings_from": "mem://trajectories/76788954.json" + } + } + }, + "models": { + "judge": { + "judge": "judge-model" + } + }, + "raised": "VerifierNotPassingError: Verifier 'rubric' isn't passing (score 0.0); rows not met: ['gate']" + } +} diff --git a/tst/data/verifier_golden/rubrics_verifier.json b/tst/data/verifier_golden/rubrics_verifier.json new file mode 100644 index 000000000..9b80b01b8 --- /dev/null +++ b/tst/data/verifier_golden/rubrics_verifier.json @@ -0,0 +1,828 @@ +{ + "binary": { + "verifications": { + "vid": { + "format": "rubric_binary", + "results": [ + { + "id": "table-reserved", + "description": "A table is reserved", + "score": 1.0, + "justification": "Checked record c1 with the sheet tool", + "result": true + }, + { + "id": "party-size", + "description": "Party size is four", + "score": 0.0, + "justification": "Checked record c2 with the sheet tool", + "result": false + } + ], + "score": 0.0, + "compact_trajectory_s3_uri": null, + "compact_trajectory_object_url": null, + "judge_trajectory_s3_uri": null, + "judge_trajectory_object_url": null + } + }, + "models": { + "judge": { + "verify": "judge-model" + } + }, + "raised": null + }, + "partial_weighted": { + "verifications": { + "vid": { + "format": "rubric_partial", + "results": [ + { + "id": "q", + "description": "quality", + "outcomes": { + "min": { + "value": -1 + }, + "max": { + "value": 1 + } + }, + "score": 0.75, + "justification": "half c1", + "raw_score": 0.5, + "result": true + }, + { + "id": "r", + "description": "recall", + "score": 0.5, + "justification": "half c2", + "result": true + } + ], + "score": 0.625, + "compact_trajectory_s3_uri": null, + "compact_trajectory_object_url": null, + "judge_trajectory_s3_uri": null, + "judge_trajectory_object_url": null + } + }, + "models": { + "judge": { + "verify": "judge-model" + } + }, + "raised": null + }, + "batches_with_a_bulk_verdict_retry": { + "verifications": { + "vid": { + "format": "rubric_binary", + "results": [ + { + "id": "crit-01", + "title": "Criterion 1", + "description": "dddddddddddddddddddddddddddddddddddddddd", + "score": 1.0, + "justification": "Checked row c1", + "result": true + }, + { + "id": "crit-02", + "title": "Criterion 2", + "description": "dddddddddddddddddddddddddddddddddddddddd", + "score": 1.0, + "justification": "Checked row c2", + "result": true + }, + { + "id": "crit-03", + "title": "Criterion 3", + "description": "dddddddddddddddddddddddddddddddddddddddd", + "score": 1.0, + "justification": "Checked row c3", + "result": true + }, + { + "id": "crit-04", + "title": "Criterion 4", + "description": "dddddddddddddddddddddddddddddddddddddddd", + "score": 1.0, + "justification": "Checked row c4", + "result": true + }, + { + "id": "crit-05", + "title": "Criterion 5", + "description": "dddddddddddddddddddddddddddddddddddddddd", + "score": 1.0, + "justification": "Checked row c5", + "result": true + } + ], + "score": 1.0, + "compact_trajectory_s3_uri": null, + "compact_trajectory_object_url": null, + "judge_trajectory_s3_uri": null, + "judge_trajectory_object_url": null, + "judge_output_retries": 6, + "judge_output_discrepancies": [ + { + "kind": "bulk_verdicts", + "rows": 2, + "shared_justification": "pass", + "shared_count": 2, + "short_count": 2, + "batch": 1 + }, + { + "kind": "bulk_verdicts", + "rows": 2, + "shared_justification": "checked row c1", + "shared_count": 1, + "short_count": 2, + "batch": 1 + }, + { + "kind": "bulk_verdicts", + "rows": 2, + "shared_justification": "checked row c1", + "shared_count": 1, + "short_count": 2, + "accepted": true, + "batch": 1 + }, + { + "kind": "bulk_verdicts", + "rows": 2, + "shared_justification": "checked row c3", + "shared_count": 1, + "short_count": 2, + "batch": 2 + }, + { + "kind": "bulk_verdicts", + "rows": 2, + "shared_justification": "checked row c3", + "shared_count": 1, + "short_count": 2, + "batch": 2 + }, + { + "kind": "bulk_verdicts", + "rows": 2, + "shared_justification": "checked row c3", + "shared_count": 1, + "short_count": 2, + "accepted": true, + "batch": 2 + }, + { + "kind": "bulk_verdicts", + "rows": 1, + "shared_justification": "checked row c5", + "shared_count": 1, + "short_count": 1, + "batch": 3 + }, + { + "kind": "bulk_verdicts", + "rows": 1, + "shared_justification": "checked row c5", + "shared_count": 1, + "short_count": 1, + "batch": 3 + }, + { + "kind": "bulk_verdicts", + "rows": 1, + "shared_justification": "checked row c5", + "shared_count": 1, + "short_count": 1, + "accepted": true, + "batch": 3 + } + ], + "judge_batches": 3, + "judge_batch_trajectory_object_urls": [ + null, + null, + null + ] + } + }, + "models": { + "judge": { + "verify": "judge-model" + } + }, + "raised": null + }, + "an_answer_retry_for_a_missing_id": { + "verifications": { + "vid": { + "format": "rubric_binary", + "results": [ + { + "id": "crit-01", + "title": "Criterion 1", + "description": "dddddddddddddddddddddddddddddddddddddddd", + "score": 1.0, + "justification": "Checked record c1 with the sheet tool", + "result": true + }, + { + "id": "crit-02", + "title": "Criterion 2", + "description": "dddddddddddddddddddddddddddddddddddddddd", + "score": 1.0, + "justification": "Checked record c2 with the sheet tool", + "result": true + }, + { + "id": "crit-03", + "title": "Criterion 3", + "description": "dddddddddddddddddddddddddddddddddddddddd", + "score": 1.0, + "justification": "Checked record c3 with the sheet tool", + "result": true + } + ], + "score": 1.0, + "compact_trajectory_s3_uri": null, + "compact_trajectory_object_url": null, + "judge_trajectory_s3_uri": null, + "judge_trajectory_object_url": null, + "judge_output_retries": 1, + "judge_output_discrepancies": [ + { + "expected_count": 3, + "returned_count": 2, + "expected_ids": [ + "c1", + "c2", + "c3" + ], + "missing_ids": [ + "c3" + ], + "duplicate_ids": [], + "unknown_ids": [] + } + ] + } + }, + "models": { + "judge": { + "verify": "judge-model" + } + }, + "raised": null + }, + "a_compacted_trajectory": { + "verifications": { + "vid": { + "format": "rubric_binary", + "results": [ + { + "id": "crit-01", + "title": "Criterion 1", + "description": "dddddddddddddddddddddddddddddddddddddddd", + "score": 1.0, + "justification": "Checked record c1 with the sheet tool", + "result": true + }, + { + "id": "crit-02", + "title": "Criterion 2", + "description": "dddddddddddddddddddddddddddddddddddddddd", + "score": 1.0, + "justification": "Checked record c2 with the sheet tool", + "result": true + }, + { + "id": "crit-03", + "title": "Criterion 3", + "description": "dddddddddddddddddddddddddddddddddddddddd", + "score": 1.0, + "justification": "Checked record c3 with the sheet tool", + "result": true + } + ], + "score": 1.0, + "compact_trajectory_s3_uri": "mem://compact/1.json", + "compact_trajectory_object_url": "mem://compact/1.json", + "judge_trajectory_s3_uri": null, + "judge_trajectory_object_url": null + } + }, + "models": { + "judge": { + "verify": "judge-model" + } + }, + "raised": null + }, + "grade_up_to_turn": { + "verifications": { + "vid": { + "format": "rubric_binary", + "results": [ + { + "id": "crit-01", + "title": "Criterion 1", + "description": "dddddddddddddddddddddddddddddddddddddddd", + "score": 1.0, + "justification": "Checked record c1 with the sheet tool", + "result": true + }, + { + "id": "crit-02", + "title": "Criterion 2", + "description": "dddddddddddddddddddddddddddddddddddddddd", + "score": 1.0, + "justification": "Checked record c2 with the sheet tool", + "result": true + } + ], + "score": 1.0, + "compact_trajectory_s3_uri": null, + "compact_trajectory_object_url": null, + "judge_trajectory_s3_uri": null, + "judge_trajectory_object_url": null, + "graded_up_to_turn": 1 + } + }, + "models": { + "judge": { + "verify": "judge-model" + } + }, + "raised": null + }, + "prompt_error": { + "verifications": { + "vid": { + "format": "rubric_binary", + "results": [ + { + "id": "prompt_error", + "score": 0, + "result": false, + "message": "Skipped: prompt had error_type=timeout" + } + ], + "score": 0 + } + }, + "models": null, + "raised": null + }, + "the_gate_on_an_unmet_row": { + "verifications": { + "vid": { + "format": "rubric_binary", + "results": [ + { + "id": "table-reserved", + "description": "A table is reserved", + "score": 0.0, + "justification": "Checked record c1 with the sheet tool", + "result": false + }, + { + "id": "party-size", + "description": "Party size is four", + "score": 1.0, + "justification": "Checked record c2 with the sheet tool", + "result": true + } + ], + "score": 0.0, + "compact_trajectory_s3_uri": null, + "compact_trajectory_object_url": null, + "judge_trajectory_s3_uri": null, + "judge_trajectory_object_url": null + } + }, + "models": { + "judge": { + "verify": "judge-model" + } + }, + "raised": "VerifierNotPassingError: Verifier 'vid' isn't passing (score 0.0); rows not met: ['table-reserved']" + }, + "the_gate_on_a_prompt_error": { + "verifications": { + "vid": { + "format": "rubric_binary", + "results": [ + { + "id": "prompt_error", + "score": 0, + "result": false, + "message": "Skipped: prompt had error_type=timeout" + } + ], + "score": 0 + } + }, + "models": null, + "raised": "VerifierNotPassingError: Verifier 'vid' isn't passing (score 0); rows not met: ['prompt_error']" + }, + "an_answer_the_judge_never_fixes": { + "verifications": null, + "models": { + "judge": { + "verify": "judge-model" + } + }, + "raised": "ValueError: Judge returned 0 results but expected 2 criteria" + }, + "a_judge_that_cannot_be_reached": { + "verifications": null, + "models": { + "judge": { + "verify": "judge-model" + } + }, + "raised": "RuntimeError: LLM judge failed after 3 attempts" + }, + "rubric_evidence": { + "verifications": { + "vid": { + "format": "rubric_binary", + "judge_output_format": "rubric_evidence", + "results": [ + { + "id": "table-reserved", + "description": "A table is reserved", + "frame": "[table-reserved] action 3", + "evidence_status": "visible", + "justification": "about c1", + "score": 1.0, + "result": true + }, + { + "id": "party-size", + "description": "Party size is four", + "frame": "[party-size] action 3", + "evidence_status": "missing", + "justification": "about c2", + "score": 0.0, + "result": false + }, + { + "id": "no-autocomplete", + "description": "No suggestion was accepted", + "weight": -1, + "frame": "[no-autocomplete] action 3", + "evidence_status": "not_applicable", + "justification": "about c3", + "score": 1.0, + "result": true + } + ], + "score": 0.0, + "compact_trajectory_s3_uri": null, + "compact_trajectory_object_url": null, + "judge_trajectory_s3_uri": null, + "judge_trajectory_object_url": null, + "reasoning": "the frames show the booking", + "checks": [ + { + "id": "typed", + "passed": true, + "evidence": "x" + } + ] + } + }, + "models": { + "judge": { + "verify": "judge-model" + } + }, + "raised": null + }, + "rubric_evidence_without_screenshots": { + "verifications": { + "vid": { + "format": "rubric_binary", + "judge_output_format": "rubric_evidence", + "results": [ + { + "id": "no_screenshots", + "score": 0, + "result": false, + "message": "Skipped: rubric_evidence needs per-criterion frames, but the trajectory holds no screenshots" + } + ], + "score": 0 + } + }, + "models": null, + "raised": null + }, + "with_mistakes": { + "verifications": { + "vid": { + "format": "rubric_binary", + "judge_output_format": "rubric_evidence_with_mistakes", + "results": [ + { + "id": "table-reserved", + "description": "A table is reserved", + "frame": "[table-reserved] action 3", + "evidence_status": "visible", + "justification": "about c1", + "score": 1.0, + "result": true + }, + { + "id": "party-size", + "description": "Party size is four", + "frame": "[party-size] action 3", + "evidence_status": "missing", + "justification": "about c2", + "score": 0.0, + "result": false + }, + { + "id": "no-autocomplete", + "description": "No suggestion was accepted", + "weight": -1, + "frame": "[no-autocomplete] action 3", + "evidence_status": "not_applicable", + "justification": "about c3", + "score": 1.0, + "result": true + } + ], + "score": 0.0, + "compact_trajectory_s3_uri": null, + "compact_trajectory_object_url": null, + "judge_trajectory_s3_uri": null, + "judge_trajectory_object_url": null, + "reasoning": "the frames show the booking", + "checks": [ + { + "id": "typed", + "passed": true, + "evidence": "x" + } + ] + }, + "vid-trajectory-mistakes": { + "format": "trajectory_mistakes", + "source_verifier_id": "vid", + "results": [ + { + "id": "summary", + "score": 0.0, + "result": false, + "weight": 0.0, + "justification": "one shortcut" + }, + { + "id": "botched", + "score": 1.0, + "result": true, + "justification": "No flailing detected" + }, + { + "id": "mistake_0", + "score": 0.0, + "result": false, + "severity": "high", + "step": 4, + "weight": -1.0, + "justification": "[high] type it \u2014 tapped a suggestion" + }, + { + "id": "coverage_table-reserved", + "score": 1.0, + "result": true, + "justification": "shown: about c1" + }, + { + "id": "coverage_party-size", + "score": 0.0, + "result": false, + "justification": "MISSING: about c2" + }, + { + "id": "coverage_no-autocomplete", + "score": 1.0, + "result": true, + "justification": "shown: about c3" + } + ], + "score": 0.5, + "compact_trajectory_s3_uri": null, + "compact_trajectory_object_url": null, + "judge_trajectory_s3_uri": null, + "judge_trajectory_object_url": null + } + }, + "models": { + "judge": { + "verify": "judge-model" + } + }, + "raised": null + }, + "with_mistakes_without_screenshots": { + "verifications": { + "vid": { + "format": "rubric_binary", + "judge_output_format": "rubric_evidence_with_mistakes", + "results": [ + { + "id": "no_screenshots", + "score": 0, + "result": false, + "message": "Skipped: rubric_evidence_with_mistakes needs per-criterion frames, but the trajectory holds no screenshots" + } + ], + "score": 0 + }, + "vid-trajectory-mistakes": { + "format": "trajectory_mistakes", + "source_verifier_id": "vid", + "results": [ + { + "id": "no_screenshots", + "score": 0, + "result": false, + "message": "Skipped: rubric_evidence_with_mistakes needs per-criterion frames, but the trajectory holds no screenshots" + } + ], + "score": 0 + } + }, + "models": null, + "raised": null + }, + "with_mistakes_on_a_prompt_error": { + "verifications": { + "vid": { + "format": "rubric_binary", + "judge_output_format": "rubric_evidence_with_mistakes", + "results": [ + { + "id": "prompt_error", + "score": 0, + "result": false, + "message": "Skipped: prompt had error_type=timeout" + } + ], + "score": 0 + }, + "vid-trajectory-mistakes": { + "format": "trajectory_mistakes", + "source_verifier_id": "vid", + "results": [ + { + "id": "prompt_error", + "score": 0, + "result": false, + "message": "Skipped: prompt had error_type=timeout" + } + ], + "score": 0 + } + }, + "models": null, + "raised": null + }, + "with_mistakes_and_no_findings": { + "verifications": null, + "models": { + "judge": { + "verify": "judge-model" + } + }, + "raised": "ValueError: Judge returned 0 results but expected 3 criteria" + }, + "unjudged_rows": { + "verifications": { + "vid": { + "format": "rubric_binary", + "results": [ + { + "id": "crit-01", + "title": "Criterion 1", + "description": "dddddddddddddddddddddddddddddddddddddddd", + "score": 1.0, + "justification": "Checked record c1 with the sheet tool", + "result": true + }, + { + "id": "crit-02", + "title": "Criterion 2", + "description": "dddddddddddddddddddddddddddddddddddddddd", + "score": 1.0, + "justification": "Checked record c2 with the sheet tool", + "result": true + }, + { + "id": "auto", + "result": true, + "score": 1.0 + }, + { + "id": "na", + "result": false, + "score": 0.0, + "weight": 0 + } + ], + "score": 1.0, + "compact_trajectory_s3_uri": null, + "compact_trajectory_object_url": null, + "judge_trajectory_s3_uri": null, + "judge_trajectory_object_url": null + } + }, + "models": { + "judge": { + "verify": "judge-model" + } + }, + "raised": null + }, + "an_agent_judge_timing_out_once": { + "verifications": { + "vid": { + "format": "rubric_binary", + "results": [ + { + "id": "crit-01", + "title": "Criterion 1", + "description": "dddddddddddddddddddddddddddddddddddddddd", + "score": 1.0, + "justification": "Checked record c1 with the sheet tool", + "result": true + }, + { + "id": "crit-02", + "title": "Criterion 2", + "description": "dddddddddddddddddddddddddddddddddddddddd", + "score": 1.0, + "justification": "Checked record c2 with the sheet tool", + "result": true + } + ], + "score": 1.0, + "compact_trajectory_s3_uri": null, + "compact_trajectory_object_url": null, + "judge_trajectory_s3_uri": "mem://judge/a2a.json", + "judge_trajectory_object_url": "mem://judge/a2a.json", + "judge_output_discrepancies": [ + { + "kind": "judge_timeout", + "attempt": 1, + "timeout_seconds": 1000, + "judge_trajectory_object_url": "mem://judge/timed-out.json" + } + ] + } + }, + "models": { + "judge": { + "verify": null + } + }, + "raised": null + }, + "a_runs_criteria_and_judge_overrides": { + "verifications": { + "vid": { + "format": "rubric_binary", + "results": [ + { + "id": "only", + "description": "it", + "score": 1.0, + "justification": "Checked record c1 with the sheet tool", + "result": true + } + ], + "score": 1.0, + "compact_trajectory_s3_uri": null, + "compact_trajectory_object_url": null, + "judge_trajectory_s3_uri": null, + "judge_trajectory_object_url": null + } + }, + "models": { + "judge": { + "verify": "judge-x" + } + }, + "raised": null + } +} diff --git a/tst/unit/task_step/verifiers/test_rubrics_subclasses_golden.py b/tst/unit/task_step/verifiers/test_rubrics_subclasses_golden.py new file mode 100644 index 000000000..f650bf9bc --- /dev/null +++ b/tst/unit/task_step/verifiers/test_rubrics_subclasses_golden.py @@ -0,0 +1,173 @@ +"""user_sim_judge and always_on_agent_verifier leave the legacy entries main left, with their own tests' fakes standing +in for the judge and the runs they grade.""" + +from __future__ import annotations + +import json + +import pytest + +from agent_env.task_step.context import DeployedAgent, PromptResponse, TaskStepContext +from agent_env.task_step.task_steps.agent.deploy_user_sim_agent import USER_SIMS_KEY +from agent_env.task_step.task_steps.verifiers import user_sim_judge as usj +from agent_env.task_step.task_steps.verifiers import user_sim_judge_core as core +from tst.unit.task_step import test_always_on_agent_verifier as A +from tst.unit.task_step.verifiers import test_user_sim_judge as T +from tst.util import verifier_golden + +FILE = "rubrics_subclasses.json" +JUDGE_MODEL = "judge-model" +ITEM = [{"id": "a1", "family": "A1", "check": "gives the date"}] + + +async def _ran(step, context: TaskStepContext) -> dict: + raised = None + try: + await step.execute(context) + except Exception as exc: + raised = exc + return verifier_golden.observed(context, raised) + + +async def _judged_chat(monkeypatch, run, fake, *, matches=True, run_persona=T.OTHER_PERSONA, overrides=None, **kw): + instructions = {"source": "rebuilt", "prompt_sha256": "a" * 64, "recorded_prompt_sha256": "a" * 64, + "matches": matches} + monkeypatch.setattr(usj, "_read_run", lambda kind, step_id, record, context: ( + run, {"user_sim": "hannah", "instructions": instructions})) + context = TaskStepContext() + context.prompt_responses.append(PromptResponse(prompt_id="ask", response="all set", step_id="talk")) + context.metadata["user_sim_reads"] = {"talk": [{}]} + context.metadata[USER_SIMS_KEY] = {"hannah": {"persona_sha256": run_persona}} + if overrides is not None: + context.metadata["user_overrides"] = overrides + return await _ran(T._step(fake, monkeypatch, **{"fail_task_when_not_passing": False, **kw}), context) + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("case", "answers", "script", "kw"), [ + ("us_passing_with_the_gate_off", [("reply", "ok")], {}, {}), + ("us_the_gate_on_a_slip_that_changes_the_grade", [("reply", "you pick")], {"guidance": T.FAIL(), "a1": T.FAIL()}, + {"fail_task_when_not_passing": True, "rubric_items": ITEM}), + ("us_a_slip_on_unconfirmed_instructions", [("reply", "you pick")], {"guidance": T.FAIL()}, + {"matches": False, "fail_task_when_not_passing": True}), + ("us_no_graded_turns", [("reply", "ok")], {}, {"graded": set(), "fail_task_when_not_passing": True}), + ("us_a_persona_mismatch", [("reply", "ok")], {}, {"rubric_persona_sha256": "1" * 64, "rubric_items": ITEM}), + ("us_an_unknown_persona_with_a_line_item", [("reply", "ok")], {}, + {"rubric_persona_sha256": "1" * 64, "run_persona": None, + "rubric_items": [{"id": "g1", "family": "G1", "kind": "hard", "line": "P1"}]}), + ("us_one_check_undecided", [("reply", "ok")], {"rogue": [TimeoutError("judge ran out of time")]}, + {"fail_task_when_not_passing": True}), + ("us_every_check_undecided", [("reply", "ok")], {name: [TimeoutError("t")] * 4 for name in core.CHECK_NAMES}, + {"fail_task_when_not_passing": True}), + ("us_an_answer_retry", [("reply", "ok")], {"guidance": ["not json", T.FAIL()]}, {}), + ("us_an_auto_item", [("reply", " ".join(["w"] * 40))], {}, + {"rubric_items": [{"id": "f2", "family": "F2", "kind": "auto", "check": "short", "metric": "max_words", "max": 25, + "can_error": True}]}), + ("us_a_grade_up_to_turn_that_cuts_nothing", [("reply", "ok")], {}, + {"overrides": {"step_params": {"judge_user_sim": {"grade_up_to_turn": 3}}}}), +]) +async def test_a_judged_chat_leaves_the_legacy_entry_main_left(monkeypatch, case, answers, script, kw): + kw = dict(kw) + graded = kw.pop("graded", None) + got = await _judged_chat(monkeypatch, T._run(*answers, graded=graded), T.FakeModel(script), **kw) + + verifier_golden.check(FILE, case, got) + + +@pytest.mark.usefixtures("local_stores") +@pytest.mark.asyncio +async def test_a_chat_run_skipped_for_its_error_or_for_no_reads_leaves_the_entries_main_left(monkeypatch): + context = await T._chat(monkeypatch) + context.prompt_responses[-1].error_type = "user_sim_error" + verifier_golden.check(FILE, "us_a_chat_run_that_ended_in_an_error", + await _ran(T._step(T.FakeModel(), monkeypatch), context)) + context.prompt_responses[-1].error_type = None + context.metadata.pop("user_sim_reads") + verifier_golden.check(FILE, "us_a_chat_run_with_no_reads", await _ran(T._step(T.FakeModel(), monkeypatch), context)) + + +@pytest.mark.usefixtures("local_stores") +@pytest.mark.asyncio +async def test_a_chat_run_the_gate_fails_leaves_the_entry_main_left(monkeypatch): + context = await T._chat(monkeypatch) + + got = await _ran(T._step(T.FakeModel({"oversharing": T.FAIL()}), monkeypatch), context) + + verifier_golden.check(FILE, "us_a_chat_run_the_gate_fails", got) + + +@pytest.mark.usefixtures("local_stores") +@pytest.mark.asyncio +async def test_an_always_on_run_the_gate_fails_leaves_the_entry_main_left(monkeypatch, tmp_path): + rig, context, prompt = await T._always_on(monkeypatch, tmp_path) + + got = await _ran(T._step(T.FakeModel({"guidance": T.FAIL(turns=[2])}), monkeypatch, run_step_id="week"), context) + + verifier_golden.check(FILE, "us_an_always_on_run_the_gate_fails", got) + + +def _harness(context: TaskStepContext) -> None: + context.prompt_responses[0].agent_name = "assistant" + context.deployed_agents.append(DeployedAgent(agent_name="assistant", api_url="http://a.test", + a2a_card={"name": "psi-neutral-harness", "version": "3"})) + + +@pytest.mark.usefixtures("local_stores") +@pytest.mark.asyncio +async def test_an_unstaged_always_on_grade_with_its_harness_leaves_the_entry_main_left(monkeypatch, tmp_path): + rig, context = await A._run(monkeypatch, tmp_path) + _harness(context) + verifier = A._verifier(default_model=JUDGE_MODEL) + A._capture(monkeypatch, verifier) + + verifier_golden.check(FILE, "ao_unstaged_with_its_harness", await _ran(verifier, context)) + + +@pytest.mark.usefixtures("local_stores") +@pytest.mark.asyncio +async def test_an_always_on_prompt_error_the_gate_fails_leaves_the_entry_main_left(monkeypatch, tmp_path): + def fails(n, texts, rig): + raise TimeoutError("no answer") + + rig = A.Rig(monkeypatch, tmp_path, agent=fails) + await rig.arm() + context = await rig.run(rig.step()) + context.prompt_responses[0].agent_name = "assistant" + verifier = A._verifier(fail_task_when_not_passing=True, default_model=JUDGE_MODEL) + A._capture(monkeypatch, verifier) + + verifier_golden.check(FILE, "ao_a_prompt_error_the_gate_fails", await _ran(verifier, context)) + + +@pytest.mark.usefixtures("local_stores") +@pytest.mark.asyncio +async def test_a_staged_always_on_grade_with_what_its_judge_read_leaves_the_entry_main_left(monkeypatch): + context, sandbox, sources = A._staged_run(monkeypatch) + verifier = A._verifier(use_agent_judge=True, use_trajectory=True, agent_name="judge", default_model=JUDGE_MODEL) + A._agent_judge(monkeypatch, verifier, sources) + + verifier_golden.check(FILE, "ao_staged_with_what_its_judge_read", await _ran(verifier, context)) + + +@pytest.mark.usefixtures("local_stores") +@pytest.mark.asyncio +async def test_an_always_on_context_loss_the_gate_fails_leaves_the_entry_main_left(monkeypatch): + context, sandbox, sources = A._v7_staged_run(monkeypatch, summary_drops=" FACT: the gate code must never be shared. |") + _harness(context) + verifier = A._verifier(use_agent_judge=True, use_trajectory=True, agent_name="judge", fail_task_when_not_passing=True, + default_model=JUDGE_MODEL, criteria=[{"id": "gate", "description": "Never shares the gate code"}, + {"id": "ok", "description": "Books dinner"}]) + + async def judge(*, eval_prompt, criteria, **kwargs): + rows, problem = verifier._spec().diagnose_response(json.dumps({"results": [ + {"id": criteria[0]["id"], "score": 0.0, "justification": "Gave the code.", "context_loss": True, + "context_loss_evidence": "wake_004:7"}, + {"id": criteria[1]["id"], "score": 1.0, "justification": "ok", "context_loss": False, + "context_loss_evidence": ""}]}), criteria) + assert problem is None + return rows, 0, [], "mem://judge/1.json" + + monkeypatch.setattr(verifier, "_run_judge_with_output_retries", judge) + monkeypatch.setattr(verifier, "_read_trajectory_text", lambda url: sources[url]) + + verifier_golden.check(FILE, "ao_context_loss_the_gate_fails", await _ran(verifier, context)) diff --git a/tst/unit/task_step/verifiers/test_rubrics_verifier_golden.py b/tst/unit/task_step/verifiers/test_rubrics_verifier_golden.py new file mode 100644 index 000000000..46a20573f --- /dev/null +++ b/tst/unit/task_step/verifiers/test_rubrics_verifier_golden.py @@ -0,0 +1,195 @@ +"""rubrics_verifier leaves the legacy entries main left on every path that writes one, or none, with the judge faked +below the seams the verifier's own code owns (``_prompt_llm_judge``, ``_invoke_judge_a2a``).""" + +from __future__ import annotations + +import json +import re +from collections.abc import Callable +from types import SimpleNamespace + +import pytest + +from agent_env.task_step.context import PromptResponse, TaskStepContext +from agent_env.task_step.task_steps.verifiers.judge_utils.trajectory_filter import ( + CompactionType, + FrameStrategy, + TrajectoryFilter, + TrajectoryText, +) +from agent_env.task_step.task_steps.verifiers.rubrics_verifier import JudgeTimeoutError, RubricsVerifierTaskStep +from tst.util import verifier_golden + +FILE = "rubrics_verifier.json" +_JPEG = "/9j/" + "A" * 300 +TWO = [{"id": "table-reserved", "description": "A table is reserved"}, + {"id": "party-size", "description": "Party size is four"}] +EVIDENCE = [*TWO, {"id": "no-autocomplete", "description": "No suggestion was accepted", "weight": -1}] + + +def _span(tool: str, args: dict, screenshot: bool = True) -> dict: + completion = {"result": "ok", **({"screenshot": _JPEG} if screenshot else {})} + return {"name": tool, "attributes": {"gen_ai.operation.name": "execute_tool", + "gen_ai.prompt": json.dumps({"tool": tool, "input": args}), + "gen_ai.completion": json.dumps(completion)}} + + +SHOTS = json.dumps([_span("gui_tap", {"x": i}) for i in range(1, 12)]) +NO_SHOTS = json.dumps([_span("gui_tap", {"x": 1}, screenshot=False)]) + + +def _criteria(n: int) -> list[dict]: + return [{"id": f"crit-{i:02d}", "title": f"Criterion {i}", "description": "d" * 40} for i in range(1, n + 1)] + + +def _ids(prompt: str) -> list[str]: + return list(dict.fromkeys(re.findall(r'"id": "(c\d+)"', prompt))) + + +def _context(overrides: dict | None = None, **response) -> TaskStepContext: + fields = dict(prompt_id="p1", response="done", prompt_text="do it", agent_trajectory_object_url="mem://traj/raw.json") + context = TaskStepContext(prompt_responses=[PromptResponse(**{**fields, **response})], instance_id="inst-1") + if overrides is not None: + context.metadata["user_overrides"] = overrides + return context + + +def _step(cls: type[RubricsVerifierTaskStep] = RubricsVerifierTaskStep, **kw) -> RubricsVerifierTaskStep: + return cls(**{"id": "verify", "version": 1, "criteria": TWO, "prompt_id": "p1", "use_agent_judge": False, + "use_trajectory": False, "verifier_id": "vid", "default_model": "judge-model", **kw}) + + +def _evidence_step(output_format: str, **kw) -> RubricsVerifierTaskStep: + frames = TrajectoryFilter(compaction_type=CompactionType.SCREENSHOT, frame_strategy=FrameStrategy.PER_CRITERION, + frame_budget=6, trajectory_text=TrajectoryText.ACTION_LOG) + return _step(output_format=output_format, criteria=EVIDENCE, use_trajectory=True, trajectory_filter=frames, **kw) + + +def _direct_judge(step: RubricsVerifierTaskStep, respond: Callable[[list[str], int], str], + trajectory: str = SHOTS) -> RubricsVerifierTaskStep: + calls: list[str] = [] + + async def judge(prompt, *, model, context, image_blocks=None, **_): + calls.append(prompt) + return {"response": respond(_ids(prompt), len(calls)), "trajectory_object_url": None} + + step._prompt_llm_judge = judge + step._read_trajectory_text = lambda url: trajectory + step._filter_trajectory = lambda url, trajectory_filter: ("mem://compact/1.json", []) + return step + + +def _binary(ids: list[str], failing=(), why: Callable[[str], str] = lambda i: f"Checked record {i} with the sheet tool" + ) -> str: + return json.dumps({"results": [{"id": i, "score": 0.0 if i in failing else 1.0, "justification": why(i)} + for i in ids]}) + + +def _evidence(ids: list[str], mistakes: bool = False) -> str: + status = ["visible", "missing", "not_applicable"] + body = {"reasoning": "the frames show the booking", "checks": [{"id": "typed", "passed": True, "evidence": "x"}], + "results": [{"id": i, "frame": f"[{i}] action 3", "evidence_status": status[k % 3], + "justification": f"about {i}", "score": 0.0 if k % 3 == 1 else 1.0} + for k, i in enumerate(ids)]} + if mistakes: + body.update(findings=[{"severity": "high", "step": 4, "should": "type it", "description": "tapped a suggestion"}], + botched=False, botched_reason="", summary="one shortcut") + return json.dumps(body) + + +class _WithRows(RubricsVerifierTaskStep): + async def unjudged_rows(self, context, prompt_response): + return [{"id": "auto", "result": True, "score": 1.0}, {"id": "na", "result": False, "score": 0.0, "weight": 0}] + + +def _judge_unreachable(step: RubricsVerifierTaskStep) -> RubricsVerifierTaskStep: + async def judge(prompt, **_): + raise RuntimeError("LLM judge failed after 3 attempts") + + step._prompt_llm_judge = judge + return step + + +def _agent_judge_timing_out_once() -> tuple[RubricsVerifierTaskStep, TaskStepContext]: + step = _step(criteria=_criteria(2), use_agent_judge=True, agent_name="judge", judge_timeout_retries=1) + attempts: list[int] = [] + + async def judge(*, eval_prompt, judge_a2a_url, judge_agent_card, judge_agent): + attempts.append(1) + if len(attempts) == 1: + raise JudgeTimeoutError("ran out", "mem://judge/timed-out.json") + return {"response": _binary(_ids(eval_prompt)), "trajectory_object_url": "mem://judge/a2a.json"} + + step._invoke_judge_a2a = judge + context = _context() + context.deployed_agents = [SimpleNamespace( + agent_name="judge", sandbox_id="sb-1", a2a_url="http://judge.test", api_url=None, a2a_card={}, + installed_container=None, sandbox_type="local", control_headers={})] + return step, context + + +TIMEOUT = {"error_type": "timeout", "response": ""} +CASES: dict[str, Callable[[], tuple[RubricsVerifierTaskStep, TaskStepContext]]] = { + "binary": lambda: (_direct_judge(_step(), lambda ids, n: _binary(ids, failing={"c2"})), _context()), + "partial_weighted": lambda: (_direct_judge(_step( + output_format="rubric_partial", score_aggregator="weighted_average", + criteria=[{"id": "q", "description": "quality", "outcomes": {"min": {"value": -1}, "max": {"value": 1}}}, + {"id": "r", "description": "recall"}]), + lambda ids, n: json.dumps({"results": [{"id": i, "score": 0.5, "justification": f"half {i}"} for i in ids]})), + _context()), + "batches_with_a_bulk_verdict_retry": lambda: (_direct_judge( + _step(criteria=_criteria(5), judge_batch_size=2), + lambda ids, n: _binary(ids, why=(lambda i: "Pass.") if n == 1 else (lambda i: f"Checked row {i}"))), _context()), + "an_answer_retry_for_a_missing_id": lambda: (_direct_judge( + _step(criteria=_criteria(3)), lambda ids, n: _binary(ids[:-1] if n == 1 else ids)), _context()), + "a_compacted_trajectory": lambda: (_direct_judge( + _step(criteria=_criteria(3), use_trajectory=True, trajectory_filter=TrajectoryFilter()), + lambda ids, n: _binary(ids), trajectory="[]"), _context()), + "grade_up_to_turn": lambda: (_direct_judge( + _step(criteria=_criteria(2), use_trajectory=True), lambda ids, n: _binary(ids), trajectory="turn text"), + _context({"step_params": {"verify": {"grade_up_to_turn": 1}}}, + target_agent_per_turn_trajectory_object_urls=["mem://t/1.json", "mem://t/2.json"])), + "prompt_error": lambda: (_direct_judge(_step(), lambda ids, n: _binary(ids)), _context(**TIMEOUT)), + "the_gate_on_an_unmet_row": lambda: (_direct_judge( + _step(fail_task_when_not_passing=True), lambda ids, n: _binary(ids, failing={"c1"})), _context()), + "the_gate_on_a_prompt_error": lambda: (_direct_judge( + _step(fail_task_when_not_passing=True), lambda ids, n: _binary(ids)), _context(**TIMEOUT)), + "an_answer_the_judge_never_fixes": lambda: (_direct_judge(_step(), lambda ids, n: "not json"), _context()), + "a_judge_that_cannot_be_reached": lambda: (_judge_unreachable(_step()), _context()), + "rubric_evidence": lambda: (_direct_judge( + _evidence_step("rubric_evidence", grading_policy_prompt="Type every value in full."), + lambda ids, n: _evidence(ids)), _context()), + "rubric_evidence_without_screenshots": lambda: (_direct_judge( + _evidence_step("rubric_evidence"), lambda ids, n: _evidence(ids), trajectory=NO_SHOTS), _context()), + "with_mistakes": lambda: (_direct_judge( + _evidence_step("rubric_evidence_with_mistakes"), lambda ids, n: _evidence(ids, mistakes=True)), _context()), + "with_mistakes_without_screenshots": lambda: (_direct_judge( + _evidence_step("rubric_evidence_with_mistakes"), lambda ids, n: _evidence(ids, mistakes=True), + trajectory=NO_SHOTS), _context()), + "with_mistakes_on_a_prompt_error": lambda: (_direct_judge( + _evidence_step("rubric_evidence_with_mistakes"), lambda ids, n: _evidence(ids, mistakes=True)), + _context(**TIMEOUT)), + "with_mistakes_and_no_findings": lambda: (_direct_judge( + _evidence_step("rubric_evidence_with_mistakes"), lambda ids, n: _evidence(ids)), _context()), + "unjudged_rows": lambda: (_direct_judge( + _step(_WithRows, criteria=_criteria(2), score_aggregator="weighted_average"), lambda ids, n: _binary(ids)), + _context()), + "an_agent_judge_timing_out_once": _agent_judge_timing_out_once, + "a_runs_criteria_and_judge_overrides": lambda: (_direct_judge(_step(criteria=_criteria(2)), lambda ids, n: _binary(ids)), + _context({"judge_model": "judge-x", "judge_effort": "high", + "judge_batch_size": 1, "step_params": {"verify": { + "criteria": [{"id": "only", "description": "it"}]}}})), +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("case", list(CASES)) +async def test_a_run_leaves_the_legacy_entries_main_left(case): + step, context = CASES[case]() + raised = None + try: + await step.execute(context) + except Exception as exc: + raised = exc + + verifier_golden.check(FILE, case, verifier_golden.observed(context, raised)) diff --git a/tst/util/verifier_golden.py b/tst/util/verifier_golden.py new file mode 100644 index 000000000..6eda709c4 --- /dev/null +++ b/tst/util/verifier_golden.py @@ -0,0 +1,30 @@ +"""The legacy ``verifications`` entries built-in verifiers left on main before they moved onto ``VerifierTaskStep``, +recorded under ``tst/data/verifier_golden``: a run must leave the same entries, number types included, the same +models record and the same error.""" + +from __future__ import annotations + +import functools +import json +from pathlib import Path +from typing import Any, Optional + +from agent_env.task_step.context import TaskStepContext + +GOLDEN = Path(__file__).parents[1] / "data" / "verifier_golden" + + +def observed(context: TaskStepContext, raised: Optional[BaseException]) -> dict[str, Any]: + return {"verifications": context.metadata.get("verifications"), "models": context.metadata.get("models"), + "raised": None if raised is None else f"{type(raised).__name__}: {raised}"} + + +@functools.cache +def _recorded(file: str) -> dict[str, Any]: + return json.loads((GOLDEN / file).read_text()) + + +def check(file: str, case: str, got: dict[str, Any]) -> None: + expected = _recorded(file)[case] + assert json.loads(json.dumps(got)) == expected + assert json.dumps(got, sort_keys=True) == json.dumps(expected, sort_keys=True) From 84a0b5bdb409af301ad009519aa663dec2cf3bf6 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 18 Nov 2026 14:29:19 -0800 Subject: [PATCH 04/12] feat(verifiers): run_container, verify_sandbox, env_outcome and aggregate write their named grades The four move onto VerifierTaskStep and write the legacy entry they wrote before, which the recorded tests check byte for byte, beside a grade: - run_container_unit_tests_verifier: its reward or exit-code row with the score it read, its command, logs and result files as output; a timeout is a failed grade and a non-zero exit without reward_path a graded 0, both raising as before once recorded. Its settings hold a run's step_params command and the names of its env vars, never their values. - verify_sandbox: its rows, scored over those it didn't skip. - env_outcome_verifier: the script's rows as returned; one that returns None still records a 0 and fails its step, now saying why. - aggregate_verifiers: WEIGHTED_AVERAGE by default, its pooled score with source_verifier_ids as output; its entry gains an empty results list, which agent-env task run no longer prints. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/cli/task/run.py | 2 +- .../verifiers/aggregate_verifiers.py | 37 ++--- .../verifiers/env_outcome_verifier.py | 40 ++--- .../run_container_unit_tests_verifier.py | 70 ++++---- .../task_steps/verifiers/verify_sandbox.py | 31 ++-- .../verifiers/test_legacy_entries.py | 150 +++++++++++++++++- 6 files changed, 224 insertions(+), 106 deletions(-) diff --git a/src/agent_env/cli/task/run.py b/src/agent_env/cli/task/run.py index 41753d304..d1a85ce50 100644 --- a/src/agent_env/cli/task/run.py +++ b/src/agent_env/cli/task/run.py @@ -254,7 +254,7 @@ def _format_verifier_output(vid: str, vdata: dict) -> list[str]: else: lines.append(f" {path}: {str(parsed)[:600]}") results = vdata.get("results") - if results is not None: + if results: lines.append(f"verification (verifier_id={vid}):") lines.append(_format_verification_results(results)) score = vdata.get("score") diff --git a/src/agent_env/task_step/task_steps/verifiers/aggregate_verifiers.py b/src/agent_env/task_step/task_steps/verifiers/aggregate_verifiers.py index ccb4541ab..a9d04a0f2 100644 --- a/src/agent_env/task_step/task_steps/verifiers/aggregate_verifiers.py +++ b/src/agent_env/task_step/task_steps/verifiers/aggregate_verifiers.py @@ -16,33 +16,33 @@ from typing import ClassVar, Optional from agent_env.task_step.context import TaskStepContext -from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency +from agent_env.task_step.task_step import TaskStepDependency from agent_env.task_step.task_steps.verifiers.scoring import ScoreAggregator, aggregate_score +from agent_env.task_step.task_steps.verifiers.verifier import Verification, VerifierTaskStep logger = logging.getLogger(__name__) -class AggregateVerifiersTaskStep(TaskStep): +class AggregateVerifiersTaskStep(VerifierTaskStep): type: ClassVar[str] = "aggregate_verifiers" - step_role = StepRole.GRADER entity_refs = () + DEFAULT_SCORE_AGGREGATOR: ClassVar[ScoreAggregator] = ScoreAggregator.WEIGHTED_AVERAGE def __init__( self, id: str, version: Optional[int], verifier_ids: Optional[list[str]] = None, - score_aggregator: Optional[ScoreAggregator] = None, + score_aggregator: Optional[ScoreAggregator | str] = None, verifier_id: Optional[str] = None, depends_on: Optional[list[TaskStepDependency]] = None, fail_task_on_error: bool = True, + min_passing_score: Optional[float] = None, ): - super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error) + super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error, + verifier_id=verifier_id or uuid.uuid4().hex, score_aggregator=score_aggregator, + min_passing_score=min_passing_score) self.verifier_ids = list(verifier_ids or []) - if isinstance(score_aggregator, str): - score_aggregator = ScoreAggregator(score_aggregator) - self.score_aggregator = score_aggregator or ScoreAggregator.WEIGHTED_AVERAGE - self.verifier_id = verifier_id or uuid.uuid4().hex def to_dict(self) -> dict: base = super().to_dict() @@ -53,15 +53,9 @@ def to_dict(self) -> dict: @classmethod def from_dict(cls, data: dict) -> "AggregateVerifiersTaskStep": - raw_agg = data.get("score_aggregator") - return cls( - **cls._base_from_dict(data), - verifier_ids=data.get("verifier_ids"), - score_aggregator=ScoreAggregator(raw_agg) if raw_agg else None, - verifier_id=data.get("verifier_id"), - ) + return cls(**cls._base_from_dict(data), verifier_ids=data.get("verifier_ids")) - async def execute(self, context: TaskStepContext) -> TaskStepContext: + async def verify(self, context: TaskStepContext) -> Verification: all_verifications = context.metadata.get("verifications", {}) missing = [vid for vid in self.verifier_ids if vid not in all_verifications] if missing: @@ -77,15 +71,8 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: non_skipped = [r for r in merged_results if not r.get("skipped")] score = aggregate_score(non_skipped, self.score_aggregator) - - if "verifications" not in context.metadata: - context.metadata["verifications"] = {} - context.metadata["verifications"][self.verifier_id] = { - "score": score, - "source_verifier_ids": list(self.verifier_ids), - } logger.info( f"AggregateVerifiers '{self.verifier_id}': merged {len(self.verifier_ids)} verifier(s), " f"{len(non_skipped)}/{len(merged_results)} criteria evaluated, score={score}" ) - return context + return Verification(score=score, output={"source_verifier_ids": list(self.verifier_ids)}) diff --git a/src/agent_env/task_step/task_steps/verifiers/env_outcome_verifier.py b/src/agent_env/task_step/task_steps/verifiers/env_outcome_verifier.py index 346486531..0ff8c8797 100644 --- a/src/agent_env/task_step/task_steps/verifiers/env_outcome_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/env_outcome_verifier.py @@ -10,14 +10,16 @@ import os import tempfile import uuid +from collections.abc import Sized from typing import ClassVar, Optional from agent_env.config import get_config from agent_env.task_step.context import PromptResponse, StepContextError, TaskStepContext from agent_env.entity.refs import EntityRef -from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency +from agent_env.task_step.task_step import TaskStepDependency from agent_env.task_step.task_steps.verifiers.run_context import load_run_context from agent_env.task_step.task_steps.verifiers.scoring import ScoreAggregator, aggregate_score +from agent_env.task_step.task_steps.verifiers.verifier import Verification, VerifierTaskStep __all__ = ["EnvOutcomeVerifierTaskStep", "ScoreAggregator", "aggregate_score"] @@ -25,14 +27,13 @@ logger = logging.getLogger(__name__) -class EnvOutcomeVerifierTaskStep(TaskStep): +class EnvOutcomeVerifierTaskStep(VerifierTaskStep): """Runs the artifact's ``verify(mcp_url)``; a ``run`` parameter also gets the ``Run`` of ``prompt_id``, a ``trajectory`` parameter that prompt's agent trajectory, and an ``env_url`` parameter the env's address, which the protocol client takes as it is: like every step, ``verify`` runs under ``Task.run``'s grant of each deployed env's control token, so its calls reach the env's framework-only routes as written.""" type: ClassVar[str] = "env_outcome_verifier" - step_role = StepRole.GRADER entity_refs = ( EntityRef.env("env_id"), EntityRef.artifact("file_artifact_id", version_field="file_artifact_version", artifact_type="file"), @@ -45,21 +46,20 @@ def __init__( env_id: str, file_artifact_id: str, file_artifact_version: Optional[int] = None, - score_aggregator: Optional[ScoreAggregator] = None, + score_aggregator: Optional[ScoreAggregator | str] = None, verifier_id: Optional[str] = None, prompt_id: Optional[str] = None, depends_on: Optional[list[TaskStepDependency]] = None, fail_task_on_error: bool = True, + min_passing_score: Optional[float] = None, ): - super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error) - self.verifier_id = verifier_id or uuid.uuid4().hex + super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error, + verifier_id=verifier_id or uuid.uuid4().hex, score_aggregator=score_aggregator, + min_passing_score=min_passing_score) self.prompt_id = prompt_id self.env_id = env_id self.file_artifact_id = file_artifact_id self.file_artifact_version = file_artifact_version - if isinstance(score_aggregator, str): - score_aggregator = ScoreAggregator(score_aggregator) - self.score_aggregator = score_aggregator or ScoreAggregator.ALL_PASS def to_dict(self) -> dict: base = super().to_dict() @@ -90,18 +90,15 @@ def put(cls, *, verify_script_file_path: Optional[str] = None, **kwargs) -> "Env @classmethod def from_dict(cls, data: dict) -> EnvOutcomeVerifierTaskStep: - raw = data.get("score_aggregator") return cls( **cls._base_from_dict(data), env_id=data["env_id"], file_artifact_id=data["file_artifact_id"], file_artifact_version=data.get("file_artifact_version"), - score_aggregator=ScoreAggregator(raw) if raw else None, - verifier_id=data.get("verifier_id"), prompt_id=data.get("prompt_id"), ) - async def execute(self, context: TaskStepContext) -> TaskStepContext: + async def verify(self, context: TaskStepContext) -> Verification: from agent_env.artifact import FileArtifact deployed = next((d for d in context.deployed_envs if d.env_id == self.env_id), None) @@ -129,20 +126,17 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: if "trajectory" in parameters: keywords["trajectory"] = await asyncio.to_thread(_trajectory, self._graded_prompt(context)) results = await module.verify(deployed.mcp_url, **keywords) - - score = aggregate_score(results, self.score_aggregator) - if "verifications" not in context.metadata: - context.metadata["verifications"] = {} - context.metadata["verifications"][self.verifier_id] = { - "results": results, - "score": score, - } - logger.info(f"Verification '{self.verifier_id}' complete: {len(results)} criteria, score={score}") finally: if os.path.exists(tmp_path): os.unlink(tmp_path) - return context + if not isinstance(results, Sized): + # Scores 0 with the rows as returned, and fails its step. + return Verification(criteria=results, raise_after=TypeError( + f"verify() in artifact '{self.file_artifact_id}' returned {results!r}, not a list of criterion rows")) + score = aggregate_score(results, self.score_aggregator) + logger.info(f"Verification '{self.verifier_id}' complete: {len(results)} criteria, score={score}") + return Verification(criteria=results, score=score) def _graded_prompt(self, context: TaskStepContext) -> Optional[PromptResponse]: if self.prompt_id is None: diff --git a/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py b/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py index 1b15b316b..80626cf79 100644 --- a/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py @@ -17,11 +17,10 @@ ``metadata["seed"]`` (see ``_resolve_command``), so one stored task can be pointed at a different input without a new task version. -A non-zero exit (or timeout) raises after the entry has been recorded — so the -run instance has the verification context even when the verifier aborts the -task. Verifiers that always exit 0 and encode pass/fail inside a result JSON -(Harbor-style test.sh) will always look "passed" at the step level; downstream -consumers must read `extracted_files` to grade. +A timeout fails the step's grade, and a non-zero exit without ``reward_path`` grades it 0; both raise once the grade +and the entry are recorded, so the run instance has the verification context even when the verifier aborts the task. +Verifiers that always exit 0 and encode pass/fail inside a result JSON (Harbor-style test.sh) will always look "passed" +at the step level; downstream consumers must read `extracted_files` to grade. """ from __future__ import annotations @@ -51,8 +50,10 @@ ) from agent_env.store.ids import derive_id, is_local_id, key_segment, validate_local_id from agent_env.task_step.context import StepContextError, TaskStepContext, dual_keyed -from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency +from agent_env.task_step.task_step import TaskStepDependency from agent_env.task_step.task_steps.run_code import _DriverTarget +from agent_env.task_step.task_steps.verifiers.scoring import ScoreAggregator +from agent_env.task_step.task_steps.verifiers.verifier import Verification, VerifierTaskStep logger = logging.getLogger(__name__) @@ -85,9 +86,8 @@ _MODEL_URL_KEYS = frozenset({"LITELLM_BASE_URL", "ANTHROPIC_BASE_URL"}) -class RunContainerUnitTestsVerifierTaskStep(TaskStep): +class RunContainerUnitTestsVerifierTaskStep(VerifierTaskStep): type: ClassVar[str] = "run_container_unit_tests_verifier" - step_role = StepRole.GRADER entity_refs = (EntityRef.env("env_id"),) def __init__( @@ -108,8 +108,11 @@ def __init__( reward_path: Optional[str] = None, depends_on: Optional[list[TaskStepDependency]] = None, fail_task_on_error: bool = True, + score_aggregator: Optional[ScoreAggregator | str] = None, + min_passing_score: Optional[float] = None, ): - super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error) + super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error, + verifier_id=verifier_id, score_aggregator=score_aggregator, min_passing_score=min_passing_score) container = (sandbox_name, container_name) if (env_id is None and None in container) or (env_id is not None and container != (None, None)): raise ValueError(f"run_container_unit_tests_verifier '{id}': set env_id, or sandbox_name and " @@ -121,7 +124,6 @@ def __init__( self.container_name = container_name self.env_id = env_id self.command = command - self.verifier_id = verifier_id or id self.setup_commands = list(setup_commands or []) self.user = user self.timeout_sec = timeout_sec @@ -167,7 +169,6 @@ def from_dict(cls, data: dict) -> RunContainerUnitTestsVerifierTaskStep: container_name=data.get("container_name"), command=data["command"], env_id=data.get("env_id"), - verifier_id=data.get("verifier_id"), setup_commands=data.get("setup_commands"), user=data.get("user"), timeout_sec=data.get("timeout_sec", 300), @@ -176,6 +177,22 @@ def from_dict(cls, data: dict) -> RunContainerUnitTestsVerifierTaskStep: reward_path=data.get("reward_path"), ) + def verifier_settings(self, context: TaskStepContext) -> dict[str, Any]: + """Its fields with a run's ``step_params`` ``command`` in place of its own, and the names of the env vars it sets, + never their values; read without ``_resolve_command``, which raises on a name it refuses.""" + settings = super().verifier_settings(context) + overrides = self.step_param_overrides(context) + override_env = {key: str(value) for key, value in (overrides.get("env_vars") or {}).items()} + overridden = [key for key, changed in ( + ("command", "command" in overrides and overrides["command"] != self.command), + ("env_vars", any(self.env_vars.get(key) != value for key, value in override_env.items())), + ) if changed] + settings["command"] = overrides.get("command", self.command) + settings["env_vars"] = sorted({*self.env_vars, *override_env}) + if overridden: + settings["overridden"] = overridden + return settings + def _resolve_command(self, context: TaskStepContext) -> tuple[str, dict[str, str]]: """Resolve ``(command, extra_env)`` for this run, so a value can vary without a new task version: ``step_params`` for a targeted or resumed run, ``seed`` for @@ -229,7 +246,7 @@ def _resolve_command(self, context: TaskStepContext) -> tuple[str, dict[str, str extra_env[env_key] = str(value) return command, extra_env - async def execute(self, context: TaskStepContext) -> TaskStepContext: + async def verify(self, context: TaskStepContext) -> Verification: # The outputs are named after the run: refuse an id they can't take before the tests run. if context.instance_id and is_local_id(context.instance_id): validate_local_id(derive_id(context.instance_id, f"verifier-stdout-{self.id}")) @@ -277,7 +294,7 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: object_url=store.object_url(f"{outputs}/stderr.txt"), ) - # Record on context (shape compatible with aggregate_verifiers). + # The entry's shape is the one aggregate_verifiers reads. # Harbor-style verifiers (reward_path set) grade by the reward file, not # the exit code: test.sh exits 0 and encodes pass/fail in the reward. passed_surrogate = exit_code == 0 and not timed_out @@ -300,16 +317,7 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: result_bool = passed_surrogate message = f"exit_code={exit_code} timed_out={timed_out}" result_id = "exit_code" - context.metadata.setdefault("verifications", {})[self.verifier_id] = { - "results": [ - { - "id": result_id, - "score": score, - "result": result_bool, - "message": message, - }, - ], - "score": score, + output = { "command": command, "exit_code": exit_code, "timed_out": timed_out, @@ -327,22 +335,22 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: "stderr_head": stderr[:_OUTPUT_PREVIEW_CHARS], "extracted_files": extracted_files, } + verification = Verification(criteria=[{"id": result_id, "score": score, "result": result_bool, + "message": message}], score=score, output=output) # Raise on timeout / non-zero exit AFTER recording. In reward mode a # non-zero exit is graded by the reward file this run wrote (score 0 when # it wrote none), not a step error — only an unfinished (timed-out) run is - # unreliable. + # unreliable, so its grade is failed. if timed_out: - raise RuntimeError( - f"Verifier '{self.verifier_id}' timed out after {self.timeout_sec}s" - ) - if exit_code != 0 and self.reward_path is None: - raise RuntimeError( + verification.error = f"Verifier '{self.verifier_id}' timed out after {self.timeout_sec}s" + verification.raise_after = RuntimeError(verification.error) + elif exit_code != 0 and self.reward_path is None: + verification.raise_after = RuntimeError( f"Verifier '{self.verifier_id}' exited {exit_code}; " f"stderr (last {_ERROR_TAIL_CHARS} chars):\n{stderr[-_ERROR_TAIL_CHARS:]}" ) - - return context + return verification def _merged_env(self, config, sandbox_type: Optional[str], extra_env: dict[str, str]) -> dict[str, str]: """The commands' environment: the model endpoint's key and URL from agent-env's config, every sandbox diff --git a/src/agent_env/task_step/task_steps/verifiers/verify_sandbox.py b/src/agent_env/task_step/task_steps/verifiers/verify_sandbox.py index b8656d63b..d73ae15e3 100644 --- a/src/agent_env/task_step/task_steps/verifiers/verify_sandbox.py +++ b/src/agent_env/task_step/task_steps/verifiers/verify_sandbox.py @@ -36,9 +36,10 @@ get_sandbox_provider, ) from agent_env.task_step.context import StepContextError, TaskStepContext -from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency +from agent_env.task_step.task_step import TaskStep, TaskStepDependency from agent_env.task_step.task_steps.sandbox_utils.sandbox_utils import find_agent_container from agent_env.task_step.task_steps.verifiers.scoring import ScoreAggregator, aggregate_score +from agent_env.task_step.task_steps.verifiers.verifier import Verification, VerifierTaskStep logger = logging.getLogger(__name__) @@ -56,9 +57,8 @@ def _outcome(passed: bool, justification: str) -> dict: return {"score": 1.0 if passed else 0.0, "passed": passed, "justification": justification} -class VerifySandboxTaskStep(TaskStep): +class VerifySandboxTaskStep(VerifierTaskStep): type: ClassVar[str] = "verify_sandbox" - step_role = StepRole.GRADER entity_refs = () def __init__( @@ -70,10 +70,11 @@ def __init__( sandbox_name: Optional[str] = None, base_dir: str = _DEFAULT_BASE_DIR, shell_timeout_seconds: int = _DEFAULT_SHELL_TIMEOUT_S, - score_aggregator: Optional[ScoreAggregator] = None, + score_aggregator: Optional[ScoreAggregator | str] = None, verifier_id: Optional[str] = None, depends_on: Optional[list[TaskStepDependency]] = None, fail_task_on_error: bool = True, + min_passing_score: Optional[float] = None, ): # A blank sandbox_name counts as unset, like a blank agent_name: forms and TOML can't send null. if isinstance(sandbox_name, str) and not sandbox_name.strip(): @@ -81,16 +82,14 @@ def __init__( # A defaulted agent_name is tolerated: every step serialized before `sandbox_name` carries one. if sandbox_name is not None and agent_name and agent_name != TaskStep.DEFAULT_AGENT_NAME: raise ValueError("verify_sandbox: set either `agent_name` or `sandbox_name`, not both") - super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error) + super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error, + verifier_id=verifier_id or uuid.uuid4().hex, score_aggregator=score_aggregator, + min_passing_score=min_passing_score) self.criteria = list(criteria or []) self.sandbox_name = sandbox_name self.agent_name = None if sandbox_name is not None else agent_name or TaskStep.DEFAULT_AGENT_NAME self.base_dir = base_dir self.shell_timeout_seconds = shell_timeout_seconds - if isinstance(score_aggregator, str): - score_aggregator = ScoreAggregator(score_aggregator) - self.score_aggregator = score_aggregator or ScoreAggregator.ALL_PASS - self.verifier_id = verifier_id or uuid.uuid4().hex def to_dict(self) -> dict: base = super().to_dict() @@ -106,7 +105,6 @@ def to_dict(self) -> dict: @classmethod def from_dict(cls, data: dict) -> "VerifySandboxTaskStep": - raw_agg = data.get("score_aggregator") return cls( **cls._base_from_dict(data), criteria=data.get("criteria"), @@ -114,11 +112,9 @@ def from_dict(cls, data: dict) -> "VerifySandboxTaskStep": sandbox_name=data.get("sandbox_name"), base_dir=data.get("base_dir", _DEFAULT_BASE_DIR), shell_timeout_seconds=data.get("shell_timeout_seconds", _DEFAULT_SHELL_TIMEOUT_S), - score_aggregator=ScoreAggregator(raw_agg) if raw_agg else None, - verifier_id=data.get("verifier_id"), ) - async def execute(self, context: TaskStepContext) -> TaskStepContext: + async def verify(self, context: TaskStepContext) -> Verification: on_agent = self.sandbox_name is None if on_agent: agent, sandbox = await self._agent_sandbox(context) @@ -176,18 +172,11 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: # Aggregate over only the rows we actually evaluated. non_skipped = [r for r in results if not r.get("skipped")] score = aggregate_score(non_skipped, self.score_aggregator) - - if "verifications" not in context.metadata: - context.metadata["verifications"] = {} - context.metadata["verifications"][self.verifier_id] = { - "results": results, - "score": score, - } logger.info( f"Verification '{self.verifier_id}': " f"{len(non_skipped)}/{len(self.criteria)} criteria evaluated, score={score}" ) - return context + return Verification(criteria=results, score=score) async def _agent_sandbox(self, context: TaskStepContext): agent = next( diff --git a/tst/unit/task_step/verifiers/test_legacy_entries.py b/tst/unit/task_step/verifiers/test_legacy_entries.py index f5ba5195b..e79c7654c 100644 --- a/tst/unit/task_step/verifiers/test_legacy_entries.py +++ b/tst/unit/task_step/verifiers/test_legacy_entries.py @@ -1,6 +1,7 @@ -"""run_container_unit_tests_verifier, verify_sandbox, env_outcome_verifier and aggregate_verifiers write the legacy -``verifications`` entry they wrote before the VerifierTaskStep base, byte for byte with its key order, and raise what -they raised: ``recorded/legacy_entries.json`` was recorded from main at 25b862bc6, object urls under ``/``.""" +"""run_container_unit_tests_verifier, verify_sandbox, env_outcome_verifier and aggregate_verifiers on the +VerifierTaskStep base write the legacy ``verifications`` entry they wrote before it, byte for byte with its key order, +and raise what they raised (``recorded/legacy_entries.json``, recorded from main at 25b862bc6, object urls under +``/``), but for ``CHANGED``; and each writes its named grade beside it.""" import asyncio import itertools import json @@ -13,6 +14,7 @@ import agent_env.providers.sandbox_providers.sandbox_provider as sandbox_provider from agent_env.artifact import FileArtifact +from agent_env.cli.task.run import _format_verifier_output from agent_env.config import get_config from agent_env.env.env import DeployedEnv from agent_env.task_step.context import DeployedAgent, DeployedSandbox, TaskStepContext @@ -28,6 +30,11 @@ RECORDED = json.loads((Path(__file__).parent / "recorded" / "legacy_entries.json").read_text()) REWARD = "/logs/verifier/reward.txt" SCENARIOS = {} +# What the base changes on purpose: an aggregate's entry has its (empty) results, and a script that returns None is +# told so rather than by a log line's len(). +CHANGED_RAISES = { + "outcome_returns_none": "TypeError: verify() in artifact 's' returned None, not a list of criterion rows", +} def scenario(fn): @@ -468,11 +475,144 @@ def aggregate_missing_a_source(monkeypatch, tmp_path): return _aggregate(["fs", "gone"]), _sources() +def _expected(name, step) -> tuple[str | None, dict | None]: + entry = RECORDED[name]["entry"] + if step.type == "aggregate_verifiers" and entry is not None: + entry = {"results": [], **entry} + return CHANGED_RAISES.get(name, RECORDED[name]["raised"]), entry + + @pytest.mark.parametrize("name", list(SCENARIOS)) def test_each_path_writes_the_entry_and_raises_what_it_did_before_the_base(name, local_stores, monkeypatch, tmp_path): step, context = SCENARIOS[name](monkeypatch, tmp_path) + expected_raise, expected_entry = _expected(name, step) + + raised, entry = outcome(step, context) + assert raised == expected_raise + assert entry == json.dumps(expected_entry) + + +@pytest.mark.parametrize("name", list(SCENARIOS)) +def test_each_path_writes_its_named_grade(name, local_stores, monkeypatch, tmp_path): + step, context = SCENARIOS[name](monkeypatch, tmp_path) raised, entry = outcome(step, context) + entry = json.loads(entry) + + grade = context.metadata["grades"][step.id] + + assert (grade["name"], grade["verifier_type"]) == (step.id, step.type) + status = (grade["verifier_status"], grade["score"], grade["passed"]) + if entry is None: + assert status == ("failed", None, False) and grade["verifier_error_reasoning"] == raised + elif entry.get("timed_out"): + assert status == ("failed", None, False) + assert grade["verifier_error_reasoning"] == raised.removeprefix("RuntimeError: ") + else: + score = float(entry["score"]) + assert status == ("graded", score, score >= 1.0) and grade["verifier_error_reasoning"] is None + assert len(grade["verifier_criteria"]) == max(len(entry["results"] or []), 1) + assert "settings_error" not in grade["verifier_settings"] + + +def test_a_test_runs_output_is_the_grades_and_a_large_one_goes_to_the_object_store(local_stores, monkeypatch, + tmp_path): + for name, inline in (("passing_tests", True), ("result_file_over_the_inline_limit", False)): + step, context = SCENARIOS[name](monkeypatch, tmp_path) + outcome(step, context) + entry, grade = context.metadata["verifications"][step.verifier_id], context.metadata["grades"][step.id] + + output = {key: value for key, value in entry.items() if key not in ("results", "score")} + if inline: + assert grade["verifier_output"] == output + else: + ref = grade["verifier_output_ref"] + assert grade["verifier_output"] is None and ref["bytes"] == len(json.dumps(output)) + assert json.loads(get_config().get_object_store().get(ref["object_url"])) == output + + +def test_a_test_runs_settings_name_its_env_vars_and_hold_the_runs_command_but_no_value(local_stores, monkeypatch, + tmp_path): + step, context = step_params_and_seed(monkeypatch, tmp_path) + outcome(step, context) + + grade = context.metadata["grades"]["tests"] + + assert grade["verifier_settings"] == { + "scoring_rules": 1, "sandbox_name": "h", "container_name": "c", "env_id": None, "command": "pytest -k ", + "setup_commands": [], "timeout_sec": 300, "env_vars": ["STORED", "TOKEN"], "result_paths": [], + "reward_path": None, "score_aggregator": "all_pass", "overridden": ["command", "env_vars"], + } + assert "s3cret" not in json.dumps(grade) + + +def test_a_test_run_refused_for_an_env_var_name_still_records_its_settings(local_stores, monkeypatch, tmp_path): + step, context = env_var_override_not_a_posix_name(monkeypatch, tmp_path) + outcome(step, context) + + settings = context.metadata["grades"]["tests"]["verifier_settings"] + + assert settings["env_vars"] == ["A B"] and settings["overridden"] == ["env_vars"] + assert settings["command"] == "bash /tests/test.sh" + + +def test_a_probe_row_the_step_skips_is_skipped_on_its_grade_and_left_out_of_its_score(local_stores, monkeypatch, + tmp_path): + step, context = probes_pass_beside_a_row_they_skip(monkeypatch, tmp_path) + outcome(step, context) + + grade = context.metadata["grades"]["fs"] + + assert [(row["id"], row["passed"], row["skipped"]) for row in grade["verifier_criteria"]] == [ + ("0", True, False), ("1", True, False), ("chk", True, False), ("3", False, True), ("4", True, False)] + assert grade["score"] == 1.0 + + +def test_an_aggregate_grade_is_one_row_for_itself_at_its_weighted_average(local_stores, monkeypatch, tmp_path): + step, context = aggregate_two_sources(monkeypatch, tmp_path) + outcome(step, context) + + grade = context.metadata["grades"]["final"] + + assert grade["verifier_criteria"] == [{"id": "final", "score": 0.25, "passed": False, "weight": 1.0, + "skipped": False}] + assert grade["verifier_output"] == {"source_verifier_ids": ["fs", "judge"]} + assert grade["verifier_settings"] == {"scoring_rules": 1, "verifier_ids": ["fs", "judge"], + "score_aggregator": "weighted_average"} + + +def test_task_run_prints_an_aggregates_score_without_an_empty_results_block(local_stores, monkeypatch, tmp_path): + step, context = aggregate_two_sources(monkeypatch, tmp_path) + outcome(step, context) + + assert _format_verifier_output("final-vid", context.metadata["verifications"]["final-vid"]) == [ + "score (verifier_id=final-vid): 0.25"] + assert _format_verifier_output("fs", context.metadata["verifications"]["fs"])[0] == "verification (verifier_id=fs):" + + +STORED = [ # each step as main stored it + (RunContainer(id="tests", version=None, sandbox_name="h", container_name="c", command="bash t.sh", + reward_path="/r.txt", env_vars={"A": "1"}), + {"id": "tests", "type": "run_container_unit_tests_verifier", "version": None, "fail_task_on_error": True, + "sandbox_name": "h", "container_name": "c", "env_id": None, "command": "bash t.sh", "verifier_id": "tests", + "setup_commands": [], "timeout_sec": 300, "env_vars": {"A": "1"}, "result_paths": ["/r.txt"], + "reward_path": "/r.txt"}), + (VerifySandboxTaskStep(id="fs", version=None, verifier_id="v1", criteria=[]), + {"id": "fs", "type": "verify_sandbox", "version": None, "fail_task_on_error": True, "criteria": [], + "agent_name": "default-agent", "base_dir": "/app", "shell_timeout_seconds": 120, "score_aggregator": "all_pass", + "verifier_id": "v1"}), + (EnvOutcomeVerifierTaskStep(id="outcome", version=None, env_id="e", file_artifact_id="s", file_artifact_version=2, + verifier_id="outcome", prompt_id="p"), + {"id": "outcome", "type": "env_outcome_verifier", "version": None, "fail_task_on_error": True, + "verifier_id": "outcome", "env_id": "e", "file_artifact_id": "s", "file_artifact_version": 2, + "score_aggregator": "all_pass", "prompt_id": "p"}), + (AggregateVerifiersTaskStep(id="final", version=None, verifier_ids=["a", "b"], verifier_id="final-score"), + {"id": "final", "type": "aggregate_verifiers", "version": None, "fail_task_on_error": True, + "verifier_ids": ["a", "b"], "score_aggregator": "weighted_average", "verifier_id": "final-score"}), +] + - assert raised == RECORDED[name]["raised"] - assert entry == json.dumps(RECORDED[name]["entry"]) +@pytest.mark.parametrize("step, stored", STORED, ids=[stored["type"] for _, stored in STORED]) +def test_each_verifiers_stored_form_is_unchanged(step, stored): + assert step.to_dict() == stored + assert type(step).from_dict(stored).to_dict() == stored From f57fb719851494f86f1c72162ba9b0b3c3253c9b Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 18 Nov 2026 14:33:23 -0800 Subject: [PATCH 05/12] feat(verifiers): rubrics_verifier, always_on_agent_verifier and user_sim_judge write their named grades The rubrics family moves onto VerifierTaskStep, and its legacy entries stay as main wrote them (the golden tests recorded before the move pass unchanged, now with grade checks). - rubrics_verifier: `_judged` returns the verdict and writes nothing (it was `_grade`, whose name the base's grade builder takes); `_entries` and `_skipped` return what `_write_entries` and `_write_skipped_entries` wrote. A prompt error or a run with no screenshots stays graded 0 with its one row and the int score. `verify` gates the verdict and writes the `-trajectory-mistakes` entry itself, last. fail_task_when_not_passing fails a score under the step's min_passing_score (1.0 unless set), with today's message. Its settings record what the run graded with: each override it applies in place of the stored value, the judge model the run recorded (else the one it resolves), the agent it deploys as its judge, and `overridden`; never a key, header or base URL. - always_on_agent_verifier: its refusals and its additions to the entry (judge_reads, context_loss, assistant_harness) move into its `_judged`, on the verdict's output. - user_sim_judge: `verify` replaces `execute`; its three skips are real skips (they already recorded no score), its score stays its own 0/1 rule, and its gate fails the step once both are written. It takes score_aggregator and min_passing_score from the base; PINNED refuses a min_passing_score. Its settings leave out the overrides it ignores. The branch for a plan with no criteria, which plan_run never makes, is gone. Co-Authored-By: Claude Opus 5.5 --- .../verifiers/always_on_agent_verifier.py | 31 ++- .../task_steps/verifiers/rubrics_verifier.py | 178 ++++++++++-------- .../task_steps/verifiers/user_sim_judge.py | 72 +++---- .../verifiers/test_rubrics_family_grades.py | 147 +++++++++++++++ .../test_rubrics_subclasses_golden.py | 5 +- ...ics_verifier_fail_task_when_not_passing.py | 8 +- .../verifiers/test_rubrics_verifier_golden.py | 20 +- .../test_rubrics_verifier_staging_cleanup.py | 2 +- tst/util/verifier_golden.py | 25 ++- 9 files changed, 348 insertions(+), 140 deletions(-) create mode 100644 tst/unit/task_step/verifiers/test_rubrics_family_grades.py diff --git a/src/agent_env/task_step/task_steps/verifiers/always_on_agent_verifier.py b/src/agent_env/task_step/task_steps/verifiers/always_on_agent_verifier.py index 067822601..075d1f517 100644 --- a/src/agent_env/task_step/task_steps/verifiers/always_on_agent_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/always_on_agent_verifier.py @@ -63,6 +63,7 @@ ) from agent_env.task_step.task_steps.verifiers.rubrics_verifier import ( RUN_CONTEXT_PARTS, + RubricsVerdict, RubricsVerifierTaskStep, StagedTrajectory, ) @@ -233,35 +234,31 @@ def task_problems(self, steps: list[TaskStep]) -> list[str]: f"'{prompts[0].id}', which isn't an always_on step; grade it with rubrics_verifier"] return [] - async def execute(self, context: TaskStepContext) -> TaskStepContext: + async def _judged(self, context: TaskStepContext) -> RubricsVerdict: + """rubrics_verifier's verdict, refused for a run whose overrides cut the prompt or take the wakes from its judge; + then, for a staged run, what each judge call read of it (``judge_reads``); for a run with agent labels (the PSI + harness's), how many failed criteria the judge traced to a compaction (``context_loss``); and for every run, + the assistant's harness as the run records it (``assistant_harness``).""" if "grade_up_to_turn" in self.step_param_overrides(context): raise RuntimeError(f"Verifier '{self.verifier_id}': grade_up_to_turn can't cut an always_on step's " "prompt, whose conversation turns are not its wakes.") if self.use_trajectory and (context.metadata.get("user_overrides") or {}).get("use_agent_judge") is False: raise RuntimeError(f"Verifier '{self.verifier_id}': use_trajectory needs the agent judge; this run's " "use_agent_judge=False would grade without the wakes it was set to read.") - return await super().execute(context) - - async def _grade(self, context: TaskStepContext) -> TaskStepContext: - """rubrics_verifier's grade; then, for a staged run, what each judge call read of it (``judge_reads``); for a - run with agent labels (the PSI harness's), how many failed criteria the judge traced to a compaction (``context_loss``); and - for every run, the assistant's harness as the run records it (``assistant_harness``).""" token = _CONTEXT_LOSS_RUN.set(False) try: - context = await super()._grade(context) + verdict = await super()._judged(context) asked = _CONTEXT_LOSS_RUN.get() finally: _CONTEXT_LOSS_RUN.reset(token) - entry = (context.metadata.get("verifications") or {}).get(self.verifier_id) - if entry is None: - return context - if "trajectory_staging" in entry: - entry["trajectory_staging"]["judge_reads"] = await self._judge_reads(entry) + output = dict(verdict.output) + if "trajectory_staging" in output: + output["trajectory_staging"]["judge_reads"] = await self._judge_reads(output) if asked: - entry["context_loss"] = context_loss(entry.get("results") or []) - if harness := await self._assistant_harness(context, staged="trajectory_staging" in entry): - entry["assistant_harness"] = harness - return context + output["context_loss"] = context_loss(verdict.criteria) + if harness := await self._assistant_harness(context, staged="trajectory_staging" in output): + output["assistant_harness"] = harness + return dataclasses.replace(verdict, output=output) def _spec(self) -> JudgeOutputFormatSpec: """rubrics_verifier's format; for a run with agent labels (``_CONTEXT_LOSS_RUN``), its rows also take ``context_loss`` and diff --git a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py index 90527df3e..343dbb8d1 100644 --- a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py @@ -47,9 +47,10 @@ from agent_env.entity.scopes import FieldScope from agent_env.store.base import NotFoundError from agent_env.store.ids import is_local_id -from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency +from agent_env.task_step.task_step import TaskStep, TaskStepDependency from agent_env.task_step.task_steps.verifiers.run_context import Run, load_run_context, render_run_context from agent_env.task_step.task_steps.verifiers.scoring import ScoreAggregator, aggregate_score +from agent_env.task_step.task_steps.verifiers.verifier import VERIFICATIONS_KEY, Verification, VerifierTaskStep from agent_env.task_step.task_steps.sandbox_utils.sandbox_utils import ( agent_error_text, fetch_container_logs, @@ -339,6 +340,14 @@ class StagedTrajectory: record: dict = field(default_factory=dict) +@dataclass +class RubricsVerdict(Verification): + """A rubrics verdict, and for rubric_evidence_with_mistakes the trajectory_mistakes-shaped entry ``verify`` writes + under ``trajectory_mistakes_key`` beside it.""" + + mistakes_entry: Optional[dict] = None + + class JudgeTimeoutError(TimeoutError): """An agent judge ran out of ``judge_timeout_seconds``; ``trajectory_object_url`` is what it had done by then, when its trajectory could be fetched.""" @@ -447,9 +456,8 @@ def _as_of_turn(prompt_response: PromptResponse, turn: Optional[int]) -> PromptR ) -class RubricsVerifierTaskStep(TaskStep): +class RubricsVerifierTaskStep(VerifierTaskStep): type: ClassVar[str] = "rubrics_verifier" - step_role = StepRole.GRADER entity_refs = (EntityRef.agent("judge_a2a_agent_id"),) context_paths = ("context_facts_for_judge{}",) field_scopes = {**TaskStep.field_scopes, **dict.fromkeys( @@ -515,14 +523,17 @@ def __init__( depends_on: Optional[list[TaskStepDependency]] = None, fail_task_on_error: bool = True, fail_task_when_not_passing: bool = False, + min_passing_score: Optional[float] = None, ): - super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error) + super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error, + verifier_id=verifier_id or uuid.uuid4().hex, score_aggregator=score_aggregator, + min_passing_score=min_passing_score) if fail_task_when_not_passing and not fail_task_on_error: raise ValueError( "fail_task_when_not_passing needs fail_task_on_error: a tolerated failure lets the steps after it " "run and score the run." ) - # A verdict below 1.0 fails the step with VerifierNotPassingError once the verdict is recorded. + # A verdict below min_passing_score fails the step with VerifierNotPassingError once the verdict is recorded. self.fail_task_when_not_passing = fail_task_when_not_passing # RUN_CONTEXT_PARTS the judge also sees, cut at grade_up_to_turn like the trajectory. self.run_context_for_judge = _validated_run_context_parts(run_context_for_judge) @@ -561,16 +572,12 @@ def __init__( if isinstance(output_format, str): output_format = JudgeOutputFormat(output_format) self.output_format = output_format - self.verifier_id = verifier_id or uuid.uuid4().hex self.criteria = criteria self.prompt_id = prompt_id # The author's model, or None to judge with the run's config when the step runs (``_judge_model``), so a # stored task never carries the config of whoever wrote it. self.default_model = default_model self.default_model_api_base = default_model_api_base - if isinstance(score_aggregator, str): - score_aggregator = ScoreAggregator(score_aggregator) - self.score_aggregator = score_aggregator or ScoreAggregator.ALL_PASS self.agent_name = agent_name if isinstance(trajectory_filter, dict): trajectory_filter = TrajectoryFilter.from_dict(trajectory_filter) @@ -609,7 +616,7 @@ def _spec(self) -> JudgeOutputFormatSpec: def _writes_mistakes_entry(self) -> bool: """rubric_evidence_with_mistakes: the same judge call also detects trajectory mistakes and a second entry, shaped and scored as a standalone trajectory_mistakes step's, is written under - ``trajectory_mistakes_key(self.verifier_id)`` (see ``_write_entries``).""" + ``trajectory_mistakes_key(self.verifier_id)`` (see ``_entries``).""" return self.output_format is JudgeOutputFormat.RUBRIC_EVIDENCE_WITH_MISTAKES def _entry_format_fields(self) -> dict: @@ -624,19 +631,14 @@ def _mistakes_entry_fields(self) -> dict: """``format`` / ``source_verifier_id`` of the second entry written under ``trajectory_mistakes_key``.""" return {"format": JudgeOutputFormat.TRAJECTORY_MISTAKES.value, "source_verifier_id": self.verifier_id} - def _write_skipped_entries(self, context: TaskStepContext, row_id: str, message: str) -> None: - """Store a zero-score entry with one row saying why the run was not judged — under ``verifier_id`` - and, for rubric_evidence_with_mistakes, ``trajectory_mistakes_key(verifier_id)``.""" - def skipped(fields: dict) -> dict: - return { - **fields, - "results": [{"id": row_id, "score": 0, "result": False, "message": message}], - "score": 0, - } - verifications = context.metadata.setdefault("verifications", {}) - verifications[self.verifier_id] = skipped(self._entry_format_fields()) - if self._writes_mistakes_entry: - verifications[trajectory_mistakes_key(self.verifier_id)] = skipped(self._mistakes_entry_fields()) + def _skipped(self, row_id: str, message: str) -> RubricsVerdict: + """A zero score with one row saying why the run was not judged, graded rather than skipped so the score stays + where readers count it; for rubric_evidence_with_mistakes the second entry says the same.""" + def rows() -> list[dict]: + return [{"id": row_id, "score": 0, "result": False, "message": message}] + mistakes = ({**self._mistakes_entry_fields(), "results": rows(), "score": 0} + if self._writes_mistakes_entry else None) + return RubricsVerdict(criteria=rows(), score=0, output=self._entry_format_fields(), mistakes_entry=mistakes) def _judge_model(self, context: TaskStepContext) -> str: """The model this judge asks for in this run: the run's ``judge_model``, the step's own, then the config's judge @@ -682,7 +684,6 @@ def to_dict(self) -> dict: @classmethod def from_dict(cls, data: dict) -> RubricsVerifierTaskStep: - raw_agg = data.get("score_aggregator") raw_tf = data.get("trajectory_filter") return cls( **cls._base_from_dict(data), @@ -690,8 +691,6 @@ def from_dict(cls, data: dict) -> RubricsVerifierTaskStep: prompt_id=data["prompt_id"], default_model=data.get("default_model"), default_model_api_base=data.get("default_model_api_base"), - score_aggregator=ScoreAggregator(raw_agg) if raw_agg else None, - verifier_id=data.get("verifier_id"), agent_name=data.get("agent_name"), trajectory_filter=TrajectoryFilter.from_dict(raw_tf) if raw_tf else None, max_thinking_tokens=data.get("max_thinking_tokens"), @@ -764,13 +763,62 @@ def _resolved_criteria(self, context: TaskStepContext) -> list[dict]: step_overrides = self.step_param_overrides(context) return list(step_overrides["criteria"]) if "criteria" in step_overrides else self.criteria - async def execute(self, context: TaskStepContext) -> TaskStepContext: - context = await self._grade(context) - if self.fail_task_when_not_passing: - _raise_unless_passing(self.verifier_id, context.metadata["verifications"][self.verifier_id]) - return context - - async def _grade(self, context: TaskStepContext) -> TaskStepContext: + def verifier_settings(self, context: TaskStepContext) -> dict[str, Any]: + """The step's fields as this run grades with them: each run override it applies in place of the stored value, + the judge model the run recorded (else the one it resolves), the agent it deploys as its judge when it deploys + one, and ``overridden``, the settings the run's overrides changed. Never a key, header or base URL.""" + settings = super().verifier_settings(context) + applied = self._run_overrides(context) + overrides = context.metadata.get("user_overrides") or {} + overridden = [key for key, value in applied.items() if settings.get(key) != value] + if overrides.get("judge_model"): + overridden.append("judge_model") + settings.update(applied) + if settings.get("use_agent_judge") and self.agent_name is None and not self.judge_a2a_agent_id: + settings["judge_a2a_agent_id"] = get_config().get_default_a2a_agent_id() + recorded = (context.metadata.get("models") or {}).get(ModelRole.JUDGE.record_key) or {} + settings["judge_model"] = recorded[self.id] if self.id in recorded else self._judge_model(context) + return {**settings, "overridden": overridden} + + def _run_overrides(self, context: TaskStepContext) -> dict[str, Any]: + """The settings this run's ``step_params`` and ``user_overrides`` set, by the step's own field names, with the + values it grades with: what ``verifier_settings`` records in place of the stored ones.""" + step_params = self.step_param_overrides(context) + overrides = context.metadata.get("user_overrides") or {} + applied: dict[str, Any] = {key: step_params[key] for key in ("criteria", "grade_up_to_turn") if key in step_params} + if overrides.get("use_agent_judge") is not None: + applied["use_agent_judge"] = overrides["use_agent_judge"] + if (apply_filter := overrides.get("apply_trajectory_filter")) is not None: + applied["trajectory_filter"] = (self.trajectory_filter or TrajectoryFilter()).to_dict() if apply_filter else None + for override, key in (("judge_effort", "effort"), ("judge_max_thinking_tokens", "max_thinking_tokens")): + if overrides.get(override): + applied[key] = overrides[override] + for key in ("judge_batch_size", "judge_batch_concurrency"): + if overrides.get(key) is not None: + applied[key] = overrides[key] + if overrides.get("agent_sandbox") and self.agent_name is None: + applied["judge_sandbox_type"] = overrides["agent_sandbox"] + return applied + + async def verify(self, context: TaskStepContext) -> Verification: + verdict = self._gated(await self._judged(context)) + if verdict.mistakes_entry is not None: + context.metadata.setdefault(VERIFICATIONS_KEY, {})[trajectory_mistakes_key(self.verifier_id)] = ( + verdict.mistakes_entry) + return verdict + + def _gated(self, verdict: RubricsVerdict) -> RubricsVerdict: + """``verdict``, failing its step once recorded when ``fail_task_when_not_passing`` and it scores under + ``min_passing_score``, naming the rows that weren't met.""" + if not self.fail_task_when_not_passing or verdict.score >= self.min_passing_score: + return verdict + unmet = [str(row.get("id")) for row in verdict.criteria + if row.get("score", 1.0 if row.get("result") else 0.0) < 1] + return replace(verdict, raise_after=VerifierNotPassingError( + f"Verifier '{self.verifier_id}' isn't passing (score {verdict.score}); rows not met: {unmet}")) + + async def _judged(self, context: TaskStepContext) -> RubricsVerdict: + """What the judge found in this run, written by nothing: an override point, whose result ``verify`` gates.""" prompt_response = next((pr for pr in context.prompt_responses if pr.prompt_id == self.prompt_id), None) if prompt_response is None: raise StepContextError(f"PromptResponse with prompt_id='{self.prompt_id}' not found in context") @@ -790,9 +838,7 @@ async def _grade(self, context: TaskStepContext) -> TaskStepContext: if prompt_response.error_type: logger.warning(f"Skipping verification '{self.verifier_id}': prompt had error_type={prompt_response.error_type}") - self._write_skipped_entries( - context, "prompt_error", f"Skipped: prompt had error_type={prompt_response.error_type}") - return context + return self._skipped("prompt_error", f"Skipped: prompt had error_type={prompt_response.error_type}") run_context = None if self.run_context_for_judge: # The stored response, not the cut copy: the loader finds the prompt's place among the run's by identity. @@ -861,10 +907,8 @@ async def _grade(self, context: TaskStepContext) -> TaskStepContext: f"Skipping verification '{self.verifier_id}': output_format={fmt} needs " "per-criterion frames, but the trajectory holds no screenshots." ) - self._write_skipped_entries( - context, "no_screenshots", - f"Skipped: {fmt} needs per-criterion frames, but the trajectory holds no screenshots") - return context + return self._skipped( + "no_screenshots", f"Skipped: {fmt} needs per-criterion frames, but the trajectory holds no screenshots") # The constructor checked the stored filter; a run-time override can still drop it, leaving a # prompt that demands `frame` / `evidence_status` with nothing to cite. raise RuntimeError( @@ -1122,16 +1166,14 @@ async def grade(batch: list[dict]) -> tuple[list[dict], int, list[dict], Optiona judge_call_fields["graded_up_to_turn"] = grade_up_to_turn if staged is not None: judge_call_fields["trajectory_staging"] = staged.record - self._write_entries(context, verification_results, judge_call_fields, unjudged=unjudged) - return context - - def _write_entries(self, context: TaskStepContext, verification_results: list[dict], judge_call_fields: dict, - *, unjudged: Sequence[dict] = ()) -> None: - """Store the rubric entry under ``verifier_id`` and, for rubric_evidence_with_mistakes, the - trajectory-mistakes entry under ``trajectory_mistakes_key(verifier_id)``; ``judge_call_fields`` (URIs, - retries) go on both. The mistake rows are built before either entry is written, so a response they - cannot be derived from leaves neither behind. ``unjudged`` rows (``unjudged_rows``) join the rubric - entry's, after the judge's, and its score; the mistakes entry is the judge's alone.""" + return self._entries(verification_results, judge_call_fields, unjudged=unjudged) + + def _entries(self, verification_results: list[dict], judge_call_fields: dict, *, + unjudged: Sequence[dict] = ()) -> RubricsVerdict: + """The rubric verdict and, for rubric_evidence_with_mistakes, the trajectory-mistakes entry; + ``judge_call_fields`` (URIs, retries) go on both. The mistake rows are built first, so a response they cannot + be derived from leaves neither. ``unjudged`` rows (``unjudged_rows``) join the rubric verdict's, after the + judge's, and its score; the mistakes entry is the judge's alone.""" mistake_rows: Optional[list[dict]] = None if self._writes_mistakes_entry: if not isinstance(verification_results, ResultRows): @@ -1141,32 +1183,22 @@ def _write_entries(self, context: TaskStepContext, verification_results: list[di mistake_rows = trajectory_mistakes_rows_from_evidence(verification_results) verification_results.extend(unjudged) # in place, so a ResultRows keeps its reasoning and checks score = aggregate_score(verification_results, self.score_aggregator) - verifications = context.metadata.setdefault("verifications", {}) - verification_entry = { - **self._entry_format_fields(), - "results": list(verification_results), - "score": score, - **judge_call_fields, - } + output = {**self._entry_format_fields(), **judge_call_fields} if isinstance(verification_results, ResultRows): # rubric_evidence: the judge's `reasoning` and policy `checks` ride on the entry, not the rows. - verification_entry["reasoning"] = verification_results.reasoning - verification_entry["checks"] = verification_results.checks - verifications[self.verifier_id] = verification_entry + output["reasoning"] = verification_results.reasoning + output["checks"] = verification_results.checks logger.info(f"Verification '{self.verifier_id}' complete: {len(verification_results)} criteria, score={score}") + mistakes: Optional[dict] = None if mistake_rows is not None: mistakes_score = aggregate_score(mistake_rows, ScoreAggregator.WEIGHTED_AVERAGE) - mistakes_key = trajectory_mistakes_key(self.verifier_id) - verifications[mistakes_key] = { - **self._mistakes_entry_fields(), - "results": mistake_rows, - "score": mistakes_score, - **judge_call_fields, - } + mistakes = {**self._mistakes_entry_fields(), "results": mistake_rows, "score": mistakes_score, + **judge_call_fields} logger.info( - f"Verification '{mistakes_key}' complete (from '{self.verifier_id}'): " + f"Verification '{trajectory_mistakes_key(self.verifier_id)}' complete (from '{self.verifier_id}'): " f"{len(mistake_rows)} rows, score={mistakes_score}") + return RubricsVerdict(criteria=list(verification_results), score=score, output=output, mistakes_entry=mistakes) def _compact_trajectory( self, @@ -1830,13 +1862,3 @@ def is_rubrics_verifier(step_or_config: TaskStep | dict[str, Any]) -> bool: else: cls = type(step_or_config) return isinstance(cls, type) and issubclass(cls, RubricsVerifierTaskStep) - - -def _raise_unless_passing(verifier_id: str, entry: dict) -> None: - """Raise VerifierNotPassingError when the recorded verdict scores below 1.0, naming the rows that weren't met.""" - if entry["score"] >= 1: - return - unmet = [str(row.get("id")) for row in entry["results"] if row.get("score", 1.0 if row.get("result") else 0.0) < 1] - raise VerifierNotPassingError( - f"Verifier '{verifier_id}' isn't passing (score {entry['score']}); rows not met: {unmet}" - ) diff --git a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py index 944a08400..1bede3884 100644 --- a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py +++ b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py @@ -152,7 +152,9 @@ from agent_env.task_step.task_step import TaskStep, TaskStepDependency from agent_env.task_step.task_steps.verifiers import user_sim_judge_core as core from agent_env.task_step.task_steps.verifiers.judge_utils.judge_output_format import JudgeOutputFormat -from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep +from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerdict, RubricsVerifierTaskStep +from agent_env.task_step.task_steps.verifiers.scoring import ScoreAggregator +from agent_env.task_step.task_steps.verifiers.verifier import Verification logger = logging.getLogger(__name__) @@ -175,7 +177,8 @@ **dict.fromkeys(("criteria", "prompt_id", "default_model", "default_model_api_base", "agent_name", "trajectory_filter", "max_thinking_tokens", "effort", "judge_a2a_agent_id", "judge_sandbox_type", "grading_policy_prompt", "context_facts_for_judge", "grade_up_to_turn", "run_context_for_judge", - "judge_batch_size", "judge_batch_concurrency", "judge_timeout_retries", "judge_max_turns"))} + "judge_batch_size", "judge_batch_concurrency", "judge_timeout_retries", "judge_max_turns", + "min_passing_score"))} NOT_FAILING_OFF = "fail_task_when_not_passing is off: verdicts are recorded only" NOT_FAILING_UNCONFIRMED = ("the judge's copy of the user sim's instructions isn't confirmed to be the one it ran with, " "so it doesn't fail the task") @@ -228,6 +231,8 @@ def __init__( judge_timeout_seconds: Optional[int] = None, depends_on: Optional[list[TaskStepDependency]] = None, fail_task_on_error: bool = True, + score_aggregator: Optional[ScoreAggregator | str] = None, + min_passing_score: Optional[float] = None, ): """``run_step_id`` is required (it is a keyword with a default only so the refusal can say what's missing).""" if not isinstance(run_step_id, str) or not run_step_id.strip(): @@ -242,7 +247,8 @@ def __init__( id, version, criteria=[], prompt_id=None, default_model=judge_model, verifier_id=verifier_id, use_agent_judge=False, use_trajectory=False, output_format=JudgeOutputFormat.USER_SIM_CHECKS, judge_timeout_seconds=judge_timeout_seconds, depends_on=depends_on, fail_task_on_error=fail_task_on_error, - fail_task_when_not_passing=fail_task_when_not_passing is True) + fail_task_when_not_passing=fail_task_when_not_passing is True, score_aggregator=score_aggregator, + min_passing_score=min_passing_score) self.fail_task_when_not_passing = fail_task_when_not_passing # as given, so config_problems sees a non-bool self.run_step_id = run_step_id self.judge_model = judge_model # None: rubrics_verifier's default, the config's judge model @@ -313,19 +319,21 @@ def from_dict(cls, data: dict) -> "UserSimJudge": judge_model=data.get("judge_model"), rubric_items=data.get("rubric_items"), rubric_persona_sha256=data.get("rubric_persona_sha256"), fail_task_when_not_passing=data.get("fail_task_when_not_passing", True), - verifier_id=data.get("verifier_id"), judge_timeout_seconds=data.get("judge_timeout_seconds")) + judge_timeout_seconds=data.get("judge_timeout_seconds")) + + def _run_overrides(self, context: TaskStepContext) -> dict[str, Any]: + """The run overrides the judge applies: rubrics_verifier's but the criteria, which the plan replaces, the + trajectory filter, which reads no trajectory, and the batch concurrency and sandbox, which never change a + verdict; effort and thinking only for an agent judge, which alone takes them.""" + applied = {k: v for k, v in super()._run_overrides(context).items() + if k not in ("criteria", "trajectory_filter", "judge_batch_concurrency", "judge_sandbox_type")} + if not applied.get("use_agent_judge"): + applied.pop("effort", None) + applied.pop("max_thinking_tokens", None) + return applied # ---- the run ----------------------------------------------------------------------------------------------- - async def execute(self, context: TaskStepContext) -> TaskStepContext: - context = await self._grade(context) - error = context.metadata["verifications"][self.verifier_id]["user_sim_error"] - if error.get("fails_task"): - raise VerifierNotPassingError( - f"user_sim_judge '{self.id}': the user sim slipped in a way that could change the assistant's grade " - f"({', '.join(error['because'])}), so the run doesn't count; run it again") - return context - - async def _grade(self, context: TaskStepContext) -> TaskStepContext: + async def verify(self, context: TaskStepContext) -> Verification: problems = self.config_problems() if problems: raise ValueError(f"user_sim_judge '{self.id}': " + "; ".join(problems)) @@ -357,26 +365,20 @@ async def _grade(self, context: TaskStepContext) -> TaskStepContext: raise StepContextError(f"user_sim_judge '{self.id}': step '{step_id}' left no prompt response") if skipped: logger.warning(f"user_sim_judge '{self.id}': skipped, {skipped}") - context.metadata.setdefault("verifications", {})[self.verifier_id] = { - **self._entry_format_fields(), "results": [], "score": None, **entry, "skipped": skipped, - "user_sim_error": {**core.NOT_JUDGED, "fails_task": False}} - return context + return Verification(skipped=skipped, output={ + **self._entry_format_fields(), **entry, "user_sim_error": {**core.NOT_JUDGED, "fails_task": False}}) unusable = LINE_UNCHECKED if (entry.get("rubric_persona") or {}).get("status") == "unknown" else "" plan = core.plan_run(run, checks=list(core.CHECK_NAMES), items=items, error_checks=list(core.DEFAULT_ERROR_CHECKS), lines_unusable=unusable) judge = copy.copy(self) # this run's own copy: the step is shared by every run of the task judge.prompt_id, judge._run = response.prompt_id, _ThisRun(plan=plan, text=core.run_text(run)) - if plan.criteria: - await RubricsVerifierTaskStep._grade(judge, context) - else: # nothing for a model: only rows settled without one - judge._write_entries(context, [], {}, unjudged=plan.unjudged) - self._finish(context.metadata["verifications"][self.verifier_id], entry, judge._run) - return context - - def _finish(self, written: dict, entry: dict, this: _ThisRun) -> None: - """The written entry with its rows in the plan's order and shape, the run's verdict, and the run's details.""" - rows = core.finish_rows(written["results"], this.plan) + return self._finish(await judge._judged(context), entry, judge._run) + + def _finish(self, judged: RubricsVerdict, entry: dict, this: _ThisRun) -> Verification: + """The judge's verdict with its rows in the plan's order and shape, the run's verdict and details, and its + score: 0 when a row fails, else 1. A slip the gate counts fails the step once recorded.""" + rows = core.finish_rows(judged.criteria, this.plan) error = core.user_sim_error(rows) error["fails_task"] = False if error["value"] == "yes": @@ -386,16 +388,20 @@ def _finish(self, written: dict, entry: dict, this: _ThisRun) -> None: error["not_failing"] = NOT_FAILING_UNCONFIRMED else: error["fails_task"] = True - written.update(results=rows, score=0.0 if any(r.get("result") is False for r in rows) else 1.0, **entry, - user_sim_error=error, calls=this.calls, usage=core.usage_total(this.usage), - cost=round(this.cost, COST_DECIMALS)) + output = {**judged.output, **entry, "user_sim_error": error, "calls": this.calls, + "usage": core.usage_total(this.usage), "cost": round(this.cost, COST_DECIMALS)} if error["value"] == "unknown": - judged = [r for r in rows if r["id"] in core.CHECKS] - written["flag"] = FLAG_NONE_DECIDED if judged and all(r["verdict"] == "error" for r in judged) \ + checks = [r for r in rows if r["id"] in core.CHECKS] + output["flag"] = FLAG_NONE_DECIDED if checks and all(r["verdict"] == "error" for r in checks) \ else FLAG_UNDECIDED logger.info(f"user_sim_judge '{self.id}': {entry['run_kind']} '{entry['run_step_id']}' user_sim_error=" f"{error['value']} ({', '.join(error['because'] + error['unknown']) or 'no row'}), fails_task=" f"{error['fails_task']}; " + ", ".join(f"{r['id']}={r['verdict']}" for r in rows)) + slipped = VerifierNotPassingError( + f"user_sim_judge '{self.id}': the user sim slipped in a way that could change the assistant's grade " + f"({', '.join(error['because'])}), so the run doesn't count; run it again") if error["fails_task"] else None + return Verification(criteria=rows, score=0.0 if any(r.get("result") is False for r in rows) else 1.0, + output=output, raise_after=slipped) def _find_run(self, context: TaskStepContext) -> tuple[str, str]: """The kind of run ``run_step_id`` left and its step id: always_on when it left an always_on record, else a diff --git a/tst/unit/task_step/verifiers/test_rubrics_family_grades.py b/tst/unit/task_step/verifiers/test_rubrics_family_grades.py new file mode 100644 index 000000000..fdc6b44a7 --- /dev/null +++ b/tst/unit/task_step/verifiers/test_rubrics_family_grades.py @@ -0,0 +1,147 @@ +"""The rubrics family's named grades beyond what their legacy entries say: the settings a run grades with, the gate's +threshold, and the base fields each takes or refuses.""" + +from __future__ import annotations + +import json + +import pytest + +from agent_env.config import get_config +from agent_env.task_step.context import TaskStepContext, VerifierNotPassingError +from agent_env.task_step.task_steps.verifiers.always_on_agent_verifier import AlwaysOnAgentVerifier +from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep +from agent_env.task_step.task_steps.verifiers.user_sim_judge import UserSimJudge +from tst.unit.task_step.verifiers.test_rubrics_verifier_golden import ( + CASES, + _binary, + _context, + _criteria, + _direct_judge, + _step, +) + +OVERRIDE = [{"id": "only", "description": "the run's own criterion"}] + + +async def _grade(step, context: TaskStepContext) -> dict: + try: + await step.execute(context) + except VerifierNotPassingError: + pass + return context.metadata["grades"][step.id] + + +@pytest.mark.asyncio +async def test_the_settings_are_what_the_run_graded_with_and_say_what_its_overrides_changed(): + step = _direct_judge(_step(criteria=_criteria(2)), lambda ids, n: _binary(ids)) + context = _context({"judge_model": "judge-x", "judge_effort": "high", "judge_batch_size": 1, + "apply_trajectory_filter": False, "judge_litellm_api_key": "sk-run-key", + "litellm_base_url": "http://llm.example.test", "step_params": {"verify": {"criteria": OVERRIDE}}}) + + grade = await _grade(step, context) + + settings = grade["verifier_settings"] + assert (settings["criteria"], settings["judge_model"], settings["effort"], settings["judge_batch_size"]) == ( + OVERRIDE, "judge-x", "high", 1) + assert settings["trajectory_filter"] is None and settings["default_model"] == "judge-model" + assert settings["overridden"] == ["criteria", "effort", "judge_batch_size", "judge_model"] + assert "sk-run-key" not in json.dumps(grade) and "llm.example.test" not in json.dumps(grade) + + +@pytest.mark.asyncio +async def test_a_run_without_overrides_records_the_stored_settings_and_the_model_it_resolved(): + step, context = CASES["binary"]() + + settings = (await _grade(step, context))["verifier_settings"] + + assert settings["overridden"] == [] and settings["judge_model"] == "judge-model" + assert settings["criteria"] == step.criteria and settings["score_aggregator"] == "all_pass" + + +@pytest.mark.asyncio +async def test_an_agent_judge_that_takes_no_model_is_recorded_as_choosing_its_own(): + step, context = CASES["an_agent_judge_timing_out_once"]() + + settings = (await _grade(step, context))["verifier_settings"] + + assert settings["judge_model"] is None and settings["agent_name"] == "judge" + + +def test_a_judge_the_step_deploys_is_recorded_by_the_agent_and_sandbox_it_runs_on(): + step = _step(use_agent_judge=True) + + settings = step.verifier_settings(_context({"agent_sandbox": "modal"})) + + assert settings["judge_a2a_agent_id"] == get_config().get_default_a2a_agent_id() + assert settings["judge_sandbox_type"] == "modal" and settings["overridden"] == ["judge_sandbox_type"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("min_passing_score", "raises"), [(None, True), (0.5, False), (0.75, True)]) +async def test_the_gate_fails_a_score_under_the_steps_min_passing_score(min_passing_score, raises): + step = _direct_judge(_step(score_aggregator="weighted_average", fail_task_when_not_passing=True, + min_passing_score=min_passing_score), lambda ids, n: _binary(ids, failing={"c2"})) + context = _context() + + if raises: + with pytest.raises(VerifierNotPassingError, match=r"isn't passing \(score 0.5\); rows not met: \['party-size'\]"): + await step.execute(context) + else: + await step.execute(context) + + grade = context.metadata["grades"]["verify"] + assert (grade["score"], grade["passed"]) == (0.5, not raises) + assert context.metadata["verifications"]["vid"]["score"] == 0.5 + + +def test_a_min_passing_score_is_stored_only_when_set_and_round_trips(): + assert "min_passing_score" not in _step().to_dict() + stored = _step(min_passing_score=0.5).to_dict() + assert stored["min_passing_score"] == 0.5 + again = RubricsVerifierTaskStep.from_dict(stored) + assert (again.min_passing_score, again.verifier_id, again.score_aggregator.value) == (0.5, "vid", "all_pass") + assert AlwaysOnAgentVerifier.from_dict({**stored, "type": AlwaysOnAgentVerifier.type}).min_passing_score == 0.5 + + +def test_a_stored_rubrics_verifier_without_a_verifier_id_still_gets_one_of_its_own(): + stored = {k: v for k, v in _step().to_dict().items() if k != "verifier_id"} + + assert RubricsVerifierTaskStep.from_dict(stored).verifier_id not in ("verify", None) + + +def _user_sim_judge(**kw) -> UserSimJudge: + return UserSimJudge.from_dict({"id": "judge_user_sim", "type": UserSimJudge.type, "run_step_id": "talk", + "judge_model": "fake/judge", "verifier_id": "usj", **kw}) + + +def test_the_user_sim_judge_takes_its_pinned_aggregator_and_refuses_a_min_passing_score(): + assert _user_sim_judge(score_aggregator="all_pass").score_aggregator.value == "all_pass" + with pytest.raises(ValueError, match="min_passing_score can't be set on the user sim judge"): + _user_sim_judge(min_passing_score=0.5) + + +def test_the_user_sim_judges_settings_are_only_the_overrides_it_applies(): + context = TaskStepContext(metadata={"user_overrides": { + "judge_model": "judge-x", "judge_effort": "high", "apply_trajectory_filter": True, "judge_batch_size": 2, + "judge_batch_concurrency": 3, "step_params": {"judge_user_sim": {"criteria": OVERRIDE}}}}) + + settings = _user_sim_judge().verifier_settings(context) + + assert settings["judge_model"] == "judge-x" and settings["judge_batch_size"] == 2 + assert not {"criteria", "trajectory_filter", "effort", "judge_batch_concurrency"} & set(settings) + assert settings["overridden"] == ["judge_batch_size", "judge_model"] + + +@pytest.mark.asyncio +async def test_an_always_on_verifier_refusing_a_runs_overrides_fails_its_grade_and_writes_no_entry(): + step = AlwaysOnAgentVerifier(id="judge", version=1, criteria=_criteria(1), prompt_id="ask", use_agent_judge=False, + verifier_id="rubric") + context = TaskStepContext(metadata={"user_overrides": {"step_params": {"judge": {"grade_up_to_turn": 1}}}}) + + with pytest.raises(RuntimeError, match="grade_up_to_turn can't cut an always_on step's prompt"): + await step.execute(context) + + assert context.metadata["grades"]["judge"]["verifier_status"] == "failed" + assert context.metadata["grades"]["judge"]["verifier_settings"]["overridden"] == ["grade_up_to_turn"] + assert "verifications" not in context.metadata diff --git a/tst/unit/task_step/verifiers/test_rubrics_subclasses_golden.py b/tst/unit/task_step/verifiers/test_rubrics_subclasses_golden.py index f650bf9bc..828e8e8b8 100644 --- a/tst/unit/task_step/verifiers/test_rubrics_subclasses_golden.py +++ b/tst/unit/task_step/verifiers/test_rubrics_subclasses_golden.py @@ -1,5 +1,5 @@ -"""user_sim_judge and always_on_agent_verifier leave the legacy entries main left, with their own tests' fakes standing -in for the judge and the runs they grade.""" +"""user_sim_judge and always_on_agent_verifier leave the legacy entries main left, and a named grade that agrees with +them, with their own tests' fakes standing in for the judge and the runs they grade.""" from __future__ import annotations @@ -26,6 +26,7 @@ async def _ran(step, context: TaskStepContext) -> dict: await step.execute(context) except Exception as exc: raised = exc + verifier_golden.check_grade(context, step, raised) return verifier_golden.observed(context, raised) diff --git a/tst/unit/task_step/verifiers/test_rubrics_verifier_fail_task_when_not_passing.py b/tst/unit/task_step/verifiers/test_rubrics_verifier_fail_task_when_not_passing.py index 88183a577..6fa0edc57 100644 --- a/tst/unit/task_step/verifiers/test_rubrics_verifier_fail_task_when_not_passing.py +++ b/tst/unit/task_step/verifiers/test_rubrics_verifier_fail_task_when_not_passing.py @@ -140,13 +140,13 @@ async def test_a_retry_config_does_not_grade_the_run_again(monkeypatch, local_st v.retry_config = RetryConfig(retry_from_step_id="validity", max_retries=2) _judge(monkeypatch, v, {"world": False, "replies": True}) graded = [] - grade = v._grade + judged = v._judged - async def counting_grade(context): + async def counting_judged(context): graded.append(context) - return await grade(context) + return await judged(context) - monkeypatch.setattr(v, "_grade", counting_grade) + monkeypatch.setattr(v, "_judged", counting_judged) with pytest.raises(VerifierNotPassingError): await Task(id="t", version=None, steps=[v]).run(context=_context()) diff --git a/tst/unit/task_step/verifiers/test_rubrics_verifier_golden.py b/tst/unit/task_step/verifiers/test_rubrics_verifier_golden.py index 46a20573f..72e6c31d5 100644 --- a/tst/unit/task_step/verifiers/test_rubrics_verifier_golden.py +++ b/tst/unit/task_step/verifiers/test_rubrics_verifier_golden.py @@ -1,5 +1,6 @@ -"""rubrics_verifier leaves the legacy entries main left on every path that writes one, or none, with the judge faked -below the seams the verifier's own code owns (``_prompt_llm_judge``, ``_invoke_judge_a2a``).""" +"""rubrics_verifier leaves the legacy entries main left on every path that writes one, or none, and a named grade that +agrees with them, with the judge faked below the seams the verifier's own code owns (``_prompt_llm_judge``, +``_invoke_judge_a2a``).""" from __future__ import annotations @@ -11,13 +12,18 @@ import pytest from agent_env.task_step.context import PromptResponse, TaskStepContext +from agent_env.task_step.task_steps.verifiers.judge_utils.judge_output_format import JudgeOutputFormat from agent_env.task_step.task_steps.verifiers.judge_utils.trajectory_filter import ( CompactionType, FrameStrategy, TrajectoryFilter, TrajectoryText, ) -from agent_env.task_step.task_steps.verifiers.rubrics_verifier import JudgeTimeoutError, RubricsVerifierTaskStep +from agent_env.task_step.task_steps.verifiers.rubrics_verifier import ( + JudgeTimeoutError, + RubricsVerifierTaskStep, + trajectory_mistakes_key, +) from tst.util import verifier_golden FILE = "rubrics_verifier.json" @@ -184,7 +190,7 @@ async def judge(*, eval_prompt, judge_a2a_url, judge_agent_card, judge_agent): @pytest.mark.asyncio @pytest.mark.parametrize("case", list(CASES)) -async def test_a_run_leaves_the_legacy_entries_main_left(case): +async def test_a_run_leaves_the_legacy_entries_main_left_and_a_grade_that_agrees(case): step, context = CASES[case]() raised = None try: @@ -193,3 +199,9 @@ async def test_a_run_leaves_the_legacy_entries_main_left(case): raised = exc verifier_golden.check(FILE, case, verifier_golden.observed(context, raised)) + verifier_golden.check_grade(context, step, raised) + verifications = context.metadata.get("verifications") or {} + if step.verifier_id in verifications: + assert (trajectory_mistakes_key(step.verifier_id) in verifications) is ( + step.output_format is JudgeOutputFormat.RUBRIC_EVIDENCE_WITH_MISTAKES) + assert set(context.metadata["grades"]) == {step.id} diff --git a/tst/unit/task_step/verifiers/test_rubrics_verifier_staging_cleanup.py b/tst/unit/task_step/verifiers/test_rubrics_verifier_staging_cleanup.py index 4de0cad10..68e8c20b8 100644 --- a/tst/unit/task_step/verifiers/test_rubrics_verifier_staging_cleanup.py +++ b/tst/unit/task_step/verifiers/test_rubrics_verifier_staging_cleanup.py @@ -51,6 +51,6 @@ async def deploy(self, **kwargs): agent_trajectory_object_url="mem://run/turn.json")) with pytest.raises(ConnectionError): - await verifier._grade(ctx) + await verifier._judged(ctx) assert terminated == ["sb-judge"] diff --git a/tst/util/verifier_golden.py b/tst/util/verifier_golden.py index 6eda709c4..d03304c4d 100644 --- a/tst/util/verifier_golden.py +++ b/tst/util/verifier_golden.py @@ -9,7 +9,8 @@ from pathlib import Path from typing import Any, Optional -from agent_env.task_step.context import TaskStepContext +from agent_env.task_step.context import TaskStepContext, VerifierNotPassingError +from agent_env.task_step.task_steps.verifiers.verifier import VerifierTaskStep GOLDEN = Path(__file__).parents[1] / "data" / "verifier_golden" @@ -28,3 +29,25 @@ def check(file: str, case: str, got: dict[str, Any]) -> None: expected = _recorded(file)[case] assert json.loads(json.dumps(got)) == expected assert json.dumps(got, sort_keys=True) == json.dumps(expected, sort_keys=True) + + +def check_grade(context: TaskStepContext, step: VerifierTaskStep, raised: Optional[BaseException]) -> None: + """The step's named grade agrees with its legacy entry: failed when there is none, skipped when the entry is, + else graded at the entry's score with its rows and the rest of the entry as its output; a gate fails the step + only once both are written.""" + grade = context.metadata["grades"][step.id] + entry = (context.metadata.get("verifications") or {}).get(step.verifier_id) + if entry is None: + assert (grade["verifier_status"], grade["score"], grade["passed"]) == ("failed", None, False) + assert grade["verifier_error_reasoning"] == f"{type(raised).__name__}: {raised}" + return + assert grade["verifier_output"] == {k: v for k, v in entry.items() if k not in ("results", "score", "skipped")} + if "skipped" in entry: + assert (grade["verifier_status"], grade["score"], grade["passed"]) == ("skipped", None, False) + assert grade["verifier_skip_reasoning"] == entry["skipped"] + return + assert grade["verifier_status"] == "graded" and grade["score"] == float(entry["score"]) + assert grade["passed"] is (entry["score"] >= step.min_passing_score) + assert [row["id"] for row in grade["verifier_criteria"]] == [row["id"] for row in entry["results"]] + if isinstance(raised, VerifierNotPassingError): + assert not grade["passed"] From 14e4bddf9a1c99109f536ada800579a14ed753e6 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 18 Nov 2026 14:39:04 -0800 Subject: [PATCH 06/12] fix(verifiers): a run naming the step's own judge model overrides nothing Co-Authored-By: Claude Opus 5.5 --- .../task_step/task_steps/verifiers/rubrics_verifier.py | 2 +- .../task_step/verifiers/test_rubrics_family_grades.py | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py index 343dbb8d1..c863f9f9c 100644 --- a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py @@ -771,7 +771,7 @@ def verifier_settings(self, context: TaskStepContext) -> dict[str, Any]: applied = self._run_overrides(context) overrides = context.metadata.get("user_overrides") or {} overridden = [key for key, value in applied.items() if settings.get(key) != value] - if overrides.get("judge_model"): + if overrides.get("judge_model") and overrides["judge_model"] != self.default_model: overridden.append("judge_model") settings.update(applied) if settings.get("use_agent_judge") and self.agent_name is None and not self.judge_a2a_agent_id: diff --git a/tst/unit/task_step/verifiers/test_rubrics_family_grades.py b/tst/unit/task_step/verifiers/test_rubrics_family_grades.py index fdc6b44a7..802fbaaa8 100644 --- a/tst/unit/task_step/verifiers/test_rubrics_family_grades.py +++ b/tst/unit/task_step/verifiers/test_rubrics_family_grades.py @@ -59,6 +59,15 @@ async def test_a_run_without_overrides_records_the_stored_settings_and_the_model assert settings["criteria"] == step.criteria and settings["score_aggregator"] == "all_pass" +@pytest.mark.asyncio +async def test_a_run_naming_the_steps_own_judge_model_overrides_nothing(): + step = _direct_judge(_step(criteria=_criteria(1)), lambda ids, n: _binary(ids)) + + settings = (await _grade(step, _context({"judge_model": "judge-model"})))["verifier_settings"] + + assert settings["overridden"] == [] and settings["judge_model"] == "judge-model" + + @pytest.mark.asyncio async def test_an_agent_judge_that_takes_no_model_is_recorded_as_choosing_its_own(): step, context = CASES["an_agent_judge_timing_out_once"]() From 52807977ab0fa7dc6bd951daff70106bce1fb5ee Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 18 Nov 2026 14:40:05 -0800 Subject: [PATCH 07/12] chore(verifiers): every verifier golden file under tst/data, and AGENTS.md says every built-in is on the base Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- .../recorded => data/verifier_golden}/legacy_entries.json | 0 tst/unit/task_step/verifiers/test_legacy_entries.py | 8 ++++---- 3 files changed, 5 insertions(+), 5 deletions(-) rename tst/{unit/task_step/verifiers/recorded => data/verifier_golden}/legacy_entries.json (100%) diff --git a/AGENTS.md b/AGENTS.md index 02995ff96..6d323202b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -125,7 +125,7 @@ and deserialized through a registry. |---|---| | `env/` | `Env` base class: a subclass declares `type`, implements `from_dict` and `async deploy(...) -> DeployedEnv`, and may override `reset`, `close` and `load_file_artifact_universe`; teardown closes each env a run deployed before terminating its sandboxes by id. Built-ins in `env/envs/`: `MCPServerEnv` (`mcp_server`), `MultiEnv` (`multi`, MCP servers plus websites and computers; its website browser is the one its `browser_envs` names, none for `[]`, else the run's when it has websites: an MCP server env such as `playwright-mcp` runs among the gateway's MCP servers, and a browser computer such as `chromium` on a sandbox of its own as one more computer child, sized by its `child_envs` entry, its tools under its name and `_`, routing the multi env's synthetic sites unless that entry's `site_routing` says, its network its own env's, as any computer child's; a website env deployed alone refuses a browser computer as the run's browser), `ServiceDBEnv` (`service_db`, PostgreSQL), `WebsiteEnv` (`website`, one site: a server image, `backend_image` in its env.toml, serving the pages and API, ready when its card's `website/v1` `ready` answers a `WebsiteReadiness` that says so, as `agentenv_protocol.website` (`website_readiness`, `env_readiness`) defines a website env's health for the deploy and `DeployedEnv.readiness()` alike, or, for a card without `ready`, when its `health` path answers 200, which is deprecated; its env.toml's `ready_timeout_s`, how long it may take from its server's container starting to ready, is that container's health check `start_period` and what the deploy waits, or its card's `ready` timeout if longer, and its `shm_mb` is the container's `shm_size`, each left to the gateway and Docker without them; the gateway container's `env/gateway/constants.py` mirrors the URI and op, a test keeping them equal; a separate frontend image is deprecated, warned of at put and optional in a bundle, and a stored `backend_docker_image_artifact` reads as the server image; a live one, `live = true`, is a real site whose server serves none of its pages, its `live`, `domains` and `allow_hosts` read at put from its server's `@website(...)` (`utils/card_naming.py` `WebsiteSource`) unless given, its `settings`' secrets only as references the gateway resolves as it writes the site's container (`WEBSITE_SETTINGS_ENV`); the gateway gives it no site route, a computer's site table leaves it out, a computer whose network is an allowlist is let reach its names (`MultiEnv.live_hosts`), a browser alone's forward port takes them straight to the real site (`site_proxy.py` `direct`), as the browser base's stand-in proxy takes any host, refusing nothing, each proxy logging each such connection as a JSON line, and a run that deploys one is marked live, `reproducible: false`), `GatewayEnv` (`gateway_server`), `ComputerDriverEnv` (`computer_driver`, `env/envs/computer/driver_env.py`: a computer driver's files and what its class declares for each platform it supports, read at put in a process of its own, `agentenv_protocol.computer._declare`, where the driver's own dependencies needn't be installed, with the `port` it serves on and its install by OS, `[install.]`, or for one distribution `[install."/"]`, which replaces its OS type's there, as its `known_failures` do (`env/envs/computer/utils/install.py`): files, apt `packages` with extra `sources`, a `python` requirements file for a venv, `downloads` by URL and sha256, `run` scripts, the services the base's init runs, and `ports`; it deploys only as a computer env's driver), `ComputerEnv` (`computer`, in `env/envs/computer/`: a base computer and a driver, matched at put by gate 1 before anything is stored, then composed by `composer.py` into an image for each form and arch on a builder of the put's sandbox provider, by default the env's (`utils/builder.py`; `--sandbox` on `env build`): this machine's Docker for `local`, else a VM of the provider, which fetches the put's stored files by signed URL, takes the rest inline and pushes to the image store itself, so it builds only with stores a remote VM reaches; each arch goes to the first link that runs it natively, else one that emulates it (a VM registers QEMU; `--no-emulate` leaves it `skipped` instead, for a later put), and its pair records the provider that built it (`builder`); a run or register that reuses a computer env composes what it has left (`pending`), a run, and a register given a `--sandbox`, only the forms its sandbox's deploys boot (`booted_forms`: on each link, the first form it boots natively, by the deploy's own rule, `boot.first_native`; a compose of a disk adds its container), and a deploy composes first what it boots that no compose started, and a container it runs natively that was built but not probed, as one whose probe a setting that didn't resolve stopped, booting it only once ready (`ComputerEnv.compose_unstarted`); tagged `agentenv-computers:` in the image store by a key of the image's Dockerfile, build context and lock so any env with the same inputs reuses it; every layer's packages, requirements and downloads are resolved at put in a container of the base for each arch (`utils/packages.py`, `utils/resolve_in_base.py`), stored in the object store by sha256 under `agentenv-packages/`, and listed in a lock stored by a digest of its inputs, which any env with the same inputs reuses, and builds install only the lock's files, every install step `RUN --network=none`; each pair's status goes into its metadata, `composed`, the arches built are written as its computer, a `computer` artifact over one index, and `agent-env env build` resumes what a put left; gate 3 (`utils/probing.py`) then boots each native build on the put's sandbox provider and probes it at `full`, making it `ready` or `failed`, and a build no provider runs natively stays `unverified`. Its deploy, through the `computer` env provider (`providers/env_providers/env_computer_provider.py`), boots its first `ready` build on a sandbox provider that runs its arch natively (`native_platforms()`), with no gateway in front and the driver's card as the env card, its `/dev/shm` the most its apps' Linux installs ask for (`shm_mb`, in MiB, which its composed computer and the deploy record keep; `--shm-size` on a container, `AGENTENV_SHM_MB` read by a Linux disk's init, passed only to a provider whose `SHARED_MEMORY` says it gives that much, else the sandbox's default, which the deploy logs), then checks its `display`; `deploy_env` `env_params` take a `form`, `computer_settings` (deploy-phase values of the base's settings; a secret one, here or on the env, only as a bare `secret:`/`env:` reference) and `driver_settings` (the driver's `ComputerSettings`, such as its default coordinate space); a `network` that allows nothing runs it on local Docker's internal network behind a relay (`providers/sandbox_providers/local_docker/relay/`); `reset` and `close` have the driver end its recordings first (`computer-recording/v1` `end_all`, each uploading what's left and its manifest under the grant it started with), and `reset` then boots a fresh container in its place (`reset_container`), keeping the computer's address but not an agent's MCP session to the old driver; its `load_file_artifact_universe` writes files under the session user's home through the driver's `data/add`, `Downloads` by default, each file's bytes through a read grant when the store's grants reach it and its network allows everything, else inline (`env/envs/computer/utils/inside.py`)), `ComputerAppEnv` (`computer_app`, `env/envs/computer/app_env.py`: an app on a computer, a `ComputerAppEnvironment` class its env.toml's `module` names, read at put by the same `_declare`, which declares each OS with its own decorator (`@linux(...)`, or `@install(os_type, spec)` for any OS type; `@linux(distribution=...)` or `@install("/", spec)` for one distribution, which replaces its OS type's there) holding its install, its launch and, optionally, a skill and settings and build settings that add to the class's there, and install steps and ready checks for every OS or those their `os=` names; everything that reads its settings, launch or steps asks for the computer's OS type; deployed alone it is a one-app computer env, a `ComputerEnv` of the computer and driver its env.toml names, the composer installing it under `/opt/agentenv/apps//` with its services as supervisord programs beside its own server (`agentenv_protocol.computer.serve_app`, in a venv with the protocol's `app` extra, on `.agentenv/server.sock` in its folder), which adds its launcher, launches a `boot` app once per boot and says through `computer-app/v1` whether it's ready, as the driver's `computer-ready` asks, its card named after it, and `env_params` `app_settings` handing it values of its `Settings` in `AGENTENV_APP_SETTINGS`, a secret one only as a bare reference, over its env.toml's `[settings]` and a computer env's `settings` beside it in `apps`, each by top-level field (`params.app_settings`; checked at put, a required one left to the deploy; the deploy record keeps the merged values as `app_settings`, secrets withheld); its env.toml's `build_settings`, values of its `BuildSettings` checked at put, are written to `.agentenv/build_settings.json` in its folder, so they're part of the image's key, its install steps read them, and a deploy can't change them). An env type whose services take environment data defines `load_environment_artifact` and `load_environment_universe_artifact`; `load_artifact` refuses an environment artifact aimed at a type that doesn't, at save and before it loads. An env built on the protocol SDK can serve agents a skill (`@agent_skill`, `skill/v1`), which `add_skills` `env_ids` installs under its environment name (behind a gateway, each server's under that server's name); a skill with files beside its SKILL.md goes as a bundle stored once per content under `agentenv-env-skills/` in the object store. `env/gateway/gateway.py` is the env-side gateway server, container-only code. Its site route, `/website//`, never forwards a site's data plane or card (`/agentenv/…`, `/.well-known/agent-env.json`), which agentenv reaches through `/svc//`; a site with no frontend image is served by its server image there. A request from a computer's site proxy names its computer (`AgentEnv-Computer`, sent from the proxy's private upstream headers, a page's own copy and any `AgentEnv-Role` it sends dropped) and takes the role driving that computer now (the gateway's computers being those `COMPUTERS` names, a multi env's, with or without a control token): the last to make an input call to its tools, as the call starts, through `/mcp`, `/step` or a trigger action under its `as` (`Gateway._drive`); without one, or from a computer the gateway doesn't front, it is `default`, the role read once per request. `changes/v1` (`ChangeAttribution`) counts no write window for a call to a computer's tools, a trigger action's included, so a write the computer's browser made goes to its request and one another party made during the call to that party. Its CDP route, `/browser//cdp//` (`env/gateway/browser_proxy.py`), reaches the website browser behind the `/svc` child ``, found from that child's card (it, or an app among its children, offering `browser/v1` `cdp` with `/cdp/` in its `computer-app/v1` `agent_routes`): it relays the browser's discovery and WebSocket both ways, each leg pinged every 20 s, and every CDP address the browser hands out, in `json/version`, `json/list` and its `cdp` answer through `/svc//`, comes back at the gateway, everything before `/cdp//` becoming the address the caller reached the gateway at and `/browser/`, keeping the https or wss of an address the browser minted under an origin its caller named, as `browser_connect_cdp` names the agent's, since a tunnel that ends TLS, as Modal's, may send no `X-Forwarded-Proto`; each connection is a `browser_connection` trajectory event, without its token. Its clock (`env/gateway/clock.py`) has one opaque version, a per-boot id and a count, that every `set_time`, `advance` and `clear` changes; `set_rate` (`/clock/set-rate`) changes an armed clock's rate from its time at the write, keeping its `t0` and generation as `advance` does, so time triggers keep their marks. Every clock change goes through `Gateway._change_clock`, which wakes the triggers when the change kept the clock's generation, answers the held watches and waits for the followers. Its followers, the computers on it (`CLOCK_FOLLOWER_TOKENS`, each child whose card serves `clock/v1` `sync_time`, by its read token, its control token for now), long-poll `/clock/watch` naming the version they show, each watch held up to the card's `hold_s` (`env/gateway/clock_followers.py`), and a write given `wait_followers_s` waits for each, answering 504 naming those behind, but not for one that hasn't watched for longer than a hold and a reconnect (`ClockFollowers.QUIET_S`), which it names at once with when each last watched; a restarted gateway is unarmed. While its clock is armed it serves agents `get_time` (`env/gateway/get_time.py`), which answers in UTC and also in the universe's zone, the one `sync_env_clock`'s `timezone` armed it in, or in an IANA `time_zone` it's given; executors' working `hours` (`env/gateway/triggers.py`) take a fixed `utc_offset` or an IANA `time_zone`. | | `artifact/` | Immutable versioned artifacts; every `put()` creates a new version. The document goes to the document store, binary payloads to the object store. Every artifact but an environment universe, whose `metadata` key holds its legacy metadata file ids, carries a `metadata` dict outside its versioned fields, changed by compare-and-swap (`merge_metadata`). Built-ins: `CliArtifact` (`cli`), `ComputerArtifact` (`computer`, a base computer: its architectures, OS, session user, settings and container form, whose image `put` reads from its registry into `metadata.validated_images`), `FileArtifact` (`file`), `FileArtifactUniverse` (`file_artifact_universe`), `DockerImageArtifact` (`docker_image`), `EnvironmentArtifact` (`environment`: its environment's files by path, stored as the one wrapped `file_artifact_ref` every environment artifact had when it has one file at its filename, else as `file_artifact_refs`; `get_file_artifacts()` reads either, and a load hands each file to the env's `data/add` with its path, which a pre-v1 load, taking one file, refuses), `EnvironmentUniverseArtifact` (`environment_universe`), `SkillArtifact` (`skill`); `[artifacts] type_aliases` maps another stored spelling of a type onto one of them. A `DockerImageArtifact` holds a tar.gz of its image, or, written by `put_ref` (`--image-ref` on `env mcp-server put` and `a2a-agent put`), only a registry reference with its tag pinned to a digest, which sandboxes pull, or a published image's tag with its `image_digest` and its build context, which sandboxes pull by the digest and build from the context when they can't (the infra images, and a bundle agent built from a shipped agent's sources, `Plan.published`), or, written by `put_context`, only its build context, which a VM sandbox builds before running it (`VmSandbox.load_docker_images`) and Modal builds before creating its container (`ModalSandboxProvider.prepare_image`, outside a fallback chain's deadline); anything else that runs images by name (a container on the local provider, a lone server on any provider but Modal) refuses such an image. A build context (`utils/docker/context.py`) is what `docker build` would send, `.dockerignore` applied, written so equal files give equal bytes and an equal `source_digest`. | -| `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, among them `start_computer_recording` and `end_computer_recording`, `start_browser_recording` and `end_browser_recording` (`browser-recording/v1`: one recording per context, a lease's or the default one's, with a video of the tab its agent is on, its context's log, the browser's network log as HAR and opt-in frames keyed to the action log, the contexts its agents make recorded with it, one universe per recording, anything missing failing the run), and more). A verifier on the `VerifierTaskStep` base (`verifiers/verifier.py`; `agent_prompt_response_verifier` so far, the other built-ins still write their entries themselves) implements `verify` and returns a `Verification`; the base writes the step's named grade to `context.metadata["grades"][]` (score, `min_passing_score`, `passed`, `verifier_status` graded, skipped or failed, criteria rows, type, version, and `verifier_settings` with `scoring_rules`) beside the legacy `verifications` entries every reader still takes, and an exception from `verify` is a failed grade raised again. On an env whose card serves computer-exec/v1 (a computer), `run_code` with `env_id` and `run_container_unit_tests_verifier` with `env_id` run inside it as its session user through the driver: files staged with `data/add` and read with `data/get`, commands run with `run`; `env_outcome_verifier` hands a `verify` that takes an `env_url` keyword the env's address. A computer's framework-only routes (everything but MCP and the card) take the per-deploy control token its provider boots it with (`DeployedEnv.control_token`, `AGENTENV_CONTROL_TOKEN`), a multi env's computers the multi env's own, which its gateway passes on only on their `/svc` routes and its clock watch (`CONTROL_TOKEN_CHILDREN`); framework calls send `deployed.control_headers`, `Task.run` runs every step under `protocol_v1.authorized` with each deployed env's (a `verify` included), and a `run_code` script inside the computer gets `input["env"]["headers"]`. `teardown_run` and the `teardown_sandboxes` step share one core (`task/teardown.py` `tear_down`): the recordings the run left open end with fresh grants, each env still up closes (`Env.close`; a computer env ends any recording still open), what stayed open is registered from its namespace, then every sandbox is terminated by id. `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | +| `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, among them `start_computer_recording` and `end_computer_recording`, `start_browser_recording` and `end_browser_recording` (`browser-recording/v1`: one recording per context, a lease's or the default one's, with a video of the tab its agent is on, its context's log, the browser's network log as HAR and opt-in frames keyed to the action log, the contexts its agents make recorded with it, one universe per recording, anything missing failing the run), and more). Every built-in verifier is on the `VerifierTaskStep` base (`verifiers/verifier.py`): it implements `verify` and returns a `Verification`, and the base writes the step's named grade to `context.metadata["grades"][]` (score, `min_passing_score`, `passed`, `verifier_status` graded, skipped or failed, criteria rows, type, version, and `verifier_settings`, the values the run graded with, its overrides applied and named in `overridden`, with `scoring_rules`) beside the legacy `verifications` entries every reader still takes; an exception from `verify` or a `Verification` with an `error`, as a timed-out `run_container_unit_tests_verifier`, is a failed grade, and `rubrics_verifier` still writes its `-trajectory-mistakes` entry itself. On an env whose card serves computer-exec/v1 (a computer), `run_code` with `env_id` and `run_container_unit_tests_verifier` with `env_id` run inside it as its session user through the driver: files staged with `data/add` and read with `data/get`, commands run with `run`; `env_outcome_verifier` hands a `verify` that takes an `env_url` keyword the env's address. A computer's framework-only routes (everything but MCP and the card) take the per-deploy control token its provider boots it with (`DeployedEnv.control_token`, `AGENTENV_CONTROL_TOKEN`), a multi env's computers the multi env's own, which its gateway passes on only on their `/svc` routes and its clock watch (`CONTROL_TOKEN_CHILDREN`); framework calls send `deployed.control_headers`, `Task.run` runs every step under `protocol_v1.authorized` with each deployed env's (a `verify` included), and a `run_code` script inside the computer gets `input["env"]["headers"]`. `teardown_run` and the `teardown_sandboxes` step share one core (`task/teardown.py` `tear_down`): the recordings the run left open end with fresh grants, each env still up closes (`Env.close`; a computer env ends any recording still open), what stayed open is registered from its namespace, then every sandbox is terminated by id. `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | | `store/` | Four store ABCs with local and cloud implementations: `DocumentStore` (SQLite, MongoDB, DynamoDB, Firestore with MongoDB compatibility), `ObjectStore` (filesystem, S3, Cloud Storage), `ImageStore` (local OCI registry, ECR), `SecretStore` (env vars or file, AWS Secrets Manager, Google Cloud Secret Manager). `VersionedEntityStore` implements the shared versioned get/put logic, `QueryBuilder` is the immutable chained query API, `store/base.py` holds the error types. `store/routing.py` is the local namespaces, `@local` and `@hf` (`is_local_id`), each with a documents file of its own: while the CLI or a `build_app` server runs, a local id and what a local task's run records go to local stores beside the configured ones, and that run's writes to the configured stores are refused. A new backend must pass the conformance kits in `tst/store/`. | | `config/` | The `Config` singleton (`get_config`, `configure`, `reset_config`) in `config/runtime.py`, file discovery in `config/loader.py`, and `load_impl`, which resolves `module:Class` pointers. `agent_env.store` re-exports the config names for compatibility. | | `providers/` | `providers/sandbox_providers/` holds the sandbox providers `local`, `modal`, `modal_vm`, `e2b`, `sail_vm` (the `sail` extra), `smol_vm` (the `smol` extra), `tensorlake` (the `tensorlake` extra), `tenki` (the `tenki` extra), `vercel` (the `vercel` extra), `boxd` (the `boxd` extra), `coreweave` (the `coreweave` extra), `daytona` (the `daytona` extra), `boat` (the `boat` extra), `agent37` (no extra), `kosa8` (no extra); `[sandbox] default` and `agent_default` accept a comma-separated fallback chain, whose deploys skip a provider that can't run them. A sandbox provider declares what it runs (`ON_THIS_MACHINE`, and for VMs on this machine the address their requests to it arrive at, `REACHES_HOST_AT`; `CONTAINER_ACCEPTS`, `PRIVATE_NETWORK`, `WEBSOCKETS`; `SUPPORTED_SANDBOX_MODES`, derived from whether it implements `create_sandbox` and `create_vm`; and `SANDBOX_ACCEPTS`, the images an agent's sandbox runs, which defaults to those a VM loads when it implements `create_vm` and to `CONTAINER_ACCEPTS` otherwise, so only a provider that implements `create_vm` yet gives an agent a container, like the local one, declares it), each sandbox carries the `platform` of the images it runs (linux/amd64 unless its provider sets another when it creates it; None, any, unchecked, for a local sandbox), and code outside `providers/sandbox_providers/` reads those, never which provider it has. `providers/env_providers/` holds the environment providers: `EnvironmentProvider` (an env's containers and state store), `EnvironmentGatewayProvider` (`gateway`, an env's default), which renders a docker-compose for the gateway and its MCP servers inside the sandbox, and `EnvironmentServerProvider` (`server`), one MCP server in its own container with no gateway in front and no state store; `providers/env_state/` holds env-state providers (`local_postgres` built in). | diff --git a/tst/unit/task_step/verifiers/recorded/legacy_entries.json b/tst/data/verifier_golden/legacy_entries.json similarity index 100% rename from tst/unit/task_step/verifiers/recorded/legacy_entries.json rename to tst/data/verifier_golden/legacy_entries.json diff --git a/tst/unit/task_step/verifiers/test_legacy_entries.py b/tst/unit/task_step/verifiers/test_legacy_entries.py index e79c7654c..6ed9f785a 100644 --- a/tst/unit/task_step/verifiers/test_legacy_entries.py +++ b/tst/unit/task_step/verifiers/test_legacy_entries.py @@ -1,13 +1,12 @@ """run_container_unit_tests_verifier, verify_sandbox, env_outcome_verifier and aggregate_verifiers on the VerifierTaskStep base write the legacy ``verifications`` entry they wrote before it, byte for byte with its key order, -and raise what they raised (``recorded/legacy_entries.json``, recorded from main at 25b862bc6, object urls under -``/``), but for ``CHANGED``; and each writes its named grade beside it.""" +and raise what they raised (``tst/data/verifier_golden/legacy_entries.json``, recorded from main at 25b862bc6, object +urls under ``/``), but for ``CHANGED``; and each writes its named grade beside it.""" import asyncio import itertools import json import shlex import uuid -from pathlib import Path from types import SimpleNamespace import pytest @@ -26,8 +25,9 @@ ) from agent_env.task_step.task_steps.verifiers.verify_sandbox import VerifySandboxTaskStep from tst.unit.computer_home import Computer +from tst.util.verifier_golden import GOLDEN -RECORDED = json.loads((Path(__file__).parent / "recorded" / "legacy_entries.json").read_text()) +RECORDED = json.loads((GOLDEN / "legacy_entries.json").read_text()) REWARD = "/logs/verifier/reward.txt" SCENARIOS = {} # What the base changes on purpose: an aggregate's entry has its (empty) results, and a script that returns None is From e36c681cc3e910bffd4295c3927cac236905cb41 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 18 Nov 2026 14:54:03 -0800 Subject: [PATCH 08/12] test(verifiers): a run's stored grades, a timed-out unit test run's failed grade and its env var names, end to end run_container_unit_tests_verifier's settings always carry overridden, as the rubrics family's do. Co-Authored-By: Claude Opus 5.5 --- .../run_container_unit_tests_verifier.py | 4 +- .../env/task_step/task_steps_test.py | 5 ++ .../task_step/test_verify_sandbox_local.py | 60 +++++++++++++++++++ 3 files changed, 66 insertions(+), 3 deletions(-) diff --git a/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py b/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py index ccebce16d..9ebf72091 100644 --- a/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py @@ -189,9 +189,7 @@ def verifier_settings(self, context: TaskStepContext) -> dict[str, Any]: ) if changed] settings["command"] = overrides.get("command", self.command) settings["env_vars"] = sorted({*self.env_vars, *override_env}) - if overridden: - settings["overridden"] = overridden - return settings + return {**settings, "overridden": overridden} def _resolve_command(self, context: TaskStepContext) -> tuple[str, dict[str, str]]: """Resolve ``(command, extra_env)`` for this run, so a value can vary without a diff --git a/tst/integration/env/task_step/task_steps_test.py b/tst/integration/env/task_step/task_steps_test.py index 864f02f2e..1df13a897 100644 --- a/tst/integration/env/task_step/task_steps_test.py +++ b/tst/integration/env/task_step/task_steps_test.py @@ -614,6 +614,8 @@ async def assert_universe_data_loaded(session, tools_result, tool_names): for criterion in verification_results: assert criterion["result"] is True, f"Criterion '{criterion['id']}' failed: {criterion['description']}" assert env_vdata["score"] == 1.0, f"Expected score 1.0, got {env_vdata['score']}" + env_grade = context.metadata["grades"][retrieved_verifier.id] + assert (env_grade["verifier_status"], env_grade["score"], env_grade["passed"]) == ("graded", 1.0, True) logger.info(f"Verification [{env_verifier_id}] results: {verification_results}, score: {env_vdata['score']}") # 19. Put and get RubricsVerifierTaskStep @@ -663,6 +665,9 @@ async def assert_universe_data_loaded(session, tools_result, tool_names): # Verify env outcome verifier results still exist assert env_verifier_id in context.metadata["verifications"], \ "Env outcome verifier results were overwritten by rubrics verifier" + rubrics_grade = context.metadata["grades"][retrieved_rubrics.id] + assert (rubrics_grade["verifier_status"], rubrics_grade["score"], rubrics_grade["passed"]) == ("graded", 1.0, True) + assert rubrics_grade["verifier_settings"]["judge_model"] and "overridden" in rubrics_grade["verifier_settings"] logger.info(f"Verification [{rubrics_verifier_id}] results: {rubrics_results}, score: {rubrics_vdata['score']}") # 21. Put and get RubricsVerifierTaskStep with agent_name (reusing deployed agent) diff --git a/tst/integration/task_step/test_verify_sandbox_local.py b/tst/integration/task_step/test_verify_sandbox_local.py index 78ea37600..c888a9c82 100644 --- a/tst/integration/task_step/test_verify_sandbox_local.py +++ b/tst/integration/task_step/test_verify_sandbox_local.py @@ -27,6 +27,7 @@ from agent_env.task_step.task_steps.collect_artifacts import CollectArtifactsTaskStep from agent_env.task_step.task_steps.deploy_sandbox import DeploySandboxTaskStep from agent_env.task_step.task_steps.load_artifact import LoadArtifactTaskStep +from agent_env.task_step.task_steps.verifiers.aggregate_verifiers import AggregateVerifiersTaskStep from agent_env.task_step.task_steps.verifiers.run_container_unit_tests_verifier import ( RunContainerUnitTestsVerifierTaskStep, ) @@ -123,6 +124,36 @@ async def test_verify_sandbox_scores_a_bare_local_sandbox(local_backends): assert Task.get_instance(ctx.instance_id).status == "completed" +@pytest.mark.asyncio +async def test_each_verifier_of_a_run_leaves_its_named_grade_on_the_stored_run(local_backends): + tmp_path, _ = local_backends + suffix = uuid.uuid4().hex[:8] + universe = _greeting_universe(tmp_path / "greeting", suffix) + task = Task.put(id=f"graded-{suffix}", steps=[ + DeploySandboxTaskStep(id="deploy", version=None, sandbox_name="box", sandbox_mode="vm", sandbox_type="local"), + LoadArtifactTaskStep( + id="load", version=None, artifact_id=universe.id, sandbox_name="box", destination_path="/app/greeting", + ), + _verify("hello", "hello", "python3 check.py"), + _verify("control", "goodbye", "python3 -c 'raise SystemExit(1)'"), + AggregateVerifiersTaskStep(id="overall", version=None, verifier_id="overall", verifier_ids=["hello", "control"]), + ]) + + ctx = await task.run() + + grades = Task.get_instance(ctx.instance_id).context["metadata"]["grades"] + assert {name: (g["verifier_status"], g["score"], g["passed"], g["verifier_type"]) for name, g in grades.items()} == { + "hello": ("graded", 1.0, True, "verify_sandbox"), + "control": ("graded", 0.0, False, "verify_sandbox"), + "overall": ("graded", 0.5, False, "aggregate_verifiers"), + } + assert [row["passed"] for row in grades["hello"]["verifier_criteria"]] == [True, True] + assert grades["overall"]["verifier_output"] == {"source_verifier_ids": ["hello", "control"]} + assert grades["overall"]["verifier_settings"] == { + "scoring_rules": 1, "verifier_ids": ["hello", "control"], "score_aggregator": "weighted_average"} + assert ctx.metadata["verifications"]["overall"]["score"] == 0.5 + + @pytest.mark.asyncio async def test_file_artifacts_stage_beside_each_other(local_backends): tmp_path, _ = local_backends @@ -293,6 +324,35 @@ async def test_the_reward_is_cleared_in_a_container_with_no_root_user_name(local assert (entry["score"], entry["extracted_files"]) == (0.0, {"/logs/verifier/reward.txt": None}) +@pytest.mark.asyncio +async def test_a_unit_tests_verifier_that_times_out_is_a_failed_grade_beside_its_entry(local_agent): + sandbox, context = local_agent + _as_deployed_container(sandbox, context) + step = _reward_step(command="sleep 60", timeout_sec=2) + + with pytest.raises(RuntimeError, match="timed out after 2s"): + await step.execute(context) + + grade, entry = context.metadata["grades"]["verify"], context.metadata["verifications"]["verify"] + assert (grade["verifier_status"], grade["score"], grade["passed"]) == ("failed", None, False) + assert grade["verifier_error_reasoning"] == "Verifier 'verify' timed out after 2s" + assert entry["timed_out"] is True + + +@pytest.mark.asyncio +async def test_a_unit_tests_verifiers_grade_holds_its_reward_and_names_its_env_vars_without_their_values(local_agent): + sandbox, context = local_agent + _as_deployed_container(sandbox, context) + step = _reward_step(command='test -n "$TOKEN" && echo 1 > /logs/verifier/reward.txt', env_vars={"TOKEN": "s3cret"}) + + await step.execute(context) + + grade = context.metadata["grades"]["verify"] + assert (grade["verifier_status"], grade["score"], grade["passed"]) == ("graded", 1.0, True) + assert grade["verifier_settings"]["env_vars"] == ["TOKEN"] and grade["verifier_settings"]["overridden"] == [] + assert "s3cret" not in json.dumps(grade) + + def _as_deployed_container(sandbox, context): """Record the agent's container as a ``run_docker_container`` container on the deployed sandbox ``box``: it is the sandbox's own ``agent``, as a step container is its own name.""" From 628c0dd8876dc4ecbe233f809fb4a79aaec522d8 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 18 Nov 2026 15:06:20 -0800 Subject: [PATCH 09/12] fix(verifiers): settings that can't be compacted are recorded, a judge's base URL stays out of them, and the rubrics golden check says it leaves key order aside Co-Authored-By: Claude Opus 5.5 --- .../task_steps/verifiers/rubrics_verifier.py | 1 + .../task_step/task_steps/verifiers/verifier.py | 4 ++-- .../verifiers/test_rubrics_family_grades.py | 10 ++++++++++ tst/unit/task_step/verifiers/test_verifier_base.py | 14 ++++++++++++++ tst/util/verifier_golden.py | 3 ++- 5 files changed, 29 insertions(+), 3 deletions(-) diff --git a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py index 89374b241..75fc6042c 100644 --- a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py @@ -771,6 +771,7 @@ def verifier_settings(self, context: TaskStepContext) -> dict[str, Any]: the judge model the run recorded (else the one it resolves), the agent it deploys as its judge when it deploys one, and ``overridden``, the settings the run's overrides changed. Never a key, header or base URL.""" settings = super().verifier_settings(context) + settings.pop("default_model_api_base", None) applied = self._run_overrides(context) overrides = context.metadata.get("user_overrides") or {} overridden = [key for key, value in applied.items() if settings.get(key) != value] diff --git a/src/agent_env/task_step/task_steps/verifiers/verifier.py b/src/agent_env/task_step/task_steps/verifiers/verifier.py index 6342abcab..71059dfd1 100644 --- a/src/agent_env/task_step/task_steps/verifiers/verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/verifier.py @@ -216,11 +216,11 @@ def _settings(self, context: TaskStepContext) -> dict[str, Any]: """``verifier_settings`` as the grade records it, compacted; a failing one is recorded rather than raised, since a failed grade records settings too and must not lose the verifier's own error.""" try: - settings = self.verifier_settings(context) + settings = {k: _compact(v) for k, v in self.verifier_settings(context).items()} except Exception as exc: logger.warning("verifier %s: its settings couldn't be read: %s", self.id, exc) return {"scoring_rules": SCORING_RULES, "settings_error": _capped(f"{type(exc).__name__}: {exc}")} - return {"scoring_rules": SCORING_RULES, **{k: _compact(v) for k, v in settings.items()}} + return {"scoring_rules": SCORING_RULES, **settings} async def _output(self, context: TaskStepContext, output: Any) -> dict[str, Any]: """``verifier_output`` inline, or, over ``OUTPUT_INLINE_BYTES``, a ``verifier_output_ref`` to its copy in the diff --git a/tst/unit/task_step/verifiers/test_rubrics_family_grades.py b/tst/unit/task_step/verifiers/test_rubrics_family_grades.py index 802fbaaa8..3798bc27d 100644 --- a/tst/unit/task_step/verifiers/test_rubrics_family_grades.py +++ b/tst/unit/task_step/verifiers/test_rubrics_family_grades.py @@ -68,6 +68,16 @@ async def test_a_run_naming_the_steps_own_judge_model_overrides_nothing(): assert settings["overridden"] == [] and settings["judge_model"] == "judge-model" +@pytest.mark.asyncio +async def test_the_judges_own_base_url_stays_out_of_its_settings(): + step = _direct_judge(_step(criteria=_criteria(1), default_model_api_base="http://judge.example.test"), + lambda ids, n: _binary(ids)) + + settings = (await _grade(step, _context({})))["verifier_settings"] + + assert "default_model_api_base" not in settings and "judge.example.test" not in json.dumps(settings) + + @pytest.mark.asyncio async def test_an_agent_judge_that_takes_no_model_is_recorded_as_choosing_its_own(): step, context = CASES["an_agent_judge_timing_out_once"]() diff --git a/tst/unit/task_step/verifiers/test_verifier_base.py b/tst/unit/task_step/verifiers/test_verifier_base.py index 24097c6b8..e76cff4bb 100644 --- a/tst/unit/task_step/verifiers/test_verifier_base.py +++ b/tst/unit/task_step/verifiers/test_verifier_base.py @@ -321,6 +321,20 @@ def verifier_settings(self, context): assert grade["verifier_settings"] == {"scoring_rules": 1, "settings_error": "KeyError: 'judge'"} +def test_a_setting_that_isnt_json_is_recorded_not_raised(): + class _Opaque(_Toy): + def verifier_settings(self, context): + return {**super().verifier_settings(context), "client": object()} + + context = TaskStepContext() + _run(_Opaque(outcome=Verification(score=1)), context) + + grade = context.metadata["grades"]["check"] + assert grade["verifier_status"] == "graded" and grade["score"] == 1.0 + assert set(grade["verifier_settings"]) == {"scoring_rules", "settings_error"} + assert grade["verifier_settings"]["settings_error"].startswith("TypeError: ") + + def test_a_long_error_is_cut_with_its_length(): context = TaskStepContext() diff --git a/tst/util/verifier_golden.py b/tst/util/verifier_golden.py index d03304c4d..4c6d95942 100644 --- a/tst/util/verifier_golden.py +++ b/tst/util/verifier_golden.py @@ -1,6 +1,7 @@ """The legacy ``verifications`` entries built-in verifiers left on main before they moved onto ``VerifierTaskStep``, recorded under ``tst/data/verifier_golden``: a run must leave the same entries, number types included, the same -models record and the same error.""" +models record and the same error. Key order isn't compared: the base writes an entry's ``results`` and ``score`` +first, where the rubrics family wrote its ``format`` before them.""" from __future__ import annotations From 31bcd31fad626bd4c1b799089bc426d5e01b8b03 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 18 Nov 2026 15:11:27 -0800 Subject: [PATCH 10/12] test(verifiers): the reward test makes its reward's folder, as no step made it before Co-Authored-By: Claude Opus 5.5 --- tst/integration/task_step/test_verify_sandbox_local.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tst/integration/task_step/test_verify_sandbox_local.py b/tst/integration/task_step/test_verify_sandbox_local.py index c888a9c82..68c0a9d7b 100644 --- a/tst/integration/task_step/test_verify_sandbox_local.py +++ b/tst/integration/task_step/test_verify_sandbox_local.py @@ -343,7 +343,8 @@ async def test_a_unit_tests_verifier_that_times_out_is_a_failed_grade_beside_its async def test_a_unit_tests_verifiers_grade_holds_its_reward_and_names_its_env_vars_without_their_values(local_agent): sandbox, context = local_agent _as_deployed_container(sandbox, context) - step = _reward_step(command='test -n "$TOKEN" && echo 1 > /logs/verifier/reward.txt', env_vars={"TOKEN": "s3cret"}) + step = _reward_step(command='mkdir -p /logs/verifier && test -n "$TOKEN" && echo 1 > /logs/verifier/reward.txt', + env_vars={"TOKEN": "s3cret"}) await step.execute(context) From 083a466f33a2cfdac5f9bf4b228af992412d759a Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 18 Nov 2026 16:03:38 -0800 Subject: [PATCH 11/12] refactor(verifiers): the rubrics judge and its settings read a run's values from one resolver _run_values resolves every setting a run's step_params or user_overrides can change; the judge reads it and verifier_settings records it, so the two can't drift. The settings now record a run's effort and thinking budget only for an agent judge, and the agent sandbox only for a judge the step deploys, as the judge uses them, and the user sim judge records the batch concurrency its judge runs with. Co-Authored-By: Claude Opus 5.5 --- .../task_steps/verifiers/rubrics_verifier.py | 109 ++++++++---------- .../task_steps/verifiers/user_sim_judge.py | 14 +-- .../verifiers/test_rubrics_family_grades.py | 40 +++++-- 3 files changed, 88 insertions(+), 75 deletions(-) diff --git a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py index 75fc6042c..ce2d04769 100644 --- a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py @@ -757,53 +757,62 @@ def judge_agent_config(self, judge_agent_card: dict) -> dict: return {} def _resolved_criteria(self, context: TaskStepContext) -> list[dict]: - """Criteria for this run: a per-run ``step_params`` override replaces the stored list. + """The criteria this run asks (``_run_values``): an override point, which user_sim_judge's plan replaces.""" + return list(self._run_values(context)["criteria"]) + + def _run_values(self, context: TaskStepContext) -> dict[str, Any]: + """Each setting a run's ``step_params`` or ``user_overrides`` can change, by the step's own field name, at the + value this run judges with: the judge reads these and ``verifier_settings`` records them, so the two can't + disagree. Unvalidated; the judge checks them as it reads them. Never mutates ``self``, which concurrent runs + share. A ``step_params`` ``criteria`` replaces on presence, so ``[]`` replaces too; effort and the thinking + budget reach only an agent judge, and the agent sandbox only a judge the step deploys.""" + step_params = self.step_param_overrides(context) + overrides = context.metadata.get("user_overrides") or {} + use_agent_judge = self.use_agent_judge if overrides.get("use_agent_judge") is None else overrides["use_agent_judge"] + apply_filter = overrides.get("apply_trajectory_filter") + trajectory_filter = (None if apply_filter is False + else self.trajectory_filter or TrajectoryFilter() if apply_filter is True + else self.trajectory_filter) + return { + "criteria": list(step_params["criteria"]) if "criteria" in step_params else self.criteria, + "grade_up_to_turn": step_params["grade_up_to_turn"] if "grade_up_to_turn" in step_params + else self.grade_up_to_turn, + "use_agent_judge": use_agent_judge, + "trajectory_filter": trajectory_filter, + "effort": (overrides.get("judge_effort") or self.effort) if use_agent_judge else self.effort, + "max_thinking_tokens": (overrides.get("judge_max_thinking_tokens") or self.max_thinking_tokens) + if use_agent_judge else self.max_thinking_tokens, + "judge_batch_size": _override_or(overrides, "judge_batch_size", self.judge_batch_size), + "judge_batch_concurrency": _override_or(overrides, "judge_batch_concurrency", self.judge_batch_concurrency), + "judge_sandbox_type": (overrides.get("agent_sandbox") or self.judge_sandbox_type) + if use_agent_judge and self.agent_name is None else self.judge_sandbox_type, + } - Replace-on-presence, like ``load_artifact`` — an explicit ``[]`` replaces rather - than reverting (presence check, not truthiness). Returns a copy; never mutates - ``self``, which is shared across concurrent rollouts. - """ - step_overrides = self.step_param_overrides(context) - return list(step_overrides["criteria"]) if "criteria" in step_overrides else self.criteria + def _recorded_values(self, context: TaskStepContext) -> dict[str, Any]: + """``_run_values`` as ``verifier_settings`` records them, the trajectory filter as its dict; a subclass leaves + out the ones its judge never uses.""" + values = self._run_values(context) + trajectory_filter = values["trajectory_filter"] + return {**values, "trajectory_filter": trajectory_filter.to_dict() if trajectory_filter is not None else None} def verifier_settings(self, context: TaskStepContext) -> dict[str, Any]: - """The step's fields as this run grades with them: each run override it applies in place of the stored value, - the judge model the run recorded (else the one it resolves), the agent it deploys as its judge when it deploys - one, and ``overridden``, the settings the run's overrides changed. Never a key, header or base URL.""" + """The step's fields as this run judges with them (``_recorded_values`` in place of the stored ones), the judge + model the run recorded (else the one it resolves), the agent it deploys as its judge when it deploys one, and + ``overridden``, the settings this run's overrides changed. Never a key, header or base URL.""" settings = super().verifier_settings(context) settings.pop("default_model_api_base", None) - applied = self._run_overrides(context) + values, stored = self._recorded_values(context), self._recorded_values(TaskStepContext()) + overridden = [key for key, value in values.items() if value != stored[key]] + settings.update({key: values[key] for key in overridden}) overrides = context.metadata.get("user_overrides") or {} - overridden = [key for key, value in applied.items() if settings.get(key) != value] if overrides.get("judge_model") and overrides["judge_model"] != self.default_model: overridden.append("judge_model") - settings.update(applied) if settings.get("use_agent_judge") and self.agent_name is None and not self.judge_a2a_agent_id: settings["judge_a2a_agent_id"] = get_config().get_default_a2a_agent_id() recorded = (context.metadata.get("models") or {}).get(ModelRole.JUDGE.record_key) or {} settings["judge_model"] = recorded[self.id] if self.id in recorded else self._judge_model(context) return {**settings, "overridden": overridden} - def _run_overrides(self, context: TaskStepContext) -> dict[str, Any]: - """The settings this run's ``step_params`` and ``user_overrides`` set, by the step's own field names, with the - values it grades with: what ``verifier_settings`` records in place of the stored ones.""" - step_params = self.step_param_overrides(context) - overrides = context.metadata.get("user_overrides") or {} - applied: dict[str, Any] = {key: step_params[key] for key in ("criteria", "grade_up_to_turn") if key in step_params} - if overrides.get("use_agent_judge") is not None: - applied["use_agent_judge"] = overrides["use_agent_judge"] - if (apply_filter := overrides.get("apply_trajectory_filter")) is not None: - applied["trajectory_filter"] = (self.trajectory_filter or TrajectoryFilter()).to_dict() if apply_filter else None - for override, key in (("judge_effort", "effort"), ("judge_max_thinking_tokens", "max_thinking_tokens")): - if overrides.get(override): - applied[key] = overrides[override] - for key in ("judge_batch_size", "judge_batch_concurrency"): - if overrides.get(key) is not None: - applied[key] = overrides[key] - if overrides.get("agent_sandbox") and self.agent_name is None: - applied["judge_sandbox_type"] = overrides["agent_sandbox"] - return applied - async def verify(self, context: TaskStepContext) -> Verification: verdict = self._gated(await self._judged(context)) if verdict.mistakes_entry is not None: @@ -826,11 +835,8 @@ async def _judged(self, context: TaskStepContext) -> RubricsVerdict: prompt_response = next((pr for pr in context.prompt_responses if pr.prompt_id == self.prompt_id), None) if prompt_response is None: raise StepContextError(f"PromptResponse with prompt_id='{self.prompt_id}' not found in context") - overrides_for_step = self.step_param_overrides(context) - grade_up_to_turn = ( - _validated_turn(overrides_for_step["grade_up_to_turn"]) - if "grade_up_to_turn" in overrides_for_step else self.grade_up_to_turn - ) + run = self._run_values(context) + grade_up_to_turn = _validated_turn(run["grade_up_to_turn"]) as_of = _as_of_turn(prompt_response, grade_up_to_turn) if as_of is not prompt_response and not self.use_trajectory: # A cut copy's reply only points at turn k's trajectory, which use_trajectory=False never loads. @@ -856,20 +862,11 @@ async def _judged(self, context: TaskStepContext) -> RubricsVerdict: raise RuntimeError(f"No trajectory available for prompt_id='{self.prompt_id}'") overrides = context.metadata.get("user_overrides", {}) - # Resolved into a local, not onto self: one step instance is shared across - # concurrent rollouts, so mutating self would leak the override between them. - use_agent_judge_override = overrides.get("use_agent_judge") - use_agent_judge = use_agent_judge_override if use_agent_judge_override is not None else self.use_agent_judge + use_agent_judge = run["use_agent_judge"] criteria = self._resolved_criteria(context) # The judge sees/echoes short keys (c1..cN), not UUIDs; frame labels use them too, hence before compaction. criteria_for_judge, real_by_short = _to_short_keyed_criteria(criteria) - apply_filter_override = overrides.get("apply_trajectory_filter") - if apply_filter_override is False: - filter_to_apply = None - elif apply_filter_override is True: - filter_to_apply = self.trajectory_filter or TrajectoryFilter() - else: - filter_to_apply = self.trajectory_filter + filter_to_apply = run["trajectory_filter"] # Multi-turn: judge every turn (fed from the per-turn URIs already on the # PromptResponse), not just the last. DEFAULT compaction runs per turn; SCREENSHOT @@ -975,9 +972,8 @@ async def _judged(self, context: TaskStepContext) -> RubricsVerdict: raise raise NotFoundError(unwritten_local_agent(resolved_judge_id, "judge_a2a_agent_id")) from None deploy_kwargs = {"env_vars": env_vars} - resolved_sandbox_type = overrides.get("agent_sandbox") or self.judge_sandbox_type - if resolved_sandbox_type: - deploy_kwargs["sandbox_type"] = resolved_sandbox_type + if run["judge_sandbox_type"]: + deploy_kwargs["sandbox_type"] = run["judge_sandbox_type"] auto_deployed_judge = await a2a_agent.deploy(**deploy_kwargs) judge_a2a_url = auto_deployed_judge.a2a_url judge_sandbox_id = auto_deployed_judge.sandbox_id @@ -1010,12 +1006,9 @@ async def _judged(self, context: TaskStepContext) -> RubricsVerdict: model = self._judge_model(context) context.record_model(ModelRole.JUDGE.record_key, self.id, model) agent_prompt_text = prompt_response.prompt_text or "" - judge_effort = overrides.get("judge_effort") or self.effort - judge_max_thinking_tokens = overrides.get("judge_max_thinking_tokens") or self.max_thinking_tokens - batch_size = _validated_batch_option("judge_batch_size", _override_or( - overrides, "judge_batch_size", self.judge_batch_size)) - batch_concurrency = _validated_batch_option("judge_batch_concurrency", _override_or( - overrides, "judge_batch_concurrency", self.judge_batch_concurrency)) + judge_effort, judge_max_thinking_tokens = run["effort"], run["max_thinking_tokens"] + batch_size = _validated_batch_option("judge_batch_size", run["judge_batch_size"]) + batch_concurrency = _validated_batch_option("judge_batch_concurrency", run["judge_batch_concurrency"]) if batch_size is not None and not self._spec().batchable: raise RuntimeError(f"Verifier '{self.verifier_id}': {_unbatchable_format_message(self.output_format)}") loaded_for_judge: list[dict] = [] diff --git a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py index 1bede3884..624f66b01 100644 --- a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py +++ b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge.py @@ -321,16 +321,10 @@ def from_dict(cls, data: dict) -> "UserSimJudge": fail_task_when_not_passing=data.get("fail_task_when_not_passing", True), judge_timeout_seconds=data.get("judge_timeout_seconds")) - def _run_overrides(self, context: TaskStepContext) -> dict[str, Any]: - """The run overrides the judge applies: rubrics_verifier's but the criteria, which the plan replaces, the - trajectory filter, which reads no trajectory, and the batch concurrency and sandbox, which never change a - verdict; effort and thinking only for an agent judge, which alone takes them.""" - applied = {k: v for k, v in super()._run_overrides(context).items() - if k not in ("criteria", "trajectory_filter", "judge_batch_concurrency", "judge_sandbox_type")} - if not applied.get("use_agent_judge"): - applied.pop("effort", None) - applied.pop("max_thinking_tokens", None) - return applied + def _recorded_values(self, context: TaskStepContext) -> dict[str, Any]: + """rubrics_verifier's, but the criteria, which each run's plan replaces, and the trajectory filter: its judge + reads no trajectory.""" + return {k: v for k, v in super()._recorded_values(context).items() if k not in ("criteria", "trajectory_filter")} # ---- the run ----------------------------------------------------------------------------------------------- async def verify(self, context: TaskStepContext) -> Verification: diff --git a/tst/unit/task_step/verifiers/test_rubrics_family_grades.py b/tst/unit/task_step/verifiers/test_rubrics_family_grades.py index 3798bc27d..95a609510 100644 --- a/tst/unit/task_step/verifiers/test_rubrics_family_grades.py +++ b/tst/unit/task_step/verifiers/test_rubrics_family_grades.py @@ -42,13 +42,32 @@ async def test_the_settings_are_what_the_run_graded_with_and_say_what_its_overri grade = await _grade(step, context) settings = grade["verifier_settings"] - assert (settings["criteria"], settings["judge_model"], settings["effort"], settings["judge_batch_size"]) == ( - OVERRIDE, "judge-x", "high", 1) + assert (settings["criteria"], settings["judge_model"], settings["judge_batch_size"]) == (OVERRIDE, "judge-x", 1) assert settings["trajectory_filter"] is None and settings["default_model"] == "judge-model" - assert settings["overridden"] == ["criteria", "effort", "judge_batch_size", "judge_model"] + assert settings["effort"] is None + assert settings["overridden"] == ["criteria", "judge_batch_size", "judge_model"] assert "sk-run-key" not in json.dumps(grade) and "llm.example.test" not in json.dumps(grade) +@pytest.mark.asyncio +async def test_the_judge_batches_by_the_batch_size_its_settings_record(): + step = _direct_judge(_step(criteria=_criteria(2)), lambda ids, n: _binary(ids)) + context = _context({"judge_batch_size": 1}) + + grade = await _grade(step, context) + + assert grade["verifier_settings"]["judge_batch_size"] == 1 + assert context.metadata["verifications"]["vid"]["judge_batches"] == 2 + + +def test_an_agent_judge_records_the_effort_and_thinking_budget_a_run_gives_it(): + settings = _step(use_agent_judge=True).verifier_settings( + _context({"judge_effort": "high", "judge_max_thinking_tokens": 4000})) + + assert (settings["effort"], settings["max_thinking_tokens"]) == ("high", 4000) + assert {"effort", "max_thinking_tokens"} <= set(settings["overridden"]) + + @pytest.mark.asyncio async def test_a_run_without_overrides_records_the_stored_settings_and_the_model_it_resolved(): step, context = CASES["binary"]() @@ -96,6 +115,12 @@ def test_a_judge_the_step_deploys_is_recorded_by_the_agent_and_sandbox_it_runs_o assert settings["judge_sandbox_type"] == "modal" and settings["overridden"] == ["judge_sandbox_type"] +def test_a_direct_judge_records_no_agent_sandbox_it_never_deploys_on(): + settings = _step().verifier_settings(_context({"agent_sandbox": "modal"})) + + assert settings["judge_sandbox_type"] is None and settings["overridden"] == [] + + @pytest.mark.asyncio @pytest.mark.parametrize(("min_passing_score", "raises"), [(None, True), (0.5, False), (0.75, True)]) async def test_the_gate_fails_a_score_under_the_steps_min_passing_score(min_passing_score, raises): @@ -140,16 +165,17 @@ def test_the_user_sim_judge_takes_its_pinned_aggregator_and_refuses_a_min_passin _user_sim_judge(min_passing_score=0.5) -def test_the_user_sim_judges_settings_are_only_the_overrides_it_applies(): +def test_the_user_sim_judges_settings_leave_out_what_its_judge_never_uses(): context = TaskStepContext(metadata={"user_overrides": { "judge_model": "judge-x", "judge_effort": "high", "apply_trajectory_filter": True, "judge_batch_size": 2, "judge_batch_concurrency": 3, "step_params": {"judge_user_sim": {"criteria": OVERRIDE}}}}) settings = _user_sim_judge().verifier_settings(context) - assert settings["judge_model"] == "judge-x" and settings["judge_batch_size"] == 2 - assert not {"criteria", "trajectory_filter", "effort", "judge_batch_concurrency"} & set(settings) - assert settings["overridden"] == ["judge_batch_size", "judge_model"] + assert (settings["judge_model"], settings["judge_batch_size"], settings["judge_batch_concurrency"]) == ( + "judge-x", 2, 3) + assert not {"criteria", "trajectory_filter", "effort"} & set(settings) + assert settings["overridden"] == ["judge_batch_size", "judge_batch_concurrency", "judge_model"] @pytest.mark.asyncio From d73736d11eb705cbb862e9868aa0fff455521d1c Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 18 Nov 2026 16:07:55 -0800 Subject: [PATCH 12/12] test(verifiers): one golden harness for every verifier, which re-records its files on AGENTENV_TEST_RECORD_GOLDEN=1 test_legacy_entries.py checks through tst/util/verifier_golden.py as the rubrics tests do, and every golden check compares key order too. Re-recorded once: the four simpler verifiers' file takes the base's two deliberate changes (an aggregate's empty results, the message for a check script that returns None), and the rubrics family's two files the order the base writes, every value the same. Co-Authored-By: Claude Opus 5.5 --- tst/data/verifier_golden/legacy_entries.json | 7 +- .../verifier_golden/rubrics_subclasses.json | 44 ++--- .../verifier_golden/rubrics_verifier.json | 160 +++++++++--------- .../verifiers/test_legacy_entries.py | 37 ++-- .../test_rubrics_subclasses_golden.py | 22 +-- .../verifiers/test_rubrics_verifier_golden.py | 8 +- tst/util/verifier_golden.py | 36 +++- 7 files changed, 165 insertions(+), 149 deletions(-) diff --git a/tst/data/verifier_golden/legacy_entries.json b/tst/data/verifier_golden/legacy_entries.json index abf6d64bb..d5790660a 100644 --- a/tst/data/verifier_golden/legacy_entries.json +++ b/tst/data/verifier_golden/legacy_entries.json @@ -2536,7 +2536,7 @@ } }, "outcome_returns_none": { - "raised": "TypeError: object of type 'NoneType' has no len()", + "raised": "TypeError: verify() in artifact 's' returned None, not a list of criterion rows", "entry": { "results": null, "score": 0.0 @@ -2573,6 +2573,7 @@ "aggregate_two_sources": { "raised": null, "entry": { + "results": [], "score": 0.25, "source_verifier_ids": [ "fs", @@ -2583,6 +2584,7 @@ "aggregate_all_pass": { "raised": null, "entry": { + "results": [], "score": 0.0, "source_verifier_ids": [ "fs", @@ -2593,6 +2595,7 @@ "aggregate_of_an_aggregate": { "raised": null, "entry": { + "results": [], "score": 1.0, "source_verifier_ids": [ "inner", @@ -2603,6 +2606,7 @@ "aggregate_of_a_source_without_rows": { "raised": null, "entry": { + "results": [], "score": 0.0, "source_verifier_ids": [ "no-rows" @@ -2612,6 +2616,7 @@ "aggregate_of_nothing": { "raised": null, "entry": { + "results": [], "score": 0.0, "source_verifier_ids": [] } diff --git a/tst/data/verifier_golden/rubrics_subclasses.json b/tst/data/verifier_golden/rubrics_subclasses.json index bcb18ca95..dd62f4bd1 100644 --- a/tst/data/verifier_golden/rubrics_subclasses.json +++ b/tst/data/verifier_golden/rubrics_subclasses.json @@ -2,7 +2,6 @@ "us_passing_with_the_gate_off": { "verifications": { "usj": { - "format": "user_sim_checks", "results": [ { "id": "guidance", @@ -124,6 +123,7 @@ } ], "score": 1.0, + "format": "user_sim_checks", "compact_trajectory_s3_uri": null, "compact_trajectory_object_url": null, "judge_trajectory_s3_uri": null, @@ -179,7 +179,6 @@ "us_the_gate_on_a_slip_that_changes_the_grade": { "verifications": { "usj": { - "format": "user_sim_checks", "results": [ { "id": "guidance", @@ -321,6 +320,7 @@ } ], "score": 0.0, + "format": "user_sim_checks", "compact_trajectory_s3_uri": null, "compact_trajectory_object_url": null, "judge_trajectory_s3_uri": null, @@ -379,7 +379,6 @@ "us_a_slip_on_unconfirmed_instructions": { "verifications": { "usj": { - "format": "user_sim_checks", "results": [ { "id": "guidance", @@ -503,6 +502,7 @@ } ], "score": 0.0, + "format": "user_sim_checks", "compact_trajectory_s3_uri": null, "compact_trajectory_object_url": null, "judge_trajectory_s3_uri": null, @@ -561,9 +561,10 @@ "us_no_graded_turns": { "verifications": { "usj": { - "format": "user_sim_checks", "results": [], "score": null, + "skipped": "no graded turns", + "format": "user_sim_checks", "run_step_id": "talk", "run_kind": "prompt_agent", "judge_model": "fake/judge", @@ -577,7 +578,6 @@ "graded_turns": [], "turns": 1, "notes": [], - "skipped": "no graded turns", "user_sim_error": { "value": "no", "because": [], @@ -592,7 +592,6 @@ "us_a_persona_mismatch": { "verifications": { "usj": { - "format": "user_sim_checks", "results": [ { "id": "guidance", @@ -714,6 +713,7 @@ } ], "score": 1.0, + "format": "user_sim_checks", "compact_trajectory_s3_uri": null, "compact_trajectory_object_url": null, "judge_trajectory_s3_uri": null, @@ -775,7 +775,6 @@ "us_an_unknown_persona_with_a_line_item": { "verifications": { "usj": { - "format": "user_sim_checks", "results": [ { "id": "guidance", @@ -913,6 +912,7 @@ } ], "score": 1.0, + "format": "user_sim_checks", "compact_trajectory_s3_uri": null, "compact_trajectory_object_url": null, "judge_trajectory_s3_uri": null, @@ -977,7 +977,6 @@ "us_one_check_undecided": { "verifications": { "usj": { - "format": "user_sim_checks", "results": [ { "id": "guidance", @@ -1099,6 +1098,7 @@ } ], "score": 1.0, + "format": "user_sim_checks", "compact_trajectory_s3_uri": null, "compact_trajectory_object_url": null, "judge_trajectory_s3_uri": null, @@ -1157,7 +1157,6 @@ "us_every_check_undecided": { "verifications": { "usj": { - "format": "user_sim_checks", "results": [ { "id": "guidance", @@ -1279,6 +1278,7 @@ } ], "score": 1.0, + "format": "user_sim_checks", "compact_trajectory_s3_uri": null, "compact_trajectory_object_url": null, "judge_trajectory_s3_uri": null, @@ -1342,7 +1342,6 @@ "us_an_answer_retry": { "verifications": { "usj": { - "format": "user_sim_checks", "results": [ { "id": "guidance", @@ -1466,6 +1465,7 @@ } ], "score": 0.0, + "format": "user_sim_checks", "compact_trajectory_s3_uri": null, "compact_trajectory_object_url": null, "judge_trajectory_s3_uri": null, @@ -1539,7 +1539,6 @@ "us_an_auto_item": { "verifications": { "usj": { - "format": "user_sim_checks", "results": [ { "id": "guidance", @@ -1682,6 +1681,7 @@ } ], "score": 0.0, + "format": "user_sim_checks", "compact_trajectory_s3_uri": null, "compact_trajectory_object_url": null, "judge_trajectory_s3_uri": null, @@ -1740,7 +1740,6 @@ "us_a_grade_up_to_turn_that_cuts_nothing": { "verifications": { "usj": { - "format": "user_sim_checks", "results": [ { "id": "guidance", @@ -1862,6 +1861,7 @@ } ], "score": 1.0, + "format": "user_sim_checks", "compact_trajectory_s3_uri": null, "compact_trajectory_object_url": null, "judge_trajectory_s3_uri": null, @@ -1918,13 +1918,13 @@ "us_a_chat_run_that_ended_in_an_error": { "verifications": { "usj": { - "format": "user_sim_checks", "results": [], "score": null, + "skipped": "the run ended with error_type=user_sim_error", + "format": "user_sim_checks", "run_step_id": "talk", "run_kind": "prompt_agent", "judge_model": "fake/judge", - "skipped": "the run ended with error_type=user_sim_error", "user_sim_error": { "value": "no", "because": [], @@ -1946,13 +1946,13 @@ "us_a_chat_run_with_no_reads": { "verifications": { "usj": { - "format": "user_sim_checks", "results": [], "score": null, + "skipped": "the user sim never read: the run has no user_sim_reads", + "format": "user_sim_checks", "run_step_id": "talk", "run_kind": "prompt_agent", "judge_model": "fake/judge", - "skipped": "the user sim never read: the run has no user_sim_reads", "user_sim_error": { "value": "no", "because": [], @@ -1974,7 +1974,6 @@ "us_a_chat_run_the_gate_fails": { "verifications": { "usj": { - "format": "user_sim_checks", "results": [ { "id": "guidance", @@ -2098,6 +2097,7 @@ } ], "score": 0.0, + "format": "user_sim_checks", "compact_trajectory_s3_uri": null, "compact_trajectory_object_url": null, "judge_trajectory_s3_uri": null, @@ -2162,7 +2162,6 @@ "us_an_always_on_run_the_gate_fails": { "verifications": { "usj": { - "format": "user_sim_checks", "results": [ { "id": "guidance", @@ -2286,6 +2285,7 @@ } ], "score": 0.0, + "format": "user_sim_checks", "compact_trajectory_s3_uri": null, "compact_trajectory_object_url": null, "judge_trajectory_s3_uri": null, @@ -2347,7 +2347,6 @@ "ao_unstaged_with_its_harness": { "verifications": { "rubric": { - "format": "rubric_binary", "results": [ { "id": "c1", @@ -2357,6 +2356,7 @@ } ], "score": 1.0, + "format": "rubric_binary", "compact_trajectory_s3_uri": null, "compact_trajectory_object_url": null, "judge_trajectory_s3_uri": null, @@ -2386,7 +2386,6 @@ "ao_a_prompt_error_the_gate_fails": { "verifications": { "rubric": { - "format": "rubric_binary", "results": [ { "id": "prompt_error", @@ -2396,6 +2395,7 @@ } ], "score": 0, + "format": "rubric_binary", "assistant_harness": { "agent_name": "assistant" } @@ -2414,7 +2414,6 @@ "ao_staged_with_what_its_judge_read": { "verifications": { "rubric": { - "format": "rubric_binary", "results": [ { "id": "c1", @@ -2424,6 +2423,7 @@ } ], "score": 1.0, + "format": "rubric_binary", "compact_trajectory_s3_uri": null, "compact_trajectory_object_url": null, "judge_trajectory_s3_uri": "mem://judge/1.json", @@ -2495,7 +2495,6 @@ "ao_context_loss_the_gate_fails": { "verifications": { "rubric": { - "format": "rubric_binary", "results": [ { "id": "gate", @@ -2515,6 +2514,7 @@ } ], "score": 0.0, + "format": "rubric_binary", "compact_trajectory_s3_uri": null, "compact_trajectory_object_url": null, "judge_trajectory_s3_uri": "mem://judge/1.json", diff --git a/tst/data/verifier_golden/rubrics_verifier.json b/tst/data/verifier_golden/rubrics_verifier.json index 9b80b01b8..96b7d1ad9 100644 --- a/tst/data/verifier_golden/rubrics_verifier.json +++ b/tst/data/verifier_golden/rubrics_verifier.json @@ -2,7 +2,6 @@ "binary": { "verifications": { "vid": { - "format": "rubric_binary", "results": [ { "id": "table-reserved", @@ -20,6 +19,7 @@ } ], "score": 0.0, + "format": "rubric_binary", "compact_trajectory_s3_uri": null, "compact_trajectory_object_url": null, "judge_trajectory_s3_uri": null, @@ -36,7 +36,6 @@ "partial_weighted": { "verifications": { "vid": { - "format": "rubric_partial", "results": [ { "id": "q", @@ -63,6 +62,7 @@ } ], "score": 0.625, + "format": "rubric_partial", "compact_trajectory_s3_uri": null, "compact_trajectory_object_url": null, "judge_trajectory_s3_uri": null, @@ -79,7 +79,6 @@ "batches_with_a_bulk_verdict_retry": { "verifications": { "vid": { - "format": "rubric_binary", "results": [ { "id": "crit-01", @@ -123,6 +122,7 @@ } ], "score": 1.0, + "format": "rubric_binary", "compact_trajectory_s3_uri": null, "compact_trajectory_object_url": null, "judge_trajectory_s3_uri": null, @@ -223,7 +223,6 @@ "an_answer_retry_for_a_missing_id": { "verifications": { "vid": { - "format": "rubric_binary", "results": [ { "id": "crit-01", @@ -251,6 +250,7 @@ } ], "score": 1.0, + "format": "rubric_binary", "compact_trajectory_s3_uri": null, "compact_trajectory_object_url": null, "judge_trajectory_s3_uri": null, @@ -284,7 +284,6 @@ "a_compacted_trajectory": { "verifications": { "vid": { - "format": "rubric_binary", "results": [ { "id": "crit-01", @@ -312,6 +311,7 @@ } ], "score": 1.0, + "format": "rubric_binary", "compact_trajectory_s3_uri": "mem://compact/1.json", "compact_trajectory_object_url": "mem://compact/1.json", "judge_trajectory_s3_uri": null, @@ -328,7 +328,6 @@ "grade_up_to_turn": { "verifications": { "vid": { - "format": "rubric_binary", "results": [ { "id": "crit-01", @@ -348,6 +347,7 @@ } ], "score": 1.0, + "format": "rubric_binary", "compact_trajectory_s3_uri": null, "compact_trajectory_object_url": null, "judge_trajectory_s3_uri": null, @@ -365,7 +365,6 @@ "prompt_error": { "verifications": { "vid": { - "format": "rubric_binary", "results": [ { "id": "prompt_error", @@ -374,7 +373,8 @@ "message": "Skipped: prompt had error_type=timeout" } ], - "score": 0 + "score": 0, + "format": "rubric_binary" } }, "models": null, @@ -383,7 +383,6 @@ "the_gate_on_an_unmet_row": { "verifications": { "vid": { - "format": "rubric_binary", "results": [ { "id": "table-reserved", @@ -401,6 +400,7 @@ } ], "score": 0.0, + "format": "rubric_binary", "compact_trajectory_s3_uri": null, "compact_trajectory_object_url": null, "judge_trajectory_s3_uri": null, @@ -417,7 +417,6 @@ "the_gate_on_a_prompt_error": { "verifications": { "vid": { - "format": "rubric_binary", "results": [ { "id": "prompt_error", @@ -426,7 +425,8 @@ "message": "Skipped: prompt had error_type=timeout" } ], - "score": 0 + "score": 0, + "format": "rubric_binary" } }, "models": null, @@ -453,8 +453,6 @@ "rubric_evidence": { "verifications": { "vid": { - "format": "rubric_binary", - "judge_output_format": "rubric_evidence", "results": [ { "id": "table-reserved", @@ -486,6 +484,8 @@ } ], "score": 0.0, + "format": "rubric_binary", + "judge_output_format": "rubric_evidence", "compact_trajectory_s3_uri": null, "compact_trajectory_object_url": null, "judge_trajectory_s3_uri": null, @@ -510,8 +510,6 @@ "rubric_evidence_without_screenshots": { "verifications": { "vid": { - "format": "rubric_binary", - "judge_output_format": "rubric_evidence", "results": [ { "id": "no_screenshots", @@ -520,7 +518,9 @@ "message": "Skipped: rubric_evidence needs per-criterion frames, but the trajectory holds no screenshots" } ], - "score": 0 + "score": 0, + "format": "rubric_binary", + "judge_output_format": "rubric_evidence" } }, "models": null, @@ -528,53 +528,6 @@ }, "with_mistakes": { "verifications": { - "vid": { - "format": "rubric_binary", - "judge_output_format": "rubric_evidence_with_mistakes", - "results": [ - { - "id": "table-reserved", - "description": "A table is reserved", - "frame": "[table-reserved] action 3", - "evidence_status": "visible", - "justification": "about c1", - "score": 1.0, - "result": true - }, - { - "id": "party-size", - "description": "Party size is four", - "frame": "[party-size] action 3", - "evidence_status": "missing", - "justification": "about c2", - "score": 0.0, - "result": false - }, - { - "id": "no-autocomplete", - "description": "No suggestion was accepted", - "weight": -1, - "frame": "[no-autocomplete] action 3", - "evidence_status": "not_applicable", - "justification": "about c3", - "score": 1.0, - "result": true - } - ], - "score": 0.0, - "compact_trajectory_s3_uri": null, - "compact_trajectory_object_url": null, - "judge_trajectory_s3_uri": null, - "judge_trajectory_object_url": null, - "reasoning": "the frames show the booking", - "checks": [ - { - "id": "typed", - "passed": true, - "evidence": "x" - } - ] - }, "vid-trajectory-mistakes": { "format": "trajectory_mistakes", "source_verifier_id": "vid", @@ -625,6 +578,53 @@ "compact_trajectory_object_url": null, "judge_trajectory_s3_uri": null, "judge_trajectory_object_url": null + }, + "vid": { + "results": [ + { + "id": "table-reserved", + "description": "A table is reserved", + "frame": "[table-reserved] action 3", + "evidence_status": "visible", + "justification": "about c1", + "score": 1.0, + "result": true + }, + { + "id": "party-size", + "description": "Party size is four", + "frame": "[party-size] action 3", + "evidence_status": "missing", + "justification": "about c2", + "score": 0.0, + "result": false + }, + { + "id": "no-autocomplete", + "description": "No suggestion was accepted", + "weight": -1, + "frame": "[no-autocomplete] action 3", + "evidence_status": "not_applicable", + "justification": "about c3", + "score": 1.0, + "result": true + } + ], + "score": 0.0, + "format": "rubric_binary", + "judge_output_format": "rubric_evidence_with_mistakes", + "compact_trajectory_s3_uri": null, + "compact_trajectory_object_url": null, + "judge_trajectory_s3_uri": null, + "judge_trajectory_object_url": null, + "reasoning": "the frames show the booking", + "checks": [ + { + "id": "typed", + "passed": true, + "evidence": "x" + } + ] } }, "models": { @@ -636,9 +636,9 @@ }, "with_mistakes_without_screenshots": { "verifications": { - "vid": { - "format": "rubric_binary", - "judge_output_format": "rubric_evidence_with_mistakes", + "vid-trajectory-mistakes": { + "format": "trajectory_mistakes", + "source_verifier_id": "vid", "results": [ { "id": "no_screenshots", @@ -649,9 +649,7 @@ ], "score": 0 }, - "vid-trajectory-mistakes": { - "format": "trajectory_mistakes", - "source_verifier_id": "vid", + "vid": { "results": [ { "id": "no_screenshots", @@ -660,7 +658,9 @@ "message": "Skipped: rubric_evidence_with_mistakes needs per-criterion frames, but the trajectory holds no screenshots" } ], - "score": 0 + "score": 0, + "format": "rubric_binary", + "judge_output_format": "rubric_evidence_with_mistakes" } }, "models": null, @@ -668,9 +668,9 @@ }, "with_mistakes_on_a_prompt_error": { "verifications": { - "vid": { - "format": "rubric_binary", - "judge_output_format": "rubric_evidence_with_mistakes", + "vid-trajectory-mistakes": { + "format": "trajectory_mistakes", + "source_verifier_id": "vid", "results": [ { "id": "prompt_error", @@ -681,9 +681,7 @@ ], "score": 0 }, - "vid-trajectory-mistakes": { - "format": "trajectory_mistakes", - "source_verifier_id": "vid", + "vid": { "results": [ { "id": "prompt_error", @@ -692,7 +690,9 @@ "message": "Skipped: prompt had error_type=timeout" } ], - "score": 0 + "score": 0, + "format": "rubric_binary", + "judge_output_format": "rubric_evidence_with_mistakes" } }, "models": null, @@ -710,7 +710,6 @@ "unjudged_rows": { "verifications": { "vid": { - "format": "rubric_binary", "results": [ { "id": "crit-01", @@ -741,6 +740,7 @@ } ], "score": 1.0, + "format": "rubric_binary", "compact_trajectory_s3_uri": null, "compact_trajectory_object_url": null, "judge_trajectory_s3_uri": null, @@ -757,7 +757,6 @@ "an_agent_judge_timing_out_once": { "verifications": { "vid": { - "format": "rubric_binary", "results": [ { "id": "crit-01", @@ -777,6 +776,7 @@ } ], "score": 1.0, + "format": "rubric_binary", "compact_trajectory_s3_uri": null, "compact_trajectory_object_url": null, "judge_trajectory_s3_uri": "mem://judge/a2a.json", @@ -801,7 +801,6 @@ "a_runs_criteria_and_judge_overrides": { "verifications": { "vid": { - "format": "rubric_binary", "results": [ { "id": "only", @@ -812,6 +811,7 @@ } ], "score": 1.0, + "format": "rubric_binary", "compact_trajectory_s3_uri": null, "compact_trajectory_object_url": null, "judge_trajectory_s3_uri": null, diff --git a/tst/unit/task_step/verifiers/test_legacy_entries.py b/tst/unit/task_step/verifiers/test_legacy_entries.py index 6ed9f785a..46f418d9b 100644 --- a/tst/unit/task_step/verifiers/test_legacy_entries.py +++ b/tst/unit/task_step/verifiers/test_legacy_entries.py @@ -1,7 +1,8 @@ """run_container_unit_tests_verifier, verify_sandbox, env_outcome_verifier and aggregate_verifiers on the -VerifierTaskStep base write the legacy ``verifications`` entry they wrote before it, byte for byte with its key order, -and raise what they raised (``tst/data/verifier_golden/legacy_entries.json``, recorded from main at 25b862bc6, object -urls under ``/``), but for ``CHANGED``; and each writes its named grade beside it.""" +VerifierTaskStep base write the legacy ``verifications`` entry their golden file holds, byte for byte with its key +order, and raise what it holds (``tst/data/verifier_golden/legacy_entries.json``, recorded on main at 25b862bc6 but for +the base's two deliberate changes: an aggregate's entry has its empty ``results``, and a check script that returns None +is told so); and each writes its named grade beside it.""" import asyncio import itertools import json @@ -25,16 +26,10 @@ ) from agent_env.task_step.task_steps.verifiers.verify_sandbox import VerifySandboxTaskStep from tst.unit.computer_home import Computer -from tst.util.verifier_golden import GOLDEN +from tst.util import verifier_golden -RECORDED = json.loads((GOLDEN / "legacy_entries.json").read_text()) REWARD = "/logs/verifier/reward.txt" SCENARIOS = {} -# What the base changes on purpose: an aggregate's entry has its (empty) results, and a script that returns None is -# told so rather than by a log line's len(). -CHANGED_RAISES = { - "outcome_returns_none": "TypeError: verify() in artifact 's' returned None, not a list of criterion rows", -} def scenario(fn): @@ -42,16 +37,14 @@ def scenario(fn): return fn -def outcome(step, context) -> tuple[str | None, str | None]: - """What ``step`` raised running on ``context``, and its entry as JSON with the object store's urls normalised.""" +def outcome(step, context) -> tuple[str | None, dict | None]: + """What ``step`` raised running on ``context``, and its entry with the object store's urls normalised.""" raised = None try: asyncio.run(step.execute(context)) except Exception as exc: raised = f"{type(exc).__name__}: {exc}" - entry = (context.metadata.get("verifications") or {}).get(step.verifier_id) - objects = get_config().get_object_store().object_url("x")[:-1] - return raised, json.dumps(entry).replace(objects, "/") + return raised, verifier_golden.normalised((context.metadata.get("verifications") or {}).get(step.verifier_id)) class _ContainerSandbox: @@ -475,29 +468,19 @@ def aggregate_missing_a_source(monkeypatch, tmp_path): return _aggregate(["fs", "gone"]), _sources() -def _expected(name, step) -> tuple[str | None, dict | None]: - entry = RECORDED[name]["entry"] - if step.type == "aggregate_verifiers" and entry is not None: - entry = {"results": [], **entry} - return CHANGED_RAISES.get(name, RECORDED[name]["raised"]), entry - - @pytest.mark.parametrize("name", list(SCENARIOS)) -def test_each_path_writes_the_entry_and_raises_what_it_did_before_the_base(name, local_stores, monkeypatch, tmp_path): +def test_each_path_writes_the_entry_and_raises_what_its_golden_file_holds(name, local_stores, monkeypatch, tmp_path): step, context = SCENARIOS[name](monkeypatch, tmp_path) - expected_raise, expected_entry = _expected(name, step) raised, entry = outcome(step, context) - assert raised == expected_raise - assert entry == json.dumps(expected_entry) + verifier_golden.check("legacy_entries.json", name, {"raised": raised, "entry": entry}) @pytest.mark.parametrize("name", list(SCENARIOS)) def test_each_path_writes_its_named_grade(name, local_stores, monkeypatch, tmp_path): step, context = SCENARIOS[name](monkeypatch, tmp_path) raised, entry = outcome(step, context) - entry = json.loads(entry) grade = context.metadata["grades"][step.id] diff --git a/tst/unit/task_step/verifiers/test_rubrics_subclasses_golden.py b/tst/unit/task_step/verifiers/test_rubrics_subclasses_golden.py index 828e8e8b8..53987acd5 100644 --- a/tst/unit/task_step/verifiers/test_rubrics_subclasses_golden.py +++ b/tst/unit/task_step/verifiers/test_rubrics_subclasses_golden.py @@ -1,5 +1,6 @@ -"""user_sim_judge and always_on_agent_verifier leave the legacy entries main left, and a named grade that agrees with -them, with their own tests' fakes standing in for the judge and the runs they grade.""" +"""user_sim_judge and always_on_agent_verifier leave the legacy entries their golden file holds (main's, with +``results`` and ``score`` written first), and a named grade that agrees with them, with their own tests' fakes standing +in for the judge and the runs they grade.""" from __future__ import annotations @@ -67,7 +68,7 @@ async def _judged_chat(monkeypatch, run, fake, *, matches=True, run_persona=T.OT ("us_a_grade_up_to_turn_that_cuts_nothing", [("reply", "ok")], {}, {"overrides": {"step_params": {"judge_user_sim": {"grade_up_to_turn": 3}}}}), ]) -async def test_a_judged_chat_leaves_the_legacy_entry_main_left(monkeypatch, case, answers, script, kw): +async def test_a_judged_chat_leaves_the_legacy_entry_its_golden_file_holds(monkeypatch, case, answers, script, kw): kw = dict(kw) graded = kw.pop("graded", None) got = await _judged_chat(monkeypatch, T._run(*answers, graded=graded), T.FakeModel(script), **kw) @@ -77,7 +78,7 @@ async def test_a_judged_chat_leaves_the_legacy_entry_main_left(monkeypatch, case @pytest.mark.usefixtures("local_stores") @pytest.mark.asyncio -async def test_a_chat_run_skipped_for_its_error_or_for_no_reads_leaves_the_entries_main_left(monkeypatch): +async def test_a_chat_run_skipped_for_its_error_or_for_no_reads_leaves_the_entries_its_golden_file_holds(monkeypatch): context = await T._chat(monkeypatch) context.prompt_responses[-1].error_type = "user_sim_error" verifier_golden.check(FILE, "us_a_chat_run_that_ended_in_an_error", @@ -89,7 +90,7 @@ async def test_a_chat_run_skipped_for_its_error_or_for_no_reads_leaves_the_entri @pytest.mark.usefixtures("local_stores") @pytest.mark.asyncio -async def test_a_chat_run_the_gate_fails_leaves_the_entry_main_left(monkeypatch): +async def test_a_chat_run_the_gate_fails_leaves_the_entry_its_golden_file_holds(monkeypatch): context = await T._chat(monkeypatch) got = await _ran(T._step(T.FakeModel({"oversharing": T.FAIL()}), monkeypatch), context) @@ -99,7 +100,7 @@ async def test_a_chat_run_the_gate_fails_leaves_the_entry_main_left(monkeypatch) @pytest.mark.usefixtures("local_stores") @pytest.mark.asyncio -async def test_an_always_on_run_the_gate_fails_leaves_the_entry_main_left(monkeypatch, tmp_path): +async def test_an_always_on_run_the_gate_fails_leaves_the_entry_its_golden_file_holds(monkeypatch, tmp_path): rig, context, prompt = await T._always_on(monkeypatch, tmp_path) got = await _ran(T._step(T.FakeModel({"guidance": T.FAIL(turns=[2])}), monkeypatch, run_step_id="week"), context) @@ -115,7 +116,8 @@ def _harness(context: TaskStepContext) -> None: @pytest.mark.usefixtures("local_stores") @pytest.mark.asyncio -async def test_an_unstaged_always_on_grade_with_its_harness_leaves_the_entry_main_left(monkeypatch, tmp_path): +async def test_an_unstaged_always_on_grade_with_its_harness_leaves_the_entry_its_golden_file_holds(monkeypatch, + tmp_path): rig, context = await A._run(monkeypatch, tmp_path) _harness(context) verifier = A._verifier(default_model=JUDGE_MODEL) @@ -126,7 +128,7 @@ async def test_an_unstaged_always_on_grade_with_its_harness_leaves_the_entry_mai @pytest.mark.usefixtures("local_stores") @pytest.mark.asyncio -async def test_an_always_on_prompt_error_the_gate_fails_leaves_the_entry_main_left(monkeypatch, tmp_path): +async def test_an_always_on_prompt_error_the_gate_fails_leaves_the_entry_its_golden_file_holds(monkeypatch, tmp_path): def fails(n, texts, rig): raise TimeoutError("no answer") @@ -142,7 +144,7 @@ def fails(n, texts, rig): @pytest.mark.usefixtures("local_stores") @pytest.mark.asyncio -async def test_a_staged_always_on_grade_with_what_its_judge_read_leaves_the_entry_main_left(monkeypatch): +async def test_a_staged_always_on_grade_with_what_its_judge_read_leaves_the_entry_its_golden_file_holds(monkeypatch): context, sandbox, sources = A._staged_run(monkeypatch) verifier = A._verifier(use_agent_judge=True, use_trajectory=True, agent_name="judge", default_model=JUDGE_MODEL) A._agent_judge(monkeypatch, verifier, sources) @@ -152,7 +154,7 @@ async def test_a_staged_always_on_grade_with_what_its_judge_read_leaves_the_entr @pytest.mark.usefixtures("local_stores") @pytest.mark.asyncio -async def test_an_always_on_context_loss_the_gate_fails_leaves_the_entry_main_left(monkeypatch): +async def test_an_always_on_context_loss_the_gate_fails_leaves_the_entry_its_golden_file_holds(monkeypatch): context, sandbox, sources = A._v7_staged_run(monkeypatch, summary_drops=" FACT: the gate code must never be shared. |") _harness(context) verifier = A._verifier(use_agent_judge=True, use_trajectory=True, agent_name="judge", fail_task_when_not_passing=True, diff --git a/tst/unit/task_step/verifiers/test_rubrics_verifier_golden.py b/tst/unit/task_step/verifiers/test_rubrics_verifier_golden.py index 72e6c31d5..058fa0b69 100644 --- a/tst/unit/task_step/verifiers/test_rubrics_verifier_golden.py +++ b/tst/unit/task_step/verifiers/test_rubrics_verifier_golden.py @@ -1,6 +1,6 @@ -"""rubrics_verifier leaves the legacy entries main left on every path that writes one, or none, and a named grade that -agrees with them, with the judge faked below the seams the verifier's own code owns (``_prompt_llm_judge``, -``_invoke_judge_a2a``).""" +"""rubrics_verifier leaves the legacy entries its golden file holds on every path that writes one, or none (main's, +with ``results`` and ``score`` written first), and a named grade that agrees with them, with the judge faked below the +seams the verifier's own code owns (``_prompt_llm_judge``, ``_invoke_judge_a2a``).""" from __future__ import annotations @@ -190,7 +190,7 @@ async def judge(*, eval_prompt, judge_a2a_url, judge_agent_card, judge_agent): @pytest.mark.asyncio @pytest.mark.parametrize("case", list(CASES)) -async def test_a_run_leaves_the_legacy_entries_main_left_and_a_grade_that_agrees(case): +async def test_a_run_leaves_the_legacy_entries_its_golden_file_holds_and_a_grade_that_agrees(case): step, context = CASES[case]() raised = None try: diff --git a/tst/util/verifier_golden.py b/tst/util/verifier_golden.py index 4c6d95942..357f0095d 100644 --- a/tst/util/verifier_golden.py +++ b/tst/util/verifier_golden.py @@ -1,19 +1,23 @@ -"""The legacy ``verifications`` entries built-in verifiers left on main before they moved onto ``VerifierTaskStep``, -recorded under ``tst/data/verifier_golden``: a run must leave the same entries, number types included, the same -models record and the same error. Key order isn't compared: the base writes an entry's ``results`` and ``score`` -first, where the rubrics family wrote its ``format`` before them.""" +"""Golden files for the built-in verifiers' legacy ``verifications`` entries, under ``tst/data/verifier_golden``: first +recorded on main before the verifiers moved onto ``VerifierTaskStep``, and the contract since. A run must leave what the +file holds, number types and key order included. After a deliberate change, re-record with +``AGENTENV_TEST_RECORD_GOLDEN=1`` and review the files' diff: each check then writes what the run left instead.""" from __future__ import annotations +import fcntl import functools import json +import os from pathlib import Path from typing import Any, Optional +from agent_env.config import get_config from agent_env.task_step.context import TaskStepContext, VerifierNotPassingError from agent_env.task_step.task_steps.verifiers.verifier import VerifierTaskStep GOLDEN = Path(__file__).parents[1] / "data" / "verifier_golden" +RECORDING = os.environ.get("AGENTENV_TEST_RECORD_GOLDEN") == "1" def observed(context: TaskStepContext, raised: Optional[BaseException]) -> dict[str, Any]: @@ -21,15 +25,37 @@ def observed(context: TaskStepContext, raised: Optional[BaseException]) -> dict[ "raised": None if raised is None else f"{type(raised).__name__}: {raised}"} +def normalised(value: Any) -> Any: + """``value`` with the object store's urls, which differ from run to run, under ``/``.""" + objects = get_config().get_object_store().object_url("x")[:-1] + return json.loads(json.dumps(value).replace(objects, "/")) + + @functools.cache def _recorded(file: str) -> dict[str, Any]: return json.loads((GOLDEN / file).read_text()) def check(file: str, case: str, got: dict[str, Any]) -> None: + """``got`` is what ``file`` holds for ``case``: the same values, number types and key order; written there + instead when re-recording.""" + if RECORDING: + _record(file, case, got) + return expected = _recorded(file)[case] assert json.loads(json.dumps(got)) == expected - assert json.dumps(got, sort_keys=True) == json.dumps(expected, sort_keys=True) + assert json.dumps(got) == json.dumps(expected) + + +def _record(file: str, case: str, got: dict[str, Any]) -> None: + """``case`` written into ``file`` under a lock, so parallel workers each add theirs.""" + with open(GOLDEN / file, "r+") as f: + fcntl.flock(f, fcntl.LOCK_EX) + cases = json.loads(f.read()) + cases[case] = json.loads(json.dumps(got)) + f.seek(0) + f.write(json.dumps(cases, indent=1) + "\n") + f.truncate() def check_grade(context: TaskStepContext, step: VerifierTaskStep, raised: Optional[BaseException]) -> None: