Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
39 commits
Select commit Hold shift + click to select a range
98a7493
feat(verifiers): read an agent turn's tool calls and text from any st…
sarah-chitty-1 Nov 13, 2026
24c5385
feat(rubrics_verifier): opt-in judge timeout retries, a judge turn ca…
sarah-chitty-1 Nov 14, 2026
1ccdf35
feat(always_on_agent_verifier): with use_trajectory, the agent judge …
sarah-chitty-1 Nov 14, 2026
a732ea6
fix(claude-code): keep a timed-out turn's error code cli_error
sarah-chitty-1 Nov 14, 2026
b3f15e4
Merge #679's timeout error-code revert into #680
sarah-chitty-1 Nov 14, 2026
7dc7dd8
fix(turn_events): a generically named tool span takes the prompt's to…
sarah-chitty-1 Nov 14, 2026
0f3e2e7
fix(rubrics_verifier): a judge the verifier deployed is terminated wh…
sarah-chitty-1 Nov 14, 2026
6fcaa72
Merge #678's Greptile fixes into #679
sarah-chitty-1 Nov 14, 2026
222407d
Merge #679's Greptile fixes into #680
sarah-chitty-1 Nov 14, 2026
6b5828d
fix(always_on_staging): split long side-file lines, label unlogged wr…
sarah-chitty-1 Nov 14, 2026
c0ab41f
feat(turn_events): keep the agent's thinking when the trajectory reco…
sarah-chitty-1 Nov 14, 2026
a8abe2d
Merge #679 into #680
sarah-chitty-1 Nov 14, 2026
1aa7336
Merge #678's thinking reader into #679
sarah-chitty-1 Nov 14, 2026
86408d7
feat(always_on_staging): the agent's thinking is an assistant_thinkin…
sarah-chitty-1 Nov 14, 2026
34e3f11
feat(turn_events): read the PSI harness's psi.reasoning thinking bloc…
sarah-chitty-1 Nov 14, 2026
ae4aa3d
Merge #678 into #679
sarah-chitty-1 Nov 14, 2026
075041b
Merge #679 into #680
sarah-chitty-1 Nov 14, 2026
36a8648
fix(turn_events): read an OpenInference Responses-API span's text (me…
sarah-chitty-1 Nov 14, 2026
50d9256
Merge #678 into #679
sarah-chitty-1 Nov 14, 2026
a2bd86a
Merge #679 into #680
sarah-chitty-1 Nov 14, 2026
a7f5e57
fix(rubrics_verifier): with timeout retries, the judge gets a 60 s ma…
sarah-chitty-1 Nov 16, 2026
d9b567d
Merge #679 into #680
sarah-chitty-1 Nov 16, 2026
869c175
refactor(rubrics_verifier): judge_agent_config is given the judge's c…
sarah-chitty-1 Nov 16, 2026
caf8e2d
Merge #679 into #680
sarah-chitty-1 Nov 16, 2026
8b0778d
fix(always_on_agent_verifier): narrow the judge to Bash and Read only…
sarah-chitty-1 Nov 16, 2026
fade3df
feat(verifiers): read the PSI neutral harness v3 trace (compaction, m…
sarah-chitty-1 Nov 16, 2026
8f85f62
feat(always_on_staging): a line in index.md for each compaction the h…
sarah-chitty-1 Nov 16, 2026
88e3b5e
fix(always_on staging): v3 review fixes — judge instructions, wake or…
sarah-chitty-1 Nov 16, 2026
4bc67a8
feat(always_on_agent_verifier): v3 runs — the assistant owns its comp…
sarah-chitty-1 Nov 16, 2026
33a2aa0
fix(always_on_agent_verifier): read the harness's settings from the t…
sarah-chitty-1 Nov 16, 2026
621eca5
fix(always_on judge): search every wake before claiming absence; judg…
sarah-chitty-1 Nov 16, 2026
c668603
Merge main into #716
sarah-chitty-1 Nov 16, 2026
5f2fee9
refactor(verifiers): read the PSI harness v5 trace names; no psi.* na…
sarah-chitty-1 Nov 16, 2026
e5ae1eb
feat(verifiers): a tool the harness provides itself (harness.tool) is…
sarah-chitty-1 Nov 16, 2026
3de93f9
fix(always_on_agent_verifier): the absence rule searches the long tex…
sarah-chitty-1 Nov 17, 2026
2980575
fix(turn_events): output messages that aren't a list add no parts ins…
sarah-chitty-1 Nov 17, 2026
6de93ca
fix(turn_events): the harness reading turns on only for a trace with …
sarah-chitty-1 Nov 17, 2026
12e9956
feat(verifiers): read the PSI harness v7 trace — who wrote a summary,…
sarah-chitty-1 Nov 17, 2026
2b69c88
fix(always_on_staging): a turn staged by position keeps how it ended,…
sarah-chitty-1 Nov 17, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view

Large diffs are not rendered by default.

163 changes: 127 additions & 36 deletions src/agent_env/task_step/task_steps/verifiers/always_on_staging.py

Large diffs are not rendered by default.

Large diffs are not rendered by default.

276 changes: 264 additions & 12 deletions tst/unit/task_step/test_always_on_agent_verifier.py

Large diffs are not rendered by default.

211 changes: 200 additions & 11 deletions tst/unit/task_step/verifiers/judge_utils/test_turn_events.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,16 @@
import json
from pathlib import Path

import pytest

from agent_env.task_step.task_steps.verifiers.judge_utils import trajectory_filter
from agent_env.task_step.task_steps.verifiers.judge_utils.turn_events import (
CLAUDE_CODE,
GEMINI,
IMAGE_OMITTED,
OTEL,
UNKNOWN,
TrajectoryFormatError,
readable_result,
turn_events,
)
Expand All @@ -26,18 +29,18 @@ def _calls(events) -> list[dict]:


def test_openinference_spans_keep_the_calls_and_the_model_s_text_and_drop_what_every_span_repeats():
raw = (RECORDED / "turn_otel_wake4.json").read_bytes()
raw = (RECORDED / "turn_otel_v7_compaction_claude.json").read_bytes()
events = turn_events(raw)

assert events.format == OTEL and events.raw_bytes == len(raw)
calls = _calls(events)
assert [c["tool"] for c in calls] == ["porchbite_get_order", "porchbite_list_conversations",
"porchbite_spending_by_month", "stay_quiet"]
assert calls[0]["input"] == {"order_key": "PB-C2SHSLHX"} and calls[0]["result"]["order"]["order_key"] == "PB-C2SHSLHX"
assert [c["tool"] for c in calls] == ["memory_add", "slow_search"]
assert calls[1]["input"] == {"seconds": 0.0} and calls[1]["result"] == (
"slow_search finished after 0 s: 3 flights found")
texts = [e["text"] for e in events.events if e["type"] == "text"]
assert len(texts) == 2 and texts[-1].startswith("Nothing has changed since Nina")
assert texts == ["Let me look.", "Done (11).", "Let me look.", "Done (14)."]
staged = json.dumps(events.events)
for repeated in ("llm.input_messages", "gmail_archive_thread", "harness.usage"):
for repeated in ("llm.input_messages", "llm.token_count", "harness.usage"):
assert repeated in raw.decode() and repeated not in staged
assert len(staged) < len(raw) / 4

Expand Down Expand Up @@ -116,17 +119,31 @@ def test_the_agent_s_thinking_is_kept_when_it_was_recorded_and_an_empty_block_is
("thinking", "Ask before paying $855."), ("text", "Asking the user.")]


def test_the_psi_harness_s_reasoning_on_an_llm_span_comes_before_that_call_s_text():
def test_an_llm_span_s_reasoning_parts_come_in_order_before_that_call_s_text_and_an_empty_one_is_dropped():
spans = [{"name": "ChatCompletion", "start_time": "1", "attributes": {
"openinference.span.kind": "LLM",
"llm.output_messages.0.message.role": "assistant",
"llm.output_messages.0.message.content": "I'll ask Nina first.",
"psi.reasoning": json.dumps([{"type": "thinking", "thinking": ""},
{"type": "thinking", "thinking": "Paying $855 needs her OK."}])}}]
"gen_ai.output.messages": json.dumps([{"role": "assistant", "finish_reason": "stop", "parts": [
{"type": "reasoning", "content": ""}, {"type": "reasoning", "content": "Paying $855 needs her OK."},
{"type": "reasoning", "content": "So ask."}, {"type": "text", "content": "I'll ask Nina first."}]}])}}]

events = [(e["type"], e["text"]) for e in turn_events(json.dumps(spans)).events if e["type"] != "final"]

assert events == [("thinking", "Paying $855 needs her OK."), ("text", "I'll ask Nina first.")]
assert events == [("thinking", "Paying $855 needs her OK.\nSo ask."), ("text", "I'll ask Nina first.")]


def test_output_messages_that_aren_t_a_list_add_nothing_and_the_span_still_reads():
for bad in ("null", "7", json.dumps({"role": "assistant"})):
spans = [{"name": "ChatCompletion", "start_time": "1", "attributes": {
"openinference.span.kind": "LLM",
"llm.output_messages.0.message.role": "assistant",
"llm.output_messages.0.message.content": "Ordered the pie.",
"gen_ai.output.messages": bad}}]

events = [(e["type"], e["text"]) for e in turn_events(json.dumps(spans)).events if e["type"] != "final"]

assert events == [("text", "Ordered the pie.")], bad


def test_a_responses_api_llm_span_s_text_and_reasoning_are_read():
Expand All @@ -135,7 +152,9 @@ def test_a_responses_api_llm_span_s_text_and_reasoning_are_read():
"llm.output_messages.0.message.role": "assistant",
"llm.output_messages.0.message.contents.0.message_content.type": "text",
"llm.output_messages.0.message.contents.0.message_content.text": "Booked for Friday.",
"psi.reasoning": json.dumps([{"type": "thinking", "thinking": "She said Friday works."}])}}]
"gen_ai.output.messages": json.dumps([{"role": "assistant", "finish_reason": "stop", "parts": [
{"type": "reasoning", "content": "She said Friday works."},
{"type": "text", "content": "Booked for Friday."}]}])}}]

events = [(e["type"], e["text"]) for e in turn_events(json.dumps(spans)).events if e["type"] != "final"]

Expand Down Expand Up @@ -171,3 +190,173 @@ def test_the_compactor_rubrics_verifier_uses_still_reads_tool_spans_the_same_way
assert events == [{"type": "tool_call", "tool": "email_send", "input": {"to": "a@b"}, "timestamp": "1"},
{"type": "tool_result", "tool": "email_send", "output": {"ok": True}, "timestamp": "2"}]
assert files == [] and trajectory_filter.tool_span_io(span) == ("email_send", {"to": "a@b"}, {"ok": True})


def _v7(name: str) -> list[dict]:
"""A recorded wake of the PSI neutral harness v7 (redacted from a fake-model run): it starts with a compaction whose
summary the assistant's own model wrote."""
return json.loads((RECORDED / f"turn_otel_v7_compaction_{name}.json").read_text())


def _summary(spans: list[dict]) -> str:
span = next(s for s in spans if s["name"] == "harness.compaction")
return json.loads(span["attributes"]["gen_ai.output.messages"])[0]["parts"][0]["content"]


def test_a_v7_compaction_wake_keeps_the_assistant_s_words_and_makes_the_harness_s_parts_notes():
spans = _v7("claude")
summary = _summary(spans)

traj = turn_events(json.dumps(spans))
events = traj.events

assert traj.agent_labels is True
assert [(e["type"], e.get("step")) for e in events] == [
("note", "pre_compaction_save"), ("thinking", "pre_compaction_save"), ("text", "pre_compaction_save"),
("tool_call", "pre_compaction_save"), ("thinking", "pre_compaction_save"), ("text", "pre_compaction_save"),
("note", None), ("note", "after_compaction"),
("thinking", None), ("text", None), ("tool_call", None), ("thinking", None), ("text", None)]
save, compaction, after = (e["text"] for e in events if e["type"] == "note")
assert save.startswith("Message the harness sent the assistant (step pre_compaction_save; not from the user):\n"
"[Harness: older history is about to be summarized")
assert compaction == ("Compaction summary (written by the assistant's own model, in a summary turn the harness "
"asked for; it replaces the older history the assistant sees; wakes 1–3 are now in the "
f"summary; wakes 4–5 kept word for word):\n{summary}")
assert next(e for e in events if "compaction" in e)["compaction"] == {
"written_by": "assistant", "wakes_summarized": 3, "wakes_kept": 2, "first_kept_wake": 4}
assert after.startswith("Message the harness sent the assistant (step after_compaction; re-sent by the harness, "
"it repeats the user's earlier words):\n[Harness: older history was summarized")
# The summary call is a step of the harness's (agent "harness", under no tool call): its text and reasoning are not
# among the turn's own words (the compaction note holds the summary), though its OpenInference output has it.
own = [e["text"] for e in events if e["type"] in ("text", "thinking")]
assert not any(summary in t or "step 12" in t for t in own)
assert [(c["tool"], c.get("internal")) for c in _calls(traj)] == [("memory_add", True), ("slow_search", None)]


def _compaction_span(spans: list[dict]) -> dict:
return next(s for s in spans if s["name"] == "harness.compaction")


def test_a_compaction_s_failed_save_turn_and_the_wait_it_caused_are_said():
spans = _v7("claude")
_compaction_span(spans)["attributes"].update({"harness.compaction.next_wake_waited_s": 8.694,
"harness.compaction.save_error": "APITimeoutError: timed out"})

compaction = next(e["text"] for e in turn_events(json.dumps(spans)).events if "compaction" in e)

assert ("; wakes 1–3 are now in the summary; wakes 4–5 kept word for word; the save turn before it failed: "
"APITimeoutError: timed out; this wake started 8.694 s late (real time) waiting for it):") in compaction


def test_a_failed_compaction_says_the_history_was_left_whole():
# As the harness records one: its summary call raised, so the span has the error and no summary, numbers or
# message after it.
spans = [s for s in _v7("claude") if s.get("attributes", {}).get("harness.message.purpose") != "after_summary"]
attrs = _compaction_span(spans)["attributes"]
for name in ("gen_ai.output.messages", "harness.compaction.chars_after", "harness.compaction.wakes_summarized",
"harness.compaction.first_kept_wake"):
del attrs[name]
attrs["harness.compaction.error"] = "APIConnectionError: Connection error."

[note] = [e for e in turn_events(json.dumps(spans)).events if "compaction" in e]

assert note["compaction"] == {"written_by": "assistant", "wakes_kept": 2,
"error": "APIConnectionError: Connection error."}
assert note["text"].endswith("; it failed, so the history was left whole: APIConnectionError: Connection error.): "
"no summary recorded")


def test_a_summary_the_harness_wrote_is_said_to_be_the_harness_s():
spans = _v7("gpt")
_compaction_span(spans)["attributes"]["harness.compaction.written_by"] = "harness"

[note] = [e for e in turn_events(json.dumps(spans)).events if "compaction" in e]

assert note["compaction"]["written_by"] == "harness"
assert note["text"].startswith("Compaction summary (written by the harness, not the assistant; it replaces the "
"older history the assistant sees; wakes 1–3 are now in the summary;")


@pytest.mark.parametrize("written_by", [None, "", "model", "Assistant"])
def test_a_compaction_that_doesn_t_say_who_wrote_its_summary_is_refused(written_by):
spans = _v7("claude")
attrs = _compaction_span(spans)["attributes"]
if written_by is None:
del attrs["harness.compaction.written_by"]
else:
attrs["harness.compaction.written_by"] = written_by

with pytest.raises(TrajectoryFormatError, match=r"harness\.compaction\.written_by=.*not one of \['assistant', "
r"'harness'\]"):
turn_events(json.dumps(spans))


def test_a_v7_gpt_save_turn_keeps_its_reasoning_and_calls_marked_as_the_harness_s_step():
events = turn_events(json.dumps(_v7("gpt"))).events

save = [e for e in events if e.get("step") == "pre_compaction_save" and e["type"] != "note"]
assert [(e["type"], e.get("tool"), e.get("text")) for e in save] == [
("thinking", None, "GPT reasoning 10."), ("tool_call", "memory_add", None),
("thinking", None, "GPT reasoning 11."), ("text", None, "Done (11).")]
assert [e["text"] for e in events if e["type"] == "text"] == ["Done (11).", "Done (14)."]


def test_a_helper_agent_s_spans_are_named_by_its_agent_and_the_call_that_started_it():
spans = _v7("claude")
delegate = next(s for s in spans if s["name"] == "slow_search")
helper = [s for s in spans if s["name"] == "generation" and s["start_time"] > delegate["start_time"]]
for s in helper:
s["parent_id"] = delegate["context"]["span_id"]
s["attributes"]["gen_ai.agent.name"] = "researcher"

events = turn_events(json.dumps(spans)).events

assert [(e["type"], e.get("subagent")) for e in events if e.get("subagent")] == [
("thinking", f"researcher@{delegate['context']['span_id']}"), ("text", f"researcher@{delegate['context']['span_id']}")]


def test_a_failed_wake_s_message_is_a_note_in_its_step():
span = {"name": "harness.message", "start_time": "1", "attributes": {
"gen_ai.agent.name": "harness", "gen_ai.main_agent.name": "Orchestrator",
"harness.message.purpose": "failed_wake",
"harness.message.text": "[Harness: your last wake ended with an error before it finished (TimeoutError).]"}}

[note] = turn_events(json.dumps([span])).events

assert note == {"type": "note", "step": "failed_wake", "text": (
"Message the harness sent the assistant (step failed_wake; not from the user):\n"
"[Harness: your last wake ended with an error before it finished (TimeoutError).]")}


def test_a_harness_wake_with_no_compaction_messages_or_helpers_gets_no_notes_or_marks():
# A wake with no compaction before it: the recorded wake's own turn and its usage record.
spans = _v7("claude")
turn_start = next(s["start_time"] for s in spans if s["name"] == "mcp_tools")
spans = [s for s in spans if s["start_time"] >= turn_start]
next(s for s in spans if s["name"] == "harness.usage")["attributes"]["harness.usage.compactions"] = 0

traj = turn_events(json.dumps(spans))

assert [(e["type"], e.get("tool")) for e in traj.events] == [
("thinking", None), ("text", None), ("tool_call", "slow_search"), ("thinking", None), ("text", None)]
assert not any("step" in e or "subagent" in e for e in traj.events)
assert traj.agent_labels is True # it is a harness's trace: its harness.* labels count


def test_a_hand_off_between_agents_without_harness_labels_keeps_both_agents_words():
# An OpenAI Agents SDK hand-off: two agents, standard GenAI names only, no ``harness.*`` labels.
spans = [
{"name": "triage", "start_time": "1", "context": {"span_id": "a"}, "attributes": {
"openinference.span.kind": "LLM", "gen_ai.agent.name": "triage", "gen_ai.main_agent.name": "triage",
"llm.output_messages.0.message.role": "assistant",
"llm.output_messages.0.message.content": "Handing you to billing."}},
{"name": "billing", "start_time": "2", "context": {"span_id": "b"}, "attributes": {
"openinference.span.kind": "LLM", "gen_ai.agent.name": "billing", "gen_ai.main_agent.name": "triage",
"llm.output_messages.0.message.role": "assistant",
"llm.output_messages.0.message.content": "Refunded $40 to your card."}}]

traj = turn_events(json.dumps(spans))
texts = [e["text"] for e in traj.events if e["type"] == "text"]

assert traj.agent_labels is False
assert texts == ["Handing you to billing.", "Refunded $40 to your card."]
Loading
Loading