Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -370,6 +370,9 @@ def dated(r: dict, items: list) -> list[dict]: # an earlier read's items, each
out += [f" [{_stamp(at(s), now)}] {s['text']}{attached_note(s.get('files'))}" for s in own]
out.append("\nNEW FROM YOUR AI ASSISTANT (this is what you're answering):")
out += [_message(m, _stamp(at(m), now)) for m in messages]
if read.get("user_note"): # from a post-turn hook: what the user saw checking the work, never sent to the assistant
out.append("\nWHAT YOU FOUND WHEN YOU CHECKED ITS WORK YOURSELF (the assistant doesn't see this):")
out += [" " + line for line in read["user_note"].strip().splitlines()]
out.append("\n" + CLOSING.format(first=_first(person)))
out.append("Answer in JSON: " + ANSWER_LINE)
return "\n".join(out)
Expand Down
19 changes: 12 additions & 7 deletions src/agent_env/task_step/task_steps/post_turn_hooks.py
Original file line number Diff line number Diff line change
Expand Up @@ -163,17 +163,17 @@ async def call_tool(self, tool: str, args: Optional[dict], role: str) -> dict:
class _Answer:
"""The user side's answer this turn: asked at most once, then the same for every later call."""

def __init__(self, ask_user: Callable[[], Awaitable[Answer]],
def __init__(self, ask_user: Callable[..., Awaitable[Answer]],
user_triggers: Callable[[], Awaitable[Optional[Answer]]]) -> None:
self.value: Optional[Answer] = None
self._ask_user = ask_user
self._user_triggers = user_triggers
self._asking = asyncio.Lock()

async def ask_user(self) -> Answer:
async def ask_user(self, note: Optional[str] = None) -> Answer:
async with self._asking:
if self.value is None:
self.value = await self._ask_user()
self.value = await (self._ask_user(note=note) if note is not None else self._ask_user())
return self.value

async def user_triggers(self) -> Optional[Answer]:
Expand Down Expand Up @@ -204,9 +204,14 @@ def metadata(self) -> Mapping[str, Any]:
"""A read-only copy of the run's metadata, its secret keys stripped."""
return MappingProxyType(_strip_redacted_keys(self._context.metadata))

async def ask_user(self) -> Answer:
"""Today's user side: agent triggers, then a human or the LLM user sim, which sees only replies sent here."""
return await self._answer.ask_user()
async def ask_user(self, note: Optional[str] = None) -> Answer:
"""Today's user side: agent triggers, then a human or the LLM user sim, which sees only replies sent here.

``note`` is shown to an LLM user sim with the agent's reply, as something only it sees (what the person would
find by checking the work themselves); the agent never sees it. A human peer gets no note."""
if note is not None and (not isinstance(note, str) or not note.strip()):
raise ValueError("a note for the user side is non-empty text")
return await self._answer.ask_user(note)

async def user_triggers(self) -> Optional[Answer]:
"""Agent triggers alone; None, leaving no trace, when none fires."""
Expand Down Expand Up @@ -263,7 +268,7 @@ def _parts(message: Any, hook: PostTurnHook) -> list[dict]:

async def run_post_turn_hooks(hooks: PostTurnHooks, *, context: TaskStepContext, step_id: str, number: int, last: bool,
agent_reply: str, agent_parts: list[dict],
ask_user: Callable[[], Awaitable[Answer]],
ask_user: Callable[..., Awaitable[Answer]],
user_triggers: Callable[[], Awaitable[Optional[Answer]]]) -> Optional[Answer]:
"""Run the hooks in order; the first to return a message or an end decides, and one that asked returns that."""
answer = _Answer(ask_user, user_triggers)
Expand Down
27 changes: 19 additions & 8 deletions src/agent_env/task_step/task_steps/prompt_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -1109,6 +1109,10 @@ async def _get_trigger_state(self, deployed) -> dict:
raise RuntimeError(f"could not read /triggers/state for env '{deployed.env_id}': {last_exc}")


# How a post-turn hook's note reaches a peer user sim: one more text part after the agent's reply.
PEER_NOTE = ("[Only you see this; the assistant doesn't. What you found when you checked its work yourself: {note}]")


class _UserSide:
"""Who answers the agent between turns: agent triggers first, then the LLM user simulator or a human peer."""

Expand All @@ -1128,10 +1132,11 @@ def __init__(self, step: PromptAgentTaskStep, context: TaskStepContext, *, url:
self._shareable = shareable
self._end_after: Optional[str] = None

async def ask(self, number: int, solver_message: str, agent_parts: list[dict]) -> dict:
async def ask(self, number: int, solver_message: str, agent_parts: list[dict], note: Optional[str] = None) -> dict:
"""The user side's answer: ``{"message": parts}``, or ``{"end": reason}`` with any closing message to record. A
user deployed by deploy_user_sim_agent may add ``end_after``: the agent answers the message, then the
conversation ends for that reason, and every later ask gets that end without asking anyone."""
conversation ends for that reason, and every later ask gets that end without asking anyone. ``note`` (from a
post-turn hook) is shown to an LLM user sim with the reply, as something only it sees; a human peer gets none."""
if self._end_after is not None:
return {"end": self._end_after}
if self._decide_url is not None:
Expand All @@ -1146,10 +1151,11 @@ async def ask(self, number: int, solver_message: str, agent_parts: list[dict]) -
return answer
# no trigger fired -> fall through to the LLM user-sim for this turn (hybrid)
if self._built_in is not None:
answer = await self._built_in.ask(number, _agent_text(solver_message, agent_parts), self._ask_built_in)
answer = await self._built_in.ask(number, _agent_text(solver_message, agent_parts), self._ask_built_in,
note=note)
self._end_after = answer.get("end_after")
return answer
return await self._ask_peer(number, agent_parts)
return await self._ask_peer(number, agent_parts, note)

async def _ask_built_in(self, text: str) -> dict:
"""One turn of the user sim deploy_user_sim_agent deployed, in this conversation's session: ``{"text",
Expand Down Expand Up @@ -1209,8 +1215,12 @@ async def triggers(self, number: int, solver_message: str) -> Optional[dict]:
self._record("agent_trigger_firings", {"turn": number, "fired": resp.get("fired", [])})
return answer

async def _ask_peer(self, number: int, agent_parts: list[dict]) -> dict:
async def _ask_peer(self, number: int, agent_parts: list[dict], note: Optional[str] = None) -> dict:
step = self._step
if note is not None and self._is_user_sim:
# after the reply, so the sim reads it as its own check of the work; the agent never sees this message
agent_parts = [*agent_parts, {"kind": "text", "text": PEER_NOTE.format(note=note.strip())}]
self._record("user_side_notes", {"turn": number, "note": note})
if self._url is None:
self._url = get_config().get_default_human_a2a_url()
user_a2a_task_id = uuid.uuid4().hex
Expand Down Expand Up @@ -1291,12 +1301,13 @@ async def start(cls, step: PromptAgentTaskStep, context: TaskStepContext, record
"opening": opening, "done_ends_run": True}
return cls(step, context, person, opening, clock_envs)

async def ask(self, number: int, reply: str, ask_user_sim) -> dict:
async def ask(self, number: int, reply: str, ask_user_sim, note: Optional[str] = None) -> dict:
"""The user's answer to the agent's ``reply``: ``{"message": parts}``, with ``end_after`` when the agent's answer
to it ends the conversation, or ``{"end": reason}``."""
to it ends the conversation, or ``{"end": reason}``. ``note`` goes in the read as ``user_note``: what the user
found checking the work themselves."""
now = await _clock_now(self._step, self._clock_envs)
read = {"time": now, "assistant_messages": [{"at": now, "text": reply}], "phone": [],
"you_sent": self._you_sent, "done_ends_run": True}
"you_sent": self._you_sent, "done_ends_run": True, **({"user_note": note.strip()} if note else {})}
self._you_sent = []
calls: list[dict] = []

Expand Down
23 changes: 23 additions & 0 deletions tst/unit/task_step/test_prompt_agent_built_in_user_sim.py
Original file line number Diff line number Diff line change
Expand Up @@ -411,3 +411,26 @@ async def test_agent_triggers_registered_for_the_built_in_user_are_refused_befor
"user sim aren't supported yet"):
await _step().execute(ctx)
assert "user_sim_reads" not in ctx.metadata


@pytest.mark.asyncio
async def test_a_hook_note_is_in_the_users_read_after_the_reply_and_never_reaches_the_agent(monkeypatch):
from tst.unit.task_step.test_prompt_agent_post_turn_hooks import _hook
hook = _hook("notes", """
async def post_turn(turn, params):
return None if turn.last else await turn.ask_user(note="The sheet has no total.")
""")
peers = Peers(monkeypatch, {AGENT: [text("Sheet's in the folder."), text("Added it.")],
USER: [said("reply", "can it have a total?")]})
ctx = await _context(monkeypatch)

await _step(turns=2, post_turn_hooks=[{"file_artifact_id": hook, "params": {}}]).execute(ctx)

stored = _full(_reads(ctx)[0])
assert stored["read"]["user_note"] == "The sheet has no total."
rendered = stored["input"]
assert rendered.index("Sheet's in the folder.") < rendered.index(
"WHAT YOU FOUND WHEN YOU CHECKED ITS WORK YOURSELF (the assistant doesn't see this):\n The sheet has no total.")
sent_to_agent = [r[2]["params"]["message"]["parts"] for r in peers.requests(ctx) if r[1] == f"{AGENT}/a2a"
and r[2].get("method") == "message/send"]
assert all("no total" not in json.dumps(parts) for parts in sent_to_agent)
45 changes: 45 additions & 0 deletions tst/unit/task_step/test_prompt_agent_post_turn_hooks.py
Original file line number Diff line number Diff line change
Expand Up @@ -264,6 +264,51 @@ async def post_turn(turn, params):
]


NOTES = """
async def post_turn(turn, params):
return None if turn.last else await turn.ask_user(note=f"the sheet has no total (turn {turn.number})")
"""


@pytest.mark.asyncio
async def test_a_hook_note_reaches_the_user_sim_after_the_reply_and_never_the_agent(monkeypatch):
peers = EnvPeers(monkeypatch, {AGENT: [text("one"), text("two")], USER: [sim("add a total?")]})
ctx = _context(user_card=_USER_SIM_CARD)

await _step(_hook("notes", NOTES), turns=2).execute(ctx)

note = ("[Only you see this; the assistant doesn't. What you found when you checked its work yourself: "
"the sheet has no total (turn 1)]")
assert peers.requests(ctx)[1:] == [
*exchange(AGENT, text("plan my week"), "<id1>", "<ctx>", "task-1"),
*exchange(USER, [*text("one"), *text(note)], "<id2>", "<ctx>", "task-2"),
*exchange(AGENT, text("add a total?"), "<id3>", "<ctx>", "task-3"),
]
assert ctx.metadata["user_side_notes"]["talk"] == [{"turn": 1, "note": "the sheet has no total (turn 1)"}]


@pytest.mark.asyncio
async def test_a_human_peer_gets_no_hook_note(monkeypatch):
monkeypatch.setenv("AGENT_ENV_HUMAN_A2A_URL", "http://human.test")
peers = EnvPeers(monkeypatch, {AGENT: [text("one"), text("two")], "http://human.test": [text("go on")]})
ctx = _context()

await _step(_hook("notes", NOTES), turns=2).execute(ctx)

assert peers.requests(ctx)[2:4] == exchange("http://human.test", text("one"), "<id2>", "<ctx>", "task-2")
assert "user_side_notes" not in ctx.metadata


@pytest.mark.asyncio
@pytest.mark.parametrize("note", ["''", "' '", "7"])
async def test_a_hook_note_that_is_not_text_fails_the_step(monkeypatch, note):
EnvPeers(monkeypatch, {AGENT: [text("one")], USER: [sim("ok")]})
hook = _hook("bad", f"async def post_turn(turn, params):\n return await turn.ask_user(note={note})\n")

with pytest.raises(ValueError, match="a note for the user side is non-empty text"):
await _step(hook, turns=2).execute(_context(user_card=_USER_SIM_CARD))


@pytest.mark.asyncio
@pytest.mark.parametrize("returned, fragment", [
("'wake'", "returned 'wake'"),
Expand Down
Loading