diff --git a/src/agent_env/task_step/task_steps/agent/user_sim_versions.py b/src/agent_env/task_step/task_steps/agent/user_sim_versions.py index c1bf4da53..785905870 100644 --- a/src/agent_env/task_step/task_steps/agent/user_sim_versions.py +++ b/src/agent_env/task_step/task_steps/agent/user_sim_versions.py @@ -370,6 +370,9 @@ def dated(r: dict, items: list) -> list[dict]: # an earlier read's items, each out += [f" [{_stamp(at(s), now)}] {s['text']}{attached_note(s.get('files'))}" for s in own] out.append("\nNEW FROM YOUR AI ASSISTANT (this is what you're answering):") out += [_message(m, _stamp(at(m), now)) for m in messages] + if read.get("user_note"): # from a post-turn hook: what the user saw checking the work, never sent to the assistant + out.append("\nWHAT YOU FOUND WHEN YOU CHECKED ITS WORK YOURSELF (the assistant doesn't see this):") + out += [" " + line for line in read["user_note"].strip().splitlines()] out.append("\n" + CLOSING.format(first=_first(person))) out.append("Answer in JSON: " + ANSWER_LINE) return "\n".join(out) diff --git a/src/agent_env/task_step/task_steps/post_turn_hooks.py b/src/agent_env/task_step/task_steps/post_turn_hooks.py index 0afb54b71..9a8549f11 100644 --- a/src/agent_env/task_step/task_steps/post_turn_hooks.py +++ b/src/agent_env/task_step/task_steps/post_turn_hooks.py @@ -163,17 +163,17 @@ async def call_tool(self, tool: str, args: Optional[dict], role: str) -> dict: class _Answer: """The user side's answer this turn: asked at most once, then the same for every later call.""" - def __init__(self, ask_user: Callable[[], Awaitable[Answer]], + def __init__(self, ask_user: Callable[..., Awaitable[Answer]], user_triggers: Callable[[], Awaitable[Optional[Answer]]]) -> None: self.value: Optional[Answer] = None self._ask_user = ask_user self._user_triggers = user_triggers self._asking = asyncio.Lock() - async def ask_user(self) -> Answer: + async def ask_user(self, note: Optional[str] = None) -> Answer: async with self._asking: if self.value is None: - self.value = await self._ask_user() + self.value = await (self._ask_user(note=note) if note is not None else self._ask_user()) return self.value async def user_triggers(self) -> Optional[Answer]: @@ -204,9 +204,14 @@ def metadata(self) -> Mapping[str, Any]: """A read-only copy of the run's metadata, its secret keys stripped.""" return MappingProxyType(_strip_redacted_keys(self._context.metadata)) - async def ask_user(self) -> Answer: - """Today's user side: agent triggers, then a human or the LLM user sim, which sees only replies sent here.""" - return await self._answer.ask_user() + async def ask_user(self, note: Optional[str] = None) -> Answer: + """Today's user side: agent triggers, then a human or the LLM user sim, which sees only replies sent here. + + ``note`` is shown to an LLM user sim with the agent's reply, as something only it sees (what the person would + find by checking the work themselves); the agent never sees it. A human peer gets no note.""" + if note is not None and (not isinstance(note, str) or not note.strip()): + raise ValueError("a note for the user side is non-empty text") + return await self._answer.ask_user(note) async def user_triggers(self) -> Optional[Answer]: """Agent triggers alone; None, leaving no trace, when none fires.""" @@ -263,7 +268,7 @@ def _parts(message: Any, hook: PostTurnHook) -> list[dict]: async def run_post_turn_hooks(hooks: PostTurnHooks, *, context: TaskStepContext, step_id: str, number: int, last: bool, agent_reply: str, agent_parts: list[dict], - ask_user: Callable[[], Awaitable[Answer]], + ask_user: Callable[..., Awaitable[Answer]], user_triggers: Callable[[], Awaitable[Optional[Answer]]]) -> Optional[Answer]: """Run the hooks in order; the first to return a message or an end decides, and one that asked returns that.""" answer = _Answer(ask_user, user_triggers) diff --git a/src/agent_env/task_step/task_steps/prompt_agent.py b/src/agent_env/task_step/task_steps/prompt_agent.py index 64f862c81..2aa383fa1 100644 --- a/src/agent_env/task_step/task_steps/prompt_agent.py +++ b/src/agent_env/task_step/task_steps/prompt_agent.py @@ -1109,6 +1109,10 @@ async def _get_trigger_state(self, deployed) -> dict: raise RuntimeError(f"could not read /triggers/state for env '{deployed.env_id}': {last_exc}") +# How a post-turn hook's note reaches a peer user sim: one more text part after the agent's reply. +PEER_NOTE = ("[Only you see this; the assistant doesn't. What you found when you checked its work yourself: {note}]") + + class _UserSide: """Who answers the agent between turns: agent triggers first, then the LLM user simulator or a human peer.""" @@ -1128,10 +1132,11 @@ def __init__(self, step: PromptAgentTaskStep, context: TaskStepContext, *, url: self._shareable = shareable self._end_after: Optional[str] = None - async def ask(self, number: int, solver_message: str, agent_parts: list[dict]) -> dict: + async def ask(self, number: int, solver_message: str, agent_parts: list[dict], note: Optional[str] = None) -> dict: """The user side's answer: ``{"message": parts}``, or ``{"end": reason}`` with any closing message to record. A user deployed by deploy_user_sim_agent may add ``end_after``: the agent answers the message, then the - conversation ends for that reason, and every later ask gets that end without asking anyone.""" + conversation ends for that reason, and every later ask gets that end without asking anyone. ``note`` (from a + post-turn hook) is shown to an LLM user sim with the reply, as something only it sees; a human peer gets none.""" if self._end_after is not None: return {"end": self._end_after} if self._decide_url is not None: @@ -1146,10 +1151,11 @@ async def ask(self, number: int, solver_message: str, agent_parts: list[dict]) - return answer # no trigger fired -> fall through to the LLM user-sim for this turn (hybrid) if self._built_in is not None: - answer = await self._built_in.ask(number, _agent_text(solver_message, agent_parts), self._ask_built_in) + answer = await self._built_in.ask(number, _agent_text(solver_message, agent_parts), self._ask_built_in, + note=note) self._end_after = answer.get("end_after") return answer - return await self._ask_peer(number, agent_parts) + return await self._ask_peer(number, agent_parts, note) async def _ask_built_in(self, text: str) -> dict: """One turn of the user sim deploy_user_sim_agent deployed, in this conversation's session: ``{"text", @@ -1209,8 +1215,12 @@ async def triggers(self, number: int, solver_message: str) -> Optional[dict]: self._record("agent_trigger_firings", {"turn": number, "fired": resp.get("fired", [])}) return answer - async def _ask_peer(self, number: int, agent_parts: list[dict]) -> dict: + async def _ask_peer(self, number: int, agent_parts: list[dict], note: Optional[str] = None) -> dict: step = self._step + if note is not None and self._is_user_sim: + # after the reply, so the sim reads it as its own check of the work; the agent never sees this message + agent_parts = [*agent_parts, {"kind": "text", "text": PEER_NOTE.format(note=note.strip())}] + self._record("user_side_notes", {"turn": number, "note": note}) if self._url is None: self._url = get_config().get_default_human_a2a_url() user_a2a_task_id = uuid.uuid4().hex @@ -1291,12 +1301,13 @@ async def start(cls, step: PromptAgentTaskStep, context: TaskStepContext, record "opening": opening, "done_ends_run": True} return cls(step, context, person, opening, clock_envs) - async def ask(self, number: int, reply: str, ask_user_sim) -> dict: + async def ask(self, number: int, reply: str, ask_user_sim, note: Optional[str] = None) -> dict: """The user's answer to the agent's ``reply``: ``{"message": parts}``, with ``end_after`` when the agent's answer - to it ends the conversation, or ``{"end": reason}``.""" + to it ends the conversation, or ``{"end": reason}``. ``note`` goes in the read as ``user_note``: what the user + found checking the work themselves.""" now = await _clock_now(self._step, self._clock_envs) read = {"time": now, "assistant_messages": [{"at": now, "text": reply}], "phone": [], - "you_sent": self._you_sent, "done_ends_run": True} + "you_sent": self._you_sent, "done_ends_run": True, **({"user_note": note.strip()} if note else {})} self._you_sent = [] calls: list[dict] = [] diff --git a/tst/unit/task_step/test_prompt_agent_built_in_user_sim.py b/tst/unit/task_step/test_prompt_agent_built_in_user_sim.py index 5eb2490d3..40d9b0af6 100644 --- a/tst/unit/task_step/test_prompt_agent_built_in_user_sim.py +++ b/tst/unit/task_step/test_prompt_agent_built_in_user_sim.py @@ -411,3 +411,26 @@ async def test_agent_triggers_registered_for_the_built_in_user_are_refused_befor "user sim aren't supported yet"): await _step().execute(ctx) assert "user_sim_reads" not in ctx.metadata + + +@pytest.mark.asyncio +async def test_a_hook_note_is_in_the_users_read_after_the_reply_and_never_reaches_the_agent(monkeypatch): + from tst.unit.task_step.test_prompt_agent_post_turn_hooks import _hook + hook = _hook("notes", """ + async def post_turn(turn, params): + return None if turn.last else await turn.ask_user(note="The sheet has no total.") + """) + peers = Peers(monkeypatch, {AGENT: [text("Sheet's in the folder."), text("Added it.")], + USER: [said("reply", "can it have a total?")]}) + ctx = await _context(monkeypatch) + + await _step(turns=2, post_turn_hooks=[{"file_artifact_id": hook, "params": {}}]).execute(ctx) + + stored = _full(_reads(ctx)[0]) + assert stored["read"]["user_note"] == "The sheet has no total." + rendered = stored["input"] + assert rendered.index("Sheet's in the folder.") < rendered.index( + "WHAT YOU FOUND WHEN YOU CHECKED ITS WORK YOURSELF (the assistant doesn't see this):\n The sheet has no total.") + sent_to_agent = [r[2]["params"]["message"]["parts"] for r in peers.requests(ctx) if r[1] == f"{AGENT}/a2a" + and r[2].get("method") == "message/send"] + assert all("no total" not in json.dumps(parts) for parts in sent_to_agent) diff --git a/tst/unit/task_step/test_prompt_agent_post_turn_hooks.py b/tst/unit/task_step/test_prompt_agent_post_turn_hooks.py index f54d8a200..a03f80bb4 100644 --- a/tst/unit/task_step/test_prompt_agent_post_turn_hooks.py +++ b/tst/unit/task_step/test_prompt_agent_post_turn_hooks.py @@ -264,6 +264,51 @@ async def post_turn(turn, params): ] +NOTES = """ + async def post_turn(turn, params): + return None if turn.last else await turn.ask_user(note=f"the sheet has no total (turn {turn.number})") +""" + + +@pytest.mark.asyncio +async def test_a_hook_note_reaches_the_user_sim_after_the_reply_and_never_the_agent(monkeypatch): + peers = EnvPeers(monkeypatch, {AGENT: [text("one"), text("two")], USER: [sim("add a total?")]}) + ctx = _context(user_card=_USER_SIM_CARD) + + await _step(_hook("notes", NOTES), turns=2).execute(ctx) + + note = ("[Only you see this; the assistant doesn't. What you found when you checked its work yourself: " + "the sheet has no total (turn 1)]") + assert peers.requests(ctx)[1:] == [ + *exchange(AGENT, text("plan my week"), "", "", "task-1"), + *exchange(USER, [*text("one"), *text(note)], "", "", "task-2"), + *exchange(AGENT, text("add a total?"), "", "", "task-3"), + ] + assert ctx.metadata["user_side_notes"]["talk"] == [{"turn": 1, "note": "the sheet has no total (turn 1)"}] + + +@pytest.mark.asyncio +async def test_a_human_peer_gets_no_hook_note(monkeypatch): + monkeypatch.setenv("AGENT_ENV_HUMAN_A2A_URL", "http://human.test") + peers = EnvPeers(monkeypatch, {AGENT: [text("one"), text("two")], "http://human.test": [text("go on")]}) + ctx = _context() + + await _step(_hook("notes", NOTES), turns=2).execute(ctx) + + assert peers.requests(ctx)[2:4] == exchange("http://human.test", text("one"), "", "", "task-2") + assert "user_side_notes" not in ctx.metadata + + +@pytest.mark.asyncio +@pytest.mark.parametrize("note", ["''", "' '", "7"]) +async def test_a_hook_note_that_is_not_text_fails_the_step(monkeypatch, note): + EnvPeers(monkeypatch, {AGENT: [text("one")], USER: [sim("ok")]}) + hook = _hook("bad", f"async def post_turn(turn, params):\n return await turn.ask_user(note={note})\n") + + with pytest.raises(ValueError, match="a note for the user side is non-empty text"): + await _step(hook, turns=2).execute(_context(user_card=_USER_SIM_CARD)) + + @pytest.mark.asyncio @pytest.mark.parametrize("returned, fragment", [ ("'wake'", "returned 'wake'"),