Skip to content
Open
4 changes: 4 additions & 0 deletions src/agent_env/agents/agents/claude-code/a2a_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,9 @@ class ClaudeCodeConfig(AgentConfig):
max_thinking_tokens: int | None = None
# Which of the CLI's own tools it may use. With any selection, its MCP servers are only the ones agent-env gives it.
builtin_tools: ClaudeCodeTools | None = None
# What a turn cut short by a transient API error is resumed with; unset, the bridge asks the model to finish the
# request it was given, which suits a task but not a simulated user.
resume_prompt: str | None = None
# Whether a turn gets the SDK's peer_agents MCP tools, to message the task's other agents.
peer_tools: bool = True

Expand Down Expand Up @@ -496,6 +499,7 @@ async def _turn(self, request: TaskRequest[ClaudeCodeConfig], cfg: ClaudeCodeCon
max_turns=cfg.max_turns,
max_thinking_tokens=cfg.max_thinking_tokens,
builtin_tools=cfg.builtin_tools,
resume_prompt=cfg.resume_prompt,
event_callback=request.trajectory.append,
)
except Exception as e:
Expand Down
6 changes: 4 additions & 2 deletions src/agent_env/agents/agents/claude-code/claude_bridge.py
Original file line number Diff line number Diff line change
Expand Up @@ -421,10 +421,12 @@ async def run_claude(
max_thinking_tokens: int | None = None,
builtin_tools: ClaudeCodeTools | None = None,
event_callback: Callable[[dict], None] | None = None,
resume_prompt: str | None = None,
) -> ClaudeResult:
"""Run one turn, resuming the CLI session when a transient upstream failure cuts it short; a resume spends what
is left of ``timeout_s``, so the turn never outlives its caller's deadline. ``CLAUDE_API_ERROR_RETRIES=0`` turns
resumes off. ``event_callback`` gets each stdout event as the CLI prints it; an attempt's trailing `result` is
resumes off. The resume is prompted with ``resume_prompt``, else ``_RESUME_PROMPT``, which asks the model to
finish the request it was given: a caller whose model isn't doing a task, such as a simulated user, sets its own. ``event_callback`` gets each stdout event as the CLI prints it; an attempt's trailing `result` is
held until it's known not to be resumed."""
deadline = time.monotonic() + timeout_s
retries_left = _configured_api_error_retries()
Expand Down Expand Up @@ -474,7 +476,7 @@ async def run_claude(
trajectory.extend(_without_terminal_result(result.trajectory))
first_failure = first_failure or result.error_detail
retries_left -= 1
attempt_prompt, attempt_resume, attempt_session = _RESUME_PROMPT, True, result.session_id
attempt_prompt, attempt_resume, attempt_session = resume_prompt or _RESUME_PROMPT, True, result.session_id
backoff = _RETRY_BACKOFF_S[min(attempts - 1, len(_RETRY_BACKOFF_S) - 1)]
logger.warning(f"claude CLI turn cut short ({_redact(failure)!r}); resuming session {attempt_session} "
f"in {backoff}s with {int(budget_left)}s of the original {timeout_s}s budget left")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,8 @@
from agent_env.entity.scopes import FieldScope
from agent_env.task_step.context import TaskStepContext
from agent_env.task_step.task_steps.agent import user_sim_versions
from agent_env.task_step.task_steps.agent.user_sim_common import (attached, opening_parts, opening_problems, opening_text,
from agent_env.task_step.task_steps.agent.user_sim_common import (USER_SIM_RESUME_PROMPT, attached, opening_parts,
opening_problems, opening_text,
persona_fingerprint, recorded_opening)
from agent_env.task_step.task_steps.deploy_agent import DeployAgentTaskStep

Expand Down Expand Up @@ -320,7 +321,9 @@ async def _set_user(self, context: TaskStepContext, agent, setup: dict) -> TaskS
from agent_env.a2a_agent import A2AAgent

model, source = self._user_model(context)
wanted = {"model": model, "output_format": setup.get("output_format")}
# The resume prompt keeps a turn cut short by an API error the user's (user_sim_common); a card that takes none
# gets none, and its agent resumes with its own.
wanted = {"model": model, "output_format": setup.get("output_format"), "resume_prompt": USER_SIM_RESUME_PROMPT}
negotiated = A2AAgent.negotiate_agent_config(agent.a2a_card or {}, {k: v for k, v in wanted.items() if v})
if negotiated is None or "model" not in negotiated.fields:
raise RuntimeError(f"{self.type} '{self.id}': agent '{self.agent_name}' takes no model through "
Expand Down
13 changes: 13 additions & 0 deletions src/agent_env/task_step/task_steps/agent/user_sim_common.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,19 @@
SATISFACTION_KEY = "user_sim_satisfaction"


# What a user-sim turn cut short by a transient API error is resumed with, for an agent that takes `resume_prompt`
# (prompt_agent's peer user sim, and the user deploy_user_sim_agent deploys).
# An agent's own resume prompt asks it to finish the request it was given; a user-sim's request is the agent's reply,
# so that prompt has it carry on as the agent. The last sentence covers a turn that isn't a reply to the assistant: the
# rating rate_user_satisfaction asks for in the same session.
USER_SIM_RESUME_PROMPT = (
"Your previous turn was interrupted by a transient API error before you sent your message. You are still the "
"user. Write the user's next message, as you were about to; the assistant's last reply above is what you are "
"answering. If what you were last sent asks for something else, such as a rating, give that instead, in the "
"format it asks for."
)


def person_of(us: dict) -> dict:
"""The user a deploy_user_sim_agent record describes, as the user sim's ``answer`` takes them; the caller adds the
run's ``time_zone``, ``opening`` and ``done_ends_run``."""
Expand Down
35 changes: 32 additions & 3 deletions src/agent_env/task_step/task_steps/prompt_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@
from agent_env.task_step.task_steps.notification_delivery import delivered_parts, labels_zone, next_message, take_due
from agent_env.task_step.task_steps.post_turn_hooks import PostTurnHooks, run_post_turn_hooks, validate_hook_specs
from agent_env.task_step.task_steps.agent import user_sim_versions
from agent_env.task_step.task_steps.agent.user_sim_common import iso, person_of, task_zone, utc
from agent_env.task_step.task_steps.agent.user_sim_common import USER_SIM_RESUME_PROMPT, iso, person_of, task_zone, utc
from agent_env.task_step.task_steps.agent.user_sim_common import attached, for_agent, opening_text, recorded_opening
from agent_env.task_step.task_steps.sandbox_utils.sandbox_utils import (
agent_error_text,
Expand Down Expand Up @@ -564,7 +564,8 @@ async def _converse(
elif is_user_sim:
user_output_format = self.user_output_format or _DEFAULT_USER_SIM_OUTPUT_FORMAT
user_model = self._resolve_user_model(context)
user_config = {k: v for k, v in {"output_format": user_output_format, "model": user_model}.items() if v is not None}
user_config = {k: v for k, v in {"output_format": user_output_format, "model": user_model,
"resume_prompt": USER_SIM_RESUME_PROMPT}.items() if v is not None}
negotiated = A2AAgent.negotiate_agent_config(user_sim.a2a_card or {}, user_config)
if negotiated:
await protocol.post_agent_config(user_url + negotiated.endpoint, negotiated.fields)
Expand Down Expand Up @@ -1208,7 +1209,7 @@ async def _ask_peer(self, number: int, agent_parts: list[dict]) -> dict:
try:
# A user-sim runs in a sandbox agent-env deployed; a human peer, registered or named by
# user_a2a_url, has none and reads the store itself.
_, user_result = await send_and_wait(
user_task_id, user_result = await send_and_wait(
self._url, agent_parts,
agent=self._user_sim if self._is_user_sim and self._user_sim.sandbox_id else None,
shareable=self._shareable, message_id=user_a2a_task_id, context_id=self._conversation_id,
Expand All @@ -1219,6 +1220,9 @@ async def _ask_peer(self, number: int, agent_parts: list[dict]) -> dict:
logger.warning(f"user_a2a_url timeout for conversation {self._conversation_id} ({e}); marking abandoned")
return {"end": "user_timeout"}

if self._is_user_sim:
await self._save_user_sim_trajectory(number, user_task_id, user_a2a_task_id)

user_state = (user_result.get("status") or {}).get("state")
if user_state != TaskState.completed:
logger.info(f"user_a2a_url returned state={user_state}; ending multi-turn")
Expand Down Expand Up @@ -1248,6 +1252,31 @@ async def _ask_peer(self, number: int, agent_parts: list[dict]) -> dict:
return {"end": "user_done"}
return {"message": [{"kind": "text", "text": message_text}]}

async def _save_user_sim_trajectory(self, number: int, user_task_id: str, user_a2a_task_id: str) -> None:
"""Store the user-sim's own trajectory for this answer, when its card serves one, and record where it went.
Best-effort: a failed fetch is logged and the conversation goes on."""
card = self._user_sim.a2a_card or {}
traj_ext = A2AAgent.find_extension(card, A2AAgent.EXT_TRAJECTORY)
if not traj_ext:
return
prefix = self._step._trajectory_prefix().rstrip("/") + "/usersim/"
try:
# Bounded like _user_sim_trajectory: the next turn waits on this, and the trace is optional.
url = await asyncio.wait_for(
self._step._fetch_trajectory(
self._url, traj_ext, user_task_id, prefix, user_a2a_task_id,
card=card, sandbox_type=self._user_sim.sandbox_type,
),
_USER_SIM_TRAJECTORY_FETCH_SECONDS,
)
except Exception as e:
logger.warning(f"user-sim trajectory fetch at turn {number} failed (continuing): {e}")
return
if url is None:
logger.warning(f"user-sim trajectory fetch at turn {number} returned no trajectory; nothing stored")
return
self._record("usersim_trajectories", {"turn": number, "a2a_task_id": user_a2a_task_id, "url": url})

def _record(self, key: str, entry: dict) -> None:
self._context.metadata.setdefault(key, {}).setdefault(self._step.id, []).append(entry)

Expand Down
2 changes: 1 addition & 1 deletion tst/unit/agents/claude_code/test_card_contract.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ def test_the_config_takes_every_field_prompt_agent_sends_that_the_cli_has(app):
config = _extensions(app)["urn:agentenv:agent-config/v1"]["params"]["methods"]
assert config["set"]["request"]["supported"] == [
"allowed_tools", "builtin_tools", "description", "effort", "max_thinking_tokens", "max_turns", "model", "name",
"output_format", "peer_tools", "project_id", "role", "system_prompt", "task_id", "timeout_seconds",
"output_format", "peer_tools", "project_id", "resume_prompt", "role", "system_prompt", "task_id", "timeout_seconds",
]
assert set(config["set"]["request"]["schema"]["properties"]) == set(config["set"]["request"]["supported"])
assert config["get"] == {"method": "GET", "response": {"required": ["config"]}}
Expand Down
12 changes: 12 additions & 0 deletions tst/unit/agents/claude_code/test_claude_bridge_retry.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,18 @@ def _write_session(scripted, session_id="sess-1"):
(scripted["session_dir"] / f"{session_id}.jsonl").write_text('{"type":"assistant"}\n')


def test_a_resume_prompt_given_replaces_the_default_one(scripted):
_write_session(scripted)
scripted["queue"].append((_api_error_stdout("API Error: The operation timed out.", tool_uses=1), 1))
scripted["queue"].append((_ok_stdout("resumed"), 0))

result = asyncio.run(claude_bridge.run_claude(prompt="p", session_id="sess-1", resume=False,
resume_prompt="stay the user"))

assert result.attempts == 2
assert scripted["procs"][1].stdin.data == b"stay the user"


def test_success_makes_exactly_one_invocation(scripted):
scripted["queue"].append((_ok_stdout(), 0))
result = asyncio.run(claude_bridge.run_claude(prompt="p", session_id="sess-1", resume=False))
Expand Down
17 changes: 17 additions & 0 deletions tst/unit/task_step/test_deploy_user_sim_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
from agent_env.task_step.context import TaskStepContext
from agent_env.task_step.task_steps import always_on as ao
from agent_env.task_step.task_steps.agent import user_sim_versions
from agent_env.task_step.task_steps.agent.user_sim_common import USER_SIM_RESUME_PROMPT
from agent_env.task_step.task_steps.deploy_agent import DeployAgentTaskStep
from agent_env.task_step.task_steps.agent.deploy_user_sim_agent import USER_SIMS_KEY, DeployUserSimAgentTaskStep
from agent_env.task_step.task_steps.agent.user_sim_common import persona_fingerprint
Expand Down Expand Up @@ -98,6 +99,22 @@ async def test_the_prompt_set_on_the_user_s_agent_is_the_tested_one_byte_for_byt
assert context.metadata["models"]["user simulator"]["user"] == "sim-model"


@pytest.mark.asyncio
async def test_a_user_whose_card_takes_a_resume_prompt_is_sent_the_one_that_keeps_it_the_user(monkeypatch):
# An agent's own resume prompt asks it to finish its request; a user sim's request is the assistant's reply.
deploy = _Deploy(monkeypatch, _config_card("name", "model", "system_prompt", "output_format", "resume_prompt"))
context = await deploy.run(_step())
assert deploy.fields("resume_prompt") == [USER_SIM_RESUME_PROMPT]
assert "resume_prompt" in context.metadata[USER_SIMS_KEY]["hannah"]["setup"]["accepted_fields"]


@pytest.mark.asyncio
async def test_a_user_whose_card_takes_no_resume_prompt_is_sent_none(monkeypatch):
deploy = _Deploy(monkeypatch)
await deploy.run(_step())
assert deploy.fields("resume_prompt") == []


@pytest.mark.asyncio
async def test_the_record_keeps_each_tag_s_line_so_an_old_used_resolves_after_the_persona_is_reordered(monkeypatch):
record = await _published(monkeypatch, _step())
Expand Down
81 changes: 81 additions & 0 deletions tst/unit/task_step/test_prompt_agent_user_side.py
Original file line number Diff line number Diff line change
Expand Up @@ -448,6 +448,87 @@ async def test_with_no_agent_trigger_fired_the_llm_user_sim_answers(monkeypatch)
assert ctx.metadata["agent_trigger_firings"] == {"talk": [{"turn": 1, "fired": []}]}


# ---- the user-sim's own trajectory -----------------------------------------------------------------------------------

_TRAJ = {"uri": "urn:agentenv:trajectory/v1", "params": {"endpoint": "/ext/trajectory"}}
_USER_SIM_CARD_WITH_TRAJ = {"capabilities": {"extensions": [_AGENT_CONFIG, _TRAJ]}}


@pytest.mark.asyncio
async def test_the_user_sims_own_trajectory_is_stored_per_answer_under_the_steps_prefix(monkeypatch):
fetched = []

async def fake_fetch(self, a2a_url, traj_ext, server_task_id, prefix, name, *, sandbox_type, card=None):
fetched.append((a2a_url, server_task_id, prefix, sandbox_type))
return f"{prefix}{name}.json"

monkeypatch.setattr(PromptAgentTaskStep, "_fetch_trajectory", fake_fetch)
peers = Peers(monkeypatch, {AGENT: [text("draft one"), text("draft two"), text("draft three")], USER: [sim("one"), sim("two")]})
ctx = _context(user_card=_USER_SIM_CARD_WITH_TRAJ)

await _step(turns=3, trajectory_output_prefix="s3://bucket/traj").execute(ctx)

peers.requests(ctx)
assert fetched == [(USER, "task-2", "s3://bucket/traj/usersim/", "local"),
(USER, "task-4", "s3://bucket/traj/usersim/", "local")]
# Stored under the id the conversation's user-side message was sent with.
sent_ids = [r[2]["params"]["message"]["messageId"] for r in peers.raw
if r[1] == f"{USER}/a2a" and r[2]["method"] == "message/send"]
assert ctx.metadata["usersim_trajectories"] == {"talk": [
{"turn": turn, "a2a_task_id": i, "url": f"s3://bucket/traj/usersim/{i}.json"}
for turn, i in enumerate(sent_ids, start=1)]}
assert len(sent_ids) == 2


@pytest.mark.asyncio
async def test_a_failed_user_sim_trajectory_fetch_is_not_recorded_and_the_conversation_goes_on(monkeypatch):
async def failing_fetch(*args, **kwargs):
raise RuntimeError("no trajectory")

monkeypatch.setattr(PromptAgentTaskStep, "_fetch_trajectory", failing_fetch)
peers = Peers(monkeypatch, {AGENT: [text("draft one"), text("draft two")], USER: [sim("shorter")]})
ctx = _context(user_card=_USER_SIM_CARD_WITH_TRAJ)

await _step(turns=2).execute(ctx)

assert peers.requests(ctx)[-2] == send(AGENT, text("shorter"), "<id3>", "<ctx>")
assert "usersim_trajectories" not in ctx.metadata


@pytest.mark.asyncio
async def test_a_user_sim_trajectory_fetch_that_hangs_is_cut_off_and_the_conversation_goes_on(monkeypatch):
import asyncio
from agent_env.task_step.task_steps import prompt_agent as prompt_agent_module

async def hanging_fetch(*args, **kwargs):
await asyncio.sleep(3600)

monkeypatch.setattr(PromptAgentTaskStep, "_fetch_trajectory", hanging_fetch)
monkeypatch.setattr(prompt_agent_module, "_USER_SIM_TRAJECTORY_FETCH_SECONDS", 0.05)
peers = Peers(monkeypatch, {AGENT: [text("draft one"), text("draft two")], USER: [sim("shorter")]})
ctx = _context(user_card=_USER_SIM_CARD_WITH_TRAJ)

await asyncio.wait_for(_step(turns=2).execute(ctx), 10)

assert peers.requests(ctx)[-2] == send(AGENT, text("shorter"), "<id3>", "<ctx>")
assert "usersim_trajectories" not in ctx.metadata


@pytest.mark.asyncio
async def test_a_user_sim_trajectory_fetch_that_stores_nothing_is_not_recorded(monkeypatch):
async def empty_fetch(*args, **kwargs):
return None

monkeypatch.setattr(PromptAgentTaskStep, "_fetch_trajectory", empty_fetch)
peers = Peers(monkeypatch, {AGENT: [text("draft one"), text("draft two")], USER: [sim("shorter")]})
ctx = _context(user_card=_USER_SIM_CARD_WITH_TRAJ)

await _step(turns=2).execute(ctx)

assert peers.requests(ctx)[-2] == send(AGENT, text("shorter"), "<id3>", "<ctx>")
assert "usersim_trajectories" not in ctx.metadata


# ---- a human peer ---------------------------------------------------------------------------------------------------

@pytest.mark.asyncio
Expand Down
Loading
Loading