diff --git a/src/agent_env/task_step/task_steps/always_on.py b/src/agent_env/task_step/task_steps/always_on.py index 1063c8779..fce42352a 100644 --- a/src/agent_env/task_step/task_steps/always_on.py +++ b/src/agent_env/task_step/task_steps/always_on.py @@ -66,7 +66,10 @@ ``user_reads_at``. - The run's metadata keeps a summary in ``metadata["always_on"][step_id]`` (small enough for a running job to report back); the full timeline goes to the object store, at the summary's ``object_url``. Its ``user_sim`` names the user - sim's session (``context_id``), which rate_user_satisfaction asks in once the run is over. + sim's session (``context_id``), which rate_user_satisfaction asks in once the run is over. Its + ``agent_system_prompt`` says what the agent was told before the run: the agent, whether the step sent it a system + prompt (``why_not`` when it didn't) and its length; the text itself is in the stored record + (``stored_system_prompt``). - An always_on_agent_verifier (``verifiers/always_on_agent_verifier.py``) grades the step's prompt from the user's chat and the run's context, the run wake by wake among it (``render_for_judge``), and, with use_trajectory, every wake's tool calls staged as files; a plain rubrics_verifier on it is refused when the task loads. @@ -410,6 +413,7 @@ def __init__(self, step: AlwaysOnTaskStep, context: TaskStepContext): self.clock_rates: list[tuple[Any, float]] = [] # (env, the task's rate) for each clock the step freezes self.undo: list[tuple[str, Any]] = [] # (what, coroutine function): the gateway settings to put back at the end self.rules_in_system_prompt = False # whether the agent took the rules in its system prompt + self.agent_system_prompt: Optional[dict] = None # what the agent was configured with, for the record async def restore(self) -> None: """Put back what the step changed on the gateways, latest first, so later steps of the task see the settings @@ -599,6 +603,12 @@ async def _configure_agents(self) -> None: if desired.get("system_prompt") and not self.rules_in_system_prompt: logger.warning("always_on: the agent's card did not accept system_prompt, so the message_user rules go " "on its first wake instead") + self.agent_system_prompt = {"agent_name": step.agent_name, "sent": self.rules_in_system_prompt} + if self.rules_in_system_prompt: + self.agent_system_prompt["text"] = negotiated.fields["system_prompt"] + else: + self.agent_system_prompt["why_not"] = ("the agent's card doesn't take one" if desired.get("system_prompt") + else "the step sets none") if self.us.get("prompt_set"): # deploy_user_sim_agent set the user sim's prompt and model at deploy self.user_model, self.user_model_from = self.us["model"], self.us["model_source"] @@ -1703,6 +1713,9 @@ async def finish(self) -> None: "scheduled_checks": self.checks, "contacts": self.contacts, "time_budget_seconds": self.time_budget, "wake_reserve_seconds": step.wake_reserve_seconds(self.contact_seconds)} + if self.agent_system_prompt is not None: + record["agent_system_prompt"] = {**{k: v for k, v in self.agent_system_prompt.items() if k != "text"}, + "chars": len(self.agent_system_prompt.get("text") or "")} await self._store_timeline(record) ctx.metadata.setdefault(ALWAYS_ON_KEY, {})[step.id] = record # The graded response is the user's whole chat from the agent, each message with its pretend time; never its @@ -1711,15 +1724,19 @@ async def finish(self) -> None: for m in self.all_messages)) async def _store_timeline(self, record: dict) -> None: - """Store the run's full record, its timeline included, and give ``record`` (what stays in the run's metadata, - kept small for the job's report) its ``object_url``. When it can't be stored, ``record`` keeps the last - ``TIMELINE_TAIL`` events instead, their bulky fields dropped.""" + """Store the run's full record, its timeline and the agent's system prompt included, and give ``record`` (what + stays in the run's metadata, kept small for the job's report) its ``object_url``. When it can't be stored, + ``record`` keeps the last ``TIMELINE_TAIL`` events instead, their bulky fields dropped, and the system prompt's + text.""" step, ctx = self.step, self.ctx + text = (self.agent_system_prompt or {}).get("text") try: store = get_config().get_object_store() key = (f"{get_config().get_artifact_key_prefix()}always_on_timeline/" f"instance_id={ctx.instance_id or 'adhoc'}/{step.id}-{uuid.uuid4().hex[:8]}.json") full = {**record, "timeline": self.timeline} + if text is not None: + full["agent_system_prompt"] = {**record["agent_system_prompt"], "text": text} record["object_url"] = await asyncio.to_thread(store.put, key, json.dumps(full, indent=1, default=str).encode(), content_type="application/json") @@ -1728,6 +1745,8 @@ async def _store_timeline(self, record: dict) -> None: record["timeline_store_error"] = f"{type(e).__name__}: {str(e)[:300]}" record["timeline_tail"] = [_small({k: v for k, v in event.items() if k not in BULKY_EVENT_FIELDS}) for event in self.timeline[-TIMELINE_TAIL:]] + if text is not None: + record["agent_system_prompt"]["text"] = text def _append_prompt_response(self, final_text: str) -> None: step, ctx = self.step, self.ctx @@ -1760,6 +1779,20 @@ def stored_timeline(record: dict) -> list[dict]: return json.loads(get_config().get_object_store_at(url).get(url))["timeline"] +def stored_system_prompt(record: dict) -> Optional[str]: + """The system prompt the run's agent was configured with, as the step sent it (its own, then the rules), from the + stored record at ``record["object_url"]`` (else the record's own copy, kept when the store failed); None when the + step sent none (``record["agent_system_prompt"]["why_not"]`` says why). Raises KeyError for a record that doesn't + keep it, and the object store's error when the stored record can't be read.""" + entry = record["agent_system_prompt"] + if not entry.get("sent"): + return None + if "text" in entry: + return entry["text"] + url = record["object_url"] + return json.loads(get_config().get_object_store_at(url).get(url))["agent_system_prompt"]["text"] + + def woke_by_text(woke_by: list[str]) -> str: """What woke the agent, in words: "the user's reply, an alert (Gmail)".""" return ", ".join(_WOKE_BY.get(w, f"an alert ({w.split(':', 1)[1]})" if w.startswith("notification:") else w) diff --git a/src/agent_env/task_step/task_steps/verifiers/always_on_agent_verifier.py b/src/agent_env/task_step/task_steps/verifiers/always_on_agent_verifier.py index 067822601..6ad078582 100644 --- a/src/agent_env/task_step/task_steps/verifiers/always_on_agent_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/always_on_agent_verifier.py @@ -15,6 +15,12 @@ A run that kept no timeline has its turns staged the same way by position, one wake file a turn, so the judge never reads a trajectory as stored. +``run_context_for_judge`` may also name ``assistant_instructions`` (not among its defaults): what the assistant was +told before the run, the system prompt the always_on step configured it with (``stored_system_prompt``), so the judge +can tell what it was told up front from what it had to learn. A staged run has it as a file beside the index +(``INSTRUCTIONS_FILE``), which the files note names; otherwise it is a section of the judge's prompt. A run whose +record doesn't keep it, or whose step sent none, has a line saying so instead; it never stops the grading. + For a run of the PSI neutral harness (its trace carries ``harness.*`` labels), the judge is told whose a compaction summary's losses are, by who wrote it (``harness.compaction.written_by``: the assistant's own model, graded as its own, or the harness, not held against the assistant), and, with a per-criterion format, says for each failed criterion @@ -39,7 +45,13 @@ from agent_env.providers.sandbox_providers.sandbox import into_container from agent_env.task_step.context import PromptResponse, TaskStepContext from agent_env.task_step.task_step import TaskStep -from agent_env.task_step.task_steps.always_on import ALWAYS_ON_KEY, AlwaysOnTaskStep, render_for_judge, stored_timeline +from agent_env.task_step.task_steps.always_on import ( + ALWAYS_ON_KEY, + AlwaysOnTaskStep, + render_for_judge, + stored_system_prompt, + stored_timeline, +) from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep from agent_env.task_step.task_steps.verifiers.always_on_staging import ( EVENTS_FILE, @@ -72,6 +84,17 @@ # Where the run's files go in the judge's sandbox, under a directory of the run's own. STAGING_DIR = "/tmp/always_on_run" +# The run_context_for_judge part this verifier adds to rubrics_verifier's (RUN_CONTEXT_PARTS, its defaults): what the +# assistant was told before the run. Staged as INSTRUCTIONS_FILE; in the judge's prompt, at most +# INSTRUCTIONS_PROMPT_CHARS of it. +ASSISTANT_INSTRUCTIONS = "assistant_instructions" +ALWAYS_ON_RUN_CONTEXT_PARTS = (*RUN_CONTEXT_PARTS, ASSISTANT_INSTRUCTIONS) +INSTRUCTIONS_FILE = "assistant_instructions.md" +INSTRUCTIONS_PROMPT_CHARS = 60_000 +INSTRUCTIONS_TITLE = "What the assistant was told before the run" +_INSTRUCTIONS_LEAD = ("Its standing instructions: the system prompt the run configured it with before its first wake, " + "as sent. It knew what is written here from the start; anything else it had to learn during the " + "run.") # Trajectories read, and files written into the judge's sandbox, at once. STAGING_CONCURRENCY = 8 # The only tools of its own the judge gets: it reads and searches the files (the CLI's search is Bash's grep), and its @@ -95,7 +118,7 @@ ("_file", such as "result_file" or "text_file") and shows its start ("_preview"). Email bodies are \ decoded (body.text). A line longer than 1,800 characters is split across lines there, nothing dropped. - {events}: every wake's events in one file, so "did it ever do X" is one search. {triggers}: every trigger firing, \ -fired or failed, with its pretend time and the wake it came after. +fired or failed, with its pretend time and the wake it came after.{instructions} How to read them: - Don't load whole files. Search with grep -n, then Read with an offset and limit. @@ -195,9 +218,14 @@ def __init__( judge_max_turns: Optional[int] = None, **fields: Any, ): - """``run_context_for_judge`` defaults to every part of the run's context; the rest is rubrics_verifier's.""" + """``run_context_for_judge`` defaults to rubrics_verifier's parts (RUN_CONTEXT_PARTS) and may also name + ``assistant_instructions``; the rest is rubrics_verifier's.""" output_format = JudgeOutputFormat(output_format) where = f"{self.type} '{id}'" + parts = run_context_for_judge or list(RUN_CONTEXT_PARTS) + if not isinstance(parts, list) or any(p not in ALWAYS_ON_RUN_CONTEXT_PARTS for p in parts): + raise ValueError(f"{where}: run_context_for_judge must be a list of {list(ALWAYS_ON_RUN_CONTEXT_PARTS)}, " + f"got {parts!r}") if use_trajectory and not use_agent_judge: raise ValueError(f"{where}: use_trajectory needs the agent judge (use_agent_judge): an always_on run's " "wakes are staged as files for a judge that searches them, never inlined") @@ -209,7 +237,8 @@ def __init__( "turns are not its wakes") super().__init__( id, version, use_trajectory=use_trajectory, use_agent_judge=use_agent_judge, - run_context_for_judge=run_context_for_judge or list(RUN_CONTEXT_PARTS), output_format=output_format, + run_context_for_judge=[p for p in parts if p != ASSISTANT_INSTRUCTIONS] or None, + output_format=output_format, default_model=default_model, default_model_api_base=default_model_api_base, effort=effort, max_thinking_tokens=max_thinking_tokens, judge_a2a_agent_id=judge_a2a_agent_id, judge_sandbox_type=judge_sandbox_type, judge_batch_size=judge_batch_size, @@ -219,6 +248,8 @@ def __init__( judge_max_turns=self.DEFAULT_JUDGE_MAX_TURNS if judge_max_turns is None else judge_max_turns, **fields, ) + # Every part, this verifier's own too: rubrics_verifier renders the parts it knows and passes the others by. + self.run_context_for_judge = list(dict.fromkeys(parts)) @classmethod def from_dict(cls, data: dict) -> AlwaysOnAgentVerifier: @@ -319,15 +350,40 @@ def _record(self, context: TaskStepContext, prompt_response: PromptResponse) -> return record async def judge_prompt_sections(self, context: TaskStepContext, prompt_response: PromptResponse) -> list[str]: - """The run wake by wake, from the always_on step's record of it.""" - return [await asyncio.to_thread(render_for_judge, self._record(context, prompt_response))] + """What the assistant was told before the run, when ``run_context_for_judge`` names it and the run isn't staged + as files (a staged run has it as a file); then the run wake by wake. Both from the always_on step's record.""" + record = self._record(context, prompt_response) + sections = [] + if ASSISTANT_INSTRUCTIONS in self.run_context_for_judge and not self.use_trajectory: + text, why = await asyncio.to_thread(self._instructions, record) + sections.append(instructions_section(text, why)) + return [*sections, await asyncio.to_thread(render_for_judge, record)] + + def _instructions(self, record: dict) -> tuple[Optional[str], str]: + """What the assistant was told before the run (``stored_system_prompt``), else None and why the judge doesn't + get it: the run's record doesn't keep it, the step sent none, or it couldn't be read. Never raises.""" + if "agent_system_prompt" not in record: + logger.warning("Verifier '%s': the always_on run's record doesn't keep the assistant's system prompt, so " + "the judge isn't shown it", self.verifier_id) + return None, "The run's record doesn't keep it." + try: + text = stored_system_prompt(record) + except Exception as e: + logger.warning("Verifier '%s': the assistant's system prompt could not be read (%s: %s)", self.verifier_id, + type(e).__name__, e) + return None, f"The run's record of it couldn't be read ({type(e).__name__}: {str(e)[:200]})." + if text is None: + why_not = record["agent_system_prompt"].get("why_not") or "no reason recorded" + return None, f"The always_on step sent the assistant no system prompt: {why_not}." + return text, "" async def stage_trajectory(self, context: TaskStepContext, prompt_response: PromptResponse, run_context: Optional[Run], sandbox: Any, container: Optional[str]) -> Optional[StagedTrajectory]: """The run as files (``stage_run``) in the judge's sandbox, each trajectory matched to its wake by the run's - stored timeline. Without that timeline (not kept, or unreadable), the turns are staged the same way by - position (``turns_by_position``). A trajectory that breaks the harness labels it carries raises + stored timeline, and what the assistant was told before the run (INSTRUCTIONS_FILE) when + ``run_context_for_judge`` names it. Without that timeline (not kept, or unreadable), the turns are staged the + same way by position (``turns_by_position``). A trajectory that breaks the harness labels it carries raises ``UngradableRunError``.""" record = self._record(context, prompt_response) timeline: Optional[list[dict]] = None @@ -352,9 +408,17 @@ async def stage_trajectory(self, context: TaskStepContext, prompt_response: Prom _CONTEXT_LOSS_RUN.set(bool(staged.record.get("agent_labeled_trajectories")) and self.output_format in _CONTEXT_LOSS_FORMATS) path = f"{STAGING_DIR}/{uuid.uuid4().hex[:12]}" - await self._write_files(sandbox, {f"{path}/{name}": text for name, text in staged.files.items()}, container) + files = dict(staged.files) + instructions = "" + if ASSISTANT_INSTRUCTIONS in self.run_context_for_judge: + told, why = await asyncio.to_thread(self._instructions, record) + if told is not None: + files[INSTRUCTIONS_FILE] = instructions_file(told) + instructions = instructions_note(f"{path}/{INSTRUCTIONS_FILE}" if told is not None else None, why) + await self._write_files(sandbox, {f"{path}/{name}": text for name, text in files.items()}, container) self._log_staging(staged) - return StagedTrajectory(path=path, prompt_note=_files_note(path, staged, context_loss=_CONTEXT_LOSS_RUN.get()), + return StagedTrajectory(path=path, prompt_note=_files_note(path, staged, context_loss=_CONTEXT_LOSS_RUN.get(), + instructions=instructions), record=staged.record) async def _read_turns(self, urls: list[str], names: dict[str, str]) -> dict[str, Optional[TurnEvents]]: @@ -448,7 +512,9 @@ def walk(node: Any) -> None: return names -def _files_note(path: str, staged: StagedRun, *, context_loss: bool = False) -> str: +def _files_note(path: str, staged: StagedRun, *, context_loss: bool = False, instructions: str = "") -> str: + """How to read the staged run at ``path``; ``instructions`` is the line on what the assistant was told before the + run (``instructions_note``), when the verifier shows it.""" r = staged.record wakes = sorted(name for name in staged.files if name.startswith(f"{WAKES_DIR}/wake_")) gaps = sorted(set(r.get("missing_trajectories", [])) | set(r.get("unreadable_trajectories", []))) @@ -461,7 +527,32 @@ def _files_note(path: str, staged: StagedRun, *, context_loss: bool = False) -> last=f"{path}/{wakes[-1] if wakes else WAKES_DIR}", results=f"{path}/{RESULTS_DIR}", events=f"{path}/{EVENTS_FILE}", triggers=f"{path}/{TRIGGERS_FILE}", by_position=_BY_POSITION_NOTE if r.get("turns_by_position") else "", gaps=gap_note, - harness=harness) + harness=harness, instructions=instructions) + + +def instructions_file(text: str) -> str: + """INSTRUCTIONS_FILE's contents: what the assistant was told before the run, under a heading saying so.""" + return f"# {INSTRUCTIONS_TITLE}\n{_INSTRUCTIONS_LEAD}\n\n---\n\n{text}\n" + + +def instructions_note(file: Optional[str], why: str) -> str: + """The files note's line on what the assistant was told before the run: the ``file`` that has it, else ``why`` + there is none.""" + if file is None: + return f"\n- {INSTRUCTIONS_TITLE} (its standing instructions) is not in the files. {why}" + return (f"\n- {file}: {INSTRUCTIONS_TITLE.lower()} (its standing instructions: the system prompt the run " + "configured it with, as sent). It knew what is written there from the start; anything else it had to " + "learn during the run.") + + +def instructions_section(text: Optional[str], why: str) -> str: + """The judge prompt's section on what the assistant was told before the run: its ``text``, at most + INSTRUCTIONS_PROMPT_CHARS of it, else ``why`` there is none.""" + if text is None: + return f"## {INSTRUCTIONS_TITLE}\n{why}" + cut = len(text) - INSTRUCTIONS_PROMPT_CHARS + shown = text if cut <= 0 else f"{text[:INSTRUCTIONS_PROMPT_CHARS]}… ({cut} more characters not shown)" + return f"## {INSTRUCTIONS_TITLE}\n{_INSTRUCTIONS_LEAD}\n\n{shown}\n" def turns_by_position(prompt_response: PromptResponse) -> list[dict]: diff --git a/tst/unit/task_step/always_on_harness.py b/tst/unit/task_step/always_on_harness.py index 15c0d30f3..a578fd830 100644 --- a/tst/unit/task_step/always_on_harness.py +++ b/tst/unit/task_step/always_on_harness.py @@ -27,6 +27,23 @@ USER_SIM = "http://user-sim.test" T0 = "2026-11-09T14:00:00Z" # Mon Nov 9, 9:00 AM in New York ROLE = "default" +# A task's system prompt for the agent, as personal-assistant tasks write it: who it works for, then its memory of the +# user's standing rules and preferences as a MEMORY.md block. +MEMORY_PROMPT = """You are the personal assistant of Linda Murphy (linda.murphy@example.test). You can use their \ +stayncook account through the stayncook tools: stayncook is the stand-in for Airbnb, and holds their own trips, \ +wishlist and account. + +Your memory from earlier conversations with Linda, as your MEMORY.md file: +```markdown +# MEMORY.md + +- Linda's standing rules: just book her own flights (nonstop, never leaving before 9am, aisle seat) and restaurant \ +reservations; don't ask. Ask her first before spending more than $500 on anything else, or before paying for anyone \ +but her. Text her only when you need a decision, plus one recap at the end. +- Linda works 9 to 6 on weekdays and is asleep by about 11:30pm; she answers texts when she can. +- Linda's best friend is Noor Haddad; Noor's partner is Sam Rivera. +- Linda books places to stay on stayncook and dinners on Tablefern. +```""" Turn = Callable[[int, list[str], "Rig"], Optional[str]] # An agent card whose agent-config extension takes the settings the step sends. diff --git a/tst/unit/task_step/test_always_on_agent_verifier.py b/tst/unit/task_step/test_always_on_agent_verifier.py index 87918c317..b6cc393c4 100644 --- a/tst/unit/task_step/test_always_on_agent_verifier.py +++ b/tst/unit/task_step/test_always_on_agent_verifier.py @@ -1,6 +1,7 @@ """always_on_agent_verifier: an always_on step's prompt is graded from the user's chat (the response and the -conversation) and the run's context (the trigger log, the run wake by wake with its app changes) and, with -use_trajectory, every wake's tool calls staged as files in an agent judge's sandbox. A direct judge with the +conversation) and the run's context (the trigger log, the run wake by wake with its app changes, and, when the task asks +for it, what the assistant was told before the run) and, with use_trajectory, every wake's tool calls staged as files +in an agent judge's sandbox. A direct judge with the trajectories, and what cites screenshots or cuts turns, is refused when the task loads, as is a plain rubrics_verifier on an always_on step's prompt; a plain rubrics_verifier grades any other prompt as before.""" @@ -21,11 +22,12 @@ from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep from agent_env.task_step.task_steps.verifiers import is_rubrics_verifier from agent_env.task_step.task_steps.verifiers.always_on_agent_verifier import ( + INSTRUCTIONS_FILE, AlwaysOnAgentVerifier, UngradableRunError, ) from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RUN_CONTEXT_PARTS, RubricsVerifierTaskStep -from tst.unit.task_step.always_on_harness import Rig +from tst.unit.task_step.always_on_harness import MEMORY_PROMPT, Rig pytestmark = pytest.mark.usefixtures("local_stores") @@ -52,7 +54,7 @@ def no_trajectories(uri): return captured -async def _run(monkeypatch, tmp_path): +async def _run(monkeypatch, tmp_path, *, system_prompt: Optional[str] = None, takes_config: bool = False): def agent(n, texts, rig): if n == 1: rig.message("Casa Jacarandas is $855. Book it?") @@ -64,11 +66,12 @@ def agent(n, texts, rig): rig.quiet() return "" - rig = Rig(monkeypatch, tmp_path, agent=agent, answers=[{"outcome": "reply", "message": "yes please"}]) + rig = Rig(monkeypatch, tmp_path, agent=agent, answers=[{"outcome": "reply", "message": "yes please"}], + takes_config=takes_config) await rig.arm() rig.keep_changelog() rig.triggers(rig.notify_at("sam", "2026-11-10T14:00:00Z", "Sam: shh", channel="noor 30")) - ctx = await rig.run(rig.step()) + ctx = await rig.run(rig.step(system_prompt=system_prompt)) [response] = ctx.prompt_responses response.target_agent_per_turn_trajectory_object_urls = [f"s3://wakes/{n}.json" for n in range(1, 4)] response.agent_trajectory_object_url = "s3://wakes/3.json" @@ -260,12 +263,19 @@ async def write_file_from_text(self, content, destination_path, **kwargs): self.files[destination_path] = content -def _staged_run(monkeypatch, *, object_url: bool = True) -> tuple[TaskStepContext, _Sandbox, dict[str, str]]: +def _staged_run(monkeypatch, *, object_url: bool = True, + system_prompt: Optional[str] = None) -> tuple[TaskStepContext, _Sandbox, dict[str, str]]: """A recorded always_on run's context with a deployed judge; ``sources`` serves each stored trajectory: wake 2's - a Claude Code turn, wake 4's a PSI neutral harness v7 wake (Claude's: a save turn, a compaction, its own turn).""" + a Claude Code turn, wake 4's a PSI neutral harness v7 wake (Claude's: a save turn, a compaction, its own turn). + The recorded run keeps no system prompt; with ``system_prompt`` it records the agent was sent that one.""" stored = json.loads((RECORDED / "always_on_timeline.json").read_text()) urls = [e["trajectory_url"] for e in stored["timeline"] if e["event"] == "agent_turn_end"] + if system_prompt is not None: + stored["agent_system_prompt"] = {"agent_name": "solver", "sent": True, "chars": len(system_prompt), + "text": system_prompt} record = {k: v for k, v in stored.items() if k != "timeline"} + if system_prompt is not None: + record["agent_system_prompt"] = {k: v for k, v in stored["agent_system_prompt"].items() if k != "text"} if object_url: record["object_url"] = get_config().get_object_store().put( "always_on_timeline/week.json", json.dumps(stored).encode(), content_type="application/json") @@ -644,3 +654,136 @@ def note(writers: dict) -> str: both = note({"assistant": 1, "harness": 1}) assert both.index(assistant) < both.index(harness) assert assistant not in note({}) and harness not in note({}) + + +# ---- assistant_instructions: what the assistant was told before the run ------------------------------------------- + +TOLD = "## What the assistant was told before the run" +WITH_INSTRUCTIONS = [*RUN_CONTEXT_PARTS, "assistant_instructions"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("parts", [WITH_INSTRUCTIONS, ["assistant_instructions"]]) +async def test_a_judge_asked_for_it_reads_what_the_assistant_was_told_before_the_run(monkeypatch, tmp_path, parts): + rig, ctx = await _run(monkeypatch, tmp_path, system_prompt=MEMORY_PROMPT, takes_config=True) + verifier = _verifier(run_context_for_judge=parts) + captured = _capture(monkeypatch, verifier) + + await verifier.execute(ctx) + + prompt = captured["prompt"] + told = prompt[prompt.index(TOLD):prompt.index("## The run, wake by wake")] + assert "It knew what is written here from the start; anything else it had to learn during the run." in told + assert f"\n{MEMORY_PROMPT}\n\n{ao.MESSAGE_RULES} {ao.QUIET_RULE}\n" in told + assert ("## Conversation" in prompt) is ("conversation" in parts) + assert ctx.metadata["verifications"]["rubric"]["score"] == 1 + + +@pytest.mark.asyncio +async def test_by_default_the_judge_is_not_shown_what_the_assistant_was_told(monkeypatch, tmp_path): + rig, ctx = await _run(monkeypatch, tmp_path, system_prompt=MEMORY_PROMPT, takes_config=True) + verifier = _verifier() + captured = _capture(monkeypatch, verifier) + + await verifier.execute(ctx) + + assert verifier.run_context_for_judge == list(RUN_CONTEXT_PARTS) == ["conversation", "trigger_log"] + assert TOLD not in captured["prompt"] and "MEMORY.md" not in captured["prompt"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("record_says,why", [ + ("nothing", "The run's record doesn't keep it."), + ("not sent", "The always_on step sent the assistant no system prompt: the agent's card doesn't take one."), + ("unreadable", "The run's record of it couldn't be read (ObjectNotFoundError: No object at " + "file:///nowhere/at/all.json.)."), +]) +async def test_without_what_the_assistant_was_told_the_judge_is_told_why_and_grades(monkeypatch, tmp_path, + record_says, why): + rig, ctx = await _run(monkeypatch, tmp_path, system_prompt=MEMORY_PROMPT, takes_config=record_says != "not sent") + record = rig.record(ctx) + if record_says == "nothing": + del record["agent_system_prompt"] + elif record_says == "unreadable": + record["object_url"] = "file:///nowhere/at/all.json" + verifier = _verifier(run_context_for_judge=WITH_INSTRUCTIONS) + captured = _capture(monkeypatch, verifier) + + await verifier.execute(ctx) + + assert f"{TOLD}\n{why}" in captured["prompt"] and "" not in captured["prompt"] + assert ctx.metadata["verifications"]["rubric"]["score"] == 1 + + +def test_a_long_system_prompt_is_cut_in_the_judge_s_prompt_and_says_so(monkeypatch): + from agent_env.task_step.task_steps.verifiers import always_on_agent_verifier as v + + monkeypatch.setattr(v, "INSTRUCTIONS_PROMPT_CHARS", 40) + section = v.instructions_section(MEMORY_PROMPT, "") + assert f"\n{MEMORY_PROMPT[:40]}… ({len(MEMORY_PROMPT) - 40} more characters not shown)\n" in section + + +@pytest.mark.asyncio +async def test_a_staged_run_has_what_the_assistant_was_told_as_a_file_the_note_names(monkeypatch): + ctx, sandbox, sources = _staged_run(monkeypatch, system_prompt=MEMORY_PROMPT) + verifier = _verifier(use_agent_judge=True, use_trajectory=True, agent_name="judge", + run_context_for_judge=WITH_INSTRUCTIONS) + captured = _agent_judge(monkeypatch, verifier, sources) + + await verifier.execute(ctx) + + [root] = {path.split("/")[3] for path in sandbox.files} + file = f"/tmp/always_on_run/{root}/{INSTRUCTIONS_FILE}" + assert sandbox.files[file].startswith("# What the assistant was told before the run\nIts standing instructions: ") + assert sandbox.files[file].endswith(f"\n---\n\n{MEMORY_PROMPT}\n") + prompt = captured["prompt"] + note = prompt[prompt.index("## The run's files"):prompt.index("How to read them:")] + assert (f"- {file}: what the assistant was told before the run (its standing instructions: the system prompt the " + "run configured it with, as sent). It knew what is written there from the start; anything else it had to " + "learn during the run.") in note + assert TOLD not in prompt and "Ask her first before spending more than $500" not in prompt + assert ctx.metadata["verifications"]["rubric"]["score"] == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("parts", [None, WITH_INSTRUCTIONS]) +async def test_a_staged_run_whose_record_doesn_t_keep_it_says_so_or_nothing_when_not_asked(monkeypatch, parts): + ctx, sandbox, sources = _staged_run(monkeypatch) # recorded before the run kept its system prompt + verifier = _verifier(use_agent_judge=True, use_trajectory=True, agent_name="judge", run_context_for_judge=parts) + captured = _agent_judge(monkeypatch, verifier, sources) + + await verifier.execute(ctx) + + assert not any(path.endswith(INSTRUCTIONS_FILE) for path in sandbox.files) + missing = ("- What the assistant was told before the run (its standing instructions) is not in the files. The " + "run's record doesn't keep it.\n\nHow to read them:") + if parts is None: + assert "standing instructions" not in captured["prompt"] + else: + assert missing in captured["prompt"] + assert ctx.metadata["verifications"]["rubric"]["score"] == 1 + + +@pytest.mark.asyncio +async def test_a_staged_run_not_asked_for_it_gets_no_file_though_its_record_keeps_it(monkeypatch): + ctx, sandbox, sources = _staged_run(monkeypatch, system_prompt=MEMORY_PROMPT) + verifier = _verifier(use_agent_judge=True, use_trajectory=True, agent_name="judge") + captured = _agent_judge(monkeypatch, verifier, sources) + + await verifier.execute(ctx) + + assert not any(path.endswith(INSTRUCTIONS_FILE) for path in sandbox.files) + assert "standing instructions" not in captured["prompt"] + + +def test_assistant_instructions_is_a_part_of_this_verifier_alone_and_round_trips(): + verifier = _verifier(run_context_for_judge=["assistant_instructions", "conversation", "assistant_instructions"]) + assert verifier.run_context_for_judge == ["assistant_instructions", "conversation"] + again = AlwaysOnAgentVerifier.from_dict(verifier.to_dict()) + assert again.run_context_for_judge == ["assistant_instructions", "conversation"] + assert again.to_dict() == verifier.to_dict() + with pytest.raises(ValueError, match="run_context_for_judge must be a list of \\['conversation', 'trigger_log', " + "'assistant_instructions'\\]"): + _verifier(run_context_for_judge=["conversation", "memory"]) + with pytest.raises(ValueError, match="run_context_for_judge"): + _verifier(RubricsVerifierTaskStep, run_context_for_judge=["assistant_instructions"]) diff --git a/tst/unit/task_step/test_always_on_record.py b/tst/unit/task_step/test_always_on_record.py index 9f0472bf9..67effb0b6 100644 --- a/tst/unit/task_step/test_always_on_record.py +++ b/tst/unit/task_step/test_always_on_record.py @@ -1,5 +1,5 @@ """What the always_on step keeps in the run's metadata (a short summary, small enough for a running job to report back) -and what it stores in full (the timeline, in the object store).""" +and what it stores in full (the timeline and the agent's system prompt, in the object store).""" from __future__ import annotations @@ -9,7 +9,7 @@ from agent_env.config import get_config from agent_env.task_step.task_steps import always_on as ao -from tst.unit.task_step.always_on_harness import Rig +from tst.unit.task_step.always_on_harness import MEMORY_PROMPT, SOLVER, Rig pytestmark = pytest.mark.usefixtures("local_stores") @@ -54,6 +54,61 @@ def fail(*a, **kw): assert not any(set(e) & set(ao.BULKY_EVENT_FIELDS) for e in record["timeline_tail"]) +@pytest.mark.asyncio +async def test_the_record_says_what_the_agent_was_told_before_the_run_and_stores_the_text(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, takes_config=True) + await rig.arm() + ctx = await rig.run(rig.step(system_prompt=MEMORY_PROMPT)) + + [sent] = [fields["system_prompt"] for url, fields in rig.configs if url.startswith(SOLVER)] + record = rig.record(ctx) + assert record["agent_system_prompt"] == {"agent_name": "solver", "sent": True, "chars": len(sent)} + assert rig.stored(ctx)["agent_system_prompt"]["text"] == sent + told = ao.stored_system_prompt(record) + assert told == sent and told.startswith(MEMORY_PROMPT) and told.endswith(f"{ao.MESSAGE_RULES} {ao.QUIET_RULE}") + assert "Ask her first before spending more than $500 on anything else" in told + + +@pytest.mark.asyncio +@pytest.mark.parametrize("takes_config,system_prompt,why_not", [ + (False, MEMORY_PROMPT, "the agent's card doesn't take one"), + (True, None, "the step sets none"), +]) +async def test_an_agent_sent_no_system_prompt_is_recorded_with_why(monkeypatch, tmp_path, takes_config, system_prompt, + why_not): + rig = Rig(monkeypatch, tmp_path, takes_config=takes_config) + await rig.arm() + ctx = await rig.run(rig.step(system_prompt=system_prompt)) + + record = rig.record(ctx) + assert record["agent_system_prompt"] == {"agent_name": "solver", "sent": False, "why_not": why_not, "chars": 0} + assert ao.stored_system_prompt(record) is None + assert "text" not in rig.stored(ctx)["agent_system_prompt"] + + +@pytest.mark.asyncio +async def test_when_the_record_cannot_be_stored_it_keeps_the_system_prompt_s_text(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, takes_config=True) + await rig.arm() + store = get_config().get_object_store() + + def fail(*a, **kw): + raise OSError("bucket gone") + + monkeypatch.setattr(type(store), "put", fail) + ctx = await rig.run(rig.step(system_prompt=MEMORY_PROMPT)) + + [sent] = [fields["system_prompt"] for url, fields in rig.configs if url.startswith(SOLVER)] + record = rig.record(ctx) + assert "object_url" not in record and record["agent_system_prompt"]["text"] == sent + assert ao.stored_system_prompt(record) == sent + + +def test_a_record_that_doesn_t_keep_the_system_prompt_raises_when_it_is_asked_for(): + with pytest.raises(KeyError): + ao.stored_system_prompt({"end_reason": "end_time", "object_url": "file:///nowhere/at/all.json"}) + + @pytest.mark.asyncio async def test_the_process_log_carries_no_one_s_words_only_the_timeline_does(monkeypatch, tmp_path, caplog): def agent(n, texts, rig):