Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 37 additions & 4 deletions src/agent_env/task_step/task_steps/always_on.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,10 @@
``user_reads_at``.
- The run's metadata keeps a summary in ``metadata["always_on"][step_id]`` (small enough for a running job to report
back); the full timeline goes to the object store, at the summary's ``object_url``. Its ``user_sim`` names the user
sim's session (``context_id``), which rate_user_satisfaction asks in once the run is over.
sim's session (``context_id``), which rate_user_satisfaction asks in once the run is over. Its
``agent_system_prompt`` says what the agent was told before the run: the agent, whether the step sent it a system
prompt (``why_not`` when it didn't) and its length; the text itself is in the stored record
(``stored_system_prompt``).
- An always_on_agent_verifier (``verifiers/always_on_agent_verifier.py``) grades the step's prompt from the user's chat
and the run's context, the run wake by wake among it (``render_for_judge``), and, with use_trajectory, every wake's
tool calls staged as files; a plain rubrics_verifier on it is refused when the task loads.
Expand Down Expand Up @@ -410,6 +413,7 @@ def __init__(self, step: AlwaysOnTaskStep, context: TaskStepContext):
self.clock_rates: list[tuple[Any, float]] = [] # (env, the task's rate) for each clock the step freezes
self.undo: list[tuple[str, Any]] = [] # (what, coroutine function): the gateway settings to put back at the end
self.rules_in_system_prompt = False # whether the agent took the rules in its system prompt
self.agent_system_prompt: Optional[dict] = None # what the agent was configured with, for the record

async def restore(self) -> None:
"""Put back what the step changed on the gateways, latest first, so later steps of the task see the settings
Expand Down Expand Up @@ -599,6 +603,12 @@ async def _configure_agents(self) -> None:
if desired.get("system_prompt") and not self.rules_in_system_prompt:
logger.warning("always_on: the agent's card did not accept system_prompt, so the message_user rules go "
"on its first wake instead")
self.agent_system_prompt = {"agent_name": step.agent_name, "sent": self.rules_in_system_prompt}
if self.rules_in_system_prompt:
self.agent_system_prompt["text"] = negotiated.fields["system_prompt"]
else:
self.agent_system_prompt["why_not"] = ("the agent's card doesn't take one" if desired.get("system_prompt")
else "the step sets none")

if self.us.get("prompt_set"): # deploy_user_sim_agent set the user sim's prompt and model at deploy
self.user_model, self.user_model_from = self.us["model"], self.us["model_source"]
Expand Down Expand Up @@ -1703,6 +1713,9 @@ async def finish(self) -> None:
"scheduled_checks": self.checks, "contacts": self.contacts,
"time_budget_seconds": self.time_budget,
"wake_reserve_seconds": step.wake_reserve_seconds(self.contact_seconds)}
if self.agent_system_prompt is not None:
record["agent_system_prompt"] = {**{k: v for k, v in self.agent_system_prompt.items() if k != "text"},
"chars": len(self.agent_system_prompt.get("text") or "")}
await self._store_timeline(record)
ctx.metadata.setdefault(ALWAYS_ON_KEY, {})[step.id] = record
# The graded response is the user's whole chat from the agent, each message with its pretend time; never its
Expand All @@ -1711,15 +1724,19 @@ async def finish(self) -> None:
for m in self.all_messages))

async def _store_timeline(self, record: dict) -> None:
"""Store the run's full record, its timeline included, and give ``record`` (what stays in the run's metadata,
kept small for the job's report) its ``object_url``. When it can't be stored, ``record`` keeps the last
``TIMELINE_TAIL`` events instead, their bulky fields dropped."""
"""Store the run's full record, its timeline and the agent's system prompt included, and give ``record`` (what
stays in the run's metadata, kept small for the job's report) its ``object_url``. When it can't be stored,
``record`` keeps the last ``TIMELINE_TAIL`` events instead, their bulky fields dropped, and the system prompt's
text."""
step, ctx = self.step, self.ctx
text = (self.agent_system_prompt or {}).get("text")
try:
store = get_config().get_object_store()
key = (f"{get_config().get_artifact_key_prefix()}always_on_timeline/"
f"instance_id={ctx.instance_id or 'adhoc'}/{step.id}-{uuid.uuid4().hex[:8]}.json")
full = {**record, "timeline": self.timeline}
if text is not None:
full["agent_system_prompt"] = {**record["agent_system_prompt"], "text": text}
record["object_url"] = await asyncio.to_thread(store.put, key,
json.dumps(full, indent=1, default=str).encode(),
content_type="application/json")
Expand All @@ -1728,6 +1745,8 @@ async def _store_timeline(self, record: dict) -> None:
record["timeline_store_error"] = f"{type(e).__name__}: {str(e)[:300]}"
record["timeline_tail"] = [_small({k: v for k, v in event.items() if k not in BULKY_EVENT_FIELDS})
for event in self.timeline[-TIMELINE_TAIL:]]
if text is not None:
record["agent_system_prompt"]["text"] = text

def _append_prompt_response(self, final_text: str) -> None:
step, ctx = self.step, self.ctx
Expand Down Expand Up @@ -1760,6 +1779,20 @@ def stored_timeline(record: dict) -> list[dict]:
return json.loads(get_config().get_object_store_at(url).get(url))["timeline"]


def stored_system_prompt(record: dict) -> Optional[str]:
"""The system prompt the run's agent was configured with, as the step sent it (its own, then the rules), from the
stored record at ``record["object_url"]`` (else the record's own copy, kept when the store failed); None when the
step sent none (``record["agent_system_prompt"]["why_not"]`` says why). Raises KeyError for a record that doesn't
keep it, and the object store's error when the stored record can't be read."""
entry = record["agent_system_prompt"]
if not entry.get("sent"):
return None
if "text" in entry:
return entry["text"]
url = record["object_url"]
return json.loads(get_config().get_object_store_at(url).get(url))["agent_system_prompt"]["text"]


def woke_by_text(woke_by: list[str]) -> str:
"""What woke the agent, in words: "the user's reply, an alert (Gmail)"."""
return ", ".join(_WOKE_BY.get(w, f"an alert ({w.split(':', 1)[1]})" if w.startswith("notification:") else w)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,12 @@
A run that kept no timeline has its turns staged the same way by position, one wake file a turn, so the judge never
reads a trajectory as stored.

``run_context_for_judge`` may also name ``assistant_instructions`` (not among its defaults): what the assistant was
told before the run, the system prompt the always_on step configured it with (``stored_system_prompt``), so the judge
can tell what it was told up front from what it had to learn. A staged run has it as a file beside the index
(``INSTRUCTIONS_FILE``), which the files note names; otherwise it is a section of the judge's prompt. A run whose
record doesn't keep it, or whose step sent none, has a line saying so instead; it never stops the grading.

For a run of the PSI neutral harness (its trace carries ``harness.*`` labels), the judge is told whose a compaction
summary's losses are, by who wrote it (``harness.compaction.written_by``: the assistant's own model, graded as its own,
or the harness, not held against the assistant), and, with a per-criterion format, says for each failed criterion
Expand All @@ -39,7 +45,13 @@
from agent_env.providers.sandbox_providers.sandbox import into_container
from agent_env.task_step.context import PromptResponse, TaskStepContext
from agent_env.task_step.task_step import TaskStep
from agent_env.task_step.task_steps.always_on import ALWAYS_ON_KEY, AlwaysOnTaskStep, render_for_judge, stored_timeline
from agent_env.task_step.task_steps.always_on import (
ALWAYS_ON_KEY,
AlwaysOnTaskStep,
render_for_judge,
stored_system_prompt,
stored_timeline,
)
from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep
from agent_env.task_step.task_steps.verifiers.always_on_staging import (
EVENTS_FILE,
Expand Down Expand Up @@ -72,6 +84,17 @@

# Where the run's files go in the judge's sandbox, under a directory of the run's own.
STAGING_DIR = "/tmp/always_on_run"
# The run_context_for_judge part this verifier adds to rubrics_verifier's (RUN_CONTEXT_PARTS, its defaults): what the
# assistant was told before the run. Staged as INSTRUCTIONS_FILE; in the judge's prompt, at most
# INSTRUCTIONS_PROMPT_CHARS of it.
ASSISTANT_INSTRUCTIONS = "assistant_instructions"
ALWAYS_ON_RUN_CONTEXT_PARTS = (*RUN_CONTEXT_PARTS, ASSISTANT_INSTRUCTIONS)
INSTRUCTIONS_FILE = "assistant_instructions.md"
INSTRUCTIONS_PROMPT_CHARS = 60_000
INSTRUCTIONS_TITLE = "What the assistant was told before the run"
_INSTRUCTIONS_LEAD = ("Its standing instructions: the system prompt the run configured it with before its first wake, "
"as sent. It knew what is written here from the start; anything else it had to learn during the "
"run.")
# Trajectories read, and files written into the judge's sandbox, at once.
STAGING_CONCURRENCY = 8
# The only tools of its own the judge gets: it reads and searches the files (the CLI's search is Bash's grep), and its
Expand All @@ -95,7 +118,7 @@
("<field>_file", such as "result_file" or "text_file") and shows its start ("<field>_preview"). Email bodies are \
decoded (body.text). A line longer than 1,800 characters is split across lines there, nothing dropped.
- {events}: every wake's events in one file, so "did it ever do X" is one search. {triggers}: every trigger firing, \
fired or failed, with its pretend time and the wake it came after.
fired or failed, with its pretend time and the wake it came after.{instructions}

How to read them:
- Don't load whole files. Search with grep -n, then Read with an offset and limit.
Expand Down Expand Up @@ -195,9 +218,14 @@ def __init__(
judge_max_turns: Optional[int] = None,
**fields: Any,
):
"""``run_context_for_judge`` defaults to every part of the run's context; the rest is rubrics_verifier's."""
"""``run_context_for_judge`` defaults to rubrics_verifier's parts (RUN_CONTEXT_PARTS) and may also name
``assistant_instructions``; the rest is rubrics_verifier's."""
output_format = JudgeOutputFormat(output_format)
where = f"{self.type} '{id}'"
parts = run_context_for_judge or list(RUN_CONTEXT_PARTS)
if not isinstance(parts, list) or any(p not in ALWAYS_ON_RUN_CONTEXT_PARTS for p in parts):
raise ValueError(f"{where}: run_context_for_judge must be a list of {list(ALWAYS_ON_RUN_CONTEXT_PARTS)}, "
f"got {parts!r}")
if use_trajectory and not use_agent_judge:
raise ValueError(f"{where}: use_trajectory needs the agent judge (use_agent_judge): an always_on run's "
"wakes are staged as files for a judge that searches them, never inlined")
Expand All @@ -209,7 +237,8 @@ def __init__(
"turns are not its wakes")
super().__init__(
id, version, use_trajectory=use_trajectory, use_agent_judge=use_agent_judge,
run_context_for_judge=run_context_for_judge or list(RUN_CONTEXT_PARTS), output_format=output_format,
run_context_for_judge=[p for p in parts if p != ASSISTANT_INSTRUCTIONS] or None,
output_format=output_format,
default_model=default_model, default_model_api_base=default_model_api_base, effort=effort,
max_thinking_tokens=max_thinking_tokens, judge_a2a_agent_id=judge_a2a_agent_id,
judge_sandbox_type=judge_sandbox_type, judge_batch_size=judge_batch_size,
Expand All @@ -219,6 +248,8 @@ def __init__(
judge_max_turns=self.DEFAULT_JUDGE_MAX_TURNS if judge_max_turns is None else judge_max_turns,
**fields,
)
# Every part, this verifier's own too: rubrics_verifier renders the parts it knows and passes the others by.
self.run_context_for_judge = list(dict.fromkeys(parts))

@classmethod
def from_dict(cls, data: dict) -> AlwaysOnAgentVerifier:
Expand Down Expand Up @@ -319,15 +350,40 @@ def _record(self, context: TaskStepContext, prompt_response: PromptResponse) ->
return record

async def judge_prompt_sections(self, context: TaskStepContext, prompt_response: PromptResponse) -> list[str]:
"""The run wake by wake, from the always_on step's record of it."""
return [await asyncio.to_thread(render_for_judge, self._record(context, prompt_response))]
"""What the assistant was told before the run, when ``run_context_for_judge`` names it and the run isn't staged
as files (a staged run has it as a file); then the run wake by wake. Both from the always_on step's record."""
record = self._record(context, prompt_response)
sections = []
if ASSISTANT_INSTRUCTIONS in self.run_context_for_judge and not self.use_trajectory:
text, why = await asyncio.to_thread(self._instructions, record)
sections.append(instructions_section(text, why))
return [*sections, await asyncio.to_thread(render_for_judge, record)]

def _instructions(self, record: dict) -> tuple[Optional[str], str]:
"""What the assistant was told before the run (``stored_system_prompt``), else None and why the judge doesn't
get it: the run's record doesn't keep it, the step sent none, or it couldn't be read. Never raises."""
if "agent_system_prompt" not in record:
logger.warning("Verifier '%s': the always_on run's record doesn't keep the assistant's system prompt, so "
"the judge isn't shown it", self.verifier_id)
return None, "The run's record doesn't keep it."
try:
text = stored_system_prompt(record)
except Exception as e:
logger.warning("Verifier '%s': the assistant's system prompt could not be read (%s: %s)", self.verifier_id,
type(e).__name__, e)
return None, f"The run's record of it couldn't be read ({type(e).__name__}: {str(e)[:200]})."

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Error preview limit is unnamed

The new error preview cuts exception text with an inline 200. The repository requires numeric limits to be named class or instance variables. Give this limit a descriptive name so readers can see what it controls without hunting through string slices. This repository requirement must be satisfied before merging.

Rule Used: Store magic numbers as class or instance variables with descriptive names rather than using them inline in the code. (source)

Learned From
scaleapi/scaleapi#126388

Prompt To Fix With AI
This is a comment left during a code review.
Path: src/agent_env/task_step/task_steps/verifiers/always_on_agent_verifier.py
Line: 374

Comment:
**Error preview limit is unnamed**

The new error preview cuts exception text with an inline `200`. The repository requires numeric limits to be named class or instance variables. Give this limit a descriptive name so readers can see what it controls without hunting through string slices. This repository requirement must be satisfied before merging.

**Rule Used:** Store magic numbers as class or instance variables with descriptive names rather than using them inline in the code. ([source](https://app.greptile.com/-/switch-tenant?tenant=scale-ai&next=%2Fcustom-context%3Fmemory%3D002e0051-41ad-46c1-9098-47433c580150))

**Learned From**
[scaleapi/scaleapi#126388](https://github.com/scaleapi/scaleapi/pull/126388)

---

For each issue above, determine whether it is valid and should be fixed. If so, fix it directly.

Note: If this suggestion doesn't match your team's coding style, reply to this and let me know. I'll remember it for next time!

Fix in Cursor Fix in Claude Code Fix in Codex

if text is None:
why_not = record["agent_system_prompt"].get("why_not") or "no reason recorded"
return None, f"The always_on step sent the assistant no system prompt: {why_not}."
return text, ""

async def stage_trajectory(self, context: TaskStepContext, prompt_response: PromptResponse,
run_context: Optional[Run], sandbox: Any,
container: Optional[str]) -> Optional[StagedTrajectory]:
"""The run as files (``stage_run``) in the judge's sandbox, each trajectory matched to its wake by the run's
stored timeline. Without that timeline (not kept, or unreadable), the turns are staged the same way by
position (``turns_by_position``). A trajectory that breaks the harness labels it carries raises
stored timeline, and what the assistant was told before the run (INSTRUCTIONS_FILE) when
``run_context_for_judge`` names it. Without that timeline (not kept, or unreadable), the turns are staged the
same way by position (``turns_by_position``). A trajectory that breaks the harness labels it carries raises
``UngradableRunError``."""
record = self._record(context, prompt_response)
timeline: Optional[list[dict]] = None
Expand All @@ -352,9 +408,17 @@ async def stage_trajectory(self, context: TaskStepContext, prompt_response: Prom
_CONTEXT_LOSS_RUN.set(bool(staged.record.get("agent_labeled_trajectories"))
and self.output_format in _CONTEXT_LOSS_FORMATS)
path = f"{STAGING_DIR}/{uuid.uuid4().hex[:12]}"
await self._write_files(sandbox, {f"{path}/{name}": text for name, text in staged.files.items()}, container)
files = dict(staged.files)
instructions = ""
if ASSISTANT_INSTRUCTIONS in self.run_context_for_judge:
told, why = await asyncio.to_thread(self._instructions, record)
if told is not None:
files[INSTRUCTIONS_FILE] = instructions_file(told)
instructions = instructions_note(f"{path}/{INSTRUCTIONS_FILE}" if told is not None else None, why)
await self._write_files(sandbox, {f"{path}/{name}": text for name, text in files.items()}, container)
self._log_staging(staged)
return StagedTrajectory(path=path, prompt_note=_files_note(path, staged, context_loss=_CONTEXT_LOSS_RUN.get()),
return StagedTrajectory(path=path, prompt_note=_files_note(path, staged, context_loss=_CONTEXT_LOSS_RUN.get(),
instructions=instructions),
record=staged.record)

async def _read_turns(self, urls: list[str], names: dict[str, str]) -> dict[str, Optional[TurnEvents]]:
Expand Down Expand Up @@ -448,7 +512,9 @@ def walk(node: Any) -> None:
return names


def _files_note(path: str, staged: StagedRun, *, context_loss: bool = False) -> str:
def _files_note(path: str, staged: StagedRun, *, context_loss: bool = False, instructions: str = "") -> str:
"""How to read the staged run at ``path``; ``instructions`` is the line on what the assistant was told before the
run (``instructions_note``), when the verifier shows it."""
r = staged.record
wakes = sorted(name for name in staged.files if name.startswith(f"{WAKES_DIR}/wake_"))
gaps = sorted(set(r.get("missing_trajectories", [])) | set(r.get("unreadable_trajectories", [])))
Expand All @@ -461,7 +527,32 @@ def _files_note(path: str, staged: StagedRun, *, context_loss: bool = False) ->
last=f"{path}/{wakes[-1] if wakes else WAKES_DIR}", results=f"{path}/{RESULTS_DIR}",
events=f"{path}/{EVENTS_FILE}", triggers=f"{path}/{TRIGGERS_FILE}",
by_position=_BY_POSITION_NOTE if r.get("turns_by_position") else "", gaps=gap_note,
harness=harness)
harness=harness, instructions=instructions)


def instructions_file(text: str) -> str:
"""INSTRUCTIONS_FILE's contents: what the assistant was told before the run, under a heading saying so."""
return f"# {INSTRUCTIONS_TITLE}\n{_INSTRUCTIONS_LEAD}\n\n---\n\n{text}\n"


def instructions_note(file: Optional[str], why: str) -> str:
"""The files note's line on what the assistant was told before the run: the ``file`` that has it, else ``why``
there is none."""
if file is None:
return f"\n- {INSTRUCTIONS_TITLE} (its standing instructions) is not in the files. {why}"
return (f"\n- {file}: {INSTRUCTIONS_TITLE.lower()} (its standing instructions: the system prompt the run "
"configured it with, as sent). It knew what is written there from the start; anything else it had to "
"learn during the run.")


def instructions_section(text: Optional[str], why: str) -> str:
"""The judge prompt's section on what the assistant was told before the run: its ``text``, at most
INSTRUCTIONS_PROMPT_CHARS of it, else ``why`` there is none."""
if text is None:
return f"## {INSTRUCTIONS_TITLE}\n{why}"
cut = len(text) - INSTRUCTIONS_PROMPT_CHARS
shown = text if cut <= 0 else f"{text[:INSTRUCTIONS_PROMPT_CHARS]}… ({cut} more characters not shown)"
return f"## {INSTRUCTIONS_TITLE}\n{_INSTRUCTIONS_LEAD}\n<system_prompt>\n{shown}\n</system_prompt>"


def turns_by_position(prompt_response: PromptResponse) -> list[dict]:
Expand Down
Loading
Loading