Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions packages/agentenv-protocol/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,16 @@ step names it. Steps that name it show and hide it as they do any tool. Steps fo
that role's earlier named steps, and a full reset (`role: "*", tools: "*"`) clears all of them, so
grant a hidden tool after such steps. Trigger tool actions call it as usual.

A config extension can reveal hidden tools the config it applies makes usable: its reply carries
`"reveal_tools": [names]`, the tools as the server lists them, and `apply_server_config` then names
them for every role (`role: "*"`), as a tool-access step would, so the agent under test sees them too.
A missing, `null` or empty list reveals nothing; any other value that is not a list of tool names (or
names `"*"`) fails the step. A later full reset (every tool, every role) hides them again. Because the
reveal names every role, it also re-enables a revealed tool for a role an earlier step denied it to. Only
a step that names the tool hides it from one role: a step for every tool skips tools listed hidden by
default, in either order. To keep a revealed tool from a role, deny that tool by name after
`apply_server_config`.

`@tool(..., annotations={"readOnlyHint": True}, structured_output=True, meta={...})` lists the tool
with MCP `ToolAnnotations`, an output schema and `_meta`, on mcp's FastMCP and on fastmcp alike.
`structured_output` follows mcp's FastMCP (None derives an output schema from the return annotation
Expand Down
27 changes: 17 additions & 10 deletions src/agent_env/openenv/environment.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
from collections.abc import Callable, Coroutine
from contextlib import AsyncExitStack
from dataclasses import dataclass, field
from typing import Any, TypedDict
from typing import Any, NotRequired, TypedDict

import httpx
from openenv.core.env_server.interfaces import Environment
Expand Down Expand Up @@ -52,6 +52,7 @@ class Toolset(TypedDict):
role: str | None
access: list[dict[str, Any]]
bash: bool
config: NotRequired[bool]


@dataclass(frozen=True)
Expand All @@ -66,18 +67,24 @@ class ServedTask:
@property
def toolset(self) -> Toolset:
"""Where the played agent's tools come from: the envs it works in and the versions the task deploys, its role
in them, the changes the task makes to that role's tool access, and whether its turns offer ``bash`` in its
sandbox. Tasks that agree on it offer the same tools, so a trainer can group them."""
in them, the changes the task makes to that role's tool access, whether its turns offer ``bash`` in its
sandbox, and whether it applies server config there (``config``, only when it does), whose replies can reveal
tools. Tasks that agree on it offer the same tools, so a trainer can group them, except tasks with config:
their replies can reveal different tools, and a revealed tool depends on whether a per-role deny of it
comes before or after the config."""
agent = self.spec.trainer_agents[0]
[deploy] = [step for step in self.task.steps if step.type in DEPLOYS and step.agent_name == agent]
role, envs = getattr(deploy, "role", None), list(getattr(deploy, "env_ids", None) or [])
return Toolset(envs=envs, role=role,
versions={step.env_id: step.env_version for step in self.task.steps
if hasattr(step, "env_version") and step.env_id in envs},
access=[{"env": step.env_id, "action": step.action, "tools": step.tools}
for step in self.task.steps
if step.type == "modify_env_tool_access" and step.role == (role or DEFAULT_ROLE)],
bash=works_in_sandbox(self.task.steps, agent))
toolset = Toolset(envs=envs, role=role,
versions={step.env_id: step.env_version for step in self.task.steps
if hasattr(step, "env_version") and step.env_id in envs},
access=[{"env": step.env_id, "action": step.action, "tools": step.tools}
for step in self.task.steps
if step.type == "modify_env_tool_access" and step.role == (role or DEFAULT_ROLE)],
bash=works_in_sandbox(self.task.steps, agent))
if any(step.type == "apply_server_config" and step.env_id in envs for step in self.task.steps):
toolset["config"] = True
return toolset


@dataclass
Expand Down
41 changes: 39 additions & 2 deletions src/agent_env/task_step/task_steps/apply_server_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,15 +23,23 @@
import httpx
from agentenv_protocol import client as protocol_v1

from agent_env.env.gateway import TOOL_ENABLE_ACTION, WILDCARD
from agent_env.env.gateway.constants import EXT_ENABLE_TOOL_URI
from agent_env.task_step.context import TaskStepContext
from agent_env.entity.refs import EntityRef
from agent_env.task_step.task_step import TaskStep, TaskStepDependency
from agent_env.task_step.task_steps.modify_env_tool_access import record_tool_access_change

if TYPE_CHECKING:
from agent_env.env.env import DeployedEnv
from agent_env.env.legacy_protocol import BackingServer

logger = logging.getLogger(__name__)

# A config extension's reply may name tools the server lists hidden by default that this config makes usable; the
# step then enables them on the env's gateway for every role, as modify_env_tool_access would.
REVEAL_TOOLS_KEY = "reveal_tools"


class ApplyServerConfigError(RuntimeError):
"""Raised when a config directive cannot be applied (bad target / extension
Expand Down Expand Up @@ -279,14 +287,43 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext:
f"apply_server_config: applied {directive.uri} on service={directive.service} "
f"env={self.env_id} args={directive.args}: {result}"
)
changes.append({
change = {
"step_id": self.id,
"env_id": self.env_id,
"service": directive.service,
"environment": directive.service,
"uri": directive.uri,
"args": directive.args,
"result": result,
})
}
changes.append(change)
revealed = await self._reveal_tools(deployed, directive, result, context)
if revealed:
change["revealed_tools"] = revealed

return context

async def _reveal_tools(self, deployed: DeployedEnv, directive: ConfigDirective, result: Any,
context: TaskStepContext) -> list[str]:
tools = result.get(REVEAL_TOOLS_KEY) if isinstance(result, dict) else None
if tools is None or tools == []:
return []
if not isinstance(tools, list) or not all(isinstance(t, str) and t and t != WILDCARD for t in tools):
raise ApplyServerConfigError(
f"Applying {directive.uri!r} on service {directive.service!r} (env={self.env_id}): its reply's "
f"{REVEAL_TOOLS_KEY!r} must be a list of tool names, got {tools!r}"
)
if deployed.environment_card and not deployed.supports(EXT_ENABLE_TOOL_URI, TOOL_ENABLE_ACTION):
# No gateway in front of the server, so nothing hides the tools.
logger.info(f"apply_server_config: env={self.env_id} has no tool access to change; {tools} stay as listed")
return []
try:
body = await deployed.invoke(EXT_ENABLE_TOOL_URI, TOOL_ENABLE_ACTION, {"role": WILDCARD, "tools": tools})
except Exception as e:
raise ApplyServerConfigError(
f"Enabling the tools {tools} that {directive.uri!r} on service {directive.service!r} revealed "
f"(env={self.env_id}) failed: {type(e).__name__}: {e}"
) from e
logger.info(f"apply_server_config: enabled {tools} for every role on env={self.env_id} after {directive.uri}")
record_tool_access_change(context, self.id, self.env_id, TOOL_ENABLE_ACTION, WILDCARD, tools, body)
return tools
21 changes: 13 additions & 8 deletions src/agent_env/task_step/task_steps/modify_env_tool_access.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,16 +70,21 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext:
raise StepContextError(f"Env '{self.env_id}' not found in context.deployed_envs")
body = await deployed.invoke(_TOOL_ACCESS_URIS[self.action], self.action, {"role": self.role, "tools": self.tools})
logger.info(f"{self.action} role={self.role!r} tools={self.tools} on env={self.env_id}: {body}")
context.metadata.setdefault("tool_access_changes", []).append({
"step_id": self.id,
"env_id": self.env_id,
"action": self.action,
"role": self.role,
"tools_requested": self.tools,
"role_state_after": {"disabled": body.get("disabled", []), "allowed": body.get("allowed", [])},
})
record_tool_access_change(context, self.id, self.env_id, self.action, self.role, self.tools, body)
return context


def record_tool_access_change(context: TaskStepContext, step_id: str, env_id: str, action: str, role: str,
tools: list[str], body: dict) -> None:
context.metadata.setdefault("tool_access_changes", []).append({
"step_id": step_id,
"env_id": env_id,
"action": action,
"role": role,
"tools_requested": tools,
"role_state_after": {"disabled": body.get("disabled", []), "allowed": body.get("allowed", [])},
})


# The gateway extension each action is advertised under on the env card.
_TOOL_ACCESS_URIS = {TOOL_DISABLE_ACTION: EXT_DISABLE_TOOL_URI, TOOL_ENABLE_ACTION: EXT_ENABLE_TOOL_URI}
13 changes: 9 additions & 4 deletions tst/integration/env/gateway/gateway_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -5653,13 +5653,18 @@ async def test_snowflake_clock_consumer(sandbox_provider):


async def _set_async_wait(result, tool_name, min_seconds, max_seconds=None, scale_to_virtual=True) -> dict:
"""Arm a submit tool's async wait range via the set-async-wait/v1 gateway ext proxy (harness control path)."""
base = f"{result.gateway_url}/svc/mcp-snowflake"
card = await protocol_v1.get_card(base)
"""Arm a submit tool's async wait range through apply_server_config, as a task does, so the tools the
reply reveals are enabled for every role before the test calls them."""
from agent_env.task_step.context import TaskStepContext
from agent_env.task_step.task_steps.apply_server_config import ApplyServerConfigStep

params = {"tool_name": tool_name, "min_seconds": min_seconds, "scale_to_virtual": scale_to_virtual}
if max_seconds is not None:
params["max_seconds"] = max_seconds
return await protocol_v1.invoke_extension(base, card, _ASYNC_WAIT_URI, params=params)
step = ApplyServerConfigStep(id="async-wait", version=None, env_id=result.env_id,
directives=[{"service": "snowflake", "uri": _ASYNC_WAIT_URI, "args": params}])
context = await step.execute(TaskStepContext(deployed_envs=[result]))
return context.metadata["server_config_changes"][-1]["result"]


async def _clock_clear(result) -> None:
Expand Down
20 changes: 20 additions & 0 deletions tst/unit/openenv/environment_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,26 @@ async def _call(env, tool, **arguments):
return await env.step_async(CallToolAction(tool_name=tool, arguments=arguments))


def test_tasks_that_apply_server_config_share_a_probe_with_each_other_and_not_with_tasks_that_do_not(teardowns):
from agent_env.training._openenv import toolsets

def config(env_id="items", **args):
return {"id": "cfg", "type": "apply_server_config", "env_id": env_id,
"directives": [{"service": "items", "uri": "urn:example:set-errors/v1", "args": args}]}

def described(*tasks):
served = [ServedTask(task, f"t{n}", EpisodeSpec.from_task(task), _score_at("replies"))
for n, task in enumerate(tasks)]
return {"all": [{"toolset": each.toolset, "index": n, "turns": 1} for n, each in enumerate(served)]}

plain, armed = _task(ITEMS, ALICE, TURN, REPLIES), _task(ITEMS, ALICE, config(), TURN, REPLIES)
assert len(toolsets("srv", described(plain, armed))) == 2
assert len(toolsets("srv", described(*(_task(ITEMS, ALICE, config(rate=rate), TURN, REPLIES)
for rate in (0.1, 0.2, 0.5))))) == 1
elsewhere = _task(ITEMS, ALICE, config("other"), TURN, REPLIES)
assert len(toolsets("srv", described(plain, elsewhere))) == 1


@pytest.mark.asyncio
async def test_the_task_api_lists_each_split_with_each_tasks_prompt_and_agent(teardowns, tools):
tasks = [_task(ITEMS, ALICE, {**TURN, "prompt": f"Add crate {n}."}, REPLIES, id=f"dock-{n}") for n in range(3)]
Expand Down
Loading
Loading