From 4b9fad6b5200216a027ca4efeba0e3f863f49835 Mon Sep 17 00:00:00 2001 From: Mateusz Cikowski Date: Thu, 19 Nov 2026 20:04:42 -0500 Subject: [PATCH 1/5] feat(task_step): apply_server_config enables the tools a config reply reveals (UGML-1707) Co-Authored-By: Claude Opus 5.5 --- packages/agentenv-protocol/README.md | 8 ++ src/agent_env/openenv/environment.py | 27 ++++--- .../task_steps/apply_server_config.py | 37 ++++++++- .../task_steps/modify_env_tool_access.py | 21 +++-- tst/integration/env/gateway/gateway_test.py | 13 +++- tst/unit/openenv/environment_test.py | 6 ++ .../task_step/test_apply_server_config.py | 76 +++++++++++++++++++ 7 files changed, 164 insertions(+), 24 deletions(-) diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index c8f1e7c27..257181e4f 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -90,6 +90,14 @@ step names it. Steps that name it show and hide it as they do any tool. Steps fo that role's earlier named steps, and a full reset (`role: "*", tools: "*"`) clears all of them, so grant a hidden tool after such steps. Trigger tool actions call it as usual. +A config extension can reveal hidden tools the config it applies makes usable: its reply carries +`"reveal_tools": [names]`, the tools as the server lists them, and `apply_server_config` then names +them for every role (`role: "*"`), as a tool-access step would, so the agent under test sees them too. +A missing, `null` or empty list reveals nothing; any other value that is not a list of tool names (or +names `"*"`) fails the step. A later step for every tool or a full reset hides them again. Because the +reveal names every role, it also re-enables a revealed tool for a role an earlier step denied it to: +apply per-role denies after `apply_server_config`. + `@tool(..., annotations={"readOnlyHint": True}, structured_output=True, meta={...})` lists the tool with MCP `ToolAnnotations`, an output schema and `_meta`, on mcp's FastMCP and on fastmcp alike. `structured_output` follows mcp's FastMCP (None derives an output schema from the return annotation diff --git a/src/agent_env/openenv/environment.py b/src/agent_env/openenv/environment.py index 7b0ca54ea..2d758cfbd 100644 --- a/src/agent_env/openenv/environment.py +++ b/src/agent_env/openenv/environment.py @@ -9,7 +9,7 @@ from collections.abc import Callable, Coroutine from contextlib import AsyncExitStack from dataclasses import dataclass, field -from typing import Any, TypedDict +from typing import Any, NotRequired, TypedDict import httpx from openenv.core.env_server.interfaces import Environment @@ -52,6 +52,7 @@ class Toolset(TypedDict): role: str | None access: list[dict[str, Any]] bash: bool + config: NotRequired[list[dict[str, Any]]] @dataclass(frozen=True) @@ -66,18 +67,24 @@ class ServedTask: @property def toolset(self) -> Toolset: """Where the played agent's tools come from: the envs it works in and the versions the task deploys, its role - in them, the changes the task makes to that role's tool access, and whether its turns offer ``bash`` in its - sandbox. Tasks that agree on it offer the same tools, so a trainer can group them.""" + in them, the changes the task makes to that role's tool access, whether its turns offer ``bash`` in its + sandbox, and the server config it applies (``config``, only when it applies any), whose replies can reveal + tools. Tasks that agree on it offer the same tools, so a trainer can group them.""" agent = self.spec.trainer_agents[0] [deploy] = [step for step in self.task.steps if step.type in DEPLOYS and step.agent_name == agent] role, envs = getattr(deploy, "role", None), list(getattr(deploy, "env_ids", None) or []) - return Toolset(envs=envs, role=role, - versions={step.env_id: step.env_version for step in self.task.steps - if hasattr(step, "env_version") and step.env_id in envs}, - access=[{"env": step.env_id, "action": step.action, "tools": step.tools} - for step in self.task.steps - if step.type == "modify_env_tool_access" and step.role == (role or DEFAULT_ROLE)], - bash=works_in_sandbox(self.task.steps, agent)) + toolset = Toolset(envs=envs, role=role, + versions={step.env_id: step.env_version for step in self.task.steps + if hasattr(step, "env_version") and step.env_id in envs}, + access=[{"env": step.env_id, "action": step.action, "tools": step.tools} + for step in self.task.steps + if step.type == "modify_env_tool_access" and step.role == (role or DEFAULT_ROLE)], + bash=works_in_sandbox(self.task.steps, agent)) + config = [{"env": step.env_id, "directives": [d.to_dict() for d in step.directives]} + for step in self.task.steps if step.type == "apply_server_config" and step.env_id in envs] + if config: + toolset["config"] = config + return toolset @dataclass diff --git a/src/agent_env/task_step/task_steps/apply_server_config.py b/src/agent_env/task_step/task_steps/apply_server_config.py index efe0444c9..c489acd24 100644 --- a/src/agent_env/task_step/task_steps/apply_server_config.py +++ b/src/agent_env/task_step/task_steps/apply_server_config.py @@ -23,15 +23,23 @@ import httpx from agentenv_protocol import client as protocol_v1 +from agent_env.env.gateway import TOOL_ENABLE_ACTION, WILDCARD +from agent_env.env.gateway.constants import EXT_ENABLE_TOOL_URI from agent_env.task_step.context import TaskStepContext from agent_env.entity.refs import EntityRef from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_steps.modify_env_tool_access import record_tool_access_change if TYPE_CHECKING: + from agent_env.env.env import DeployedEnv from agent_env.env.legacy_protocol import BackingServer logger = logging.getLogger(__name__) +# A config extension's reply may name tools the server lists hidden by default that this config makes usable; the +# step then enables them on the env's gateway for every role, as modify_env_tool_access would. +REVEAL_TOOLS_KEY = "reveal_tools" + class ApplyServerConfigError(RuntimeError): """Raised when a config directive cannot be applied (bad target / extension @@ -279,7 +287,7 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: f"apply_server_config: applied {directive.uri} on service={directive.service} " f"env={self.env_id} args={directive.args}: {result}" ) - changes.append({ + change = { "step_id": self.id, "env_id": self.env_id, "service": directive.service, @@ -287,6 +295,31 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: "uri": directive.uri, "args": directive.args, "result": result, - }) + } + changes.append(change) + revealed = await self._reveal_tools(deployed, directive, result, context) + if revealed: + change["revealed_tools"] = revealed return context + + async def _reveal_tools(self, deployed: DeployedEnv, directive: ConfigDirective, result: Any, + context: TaskStepContext) -> list[str]: + tools = result.get(REVEAL_TOOLS_KEY) if isinstance(result, dict) else None + if tools is None or tools == []: + return [] + if not isinstance(tools, list) or not all(isinstance(t, str) and t and t != WILDCARD for t in tools): + raise ApplyServerConfigError( + f"Applying {directive.uri!r} on service {directive.service!r} (env={self.env_id}): its reply's " + f"{REVEAL_TOOLS_KEY!r} must be a list of tool names, got {tools!r}" + ) + try: + body = await deployed.invoke(EXT_ENABLE_TOOL_URI, TOOL_ENABLE_ACTION, {"role": WILDCARD, "tools": tools}) + except Exception as e: + raise ApplyServerConfigError( + f"Enabling the tools {tools} that {directive.uri!r} on service {directive.service!r} revealed " + f"(env={self.env_id}) failed: {type(e).__name__}: {e}" + ) from e + logger.info(f"apply_server_config: enabled {tools} for every role on env={self.env_id} after {directive.uri}") + record_tool_access_change(context, self.id, self.env_id, TOOL_ENABLE_ACTION, WILDCARD, tools, body) + return tools diff --git a/src/agent_env/task_step/task_steps/modify_env_tool_access.py b/src/agent_env/task_step/task_steps/modify_env_tool_access.py index f45e5db09..baba2445a 100644 --- a/src/agent_env/task_step/task_steps/modify_env_tool_access.py +++ b/src/agent_env/task_step/task_steps/modify_env_tool_access.py @@ -70,16 +70,21 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: raise StepContextError(f"Env '{self.env_id}' not found in context.deployed_envs") body = await deployed.invoke(_TOOL_ACCESS_URIS[self.action], self.action, {"role": self.role, "tools": self.tools}) logger.info(f"{self.action} role={self.role!r} tools={self.tools} on env={self.env_id}: {body}") - context.metadata.setdefault("tool_access_changes", []).append({ - "step_id": self.id, - "env_id": self.env_id, - "action": self.action, - "role": self.role, - "tools_requested": self.tools, - "role_state_after": {"disabled": body.get("disabled", []), "allowed": body.get("allowed", [])}, - }) + record_tool_access_change(context, self.id, self.env_id, self.action, self.role, self.tools, body) return context +def record_tool_access_change(context: TaskStepContext, step_id: str, env_id: str, action: str, role: str, + tools: list[str], body: dict) -> None: + context.metadata.setdefault("tool_access_changes", []).append({ + "step_id": step_id, + "env_id": env_id, + "action": action, + "role": role, + "tools_requested": tools, + "role_state_after": {"disabled": body.get("disabled", []), "allowed": body.get("allowed", [])}, + }) + + # The gateway extension each action is advertised under on the env card. _TOOL_ACCESS_URIS = {TOOL_DISABLE_ACTION: EXT_DISABLE_TOOL_URI, TOOL_ENABLE_ACTION: EXT_ENABLE_TOOL_URI} diff --git a/tst/integration/env/gateway/gateway_test.py b/tst/integration/env/gateway/gateway_test.py index aae711431..750032e8e 100644 --- a/tst/integration/env/gateway/gateway_test.py +++ b/tst/integration/env/gateway/gateway_test.py @@ -5653,13 +5653,18 @@ async def test_snowflake_clock_consumer(sandbox_provider): async def _set_async_wait(result, tool_name, min_seconds, max_seconds=None, scale_to_virtual=True) -> dict: - """Arm a submit tool's async wait range via the set-async-wait/v1 gateway ext proxy (harness control path).""" - base = f"{result.gateway_url}/svc/mcp-snowflake" - card = await protocol_v1.get_card(base) + """Arm a submit tool's async wait range through apply_server_config, as a task does, so the tools the + reply reveals are enabled for every role before the test calls them.""" + from agent_env.task_step.context import TaskStepContext + from agent_env.task_step.task_steps.apply_server_config import ApplyServerConfigStep + params = {"tool_name": tool_name, "min_seconds": min_seconds, "scale_to_virtual": scale_to_virtual} if max_seconds is not None: params["max_seconds"] = max_seconds - return await protocol_v1.invoke_extension(base, card, _ASYNC_WAIT_URI, params=params) + step = ApplyServerConfigStep(id="async-wait", version=None, env_id=result.env_id, + directives=[{"service": "snowflake", "uri": _ASYNC_WAIT_URI, "args": params}]) + context = await step.execute(TaskStepContext(deployed_envs=[result])) + return context.metadata["server_config_changes"][-1]["result"] async def _clock_clear(result) -> None: diff --git a/tst/unit/openenv/environment_test.py b/tst/unit/openenv/environment_test.py index 7f9ac0f3a..fcf1303fa 100644 --- a/tst/unit/openenv/environment_test.py +++ b/tst/unit/openenv/environment_test.py @@ -103,6 +103,12 @@ async def test_the_task_api_lists_each_split_with_each_tasks_prompt_and_agent(te assert ServedTask(defaulted, "defaulted", EpisodeSpec.from_task(defaulted), _score_at("replies")).toolset == { "envs": ["items"], "versions": {}, "role": None, "access": [{"env": "items", "action": "disable", "tools": ["add_item"]}], "bash": False} + configure = {"id": "cfg", "type": "apply_server_config", "env_id": "items", + "directives": [{"service": "items", "uri": "urn:example:set-mode/v1", "args": {"mode": "slow"}}]} + configured = _task(ITEMS, ALICE, configure, TURN, REPLIES) + assert ServedTask(configured, "configured", EpisodeSpec.from_task(configured), _score_at("replies")).toolset[ + "config"] == [{"env": "items", "directives": [{"service": "items", "environment": "items", + "uri": "urn:example:set-mode/v1", "args": {"mode": "slow"}}]}] assert [task["index"] for task in env.get_task_range("all", 1)] == [1, 2] assert [task["id"] for task in env.list_tasks("smoke")] == ["dock-0"] with pytest.raises(IndexError, match="split 'all' has 3 tasks, so it has no task 3"): diff --git a/tst/unit/task_step/test_apply_server_config.py b/tst/unit/task_step/test_apply_server_config.py index 9df2a67ec..e47bf46e9 100644 --- a/tst/unit/task_step/test_apply_server_config.py +++ b/tst/unit/task_step/test_apply_server_config.py @@ -874,3 +874,79 @@ async def __aexit__(self, *a): await step.execute(TaskStepContext(deployed_envs=[record])) assert sent == [("https://desk.example/agentenv/ext/set_errors", computer.control_headers)] + + +class _RevealingGateway: + """Replies to the config extension with `reply` and to the gateway's enable-tool endpoint with the role's state.""" + + def __init__(self, reply: dict, enable_status: int = 200): + self.reply = reply + self.enable_status = enable_status + self.post_calls: list[tuple[str, dict]] = [] + + async def request(self, method, url, *, json=None, timeout=None, headers=None): + self.post_calls.append((url, json)) + if url.endswith("/tools/enable"): + return _resp(self.enable_status, url, {"role": "*", "disabled": [], "allowed": json["tools"]}) + return _resp(200, url, self.reply) + + async def __aenter__(self): + return self + + async def __aexit__(self, *a): + return False + + +async def _revealing_context() -> TaskStepContext: + return _carded_context(await _composed_card({"mcp-slack": [_ext(_SET_ERRORS_URI, "/agentenv/ext/set_errors")]})) + + +@pytest.mark.asyncio +async def test_tools_a_reply_reveals_are_enabled_for_every_role_on_the_gateway(): + gw = _RevealingGateway({"ok": True, "reveal_tools": ["slack_export", "slack_export_status"]}) + with patch("httpx.AsyncClient", return_value=gw): + out = await _step().execute(await _revealing_context()) + + assert gw.post_calls[1] == (f"{_CARD}/tools/enable", {"role": "*", "tools": ["slack_export", "slack_export_status"]}) + [change] = out.metadata["server_config_changes"] + assert change["revealed_tools"] == ["slack_export", "slack_export_status"] + [access] = out.metadata["tool_access_changes"] + assert (access["action"], access["role"], access["tools_requested"]) == ( + "enable", "*", ["slack_export", "slack_export_status"]) + assert access["role_state_after"] == {"disabled": [], "allowed": ["slack_export", "slack_export_status"]} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("reply", [{"ok": True}, {"ok": True, "reveal_tools": []}, {"ok": True, "reveal_tools": None}, + ["ok"], True], + ids=["absent", "empty", "null", "list-reply", "scalar-reply"]) +async def test_a_reply_that_reveals_nothing_enables_nothing(reply): + gw = _RevealingGateway(reply) + with patch("httpx.AsyncClient", return_value=gw): + out = await _step().execute(await _revealing_context()) + + assert [url for url, _ in gw.post_calls] == [f"{_CARD}/svc/mcp-slack/agentenv/ext/set_errors"] + assert "revealed_tools" not in out.metadata["server_config_changes"][0] + assert "tool_access_changes" not in out.metadata + + +@pytest.mark.asyncio +@pytest.mark.parametrize("tools", ["slack_export", ["slack_export", 3], [""], ["*"], ["slack_export", "*"]], + ids=["string", "non-string", "empty-name", "wildcard", "wildcard-among-names"]) +async def test_a_malformed_reveal_list_fails_the_step_after_recording_the_applied_directive(tools): + gw = _RevealingGateway({"ok": True, "reveal_tools": tools}) + ctx = await _revealing_context() + with patch("httpx.AsyncClient", return_value=gw): + with pytest.raises(ApplyServerConfigError, match="'reveal_tools' must be a list of tool names"): + await _step().execute(ctx) + + assert len(gw.post_calls) == 1 + assert ctx.metadata["server_config_changes"][0]["uri"] == _SET_ERRORS_URI + + +@pytest.mark.asyncio +async def test_a_failing_enable_fails_the_step(): + gw = _RevealingGateway({"ok": True, "reveal_tools": ["slack_export"]}, enable_status=500) + with patch("httpx.AsyncClient", return_value=gw): + with pytest.raises(ApplyServerConfigError, match="Enabling the tools \\['slack_export'\\]"): + await _step().execute(await _revealing_context()) From dedadfda950e17960e6819bb00a137029ea5c197 Mon Sep 17 00:00:00 2001 From: Mateusz Cikowski Date: Thu, 19 Nov 2026 20:21:11 -0500 Subject: [PATCH 2/5] address review round 2: toolset key orders access and config extensions without args; README says only a by-name deny hides a revealed tool; real-gateway reveal test Co-Authored-By: Claude Opus 5.5 --- packages/agentenv-protocol/README.md | 8 ++- src/agent_env/openenv/environment.py | 17 ++++-- tst/unit/openenv/environment_test.py | 17 ++++-- .../task_step/test_apply_server_config.py | 56 +++++++++++++++++++ 4 files changed, 85 insertions(+), 13 deletions(-) diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index 257181e4f..c2e56b893 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -94,9 +94,11 @@ A config extension can reveal hidden tools the config it applies makes usable: i `"reveal_tools": [names]`, the tools as the server lists them, and `apply_server_config` then names them for every role (`role: "*"`), as a tool-access step would, so the agent under test sees them too. A missing, `null` or empty list reveals nothing; any other value that is not a list of tool names (or -names `"*"`) fails the step. A later step for every tool or a full reset hides them again. Because the -reveal names every role, it also re-enables a revealed tool for a role an earlier step denied it to: -apply per-role denies after `apply_server_config`. +names `"*"`) fails the step. A later full reset (every tool, every role) hides them again. Because the +reveal names every role, it also re-enables a revealed tool for a role an earlier step denied it to. Only +a step that names the tool hides it from one role: a step for every tool skips tools listed hidden by +default, in either order. To keep a revealed tool from a role, deny that tool by name after +`apply_server_config`. `@tool(..., annotations={"readOnlyHint": True}, structured_output=True, meta={...})` lists the tool with MCP `ToolAnnotations`, an output schema and `_meta`, on mcp's FastMCP and on fastmcp alike. diff --git a/src/agent_env/openenv/environment.py b/src/agent_env/openenv/environment.py index 2d758cfbd..2cce28778 100644 --- a/src/agent_env/openenv/environment.py +++ b/src/agent_env/openenv/environment.py @@ -68,8 +68,9 @@ class ServedTask: def toolset(self) -> Toolset: """Where the played agent's tools come from: the envs it works in and the versions the task deploys, its role in them, the changes the task makes to that role's tool access, whether its turns offer ``bash`` in its - sandbox, and the server config it applies (``config``, only when it applies any), whose replies can reveal - tools. Tasks that agree on it offer the same tools, so a trainer can group them.""" + sandbox, and, when it applies server config (whose replies can reveal tools), its config extensions and those + tool-access changes in step order (``config``). Tasks that agree on it offer the same tools, so a trainer can + group them.""" agent = self.spec.trainer_agents[0] [deploy] = [step for step in self.task.steps if step.type in DEPLOYS and step.agent_name == agent] role, envs = getattr(deploy, "role", None), list(getattr(deploy, "env_ids", None) or []) @@ -80,10 +81,14 @@ def toolset(self) -> Toolset: for step in self.task.steps if step.type == "modify_env_tool_access" and step.role == (role or DEFAULT_ROLE)], bash=works_in_sandbox(self.task.steps, agent)) - config = [{"env": step.env_id, "directives": [d.to_dict() for d in step.directives]} - for step in self.task.steps if step.type == "apply_server_config" and step.env_id in envs] - if config: - toolset["config"] = config + if any(step.type == "apply_server_config" and step.env_id in envs for step in self.task.steps): + toolset["config"] = [ + {"env": step.env_id, "action": step.action, "tools": step.tools} + if step.type == "modify_env_tool_access" + else {"env": step.env_id, "extensions": [directive.uri for directive in step.directives]} + for step in self.task.steps + if (step.type == "modify_env_tool_access" and step.role == (role or DEFAULT_ROLE)) + or (step.type == "apply_server_config" and step.env_id in envs)] return toolset diff --git a/tst/unit/openenv/environment_test.py b/tst/unit/openenv/environment_test.py index fcf1303fa..db6d6b154 100644 --- a/tst/unit/openenv/environment_test.py +++ b/tst/unit/openenv/environment_test.py @@ -105,10 +105,19 @@ async def test_the_task_api_lists_each_split_with_each_tasks_prompt_and_agent(te "tools": ["add_item"]}], "bash": False} configure = {"id": "cfg", "type": "apply_server_config", "env_id": "items", "directives": [{"service": "items", "uri": "urn:example:set-mode/v1", "args": {"mode": "slow"}}]} - configured = _task(ITEMS, ALICE, configure, TURN, REPLIES) - assert ServedTask(configured, "configured", EpisodeSpec.from_task(configured), _score_at("replies")).toolset[ - "config"] == [{"env": "items", "directives": [{"service": "items", "environment": "items", - "uri": "urn:example:set-mode/v1", "args": {"mode": "slow"}}]}] + def toolset(*steps): + task = _task(ITEMS, ALICE, *steps, TURN, REPLIES) + return ServedTask(task, "t", EpisodeSpec.from_task(task), _score_at("replies")).toolset + + faster = {**configure, "directives": [{**configure["directives"][0], "args": {"mode": "fast"}}]} + assert toolset(configure)["config"] == [{"env": "items", "extensions": ["urn:example:set-mode/v1"]}] + assert toolset(configure) == toolset(faster) + assert toolset(hide, {**configure, "depends_on": ["hide"]})["config"] == [ + {"env": "items", "action": "disable", "tools": ["add_item"]}, + {"env": "items", "extensions": ["urn:example:set-mode/v1"]}] + assert toolset(hide, {**configure, "depends_on": ["hide"]}) != toolset(configure, {**hide, "depends_on": ["cfg"]}) + assert "config" not in toolset(hide) + assert "config" not in toolset({**configure, "env_id": "elsewhere"}) assert [task["index"] for task in env.get_task_range("all", 1)] == [1, 2] assert [task["id"] for task in env.list_tasks("smoke")] == ["dock-0"] with pytest.raises(IndexError, match="split 'all' has 3 tasks, so it has no task 3"): diff --git a/tst/unit/task_step/test_apply_server_config.py b/tst/unit/task_step/test_apply_server_config.py index e47bf46e9..5491d882e 100644 --- a/tst/unit/task_step/test_apply_server_config.py +++ b/tst/unit/task_step/test_apply_server_config.py @@ -950,3 +950,59 @@ async def test_a_failing_enable_fails_the_step(): with patch("httpx.AsyncClient", return_value=gw): with pytest.raises(ApplyServerConfigError, match="Enabling the tools \\['slack_export'\\]"): await _step().execute(await _revealing_context()) + + +_RealAsyncClient = httpx.AsyncClient + + +class _GatewayForwarder(_RevealingGateway): + """Replies to the config extension with `reply` and forwards the enable to a real in-process gateway.""" + + def __init__(self, reply: dict, gateway): + super().__init__(reply) + self.gateway = gateway + + async def request(self, method, url, *, json=None, timeout=None, headers=None): + if not url.endswith("/tools/enable"): + return await super().request(method, url, json=json) + transport = httpx.ASGITransport(app=self.gateway._mcp.streamable_http_app()) + async with _RealAsyncClient(transport=transport, base_url="http://gateway") as client: + return await client.request(method, "/tools/enable", json=json, headers=headers) + + +@pytest.mark.asyncio +async def test_a_tool_a_reply_reveals_is_listed_to_every_role_by_the_gateway(): + from mcp.types import Tool as MCPTool + + from agent_env.env.gateway import AGENT_ENV_HIDDEN_BY_DEFAULT_META_KEY, InternalMCPServer + from agent_env.env.gateway.caller import Caller + from agent_env.env.gateway.gateway import Gateway + + tools = [MCPTool(name=name, description="", inputSchema={"type": "object", "properties": {}}, _meta=meta) + for name, meta in (("slack_send", None), ("slack_export", {AGENT_ENV_HIDDEN_BY_DEFAULT_META_KEY: True}))] + gateway = Gateway(host="127.0.0.1", port=0, server_name="unit", + internal_mcp_servers=[InternalMCPServer("slack", "http://slack/mcp")]) + + class _Session: + async def list_tools(self): + return SimpleNamespace(tools=tools) + + async def open_session(server, stack, report): + return _Session() + + gateway._sessions.open_session = open_session + await gateway._ensure_tools_discovered() + + def listed(role: str) -> set[str]: + token = Caller(role=role).bind() + try: + return {tool.name for tool in gateway._mcp._tool_manager.list_tools()} + finally: + Caller.reset(token) + + assert "slack_export" not in listed("grader") + forwarder = _GatewayForwarder({"ok": True, "reveal_tools": ["slack_export"]}, gateway) + with patch("httpx.AsyncClient", return_value=forwarder): + await _step().execute(await _revealing_context()) + assert {"slack_send", "slack_export"} <= listed("grader") & listed("default") + await gateway._close_child_sessions() From 9381680690f10e26dbe27cbd0ab1bf32820b67e2 Mon Sep 17 00:00:00 2001 From: Mateusz Cikowski Date: Thu, 19 Nov 2026 20:50:56 -0500 Subject: [PATCH 3/5] address review round 3: drop the OpenEnv toolset-key change to a follow-up; falsy non-list reveal replies fail; a reveal on an env with no tool access to change enables nothing Co-Authored-By: Claude Opus 5.5 --- src/agent_env/openenv/environment.py | 32 ++++++------------- .../task_steps/apply_server_config.py | 4 +++ tst/unit/openenv/environment_test.py | 15 --------- .../task_step/test_apply_server_config.py | 29 +++++++++++++++-- 4 files changed, 41 insertions(+), 39 deletions(-) diff --git a/src/agent_env/openenv/environment.py b/src/agent_env/openenv/environment.py index 2cce28778..7b0ca54ea 100644 --- a/src/agent_env/openenv/environment.py +++ b/src/agent_env/openenv/environment.py @@ -9,7 +9,7 @@ from collections.abc import Callable, Coroutine from contextlib import AsyncExitStack from dataclasses import dataclass, field -from typing import Any, NotRequired, TypedDict +from typing import Any, TypedDict import httpx from openenv.core.env_server.interfaces import Environment @@ -52,7 +52,6 @@ class Toolset(TypedDict): role: str | None access: list[dict[str, Any]] bash: bool - config: NotRequired[list[dict[str, Any]]] @dataclass(frozen=True) @@ -67,29 +66,18 @@ class ServedTask: @property def toolset(self) -> Toolset: """Where the played agent's tools come from: the envs it works in and the versions the task deploys, its role - in them, the changes the task makes to that role's tool access, whether its turns offer ``bash`` in its - sandbox, and, when it applies server config (whose replies can reveal tools), its config extensions and those - tool-access changes in step order (``config``). Tasks that agree on it offer the same tools, so a trainer can - group them.""" + in them, the changes the task makes to that role's tool access, and whether its turns offer ``bash`` in its + sandbox. Tasks that agree on it offer the same tools, so a trainer can group them.""" agent = self.spec.trainer_agents[0] [deploy] = [step for step in self.task.steps if step.type in DEPLOYS and step.agent_name == agent] role, envs = getattr(deploy, "role", None), list(getattr(deploy, "env_ids", None) or []) - toolset = Toolset(envs=envs, role=role, - versions={step.env_id: step.env_version for step in self.task.steps - if hasattr(step, "env_version") and step.env_id in envs}, - access=[{"env": step.env_id, "action": step.action, "tools": step.tools} - for step in self.task.steps - if step.type == "modify_env_tool_access" and step.role == (role or DEFAULT_ROLE)], - bash=works_in_sandbox(self.task.steps, agent)) - if any(step.type == "apply_server_config" and step.env_id in envs for step in self.task.steps): - toolset["config"] = [ - {"env": step.env_id, "action": step.action, "tools": step.tools} - if step.type == "modify_env_tool_access" - else {"env": step.env_id, "extensions": [directive.uri for directive in step.directives]} - for step in self.task.steps - if (step.type == "modify_env_tool_access" and step.role == (role or DEFAULT_ROLE)) - or (step.type == "apply_server_config" and step.env_id in envs)] - return toolset + return Toolset(envs=envs, role=role, + versions={step.env_id: step.env_version for step in self.task.steps + if hasattr(step, "env_version") and step.env_id in envs}, + access=[{"env": step.env_id, "action": step.action, "tools": step.tools} + for step in self.task.steps + if step.type == "modify_env_tool_access" and step.role == (role or DEFAULT_ROLE)], + bash=works_in_sandbox(self.task.steps, agent)) @dataclass diff --git a/src/agent_env/task_step/task_steps/apply_server_config.py b/src/agent_env/task_step/task_steps/apply_server_config.py index c489acd24..4145a7527 100644 --- a/src/agent_env/task_step/task_steps/apply_server_config.py +++ b/src/agent_env/task_step/task_steps/apply_server_config.py @@ -313,6 +313,10 @@ async def _reveal_tools(self, deployed: DeployedEnv, directive: ConfigDirective, f"Applying {directive.uri!r} on service {directive.service!r} (env={self.env_id}): its reply's " f"{REVEAL_TOOLS_KEY!r} must be a list of tool names, got {tools!r}" ) + if deployed.environment_card is not None and not deployed.supports(EXT_ENABLE_TOOL_URI, TOOL_ENABLE_ACTION): + # No gateway in front of the server, so nothing hides the tools. + logger.info(f"apply_server_config: env={self.env_id} has no tool access to change; {tools} stay as listed") + return [] try: body = await deployed.invoke(EXT_ENABLE_TOOL_URI, TOOL_ENABLE_ACTION, {"role": WILDCARD, "tools": tools}) except Exception as e: diff --git a/tst/unit/openenv/environment_test.py b/tst/unit/openenv/environment_test.py index db6d6b154..7f9ac0f3a 100644 --- a/tst/unit/openenv/environment_test.py +++ b/tst/unit/openenv/environment_test.py @@ -103,21 +103,6 @@ async def test_the_task_api_lists_each_split_with_each_tasks_prompt_and_agent(te assert ServedTask(defaulted, "defaulted", EpisodeSpec.from_task(defaulted), _score_at("replies")).toolset == { "envs": ["items"], "versions": {}, "role": None, "access": [{"env": "items", "action": "disable", "tools": ["add_item"]}], "bash": False} - configure = {"id": "cfg", "type": "apply_server_config", "env_id": "items", - "directives": [{"service": "items", "uri": "urn:example:set-mode/v1", "args": {"mode": "slow"}}]} - def toolset(*steps): - task = _task(ITEMS, ALICE, *steps, TURN, REPLIES) - return ServedTask(task, "t", EpisodeSpec.from_task(task), _score_at("replies")).toolset - - faster = {**configure, "directives": [{**configure["directives"][0], "args": {"mode": "fast"}}]} - assert toolset(configure)["config"] == [{"env": "items", "extensions": ["urn:example:set-mode/v1"]}] - assert toolset(configure) == toolset(faster) - assert toolset(hide, {**configure, "depends_on": ["hide"]})["config"] == [ - {"env": "items", "action": "disable", "tools": ["add_item"]}, - {"env": "items", "extensions": ["urn:example:set-mode/v1"]}] - assert toolset(hide, {**configure, "depends_on": ["hide"]}) != toolset(configure, {**hide, "depends_on": ["cfg"]}) - assert "config" not in toolset(hide) - assert "config" not in toolset({**configure, "env_id": "elsewhere"}) assert [task["index"] for task in env.get_task_range("all", 1)] == [1, 2] assert [task["id"] for task in env.list_tasks("smoke")] == ["dock-0"] with pytest.raises(IndexError, match="split 'all' has 3 tasks, so it has no task 3"): diff --git a/tst/unit/task_step/test_apply_server_config.py b/tst/unit/task_step/test_apply_server_config.py index 5491d882e..dbe88af83 100644 --- a/tst/unit/task_step/test_apply_server_config.py +++ b/tst/unit/task_step/test_apply_server_config.py @@ -931,8 +931,10 @@ async def test_a_reply_that_reveals_nothing_enables_nothing(reply): @pytest.mark.asyncio -@pytest.mark.parametrize("tools", ["slack_export", ["slack_export", 3], [""], ["*"], ["slack_export", "*"]], - ids=["string", "non-string", "empty-name", "wildcard", "wildcard-among-names"]) +@pytest.mark.parametrize("tools", ["slack_export", ["slack_export", 3], [""], ["*"], ["slack_export", "*"], + False, "", 0, {}], + ids=["string", "non-string", "empty-name", "wildcard", "wildcard-among-names", + "false", "empty-string", "zero", "empty-object"]) async def test_a_malformed_reveal_list_fails_the_step_after_recording_the_applied_directive(tools): gw = _RevealingGateway({"ok": True, "reveal_tools": tools}) ctx = await _revealing_context() @@ -955,6 +957,29 @@ async def test_a_failing_enable_fails_the_step(): _RealAsyncClient = httpx.AsyncClient +@pytest.mark.asyncio +async def test_a_reveal_on_an_env_with_no_tool_access_to_change_enables_nothing(): + from agent_env.env.gateway.constants import EXT_ENABLE_TOOL_URI + + def without_enable(node): + if isinstance(node, list): + return [without_enable(item) for item in node + if not (isinstance(item, dict) and item.get("uri") == EXT_ENABLE_TOOL_URI)] + if isinstance(node, dict): + return {key: without_enable(value) for key, value in node.items()} + return node + + ctx = await _revealing_context() + ctx.deployed_envs[0].environment_card = without_enable(ctx.deployed_envs[0].environment_card) + gw = _RevealingGateway({"ok": True, "reveal_tools": ["slack_export"]}) + with patch("httpx.AsyncClient", return_value=gw): + out = await _step().execute(ctx) + + assert [url for url, _ in gw.post_calls] == [f"{_CARD}/svc/mcp-slack/agentenv/ext/set_errors"] + assert "revealed_tools" not in out.metadata["server_config_changes"][0] + assert "tool_access_changes" not in out.metadata + + class _GatewayForwarder(_RevealingGateway): """Replies to the config extension with `reply` and forwards the enable to a real in-process gateway.""" From 3ba02d9679654e62456a451b028a021e230d015b Mon Sep 17 00:00:00 2001 From: Mateusz Cikowski Date: Thu, 19 Nov 2026 21:06:42 -0500 Subject: [PATCH 4/5] address review round 4: toolset key marks tasks that apply server config; a reveal on a record with no or an empty stored card fails Co-Authored-By: Claude Opus 5.5 --- src/agent_env/openenv/environment.py | 26 ++++++++++++------- .../task_steps/apply_server_config.py | 2 +- tst/unit/openenv/environment_test.py | 20 ++++++++++++++ .../task_step/test_apply_server_config.py | 12 +++++++++ 4 files changed, 49 insertions(+), 11 deletions(-) diff --git a/src/agent_env/openenv/environment.py b/src/agent_env/openenv/environment.py index 7b0ca54ea..b374a7535 100644 --- a/src/agent_env/openenv/environment.py +++ b/src/agent_env/openenv/environment.py @@ -9,7 +9,7 @@ from collections.abc import Callable, Coroutine from contextlib import AsyncExitStack from dataclasses import dataclass, field -from typing import Any, TypedDict +from typing import Any, NotRequired, TypedDict import httpx from openenv.core.env_server.interfaces import Environment @@ -52,6 +52,7 @@ class Toolset(TypedDict): role: str | None access: list[dict[str, Any]] bash: bool + config: NotRequired[bool] @dataclass(frozen=True) @@ -66,18 +67,23 @@ class ServedTask: @property def toolset(self) -> Toolset: """Where the played agent's tools come from: the envs it works in and the versions the task deploys, its role - in them, the changes the task makes to that role's tool access, and whether its turns offer ``bash`` in its - sandbox. Tasks that agree on it offer the same tools, so a trainer can group them.""" + in them, the changes the task makes to that role's tool access, whether its turns offer ``bash`` in its + sandbox, and whether it applies server config there (``config``, only when it does), whose replies can reveal + tools. Tasks that agree on it offer the same tools unless their configs reveal different ones, so a trainer + can group them.""" agent = self.spec.trainer_agents[0] [deploy] = [step for step in self.task.steps if step.type in DEPLOYS and step.agent_name == agent] role, envs = getattr(deploy, "role", None), list(getattr(deploy, "env_ids", None) or []) - return Toolset(envs=envs, role=role, - versions={step.env_id: step.env_version for step in self.task.steps - if hasattr(step, "env_version") and step.env_id in envs}, - access=[{"env": step.env_id, "action": step.action, "tools": step.tools} - for step in self.task.steps - if step.type == "modify_env_tool_access" and step.role == (role or DEFAULT_ROLE)], - bash=works_in_sandbox(self.task.steps, agent)) + toolset = Toolset(envs=envs, role=role, + versions={step.env_id: step.env_version for step in self.task.steps + if hasattr(step, "env_version") and step.env_id in envs}, + access=[{"env": step.env_id, "action": step.action, "tools": step.tools} + for step in self.task.steps + if step.type == "modify_env_tool_access" and step.role == (role or DEFAULT_ROLE)], + bash=works_in_sandbox(self.task.steps, agent)) + if any(step.type == "apply_server_config" and step.env_id in envs for step in self.task.steps): + toolset["config"] = True + return toolset @dataclass diff --git a/src/agent_env/task_step/task_steps/apply_server_config.py b/src/agent_env/task_step/task_steps/apply_server_config.py index 4145a7527..2695d1975 100644 --- a/src/agent_env/task_step/task_steps/apply_server_config.py +++ b/src/agent_env/task_step/task_steps/apply_server_config.py @@ -313,7 +313,7 @@ async def _reveal_tools(self, deployed: DeployedEnv, directive: ConfigDirective, f"Applying {directive.uri!r} on service {directive.service!r} (env={self.env_id}): its reply's " f"{REVEAL_TOOLS_KEY!r} must be a list of tool names, got {tools!r}" ) - if deployed.environment_card is not None and not deployed.supports(EXT_ENABLE_TOOL_URI, TOOL_ENABLE_ACTION): + if deployed.environment_card and not deployed.supports(EXT_ENABLE_TOOL_URI, TOOL_ENABLE_ACTION): # No gateway in front of the server, so nothing hides the tools. logger.info(f"apply_server_config: env={self.env_id} has no tool access to change; {tools} stay as listed") return [] diff --git a/tst/unit/openenv/environment_test.py b/tst/unit/openenv/environment_test.py index 7f9ac0f3a..a146565e3 100644 --- a/tst/unit/openenv/environment_test.py +++ b/tst/unit/openenv/environment_test.py @@ -79,6 +79,26 @@ async def _call(env, tool, **arguments): return await env.step_async(CallToolAction(tool_name=tool, arguments=arguments)) +def test_tasks_that_apply_server_config_share_a_probe_with_each_other_and_not_with_tasks_that_do_not(teardowns): + from agent_env.training._openenv import toolsets + + def config(env_id="items", **args): + return {"id": "cfg", "type": "apply_server_config", "env_id": env_id, + "directives": [{"service": "items", "uri": "urn:example:set-errors/v1", "args": args}]} + + def described(*tasks): + served = [ServedTask(task, f"t{n}", EpisodeSpec.from_task(task), _score_at("replies")) + for n, task in enumerate(tasks)] + return {"all": [{"toolset": each.toolset, "index": n, "turns": 1} for n, each in enumerate(served)]} + + plain, armed = _task(ITEMS, ALICE, TURN, REPLIES), _task(ITEMS, ALICE, config(), TURN, REPLIES) + assert len(toolsets("srv", described(plain, armed))) == 2 + assert len(toolsets("srv", described(*(_task(ITEMS, ALICE, config(rate=rate), TURN, REPLIES) + for rate in (0.1, 0.2, 0.5))))) == 1 + elsewhere = _task(ITEMS, ALICE, config("other"), TURN, REPLIES) + assert len(toolsets("srv", described(plain, elsewhere))) == 1 + + @pytest.mark.asyncio async def test_the_task_api_lists_each_split_with_each_tasks_prompt_and_agent(teardowns, tools): tasks = [_task(ITEMS, ALICE, {**TURN, "prompt": f"Add crate {n}."}, REPLIES, id=f"dock-{n}") for n in range(3)] diff --git a/tst/unit/task_step/test_apply_server_config.py b/tst/unit/task_step/test_apply_server_config.py index dbe88af83..2d556d001 100644 --- a/tst/unit/task_step/test_apply_server_config.py +++ b/tst/unit/task_step/test_apply_server_config.py @@ -980,6 +980,18 @@ def without_enable(node): assert "tool_access_changes" not in out.metadata +@pytest.mark.asyncio +async def test_a_reveal_on_a_record_with_no_stored_card_fails_the_step(): + class _Revealing(_MockGateway): + async def request(self, method, url, *, json=None, timeout=None, headers=None): + return _resp(200, url, {"ok": True, "reveal_tools": ["slack_export"]}) + + gw = _Revealing({"slack": _card(_ext(_SET_ERRORS_URI, "/agentenv/ext/set_errors"))}) + with _patch_httpx(gw): + with pytest.raises(ApplyServerConfigError, match="Enabling the tools \\['slack_export'\\]"): + await _step().execute(_make_context()) + + class _GatewayForwarder(_RevealingGateway): """Replies to the config extension with `reply` and forwards the enable to a real in-process gateway.""" From 3f74730054431e7c713363b9a0b946400e95da75 Mon Sep 17 00:00:00 2001 From: Mateusz Cikowski Date: Thu, 19 Nov 2026 21:30:36 -0500 Subject: [PATCH 5/5] address review round 5: an empty stored card fails a reveal (tested); toolset docstring names where config can make tasks differ Co-Authored-By: Claude Opus 5.5 --- src/agent_env/openenv/environment.py | 5 +++-- tst/unit/task_step/test_apply_server_config.py | 7 +++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/agent_env/openenv/environment.py b/src/agent_env/openenv/environment.py index b374a7535..db323f657 100644 --- a/src/agent_env/openenv/environment.py +++ b/src/agent_env/openenv/environment.py @@ -69,8 +69,9 @@ def toolset(self) -> Toolset: """Where the played agent's tools come from: the envs it works in and the versions the task deploys, its role in them, the changes the task makes to that role's tool access, whether its turns offer ``bash`` in its sandbox, and whether it applies server config there (``config``, only when it does), whose replies can reveal - tools. Tasks that agree on it offer the same tools unless their configs reveal different ones, so a trainer - can group them.""" + tools. Tasks that agree on it offer the same tools, so a trainer can group them, except tasks with config: + their replies can reveal different tools, and a revealed tool depends on whether a per-role deny of it + comes before or after the config.""" agent = self.spec.trainer_agents[0] [deploy] = [step for step in self.task.steps if step.type in DEPLOYS and step.agent_name == agent] role, envs = getattr(deploy, "role", None), list(getattr(deploy, "env_ids", None) or []) diff --git a/tst/unit/task_step/test_apply_server_config.py b/tst/unit/task_step/test_apply_server_config.py index 2d556d001..30cf36ebf 100644 --- a/tst/unit/task_step/test_apply_server_config.py +++ b/tst/unit/task_step/test_apply_server_config.py @@ -981,15 +981,18 @@ def without_enable(node): @pytest.mark.asyncio -async def test_a_reveal_on_a_record_with_no_stored_card_fails_the_step(): +@pytest.mark.parametrize("stored", [None, {}], ids=["missing", "empty"]) +async def test_a_reveal_on_a_record_with_no_stored_card_fails_the_step(stored): class _Revealing(_MockGateway): async def request(self, method, url, *, json=None, timeout=None, headers=None): return _resp(200, url, {"ok": True, "reveal_tools": ["slack_export"]}) gw = _Revealing({"slack": _card(_ext(_SET_ERRORS_URI, "/agentenv/ext/set_errors"))}) + ctx = _make_context() + ctx.deployed_envs[0].environment_card = stored with _patch_httpx(gw): with pytest.raises(ApplyServerConfigError, match="Enabling the tools \\['slack_export'\\]"): - await _step().execute(_make_context()) + await _step().execute(ctx) class _GatewayForwarder(_RevealingGateway):