From 5132c116b41e3d7e07c75a09962809a74f25b521 Mon Sep 17 00:00:00 2001 From: Patricio Leonel Filice Date: Mon, 20 Jul 2026 18:52:24 -0300 Subject: [PATCH 1/5] Python: feat: add agent-framework-tenki (Tenki-backed CodeAct provider) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduces agent-framework-tenki, a third code-executor backend that runs Python inside a Tenki managed Linux microVM sandbox. Ships alongside the existing hyperlight (WASM) and monty (in-process Rust) executors, and mirrors their public API shape (TenkiCodeActProvider ContextProvider + TenkiExecuteCodeTool FunctionTool). Lifecycle: lazy provision, reuse-per-tool, and per-call reconciliation — PAUSED sandboxes are transparently resumed, TERMINATED sandboxes are replaced by a fresh provision. Includes 32 hermetic unit tests plus one opt-in integration test guarded on TENKI_API_KEY. Alpha-policy compliant: workspace uv sources only, no core[all] extra, no lazy-loading shim. --- python/packages/tenki/LICENSE | 21 + python/packages/tenki/README.md | 148 ++++ .../tenki/agent_framework_tenki/__init__.py | 19 + .../_execute_code_tool.py | 364 ++++++++++ .../tenki/agent_framework_tenki/_provider.py | 108 +++ .../tenki/agent_framework_tenki/py.typed | 0 python/packages/tenki/pyproject.toml | 94 +++ .../tenki/tests/tenki/test_tenki_codeact.py | 687 ++++++++++++++++++ python/pyproject.toml | 1 + python/uv.lock | 33 + 10 files changed, 1475 insertions(+) create mode 100644 python/packages/tenki/LICENSE create mode 100644 python/packages/tenki/README.md create mode 100644 python/packages/tenki/agent_framework_tenki/__init__.py create mode 100644 python/packages/tenki/agent_framework_tenki/_execute_code_tool.py create mode 100644 python/packages/tenki/agent_framework_tenki/_provider.py create mode 100644 python/packages/tenki/agent_framework_tenki/py.typed create mode 100644 python/packages/tenki/pyproject.toml create mode 100644 python/packages/tenki/tests/tenki/test_tenki_codeact.py diff --git a/python/packages/tenki/LICENSE b/python/packages/tenki/LICENSE new file mode 100644 index 00000000000..9e841e7a26e --- /dev/null +++ b/python/packages/tenki/LICENSE @@ -0,0 +1,21 @@ + MIT License + + Copyright (c) Microsoft Corporation. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE diff --git a/python/packages/tenki/README.md b/python/packages/tenki/README.md new file mode 100644 index 00000000000..3608c92c40b --- /dev/null +++ b/python/packages/tenki/README.md @@ -0,0 +1,148 @@ +# agent-framework-tenki + +[Tenki Sandbox](https://tenki.cloud) integration for Microsoft Agent Framework. + +## Installation + +```bash +pip install agent-framework-tenki --pre +``` + +You also need a Tenki API key. Follow the [Tenki Sandbox quick start](https://tenki.cloud/docs/sandbox/quick-start-sandbox) +to create a workspace and generate a key, then export it before running your agent: + +```bash +export TENKI_API_KEY="tk_..." +``` + +## Quick start + +### Context provider (recommended) + +Use `TenkiCodeActProvider` to inject an `execute_code` tool into every agent run. Each +run shares the same underlying Tenki sandbox until the provider is closed. + +```python +from agent_framework import Agent +from agent_framework.openai import OpenAIChatClient +from agent_framework_tenki import TenkiCodeActProvider + +async with TenkiCodeActProvider() as codeact: + agent = Agent( + client=OpenAIChatClient(), + context_providers=[codeact], + ) + result = await agent.run("Compute the 42nd Fibonacci number.") +``` + +### Standalone tool + +Use `TenkiExecuteCodeTool` directly when you want full control over how the tool is +attached to the agent. + +```python +from agent_framework import Agent +from agent_framework.openai import OpenAIChatClient +from agent_framework_tenki import TenkiExecuteCodeTool + +async with TenkiExecuteCodeTool() as execute_code: + agent = Agent( + client=OpenAIChatClient(), + tools=[execute_code], + ) + result = await agent.run("Print the SHA-256 of 'hello world'.") +``` + +Remember to `close()` (or use `async with`) so the sandbox is terminated when you're +done — otherwise it lives until your Tenki workspace timeout expires. + +## Configuration + +| Kwarg | Default | Description | +|---|---|---| +| `sandbox_name` | `agent-framework-<8-hex>` | Sandbox identifier used when creating the sandbox on Tenki. | +| `api_key` | `os.environ.get("TENKI_API_KEY")` | Overrides the environment variable. | +| `image` | Tenki default | Custom base image identifier. | +| `project_id` / `workspace_id` | `os.environ.get("TENKI_PROJECT_ID")` / `os.environ.get("TENKI_WORKSPACE_ID")` | Required when your API key has access to multiple projects. Constructor args override the env vars. | +| `cpu_cores` / `memory_mb` / `disk_size_gb` | Tenki defaults | Optional resource overrides. | +| `max_duration_seconds` | `None` | Server-side cost kill-switch. Recommended for production agent loops. | +| `exec_timeout_seconds` | `60` | Per-`execute_code` invocation timeout in seconds. | +| `extra_create_kwargs` | `{}` | Passed straight to `tenki_sandbox.Sandbox.create` for Tenki-specific options — see the section below. | + +### Tenki-specific options via `extra_create_kwargs` + +The Tenki SDK exposes platform features beyond the ones surfaced directly on +`TenkiExecuteCodeTool`. Anything you pass through `extra_create_kwargs` is +forwarded verbatim to `tenki_sandbox.Sandbox.create`. Common ones: + +| Kwarg | Type | Purpose | +|---|---|---| +| `snapshot_id` | `str` | Restore the sandbox from a previously created Tenki snapshot instead of provisioning a fresh image — preserves filesystem state across sessions. | +| `clone_repo_url` | `str` | Git-clone the URL into the sandbox on create. Pair with `github_token` for private repos. | +| `github_token` | `str` | Auth token consumed by `clone_repo_url`. | +| `env` | `dict[str, str]` | Environment variables passed into the sandbox at creation time (agent secrets, config, etc.). | +| `allow_inbound` / `allow_outbound` | `bool` | Sandbox network policy. Enable `allow_inbound` for inbound exposure workflows; disable `allow_outbound` for stricter isolation. Both default to `True`. | +| `metadata` | `dict[str, str]` | Attach arbitrary key-value tags for filtering, billing attribution, or upstream job-ID tracking. | +| `tags` | `list[str]` | Attach labels for organization/filtering in the Tenki dashboard. | +| `volumes` | `list[dict]` | Attach persistent [Tenki volumes](https://tenki.cloud/docs/sandbox/volumes) that survive sandbox termination. Each entry: `{"volume_id": str, "mount_path": str, "read_only": bool (optional)}`. Volumes are workspace-scoped and must be reattached explicitly on future sandboxes. | + +Example: + +```python +async with TenkiExecuteCodeTool( + extra_create_kwargs={ + "clone_repo_url": "https://github.com/myorg/myrepo", + "github_token": os.environ["GITHUB_TOKEN"], + "env": {"OPENAI_API_KEY": os.environ["OPENAI_API_KEY"]}, + }, +) as tool: + ... +``` + +See the [Tenki sandbox sessions](https://tenki.cloud/docs/sandbox/sessions#create-a-session) +and [Tenki volumes](https://tenki.cloud/docs/sandbox/volumes) reference for the +complete option list and semantics. + +## Lifecycle + +The sandbox is provisioned lazily on the first `execute_code` call and reused for every +subsequent call on the same tool or provider instance. Before each call the tool +reconciles remote sandbox state, so callers never need to track pause/terminate +transitions manually. Each call runs `python3 -c ` inside the sandbox, which +means: + +- **Sandbox filesystem persists** across calls — files written to `/tmp` or the user + home in one call are visible in the next. +- **Installed packages persist** — packages installed via pip or apt in one call are + available in subsequent calls (subject to the sandbox's outbound network policy). + See [Tenki's sandbox quick start](https://tenki.cloud/docs/sandbox/quick-start-sandbox) + for the recommended installation workflow (the default image ships `python3-venv`). +- **Python interpreter state does not persist** — each call is a fresh `python3` + process, so variables defined in one call are not reachable in the next. Persist + intermediate state through files or environment variables when a later call needs + it. +- **Paused sandboxes auto-resume** — if the sandbox transitions to `PAUSED` between + calls (Tenki's server-side idle policies, `idle_timeout_minutes` supplied via + `extra_create_kwargs`, or an external `tenki sandbox pause`), the next + `execute_code` call transparently resumes it before running. Filesystem and + installed packages carry across the pause unchanged. +- **Terminated sandboxes are replaced** — if the sandbox transitions to + `TERMINATING`/`TERMINATED` (workspace timeout, `max_duration_seconds`, or an + external `tenki sandbox terminate`), the next call provisions a fresh sandbox. + Filesystem and installed packages from the previous sandbox are **not** carried + over — snapshot the sandbox (via `extra_create_kwargs={"snapshot_id": ...}` on a + new tool) if you need to preserve state across a termination. + +Call `close()` on the tool or provider (or use `async with`) to terminate the sandbox +and release the underlying microVM. Different agents that share a single +`TenkiCodeActProvider` share the same sandbox — create a separate provider per agent +when isolation matters. + +## Notes + +- In-sandbox tool callbacks are not supported — code executing inside the sandbox + cannot invoke host-side tools (Tenki's SDK does not expose a callback bridge). +- File mounts and outbound network allow-lists are not surfaced as first-class + kwargs on this package. Pass them through `extra_create_kwargs` (see the table + above for `allow_inbound` / `allow_outbound` / `volumes`), or bake dependencies + into a custom Tenki image. diff --git a/python/packages/tenki/agent_framework_tenki/__init__.py b/python/packages/tenki/agent_framework_tenki/__init__.py new file mode 100644 index 00000000000..cdf2037d1c2 --- /dev/null +++ b/python/packages/tenki/agent_framework_tenki/__init__.py @@ -0,0 +1,19 @@ +# Copyright (c) Microsoft. All rights reserved. + +from __future__ import annotations + +import importlib.metadata + +from ._execute_code_tool import TenkiExecuteCodeTool +from ._provider import TenkiCodeActProvider + +try: + __version__ = importlib.metadata.version(__name__) +except importlib.metadata.PackageNotFoundError: + __version__ = "0.0.0" + +__all__ = [ + "TenkiCodeActProvider", + "TenkiExecuteCodeTool", + "__version__", +] diff --git a/python/packages/tenki/agent_framework_tenki/_execute_code_tool.py b/python/packages/tenki/agent_framework_tenki/_execute_code_tool.py new file mode 100644 index 00000000000..4702f3d3f9c --- /dev/null +++ b/python/packages/tenki/agent_framework_tenki/_execute_code_tool.py @@ -0,0 +1,364 @@ +# Copyright (c) Microsoft. All rights reserved. + +from __future__ import annotations + +import asyncio +import contextlib +import logging +import os +import threading +import uuid +from typing import Any, ClassVar + +from agent_framework import Content, FunctionTool +from agent_framework._tools import ApprovalMode +from tenki_sandbox import Sandbox + +logger = logging.getLogger(__name__) + + +EXECUTE_CODE_TOOL_DESCRIPTION = ( + "Execute Python in an isolated Tenki microVM sandbox. Each call runs the " + "code as `python3 -c `; only stdout and stderr are captured, so wrap " + "any values you want to see in `print(...)` — bare expressions are not " + "auto-printed like in a REPL." +) + +EXECUTE_CODE_INPUT_SCHEMA: dict[str, Any] = { + "type": "object", + "title": "_ExecuteCodeInput", + "properties": { + "code": { + "type": "string", + "title": "Code", + "description": ( + "Python code to execute in an isolated Tenki sandbox. The code " + "runs as `python3 -c ` — only stdout/stderr are captured, " + "so use `print(...)` to return any values." + ), + }, + }, + "required": ["code"], +} + + +def _default_sandbox_name() -> str: + """Return a unique, source-attributable sandbox name. + + The ``agent-framework-`` prefix lets operators inspecting the Tenki dashboard or the + ``tenki sandbox list`` CLI attribute the sandbox back to Agent Framework — matching + the client-attributable naming already used elsewhere in the workspace (for example + ``WorkflowBuilder-`` in ``agent-framework-durabletask``). + """ + return f"agent-framework-{uuid.uuid4().hex[:8]}" + + +class TenkiExecuteCodeTool(FunctionTool): + r"""Execute Python code inside a `Tenki `_ managed microVM sandbox. + + Requires the ``tenki-sandbox`` Python SDK (installed as a dependency of + ``agent-framework-tenki``) and a Tenki API key. Follow the + `Tenki Sandbox quick start `_ + to create a workspace and generate a key, then export it as ``TENKI_API_KEY``. + + Lifecycle: the tool lazily provisions a sandbox on the first ``_run_code`` invocation + and reuses the same sandbox for every subsequent call within the tool's lifetime. + Before each call the tool reconciles remote sandbox state — a sandbox that has + transitioned to ``PAUSED`` (Tenki server-side idle policy, ``idle_timeout_minutes`` + from ``extra_create_kwargs``, or an external ``tenki sandbox pause``) is + transparently resumed, and a sandbox that has transitioned to ``TERMINATING`` or + ``TERMINATED`` (workspace timeout, ``max_duration_seconds``, or an external + ``tenki sandbox terminate``) is replaced by a fresh provision. Callers therefore + never have to track pause/terminate transitions manually. Each call runs + ``python3 -c `` inside the sandbox, so the sandbox filesystem and installed + packages persist across calls but the Python interpreter state does not — + variables defined in one call are not reachable in the next. Filesystem and + installed packages are also lost when the sandbox is re-provisioned after a + termination; use ``extra_create_kwargs={"snapshot_id": ...}`` on a new tool if you + need to preserve state across that boundary. Persist intermediate state through + files or environment variables when a later call needs it. Call :meth:`close` (or + use the tool via ``async with``) to terminate the sandbox and release the + underlying microVM. A new sandbox is created on the next call if the tool is + reused after being closed. + + Args: + approval_mode (ApprovalMode | None): Approval policy passed through to + :class:`agent_framework.FunctionTool`. Defaults to ``"never_require"``. + api_key (str | None): Optional Tenki API key. When ``None`` the SDK reads + :envvar:`TENKI_API_KEY` from the environment. + sandbox_name (str | None): Optional sandbox identifier. When ``None`` the tool + generates ``agent-framework-<8-hex>`` — matching the naming convention already + used by other Agent Framework packages that create externally visible + resources. + image (str | None): Optional Tenki base-image identifier. When ``None`` the + Tenki service picks its default sandbox image (which ships ``python3``). + project_id (str | None): Optional Tenki project ID scoping the sandbox. + Required when the API key has access to more than one project. + workspace_id (str | None): Optional Tenki workspace ID. + cpu_cores (int | None): Optional CPU-core count. When ``None`` the Tenki + service default applies. + memory_mb (int | None): Optional memory (MB). ``None`` uses the service default. + disk_size_gb (int | None): Optional ephemeral disk (GB). ``None`` uses the + service default. + max_duration_seconds (int | None): Optional cost-safety cap on sandbox + lifetime, enforced server-side. When ``None`` the sandbox lives until + explicitly terminated (or the workspace timeout applies). Recommended for + production use to avoid runaway sandboxes in agent loops. + exec_timeout_seconds (int): Per-``_run_code`` timeout in seconds. Defaults to 60. + extra_create_kwargs (dict[str, Any] | None): Optional keyword arguments passed + straight to :meth:`tenki_sandbox.Sandbox.create` — for advanced knobs not + surfaced individually (e.g. ``snapshot_id``, ``allow_inbound``, + ``allow_outbound``). + + Notes: + * In-sandbox tool callbacks are not supported — code executing inside the sandbox + cannot invoke host-side :class:`~agent_framework.FunctionTool` instances (the + Tenki SDK does not expose a callback bridge). + * File mounts and outbound network allow-lists are not modeled by this package. + Bake dependencies and files into a custom Tenki image, or configure them + through ``extra_create_kwargs``. + * The sandbox lifecycle is *reuse-per-tool*, not per-agent-run. If the tool is + shared across many agent runs, filesystem state from run N is visible to run + N+1 (same sandbox), though each individual ``execute_code`` invocation is a + fresh Python process. Create a fresh :class:`TenkiExecuteCodeTool` per agent + instance if isolation between agents matters. + """ + + SUPPORTED_LANGUAGES: ClassVar[list[str]] = ["python"] + + def __init__( + self, + *, + approval_mode: ApprovalMode | None = None, + api_key: str | None = None, + sandbox_name: str | None = None, + image: str | None = None, + project_id: str | None = None, + workspace_id: str | None = None, + cpu_cores: int | None = None, + memory_mb: int | None = None, + disk_size_gb: int | None = None, + max_duration_seconds: int | None = None, + exec_timeout_seconds: int = 60, + extra_create_kwargs: dict[str, Any] | None = None, + ) -> None: + if exec_timeout_seconds < 1: + raise ValueError("exec_timeout_seconds must be greater than or equal to 1.") + + super().__init__( + name="execute_code", + description=EXECUTE_CODE_TOOL_DESCRIPTION, + approval_mode=approval_mode or "never_require", + func=self._run_code, + input_model=EXECUTE_CODE_INPUT_SCHEMA, + ) + + self._api_key = api_key + # An explicit ``sandbox_name`` is preserved as-is across the tool's lifetime, + # even across re-provisioning after termination — the user asked for that + # name specifically, so if Tenki rejects reuse we surface the error. An + # auto-generated name is refreshed on every re-provision to avoid + # ``ALREADY_EXISTS`` collisions when Tenki holds a name reservation after + # termination. + self._explicit_sandbox_name: str | None = sandbox_name + self._sandbox_name = sandbox_name if sandbox_name is not None else _default_sandbox_name() + self._image = image + self._project_id = project_id + self._workspace_id = workspace_id + self._cpu_cores = cpu_cores + self._memory_mb = memory_mb + self._disk_size_gb = disk_size_gb + self._max_duration_seconds = max_duration_seconds + self._exec_timeout_seconds = exec_timeout_seconds + self._extra_create_kwargs: dict[str, Any] = dict(extra_create_kwargs) if extra_create_kwargs else {} + + # A single sandbox is created lazily on first use and reused across calls. + # A lock guards the create/close race between concurrent ``_run_code`` invocations. + self._sandbox_lock = threading.Lock() + self._sandbox: Sandbox | None = None + + @property + def sandbox_name(self) -> str: + """The name of the underlying Tenki sandbox.""" + return self._sandbox_name + + @property + def exec_timeout_seconds(self) -> int: + """Per-invocation execution timeout in seconds.""" + return self._exec_timeout_seconds + + def _build_create_kwargs(self) -> dict[str, Any]: + kwargs: dict[str, Any] = {"name": self._sandbox_name} + # Only forward optional values the caller explicitly set; the Tenki SDK applies + # its own defaults when we omit a field, and it may reject ``field=None``. + # ``is not None`` (rather than ``or``) so that an explicit empty string on the + # constructor still wins over the env var — the contract is "explicit args + # override env vars when provided", not "truthy args override env vars". + api_key = self._api_key if self._api_key is not None else os.environ.get("TENKI_API_KEY") + if api_key is not None: + kwargs["auth_token"] = api_key + if self._image is not None: + kwargs["image"] = self._image + project_id = self._project_id if self._project_id is not None else os.environ.get("TENKI_PROJECT_ID") + if project_id is not None: + kwargs["project_id"] = project_id + workspace_id = ( + self._workspace_id if self._workspace_id is not None else os.environ.get("TENKI_WORKSPACE_ID") + ) + if workspace_id is not None: + kwargs["workspace_id"] = workspace_id + if self._cpu_cores is not None: + kwargs["cpu_cores"] = self._cpu_cores + if self._memory_mb is not None: + kwargs["memory_mb"] = self._memory_mb + if self._disk_size_gb is not None: + kwargs["disk_size_gb"] = self._disk_size_gb + if self._max_duration_seconds is not None: + kwargs["max_duration"] = self._max_duration_seconds + # Caller-supplied extras win last so they can override anything above. + kwargs.update(self._extra_create_kwargs) + return kwargs + + def _ensure_sandbox_sync(self) -> Sandbox: + """Return a usable sandbox — provision on first use, resume if paused, re-provision if gone. + + Tenki sandboxes can transition to ``PAUSED`` between calls (server-side idle + policies, external ``tenki sandbox pause``) and to ``TERMINATED`` if the + workspace timeout or ``max_duration`` elapsed. This method reconciles the + remote state on every call so ``_run_code`` never hands the SDK a + ``sandbox.exec`` that would fail with ``sandbox is not RUNNING``. + + Called inside ``asyncio.to_thread``. + """ + with self._sandbox_lock: + sandbox = self._sandbox + if sandbox is None: + return self._create_sandbox_locked() + + try: + sandbox.refresh() + except Exception as exc: + # Remote state is unknowable — drop the stale handle and re-provision. + logger.debug("Dropping Tenki sandbox handle after refresh failure: %s", exc) + self._sandbox = None + self._refresh_autogenerated_name_locked() + return self._create_sandbox_locked() + + state = getattr(sandbox, "state", None) + if state == "PAUSED": + try: + sandbox.resume() + except Exception as exc: + raise RuntimeError(f"Failed to resume paused Tenki sandbox: {exc}") from exc + return sandbox + if state in {"TERMINATING", "TERMINATED"}: + logger.debug("Dropping Tenki sandbox handle after remote state=%s", state) + self._sandbox = None + self._refresh_autogenerated_name_locked() + return self._create_sandbox_locked() + return sandbox + + def _refresh_autogenerated_name_locked(self) -> None: + """Mint a fresh auto-generated name so re-provision never reuses the prior one. + + Tenki may hold the terminated sandbox's name (``ALREADY_EXISTS``); a fresh + 8-hex suffix avoids the collision. Explicit user-supplied names are + preserved — the caller chose that name deliberately, and any reuse + rejection surfaces as a ``RuntimeError`` they can act on. + """ + if self._explicit_sandbox_name is None: + self._sandbox_name = _default_sandbox_name() + + def _create_sandbox_locked(self) -> Sandbox: + """Provision a fresh sandbox. The sandbox lock must be held by the caller.""" + create_kwargs = self._build_create_kwargs() + try: + sandbox = Sandbox.create(**create_kwargs) # pyright: ignore[reportUnknownMemberType] + except Exception as exc: + raise RuntimeError(f"Failed to create Tenki sandbox: {exc}") from exc + self._sandbox = sandbox + return sandbox + + async def _run_code(self, *, code: str) -> list[Content]: + """Execute a single block of Python code inside the sandbox.""" + try: + sandbox = await asyncio.to_thread(self._ensure_sandbox_sync) + except RuntimeError as exc: + return [Content.from_error(message="Failed to provision Tenki sandbox", error_details=str(exc))] + + try: + result = await asyncio.to_thread(sandbox.exec, "python3", "-c", code, timeout=self._exec_timeout_seconds) + except asyncio.CancelledError: + # Cancellation must propagate so higher-level workflows can stop promptly. + raise + except Exception as exc: + return [Content.from_error(message="Sandbox execution failed", error_details=str(exc))] + + return self._build_contents(result) + + _ERROR_MESSAGE_STDERR_LIMIT: ClassVar[int] = 500 + + # Recovery hint appended when the sandbox reports a Python SyntaxError. Only + # fires for that specific error class to keep the hint from misleading + # stronger models on unrelated failures. + _SYNTAX_ERROR_HINT: ClassVar[str] = ( + " [Common causes: (1) a compound statement (with/for/if/try/while) " + "following a semicolon at the outer level; put it on its own line " + "instead. (2) `\\n` literals in the JSON payload where actual newline " + "characters were intended.]" + ) + + def _build_contents(self, result: Any) -> list[Content]: + """Convert a Tenki exec result into a list of :class:`Content` values.""" + stdout = getattr(result, "stdout_text", "") or "" + stderr = getattr(result, "stderr_text", "") or "" + exit_code = int(getattr(result, "exit_code", 1)) + + contents: list[Content] = [] + if stdout: + contents.append(Content.from_text(stdout)) + if exit_code != 0: + # Inline a truncated stderr into the message so downstream LLM + # consumers see the traceback in the primary field, not just in + # error_details — small models tend to under-weight secondary fields. + stderr_snippet = stderr.strip()[: self._ERROR_MESSAGE_STDERR_LIMIT] if stderr else "" + message = f"Code exited with status {exit_code}" + if stderr_snippet: + message = f"{message}. stderr: {stderr_snippet}" + if stderr and "SyntaxError" in stderr: + message = f"{message}{self._SYNTAX_ERROR_HINT}" + contents.append( + Content.from_error( + message=message, + error_details=stderr or None, + ) + ) + elif stderr: + # Non-fatal stderr (e.g. warnings) — surface it as ordinary text so the model + # sees it, matching Hyperlight's behaviour. + contents.append(Content.from_text(stderr)) + if not contents: + contents.append(Content.from_text("Code executed successfully without output.")) + return contents + + async def close(self) -> None: + """Terminate the underlying Tenki sandbox and release its resources. + + Safe to call multiple times; a no-op if the sandbox was never created. After + close, a subsequent ``_run_code`` invocation lazily provisions a new sandbox. + """ + with self._sandbox_lock: + sandbox = self._sandbox + self._sandbox = None + if sandbox is None: + return + # SDK errors on shutdown are ignored — the sandbox is server-side managed and will + # be reaped by Tenki's own idle timeout even if our terminate call fails. + with contextlib.suppress(Exception): # pragma: no cover - defensive + await asyncio.to_thread(sandbox.close_if_open) + + async def __aenter__(self) -> TenkiExecuteCodeTool: + return self + + async def __aexit__(self, exc_type: Any, exc: Any, tb: Any) -> None: + await self.close() diff --git a/python/packages/tenki/agent_framework_tenki/_provider.py b/python/packages/tenki/agent_framework_tenki/_provider.py new file mode 100644 index 00000000000..3d7db27c459 --- /dev/null +++ b/python/packages/tenki/agent_framework_tenki/_provider.py @@ -0,0 +1,108 @@ +# Copyright (c) Microsoft. All rights reserved. + +from __future__ import annotations + +from typing import Any + +from agent_framework import AgentSession, ContextProvider, SessionContext +from agent_framework._tools import ApprovalMode + +from ._execute_code_tool import TenkiExecuteCodeTool + +TENKI_CODEACT_INSTRUCTIONS = ( + "You have access to `execute_code`, which runs Python inside a Tenki microVM " + "sandbox. Each call runs the code as `python3 -c ` — only stdout and " + "stderr are captured, so wrap results in `print(...)`; bare expressions are " + "not auto-printed like in a REPL. The sandbox filesystem persists across " + "calls (files under `/home/tenki` and `/tmp` are visible in later calls), but " + "Python interpreter state does not — every call starts a fresh Python process, " + "so persist intermediate state through files or environment variables when a " + "later call needs it. Do not use Jupyter/IPython magic syntax (`!command`, " + "`%magic`, `{var}` interpolation) — those do not work in plain `python3 -c` " + "and will raise `SyntaxError`. Use newlines (not semicolons) to introduce " + "compound statements like `if`, `for`, `while`, `with`, or `try`: Python's " + "grammar does not permit compound statements after `;` and `import x; with " + "open(...) as f: ...` raises `SyntaxError`. Valid single-line form: " + "`with open('/tmp/x.txt') as f: print(f.read())` — semicolons ARE allowed " + "between simple statements inside the block body (e.g. `with open(...) as " + "f: x = f.read(); print(x)`); what's forbidden is a compound statement " + "following a semicolon. For shell commands, use " + "`subprocess`; for example, to install a Python package: `import subprocess, " + "sys; subprocess.check_call([sys.executable, '-m', 'pip', 'install', " + "'--break-system-packages', ''])`." +) + + +class TenkiCodeActProvider(ContextProvider): + """Inject a Tenki-backed CodeAct surface using a provider-owned execute_code tool. + + On every agent run, the provider exposes its underlying :class:`TenkiExecuteCodeTool` + to the context so the model can run Python inside a Tenki sandbox. + + The sandbox is reused across runs of the same provider — its filesystem and installed + packages persist across calls, though each individual ``execute_code`` invocation is + a fresh Python process. Instantiate a new provider per agent instance when you need + isolation between agents. + """ + + DEFAULT_SOURCE_ID = "tenki_codeact" + + def __init__( + self, + source_id: str = DEFAULT_SOURCE_ID, + *, + approval_mode: ApprovalMode | None = None, + api_key: str | None = None, + sandbox_name: str | None = None, + image: str | None = None, + project_id: str | None = None, + workspace_id: str | None = None, + cpu_cores: int | None = None, + memory_mb: int | None = None, + disk_size_gb: int | None = None, + max_duration_seconds: int | None = None, + exec_timeout_seconds: int = 60, + extra_create_kwargs: dict[str, Any] | None = None, + ) -> None: + super().__init__(source_id) + self._execute_code_tool = TenkiExecuteCodeTool( + approval_mode=approval_mode, + api_key=api_key, + sandbox_name=sandbox_name, + image=image, + project_id=project_id, + workspace_id=workspace_id, + cpu_cores=cpu_cores, + memory_mb=memory_mb, + disk_size_gb=disk_size_gb, + max_duration_seconds=max_duration_seconds, + exec_timeout_seconds=exec_timeout_seconds, + extra_create_kwargs=extra_create_kwargs, + ) + + @property + def execute_code_tool(self) -> TenkiExecuteCodeTool: + """The underlying execute_code tool. Exposed for advanced integration.""" + return self._execute_code_tool + + async def close(self) -> None: + """Terminate the underlying Tenki sandbox and release its resources.""" + await self._execute_code_tool.close() + + async def __aenter__(self) -> TenkiCodeActProvider: + return self + + async def __aexit__(self, exc_type: Any, exc: Any, tb: Any) -> None: + await self.close() + + async def before_run( + self, + *, + agent: Any, + session: AgentSession | None, + context: SessionContext, + state: dict[str, Any], + ) -> None: + """Inject the execute_code tool and its usage instructions for this run.""" + context.extend_instructions(self.source_id, TENKI_CODEACT_INSTRUCTIONS) + context.extend_tools(self.source_id, [self._execute_code_tool]) diff --git a/python/packages/tenki/agent_framework_tenki/py.typed b/python/packages/tenki/agent_framework_tenki/py.typed new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/packages/tenki/pyproject.toml b/python/packages/tenki/pyproject.toml new file mode 100644 index 00000000000..979c586c85e --- /dev/null +++ b/python/packages/tenki/pyproject.toml @@ -0,0 +1,94 @@ +[project] +name = "agent-framework-tenki" +description = "Tenki Sandbox CodeAct integrations for Microsoft Agent Framework." +authors = [{ name = "Microsoft", email = "af-support@microsoft.com"}] +readme = "README.md" +requires-python = ">=3.10" +version = "0.1.0a260720" +license-files = ["LICENSE"] +urls.homepage = "https://aka.ms/agent-framework" +urls.source = "https://github.com/microsoft/agent-framework/tree/main/python" +urls.release_notes = "https://github.com/microsoft/agent-framework/releases?q=tag%3Apython-1&expanded=true" +urls.issues = "https://github.com/microsoft/agent-framework/issues" +classifiers = [ + "License :: OSI Approved :: MIT License", + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Typing :: Typed", +] +dependencies = [ + "agent-framework-core>=1.11.0,<2", + "tenki-sandbox>=0.1.0,<1", +] + +[tool.uv] +prerelease = "if-necessary-or-explicit" + +[tool.uv-dynamic-versioning] +fallback-version = "0.0.0" + +[tool.pytest.ini_options] +testpaths = 'tests' +addopts = "-ra -q -r fEX" +asyncio_mode = "auto" +asyncio_default_fixture_loop_scope = "function" +filterwarnings = [] +timeout = 120 +markers = [ + "integration: marks tests as integration tests that require external services", +] + +[tool.ruff] +extend = "../../pyproject.toml" + +[tool.ruff.lint.per-file-ignores] +"tests/**" = ["D", "INP", "TD", "ERA001", "RUF", "S"] + +[tool.coverage.run] +omit = [ + "**/__init__.py" +] + +[tool.pyright] +extends = "../../pyproject.toml" +include = ["agent_framework_tenki"] +exclude = ['tests'] + +[tool.mypy] +plugins = ['pydantic.mypy'] +strict = true +python_version = "3.10" +ignore_missing_imports = true +disallow_untyped_defs = true +no_implicit_optional = true +check_untyped_defs = true +warn_return_any = true +show_error_codes = true +warn_unused_ignores = false +disallow_incomplete_defs = true +disallow_untyped_decorators = true + +[tool.bandit] +targets = ["agent_framework_tenki"] +exclude_dirs = ["tests"] + +[tool.poe] +executor.type = "uv" +include = "../../shared_tasks.toml" + +[tool.poe.tasks.mypy] +help = "Run MyPy for this package." +cmd = "mypy --config-file $POE_ROOT/pyproject.toml agent_framework_tenki" + +[tool.poe.tasks.test] +help = "Run the default unit test suite for this package." +cmd = 'pytest -m "not integration" --cov=agent_framework_tenki --cov-report=term-missing:skip-covered tests' + +[build-system] +requires = ["flit-core >= 3.11,<4.0"] +build-backend = "flit_core.buildapi" diff --git a/python/packages/tenki/tests/tenki/test_tenki_codeact.py b/python/packages/tenki/tests/tenki/test_tenki_codeact.py new file mode 100644 index 00000000000..b233377faab --- /dev/null +++ b/python/packages/tenki/tests/tenki/test_tenki_codeact.py @@ -0,0 +1,687 @@ +# Copyright (c) Microsoft. All rights reserved. + +from __future__ import annotations + +import importlib.util +import os +import sys +from types import SimpleNamespace +from typing import Any, cast + +import pytest + +if sys.platform == "win32": # pragma: no cover - platform-dependent + # The integration test relies on POSIX-style commands inside the sandbox. Skip the + # whole module on Windows to match the sibling ``agent-framework-hyperlight`` tests. + pytest.skip("Tenki tests use POSIX-style shell invocations.", allow_module_level=True) + +# ``tenki-sandbox`` is a hard dep of this package, so in practice it's always present. +# Skip the module if it happens to be missing (e.g. dev checkout without a full install) +# instead of erroring pytest collection. +try: + from agent_framework_tenki import TenkiCodeActProvider, TenkiExecuteCodeTool + from agent_framework_tenki import _execute_code_tool as _tenki_module +except ImportError as _import_err: # pragma: no cover - environment-dependent + pytest.skip( + f"tenki-sandbox SDK is not importable ({_import_err}); skipping Tenki tests.", + allow_module_level=True, + ) + + +# --------------------------------------------------------------------------- +# Fake Tenki SDK — deterministic, in-memory replacement for ``tenki_sandbox.Sandbox``. +# --------------------------------------------------------------------------- + + +class _FakeExecResult(SimpleNamespace): + def __init__(self, *, stdout_text: str = "", stderr_text: str = "", exit_code: int = 0) -> None: + super().__init__(stdout_text=stdout_text, stderr_text=stderr_text, exit_code=exit_code) + + +class _FakeSandbox: + def __init__(self, *, name: str) -> None: + self.name = name + self.closed: bool = False + self.exec_calls: list[tuple[tuple[Any, ...], dict[str, Any]]] = [] + self.script_result: _FakeExecResult = _FakeExecResult(stdout_text="ok\n") + self.script_handler: Any = None + # Lifecycle state — mirrors the real SDK's ``sandbox.state`` string values. + self.state: str = "RUNNING" + self.refresh_calls: int = 0 + self.resume_calls: int = 0 + # When set, ``refresh()`` transitions ``state`` to this value on the next call. + self.refresh_transitions_to: str | None = None + # When set, ``refresh()`` raises this exception on the next call. + self.refresh_raises: BaseException | None = None + # When set, ``resume()`` raises this exception on the next call. + self.resume_raises: BaseException | None = None + + def exec(self, *args: Any, **kwargs: Any) -> _FakeExecResult: + self.exec_calls.append((args, kwargs)) + if self.script_handler is not None: + return self.script_handler(args, kwargs) # type: ignore[no-any-return] + return self.script_result + + def refresh(self) -> None: + self.refresh_calls += 1 + if self.refresh_raises is not None: + exc, self.refresh_raises = self.refresh_raises, None + raise exc + if self.refresh_transitions_to is not None: + self.state = self.refresh_transitions_to + self.refresh_transitions_to = None + + def resume(self) -> None: + self.resume_calls += 1 + if self.resume_raises is not None: + exc, self.resume_raises = self.resume_raises, None + raise exc + self.state = "RUNNING" + + def close_if_open(self) -> None: + self.closed = True + + +class _FakeSandboxFactory: + """Stand-in for :class:`tenki_sandbox.Sandbox`. Records ``create`` calls.""" + + def __init__(self) -> None: + self.create_calls: list[dict[str, Any]] = [] + self.sandboxes: list[_FakeSandbox] = [] + self.raise_on_create: BaseException | None = None + + @property + def last_sandbox(self) -> _FakeSandbox | None: + return self.sandboxes[-1] if self.sandboxes else None + + def create(self, **kwargs: Any) -> _FakeSandbox: + if self.raise_on_create is not None: + raise self.raise_on_create + self.create_calls.append(kwargs) + sandbox = _FakeSandbox(name=kwargs.get("name", "sb-fake")) + self.sandboxes.append(sandbox) + return sandbox + + +@pytest.fixture +def fake_sdk(monkeypatch: pytest.MonkeyPatch) -> _FakeSandboxFactory: + """Replace the Tenki ``Sandbox`` class in the executor module with an in-memory fake.""" + factory = _FakeSandboxFactory() + monkeypatch.setattr(_tenki_module, "Sandbox", factory) + return factory + + +# --------------------------------------------------------------------------- +# Unit tests — no network / no real Tenki service. +# --------------------------------------------------------------------------- + + +def test_default_sandbox_name_carries_agent_framework_prefix() -> None: + tool = TenkiExecuteCodeTool() + assert tool.sandbox_name.startswith("agent-framework-") + # 8-char hex suffix, per _default_sandbox_name. + suffix = tool.sandbox_name.rsplit("-", 1)[-1] + assert len(suffix) == 8 + int(suffix, 16) # raises if the suffix is not hex. + + +def test_explicit_sandbox_name_is_preserved() -> None: + tool = TenkiExecuteCodeTool(sandbox_name="my-name") + assert tool.sandbox_name == "my-name" + + +def test_exec_timeout_below_one_is_rejected() -> None: + with pytest.raises(ValueError, match="exec_timeout_seconds"): + TenkiExecuteCodeTool(exec_timeout_seconds=0) + + +async def test_run_code_lazily_creates_sandbox(fake_sdk: _FakeSandboxFactory) -> None: + tool = TenkiExecuteCodeTool(sandbox_name="lazy") + # No sandbox yet just from construction. + assert fake_sdk.create_calls == [] + + await tool._run_code(code="print('hello')") + assert len(fake_sdk.create_calls) == 1 + assert fake_sdk.create_calls[0]["name"] == "lazy" + + +async def test_run_code_reuses_the_same_sandbox_across_calls(fake_sdk: _FakeSandboxFactory) -> None: + """Reuse-per-session: subsequent calls do NOT create a new sandbox.""" + tool = TenkiExecuteCodeTool(sandbox_name="reuse") + await tool._run_code(code="x = 1") + await tool._run_code(code="print(x)") + await tool._run_code(code="print(x + 1)") + assert len(fake_sdk.create_calls) == 1 + # All exec calls landed on the single created sandbox. + assert fake_sdk.last_sandbox is not None + assert len(fake_sdk.last_sandbox.exec_calls) == 3 + + +async def test_close_terminates_sandbox_and_next_call_creates_new_one( + fake_sdk: _FakeSandboxFactory, +) -> None: + tool = TenkiExecuteCodeTool(sandbox_name="cycle") + await tool._run_code(code="print('a')") + first_sandbox = fake_sdk.last_sandbox + assert first_sandbox is not None + await tool.close() + assert first_sandbox.closed is True + + await tool._run_code(code="print('b')") + assert len(fake_sdk.create_calls) == 2 + assert fake_sdk.last_sandbox is not first_sandbox + + +async def test_paused_sandbox_is_auto_resumed_between_calls(fake_sdk: _FakeSandboxFactory) -> None: + """A sandbox paused between calls must be transparently resumed before the next exec.""" + tool = TenkiExecuteCodeTool(sandbox_name="paused") + await tool._run_code(code="print('a')") + sandbox = fake_sdk.last_sandbox + assert sandbox is not None + + # Simulate a server-side pause discovered on the next ``refresh()``. + sandbox.refresh_transitions_to = "PAUSED" + + await tool._run_code(code="print('b')") + + # Same sandbox reused (no re-provision). + assert len(fake_sdk.create_calls) == 1 + assert fake_sdk.last_sandbox is sandbox + # refresh + resume were called, and state landed back on RUNNING. + assert sandbox.refresh_calls == 1 + assert sandbox.resume_calls == 1 + assert sandbox.state == "RUNNING" + # The second exec landed on the same sandbox. + assert len(sandbox.exec_calls) == 2 + + +async def test_reprovision_regenerates_name_for_autogenerated_defaults( + fake_sdk: _FakeSandboxFactory, +) -> None: + """Auto-generated sandbox names must be refreshed on every re-provision. + + Tenki can reject ``Sandbox.create(name=)`` with + ``ALREADY_EXISTS`` if the workspace still holds the name; a fresh 8-hex + suffix keeps the re-provision path healthy for the common case where the + caller never passed an explicit ``sandbox_name``. + """ + tool = TenkiExecuteCodeTool() # no explicit sandbox_name → auto-generated + original_name = tool.sandbox_name + await tool._run_code(code="print('a')") + first = fake_sdk.last_sandbox + assert first is not None + first.refresh_transitions_to = "TERMINATED" + + await tool._run_code(code="print('b')") + + # Two ``create`` calls, each with a different name; both carry the shared prefix. + assert len(fake_sdk.create_calls) == 2 + assert fake_sdk.create_calls[0]["name"] == original_name + assert fake_sdk.create_calls[1]["name"] != original_name + assert fake_sdk.create_calls[1]["name"].startswith("agent-framework-") + + +async def test_reprovision_preserves_explicit_sandbox_name( + fake_sdk: _FakeSandboxFactory, +) -> None: + """An explicit ``sandbox_name`` must be kept as-is across re-provision. + + Regenerating would silently violate the caller's expressed intent; if Tenki + rejects reuse we let the ``RuntimeError`` surface so the caller can act on it. + """ + tool = TenkiExecuteCodeTool(sandbox_name="explicit-name") + await tool._run_code(code="print('a')") + first = fake_sdk.last_sandbox + assert first is not None + first.refresh_transitions_to = "TERMINATED" + + await tool._run_code(code="print('b')") + + assert [call["name"] for call in fake_sdk.create_calls] == ["explicit-name", "explicit-name"] + + +async def test_terminated_sandbox_is_replaced_by_fresh_provision(fake_sdk: _FakeSandboxFactory) -> None: + """A sandbox that terminated between calls must be dropped and replaced.""" + tool = TenkiExecuteCodeTool(sandbox_name="gone") + await tool._run_code(code="print('a')") + first = fake_sdk.last_sandbox + assert first is not None + + first.refresh_transitions_to = "TERMINATED" + + await tool._run_code(code="print('b')") + + # A brand new sandbox was provisioned; the terminated one was not resumed. + assert len(fake_sdk.create_calls) == 2 + assert fake_sdk.last_sandbox is not first + assert first.resume_calls == 0 + assert fake_sdk.last_sandbox is not None + assert len(fake_sdk.last_sandbox.exec_calls) == 1 + + +async def test_resume_failure_surfaces_as_error_content(fake_sdk: _FakeSandboxFactory) -> None: + """If ``resume()`` raises, the failure surfaces to the caller as error content. + + The stale handle is deliberately kept so that a subsequent call (once the underlying + condition clears, e.g. quota released) can retry ``resume()`` on the same sandbox + without losing filesystem state to an unnecessary re-provision. + """ + tool = TenkiExecuteCodeTool(sandbox_name="resume-fail") + await tool._run_code(code="print('a')") + sandbox = fake_sdk.last_sandbox + assert sandbox is not None + + sandbox.refresh_transitions_to = "PAUSED" + sandbox.resume_raises = RuntimeError("quota_exceeded: resume denied") + + contents = await tool._run_code(code="print('b')") + + # RuntimeError from _ensure_sandbox_sync is caught in _run_code and returned as + # error content — the caller sees a structured failure, not an unhandled exception. + assert len(contents) == 1 + assert contents[0].type == "error" + assert "Failed to provision Tenki sandbox" in (contents[0].message or "") + assert "quota_exceeded" in (contents[0].error_details or "") + + # Resume was attempted exactly once; no re-provision happened. + assert sandbox.resume_calls == 1 + assert len(fake_sdk.create_calls) == 1 + + +async def test_refresh_failure_falls_back_to_fresh_provision(fake_sdk: _FakeSandboxFactory) -> None: + """If ``refresh()`` raises, the stale handle is dropped and a fresh sandbox is provisioned.""" + tool = TenkiExecuteCodeTool(sandbox_name="stale") + await tool._run_code(code="print('a')") + first = fake_sdk.last_sandbox + assert first is not None + + first.refresh_raises = RuntimeError("connection reset") + + await tool._run_code(code="print('b')") + + assert len(fake_sdk.create_calls) == 2 + assert fake_sdk.last_sandbox is not first + # We did not attempt to resume a sandbox whose state we couldn't confirm. + assert first.resume_calls == 0 + + +async def test_close_is_idempotent(fake_sdk: _FakeSandboxFactory) -> None: + tool = TenkiExecuteCodeTool(sandbox_name="idempotent") + await tool.close() # no sandbox yet — no-op. + await tool._run_code(code="print('x')") + await tool.close() + await tool.close() # second close — no-op. + assert fake_sdk.last_sandbox is not None + assert fake_sdk.last_sandbox.closed is True + + +async def test_async_context_manager_closes_sandbox(fake_sdk: _FakeSandboxFactory) -> None: + async with TenkiExecuteCodeTool(sandbox_name="cm") as tool: + await tool._run_code(code="print('inside')") + assert fake_sdk.last_sandbox is not None + assert fake_sdk.last_sandbox.closed is True + + +async def test_create_kwargs_only_forward_set_optionals(fake_sdk: _FakeSandboxFactory) -> None: + tool = TenkiExecuteCodeTool( + sandbox_name="kwargs", + api_key="tk_TEST", + image="my-image", + project_id="proj-1", + workspace_id="ws-1", + cpu_cores=4, + memory_mb=2048, + disk_size_gb=10, + max_duration_seconds=300, + extra_create_kwargs={"allow_inbound": True}, + ) + await tool._run_code(code="pass") + + call = fake_sdk.create_calls[0] + assert call["name"] == "kwargs" + assert call["auth_token"] == "tk_TEST" + assert call["image"] == "my-image" + assert call["project_id"] == "proj-1" + assert call["workspace_id"] == "ws-1" + assert call["cpu_cores"] == 4 + assert call["memory_mb"] == 2048 + assert call["disk_size_gb"] == 10 + assert call["max_duration"] == 300 + assert call["allow_inbound"] is True + + +async def test_create_kwargs_omit_unset_optionals( + fake_sdk: _FakeSandboxFactory, monkeypatch: pytest.MonkeyPatch +) -> None: + # Every Tenki env var the tool consults must be scrubbed so this test doesn't + # inherit a real developer/CI value and start asserting against it. + monkeypatch.delenv("TENKI_API_KEY", raising=False) + monkeypatch.delenv("TENKI_PROJECT_ID", raising=False) + monkeypatch.delenv("TENKI_WORKSPACE_ID", raising=False) + tool = TenkiExecuteCodeTool(sandbox_name="bare") + await tool._run_code(code="pass") + + call = fake_sdk.create_calls[0] + for absent in ( + "auth_token", + "image", + "project_id", + "workspace_id", + "cpu_cores", + "memory_mb", + "disk_size_gb", + "max_duration", + ): + assert absent not in call, f"expected {absent!r} to be omitted from create kwargs" + + +async def test_env_api_key_is_forwarded_as_auth_token( + fake_sdk: _FakeSandboxFactory, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("TENKI_API_KEY", "tk_FROM_ENV") + tool = TenkiExecuteCodeTool(sandbox_name="env-key") + await tool._run_code(code="pass") + assert fake_sdk.create_calls[0]["auth_token"] == "tk_FROM_ENV" + + +async def test_env_project_id_is_forwarded_when_unset( + fake_sdk: _FakeSandboxFactory, monkeypatch: pytest.MonkeyPatch +) -> None: + """TENKI_PROJECT_ID env var populates project_id when the constructor arg is None.""" + monkeypatch.setenv("TENKI_PROJECT_ID", "proj-from-env") + tool = TenkiExecuteCodeTool(sandbox_name="env-proj") + await tool._run_code(code="pass") + assert fake_sdk.create_calls[0]["project_id"] == "proj-from-env" + + +async def test_env_workspace_id_is_forwarded_when_unset( + fake_sdk: _FakeSandboxFactory, monkeypatch: pytest.MonkeyPatch +) -> None: + """TENKI_WORKSPACE_ID env var populates workspace_id when the constructor arg is None.""" + monkeypatch.setenv("TENKI_WORKSPACE_ID", "ws-from-env") + tool = TenkiExecuteCodeTool(sandbox_name="env-ws") + await tool._run_code(code="pass") + assert fake_sdk.create_calls[0]["workspace_id"] == "ws-from-env" + + +async def test_constructor_project_id_wins_over_env( + fake_sdk: _FakeSandboxFactory, monkeypatch: pytest.MonkeyPatch +) -> None: + """Explicit constructor project_id takes precedence over TENKI_PROJECT_ID env.""" + monkeypatch.setenv("TENKI_PROJECT_ID", "proj-from-env") + monkeypatch.setenv("TENKI_WORKSPACE_ID", "ws-from-env") + tool = TenkiExecuteCodeTool( + sandbox_name="explicit-wins", + project_id="proj-explicit", + workspace_id="ws-explicit", + ) + await tool._run_code(code="pass") + call = fake_sdk.create_calls[0] + assert call["project_id"] == "proj-explicit" + assert call["workspace_id"] == "ws-explicit" + + +async def test_create_failure_returns_error_content(fake_sdk: _FakeSandboxFactory) -> None: + fake_sdk.raise_on_create = RuntimeError("resource_exhausted: no live node-agent") + tool = TenkiExecuteCodeTool(sandbox_name="fails") + contents = await tool._run_code(code="print('never runs')") + assert len(contents) == 1 + assert contents[0].type == "error" + assert "Failed to provision Tenki sandbox" in (contents[0].message or "") + assert "resource_exhausted" in (contents[0].error_details or "") + + +async def test_run_code_uses_python3_dash_c_with_timeout(fake_sdk: _FakeSandboxFactory) -> None: + tool = TenkiExecuteCodeTool(sandbox_name="argshape", exec_timeout_seconds=45) + await tool._run_code(code="print(1 + 1)") + + assert fake_sdk.last_sandbox is not None + args, kwargs = fake_sdk.last_sandbox.exec_calls[0] + assert args == ("python3", "-c", "print(1 + 1)") + assert kwargs == {"timeout": 45} + + +async def test_stdout_only_produces_single_text_content(fake_sdk: _FakeSandboxFactory) -> None: + tool = TenkiExecuteCodeTool(sandbox_name="stdout") + await tool._run_code(code="init") + assert fake_sdk.last_sandbox is not None + fake_sdk.last_sandbox.script_result = _FakeExecResult(stdout_text="hello\n", exit_code=0) + contents = await tool._run_code(code="print('hello')") + assert len(contents) == 1 + assert contents[0].type == "text" + assert contents[0].text == "hello\n" + + +async def test_non_zero_exit_produces_error_content_with_stderr(fake_sdk: _FakeSandboxFactory) -> None: + tool = TenkiExecuteCodeTool(sandbox_name="failing") + await tool._run_code(code="init") + assert fake_sdk.last_sandbox is not None + fake_sdk.last_sandbox.script_result = _FakeExecResult(stdout_text="", stderr_text="Traceback...\n", exit_code=1) + contents = await tool._run_code(code="raise ValueError('boom')") + error_contents = [c for c in contents if c.type == "error"] + assert len(error_contents) == 1 + message = error_contents[0].message or "" + assert "status 1" in message + # stderr must be inlined into the message so LLMs that under-weight + # `error_details` still see the traceback in the primary field. + assert "Traceback" in message + assert "Traceback" in (error_contents[0].error_details or "") + + +async def test_syntaxerror_appends_recovery_hint(fake_sdk: _FakeSandboxFactory) -> None: + """SyntaxError stderr must trigger the recovery hint in the message field.""" + tool = TenkiExecuteCodeTool(sandbox_name="syntax") + await tool._run_code(code="init") + assert fake_sdk.last_sandbox is not None + fake_sdk.last_sandbox.script_result = _FakeExecResult( + stdout_text="", + stderr_text=( + 'File "", line 1\n' + " import os; with open('/tmp/x') as f: pass\n" + " ^^^^\n" + "SyntaxError: invalid syntax" + ), + exit_code=1, + ) + contents = await tool._run_code(code="import os; with open('/tmp/x') as f: pass") + error_content = next(c for c in contents if c.type == "error") + message = error_content.message or "" + # Hint should mention both known failure modes. + assert "Common causes" in message + assert "compound statement" in message + assert "\\n" in message # literal-newline pitfall reference + + +async def test_non_syntax_error_does_not_append_hint(fake_sdk: _FakeSandboxFactory) -> None: + """Non-SyntaxError stderr must NOT trigger the recovery hint (no misleading advice).""" + tool = TenkiExecuteCodeTool(sandbox_name="runtime") + await tool._run_code(code="init") + assert fake_sdk.last_sandbox is not None + fake_sdk.last_sandbox.script_result = _FakeExecResult( + stdout_text="", + stderr_text=( + 'Traceback (most recent call last):\n File "", line 1, in \nValueError: bad value' + ), + exit_code=1, + ) + contents = await tool._run_code(code="raise ValueError('bad value')") + error_content = next(c for c in contents if c.type == "error") + message = error_content.message or "" + assert "ValueError" in message + assert "Common causes" not in message + + +async def test_error_message_truncates_long_stderr(fake_sdk: _FakeSandboxFactory) -> None: + """Very long stderr must be truncated in the message so it doesn't blow the field.""" + tool = TenkiExecuteCodeTool(sandbox_name="failing-long") + await tool._run_code(code="init") + assert fake_sdk.last_sandbox is not None + long_stderr = "err " * 500 # 2000 chars total + fake_sdk.last_sandbox.script_result = _FakeExecResult(stdout_text="", stderr_text=long_stderr, exit_code=1) + contents = await tool._run_code(code="raise Exception('x')") + error_contents = [c for c in contents if c.type == "error"] + assert len(error_contents) == 1 + message = error_contents[0].message or "" + # message field capped: status prefix + up to 500 chars of stderr + minor formatting. + # Verify total is well below the full stderr length so the LLM's context isn't flooded. + assert len(message) < 700 + # Full stderr still available via error_details for callers that want it. + assert len(error_contents[0].error_details or "") == len(long_stderr) + + +async def test_stderr_without_error_becomes_text_content(fake_sdk: _FakeSandboxFactory) -> None: + """Warnings to stderr should still surface to the model, matching Hyperlight.""" + tool = TenkiExecuteCodeTool(sandbox_name="warn") + await tool._run_code(code="init") + assert fake_sdk.last_sandbox is not None + fake_sdk.last_sandbox.script_result = _FakeExecResult( + stdout_text="", stderr_text="DeprecationWarning: foo\n", exit_code=0 + ) + contents = await tool._run_code(code="import warnings; warnings.warn('foo')") + assert any(c.type == "text" and "DeprecationWarning" in (c.text or "") for c in contents) + assert not any(c.type == "error" for c in contents) + + +async def test_empty_output_produces_placeholder_text(fake_sdk: _FakeSandboxFactory) -> None: + tool = TenkiExecuteCodeTool(sandbox_name="silent") + await tool._run_code(code="init") + assert fake_sdk.last_sandbox is not None + fake_sdk.last_sandbox.script_result = _FakeExecResult(stdout_text="", stderr_text="", exit_code=0) + contents = await tool._run_code(code="x = 1") + assert len(contents) == 1 + assert contents[0].type == "text" + assert "without output" in (contents[0].text or "") + + +async def test_cancelled_error_propagates(fake_sdk: _FakeSandboxFactory) -> None: + """asyncio.CancelledError must propagate so higher-level workflows can stop promptly.""" + import asyncio + + tool = TenkiExecuteCodeTool(sandbox_name="cancel") + await tool._run_code(code="init") + assert fake_sdk.last_sandbox is not None + + def handler(_args: tuple[Any, ...], _kwargs: dict[str, Any]) -> _FakeExecResult: + raise asyncio.CancelledError() + + fake_sdk.last_sandbox.script_handler = handler + with pytest.raises(asyncio.CancelledError): + await tool._run_code(code="print('x')") + + +async def test_exec_exception_returns_error_content(fake_sdk: _FakeSandboxFactory) -> None: + tool = TenkiExecuteCodeTool(sandbox_name="exec-err") + + def handler(_args: tuple[Any, ...], _kwargs: dict[str, Any]) -> _FakeExecResult: + raise RuntimeError("SDK exec transport error") + + await tool._run_code(code="init") + assert fake_sdk.last_sandbox is not None + fake_sdk.last_sandbox.script_handler = handler + + contents = await tool._run_code(code="print('x')") + assert len(contents) == 1 + assert contents[0].type == "error" + assert "Sandbox execution failed" in (contents[0].message or "") + + +async def test_tool_name_and_description() -> None: + tool = TenkiExecuteCodeTool() + assert tool.name == "execute_code" + assert "Tenki" in tool.description + + +async def test_provider_close_terminates_underlying_sandbox(fake_sdk: _FakeSandboxFactory) -> None: + provider = TenkiCodeActProvider(sandbox_name="prov") + await provider.execute_code_tool._run_code(code="print('x')") + assert fake_sdk.last_sandbox is not None + await provider.close() + assert fake_sdk.last_sandbox.closed is True + + +async def test_provider_async_context_manager_closes_sandbox(fake_sdk: _FakeSandboxFactory) -> None: + async with TenkiCodeActProvider(sandbox_name="prov-cm") as provider: + await provider.execute_code_tool._run_code(code="print('x')") + assert fake_sdk.last_sandbox is not None + assert fake_sdk.last_sandbox.closed is False + assert fake_sdk.last_sandbox is not None + assert fake_sdk.last_sandbox.closed is True + + +async def test_provider_exposes_execute_code_tool_before_run(fake_sdk: _FakeSandboxFactory) -> None: + provider = TenkiCodeActProvider(sandbox_name="ctx") + recorded: dict[str, Any] = {"tools_calls": [], "instructions_calls": []} + + class _StubContext: + def extend_tools(self, source_id: str, tools: list[Any]) -> None: + recorded["tools_calls"].append((source_id, tools)) + + def extend_instructions(self, source_id: str, instructions: str) -> None: + recorded["instructions_calls"].append((source_id, instructions)) + + # ``_StubContext`` is structurally compatible with ``SessionContext`` (implements + # extend_tools + extend_instructions) but isn't a nominal subclass. Cast to Any + # so strict type checkers accept the deliberate stub injection. + await provider.before_run(agent=None, session=None, context=cast(Any, _StubContext()), state={}) + + assert len(recorded["tools_calls"]) == 1 + tools_source_id, tools = recorded["tools_calls"][0] + assert tools_source_id == TenkiCodeActProvider.DEFAULT_SOURCE_ID + assert tools == [provider.execute_code_tool] + + assert len(recorded["instructions_calls"]) == 1 + instr_source_id, instructions = recorded["instructions_calls"][0] + assert instr_source_id == TenkiCodeActProvider.DEFAULT_SOURCE_ID + # The instructions must reinforce the print() requirement, the persistence + # rules, the "no Jupyter magic" rule, and the "newlines for compound + # statements" rule so that small models tool-call correctly against a + # fresh interpreter. + assert "print(" in instructions + assert "python3 -c" in instructions + assert "filesystem persists" in instructions + assert "subprocess" in instructions + # Match on '!command' to catch the specific Jupyter-magic footgun we hit + # in the field (small models emitted `!{sys.executable} -m pip install ...`). + assert "!command" in instructions + # Match on 'compound statements' to catch the semicolon-in-compound-stmt + # footgun (small models emitted `import os; with open(...) as f: ...`). + assert "compound statements" in instructions + # A concrete valid single-line `with` example nudges small models toward + # the shape that actually worked in the field. + assert "with open('/tmp/x.txt') as f: print(f.read())" in instructions + + +# --------------------------------------------------------------------------- +# Integration test — real Tenki service. Requires TENKI_API_KEY. Marked as +# integration so ``pytest -m "not integration"`` (the default suite) skips it. +# --------------------------------------------------------------------------- + + +def _tenki_integration_skip_reason() -> str | None: + if os.environ.get("SKIP_TENKI", "").lower() == "true": + return "SKIP_TENKI=true is set." + if importlib.util.find_spec("tenki_sandbox") is None: + return "tenki-sandbox is not installed." + if not os.environ.get("TENKI_API_KEY"): + return "TENKI_API_KEY is not set." + return None + + +skip_if_tenki_integration_disabled = pytest.mark.skipif( + _tenki_integration_skip_reason() is not None, + reason=_tenki_integration_skip_reason() or "unknown", +) + + +@pytest.mark.integration +@skip_if_tenki_integration_disabled +async def test_integration_execute_hello_world() -> None: + project_id = os.environ.get("TENKI_PROJECT_ID") + async with TenkiExecuteCodeTool( + sandbox_name=f"agent-framework-ci-{os.getpid()}", + project_id=project_id, + max_duration_seconds=300, + ) as tool: + contents = await tool._run_code(code="print('hello from tenki')") + text_contents = [c for c in contents if c.type == "text"] + assert any("hello from tenki" in (c.text or "") for c in text_contents) diff --git a/python/pyproject.toml b/python/pyproject.toml index 14fc6cbaba1..cc96a481e52 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -105,6 +105,7 @@ agent-framework-orchestrations = { workspace = true } agent-framework-purview = { workspace = true } agent-framework-redis = { workspace = true } agent-framework-azure-contentunderstanding = { workspace = true } +agent-framework-tenki = { workspace = true } agent-framework-tools = { workspace = true } [tool.ruff] diff --git a/python/uv.lock b/python/uv.lock index a0068be040b..f8b85461ecd 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -63,6 +63,7 @@ members = [ "agent-framework-orchestrations", "agent-framework-purview", "agent-framework-redis", + "agent-framework-tenki", "agent-framework-tools", ] constraints = [ @@ -926,6 +927,15 @@ requires-dist = [ { name = "redisvl", specifier = ">=0.11.0,<0.16" }, ] +[[package]] +name = "agent-framework-tenki" +version = "0.1.0a260720" +source = { editable = "packages/tenki" } +dependencies = [ + { name = "agent-framework-core" }, + { name = "tenki-sandbox" }, +] + [[package]] name = "agent-framework-tools" version = "1.0.0a260630" @@ -7977,6 +7987,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d7/c1/eb8f9debc45d3b7918a32ab756658a0904732f75e555402972246b0b8e71/tenacity-9.1.4-py3-none-any.whl", hash = "sha256:6095a360c919085f28c6527de529e76a06ad89b23659fa881ae0649b867a9d55", size = 28926, upload-time = "2026-02-07T10:45:32.24Z" }, ] +[[package]] +name = "tenki-sandbox" +version = "0.3.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "grpcio" }, + { name = "protobuf" }, + { name = "websocket-client" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c8/3e/319d553fb6bd266aed3f5e0ec5d9b1865ee7cb45388c9dc6dac1ee30a51f/tenki_sandbox-0.3.6.tar.gz", hash = "sha256:f4a6e0b7329a7fd42138d5eb582adb1356c084e6f6b3fe046530cbbcd426ffe2", size = 144662, upload-time = "2026-07-10T08:28:16.269Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/4f/860d1ba154200904b493ed4353e1994aa14627f407144d975b21e4cf51be/tenki_sandbox-0.3.6-py3-none-any.whl", hash = "sha256:324fe804a0b561ffa129e6b1b1874e0d7747ebcb566d976e8485c9dd45ef6853", size = 128658, upload-time = "2026-07-10T08:28:14.925Z" }, +] + [[package]] name = "termcolor" version = "2.4.0" @@ -8526,6 +8550,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/96/42/3e5985a0a7e57de470b320c6d6a1a67c844f6737a587f3d44dd13d1819e7/wcwidth-0.8.2-py3-none-any.whl", hash = "sha256:d63947694a0539a1d51e01eda7caf800c291020e6cdd7e28ad7b14dd33ad4f85", size = 323166, upload-time = "2026-06-29T18:11:09.888Z" }, ] +[[package]] +name = "websocket-client" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2c/41/aa4bf9664e4cda14c3b39865b12251e8e7d239f4cd0e3cc1b6c2ccde25c1/websocket_client-1.9.0.tar.gz", hash = "sha256:9e813624b6eb619999a97dc7958469217c3176312b3a16a4bd1bc7e08a46ec98", size = 70576, upload-time = "2025-10-07T21:16:36.495Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/34/db/b10e48aa8fff7407e67470363eac595018441cf32d5e1001567a7aeba5d2/websocket_client-1.9.0-py3-none-any.whl", hash = "sha256:af248a825037ef591efbf6ed20cc5faa03d3b47b9e5a2230a529eeee1c1fc3ef", size = 82616, upload-time = "2025-10-07T21:16:34.951Z" }, +] + [[package]] name = "websockets" version = "15.0.1" From 02d496f5109cb95f9a25bc5b74d23ecd6abf5ea1 Mon Sep 17 00:00:00 2001 From: Patricio Leonel Filice Date: Wed, 22 Jul 2026 18:27:38 -0300 Subject: [PATCH 2/5] Python: fix: address review feedback for agent-framework-tenki - Retry sandbox resume within a 120s poll budget: USER_SHUTDOWN pause snapshots are linked asynchronously (~60s) and the server can revert an accepted resume, so single-shot resume failed live - Keep run-scoped provider state JSON-serializable (store sandbox name in session state; live tool handles stay on the provider) - Handle CommandResult diagnostics (ok/signal/reason/errno) directly and keep the sandbox handle when close/refresh fails so it can retry - Run tool close under a single lock in a worker thread so a close during an in-flight reconcile cannot block the event loop - Make provider.close() retryable: keep failed terminates in the live-tool set instead of dropping them up front - Type __aexit__ signatures to match other resource-backed providers - Pin tenki-sandbox>=0.4.0,<0.5, finite max_duration default (900s), document pause/terminate asymmetry on expiry - Correct startup-latency docs to measured ~2s (was 10-30s) and clarify run-scoped vs standalone sandbox lifetime in the README - Expand tests to 58 unit + 4 integration (resume retry, server revert, close/run serialization, retryable provider close, cancellation teardown, terminal-state re-provision); wire tenki into integration/merge CI workflows Co-Authored-By: Claude Opus 4.7 --- .../workflows/python-integration-tests.yml | 7 +- .github/workflows/python-merge-tests.yml | 8 +- python/PACKAGE_STATUS.md | 1 + python/packages/tenki/README.md | 91 +- .../_execute_code_tool.py | 398 ++++-- .../tenki/agent_framework_tenki/_provider.py | 149 +- python/packages/tenki/pyproject.toml | 4 +- .../tenki/tests/tenki/test_tenki_codeact.py | 1260 ++++++++++++----- .../context_providers/code_act/README.md | 18 +- .../code_act/tenki_code_act.py | 175 +++ python/uv.lock | 14 +- 11 files changed, 1616 insertions(+), 509 deletions(-) create mode 100644 python/samples/02-agents/context_providers/code_act/tenki_code_act.py diff --git a/.github/workflows/python-integration-tests.yml b/.github/workflows/python-integration-tests.yml index fdc587aa59d..3710c4ee157 100644 --- a/.github/workflows/python-integration-tests.yml +++ b/.github/workflows/python-integration-tests.yml @@ -164,7 +164,7 @@ jobs: path: ./python/pytest.xml if-no-files-found: ignore - # Misc integration tests (Anthropic, Hyperlight, Ollama, MCP) + # Misc integration tests (Anthropic, Hyperlight, Ollama, MCP, Tenki) python-tests-misc-integration: name: Python Integration Tests - Misc runs-on: ubuntu-latest @@ -176,6 +176,8 @@ jobs: LOCAL_MCP_URL: ${{ vars.LOCAL_MCP__URL }} OLLAMA_MODEL: qwen2.5:1.5b OLLAMA_EMBEDDING_MODEL: nomic-embed-text + TENKI_API_KEY: ${{ secrets.TENKI_API_KEY }} + TENKI_PROJECT_ID: ${{ vars.TENKI_PROJECT_ID }} defaults: run: working-directory: python @@ -234,12 +236,13 @@ jobs: fallback_url: ${{ env.LOCAL_MCP_URL }} - name: Prefer local MCP URL when available run: echo "LOCAL_MCP_URL=${{ steps.local-mcp.outputs.effective_url }}" >> "$GITHUB_ENV" - - name: Test with pytest (Anthropic, Hyperlight, Ollama, MCP integration) + - name: Test with pytest (Anthropic, Hyperlight, Ollama, MCP, Tenki integration) run: > uv run pytest --import-mode=importlib packages/anthropic/tests packages/hyperlight/tests packages/ollama/tests + packages/tenki/tests packages/core/tests/core/test_mcp.py -m integration -n logical --dist worksteal diff --git a/.github/workflows/python-merge-tests.yml b/.github/workflows/python-merge-tests.yml index db391fa1acb..f152e20c381 100644 --- a/.github/workflows/python-merge-tests.yml +++ b/.github/workflows/python-merge-tests.yml @@ -69,6 +69,7 @@ jobs: - 'python/packages/anthropic/**' - 'python/packages/hyperlight/**' - 'python/packages/ollama/**' + - 'python/packages/tenki/**' - 'python/packages/core/agent_framework/_mcp.py' - 'python/packages/core/tests/core/test_mcp.py' - 'python/scripts/local_mcp_streamable_http_server.py' @@ -265,7 +266,7 @@ jobs: path: ./python/pytest.xml if-no-files-found: ignore - # Misc integration tests (Anthropic, Ollama, MCP) + # Misc integration tests (Anthropic, Ollama, MCP, Tenki) python-tests-misc-integration: name: Python Tests - Misc Integration needs: paths-filter @@ -283,6 +284,8 @@ jobs: LOCAL_MCP_URL: ${{ vars.LOCAL_MCP__URL }} OLLAMA_MODEL: qwen2.5:1.5b OLLAMA_EMBEDDING_MODEL: nomic-embed-text + TENKI_API_KEY: ${{ secrets.TENKI_API_KEY }} + TENKI_PROJECT_ID: ${{ vars.TENKI_PROJECT_ID }} defaults: run: working-directory: python @@ -338,12 +341,13 @@ jobs: fallback_url: ${{ env.LOCAL_MCP_URL }} - name: Prefer local MCP URL when available run: echo "LOCAL_MCP_URL=${{ steps.local-mcp.outputs.effective_url }}" >> "$GITHUB_ENV" - - name: Test with pytest (Anthropic, Hyperlight, Ollama, MCP integration) + - name: Test with pytest (Anthropic, Hyperlight, Ollama, MCP, Tenki integration) run: > uv run pytest --import-mode=importlib packages/anthropic/tests packages/hyperlight/tests packages/ollama/tests + packages/tenki/tests packages/core/tests/core/test_mcp.py -m integration -n logical --dist worksteal diff --git a/python/PACKAGE_STATUS.md b/python/PACKAGE_STATUS.md index 82416d26ad2..8c4f4210d19 100644 --- a/python/PACKAGE_STATUS.md +++ b/python/PACKAGE_STATUS.md @@ -47,6 +47,7 @@ Status is grouped into these buckets: | `agent-framework-orchestrations` | `python/packages/orchestrations` | `released` | | `agent-framework-purview` | `python/packages/purview` | `beta` | | `agent-framework-redis` | `python/packages/redis` | `beta` | +| `agent-framework-tenki` | `python/packages/tenki` | `alpha` | ## Deprecated / removed packages diff --git a/python/packages/tenki/README.md b/python/packages/tenki/README.md index 3608c92c40b..9914fc5b705 100644 --- a/python/packages/tenki/README.md +++ b/python/packages/tenki/README.md @@ -2,6 +2,10 @@ [Tenki Sandbox](https://tenki.cloud) integration for Microsoft Agent Framework. +> [!WARNING] +> This package is in **alpha**. APIs may change without notice. It is not part of +> `agent-framework[all]` yet; install it explicitly with `--pre`. + ## Installation ```bash @@ -19,17 +23,17 @@ export TENKI_API_KEY="tk_..." ### Context provider (recommended) -Use `TenkiCodeActProvider` to inject an `execute_code` tool into every agent run. Each -run shares the same underlying Tenki sandbox until the provider is closed. +Use `TenkiCodeActProvider` to inject an `execute_code` tool into every agent run. +**Each agent run gets its own fresh sandbox**, which is terminated automatically +when the run completes — state does not leak across runs. ```python from agent_framework import Agent -from agent_framework.openai import OpenAIChatClient from agent_framework_tenki import TenkiCodeActProvider async with TenkiCodeActProvider() as codeact: agent = Agent( - client=OpenAIChatClient(), + client=client, # any agent-framework chat client context_providers=[codeact], ) result = await agent.run("Compute the 42nd Fibonacci number.") @@ -37,35 +41,35 @@ async with TenkiCodeActProvider() as codeact: ### Standalone tool -Use `TenkiExecuteCodeTool` directly when you want full control over how the tool is -attached to the agent. +Use `TenkiExecuteCodeTool` directly when you want to share a single sandbox +across many agent runs (bypassing the per-run isolation of the provider): ```python from agent_framework import Agent -from agent_framework.openai import OpenAIChatClient from agent_framework_tenki import TenkiExecuteCodeTool async with TenkiExecuteCodeTool() as execute_code: agent = Agent( - client=OpenAIChatClient(), + client=client, # any agent-framework chat client tools=[execute_code], ) result = await agent.run("Print the SHA-256 of 'hello world'.") ``` Remember to `close()` (or use `async with`) so the sandbox is terminated when you're -done — otherwise it lives until your Tenki workspace timeout expires. +done — otherwise it keeps running (and billing) until Tenki's `max_duration` or idle +policies stop it. ## Configuration | Kwarg | Default | Description | |---|---|---| -| `sandbox_name` | `agent-framework-<8-hex>` | Sandbox identifier used when creating the sandbox on Tenki. | +| `sandbox_name` | `agent-framework-<8-hex>` | Literal sandbox identifier for the standalone tool; **prefix** for run-scoped names when used through `TenkiCodeActProvider` (each run gets `-<8-hex>`). | | `api_key` | `os.environ.get("TENKI_API_KEY")` | Overrides the environment variable. | | `image` | Tenki default | Custom base image identifier. | | `project_id` / `workspace_id` | `os.environ.get("TENKI_PROJECT_ID")` / `os.environ.get("TENKI_WORKSPACE_ID")` | Required when your API key has access to multiple projects. Constructor args override the env vars. | | `cpu_cores` / `memory_mb` / `disk_size_gb` | Tenki defaults | Optional resource overrides. | -| `max_duration_seconds` | `None` | Server-side cost kill-switch. Recommended for production agent loops. | +| `max_duration_seconds` | `900` (15 min) | Server-side duration cap. On expiry, project/workspace-scoped sandboxes are **paused** and unscoped ones are **terminated**; in both cases compute billing stops, including when the parent process crashed before calling `close()`. Pass a larger value for longer evals, or `None` to opt out (not recommended). | | `exec_timeout_seconds` | `60` | Per-`execute_code` invocation timeout in seconds. | | `extra_create_kwargs` | `{}` | Passed straight to `tenki_sandbox.Sandbox.create` for Tenki-specific options — see the section below. | @@ -89,6 +93,8 @@ forwarded verbatim to `tenki_sandbox.Sandbox.create`. Common ones: Example: ```python +import os + async with TenkiExecuteCodeTool( extra_create_kwargs={ "clone_repo_url": "https://github.com/myorg/myrepo", @@ -105,14 +111,13 @@ complete option list and semantics. ## Lifecycle -The sandbox is provisioned lazily on the first `execute_code` call and reused for every -subsequent call on the same tool or provider instance. Before each call the tool -reconciles remote sandbox state, so callers never need to track pause/terminate -transitions manually. Each call runs `python3 -c ` inside the sandbox, which -means: +The `execute_code` tool provisions a Tenki sandbox lazily on its first invocation. +Before each subsequent call it reconciles remote sandbox state, resuming or +re-provisioning as described below. Each call runs `python3 -c +` inside the sandbox, which means: -- **Sandbox filesystem persists** across calls — files written to `/tmp` or the user - home in one call are visible in the next. +- **Sandbox filesystem persists** across calls within the same tool instance — + files written to `/tmp` or the user home in one call are visible in the next. - **Installed packages persist** — packages installed via pip or apt in one call are available in subsequent calls (subject to the sandbox's outbound network policy). See [Tenki's sandbox quick start](https://tenki.cloud/docs/sandbox/quick-start-sandbox) @@ -121,22 +126,52 @@ means: process, so variables defined in one call are not reachable in the next. Persist intermediate state through files or environment variables when a later call needs it. -- **Paused sandboxes auto-resume** — if the sandbox transitions to `PAUSED` between - calls (Tenki's server-side idle policies, `idle_timeout_minutes` supplied via - `extra_create_kwargs`, or an external `tenki sandbox pause`), the next - `execute_code` call transparently resumes it before running. Filesystem and - installed packages carry across the pause unchanged. +- **Paused sandboxes auto-resume** — if the sandbox transitions to `PAUSED` + between calls (Tenki's server-side idle policies, `max_duration_seconds` + expiring on a project/workspace-scoped sandbox, `idle_timeout_minutes` + supplied via `extra_create_kwargs`, or an external `tenki sandbox pause`), + the next `execute_code` call transparently resumes it and polls until it + reaches `RUNNING` before executing. The same applies to `USER_SHUTDOWN` + (the guest OS was shut down from inside the VM) — note that this resume can + take a minute or more, since Tenki captures the shutdown sandbox's disk + asynchronously and the resume waits for that capture to complete. Filesystem + and installed packages carry across the pause unchanged. - **Terminated sandboxes are replaced** — if the sandbox transitions to - `TERMINATING`/`TERMINATED` (workspace timeout, `max_duration_seconds`, or an - external `tenki sandbox terminate`), the next call provisions a fresh sandbox. + `TERMINATING`/`TERMINATED` (workspace timeout, `max_duration_seconds` + expiring on an unscoped sandbox, or an external `tenki sandbox terminate`), + the next call provisions a fresh sandbox. Filesystem and installed packages from the previous sandbox are **not** carried over — snapshot the sandbox (via `extra_create_kwargs={"snapshot_id": ...}` on a new tool) if you need to preserve state across a termination. +### Provider vs standalone lifetime + +- **`TenkiCodeActProvider` mints a fresh tool per agent run** (`before_run` → + `create_run_tool()`), and terminates its sandbox in `after_run`. Within a + single run, every `execute_code` call in the agentic loop shares that run's + sandbox — files written by one call are visible to the next. Two agent + runs never share filesystem state, secrets, or a Python interpreter through + the tool. Every run that executes code pays the sandbox-provisioning cost + (a few seconds of startup latency — ~2s measured with the default image — + plus Tenki credits for the run's duration); a run where the model never + calls `execute_code` provisions no sandbox at all. +- **`TenkiExecuteCodeTool` used standalone** reuses the same sandbox across + every agent run until you `close()` it. Cheaper (one provision) and stateful + (files persist across runs) — appropriate when isolation between runs is not + required. + +The provider's per-run scoping suits **one-shot task runs**: execute a task, +return the answer, discard the environment. In a **multi-turn conversation** +(several `agent.run()` calls on one session), each message starts from a blank +filesystem — files and installed packages from earlier messages are gone, and +each code-executing message pays a fresh provision. If state should carry +across the messages of a conversation, use the standalone tool (one shared +sandbox, no per-run isolation). + Call `close()` on the tool or provider (or use `async with`) to terminate the sandbox -and release the underlying microVM. Different agents that share a single -`TenkiCodeActProvider` share the same sandbox — create a separate provider per agent -when isolation matters. +and release the underlying microVM. `close()` preserves the sandbox handle if the +terminate call itself fails, so a transient error does not leak a running microVM — +the caller can retry. ## Notes diff --git a/python/packages/tenki/agent_framework_tenki/_execute_code_tool.py b/python/packages/tenki/agent_framework_tenki/_execute_code_tool.py index 4702f3d3f9c..e16f6090aa9 100644 --- a/python/packages/tenki/agent_framework_tenki/_execute_code_tool.py +++ b/python/packages/tenki/agent_framework_tenki/_execute_code_tool.py @@ -3,16 +3,16 @@ from __future__ import annotations import asyncio -import contextlib import logging import os import threading +import time import uuid -from typing import Any, ClassVar +from typing import Any, ClassVar, cast from agent_framework import Content, FunctionTool from agent_framework._tools import ApprovalMode -from tenki_sandbox import Sandbox +from tenki_sandbox import CommandResult, Sandbox logger = logging.getLogger(__name__) @@ -41,16 +41,35 @@ "required": ["code"], } +_DEFAULT_SANDBOX_NAME_PREFIX = "agent-framework" -def _default_sandbox_name() -> str: - """Return a unique, source-attributable sandbox name. +# Server-side duration cap applied when the caller does not pass an explicit +# ``max_duration_seconds``. Without it, a crashed process or cancelled task +# would leave the sandbox running until Tenki's workspace policies stop it. +# On expiry Tenki pauses project/workspace-scoped sandboxes (compute stops, +# disk state is retained) and terminates unscoped ones. +_DEFAULT_MAX_DURATION_SECONDS = 60 * 15 + +# Poll-until-RUNNING budget for the reconcile loop (resume + transitional +# states). Sized for the slowest observed path: a ``USER_SHUTDOWN`` resume +# must first wait out the server's async rootfs capture (resume is rejected +# with "session has no pause snapshot" / "pause snapshot is not ready" until +# it completes — ~60s measured live), then cold-boot from that snapshot. +_RESUME_POLL_TIMEOUT_SECONDS = 120.0 +_RESUME_POLL_INTERVAL_SECONDS = 0.5 + +# State groupings (SDK returns strings). ``PAUSED`` and ``USER_SHUTDOWN`` are +# both stopped-but-resumable server-side: filesystem state is retained and +# ``resume()`` is a legal transition. Only ``TERMINATED`` is truly terminal; +# ``TERMINATING`` can no longer be resumed, so both are grouped for re-provision. +_RUNNABLE_STATE = "RUNNING" +_RESUMABLE_STATES = frozenset({"PAUSED", "USER_SHUTDOWN"}) +_TERMINAL_STATES = frozenset({"TERMINATING", "TERMINATED"}) - The ``agent-framework-`` prefix lets operators inspecting the Tenki dashboard or the - ``tenki sandbox list`` CLI attribute the sandbox back to Agent Framework — matching - the client-attributable naming already used elsewhere in the workspace (for example - ``WorkflowBuilder-`` in ``agent-framework-durabletask``). - """ - return f"agent-framework-{uuid.uuid4().hex[:8]}" + +def _generate_sandbox_name(prefix: str) -> str: + """Return ``-<8-hex>``. Used by both standalone and run-scoped tools.""" + return f"{prefix}-{uuid.uuid4().hex[:8]}" class TenkiExecuteCodeTool(FunctionTool): @@ -61,35 +80,36 @@ class TenkiExecuteCodeTool(FunctionTool): `Tenki Sandbox quick start `_ to create a workspace and generate a key, then export it as ``TENKI_API_KEY``. - Lifecycle: the tool lazily provisions a sandbox on the first ``_run_code`` invocation - and reuses the same sandbox for every subsequent call within the tool's lifetime. - Before each call the tool reconciles remote sandbox state — a sandbox that has - transitioned to ``PAUSED`` (Tenki server-side idle policy, ``idle_timeout_minutes`` - from ``extra_create_kwargs``, or an external ``tenki sandbox pause``) is - transparently resumed, and a sandbox that has transitioned to ``TERMINATING`` or - ``TERMINATED`` (workspace timeout, ``max_duration_seconds``, or an external - ``tenki sandbox terminate``) is replaced by a fresh provision. Callers therefore - never have to track pause/terminate transitions manually. Each call runs + Lifecycle when used standalone: the tool lazily provisions a sandbox on the first + ``execute_code`` invocation and reuses the same sandbox for every subsequent call. + Before each call the tool reconciles remote sandbox state — a sandbox stopped in + ``PAUSED`` or ``USER_SHUTDOWN`` (both resumable, filesystem retained) is + transparently resumed and polled until ``RUNNING``; a sandbox in ``TERMINATING`` + or ``TERMINATED`` is replaced by a fresh provision. A transient ``refresh()`` + failure surfaces as a ``RuntimeError`` without dropping the handle, so the next + call retries the same sandbox rather than leaking it. Each call runs ``python3 -c `` inside the sandbox, so the sandbox filesystem and installed packages persist across calls but the Python interpreter state does not — - variables defined in one call are not reachable in the next. Filesystem and - installed packages are also lost when the sandbox is re-provisioned after a - termination; use ``extra_create_kwargs={"snapshot_id": ...}`` on a new tool if you - need to preserve state across that boundary. Persist intermediate state through - files or environment variables when a later call needs it. Call :meth:`close` (or - use the tool via ``async with``) to terminate the sandbox and release the - underlying microVM. A new sandbox is created on the next call if the tool is - reused after being closed. + variables defined in one call are not reachable in the next. Persist intermediate + state through files or environment variables when a later call needs it. Call + `close` (or use the tool via ``async with``) to terminate the sandbox and release + the underlying microVM. + + Lifecycle when used through `TenkiCodeActProvider`: the provider mints a + fresh run-scoped tool per agent run via `create_run_tool`, and the run tool + provisions its own sandbox that is terminated by ``after_run``. Agent runs never + share filesystem/secret state through this tool. Args: approval_mode (ApprovalMode | None): Approval policy passed through to - :class:`agent_framework.FunctionTool`. Defaults to ``"never_require"``. + `agent_framework.FunctionTool`. Defaults to ``"never_require"``. api_key (str | None): Optional Tenki API key. When ``None`` the SDK reads - :envvar:`TENKI_API_KEY` from the environment. - sandbox_name (str | None): Optional sandbox identifier. When ``None`` the tool - generates ``agent-framework-<8-hex>`` — matching the naming convention already - used by other Agent Framework packages that create externally visible - resources. + `TENKI_API_KEY` from the environment. + sandbox_name (str | None): Optional literal sandbox identifier. Tenki does + not enforce name uniqueness — names are display labels. When ``None`` + the tool generates ``agent-framework-<8-hex>`` and refreshes the suffix + on every re-provision so successive sandboxes stay distinguishable in + the Tenki dashboard. When set, the literal name is used as-is. image (str | None): Optional Tenki base-image identifier. When ``None`` the Tenki service picks its default sandbox image (which ships ``python3``). project_id (str | None): Optional Tenki project ID scoping the sandbox. @@ -100,28 +120,31 @@ class TenkiExecuteCodeTool(FunctionTool): memory_mb (int | None): Optional memory (MB). ``None`` uses the service default. disk_size_gb (int | None): Optional ephemeral disk (GB). ``None`` uses the service default. - max_duration_seconds (int | None): Optional cost-safety cap on sandbox - lifetime, enforced server-side. When ``None`` the sandbox lives until - explicitly terminated (or the workspace timeout applies). Recommended for - production use to avoid runaway sandboxes in agent loops. - exec_timeout_seconds (int): Per-``_run_code`` timeout in seconds. Defaults to 60. + max_duration_seconds (int | None): Server-side duration cap enforced by + Tenki. Defaults to 900 (15 minutes) so a crashed process or cancelled + task cannot leave a sandbox running indefinitely. On expiry Tenki + pauses project/workspace-scoped sandboxes (compute stops, disk state + retained) and terminates unscoped ones. Pass an explicit value for + longer evals; pass ``None`` to explicitly opt out (not recommended in + production). + exec_timeout_seconds (int): Per-``execute_code`` timeout in seconds. Defaults to 60. extra_create_kwargs (dict[str, Any] | None): Optional keyword arguments passed - straight to :meth:`tenki_sandbox.Sandbox.create` — for advanced knobs not + straight to `tenki_sandbox.Sandbox.create` — for advanced knobs not surfaced individually (e.g. ``snapshot_id``, ``allow_inbound``, ``allow_outbound``). Notes: * In-sandbox tool callbacks are not supported — code executing inside the sandbox - cannot invoke host-side :class:`~agent_framework.FunctionTool` instances (the + cannot invoke host-side `agent_framework.FunctionTool` instances (the Tenki SDK does not expose a callback bridge). - * File mounts and outbound network allow-lists are not modeled by this package. - Bake dependencies and files into a custom Tenki image, or configure them - through ``extra_create_kwargs``. - * The sandbox lifecycle is *reuse-per-tool*, not per-agent-run. If the tool is - shared across many agent runs, filesystem state from run N is visible to run - N+1 (same sandbox), though each individual ``execute_code`` invocation is a - fresh Python process. Create a fresh :class:`TenkiExecuteCodeTool` per agent - instance if isolation between agents matters. + * File mounts and outbound network allow-lists are not surfaced as first-class + kwargs; pass them through ``extra_create_kwargs``. + * Cancellation latency: ``execute_code`` runs ``sandbox.exec`` inside + ``asyncio.to_thread``. Python cannot cancel an in-flight thread, so a + cancel raised during exec propagates to the awaiter immediately but the + underlying sandbox call keeps running server-side until it completes + or hits ``exec_timeout_seconds``. Worst-case wall time from cancel to + full quiescence is therefore ``exec_timeout_seconds``. """ SUPPORTED_LANGUAGES: ClassVar[list[str]] = ["python"] @@ -138,9 +161,10 @@ def __init__( cpu_cores: int | None = None, memory_mb: int | None = None, disk_size_gb: int | None = None, - max_duration_seconds: int | None = None, + max_duration_seconds: int | None = _DEFAULT_MAX_DURATION_SECONDS, exec_timeout_seconds: int = 60, extra_create_kwargs: dict[str, Any] | None = None, + _sandbox_name_prefix: str | None = None, ) -> None: if exec_timeout_seconds < 1: raise ValueError("exec_timeout_seconds must be greater than or equal to 1.") @@ -154,14 +178,19 @@ def __init__( ) self._api_key = api_key - # An explicit ``sandbox_name`` is preserved as-is across the tool's lifetime, - # even across re-provisioning after termination — the user asked for that - # name specifically, so if Tenki rejects reuse we surface the error. An - # auto-generated name is refreshed on every re-provision to avoid - # ``ALREADY_EXISTS`` collisions when Tenki holds a name reservation after - # termination. - self._explicit_sandbox_name: str | None = sandbox_name - self._sandbox_name = sandbox_name if sandbox_name is not None else _default_sandbox_name() + # Two naming modes (names are display labels; Tenki does not enforce uniqueness): + # (1) explicit literal — ``sandbox_name="my-agent"`` on standalone use, passed + # verbatim and preserved across re-provision. + # (2) prefix-with-suffix — ``_sandbox_name_prefix`` set by `create_run_tool` + # (or the default prefix): every provision mints ``-<8-hex>`` so + # sandboxes stay distinguishable in the Tenki dashboard. + self._explicit_sandbox_name: str | None = sandbox_name if _sandbox_name_prefix is None else None + self._sandbox_name_prefix: str = _sandbox_name_prefix or _DEFAULT_SANDBOX_NAME_PREFIX + if self._explicit_sandbox_name is not None: + self._sandbox_name = self._explicit_sandbox_name + else: + self._sandbox_name = _generate_sandbox_name(self._sandbox_name_prefix) + self._image = image self._project_id = project_id self._workspace_id = workspace_id @@ -173,13 +202,13 @@ def __init__( self._extra_create_kwargs: dict[str, Any] = dict(extra_create_kwargs) if extra_create_kwargs else {} # A single sandbox is created lazily on first use and reused across calls. - # A lock guards the create/close race between concurrent ``_run_code`` invocations. + # A lock guards the create/close race between concurrent ``execute_code`` invocations. self._sandbox_lock = threading.Lock() self._sandbox: Sandbox | None = None @property def sandbox_name(self) -> str: - """The name of the underlying Tenki sandbox.""" + """The name of the current or next-provisioned Tenki sandbox.""" return self._sandbox_name @property @@ -187,6 +216,34 @@ def exec_timeout_seconds(self) -> int: """Per-invocation execution timeout in seconds.""" return self._exec_timeout_seconds + def create_run_tool(self) -> TenkiExecuteCodeTool: + """Return a fresh, run-scoped ``TenkiExecuteCodeTool`` with the same config. + + Called by `TenkiCodeActProvider.before_run` to create a per-agent-run + tool. The returned tool has a unique ``-<8-hex>`` sandbox name derived + from the user's explicit ``sandbox_name`` (used as prefix) or the default + ``agent-framework`` prefix. All other kwargs are copied verbatim. + """ + prefix = self._explicit_sandbox_name if self._explicit_sandbox_name is not None else self._sandbox_name_prefix + # ``FunctionTool.__init__`` stores ``approval_mode`` after applying its own + # ``or "never_require"`` fallback, so the runtime value is always a valid + # ``ApprovalMode``. Cast so the constructor's stricter ``ApprovalMode | None`` + # type annotation stays satisfied. + return TenkiExecuteCodeTool( + approval_mode=cast("ApprovalMode | None", self.approval_mode), + api_key=self._api_key, + image=self._image, + project_id=self._project_id, + workspace_id=self._workspace_id, + cpu_cores=self._cpu_cores, + memory_mb=self._memory_mb, + disk_size_gb=self._disk_size_gb, + max_duration_seconds=self._max_duration_seconds, + exec_timeout_seconds=self._exec_timeout_seconds, + extra_create_kwargs=dict(self._extra_create_kwargs), + _sandbox_name_prefix=prefix, + ) + def _build_create_kwargs(self) -> dict[str, Any]: kwargs: dict[str, Any] = {"name": self._sandbox_name} # Only forward optional values the caller explicitly set; the Tenki SDK applies @@ -194,6 +251,12 @@ def _build_create_kwargs(self) -> dict[str, Any]: # ``is not None`` (rather than ``or``) so that an explicit empty string on the # constructor still wins over the env var — the contract is "explicit args # override env vars when provided", not "truthy args override env vars". + # + # Env-var ownership: ``TENKI_API_KEY`` is also read by the SDK + # (``resolve_auth_token``); the duplicate read here preserves the + # explicit-arg-wins semantics above. ``TENKI_PROJECT_ID`` and + # ``TENKI_WORKSPACE_ID`` are **not** read by the SDK — this module + # owns them. api_key = self._api_key if self._api_key is not None else os.environ.get("TENKI_API_KEY") if api_key is not None: kwargs["auth_token"] = api_key @@ -202,9 +265,7 @@ def _build_create_kwargs(self) -> dict[str, Any]: project_id = self._project_id if self._project_id is not None else os.environ.get("TENKI_PROJECT_ID") if project_id is not None: kwargs["project_id"] = project_id - workspace_id = ( - self._workspace_id if self._workspace_id is not None else os.environ.get("TENKI_WORKSPACE_ID") - ) + workspace_id = self._workspace_id if self._workspace_id is not None else os.environ.get("TENKI_WORKSPACE_ID") if workspace_id is not None: kwargs["workspace_id"] = workspace_id if self._cpu_cores is not None: @@ -220,15 +281,19 @@ def _build_create_kwargs(self) -> dict[str, Any]: return kwargs def _ensure_sandbox_sync(self) -> Sandbox: - """Return a usable sandbox — provision on first use, resume if paused, re-provision if gone. - - Tenki sandboxes can transition to ``PAUSED`` between calls (server-side idle - policies, external ``tenki sandbox pause``) and to ``TERMINATED`` if the - workspace timeout or ``max_duration`` elapsed. This method reconciles the - remote state on every call so ``_run_code`` never hands the SDK a - ``sandbox.exec`` that would fail with ``sandbox is not RUNNING``. - - Called inside ``asyncio.to_thread``. + """Return a runnable sandbox — provision on first use, resume if stopped, re-provision if gone. + + Called inside ``asyncio.to_thread``. Reconciles remote state on every call: + + - ``TERMINATING`` / ``TERMINATED`` → drop the handle, provision a fresh + sandbox (auto-generated names refresh their suffix; explicit names are + preserved). + - any other non-``RUNNING`` state → `_wait_for_running_locked`, which + resumes stopped-but-resumable sandboxes (``PAUSED`` / ``USER_SHUTDOWN``) + and polls transitional states (``PAUSING`` / ``RESUMING`` / ``CREATING``) + until ``RUNNING``. + - ``refresh()`` raising → surface as ``RuntimeError`` without dropping the + handle. Next call retries the same sandbox rather than leaking it. """ with self._sandbox_lock: sandbox = self._sandbox @@ -238,36 +303,79 @@ def _ensure_sandbox_sync(self) -> Sandbox: try: sandbox.refresh() except Exception as exc: - # Remote state is unknowable — drop the stale handle and re-provision. - logger.debug("Dropping Tenki sandbox handle after refresh failure: %s", exc) + # Do not drop the handle — the sandbox may still be alive. + # Surface the error so the next call retries the same session + # rather than provisioning a duplicate. + raise RuntimeError(f"Failed to refresh Tenki sandbox state: {exc}") from exc + + if sandbox.state in _TERMINAL_STATES: + logger.debug("Tenki sandbox reached remote state=%s; re-provisioning", sandbox.state) + # The session can no longer be resumed or terminated again, but the + # SDK-owned control-plane channel is still open on our side. + self._close_owning_client_sync(sandbox) self._sandbox = None self._refresh_autogenerated_name_locked() return self._create_sandbox_locked() - - state = getattr(sandbox, "state", None) - if state == "PAUSED": + return self._wait_for_running_locked(sandbox) + + def _wait_for_running_locked(self, sandbox: Sandbox) -> Sandbox: + """Poll until the sandbox reaches ``RUNNING``, resuming it if stopped. Lock must be held. + + Resume attempts are retried within the poll budget. Two failure shapes + require this — both observed live: ``USER_SHUTDOWN`` links its pause + snapshot asynchronously (the rootfs capture runs *after* the state + flips), so the server rejects resume with FAILED_PRECONDITION ("session + has no pause snapshot" / "pause snapshot is not ready") until the + capture completes; and a dispatched resume that fails on the node is + reverted server-side to the source state (``RESUMING`` → + ``USER_SHUTDOWN``/``PAUSED``), which the loop sees as the sandbox + settling back and simply retries. Unknown states are treated as + transitional so an SDK upgrade that adds new states doesn't fail + closed. Raises ``RuntimeError`` on timeout so the caller sees a clear + error rather than a "sandbox is not RUNNING" exec failure. + + Concurrency: the sandbox lock is held for the full poll window (up to + `_RESUME_POLL_TIMEOUT_SECONDS`); concurrent ``execute_code`` calls block + on the lock rather than racing the resume. + """ + deadline = time.monotonic() + _RESUME_POLL_TIMEOUT_SECONDS + while True: + state = sandbox.state + if state == _RUNNABLE_STATE: + return sandbox + if state in _TERMINAL_STATES: + raise RuntimeError(f"Tenki sandbox reached state={state} while waiting for RUNNING.") + if state in _RESUMABLE_STATES: try: sandbox.resume() except Exception as exc: - raise RuntimeError(f"Failed to resume paused Tenki sandbox: {exc}") from exc - return sandbox - if state in {"TERMINATING", "TERMINATED"}: - logger.debug("Dropping Tenki sandbox handle after remote state=%s", state) - self._sandbox = None - self._refresh_autogenerated_name_locked() - return self._create_sandbox_locked() - return sandbox + if time.monotonic() >= deadline: + raise RuntimeError(f"Failed to resume Tenki sandbox from state={state}: {exc}") from exc + logger.debug("Tenki sandbox resume from state=%s failed; retrying: %s", state, exc) + else: + # Local state is now RESUMING; fall through to poll. If the + # server reverts the resume, the next refresh shows the + # resumable state again and the loop retries. + continue + if time.monotonic() >= deadline: + raise RuntimeError( + f"Tenki sandbox did not reach RUNNING within {_RESUME_POLL_TIMEOUT_SECONDS}s (last state={state})." + ) + time.sleep(_RESUME_POLL_INTERVAL_SECONDS) + try: + sandbox.refresh() + except Exception as exc: + raise RuntimeError(f"Failed to refresh Tenki sandbox state while polling: {exc}") from exc def _refresh_autogenerated_name_locked(self) -> None: - """Mint a fresh auto-generated name so re-provision never reuses the prior one. + """Mint a fresh suffix on re-provision for auto-generated names. - Tenki may hold the terminated sandbox's name (``ALREADY_EXISTS``); a fresh - 8-hex suffix avoids the collision. Explicit user-supplied names are - preserved — the caller chose that name deliberately, and any reuse - rejection surfaces as a ``RuntimeError`` they can act on. + Tenki does not enforce name uniqueness; the fresh suffix keeps + successive sandboxes distinguishable in the Tenki dashboard and logs. + An explicit user-supplied literal name is preserved as-is. """ if self._explicit_sandbox_name is None: - self._sandbox_name = _default_sandbox_name() + self._sandbox_name = _generate_sandbox_name(self._sandbox_name_prefix) def _create_sandbox_locked(self) -> Sandbox: """Provision a fresh sandbox. The sandbox lock must be held by the caller.""" @@ -284,7 +392,7 @@ async def _run_code(self, *, code: str) -> list[Content]: try: sandbox = await asyncio.to_thread(self._ensure_sandbox_sync) except RuntimeError as exc: - return [Content.from_error(message="Failed to provision Tenki sandbox", error_details=str(exc))] + return [Content.from_error(message="Failed to prepare Tenki sandbox", error_details=str(exc))] try: result = await asyncio.to_thread(sandbox.exec, "python3", "-c", code, timeout=self._exec_timeout_seconds) @@ -308,21 +416,32 @@ async def _run_code(self, *, code: str) -> list[Content]: "characters were intended.]" ) - def _build_contents(self, result: Any) -> list[Content]: - """Convert a Tenki exec result into a list of :class:`Content` values.""" - stdout = getattr(result, "stdout_text", "") or "" - stderr = getattr(result, "stderr_text", "") or "" - exit_code = int(getattr(result, "exit_code", 1)) + def _build_contents(self, result: CommandResult) -> list[Content]: + """Convert a Tenki ``CommandResult`` into a list of `Content` values. + + ``result.ok`` accounts for the process being killed by a signal + (SIGKILL, SIGTERM, …), which ``exit_code`` alone misses. ``signal``, + ``reason``, and ``errno`` are surfaced in the error message so operators + can distinguish timeouts, OOM kills, and spawn failures. + """ + stdout = result.stdout_text + stderr = result.stderr_text contents: list[Content] = [] if stdout: contents.append(Content.from_text(stdout)) - if exit_code != 0: - # Inline a truncated stderr into the message so downstream LLM - # consumers see the traceback in the primary field, not just in - # error_details — small models tend to under-weight secondary fields. - stderr_snippet = stderr.strip()[: self._ERROR_MESSAGE_STDERR_LIMIT] if stderr else "" - message = f"Code exited with status {exit_code}" + if not result.ok: + # Inline a truncated stderr into the message so LLM consumers see the + # traceback in the primary field, not just in error_details. + stderr_snippet = stderr.strip()[: self._ERROR_MESSAGE_STDERR_LIMIT] + failure_bits: list[str] = [f"status {result.exit_code}"] + if result.signal: + failure_bits.append(f"signal {result.signal}") + if result.reason: + failure_bits.append(f"reason {result.reason}") + if result.errno is not None: + failure_bits.append(f"errno {result.errno}") + message = f"Code exited with {', '.join(failure_bits)}" if stderr_snippet: message = f"{message}. stderr: {stderr_snippet}" if stderr and "SyntaxError" in stderr: @@ -342,23 +461,78 @@ def _build_contents(self, result: Any) -> list[Content]: return contents async def close(self) -> None: - """Terminate the underlying Tenki sandbox and release its resources. + """Terminate the underlying Tenki sandbox and its owning SDK client. + + Safe to call multiple times; a no-op if the sandbox was never created. + On terminate failure the handle is **preserved** so the caller can retry + — a swallowed failure would otherwise leak the microVM. The SDK-owned + `tenki_sandbox.Client` created by ``Sandbox.create`` (when + ``_owns_client=True``) is closed after a successful terminate to release + the gRPC channel. + """ + await asyncio.to_thread(self._close_sync) - Safe to call multiple times; a no-op if the sandbox was never created. After - close, a subsequent ``_run_code`` invocation lazily provisions a new sandbox. + def _close_sync(self) -> None: + """Sync body of `close`, run inside ``asyncio.to_thread``. + + Holds ``_sandbox_lock`` for the whole terminate so it never races + `_ensure_sandbox_sync`, and — like all lock acquisitions in this class — + only ever blocks a worker thread, never the event loop (a reconcile can + hold the lock for up to the resume-poll budget). """ with self._sandbox_lock: sandbox = self._sandbox + if sandbox is None: + return + + # Terminate first. If it fails, the raised error propagates and the + # handle is kept so the caller can retry — dropping it here would + # leak the sandbox. + sandbox.close_if_open() + self._sandbox = None - if sandbox is None: + # Close the owning Client (see `_close_owning_client_sync`). + self._close_owning_client_sync(sandbox) + + def _close_owning_client_sync(self, sandbox: Sandbox) -> None: + """Close the SDK-owned control-plane `tenki_sandbox.Client`. + + Called both after a successful ``close_if_open`` and from the + terminal-state re-provision branch in `_ensure_sandbox_sync`. The + SDK's own ``Sandbox.close()`` only closes the data-plane RPC, so + without this the gRPC control channel would stay open until the + process exits — a real leak under long-lived standalone tools that + re-provision on ``TERMINATED``. + + ``_owns_client`` / ``_client`` are private SDK attributes read via + ``getattr``; if Tenki renames them we log loudly rather than silently + leak. A public ``sandbox.close_client()`` would let us drop this. + Errors are logged and suppressed — callers must not block on channel + cleanup. + """ + if not getattr(sandbox, "_owns_client", False): + return + client = getattr(sandbox, "_client", None) + client_close = getattr(client, "close", None) if client is not None else None + if not callable(client_close): + logger.warning( + "Tenki sandbox %s reports _owns_client=True but no closable Client is " + "accessible; gRPC channel may leak. Tenki SDK internals may have changed.", + sandbox.id, + ) return - # SDK errors on shutdown are ignored — the sandbox is server-side managed and will - # be reaped by Tenki's own idle timeout even if our terminate call fails. - with contextlib.suppress(Exception): # pragma: no cover - defensive - await asyncio.to_thread(sandbox.close_if_open) + try: + client_close() + except Exception as exc: + logger.debug("Ignoring error closing Tenki Client: %s", exc) async def __aenter__(self) -> TenkiExecuteCodeTool: return self - async def __aexit__(self, exc_type: Any, exc: Any, tb: Any) -> None: + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: Any, + ) -> None: await self.close() diff --git a/python/packages/tenki/agent_framework_tenki/_provider.py b/python/packages/tenki/agent_framework_tenki/_provider.py index 3d7db27c459..e0827470673 100644 --- a/python/packages/tenki/agent_framework_tenki/_provider.py +++ b/python/packages/tenki/agent_framework_tenki/_provider.py @@ -2,6 +2,8 @@ from __future__ import annotations +import asyncio +import logging from typing import Any from agent_framework import AgentSession, ContextProvider, SessionContext @@ -9,6 +11,8 @@ from ._execute_code_tool import TenkiExecuteCodeTool +logger = logging.getLogger(__name__) + TENKI_CODEACT_INSTRUCTIONS = ( "You have access to `execute_code`, which runs Python inside a Tenki microVM " "sandbox. Each call runs the code as `python3 -c ` — only stdout and " @@ -34,15 +38,27 @@ class TenkiCodeActProvider(ContextProvider): - """Inject a Tenki-backed CodeAct surface using a provider-owned execute_code tool. - - On every agent run, the provider exposes its underlying :class:`TenkiExecuteCodeTool` - to the context so the model can run Python inside a Tenki sandbox. - - The sandbox is reused across runs of the same provider — its filesystem and installed - packages persist across calls, though each individual ``execute_code`` invocation is - a fresh Python process. Instantiate a new provider per agent instance when you need - isolation between agents. + """Inject a Tenki-backed CodeAct surface with a **run-scoped** execute_code tool. + + On every agent run, `before_run` mints a fresh `TenkiExecuteCodeTool` via + `TenkiExecuteCodeTool.create_run_tool` and exposes it to the model. + `after_run` terminates that run-scoped tool's sandbox so state does not leak + across runs. + + Sandbox naming: an explicit ``sandbox_name`` on the provider is used as the + **prefix** for run-scoped sandboxes (e.g. ``sandbox_name="analysis-agent"`` → + ``"analysis-agent-<8-hex>"`` per run). Without an explicit name, run-scoped + sandboxes get ``agent-framework-<8-hex>``. + + Cost implication: every agent run provisions and terminates a new Tenki microVM. + Provisioning adds a few seconds of startup latency (~2s measured with the + default image) and consumes credits for the run's duration. Use the standalone + `TenkiExecuteCodeTool` directly (bypassing the provider) if you need to reuse + a sandbox across runs. + + ``max_duration_seconds=None`` (the default) means "use the tool's own default" + (see ``TenkiExecuteCodeTool``); there is no way to opt out of a max duration + through the provider. """ DEFAULT_SOURCE_ID = "tenki_codeact" @@ -65,34 +81,77 @@ def __init__( extra_create_kwargs: dict[str, Any] | None = None, ) -> None: super().__init__(source_id) - self._execute_code_tool = TenkiExecuteCodeTool( - approval_mode=approval_mode, - api_key=api_key, - sandbox_name=sandbox_name, - image=image, - project_id=project_id, - workspace_id=workspace_id, - cpu_cores=cpu_cores, - memory_mb=memory_mb, - disk_size_gb=disk_size_gb, - max_duration_seconds=max_duration_seconds, - exec_timeout_seconds=exec_timeout_seconds, - extra_create_kwargs=extra_create_kwargs, - ) + # ``max_duration_seconds=None`` here means "use the tool's own default" + # (see ``_execute_code_tool._DEFAULT_MAX_DURATION_SECONDS``). + tool_kwargs: dict[str, Any] = { + "approval_mode": approval_mode, + "api_key": api_key, + "sandbox_name": sandbox_name, + "image": image, + "project_id": project_id, + "workspace_id": workspace_id, + "cpu_cores": cpu_cores, + "memory_mb": memory_mb, + "disk_size_gb": disk_size_gb, + "exec_timeout_seconds": exec_timeout_seconds, + "extra_create_kwargs": extra_create_kwargs, + } + if max_duration_seconds is not None: + tool_kwargs["max_duration_seconds"] = max_duration_seconds + # Base tool acts as a template config-holder; it never provisions a + # sandbox itself. Run-scoped copies are minted per agent run. + self._execute_code_tool = TenkiExecuteCodeTool(**tool_kwargs) + + # Live run-scoped tools keyed by sandbox name, so ``close`` can terminate + # any that leaked past ``after_run`` (e.g. mid-run exception). The + # contract is "no new runs after ``close()``" — callers running + # concurrent agent runs on one provider must let all runs finish before + # invoking ``close()``. + self._live_run_tools: dict[str, TenkiExecuteCodeTool] = {} @property def execute_code_tool(self) -> TenkiExecuteCodeTool: - """The underlying execute_code tool. Exposed for advanced integration.""" + """The base template tool. Advanced: use as a config template, not for direct exec.""" return self._execute_code_tool async def close(self) -> None: - """Terminate the underlying Tenki sandbox and release its resources.""" + """Terminate any lingering run-scoped sandboxes and the base template tool. + + Called when the provider goes out of scope (via ``async with`` or an + explicit ``close()``). Sandboxes for completed runs are already terminated + by ``after_run``; this cleans up any that leaked past that hook due to + an in-run exception. + """ + leaked = list(self._live_run_tools.items()) + # Terminate orphans in parallel — each ``close()`` is one terminate RPC + # (seconds of latency), and a crashed run that leaked multiple sandboxes + # would otherwise pay N * latency on provider teardown. Entries are + # removed only on success so a failed terminate stays retryable via a + # second ``close()`` — mirroring the handle-preserving contract of + # ``TenkiExecuteCodeTool.close`` and ``after_run``. + results = await asyncio.gather(*(tool.close() for _, tool in leaked), return_exceptions=True) + for (sandbox_name, _), result in zip(leaked, results, strict=True): + if isinstance(result, BaseException): + logger.warning( + "Failed to close orphaned run-scoped Tenki sandbox %s: %s", + sandbox_name, + result, + ) + else: + self._live_run_tools.pop(sandbox_name, None) + # Base template tool never provisions, so this is a no-op in practice — + # kept for symmetry with the standalone-tool ``async with`` pattern. await self._execute_code_tool.close() async def __aenter__(self) -> TenkiCodeActProvider: return self - async def __aexit__(self, exc_type: Any, exc: Any, tb: Any) -> None: + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: Any, + ) -> None: await self.close() async def before_run( @@ -103,6 +162,40 @@ async def before_run( context: SessionContext, state: dict[str, Any], ) -> None: - """Inject the execute_code tool and its usage instructions for this run.""" + """Inject a fresh run-scoped ``execute_code`` tool for this agent run.""" + run_tool = self._execute_code_tool.create_run_tool() + self._live_run_tools[run_tool.sandbox_name] = run_tool + # Provider state is serialized with the session, so store only the + # sandbox name; the live tool handle stays in ``_live_run_tools``. + state[self.source_id] = run_tool.sandbox_name context.extend_instructions(self.source_id, TENKI_CODEACT_INSTRUCTIONS) - context.extend_tools(self.source_id, [self._execute_code_tool]) + context.extend_tools(self.source_id, [run_tool]) + + async def after_run( + self, + *, + agent: Any, + session: AgentSession | None, + context: SessionContext, + state: dict[str, Any], + ) -> None: + """Terminate this run's sandbox so state cannot leak to the next run.""" + sandbox_name = state.pop(self.source_id, None) + if sandbox_name is None: + return + run_tool = self._live_run_tools.get(sandbox_name) + if run_tool is None: + return + try: + await run_tool.close() + except Exception as exc: + # Preserve the handle so ``provider.close()`` can retry; the + # sandbox's max_duration eventually stops billing as a backstop + # (unscoped sandboxes terminate; project/workspace-scoped ones pause). + logger.warning( + "Failed to terminate run-scoped Tenki sandbox %s: %s", + sandbox_name, + exc, + ) + else: + self._live_run_tools.pop(sandbox_name, None) diff --git a/python/packages/tenki/pyproject.toml b/python/packages/tenki/pyproject.toml index 979c586c85e..97794032f0e 100644 --- a/python/packages/tenki/pyproject.toml +++ b/python/packages/tenki/pyproject.toml @@ -4,7 +4,7 @@ description = "Tenki Sandbox CodeAct integrations for Microsoft Agent Framework. authors = [{ name = "Microsoft", email = "af-support@microsoft.com"}] readme = "README.md" requires-python = ">=3.10" -version = "0.1.0a260720" +version = "1.0.0a260722" license-files = ["LICENSE"] urls.homepage = "https://aka.ms/agent-framework" urls.source = "https://github.com/microsoft/agent-framework/tree/main/python" @@ -23,7 +23,7 @@ classifiers = [ ] dependencies = [ "agent-framework-core>=1.11.0,<2", - "tenki-sandbox>=0.1.0,<1", + "tenki-sandbox>=0.4.0,<0.5", ] [tool.uv] diff --git a/python/packages/tenki/tests/tenki/test_tenki_codeact.py b/python/packages/tenki/tests/tenki/test_tenki_codeact.py index b233377faab..2a2fbfbce21 100644 --- a/python/packages/tenki/tests/tenki/test_tenki_codeact.py +++ b/python/packages/tenki/tests/tenki/test_tenki_codeact.py @@ -2,19 +2,16 @@ from __future__ import annotations +import asyncio import importlib.util import os import sys +import threading from types import SimpleNamespace from typing import Any, cast import pytest -if sys.platform == "win32": # pragma: no cover - platform-dependent - # The integration test relies on POSIX-style commands inside the sandbox. Skip the - # whole module on Windows to match the sibling ``agent-framework-hyperlight`` tests. - pytest.skip("Tenki tests use POSIX-style shell invocations.", allow_module_level=True) - # ``tenki-sandbox`` is a hard dep of this package, so in practice it's always present. # Skip the module if it happens to be missing (e.g. dev checkout without a full install) # instead of erroring pytest collection. @@ -34,13 +31,44 @@ class _FakeExecResult(SimpleNamespace): - def __init__(self, *, stdout_text: str = "", stderr_text: str = "", exit_code: int = 0) -> None: - super().__init__(stdout_text=stdout_text, stderr_text=stderr_text, exit_code=exit_code) + """Mirror ``tenki_sandbox.models.CommandResult`` — including the ``ok`` property.""" + + def __init__( + self, + *, + stdout_text: str = "", + stderr_text: str = "", + exit_code: int = 0, + signal: str | None = None, + reason: str | None = None, + errno: int | None = None, + ) -> None: + super().__init__( + stdout_text=stdout_text, + stderr_text=stderr_text, + exit_code=exit_code, + signal=signal, + reason=reason, + errno=errno, + ) + + @property + def ok(self) -> bool: + return self.exit_code == 0 and not self.signal + + +class _FakeClient: + def __init__(self) -> None: + self.closed: bool = False + + def close(self) -> None: + self.closed = True class _FakeSandbox: def __init__(self, *, name: str) -> None: - self.name = name + self.id: str = f"sandbox-{name}" + self.name: str = name self.closed: bool = False self.exec_calls: list[tuple[tuple[Any, ...], dict[str, Any]]] = [] self.script_result: _FakeExecResult = _FakeExecResult(stdout_text="ok\n") @@ -51,10 +79,23 @@ def __init__(self, *, name: str) -> None: self.resume_calls: int = 0 # When set, ``refresh()`` transitions ``state`` to this value on the next call. self.refresh_transitions_to: str | None = None - # When set, ``refresh()`` raises this exception on the next call. + # When set, ``refresh()`` walks through the sequence one value per call. + # Terminal value stays after the sequence is exhausted. + self.refresh_state_sequence: list[str] | None = None + # When set, ``refresh()`` / ``close_if_open()`` raise this once. self.refresh_raises: BaseException | None = None - # When set, ``resume()`` raises this exception on the next call. + self.close_raises: BaseException | None = None + # ``resume()`` failure knobs: ``resume_raises`` raises on EVERY call + # (persistent failure), ``resume_raises_once`` raises once then clears + # (transient failure — e.g. USER_SHUTDOWN's snapshot not yet linked). self.resume_raises: BaseException | None = None + self.resume_raises_once: BaseException | None = None + # ``resume()`` in the real SDK returns while state may still be ``RESUMING``. + # Fake matches: by default resume leaves state at ``RUNNING`` (tests can override). + self.resume_leaves_state_as: str = "RUNNING" + # SDK-owned client fields that ``Sandbox.create`` sets when it builds a Client. + self._owns_client: bool = True + self._client: _FakeClient = _FakeClient() def exec(self, *args: Any, **kwargs: Any) -> _FakeExecResult: self.exec_calls.append((args, kwargs)) @@ -67,18 +108,25 @@ def refresh(self) -> None: if self.refresh_raises is not None: exc, self.refresh_raises = self.refresh_raises, None raise exc - if self.refresh_transitions_to is not None: + if self.refresh_state_sequence: + self.state = self.refresh_state_sequence.pop(0) + elif self.refresh_transitions_to is not None: self.state = self.refresh_transitions_to self.refresh_transitions_to = None def resume(self) -> None: self.resume_calls += 1 - if self.resume_raises is not None: - exc, self.resume_raises = self.resume_raises, None + if self.resume_raises_once is not None: + exc, self.resume_raises_once = self.resume_raises_once, None raise exc - self.state = "RUNNING" + if self.resume_raises is not None: + raise self.resume_raises + self.state = self.resume_leaves_state_as def close_if_open(self) -> None: + if self.close_raises is not None: + exc, self.close_raises = self.close_raises, None + raise exc self.closed = True @@ -105,27 +153,35 @@ def create(self, **kwargs: Any) -> _FakeSandbox: @pytest.fixture def fake_sdk(monkeypatch: pytest.MonkeyPatch) -> _FakeSandboxFactory: - """Replace the Tenki ``Sandbox`` class in the executor module with an in-memory fake.""" + """Replace the Tenki ``Sandbox`` class with an in-memory fake + tight polls.""" factory = _FakeSandboxFactory() monkeypatch.setattr(_tenki_module, "Sandbox", factory) + # Shrink poll interval + timeout so tests exercising state polls stay fast. + monkeypatch.setattr(_tenki_module, "_RESUME_POLL_INTERVAL_SECONDS", 0.001) + monkeypatch.setattr(_tenki_module, "_RESUME_POLL_TIMEOUT_SECONDS", 0.5) return factory +async def _invoke(tool: TenkiExecuteCodeTool, code: str) -> list[Any]: + """Invoke the tool through the public :meth:`FunctionTool.invoke` API.""" + return await tool.invoke(arguments={"code": code}) + + # --------------------------------------------------------------------------- -# Unit tests — no network / no real Tenki service. +# Construction + naming # --------------------------------------------------------------------------- def test_default_sandbox_name_carries_agent_framework_prefix() -> None: tool = TenkiExecuteCodeTool() assert tool.sandbox_name.startswith("agent-framework-") - # 8-char hex suffix, per _default_sandbox_name. suffix = tool.sandbox_name.rsplit("-", 1)[-1] assert len(suffix) == 8 - int(suffix, 16) # raises if the suffix is not hex. + int(suffix, 16) # raises if the suffix is not hex def test_explicit_sandbox_name_is_preserved() -> None: + """Standalone use with an explicit name preserves the literal (no suffix).""" tool = TenkiExecuteCodeTool(sandbox_name="my-name") assert tool.sandbox_name == "my-name" @@ -135,191 +191,37 @@ def test_exec_timeout_below_one_is_rejected() -> None: TenkiExecuteCodeTool(exec_timeout_seconds=0) +# --------------------------------------------------------------------------- +# Sandbox creation + kwargs forwarding +# --------------------------------------------------------------------------- + + async def test_run_code_lazily_creates_sandbox(fake_sdk: _FakeSandboxFactory) -> None: tool = TenkiExecuteCodeTool(sandbox_name="lazy") - # No sandbox yet just from construction. assert fake_sdk.create_calls == [] - - await tool._run_code(code="print('hello')") + await _invoke(tool, "print('hello')") assert len(fake_sdk.create_calls) == 1 assert fake_sdk.create_calls[0]["name"] == "lazy" async def test_run_code_reuses_the_same_sandbox_across_calls(fake_sdk: _FakeSandboxFactory) -> None: - """Reuse-per-session: subsequent calls do NOT create a new sandbox.""" + """Reuse-per-tool: subsequent calls on the same tool do NOT create a new sandbox.""" tool = TenkiExecuteCodeTool(sandbox_name="reuse") - await tool._run_code(code="x = 1") - await tool._run_code(code="print(x)") - await tool._run_code(code="print(x + 1)") + await _invoke(tool, "x = 1") + await _invoke(tool, "print(x)") + await _invoke(tool, "print(x + 1)") assert len(fake_sdk.create_calls) == 1 - # All exec calls landed on the single created sandbox. assert fake_sdk.last_sandbox is not None assert len(fake_sdk.last_sandbox.exec_calls) == 3 -async def test_close_terminates_sandbox_and_next_call_creates_new_one( - fake_sdk: _FakeSandboxFactory, -) -> None: - tool = TenkiExecuteCodeTool(sandbox_name="cycle") - await tool._run_code(code="print('a')") - first_sandbox = fake_sdk.last_sandbox - assert first_sandbox is not None - await tool.close() - assert first_sandbox.closed is True - - await tool._run_code(code="print('b')") - assert len(fake_sdk.create_calls) == 2 - assert fake_sdk.last_sandbox is not first_sandbox - - -async def test_paused_sandbox_is_auto_resumed_between_calls(fake_sdk: _FakeSandboxFactory) -> None: - """A sandbox paused between calls must be transparently resumed before the next exec.""" - tool = TenkiExecuteCodeTool(sandbox_name="paused") - await tool._run_code(code="print('a')") - sandbox = fake_sdk.last_sandbox - assert sandbox is not None - - # Simulate a server-side pause discovered on the next ``refresh()``. - sandbox.refresh_transitions_to = "PAUSED" - - await tool._run_code(code="print('b')") - - # Same sandbox reused (no re-provision). - assert len(fake_sdk.create_calls) == 1 - assert fake_sdk.last_sandbox is sandbox - # refresh + resume were called, and state landed back on RUNNING. - assert sandbox.refresh_calls == 1 - assert sandbox.resume_calls == 1 - assert sandbox.state == "RUNNING" - # The second exec landed on the same sandbox. - assert len(sandbox.exec_calls) == 2 - - -async def test_reprovision_regenerates_name_for_autogenerated_defaults( - fake_sdk: _FakeSandboxFactory, -) -> None: - """Auto-generated sandbox names must be refreshed on every re-provision. - - Tenki can reject ``Sandbox.create(name=)`` with - ``ALREADY_EXISTS`` if the workspace still holds the name; a fresh 8-hex - suffix keeps the re-provision path healthy for the common case where the - caller never passed an explicit ``sandbox_name``. - """ - tool = TenkiExecuteCodeTool() # no explicit sandbox_name → auto-generated - original_name = tool.sandbox_name - await tool._run_code(code="print('a')") - first = fake_sdk.last_sandbox - assert first is not None - first.refresh_transitions_to = "TERMINATED" - - await tool._run_code(code="print('b')") - - # Two ``create`` calls, each with a different name; both carry the shared prefix. - assert len(fake_sdk.create_calls) == 2 - assert fake_sdk.create_calls[0]["name"] == original_name - assert fake_sdk.create_calls[1]["name"] != original_name - assert fake_sdk.create_calls[1]["name"].startswith("agent-framework-") - - -async def test_reprovision_preserves_explicit_sandbox_name( - fake_sdk: _FakeSandboxFactory, -) -> None: - """An explicit ``sandbox_name`` must be kept as-is across re-provision. - - Regenerating would silently violate the caller's expressed intent; if Tenki - rejects reuse we let the ``RuntimeError`` surface so the caller can act on it. - """ - tool = TenkiExecuteCodeTool(sandbox_name="explicit-name") - await tool._run_code(code="print('a')") - first = fake_sdk.last_sandbox - assert first is not None - first.refresh_transitions_to = "TERMINATED" - - await tool._run_code(code="print('b')") - - assert [call["name"] for call in fake_sdk.create_calls] == ["explicit-name", "explicit-name"] - - -async def test_terminated_sandbox_is_replaced_by_fresh_provision(fake_sdk: _FakeSandboxFactory) -> None: - """A sandbox that terminated between calls must be dropped and replaced.""" - tool = TenkiExecuteCodeTool(sandbox_name="gone") - await tool._run_code(code="print('a')") - first = fake_sdk.last_sandbox - assert first is not None - - first.refresh_transitions_to = "TERMINATED" - - await tool._run_code(code="print('b')") - - # A brand new sandbox was provisioned; the terminated one was not resumed. - assert len(fake_sdk.create_calls) == 2 - assert fake_sdk.last_sandbox is not first - assert first.resume_calls == 0 - assert fake_sdk.last_sandbox is not None - assert len(fake_sdk.last_sandbox.exec_calls) == 1 - - -async def test_resume_failure_surfaces_as_error_content(fake_sdk: _FakeSandboxFactory) -> None: - """If ``resume()`` raises, the failure surfaces to the caller as error content. - - The stale handle is deliberately kept so that a subsequent call (once the underlying - condition clears, e.g. quota released) can retry ``resume()`` on the same sandbox - without losing filesystem state to an unnecessary re-provision. - """ - tool = TenkiExecuteCodeTool(sandbox_name="resume-fail") - await tool._run_code(code="print('a')") - sandbox = fake_sdk.last_sandbox - assert sandbox is not None - - sandbox.refresh_transitions_to = "PAUSED" - sandbox.resume_raises = RuntimeError("quota_exceeded: resume denied") - - contents = await tool._run_code(code="print('b')") - - # RuntimeError from _ensure_sandbox_sync is caught in _run_code and returned as - # error content — the caller sees a structured failure, not an unhandled exception. - assert len(contents) == 1 - assert contents[0].type == "error" - assert "Failed to provision Tenki sandbox" in (contents[0].message or "") - assert "quota_exceeded" in (contents[0].error_details or "") - - # Resume was attempted exactly once; no re-provision happened. - assert sandbox.resume_calls == 1 - assert len(fake_sdk.create_calls) == 1 - - -async def test_refresh_failure_falls_back_to_fresh_provision(fake_sdk: _FakeSandboxFactory) -> None: - """If ``refresh()`` raises, the stale handle is dropped and a fresh sandbox is provisioned.""" - tool = TenkiExecuteCodeTool(sandbox_name="stale") - await tool._run_code(code="print('a')") - first = fake_sdk.last_sandbox - assert first is not None - - first.refresh_raises = RuntimeError("connection reset") - - await tool._run_code(code="print('b')") - - assert len(fake_sdk.create_calls) == 2 - assert fake_sdk.last_sandbox is not first - # We did not attempt to resume a sandbox whose state we couldn't confirm. - assert first.resume_calls == 0 - - -async def test_close_is_idempotent(fake_sdk: _FakeSandboxFactory) -> None: - tool = TenkiExecuteCodeTool(sandbox_name="idempotent") - await tool.close() # no sandbox yet — no-op. - await tool._run_code(code="print('x')") - await tool.close() - await tool.close() # second close — no-op. - assert fake_sdk.last_sandbox is not None - assert fake_sdk.last_sandbox.closed is True - - -async def test_async_context_manager_closes_sandbox(fake_sdk: _FakeSandboxFactory) -> None: - async with TenkiExecuteCodeTool(sandbox_name="cm") as tool: - await tool._run_code(code="print('inside')") - assert fake_sdk.last_sandbox is not None - assert fake_sdk.last_sandbox.closed is True +async def test_default_max_duration_is_finite(fake_sdk: _FakeSandboxFactory) -> None: + """Even without ``max_duration_seconds`` the tool applies a finite server-side backstop.""" + tool = TenkiExecuteCodeTool(sandbox_name="default-cap") + await _invoke(tool, "pass") + call = fake_sdk.create_calls[0] + assert call["max_duration"] == _tenki_module._DEFAULT_MAX_DURATION_SECONDS + assert call["max_duration"] > 0 async def test_create_kwargs_only_forward_set_optionals(fake_sdk: _FakeSandboxFactory) -> None: @@ -335,7 +237,7 @@ async def test_create_kwargs_only_forward_set_optionals(fake_sdk: _FakeSandboxFa max_duration_seconds=300, extra_create_kwargs={"allow_inbound": True}, ) - await tool._run_code(code="pass") + await _invoke(tool, "pass") call = fake_sdk.create_calls[0] assert call["name"] == "kwargs" @@ -359,9 +261,11 @@ async def test_create_kwargs_omit_unset_optionals( monkeypatch.delenv("TENKI_PROJECT_ID", raising=False) monkeypatch.delenv("TENKI_WORKSPACE_ID", raising=False) tool = TenkiExecuteCodeTool(sandbox_name="bare") - await tool._run_code(code="pass") + await _invoke(tool, "pass") call = fake_sdk.create_calls[0] + # ``max_duration`` is now unconditionally set (finite default) — so it is NOT in + # the omitted list. The other optionals should still be absent when unset. for absent in ( "auth_token", "image", @@ -370,44 +274,47 @@ async def test_create_kwargs_omit_unset_optionals( "cpu_cores", "memory_mb", "disk_size_gb", - "max_duration", ): assert absent not in call, f"expected {absent!r} to be omitted from create kwargs" +async def test_max_duration_opt_out(fake_sdk: _FakeSandboxFactory) -> None: + """Passing ``max_duration_seconds=None`` explicitly opts out of the finite cap.""" + tool = TenkiExecuteCodeTool(sandbox_name="no-cap", max_duration_seconds=None) + await _invoke(tool, "pass") + assert "max_duration" not in fake_sdk.create_calls[0] + + async def test_env_api_key_is_forwarded_as_auth_token( fake_sdk: _FakeSandboxFactory, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setenv("TENKI_API_KEY", "tk_FROM_ENV") tool = TenkiExecuteCodeTool(sandbox_name="env-key") - await tool._run_code(code="pass") + await _invoke(tool, "pass") assert fake_sdk.create_calls[0]["auth_token"] == "tk_FROM_ENV" async def test_env_project_id_is_forwarded_when_unset( fake_sdk: _FakeSandboxFactory, monkeypatch: pytest.MonkeyPatch ) -> None: - """TENKI_PROJECT_ID env var populates project_id when the constructor arg is None.""" monkeypatch.setenv("TENKI_PROJECT_ID", "proj-from-env") tool = TenkiExecuteCodeTool(sandbox_name="env-proj") - await tool._run_code(code="pass") + await _invoke(tool, "pass") assert fake_sdk.create_calls[0]["project_id"] == "proj-from-env" async def test_env_workspace_id_is_forwarded_when_unset( fake_sdk: _FakeSandboxFactory, monkeypatch: pytest.MonkeyPatch ) -> None: - """TENKI_WORKSPACE_ID env var populates workspace_id when the constructor arg is None.""" monkeypatch.setenv("TENKI_WORKSPACE_ID", "ws-from-env") tool = TenkiExecuteCodeTool(sandbox_name="env-ws") - await tool._run_code(code="pass") + await _invoke(tool, "pass") assert fake_sdk.create_calls[0]["workspace_id"] == "ws-from-env" async def test_constructor_project_id_wins_over_env( fake_sdk: _FakeSandboxFactory, monkeypatch: pytest.MonkeyPatch ) -> None: - """Explicit constructor project_id takes precedence over TENKI_PROJECT_ID env.""" monkeypatch.setenv("TENKI_PROJECT_ID", "proj-from-env") monkeypatch.setenv("TENKI_WORKSPACE_ID", "ws-from-env") tool = TenkiExecuteCodeTool( @@ -415,7 +322,7 @@ async def test_constructor_project_id_wins_over_env( project_id="proj-explicit", workspace_id="ws-explicit", ) - await tool._run_code(code="pass") + await _invoke(tool, "pass") call = fake_sdk.create_calls[0] assert call["project_id"] == "proj-explicit" assert call["workspace_id"] == "ws-explicit" @@ -424,29 +331,33 @@ async def test_constructor_project_id_wins_over_env( async def test_create_failure_returns_error_content(fake_sdk: _FakeSandboxFactory) -> None: fake_sdk.raise_on_create = RuntimeError("resource_exhausted: no live node-agent") tool = TenkiExecuteCodeTool(sandbox_name="fails") - contents = await tool._run_code(code="print('never runs')") + contents = await _invoke(tool, "print('never runs')") assert len(contents) == 1 assert contents[0].type == "error" - assert "Failed to provision Tenki sandbox" in (contents[0].message or "") + assert "Failed to prepare Tenki sandbox" in (contents[0].message or "") assert "resource_exhausted" in (contents[0].error_details or "") async def test_run_code_uses_python3_dash_c_with_timeout(fake_sdk: _FakeSandboxFactory) -> None: tool = TenkiExecuteCodeTool(sandbox_name="argshape", exec_timeout_seconds=45) - await tool._run_code(code="print(1 + 1)") - + await _invoke(tool, "print(1 + 1)") assert fake_sdk.last_sandbox is not None args, kwargs = fake_sdk.last_sandbox.exec_calls[0] assert args == ("python3", "-c", "print(1 + 1)") assert kwargs == {"timeout": 45} +# --------------------------------------------------------------------------- +# Result parsing — ok/signal/reason/errno +# --------------------------------------------------------------------------- + + async def test_stdout_only_produces_single_text_content(fake_sdk: _FakeSandboxFactory) -> None: tool = TenkiExecuteCodeTool(sandbox_name="stdout") - await tool._run_code(code="init") + await _invoke(tool, "init") assert fake_sdk.last_sandbox is not None fake_sdk.last_sandbox.script_result = _FakeExecResult(stdout_text="hello\n", exit_code=0) - contents = await tool._run_code(code="print('hello')") + contents = await _invoke(tool, "print('hello')") assert len(contents) == 1 assert contents[0].type == "text" assert contents[0].text == "hello\n" @@ -454,24 +365,58 @@ async def test_stdout_only_produces_single_text_content(fake_sdk: _FakeSandboxFa async def test_non_zero_exit_produces_error_content_with_stderr(fake_sdk: _FakeSandboxFactory) -> None: tool = TenkiExecuteCodeTool(sandbox_name="failing") - await tool._run_code(code="init") + await _invoke(tool, "init") assert fake_sdk.last_sandbox is not None fake_sdk.last_sandbox.script_result = _FakeExecResult(stdout_text="", stderr_text="Traceback...\n", exit_code=1) - contents = await tool._run_code(code="raise ValueError('boom')") + contents = await _invoke(tool, "raise ValueError('boom')") error_contents = [c for c in contents if c.type == "error"] assert len(error_contents) == 1 message = error_contents[0].message or "" assert "status 1" in message - # stderr must be inlined into the message so LLMs that under-weight - # `error_details` still see the traceback in the primary field. assert "Traceback" in message assert "Traceback" in (error_contents[0].error_details or "") +async def test_signal_termination_reports_failure_via_ok_property( + fake_sdk: _FakeSandboxFactory, +) -> None: + """SIGKILL / SIGTERM etc. leave ``exit_code=0`` but ``ok=False``. + + We must trust ``ok`` (or the derived ``exit_code == 0 and not signal`` check), + not ``exit_code`` alone — otherwise ``Killed`` reads as success. + """ + tool = TenkiExecuteCodeTool(sandbox_name="sig") + await _invoke(tool, "init") + assert fake_sdk.last_sandbox is not None + fake_sdk.last_sandbox.script_result = _FakeExecResult( + stdout_text="", stderr_text="Killed\n", exit_code=0, signal="SIGKILL", reason="oom" + ) + contents = await _invoke(tool, "hog memory") + error_contents = [c for c in contents if c.type == "error"] + assert len(error_contents) == 1 + message = error_contents[0].message or "" + assert "signal SIGKILL" in message + assert "reason oom" in message + + +async def test_errno_surfaces_in_failure_message(fake_sdk: _FakeSandboxFactory) -> None: + """Spawn failures carry ``errno`` — must be visible in the error message.""" + tool = TenkiExecuteCodeTool(sandbox_name="spawn-fail") + await _invoke(tool, "init") + assert fake_sdk.last_sandbox is not None + fake_sdk.last_sandbox.script_result = _FakeExecResult( + stdout_text="", stderr_text="", exit_code=127, errno=2, reason="spawn_failed" + ) + contents = await _invoke(tool, "nope") + error_contents = [c for c in contents if c.type == "error"] + message = error_contents[0].message or "" + assert "errno 2" in message + assert "reason spawn_failed" in message + + async def test_syntaxerror_appends_recovery_hint(fake_sdk: _FakeSandboxFactory) -> None: - """SyntaxError stderr must trigger the recovery hint in the message field.""" tool = TenkiExecuteCodeTool(sandbox_name="syntax") - await tool._run_code(code="init") + await _invoke(tool, "init") assert fake_sdk.last_sandbox is not None fake_sdk.last_sandbox.script_result = _FakeExecResult( stdout_text="", @@ -483,19 +428,17 @@ async def test_syntaxerror_appends_recovery_hint(fake_sdk: _FakeSandboxFactory) ), exit_code=1, ) - contents = await tool._run_code(code="import os; with open('/tmp/x') as f: pass") + contents = await _invoke(tool, "import os; with open('/tmp/x') as f: pass") error_content = next(c for c in contents if c.type == "error") message = error_content.message or "" - # Hint should mention both known failure modes. assert "Common causes" in message assert "compound statement" in message - assert "\\n" in message # literal-newline pitfall reference + assert "\\n" in message async def test_non_syntax_error_does_not_append_hint(fake_sdk: _FakeSandboxFactory) -> None: - """Non-SyntaxError stderr must NOT trigger the recovery hint (no misleading advice).""" tool = TenkiExecuteCodeTool(sandbox_name="runtime") - await tool._run_code(code="init") + await _invoke(tool, "init") assert fake_sdk.last_sandbox is not None fake_sdk.last_sandbox.script_result = _FakeExecResult( stdout_text="", @@ -504,7 +447,7 @@ async def test_non_syntax_error_does_not_append_hint(fake_sdk: _FakeSandboxFacto ), exit_code=1, ) - contents = await tool._run_code(code="raise ValueError('bad value')") + contents = await _invoke(tool, "raise ValueError('bad value')") error_content = next(c for c in contents if c.type == "error") message = error_content.message or "" assert "ValueError" in message @@ -512,176 +455,839 @@ async def test_non_syntax_error_does_not_append_hint(fake_sdk: _FakeSandboxFacto async def test_error_message_truncates_long_stderr(fake_sdk: _FakeSandboxFactory) -> None: - """Very long stderr must be truncated in the message so it doesn't blow the field.""" tool = TenkiExecuteCodeTool(sandbox_name="failing-long") - await tool._run_code(code="init") + await _invoke(tool, "init") assert fake_sdk.last_sandbox is not None - long_stderr = "err " * 500 # 2000 chars total + long_stderr = "err " * 500 fake_sdk.last_sandbox.script_result = _FakeExecResult(stdout_text="", stderr_text=long_stderr, exit_code=1) - contents = await tool._run_code(code="raise Exception('x')") + contents = await _invoke(tool, "raise Exception('x')") error_contents = [c for c in contents if c.type == "error"] assert len(error_contents) == 1 message = error_contents[0].message or "" - # message field capped: status prefix + up to 500 chars of stderr + minor formatting. - # Verify total is well below the full stderr length so the LLM's context isn't flooded. assert len(message) < 700 - # Full stderr still available via error_details for callers that want it. assert len(error_contents[0].error_details or "") == len(long_stderr) async def test_stderr_without_error_becomes_text_content(fake_sdk: _FakeSandboxFactory) -> None: - """Warnings to stderr should still surface to the model, matching Hyperlight.""" tool = TenkiExecuteCodeTool(sandbox_name="warn") - await tool._run_code(code="init") + await _invoke(tool, "init") assert fake_sdk.last_sandbox is not None fake_sdk.last_sandbox.script_result = _FakeExecResult( stdout_text="", stderr_text="DeprecationWarning: foo\n", exit_code=0 ) - contents = await tool._run_code(code="import warnings; warnings.warn('foo')") + contents = await _invoke(tool, "import warnings; warnings.warn('foo')") assert any(c.type == "text" and "DeprecationWarning" in (c.text or "") for c in contents) assert not any(c.type == "error" for c in contents) async def test_empty_output_produces_placeholder_text(fake_sdk: _FakeSandboxFactory) -> None: tool = TenkiExecuteCodeTool(sandbox_name="silent") - await tool._run_code(code="init") + await _invoke(tool, "init") assert fake_sdk.last_sandbox is not None fake_sdk.last_sandbox.script_result = _FakeExecResult(stdout_text="", stderr_text="", exit_code=0) - contents = await tool._run_code(code="x = 1") + contents = await _invoke(tool, "x = 1") assert len(contents) == 1 assert contents[0].type == "text" assert "without output" in (contents[0].text or "") -async def test_cancelled_error_propagates(fake_sdk: _FakeSandboxFactory) -> None: - """asyncio.CancelledError must propagate so higher-level workflows can stop promptly.""" - import asyncio - - tool = TenkiExecuteCodeTool(sandbox_name="cancel") - await tool._run_code(code="init") - assert fake_sdk.last_sandbox is not None - - def handler(_args: tuple[Any, ...], _kwargs: dict[str, Any]) -> _FakeExecResult: - raise asyncio.CancelledError() - - fake_sdk.last_sandbox.script_handler = handler - with pytest.raises(asyncio.CancelledError): - await tool._run_code(code="print('x')") - - async def test_exec_exception_returns_error_content(fake_sdk: _FakeSandboxFactory) -> None: tool = TenkiExecuteCodeTool(sandbox_name="exec-err") def handler(_args: tuple[Any, ...], _kwargs: dict[str, Any]) -> _FakeExecResult: raise RuntimeError("SDK exec transport error") - await tool._run_code(code="init") + await _invoke(tool, "init") assert fake_sdk.last_sandbox is not None fake_sdk.last_sandbox.script_handler = handler - contents = await tool._run_code(code="print('x')") + contents = await _invoke(tool, "print('x')") assert len(contents) == 1 assert contents[0].type == "error" assert "Sandbox execution failed" in (contents[0].message or "") -async def test_tool_name_and_description() -> None: - tool = TenkiExecuteCodeTool() - assert tool.name == "execute_code" - assert "Tenki" in tool.description - +# --------------------------------------------------------------------------- +# Cancellation propagation — real ``asyncio.Task.cancel()`` mid-execution +# --------------------------------------------------------------------------- -async def test_provider_close_terminates_underlying_sandbox(fake_sdk: _FakeSandboxFactory) -> None: - provider = TenkiCodeActProvider(sandbox_name="prov") - await provider.execute_code_tool._run_code(code="print('x')") - assert fake_sdk.last_sandbox is not None - await provider.close() - assert fake_sdk.last_sandbox.closed is True +async def test_task_cancel_during_exec_propagates(fake_sdk: _FakeSandboxFactory) -> None: + """Cancelling the outer task while ``sandbox.exec`` is running propagates ``CancelledError``. -async def test_provider_async_context_manager_closes_sandbox(fake_sdk: _FakeSandboxFactory) -> None: - async with TenkiCodeActProvider(sandbox_name="prov-cm") as provider: - await provider.execute_code_tool._run_code(code="print('x')") - assert fake_sdk.last_sandbox is not None - assert fake_sdk.last_sandbox.closed is False - assert fake_sdk.last_sandbox is not None - assert fake_sdk.last_sandbox.closed is True + Not a mocked raise inside the worker — a real ``asyncio.Task.cancel()`` from the + outer coroutine while the ``to_thread(exec)`` call is still running. Verifies + that our ``except asyncio.CancelledError: raise`` propagates the cancel through + to the caller, which is what higher-level workflows depend on to stop promptly, + AND that the tool remains cleanly closable afterwards (no leaked sandbox). + """ + import threading + tool = TenkiExecuteCodeTool(sandbox_name="cancel") + await _invoke(tool, "init") # provision sandbox up-front + sandbox = fake_sdk.last_sandbox + assert sandbox is not None -async def test_provider_exposes_execute_code_tool_before_run(fake_sdk: _FakeSandboxFactory) -> None: - provider = TenkiCodeActProvider(sandbox_name="ctx") - recorded: dict[str, Any] = {"tools_calls": [], "instructions_calls": []} + loop = asyncio.get_running_loop() + exec_started = asyncio.Event() + # ``threading.Event`` — set by the outer coroutine to release the worker. + # Cross-thread signalling with no private-API reach into ``asyncio.Event._loop``. + exec_release = threading.Event() - class _StubContext: - def extend_tools(self, source_id: str, tools: list[Any]) -> None: - recorded["tools_calls"].append((source_id, tools)) + def slow_exec(_args: tuple[Any, ...], _kwargs: dict[str, Any]) -> _FakeExecResult: + loop.call_soon_threadsafe(exec_started.set) + exec_release.wait(timeout=2.0) + return _FakeExecResult(stdout_text="never returned\n") - def extend_instructions(self, source_id: str, instructions: str) -> None: - recorded["instructions_calls"].append((source_id, instructions)) + sandbox.script_handler = slow_exec - # ``_StubContext`` is structurally compatible with ``SessionContext`` (implements - # extend_tools + extend_instructions) but isn't a nominal subclass. Cast to Any - # so strict type checkers accept the deliberate stub injection. - await provider.before_run(agent=None, session=None, context=cast(Any, _StubContext()), state={}) + task = asyncio.create_task(_invoke(tool, "print('never')")) + await exec_started.wait() + task.cancel() + exec_release.set() # let the worker complete so its thread can join - assert len(recorded["tools_calls"]) == 1 - tools_source_id, tools = recorded["tools_calls"][0] - assert tools_source_id == TenkiCodeActProvider.DEFAULT_SOURCE_ID - assert tools == [provider.execute_code_tool] + with pytest.raises(asyncio.CancelledError): + await task - assert len(recorded["instructions_calls"]) == 1 - instr_source_id, instructions = recorded["instructions_calls"][0] - assert instr_source_id == TenkiCodeActProvider.DEFAULT_SOURCE_ID - # The instructions must reinforce the print() requirement, the persistence - # rules, the "no Jupyter magic" rule, and the "newlines for compound - # statements" rule so that small models tool-call correctly against a - # fresh interpreter. - assert "print(" in instructions - assert "python3 -c" in instructions - assert "filesystem persists" in instructions - assert "subprocess" in instructions - # Match on '!command' to catch the specific Jupyter-magic footgun we hit - # in the field (small models emitted `!{sys.executable} -m pip install ...`). - assert "!command" in instructions - # Match on 'compound statements' to catch the semicolon-in-compound-stmt - # footgun (small models emitted `import os; with open(...) as f: ...`). - assert "compound statements" in instructions - # A concrete valid single-line `with` example nudges small models toward - # the shape that actually worked in the field. - assert "with open('/tmp/x.txt') as f: print(f.read())" in instructions + # Teardown/leak assertion: after cancellation the tool must still close + # cleanly — a cancelled run must not orphan the underlying sandbox. + await tool.close() + assert sandbox.closed is True + assert sandbox._client.closed is True + assert tool._sandbox is None # --------------------------------------------------------------------------- -# Integration test — real Tenki service. Requires TENKI_API_KEY. Marked as -# integration so ``pytest -m "not integration"`` (the default suite) skips it. +# Reconcile lifecycle — refresh / pause+resume / terminate / poll # --------------------------------------------------------------------------- -def _tenki_integration_skip_reason() -> str | None: - if os.environ.get("SKIP_TENKI", "").lower() == "true": - return "SKIP_TENKI=true is set." - if importlib.util.find_spec("tenki_sandbox") is None: - return "tenki-sandbox is not installed." - if not os.environ.get("TENKI_API_KEY"): - return "TENKI_API_KEY is not set." - return None +async def test_paused_sandbox_is_auto_resumed_between_calls(fake_sdk: _FakeSandboxFactory) -> None: + """A sandbox paused between calls must be transparently resumed before the next exec.""" + tool = TenkiExecuteCodeTool(sandbox_name="paused") + await _invoke(tool, "print('a')") + sandbox = fake_sdk.last_sandbox + assert sandbox is not None + sandbox.refresh_transitions_to = "PAUSED" + await _invoke(tool, "print('b')") -skip_if_tenki_integration_disabled = pytest.mark.skipif( - _tenki_integration_skip_reason() is not None, - reason=_tenki_integration_skip_reason() or "unknown", -) + assert len(fake_sdk.create_calls) == 1 + assert fake_sdk.last_sandbox is sandbox + assert sandbox.refresh_calls >= 1 + assert sandbox.resume_calls == 1 + assert sandbox.state == "RUNNING" + assert len(sandbox.exec_calls) == 2 -@pytest.mark.integration -@skip_if_tenki_integration_disabled -async def test_integration_execute_hello_world() -> None: - project_id = os.environ.get("TENKI_PROJECT_ID") - async with TenkiExecuteCodeTool( - sandbox_name=f"agent-framework-ci-{os.getpid()}", - project_id=project_id, - max_duration_seconds=300, - ) as tool: - contents = await tool._run_code(code="print('hello from tenki')") - text_contents = [c for c in contents if c.type == "text"] - assert any("hello from tenki" in (c.text or "") for c in text_contents) +async def test_resume_polls_until_running_when_state_lags(fake_sdk: _FakeSandboxFactory) -> None: + """``resume()`` can return while the SDK reports ``RESUMING`` — poll until ``RUNNING``. + + Uses a state sequence: refresh #1 reveals ``PAUSED``, then after resume the fake + reports ``RESUMING`` for a couple of polls before flipping to ``RUNNING``. + """ + tool = TenkiExecuteCodeTool(sandbox_name="resume-lag") + await _invoke(tool, "init") + sandbox = fake_sdk.last_sandbox + assert sandbox is not None + + # Refresh sequence drives the reconcile: PAUSED (triggers resume), then RUNNING + # (poll loop sees running). ``resume()`` itself leaves state at ``RESUMING``. + sandbox.refresh_state_sequence = ["PAUSED", "RUNNING"] + sandbox.resume_leaves_state_as = "RESUMING" + + await _invoke(tool, "print('after-resume')") + assert sandbox.resume_calls == 1 + assert sandbox.state == "RUNNING" + + +async def test_resume_timeout_raises_runtime_error(fake_sdk: _FakeSandboxFactory) -> None: + """If the sandbox never reaches RUNNING within the poll budget, surface a clear error.""" + tool = TenkiExecuteCodeTool(sandbox_name="resume-timeout") + await _invoke(tool, "init") + sandbox = fake_sdk.last_sandbox + assert sandbox is not None + + # Refresh #1 exposes PAUSED, then every subsequent refresh keeps reporting RESUMING. + sandbox.refresh_state_sequence = ["PAUSED"] + ["RESUMING"] * 200 + sandbox.resume_leaves_state_as = "RESUMING" + + contents = await _invoke(tool, "print('x')") + assert len(contents) == 1 + assert contents[0].type == "error" + assert "did not reach RUNNING" in (contents[0].error_details or "") + + +async def test_transitional_state_polls_to_running(fake_sdk: _FakeSandboxFactory) -> None: + """Transitional states (CREATING / RESUMING) resolve via the poll loop, no resume() call.""" + tool = TenkiExecuteCodeTool(sandbox_name="transitional") + await _invoke(tool, "init") + sandbox = fake_sdk.last_sandbox + assert sandbox is not None + + # Refresh sequence: CREATING (initial reconcile) → RESUMING (first poll) → RUNNING. + sandbox.refresh_state_sequence = ["CREATING", "RESUMING", "RUNNING"] + + await _invoke(tool, "print('done')") + assert sandbox.resume_calls == 0 # no resume — only polling + assert sandbox.state == "RUNNING" + + +async def test_pausing_settles_to_paused_then_resumes(fake_sdk: _FakeSandboxFactory) -> None: + """A sandbox caught mid-PAUSING must be polled to PAUSED and then resumed.""" + tool = TenkiExecuteCodeTool(sandbox_name="pausing") + await _invoke(tool, "init") + sandbox = fake_sdk.last_sandbox + assert sandbox is not None + + # Refresh sequence: PAUSING (initial reconcile) → PAUSED (first poll, triggers resume). + sandbox.refresh_state_sequence = ["PAUSING", "PAUSED"] + + await _invoke(tool, "print('done')") + assert sandbox.resume_calls == 1 + assert sandbox.state == "RUNNING" + + +async def test_user_shutdown_sandbox_is_auto_resumed(fake_sdk: _FakeSandboxFactory) -> None: + """USER_SHUTDOWN (guest OS shut down inside the VM) is resumable — treat it like PAUSED.""" + tool = TenkiExecuteCodeTool(sandbox_name="guest-shutdown") + await _invoke(tool, "print('a')") + sandbox = fake_sdk.last_sandbox + assert sandbox is not None + + sandbox.refresh_transitions_to = "USER_SHUTDOWN" + await _invoke(tool, "print('b')") + + assert len(fake_sdk.create_calls) == 1 # resumed, not re-provisioned + assert sandbox.resume_calls == 1 + assert sandbox.state == "RUNNING" + assert len(sandbox.exec_calls) == 2 + + +async def test_terminated_sandbox_is_replaced_by_fresh_provision(fake_sdk: _FakeSandboxFactory) -> None: + """A sandbox that terminated between calls must be dropped and replaced.""" + tool = TenkiExecuteCodeTool(sandbox_name="gone") + await _invoke(tool, "print('a')") + first = fake_sdk.last_sandbox + assert first is not None + first.refresh_transitions_to = "TERMINATED" + + await _invoke(tool, "print('b')") + assert len(fake_sdk.create_calls) == 2 + assert fake_sdk.last_sandbox is not first + assert first.resume_calls == 0 + assert fake_sdk.last_sandbox is not None + assert len(fake_sdk.last_sandbox.exec_calls) == 1 + + +@pytest.mark.parametrize("terminal_state", ["TERMINATED", "TERMINATING"]) +async def test_reprovision_on_terminal_state_closes_owning_client( + fake_sdk: _FakeSandboxFactory, terminal_state: str +) -> None: + """Terminal-state re-provision must close the old sandbox's owning gRPC client. + + Without this a long-lived standalone tool that repeatedly hits a terminal + state leaks one control-plane channel per cycle for the lifetime of the + process. + """ + tool = TenkiExecuteCodeTool(sandbox_name=f"leak-{terminal_state.lower()}") + await _invoke(tool, "print('a')") + first = fake_sdk.last_sandbox + assert first is not None + first.refresh_transitions_to = terminal_state + + await _invoke(tool, "print('b')") + + assert fake_sdk.last_sandbox is not first + assert first._client.closed is True + + +async def test_reprovision_regenerates_name_for_autogenerated_defaults( + fake_sdk: _FakeSandboxFactory, +) -> None: + """Auto-generated sandbox names must be refreshed on every re-provision.""" + tool = TenkiExecuteCodeTool() + original_name = tool.sandbox_name + await _invoke(tool, "print('a')") + first = fake_sdk.last_sandbox + assert first is not None + first.refresh_transitions_to = "TERMINATED" + + await _invoke(tool, "print('b')") + assert len(fake_sdk.create_calls) == 2 + assert fake_sdk.create_calls[0]["name"] == original_name + assert fake_sdk.create_calls[1]["name"] != original_name + assert fake_sdk.create_calls[1]["name"].startswith("agent-framework-") + + +async def test_reprovision_preserves_explicit_sandbox_name( + fake_sdk: _FakeSandboxFactory, +) -> None: + """An explicit ``sandbox_name`` must be kept as-is across re-provision.""" + tool = TenkiExecuteCodeTool(sandbox_name="explicit-name") + await _invoke(tool, "print('a')") + first = fake_sdk.last_sandbox + assert first is not None + first.refresh_transitions_to = "TERMINATED" + + await _invoke(tool, "print('b')") + assert [call["name"] for call in fake_sdk.create_calls] == ["explicit-name", "explicit-name"] + + +async def test_refresh_failure_raises_without_dropping_handle( + fake_sdk: _FakeSandboxFactory, +) -> None: + """A transient ``refresh()`` failure must not orphan the sandbox. + + We keep the handle and surface an error; the next call retries the same + sandbox rather than provisioning a duplicate (which would leak the first). + """ + tool = TenkiExecuteCodeTool(sandbox_name="stale") + await _invoke(tool, "print('a')") + first = fake_sdk.last_sandbox + assert first is not None + first.refresh_raises = RuntimeError("connection reset") + + contents = await _invoke(tool, "print('b')") + assert len(contents) == 1 + assert contents[0].type == "error" + assert "Failed to refresh" in (contents[0].error_details or "") + + # Handle preserved: no new sandbox, no close on the old one, no resume attempted. + assert len(fake_sdk.create_calls) == 1 + assert first.closed is False + assert first.resume_calls == 0 + + # Next call succeeds when the API recovers — same sandbox is reused. + await _invoke(tool, "print('c')") + assert len(fake_sdk.create_calls) == 1 + assert len(first.exec_calls) == 2 # 'a' and 'c' (the 'b' call never reached exec) + + +async def test_persistent_resume_failure_surfaces_as_error_content( + fake_sdk: _FakeSandboxFactory, +) -> None: + """If ``resume()`` keeps raising past the poll budget, the failure surfaces as error content.""" + tool = TenkiExecuteCodeTool(sandbox_name="resume-fail") + await _invoke(tool, "print('a')") + sandbox = fake_sdk.last_sandbox + assert sandbox is not None + sandbox.refresh_transitions_to = "PAUSED" + sandbox.resume_raises = RuntimeError("quota_exceeded: resume denied") + + contents = await _invoke(tool, "print('b')") + assert len(contents) == 1 + assert contents[0].type == "error" + assert "Failed to prepare Tenki sandbox" in (contents[0].message or "") + assert "quota_exceeded" in (contents[0].error_details or "") + assert sandbox.resume_calls >= 2 # retried within the budget before giving up + assert len(fake_sdk.create_calls) == 1 + + +async def test_transient_resume_failure_is_retried(fake_sdk: _FakeSandboxFactory) -> None: + """A transient resume rejection must be retried within the poll budget. + + ``USER_SHUTDOWN`` links its pause snapshot asynchronously — the server + rejects resume with "session has no pause snapshot" until the rootfs + capture completes, so the first rejection must not fail the call. + """ + tool = TenkiExecuteCodeTool(sandbox_name="resume-retry") + await _invoke(tool, "print('a')") + sandbox = fake_sdk.last_sandbox + assert sandbox is not None + sandbox.refresh_transitions_to = "USER_SHUTDOWN" + sandbox.resume_raises_once = RuntimeError("session has no pause snapshot") + + contents = await _invoke(tool, "print('b')") + assert not any(c.type == "error" for c in contents) + assert sandbox.resume_calls == 2 # first rejected, retry succeeded + assert sandbox.state == "RUNNING" + assert len(fake_sdk.create_calls) == 1 + + +async def test_server_reverted_resume_is_retried(fake_sdk: _FakeSandboxFactory) -> None: + """An accepted resume that the server reverts must be retried, not treated as fatal. + + A resume that fails on the node reverts the session to its source state + (``RESUMING`` → ``USER_SHUTDOWN``/``PAUSED``, with ``last_resume_error`` + recorded server-side); the loop sees the resumable state again and retries. + """ + tool = TenkiExecuteCodeTool(sandbox_name="resume-revert") + await _invoke(tool, "print('a')") + sandbox = fake_sdk.last_sandbox + assert sandbox is not None + + # Reconcile refresh reveals USER_SHUTDOWN; resume #1 is accepted (state → + # RESUMING) but the next refresh shows the server reverted it; resume #2 + # is accepted and the sandbox reaches RUNNING. + sandbox.refresh_state_sequence = ["USER_SHUTDOWN", "USER_SHUTDOWN", "RUNNING"] + sandbox.resume_leaves_state_as = "RESUMING" + + contents = await _invoke(tool, "print('b')") + assert not any(c.type == "error" for c in contents) + assert sandbox.resume_calls == 2 + assert sandbox.state == "RUNNING" + assert len(fake_sdk.create_calls) == 1 + + +# --------------------------------------------------------------------------- +# Close lifecycle +# --------------------------------------------------------------------------- + + +async def test_close_terminates_sandbox_and_closes_owning_client( + fake_sdk: _FakeSandboxFactory, +) -> None: + tool = TenkiExecuteCodeTool(sandbox_name="close-clean") + await _invoke(tool, "print('x')") + sandbox = fake_sdk.last_sandbox + assert sandbox is not None + + await tool.close() + assert sandbox.closed is True + # Owning client must also be closed — the SDK's own ``close()`` does not + # close the Client that ``Sandbox.create`` allocated, leaking the channel. + assert sandbox._client.closed is True + + +async def test_close_preserves_handle_on_terminate_failure(fake_sdk: _FakeSandboxFactory) -> None: + """Failed terminate must not silently discard the sandbox handle. + + Otherwise the sandbox stays live server-side, we can't retry, and the + caller has no way to force cleanup. + """ + tool = TenkiExecuteCodeTool(sandbox_name="close-fail") + await _invoke(tool, "print('x')") + sandbox = fake_sdk.last_sandbox + assert sandbox is not None + sandbox.close_raises = RuntimeError("terminate rpc unavailable") + + with pytest.raises(RuntimeError, match="terminate rpc unavailable"): + await tool.close() + + # Handle preserved — caller can retry ``close()``. + assert tool._sandbox is sandbox + assert sandbox._client.closed is False + + # Retry succeeds after the transient error clears. + await tool.close() + assert sandbox.closed is True + assert sandbox._client.closed is True + + +async def test_close_serializes_with_concurrent_run( + fake_sdk: _FakeSandboxFactory, monkeypatch: pytest.MonkeyPatch +) -> None: + """``close()`` holds the sandbox lock for the whole terminate, so a concurrent + run cannot race it — it blocks until close completes, then provisions fresh.""" + tool = TenkiExecuteCodeTool(sandbox_name="race") + await _invoke(tool, "print('a')") + first = fake_sdk.last_sandbox + assert first is not None + + close_entered = threading.Event() + release_close = threading.Event() + original_close_if_open = first.close_if_open + + def slow_close() -> None: + close_entered.set() + assert release_close.wait(timeout=5), "close was never released" + original_close_if_open() + + monkeypatch.setattr(first, "close_if_open", slow_close) + + close_task = asyncio.create_task(tool.close()) + assert await asyncio.to_thread(close_entered.wait, 5) + + # A run started mid-close must block on the lock, not interleave. + invoke_task = asyncio.create_task(_invoke(tool, "print('b')")) + await asyncio.sleep(0.05) + assert not invoke_task.done() + + release_close.set() + await close_task + await invoke_task + + assert first.closed is True + assert first._client.closed is True + second = fake_sdk.last_sandbox + assert second is not None + assert second is not first + assert second.closed is False + + +async def test_close_is_idempotent(fake_sdk: _FakeSandboxFactory) -> None: + tool = TenkiExecuteCodeTool(sandbox_name="idempotent") + await tool.close() # no sandbox yet — no-op + await _invoke(tool, "print('x')") + await tool.close() + await tool.close() # second close — no-op + assert fake_sdk.last_sandbox is not None + assert fake_sdk.last_sandbox.closed is True + + +async def test_close_and_next_call_creates_new_sandbox(fake_sdk: _FakeSandboxFactory) -> None: + tool = TenkiExecuteCodeTool(sandbox_name="cycle") + await _invoke(tool, "print('a')") + first_sandbox = fake_sdk.last_sandbox + assert first_sandbox is not None + await tool.close() + assert first_sandbox.closed is True + + await _invoke(tool, "print('b')") + assert len(fake_sdk.create_calls) == 2 + assert fake_sdk.last_sandbox is not first_sandbox + + +async def test_async_context_manager_closes_sandbox(fake_sdk: _FakeSandboxFactory) -> None: + async with TenkiExecuteCodeTool(sandbox_name="cm") as tool: + await _invoke(tool, "print('inside')") + assert fake_sdk.last_sandbox is not None + assert fake_sdk.last_sandbox.closed is True + + +# --------------------------------------------------------------------------- +# Provider run-scoping — before_run / after_run / close cleanup +# --------------------------------------------------------------------------- + + +class _StubContext: + """Structurally compatible with ``SessionContext``. Records the extended tools/instructions.""" + + def __init__(self) -> None: + self.tools_calls: list[tuple[str, list[Any]]] = [] + self.instructions_calls: list[tuple[str, str]] = [] + + def extend_tools(self, source_id: str, tools: list[Any]) -> None: + self.tools_calls.append((source_id, tools)) + + def extend_instructions(self, source_id: str, instructions: str) -> None: + self.instructions_calls.append((source_id, instructions)) + + +def _run_tool_from(context: _StubContext) -> TenkiExecuteCodeTool: + """Extract the run-scoped tool that ``before_run`` extended onto the context.""" + _, tools = context.tools_calls[-1] + assert len(tools) == 1 + assert isinstance(tools[0], TenkiExecuteCodeTool) + return tools[0] + + +async def test_provider_before_run_mints_fresh_run_scoped_tool( + fake_sdk: _FakeSandboxFactory, +) -> None: + provider = TenkiCodeActProvider(sandbox_name="run-scope") + state: dict[str, Any] = {} + context = _StubContext() + await provider.before_run(agent=None, session=None, context=cast(Any, context), state=state) + + # Extended tools carry a NEW ``TenkiExecuteCodeTool`` instance, not the base template. + assert len(context.tools_calls) == 1 + run_tool = _run_tool_from(context) + assert run_tool is not provider.execute_code_tool + + # Provider state is serialized with the session, so it holds only the + # sandbox name (a plain string); the live handle stays on the provider. + assert state[provider.source_id] == run_tool.sandbox_name + assert isinstance(state[provider.source_id], str) + assert provider._live_run_tools[run_tool.sandbox_name] is run_tool + + +async def test_provider_run_tool_uses_sandbox_name_as_prefix( + fake_sdk: _FakeSandboxFactory, +) -> None: + """Explicit ``sandbox_name`` on the provider becomes the prefix for run-scoped names.""" + provider = TenkiCodeActProvider(sandbox_name="analysis-agent") + state: dict[str, Any] = {} + context = _StubContext() + await provider.before_run(agent=None, session=None, context=cast(Any, context), state=state) + + run_tool = _run_tool_from(context) + assert run_tool.sandbox_name.startswith("analysis-agent-") + suffix = run_tool.sandbox_name.rsplit("-", 1)[-1] + assert len(suffix) == 8 + int(suffix, 16) + + +async def test_provider_run_tool_uses_default_prefix_when_unset( + fake_sdk: _FakeSandboxFactory, +) -> None: + provider = TenkiCodeActProvider() + state: dict[str, Any] = {} + context = _StubContext() + await provider.before_run(agent=None, session=None, context=cast(Any, context), state=state) + + run_tool = _run_tool_from(context) + assert run_tool.sandbox_name.startswith("agent-framework-") + + +async def test_provider_after_run_terminates_run_scoped_sandbox( + fake_sdk: _FakeSandboxFactory, +) -> None: + """The run's sandbox must be terminated by ``after_run`` — no cross-run bleed.""" + provider = TenkiCodeActProvider(sandbox_name="isolated") + state: dict[str, Any] = {} + context = _StubContext() + await provider.before_run(agent=None, session=None, context=cast(Any, context), state=state) + run_tool = _run_tool_from(context) + + # Exercise the run tool so it actually provisions a sandbox. + await _invoke(run_tool, "print('hi')") + sandbox = fake_sdk.last_sandbox + assert sandbox is not None + assert sandbox.closed is False + + await provider.after_run(agent=None, session=None, context=cast(Any, context), state=state) + assert sandbox.closed is True + assert provider.source_id not in state + + +async def test_provider_after_run_retains_tool_on_close_failure(fake_sdk: _FakeSandboxFactory) -> None: + """When ``after_run`` fails to terminate, the tool must stay in the live-set. + + Preserves the "handle survives a failed terminate" contract from + :meth:`TenkiExecuteCodeTool.close` on the provider path — otherwise + ``provider.close()`` has no reference to retry against and the sandbox + leaks until ``max_duration`` reaps it. + """ + provider = TenkiCodeActProvider(sandbox_name="retry") + state: dict[str, Any] = {} + context = _StubContext() + await provider.before_run(agent=None, session=None, context=cast(Any, context), state=state) + run_tool = _run_tool_from(context) + await _invoke(run_tool, "print('hi')") + sandbox = fake_sdk.last_sandbox + assert sandbox is not None + + sandbox.close_raises = RuntimeError("terminate rpc unavailable") + await provider.after_run(agent=None, session=None, context=cast(Any, context), state=state) + assert sandbox.closed is False + assert provider.source_id not in state # framework won't call after_run twice + assert provider._live_run_tools.get(run_tool.sandbox_name) is run_tool + + # provider.close() retries the failed terminate. + await provider.close() + assert sandbox.closed is True + + +async def test_provider_two_runs_get_distinct_sandboxes( + fake_sdk: _FakeSandboxFactory, +) -> None: + """Consecutive runs on the same provider must get isolated sandboxes.""" + provider = TenkiCodeActProvider(sandbox_name="two-run") + + # Run 1 + state1: dict[str, Any] = {} + context1 = _StubContext() + await provider.before_run(agent=None, session=None, context=cast(Any, context1), state=state1) + tool1 = _run_tool_from(context1) + await _invoke(tool1, "x = 1") + sandbox1 = fake_sdk.last_sandbox + await provider.after_run(agent=None, session=None, context=cast(Any, context1), state=state1) + + # Run 2 + state2: dict[str, Any] = {} + context2 = _StubContext() + await provider.before_run(agent=None, session=None, context=cast(Any, context2), state=state2) + tool2 = _run_tool_from(context2) + await _invoke(tool2, "x = 2") + sandbox2 = fake_sdk.last_sandbox + + assert tool1 is not tool2 + assert sandbox1 is not sandbox2 + assert sandbox1 is not None and sandbox1.closed is True + assert sandbox2 is not None and sandbox2.closed is False + assert sandbox1.name != sandbox2.name # per-run suffix keeps dashboard names distinguishable + + +async def test_provider_close_cleans_up_orphaned_run_tools( + fake_sdk: _FakeSandboxFactory, +) -> None: + """If ``after_run`` is skipped (e.g. mid-run exception), ``close()`` still cleans up.""" + provider = TenkiCodeActProvider(sandbox_name="orphan") + state: dict[str, Any] = {} + context = _StubContext() + await provider.before_run(agent=None, session=None, context=cast(Any, context), state=state) + run_tool = _run_tool_from(context) + await _invoke(run_tool, "print('hi')") + sandbox = fake_sdk.last_sandbox + assert sandbox is not None + + # Simulate the agent-framework runtime bailing out before after_run fires: + # user calls provider.close() to clean up. + await provider.close() + assert sandbox.closed is True + + +async def test_provider_close_retains_tool_on_failure_and_retries( + fake_sdk: _FakeSandboxFactory, +) -> None: + """A terminate that fails during ``close()`` must stay in the live-set. + + ``close()`` removes entries only on success — dropping the reference on + failure would make a second ``close()`` a no-op and leak the microVM + until ``max_duration`` reaps it. + """ + provider = TenkiCodeActProvider(sandbox_name="close-retry") + state: dict[str, Any] = {} + context = _StubContext() + await provider.before_run(agent=None, session=None, context=cast(Any, context), state=state) + run_tool = _run_tool_from(context) + await _invoke(run_tool, "print('hi')") + sandbox = fake_sdk.last_sandbox + assert sandbox is not None + + sandbox.close_raises = RuntimeError("terminate rpc unavailable") + await provider.close() + assert sandbox.closed is False + assert provider._live_run_tools.get(run_tool.sandbox_name) is run_tool + + # The transient error cleared — a second close() retries and succeeds. + await provider.close() + assert sandbox.closed is True + assert provider._live_run_tools == {} + + +async def test_provider_async_context_manager_closes_sandbox(fake_sdk: _FakeSandboxFactory) -> None: + async with TenkiCodeActProvider(sandbox_name="prov-cm") as provider: + state: dict[str, Any] = {} + context = _StubContext() + await provider.before_run(agent=None, session=None, context=cast(Any, context), state=state) + run_tool = _run_tool_from(context) + await _invoke(run_tool, "print('x')") + assert fake_sdk.last_sandbox is not None + assert fake_sdk.last_sandbox.closed is False + # Exiting the ``async with`` should have terminated the run tool's sandbox + # (even without an explicit after_run). + assert fake_sdk.last_sandbox is not None + assert fake_sdk.last_sandbox.closed is True + + +async def test_provider_injects_codeact_instructions(fake_sdk: _FakeSandboxFactory) -> None: + provider = TenkiCodeActProvider() + state: dict[str, Any] = {} + context = _StubContext() + await provider.before_run(agent=None, session=None, context=cast(Any, context), state=state) + + assert len(context.instructions_calls) == 1 + source_id, instructions = context.instructions_calls[0] + assert source_id == TenkiCodeActProvider.DEFAULT_SOURCE_ID + # Sanity: the injected instructions still cover the field-observed footguns. + assert "print(" in instructions + assert "python3 -c" in instructions + assert "filesystem persists" in instructions + assert "subprocess" in instructions + assert "!command" in instructions + assert "compound statements" in instructions + assert "with open('/tmp/x.txt') as f: print(f.read())" in instructions + + +async def test_tool_name_and_description() -> None: + tool = TenkiExecuteCodeTool() + assert tool.name == "execute_code" + assert "Tenki" in tool.description + + +# --------------------------------------------------------------------------- +# Integration test — real Tenki service. Requires TENKI_API_KEY. Marked as +# integration so ``pytest -m "not integration"`` (the default suite) skips it. +# Only the integration test is POSIX-only; unit tests above work on any OS. +# --------------------------------------------------------------------------- + + +def _tenki_integration_skip_reason() -> str | None: + if sys.platform == "win32": + return "Tenki integration tests use POSIX-style shell invocations." + if os.environ.get("SKIP_TENKI", "").lower() == "true": + return "SKIP_TENKI=true is set." + if importlib.util.find_spec("tenki_sandbox") is None: + return "tenki-sandbox is not installed." + if not os.environ.get("TENKI_API_KEY"): + return "TENKI_API_KEY is not set." + return None + + +skip_if_tenki_integration_disabled = pytest.mark.skipif( + _tenki_integration_skip_reason() is not None, + reason=_tenki_integration_skip_reason() or "unknown", +) + + +@pytest.mark.integration +@pytest.mark.flaky(reruns=2, reruns_delay=5) +@skip_if_tenki_integration_disabled +async def test_integration_execute_hello_world() -> None: + project_id = os.environ.get("TENKI_PROJECT_ID") + async with TenkiExecuteCodeTool( + sandbox_name=f"agent-framework-ci-{os.getpid()}", + project_id=project_id, + max_duration_seconds=300, + ) as tool: + contents = await _invoke(tool, "print('hello from tenki')") + text_contents = [c for c in contents if c.type == "text"] + assert any("hello from tenki" in (c.text or "") for c in text_contents) + + +@pytest.mark.integration +@pytest.mark.flaky(reruns=2, reruns_delay=5) +@skip_if_tenki_integration_disabled +async def test_integration_filesystem_persists_across_calls() -> None: + """Files written in one ``execute_code`` call are visible in the next.""" + project_id = os.environ.get("TENKI_PROJECT_ID") + async with TenkiExecuteCodeTool( + sandbox_name=f"agent-framework-ci-fs-{os.getpid()}", + project_id=project_id, + max_duration_seconds=300, + ) as tool: + marker = f"hello-{os.getpid()}" + write_contents = await _invoke(tool, f"open('/tmp/marker', 'w').write({marker!r}); print('wrote')") + assert any("wrote" in (c.text or "") for c in write_contents if c.type == "text") + + read_contents = await _invoke(tool, "print(open('/tmp/marker').read())") + assert any(marker in (c.text or "") for c in read_contents if c.type == "text") + + +@pytest.mark.integration +@pytest.mark.flaky(reruns=2, reruns_delay=5) +@skip_if_tenki_integration_disabled +async def test_integration_failure_diagnostics_carry_signal_and_exit_code() -> None: + """A killed subprocess must surface ``signal``/``exit_code`` in the error content.""" + project_id = os.environ.get("TENKI_PROJECT_ID") + async with TenkiExecuteCodeTool( + sandbox_name=f"agent-framework-ci-fail-{os.getpid()}", + project_id=project_id, + max_duration_seconds=300, + ) as tool: + # Force a non-zero exit so we can verify the diagnostic fields flow through. + contents = await _invoke(tool, "import sys; sys.exit(42)") + error_contents = [c for c in contents if c.type == "error"] + assert len(error_contents) == 1 + assert "status 42" in (error_contents[0].message or "") + + +@pytest.mark.integration +@pytest.mark.flaky(reruns=2, reruns_delay=5) +@skip_if_tenki_integration_disabled +async def test_integration_close_removes_sandbox_from_workspace() -> None: + """After ``close()``, the sandbox is gone from Tenki's workspace list.""" + from tenki_sandbox import Client + + project_id = os.environ.get("TENKI_PROJECT_ID") + workspace_id = os.environ.get("TENKI_WORKSPACE_ID") + unique_name = f"agent-framework-ci-teardown-{os.getpid()}" + + async with TenkiExecuteCodeTool( + sandbox_name=unique_name, + project_id=project_id, + max_duration_seconds=300, + ) as tool: + await _invoke(tool, "print('provisioned')") + + # After the async-with block, the sandbox should be terminated. Verify via list. + client = Client() + try: + sandboxes = client.list_workspace(workspace_id) if workspace_id else client.list() + live_names = {sb.info.name for sb in sandboxes if sb.info.state == "RUNNING"} + assert unique_name not in live_names, f"expected {unique_name} to be terminated" + finally: + client.close() diff --git a/python/samples/02-agents/context_providers/code_act/README.md b/python/samples/02-agents/context_providers/code_act/README.md index ea1ff3dc05f..6662c8aee42 100644 --- a/python/samples/02-agents/context_providers/code_act/README.md +++ b/python/samples/02-agents/context_providers/code_act/README.md @@ -1,21 +1,25 @@ # CodeAct context providers -Demonstrates the provider-owned CodeAct flow with two backends: +Demonstrates the provider-owned CodeAct flow with three backends: | File | Backend | Notes | |------|---------|-------| | [`code_act.py`](code_act.py) | [Hyperlight](https://github.com/hyperlight-dev/hyperlight) WASM sandbox via `HyperlightCodeActProvider` | Hardened sandbox with WASM isolation; sandbox tools called via `call_tool(...)`. | | [`monty_code_act.py`](monty_code_act.py) | [Monty](https://github.com/pydantic/monty) Rust-based Python interpreter via `MontyCodeActProvider` (alpha) | Cross-platform pure interpreter; sandbox tools can be called as typed async functions (`await compute(...)`) or via `call_tool(...)`. | +| [`tenki_code_act.py`](tenki_code_act.py) | [Tenki](https://tenki.cloud) managed Linux micro-VM via `TenkiCodeActProvider` (alpha) | Full Linux userland with `subprocess`, `apt`, and a persistent filesystem across `execute_code` calls; in-sandbox tool callbacks not supported. | -Both providers inject an `execute_code` tool into the agent and keep the -registered sandbox tools (`compute`, `fetch_data`) hidden from the model — the -model invokes them from inside the sandbox. +The Hyperlight and Monty providers register sandbox-only tools (`compute`, +`fetch_data`) that the model invokes from inside the sandbox. The Tenki provider +does not register sandbox-scoped host tools — the Tenki SDK has no callback +bridge — but exposes a much wider environment (real subprocesses, network, +package installs). ## Installation ```bash pip install agent-framework agent-framework-hyperlight --pre # Hyperlight sample pip install agent-framework agent-framework-monty --pre # Monty sample +pip install agent-framework agent-framework-tenki --pre # Tenki sample ``` > The Hyperlight Wasm backend is currently published only for `linux/x86_64` and @@ -26,6 +30,11 @@ pip install agent-framework agent-framework-monty --pre # Monty sample > interprets a Python subset (e.g. `os`/network/subprocess access is blocked). > `agent-framework-monty` is an alpha package and is not yet part of > `agent-framework[all]`; install it explicitly with `--pre`. +> +> Tenki runs code in a managed Linux micro-VM on the Tenki service, so it needs +> `TENKI_API_KEY` (and `TENKI_PROJECT_ID` when the key spans multiple projects). +> `agent-framework-tenki` is an alpha package and is not yet part of +> `agent-framework[all]`; install it explicitly with `--pre`. ## Prerequisites @@ -38,6 +47,7 @@ pip install agent-framework agent-framework-monty --pre # Monty sample ```bash python code_act.py # Hyperlight python monty_code_act.py # Monty +python tenki_code_act.py # Tenki (also needs TENKI_API_KEY) ``` See the source files for the full annotated examples. diff --git a/python/samples/02-agents/context_providers/code_act/tenki_code_act.py b/python/samples/02-agents/context_providers/code_act/tenki_code_act.py new file mode 100644 index 00000000000..1edabcdd91a --- /dev/null +++ b/python/samples/02-agents/context_providers/code_act/tenki_code_act.py @@ -0,0 +1,175 @@ +# Copyright (c) Microsoft. All rights reserved. + +from __future__ import annotations + +import asyncio +import logging +import os +import time +from collections.abc import Awaitable, Callable + +from agent_framework import Agent, FunctionInvocationContext, function_middleware +from agent_framework.foundry import FoundryChatClient +from agent_framework_tenki import TenkiCodeActProvider +from azure.identity import AzureCliCredential +from dotenv import load_dotenv + +"""This sample demonstrates the Tenki CodeAct provider. + +``TenkiCodeActProvider`` gives the model an ``execute_code`` tool that runs +Python inside a Tenki managed Linux micro-VM — a real userland with ``apt``, +``subprocess``, and a persistent filesystem across ``execute_code`` calls. The +provider mints a fresh sandbox per agent run and terminates it in ``after_run``, +so state does not leak across runs. + +The sample task exercises Linux-only capabilities (a subprocess pipeline into +``awk``) that in-process CodeAct backends can't run. See sibling +``monty_code_act.py`` for the Monty-backed provider, which trades Linux userland +for zero-provisioning cost. + +Note: ``agent-framework-tenki`` is an alpha package and is not yet part of +``agent-framework[all]``. Install it explicitly with: + + pip install agent-framework agent-framework-tenki --pre + +It is imported as ``agent_framework_tenki`` (no lazy-loading namespace yet). +Requires ``TENKI_API_KEY``; also ``TENKI_PROJECT_ID`` when the key spans +multiple projects. See https://tenki.cloud/docs/sandbox/quick-start-sandbox. +""" + +load_dotenv() + +_CYAN = "\033[36m" +_YELLOW = "\033[33m" +_GREEN = "\033[32m" +_DIM = "\033[2m" +_RESET = "\033[0m" + + +class _ColoredFormatter(logging.Formatter): + """Dim logger output so it does not compete with sample prints.""" + + def format(self, record: logging.LogRecord) -> str: + return f"{_DIM}{super().format(record)}{_RESET}" + + +logging.basicConfig(level=logging.WARNING) +logging.getLogger().handlers[0].setFormatter( + _ColoredFormatter("[%(asctime)s] %(levelname)s: %(message)s"), +) + + +@function_middleware +async def log_function_calls( + context: FunctionInvocationContext, + call_next: Callable[[], Awaitable[None]], +) -> None: + """Log tool calls, including readable execute_code blocks.""" + function_name = context.function.name + arguments = context.arguments if isinstance(context.arguments, dict) else {} + + if function_name == "execute_code" and "code" in arguments: + print(f"\n{_YELLOW}{'─' * 60}") + print("▶ execute_code") + print(f"{'─' * 60}{_RESET}") + print(arguments["code"]) + print(f"{_YELLOW}{'─' * 60}{_RESET}") + else: + pairs = ", ".join(f"{name}={value!r}" for name, value in arguments.items()) + print(f"\n{_YELLOW}▶ {function_name}({pairs}){_RESET}") + + start = time.perf_counter() + await call_next() + elapsed = time.perf_counter() - start + + result = context.result + if function_name == "execute_code" and isinstance(result, list): + for output in result: + if output.type == "text" and output.text: + print(f"{_GREEN}stdout:\n{output.text}{_RESET}") + elif output.type == "error" and output.error_details: + print(f"{_YELLOW}stderr:\n{output.error_details}{_RESET}") + else: + print(f"{_YELLOW}◀ {function_name} → {result!r}{_RESET}") + + print(f"{_DIM} ({elapsed:.4f}s){_RESET}") + + +async def main() -> None: + """Run the Tenki CodeAct provider sample.""" + # 1. Create the Tenki-backed provider. Explicit ``sandbox_name`` becomes the + # prefix for run-scoped sandboxes (e.g. "tenki-codeact-sample-<8-hex>"), + # making them easy to identify in the Tenki dashboard. + async with TenkiCodeActProvider(sandbox_name="tenki-codeact-sample") as codeact: + # 2. Create the client and the agent. + agent = Agent( + client=FoundryChatClient( + project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], + model=os.environ["FOUNDRY_MODEL"], + credential=AzureCliCredential(), + ), + name="TenkiCodeActProviderAgent", + instructions="You are a helpful assistant.", + context_providers=[codeact], + middleware=[log_function_calls], + ) + + # 3. Run a task that exercises the Linux userland Tenki gives us — + # a real subprocess pipeline into ``awk``, and file persistence + # across two ``execute_code`` calls. + query = ( + "Using execute_code, do this in TWO calls: " + "(1) write the numbers 1..100, one per line, to /tmp/numbers.txt " + "and print 'wrote'. " + "(2) shell out to `awk` via `subprocess.run(['awk', ...])` to sum " + "the squares of every number in /tmp/numbers.txt, then print the " + "result as 'sum_of_squares='." + ) + print(f"{_CYAN}{'=' * 60}") + print("Tenki CodeAct provider sample") + print(f"{'=' * 60}{_RESET}") + print(f"{_CYAN}User: {query}{_RESET}") + result = await agent.run(query) + print(f"{_CYAN}Agent: {result.text}{_RESET}") + + +""" +Sample output (shape only): + +============================================================ +Tenki CodeAct provider sample +============================================================ +User: Using execute_code, do this in TWO calls: ... + +──────────────────────────────────────────────────────────── +▶ execute_code +──────────────────────────────────────────────────────────── +with open('/tmp/numbers.txt', 'w') as f: + for n in range(1, 101): + f.write(f"{n}\n") +print('wrote') +──────────────────────────────────────────────────────────── +stdout: +wrote + (3.5xxx s) # first call pays the sandbox-provision cost + +──────────────────────────────────────────────────────────── +▶ execute_code +──────────────────────────────────────────────────────────── +import subprocess +result = subprocess.run( + ['awk', '{s += $1 * $1} END {print s}', '/tmp/numbers.txt'], + capture_output=True, text=True, check=True, +) +print(f"sum_of_squares={result.stdout.strip()}") +──────────────────────────────────────────────────────────── +stdout: +sum_of_squares=338350 + (0.5xxx s) # second call reuses the same sandbox + +Agent: The sum of squares from 1 to 100 is 338350. +""" + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/python/uv.lock b/python/uv.lock index f8b85461ecd..f804dfbfa44 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -929,13 +929,19 @@ requires-dist = [ [[package]] name = "agent-framework-tenki" -version = "0.1.0a260720" +version = "1.0.0a260722" source = { editable = "packages/tenki" } dependencies = [ { name = "agent-framework-core" }, { name = "tenki-sandbox" }, ] +[package.metadata] +requires-dist = [ + { name = "agent-framework-core", editable = "packages/core" }, + { name = "tenki-sandbox", specifier = ">=0.4.0,<0.5" }, +] + [[package]] name = "agent-framework-tools" version = "1.0.0a260630" @@ -7989,16 +7995,16 @@ wheels = [ [[package]] name = "tenki-sandbox" -version = "0.3.6" +version = "0.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "grpcio" }, { name = "protobuf" }, { name = "websocket-client" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c8/3e/319d553fb6bd266aed3f5e0ec5d9b1865ee7cb45388c9dc6dac1ee30a51f/tenki_sandbox-0.3.6.tar.gz", hash = "sha256:f4a6e0b7329a7fd42138d5eb582adb1356c084e6f6b3fe046530cbbcd426ffe2", size = 144662, upload-time = "2026-07-10T08:28:16.269Z" } +sdist = { url = "https://files.pythonhosted.org/packages/8d/58/f6527c63b2e4c94fd00a48ba4bd93ef52de22dbc72e247110949a17511f6/tenki_sandbox-0.4.0.tar.gz", hash = "sha256:2836e6e7100dfc81715b4d93ecfe80e3f055867498cb9f34be921a02969bb15f", size = 179392, upload-time = "2026-07-17T22:18:09.869Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e5/4f/860d1ba154200904b493ed4353e1994aa14627f407144d975b21e4cf51be/tenki_sandbox-0.3.6-py3-none-any.whl", hash = "sha256:324fe804a0b561ffa129e6b1b1874e0d7747ebcb566d976e8485c9dd45ef6853", size = 128658, upload-time = "2026-07-10T08:28:14.925Z" }, + { url = "https://files.pythonhosted.org/packages/3b/f7/3c02da98793dc0e56230c520064d7d233c6e0edab13b410c031e1cfe7fd9/tenki_sandbox-0.4.0-py3-none-any.whl", hash = "sha256:76d5ece2e1607b43705587d86277e983ef43601a86993077e379be8033a40b3e", size = 155246, upload-time = "2026-07-17T22:18:08.383Z" }, ] [[package]] From babf99c040689984094a7c0a2e0ef977f9e3ea2f Mon Sep 17 00:00:00 2001 From: Patricio Leonel Filice Date: Thu, 23 Jul 2026 17:49:55 -0300 Subject: [PATCH 3/5] Python: fix: address second-round review feedback for agent-framework-tenki - Hold the sandbox lock across reconcile + exec so close() cannot terminate a sandbox while an exec is in flight - Add pause_retention_seconds kwarg (tool + provider); run-scoped sandboxes default to 1h retention so orphans are GC'd server-side - Provider close() raises RuntimeError after attempting all terminates, retaining failed handles for retry - Harden the teardown integration test: assert provisioning succeeded and poll the live API until TERMINATED - Clarify README max_duration None-semantics and pause retention Co-Authored-By: Claude Opus 4.7 --- python/packages/tenki/README.md | 7 +- .../_execute_code_tool.py | 152 ++++++++++----- .../tenki/agent_framework_tenki/_provider.py | 22 ++- .../tenki/tests/tenki/test_tenki_codeact.py | 174 ++++++++++++++++-- 4 files changed, 291 insertions(+), 64 deletions(-) diff --git a/python/packages/tenki/README.md b/python/packages/tenki/README.md index 9914fc5b705..45efadf5621 100644 --- a/python/packages/tenki/README.md +++ b/python/packages/tenki/README.md @@ -69,7 +69,8 @@ policies stop it. | `image` | Tenki default | Custom base image identifier. | | `project_id` / `workspace_id` | `os.environ.get("TENKI_PROJECT_ID")` / `os.environ.get("TENKI_WORKSPACE_ID")` | Required when your API key has access to multiple projects. Constructor args override the env vars. | | `cpu_cores` / `memory_mb` / `disk_size_gb` | Tenki defaults | Optional resource overrides. | -| `max_duration_seconds` | `900` (15 min) | Server-side duration cap. On expiry, project/workspace-scoped sandboxes are **paused** and unscoped ones are **terminated**; in both cases compute billing stops, including when the parent process crashed before calling `close()`. Pass a larger value for longer evals, or `None` to opt out (not recommended). | +| `max_duration_seconds` | `900` (15 min) | Server-side duration cap. On expiry, project/workspace-scoped sandboxes are **paused** and unscoped ones are **terminated**; in both cases compute billing stops, including when the parent process crashed before calling `close()`. Pass a larger value for longer evals, or `None` to opt out (not recommended). `None` only opts out on the standalone tool — through `TenkiCodeActProvider` it means "use the 900s default"; the provider offers no opt-out. | +| `pause_retention_seconds` | Tenki default (7 days) standalone; `3600` (1 h) for provider run-scoped sandboxes | How long Tenki retains a stopped sandbox's pause snapshot (`PAUSED`/`USER_SHUTDOWN`) before the server GC deletes it — snapshot storage bills until then, and the sandbox cannot be resumed afterwards. The short run-scoped default caps what an orphaned sandbox (crashed run) can cost. | | `exec_timeout_seconds` | `60` | Per-`execute_code` invocation timeout in seconds. | | `extra_create_kwargs` | `{}` | Passed straight to `tenki_sandbox.Sandbox.create` for Tenki-specific options — see the section below. | @@ -135,7 +136,9 @@ re-provisioning as described below. Each call runs `python3 -c (the guest OS was shut down from inside the VM) — note that this resume can take a minute or more, since Tenki captures the shutdown sandbox's disk asynchronously and the resume waits for that capture to complete. Filesystem - and installed packages carry across the pause unchanged. + and installed packages carry across the pause unchanged. Resuming is only + possible while the pause snapshot is retained (see `pause_retention_seconds` + above); after retention expires the sandbox is gone for good. - **Terminated sandboxes are replaced** — if the sandbox transitions to `TERMINATING`/`TERMINATED` (workspace timeout, `max_duration_seconds` expiring on an unscoped sandbox, or an external `tenki sandbox terminate`), diff --git a/python/packages/tenki/agent_framework_tenki/_execute_code_tool.py b/python/packages/tenki/agent_framework_tenki/_execute_code_tool.py index e16f6090aa9..f09da8c87e6 100644 --- a/python/packages/tenki/agent_framework_tenki/_execute_code_tool.py +++ b/python/packages/tenki/agent_framework_tenki/_execute_code_tool.py @@ -58,6 +58,14 @@ _RESUME_POLL_TIMEOUT_SECONDS = 120.0 _RESUME_POLL_INTERVAL_SECONDS = 0.5 +# Pause-snapshot retention applied to run-scoped tools minted by +# ``create_run_tool`` when the caller does not set ``pause_retention_seconds``. +# A run-scoped sandbox that outlives its run only exists because run cleanup +# failed (crash, cancellation, abandoned stream), and nothing will ever resume +# it — a short retention lets Tenki's GC delete its pause snapshot (which bills +# storage while retained) after an hour instead of the server default (7 days). +_DEFAULT_RUN_SCOPED_PAUSE_RETENTION_SECONDS = 60 * 60 + # State groupings (SDK returns strings). ``PAUSED`` and ``USER_SHUTDOWN`` are # both stopped-but-resumable server-side: filesystem state is retained and # ``resume()`` is a legal transition. Only ``TERMINATED`` is truly terminal; @@ -72,6 +80,15 @@ def _generate_sandbox_name(prefix: str) -> str: return f"{prefix}-{uuid.uuid4().hex[:8]}" +class _SandboxPreparationError(RuntimeError): + """The sandbox could not be provisioned, resumed, or reconciled before exec. + + Distinguishes preparation failures from ``sandbox.exec`` failures now that + both run inside the same worker-thread call (`_execute_sync`), so + `_run_code` can keep reporting them under separate error headlines. + """ + + class TenkiExecuteCodeTool(FunctionTool): r"""Execute Python code inside a `Tenki `_ managed microVM sandbox. @@ -127,6 +144,14 @@ class TenkiExecuteCodeTool(FunctionTool): retained) and terminates unscoped ones. Pass an explicit value for longer evals; pass ``None`` to explicitly opt out (not recommended in production). + pause_retention_seconds (int | None): How long Tenki retains the pause + snapshot of a stopped sandbox (``PAUSED`` / ``USER_SHUTDOWN``) before + the server's retention GC deletes it — snapshot storage bills until + then. ``None`` (the default) uses the Tenki server default (7 days). + Run-scoped tools minted by `create_run_tool` default to 1 hour + instead, so a sandbox orphaned by a crashed run does not bill + storage for a week; note that once the snapshot is GC'd the sandbox + can no longer be resumed. exec_timeout_seconds (int): Per-``execute_code`` timeout in seconds. Defaults to 60. extra_create_kwargs (dict[str, Any] | None): Optional keyword arguments passed straight to `tenki_sandbox.Sandbox.create` — for advanced knobs not @@ -139,12 +164,14 @@ class TenkiExecuteCodeTool(FunctionTool): Tenki SDK does not expose a callback bridge). * File mounts and outbound network allow-lists are not surfaced as first-class kwargs; pass them through ``extra_create_kwargs``. - * Cancellation latency: ``execute_code`` runs ``sandbox.exec`` inside - ``asyncio.to_thread``. Python cannot cancel an in-flight thread, so a - cancel raised during exec propagates to the awaiter immediately but the - underlying sandbox call keeps running server-side until it completes - or hits ``exec_timeout_seconds``. Worst-case wall time from cancel to - full quiescence is therefore ``exec_timeout_seconds``. + * Cancellation latency: ``execute_code`` runs the reconcile and + ``sandbox.exec`` inside ``asyncio.to_thread`` under the sandbox lock. + Python cannot cancel an in-flight thread, so a cancel raised during + exec propagates to the awaiter immediately but the underlying sandbox + call keeps running server-side until it completes or hits + ``exec_timeout_seconds`` — and a subsequent `close` blocks on the same + lock until then. Worst-case wall time from cancel to full quiescence + is therefore ``exec_timeout_seconds``. """ SUPPORTED_LANGUAGES: ClassVar[list[str]] = ["python"] @@ -162,6 +189,7 @@ def __init__( memory_mb: int | None = None, disk_size_gb: int | None = None, max_duration_seconds: int | None = _DEFAULT_MAX_DURATION_SECONDS, + pause_retention_seconds: int | None = None, exec_timeout_seconds: int = 60, extra_create_kwargs: dict[str, Any] | None = None, _sandbox_name_prefix: str | None = None, @@ -198,6 +226,7 @@ def __init__( self._memory_mb = memory_mb self._disk_size_gb = disk_size_gb self._max_duration_seconds = max_duration_seconds + self._pause_retention_seconds = pause_retention_seconds self._exec_timeout_seconds = exec_timeout_seconds self._extra_create_kwargs: dict[str, Any] = dict(extra_create_kwargs) if extra_create_kwargs else {} @@ -222,9 +251,15 @@ def create_run_tool(self) -> TenkiExecuteCodeTool: Called by `TenkiCodeActProvider.before_run` to create a per-agent-run tool. The returned tool has a unique ``-<8-hex>`` sandbox name derived from the user's explicit ``sandbox_name`` (used as prefix) or the default - ``agent-framework`` prefix. All other kwargs are copied verbatim. + ``agent-framework`` prefix. ``pause_retention_seconds`` defaults to 1 hour + when not set explicitly (run-scoped sandboxes that outlive their run are + orphans of a failed cleanup; see `_DEFAULT_RUN_SCOPED_PAUSE_RETENTION_SECONDS`). + All other kwargs are copied verbatim. """ prefix = self._explicit_sandbox_name if self._explicit_sandbox_name is not None else self._sandbox_name_prefix + pause_retention_seconds = self._pause_retention_seconds + if pause_retention_seconds is None: + pause_retention_seconds = _DEFAULT_RUN_SCOPED_PAUSE_RETENTION_SECONDS # ``FunctionTool.__init__`` stores ``approval_mode`` after applying its own # ``or "never_require"`` fallback, so the runtime value is always a valid # ``ApprovalMode``. Cast so the constructor's stricter ``ApprovalMode | None`` @@ -239,6 +274,7 @@ def create_run_tool(self) -> TenkiExecuteCodeTool: memory_mb=self._memory_mb, disk_size_gb=self._disk_size_gb, max_duration_seconds=self._max_duration_seconds, + pause_retention_seconds=pause_retention_seconds, exec_timeout_seconds=self._exec_timeout_seconds, extra_create_kwargs=dict(self._extra_create_kwargs), _sandbox_name_prefix=prefix, @@ -276,14 +312,31 @@ def _build_create_kwargs(self) -> dict[str, Any]: kwargs["disk_size_gb"] = self._disk_size_gb if self._max_duration_seconds is not None: kwargs["max_duration"] = self._max_duration_seconds + if self._pause_retention_seconds is not None: + kwargs["pause_retention"] = self._pause_retention_seconds # Caller-supplied extras win last so they can override anything above. kwargs.update(self._extra_create_kwargs) return kwargs - def _ensure_sandbox_sync(self) -> Sandbox: + def _execute_sync(self, code: str) -> CommandResult: + """Reconcile sandbox state and execute the code under a single lock hold. + + Called inside ``asyncio.to_thread``. Holding ``_sandbox_lock`` across + both the reconcile and ``sandbox.exec`` means `close` (which takes the + same lock) cannot terminate the sandbox or close its gRPC client while + an exec is still in flight — it blocks until the exec finishes, bounded + by ``exec_timeout_seconds``. Concurrent ``execute_code`` calls serialize + on the lock; the sandbox is a single microVM either way. + """ + with self._sandbox_lock: + sandbox = self._ensure_sandbox_locked() + return sandbox.exec("python3", "-c", code, timeout=self._exec_timeout_seconds) + + def _ensure_sandbox_locked(self) -> Sandbox: """Return a runnable sandbox — provision on first use, resume if stopped, re-provision if gone. - Called inside ``asyncio.to_thread``. Reconciles remote state on every call: + ``_sandbox_lock`` must be held by the caller. Reconciles remote state on + every call: - ``TERMINATING`` / ``TERMINATED`` → drop the handle, provision a fresh sandbox (auto-generated names refresh their suffix; explicit names are @@ -292,31 +345,31 @@ def _ensure_sandbox_sync(self) -> Sandbox: resumes stopped-but-resumable sandboxes (``PAUSED`` / ``USER_SHUTDOWN``) and polls transitional states (``PAUSING`` / ``RESUMING`` / ``CREATING``) until ``RUNNING``. - - ``refresh()`` raising → surface as ``RuntimeError`` without dropping the - handle. Next call retries the same sandbox rather than leaking it. + - ``refresh()`` raising → surface as `_SandboxPreparationError` without + dropping the handle. Next call retries the same sandbox rather than + leaking it. """ - with self._sandbox_lock: - sandbox = self._sandbox - if sandbox is None: - return self._create_sandbox_locked() + sandbox = self._sandbox + if sandbox is None: + return self._create_sandbox_locked() - try: - sandbox.refresh() - except Exception as exc: - # Do not drop the handle — the sandbox may still be alive. - # Surface the error so the next call retries the same session - # rather than provisioning a duplicate. - raise RuntimeError(f"Failed to refresh Tenki sandbox state: {exc}") from exc - - if sandbox.state in _TERMINAL_STATES: - logger.debug("Tenki sandbox reached remote state=%s; re-provisioning", sandbox.state) - # The session can no longer be resumed or terminated again, but the - # SDK-owned control-plane channel is still open on our side. - self._close_owning_client_sync(sandbox) - self._sandbox = None - self._refresh_autogenerated_name_locked() - return self._create_sandbox_locked() - return self._wait_for_running_locked(sandbox) + try: + sandbox.refresh() + except Exception as exc: + # Do not drop the handle — the sandbox may still be alive. + # Surface the error so the next call retries the same session + # rather than provisioning a duplicate. + raise _SandboxPreparationError(f"Failed to refresh Tenki sandbox state: {exc}") from exc + + if sandbox.state in _TERMINAL_STATES: + logger.debug("Tenki sandbox reached remote state=%s; re-provisioning", sandbox.state) + # The session can no longer be resumed or terminated again, but the + # SDK-owned control-plane channel is still open on our side. + self._close_owning_client_sync(sandbox) + self._sandbox = None + self._refresh_autogenerated_name_locked() + return self._create_sandbox_locked() + return self._wait_for_running_locked(sandbox) def _wait_for_running_locked(self, sandbox: Sandbox) -> Sandbox: """Poll until the sandbox reaches ``RUNNING``, resuming it if stopped. Lock must be held. @@ -331,8 +384,8 @@ def _wait_for_running_locked(self, sandbox: Sandbox) -> Sandbox: ``USER_SHUTDOWN``/``PAUSED``), which the loop sees as the sandbox settling back and simply retries. Unknown states are treated as transitional so an SDK upgrade that adds new states doesn't fail - closed. Raises ``RuntimeError`` on timeout so the caller sees a clear - error rather than a "sandbox is not RUNNING" exec failure. + closed. Raises `_SandboxPreparationError` on timeout so the caller sees + a clear error rather than a "sandbox is not RUNNING" exec failure. Concurrency: the sandbox lock is held for the full poll window (up to `_RESUME_POLL_TIMEOUT_SECONDS`); concurrent ``execute_code`` calls block @@ -344,13 +397,15 @@ def _wait_for_running_locked(self, sandbox: Sandbox) -> Sandbox: if state == _RUNNABLE_STATE: return sandbox if state in _TERMINAL_STATES: - raise RuntimeError(f"Tenki sandbox reached state={state} while waiting for RUNNING.") + raise _SandboxPreparationError(f"Tenki sandbox reached state={state} while waiting for RUNNING.") if state in _RESUMABLE_STATES: try: sandbox.resume() except Exception as exc: if time.monotonic() >= deadline: - raise RuntimeError(f"Failed to resume Tenki sandbox from state={state}: {exc}") from exc + raise _SandboxPreparationError( + f"Failed to resume Tenki sandbox from state={state}: {exc}" + ) from exc logger.debug("Tenki sandbox resume from state=%s failed; retrying: %s", state, exc) else: # Local state is now RESUMING; fall through to poll. If the @@ -358,14 +413,14 @@ def _wait_for_running_locked(self, sandbox: Sandbox) -> Sandbox: # resumable state again and the loop retries. continue if time.monotonic() >= deadline: - raise RuntimeError( + raise _SandboxPreparationError( f"Tenki sandbox did not reach RUNNING within {_RESUME_POLL_TIMEOUT_SECONDS}s (last state={state})." ) time.sleep(_RESUME_POLL_INTERVAL_SECONDS) try: sandbox.refresh() except Exception as exc: - raise RuntimeError(f"Failed to refresh Tenki sandbox state while polling: {exc}") from exc + raise _SandboxPreparationError(f"Failed to refresh Tenki sandbox state while polling: {exc}") from exc def _refresh_autogenerated_name_locked(self) -> None: """Mint a fresh suffix on re-provision for auto-generated names. @@ -383,22 +438,19 @@ def _create_sandbox_locked(self) -> Sandbox: try: sandbox = Sandbox.create(**create_kwargs) # pyright: ignore[reportUnknownMemberType] except Exception as exc: - raise RuntimeError(f"Failed to create Tenki sandbox: {exc}") from exc + raise _SandboxPreparationError(f"Failed to create Tenki sandbox: {exc}") from exc self._sandbox = sandbox return sandbox async def _run_code(self, *, code: str) -> list[Content]: """Execute a single block of Python code inside the sandbox.""" try: - sandbox = await asyncio.to_thread(self._ensure_sandbox_sync) - except RuntimeError as exc: - return [Content.from_error(message="Failed to prepare Tenki sandbox", error_details=str(exc))] - - try: - result = await asyncio.to_thread(sandbox.exec, "python3", "-c", code, timeout=self._exec_timeout_seconds) + result = await asyncio.to_thread(self._execute_sync, code) except asyncio.CancelledError: # Cancellation must propagate so higher-level workflows can stop promptly. raise + except _SandboxPreparationError as exc: + return [Content.from_error(message="Failed to prepare Tenki sandbox", error_details=str(exc))] except Exception as exc: return [Content.from_error(message="Sandbox execution failed", error_details=str(exc))] @@ -476,9 +528,11 @@ def _close_sync(self) -> None: """Sync body of `close`, run inside ``asyncio.to_thread``. Holds ``_sandbox_lock`` for the whole terminate so it never races - `_ensure_sandbox_sync`, and — like all lock acquisitions in this class — - only ever blocks a worker thread, never the event loop (a reconcile can - hold the lock for up to the resume-poll budget). + `_execute_sync` — an in-flight reconcile or exec finishes before the + terminate starts — and, like all lock acquisitions in this class, only + ever blocks a worker thread, never the event loop (a reconcile can hold + the lock for up to the resume-poll budget; an exec for up to + ``exec_timeout_seconds``). """ with self._sandbox_lock: sandbox = self._sandbox @@ -498,7 +552,7 @@ def _close_owning_client_sync(self, sandbox: Sandbox) -> None: """Close the SDK-owned control-plane `tenki_sandbox.Client`. Called both after a successful ``close_if_open`` and from the - terminal-state re-provision branch in `_ensure_sandbox_sync`. The + terminal-state re-provision branch in `_ensure_sandbox_locked`. The SDK's own ``Sandbox.close()`` only closes the data-plane RPC, so without this the gRPC control channel would stay open until the process exits — a real leak under long-lived standalone tools that diff --git a/python/packages/tenki/agent_framework_tenki/_provider.py b/python/packages/tenki/agent_framework_tenki/_provider.py index e0827470673..9e34c290d9c 100644 --- a/python/packages/tenki/agent_framework_tenki/_provider.py +++ b/python/packages/tenki/agent_framework_tenki/_provider.py @@ -58,7 +58,10 @@ class TenkiCodeActProvider(ContextProvider): ``max_duration_seconds=None`` (the default) means "use the tool's own default" (see ``TenkiExecuteCodeTool``); there is no way to opt out of a max duration - through the provider. + through the provider. Likewise ``pause_retention_seconds=None`` (the default) + means "use the run-scoped default" (1 hour): a run-scoped sandbox that + outlives its run is an orphan of a failed cleanup, so its pause snapshot is + GC'd by Tenki after an hour instead of the server default of 7 days. """ DEFAULT_SOURCE_ID = "tenki_codeact" @@ -77,6 +80,7 @@ def __init__( memory_mb: int | None = None, disk_size_gb: int | None = None, max_duration_seconds: int | None = None, + pause_retention_seconds: int | None = None, exec_timeout_seconds: int = 60, extra_create_kwargs: dict[str, Any] | None = None, ) -> None: @@ -93,6 +97,7 @@ def __init__( "cpu_cores": cpu_cores, "memory_mb": memory_mb, "disk_size_gb": disk_size_gb, + "pause_retention_seconds": pause_retention_seconds, "exec_timeout_seconds": exec_timeout_seconds, "extra_create_kwargs": extra_create_kwargs, } @@ -121,6 +126,11 @@ async def close(self) -> None: explicit ``close()``). Sandboxes for completed runs are already terminated by ``after_run``; this cleans up any that leaked past that hook due to an in-run exception. + + Raises: + RuntimeError: If terminating one or more run-scoped sandboxes failed + — raised only after every close was attempted. Failed entries + stay tracked, so a second ``close()`` retries exactly those. """ leaked = list(self._live_run_tools.items()) # Terminate orphans in parallel — each ``close()`` is one terminate RPC @@ -130,6 +140,7 @@ async def close(self) -> None: # second ``close()`` — mirroring the handle-preserving contract of # ``TenkiExecuteCodeTool.close`` and ``after_run``. results = await asyncio.gather(*(tool.close() for _, tool in leaked), return_exceptions=True) + failures: list[tuple[str, BaseException]] = [] for (sandbox_name, _), result in zip(leaked, results, strict=True): if isinstance(result, BaseException): logger.warning( @@ -137,11 +148,20 @@ async def close(self) -> None: sandbox_name, result, ) + failures.append((sandbox_name, result)) else: self._live_run_tools.pop(sandbox_name, None) # Base template tool never provisions, so this is a no-op in practice — # kept for symmetry with the standalone-tool ``async with`` pattern. await self._execute_code_tool.close() + if failures: + # Surface the failure so ``async with`` callers learn a sandbox may + # still be live; every close was already attempted above. + names = ", ".join(sorted(name for name, _ in failures)) + raise RuntimeError( + f"Failed to terminate {len(failures)} run-scoped Tenki sandbox(es): {names}. " + "The handles are retained; call close() again to retry." + ) from failures[0][1] async def __aenter__(self) -> TenkiCodeActProvider: return self diff --git a/python/packages/tenki/tests/tenki/test_tenki_codeact.py b/python/packages/tenki/tests/tenki/test_tenki_codeact.py index 2a2fbfbce21..1a647741ad6 100644 --- a/python/packages/tenki/tests/tenki/test_tenki_codeact.py +++ b/python/packages/tenki/tests/tenki/test_tenki_codeact.py @@ -7,6 +7,7 @@ import os import sys import threading +import time from types import SimpleNamespace from typing import Any, cast @@ -285,6 +286,42 @@ async def test_max_duration_opt_out(fake_sdk: _FakeSandboxFactory) -> None: assert "max_duration" not in fake_sdk.create_calls[0] +async def test_pause_retention_omitted_by_default_standalone(fake_sdk: _FakeSandboxFactory) -> None: + """A standalone tool leaves pause retention to the Tenki server default (7 days). + + A user who deliberately pauses a long-lived standalone sandbox must not have + its snapshot GC'd early by a default they never asked for. + """ + tool = TenkiExecuteCodeTool(sandbox_name="retention-default") + await _invoke(tool, "pass") + assert "pause_retention" not in fake_sdk.create_calls[0] + + +async def test_pause_retention_forwarded_when_set(fake_sdk: _FakeSandboxFactory) -> None: + tool = TenkiExecuteCodeTool(sandbox_name="retention-explicit", pause_retention_seconds=120) + await _invoke(tool, "pass") + assert fake_sdk.create_calls[0]["pause_retention"] == 120 + + +async def test_run_tool_defaults_to_short_pause_retention(fake_sdk: _FakeSandboxFactory) -> None: + """Run-scoped sandboxes get a short pause retention by default. + + A run-scoped sandbox that outlives its run is an orphan of a failed cleanup + (crash / cancellation / abandoned stream) — nothing will resume it, so its + pause snapshot should be GC'd after an hour, not the server's 7-day default. + """ + run_tool = TenkiExecuteCodeTool(sandbox_name="prefix").create_run_tool() + await _invoke(run_tool, "pass") + expected = _tenki_module._DEFAULT_RUN_SCOPED_PAUSE_RETENTION_SECONDS + assert fake_sdk.create_calls[0]["pause_retention"] == expected + + +async def test_run_tool_preserves_explicit_pause_retention(fake_sdk: _FakeSandboxFactory) -> None: + run_tool = TenkiExecuteCodeTool(sandbox_name="prefix", pause_retention_seconds=7200).create_run_tool() + await _invoke(run_tool, "pass") + assert fake_sdk.create_calls[0]["pause_retention"] == 7200 + + async def test_env_api_key_is_forwarded_as_auth_token( fake_sdk: _FakeSandboxFactory, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -916,6 +953,48 @@ def slow_close() -> None: assert second.closed is False +async def test_close_waits_for_in_flight_exec(fake_sdk: _FakeSandboxFactory) -> None: + """``close()`` must block until an in-flight exec finishes, not terminate under it. + + Reconcile + exec run under the sandbox lock in a worker thread; ``close()`` + takes the same lock, so it cannot terminate the sandbox (or close its gRPC + client) while ``sandbox.exec`` is still running. + """ + tool = TenkiExecuteCodeTool(sandbox_name="exec-close") + await _invoke(tool, "print('warm-up')") + sandbox = fake_sdk.last_sandbox + assert sandbox is not None + + exec_entered = threading.Event() + release_exec = threading.Event() + + def blocking_exec(args: Any, kwargs: Any) -> _FakeExecResult: + exec_entered.set() + assert release_exec.wait(timeout=5), "exec was never released" + return _FakeExecResult(stdout_text="slow done\n") + + sandbox.script_handler = blocking_exec + + invoke_task = asyncio.create_task(_invoke(tool, "print('slow')")) + assert await asyncio.to_thread(exec_entered.wait, 5) + + close_task = asyncio.create_task(tool.close()) + await asyncio.sleep(0.05) + # The exec is mid-flight — close() must be parked on the lock, sandbox untouched. + assert not close_task.done() + assert sandbox.closed is False + assert sandbox._client.closed is False + + release_exec.set() + contents = await invoke_task + await close_task + + # The exec completed normally *before* the terminate ran. + assert any("slow done" in (c.text or "") for c in contents if c.type == "text") + assert sandbox.closed is True + assert sandbox._client.closed is True + + async def test_close_is_idempotent(fake_sdk: _FakeSandboxFactory) -> None: tool = TenkiExecuteCodeTool(sandbox_name="idempotent") await tool.close() # no sandbox yet — no-op @@ -1021,6 +1100,32 @@ async def test_provider_run_tool_uses_default_prefix_when_unset( assert run_tool.sandbox_name.startswith("agent-framework-") +async def test_provider_run_tool_gets_short_pause_retention( + fake_sdk: _FakeSandboxFactory, +) -> None: + """Provider-minted run tools inherit the short run-scoped pause retention.""" + provider = TenkiCodeActProvider(sandbox_name="retention-prov") + state: dict[str, Any] = {} + context = _StubContext() + await provider.before_run(agent=None, session=None, context=cast(Any, context), state=state) + run_tool = _run_tool_from(context) + await _invoke(run_tool, "pass") + expected = _tenki_module._DEFAULT_RUN_SCOPED_PAUSE_RETENTION_SECONDS + assert fake_sdk.create_calls[0]["pause_retention"] == expected + + +async def test_provider_explicit_pause_retention_reaches_run_tool( + fake_sdk: _FakeSandboxFactory, +) -> None: + provider = TenkiCodeActProvider(sandbox_name="retention-prov", pause_retention_seconds=1800) + state: dict[str, Any] = {} + context = _StubContext() + await provider.before_run(agent=None, session=None, context=cast(Any, context), state=state) + run_tool = _run_tool_from(context) + await _invoke(run_tool, "pass") + assert fake_sdk.create_calls[0]["pause_retention"] == 1800 + + async def test_provider_after_run_terminates_run_scoped_sandbox( fake_sdk: _FakeSandboxFactory, ) -> None: @@ -1122,11 +1227,12 @@ async def test_provider_close_cleans_up_orphaned_run_tools( async def test_provider_close_retains_tool_on_failure_and_retries( fake_sdk: _FakeSandboxFactory, ) -> None: - """A terminate that fails during ``close()`` must stay in the live-set. + """A terminate that fails during ``close()`` must stay in the live-set and raise. - ``close()`` removes entries only on success — dropping the reference on - failure would make a second ``close()`` a no-op and leak the microVM - until ``max_duration`` reaps it. + ``close()`` removes entries only on success and surfaces the failure to the + caller — a silently-swallowed error would tell ``async with`` users the + cleanup succeeded while the microVM is still live, and dropping the + reference would make a second ``close()`` a no-op. """ provider = TenkiCodeActProvider(sandbox_name="close-retry") state: dict[str, Any] = {} @@ -1138,7 +1244,8 @@ async def test_provider_close_retains_tool_on_failure_and_retries( assert sandbox is not None sandbox.close_raises = RuntimeError("terminate rpc unavailable") - await provider.close() + with pytest.raises(RuntimeError, match="run-scoped Tenki sandbox"): + await provider.close() assert sandbox.closed is False assert provider._live_run_tools.get(run_tool.sandbox_name) is run_tool @@ -1148,6 +1255,32 @@ async def test_provider_close_retains_tool_on_failure_and_retries( assert provider._live_run_tools == {} +async def test_provider_close_attempts_every_tool_before_raising( + fake_sdk: _FakeSandboxFactory, +) -> None: + """One failing terminate must not stop ``close()`` from closing the others.""" + provider = TenkiCodeActProvider(sandbox_name="close-all") + + tools: list[TenkiExecuteCodeTool] = [] + for _ in range(2): + state: dict[str, Any] = {} + context = _StubContext() + await provider.before_run(agent=None, session=None, context=cast(Any, context), state=state) + run_tool = _run_tool_from(context) + await _invoke(run_tool, "print('hi')") + tools.append(run_tool) + failing_sandbox, ok_sandbox = fake_sdk.sandboxes[0], fake_sdk.sandboxes[1] + + failing_sandbox.close_raises = RuntimeError("terminate rpc unavailable") + with pytest.raises(RuntimeError, match="1 run-scoped Tenki sandbox"): + await provider.close() + + # The healthy sandbox was still closed; only the failed one stays tracked. + assert ok_sandbox.closed is True + assert failing_sandbox.closed is False + assert set(provider._live_run_tools) == {tools[0].sandbox_name} + + async def test_provider_async_context_manager_closes_sandbox(fake_sdk: _FakeSandboxFactory) -> None: async with TenkiCodeActProvider(sandbox_name="prov-cm") as provider: state: dict[str, Any] = {} @@ -1269,11 +1402,17 @@ async def test_integration_failure_diagnostics_carry_signal_and_exit_code() -> N @pytest.mark.flaky(reruns=2, reruns_delay=5) @skip_if_tenki_integration_disabled async def test_integration_close_removes_sandbox_from_workspace() -> None: - """After ``close()``, the sandbox is gone from Tenki's workspace list.""" + """After ``close()``, the sandbox reaches ``TERMINATED`` server-side. + + Guards against two vacuous passes: provisioning/exec failure (the old + version never asserted the invoke succeeded, so a create error also + "removed" the sandbox from the list) and incomplete cleanup (a sandbox + left ``PAUSED`` or stuck ``TERMINATING`` is not RUNNING either). Verifies + by sandbox id, not by name absence from a filtered list. + """ from tenki_sandbox import Client project_id = os.environ.get("TENKI_PROJECT_ID") - workspace_id = os.environ.get("TENKI_WORKSPACE_ID") unique_name = f"agent-framework-ci-teardown-{os.getpid()}" async with TenkiExecuteCodeTool( @@ -1281,13 +1420,24 @@ async def test_integration_close_removes_sandbox_from_workspace() -> None: project_id=project_id, max_duration_seconds=300, ) as tool: - await _invoke(tool, "print('provisioned')") + contents = await _invoke(tool, "print('provisioned')") + errors = [c for c in contents if c.type == "error"] + assert not errors, f"provisioning/exec failed: {[e.message for e in errors]}" + assert any("provisioned" in (c.text or "") for c in contents if c.type == "text") + sandbox = tool._sandbox + assert sandbox is not None, "expected a live sandbox handle after a successful exec" + sandbox_id = sandbox.id - # After the async-with block, the sandbox should be terminated. Verify via list. client = Client() try: - sandboxes = client.list_workspace(workspace_id) if workspace_id else client.list() - live_names = {sb.info.name for sb in sandboxes if sb.info.state == "RUNNING"} - assert unique_name not in live_names, f"expected {unique_name} to be terminated" + deadline = time.monotonic() + 30 + while True: + state = client.get(sandbox_id).state + if state == "TERMINATED": + break + assert time.monotonic() < deadline, ( + f"sandbox {unique_name} ({sandbox_id}) still in state={state} 30s after close()" + ) + await asyncio.sleep(1) finally: client.close() From ca9eb1174e089357f28ae88877fd1352cb4c45e5 Mon Sep 17 00:00:00 2001 From: Patricio Leonel Filice Date: Fri, 24 Jul 2026 13:53:11 -0300 Subject: [PATCH 4/5] Python: docs: clarify snapshot seeding in tenki README MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `snapshot_id` restores from a snapshot prepared beforehand with the Tenki CLI/SDK — it does not snapshot the (already terminated) sandbox, so the previous wording suggested an impossible workflow. Co-Authored-By: Claude Opus 4.7 --- python/packages/tenki/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python/packages/tenki/README.md b/python/packages/tenki/README.md index 45efadf5621..b88594440c4 100644 --- a/python/packages/tenki/README.md +++ b/python/packages/tenki/README.md @@ -144,8 +144,9 @@ re-provisioning as described below. Each call runs `python3 -c expiring on an unscoped sandbox, or an external `tenki sandbox terminate`), the next call provisions a fresh sandbox. Filesystem and installed packages from the previous sandbox are **not** carried - over — snapshot the sandbox (via `extra_create_kwargs={"snapshot_id": ...}` on a - new tool) if you need to preserve state across a termination. + over. To start runs from a known state, seed sandboxes from a snapshot you + prepared beforehand with the Tenki CLI/SDK + (`extra_create_kwargs={"snapshot_id": ...}`). ### Provider vs standalone lifetime From 128685b74dbd1a2e708020951c42880fa1793974 Mon Sep 17 00:00:00 2001 From: Patricio Leonel Filice Date: Fri, 24 Jul 2026 17:29:36 -0300 Subject: [PATCH 5/5] Python: fix: treat empty TENKI_* env vars as unset CI systems expand unconfigured secrets/vars to "" (e.g. GitHub Actions vars.TENKI_PROJECT_ID), which was forwarded to the Tenki SDK as project_id=""/auth_token="" and failed provisioning in non-obvious ways. Env fallbacks now normalize "" to unset; explicit constructor args keep their documented precedence. Integration tests no longer pass an empty project_id as an explicit arg. Addresses Copilot review on #7312. Co-Authored-By: Claude Opus 4.7 --- .../_execute_code_tool.py | 13 ++++++-- .../tenki/tests/tenki/test_tenki_codeact.py | 31 ++++++++++++++++--- 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/python/packages/tenki/agent_framework_tenki/_execute_code_tool.py b/python/packages/tenki/agent_framework_tenki/_execute_code_tool.py index f09da8c87e6..73d179301f5 100644 --- a/python/packages/tenki/agent_framework_tenki/_execute_code_tool.py +++ b/python/packages/tenki/agent_framework_tenki/_execute_code_tool.py @@ -293,15 +293,22 @@ def _build_create_kwargs(self) -> dict[str, Any]: # explicit-arg-wins semantics above. ``TENKI_PROJECT_ID`` and # ``TENKI_WORKSPACE_ID`` are **not** read by the SDK — this module # owns them. - api_key = self._api_key if self._api_key is not None else os.environ.get("TENKI_API_KEY") + # + # Env vars set to the empty string are treated as unset (``or None``): + # CI systems expand unconfigured secrets/vars to "" (e.g. GitHub Actions + # ``${{ vars.TENKI_PROJECT_ID }}``), and forwarding ``project_id=""`` to + # the SDK fails in non-obvious ways. + api_key = self._api_key if self._api_key is not None else (os.environ.get("TENKI_API_KEY") or None) if api_key is not None: kwargs["auth_token"] = api_key if self._image is not None: kwargs["image"] = self._image - project_id = self._project_id if self._project_id is not None else os.environ.get("TENKI_PROJECT_ID") + project_id = self._project_id if self._project_id is not None else (os.environ.get("TENKI_PROJECT_ID") or None) if project_id is not None: kwargs["project_id"] = project_id - workspace_id = self._workspace_id if self._workspace_id is not None else os.environ.get("TENKI_WORKSPACE_ID") + workspace_id = ( + self._workspace_id if self._workspace_id is not None else (os.environ.get("TENKI_WORKSPACE_ID") or None) + ) if workspace_id is not None: kwargs["workspace_id"] = workspace_id if self._cpu_cores is not None: diff --git a/python/packages/tenki/tests/tenki/test_tenki_codeact.py b/python/packages/tenki/tests/tenki/test_tenki_codeact.py index 1a647741ad6..7229a244727 100644 --- a/python/packages/tenki/tests/tenki/test_tenki_codeact.py +++ b/python/packages/tenki/tests/tenki/test_tenki_codeact.py @@ -349,6 +349,21 @@ async def test_env_workspace_id_is_forwarded_when_unset( assert fake_sdk.create_calls[0]["workspace_id"] == "ws-from-env" +async def test_empty_string_env_vars_are_treated_as_unset( + fake_sdk: _FakeSandboxFactory, monkeypatch: pytest.MonkeyPatch +) -> None: + """CI systems expand unconfigured secrets/vars to "" — never forward those.""" + monkeypatch.setenv("TENKI_API_KEY", "") + monkeypatch.setenv("TENKI_PROJECT_ID", "") + monkeypatch.setenv("TENKI_WORKSPACE_ID", "") + tool = TenkiExecuteCodeTool(sandbox_name="empty-env") + await _invoke(tool, "pass") + call = fake_sdk.create_calls[0] + assert "auth_token" not in call + assert "project_id" not in call + assert "workspace_id" not in call + + async def test_constructor_project_id_wins_over_env( fake_sdk: _FakeSandboxFactory, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -1350,7 +1365,9 @@ def _tenki_integration_skip_reason() -> str | None: @pytest.mark.flaky(reruns=2, reruns_delay=5) @skip_if_tenki_integration_disabled async def test_integration_execute_hello_world() -> None: - project_id = os.environ.get("TENKI_PROJECT_ID") + # "or None": CI expands an unconfigured ``vars.TENKI_PROJECT_ID`` to "", and an + # explicit empty-string constructor arg would win over the env fallback. + project_id = os.environ.get("TENKI_PROJECT_ID") or None async with TenkiExecuteCodeTool( sandbox_name=f"agent-framework-ci-{os.getpid()}", project_id=project_id, @@ -1366,7 +1383,9 @@ async def test_integration_execute_hello_world() -> None: @skip_if_tenki_integration_disabled async def test_integration_filesystem_persists_across_calls() -> None: """Files written in one ``execute_code`` call are visible in the next.""" - project_id = os.environ.get("TENKI_PROJECT_ID") + # "or None": CI expands an unconfigured ``vars.TENKI_PROJECT_ID`` to "", and an + # explicit empty-string constructor arg would win over the env fallback. + project_id = os.environ.get("TENKI_PROJECT_ID") or None async with TenkiExecuteCodeTool( sandbox_name=f"agent-framework-ci-fs-{os.getpid()}", project_id=project_id, @@ -1385,7 +1404,9 @@ async def test_integration_filesystem_persists_across_calls() -> None: @skip_if_tenki_integration_disabled async def test_integration_failure_diagnostics_carry_signal_and_exit_code() -> None: """A killed subprocess must surface ``signal``/``exit_code`` in the error content.""" - project_id = os.environ.get("TENKI_PROJECT_ID") + # "or None": CI expands an unconfigured ``vars.TENKI_PROJECT_ID`` to "", and an + # explicit empty-string constructor arg would win over the env fallback. + project_id = os.environ.get("TENKI_PROJECT_ID") or None async with TenkiExecuteCodeTool( sandbox_name=f"agent-framework-ci-fail-{os.getpid()}", project_id=project_id, @@ -1412,7 +1433,9 @@ async def test_integration_close_removes_sandbox_from_workspace() -> None: """ from tenki_sandbox import Client - project_id = os.environ.get("TENKI_PROJECT_ID") + # "or None": CI expands an unconfigured ``vars.TENKI_PROJECT_ID`` to "", and an + # explicit empty-string constructor arg would win over the env fallback. + project_id = os.environ.get("TENKI_PROJECT_ID") or None unique_name = f"agent-framework-ci-teardown-{os.getpid()}" async with TenkiExecuteCodeTool(