From 91878d7887b30f3197283c5fa34089c96a686b91 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sat, 31 Oct 2026 18:11:33 +0000 Subject: [PATCH 1/3] Add retail-cart benchmark: guest add-to-cart flows with the browser REPL --- .github/workflows/_run.yml | 22 ++- .github/workflows/ci.yml | 2 + .github/workflows/retail-cart.yml | 45 +++++ README.md | 17 ++ arms/codex-repl-site-config.toml | 21 +++ arms/codex-repl.toml | 16 ++ benchmarks/retail-cart.toml | 5 + harness/cli.py | 35 +++- harness/definitions.py | 13 +- harness/run.py | 37 ++++- suites/retail-cart/image/Dockerfile | 13 ++ suites/retail-cart/image/bench/finalize-mcp | 157 ++++++++++++++++++ suites/retail-cart/image/bench/open-start-url | 55 ++++++ suites/retail-cart/image/bench/score | 73 ++++++++ suites/retail-cart/suite.py | 117 +++++++++++++ suites/retail-cart/tasks.toml | 99 +++++++++++ tests/test_kernel_mcp_local.py | 122 +++++++++++++- tests/test_retail_cart.py | 98 +++++++++++ tests/test_run.py | 69 ++++++++ .../kernel-mcp-local/image/bin/kernel-mcp | 8 +- .../image/bin/kernel-mcp-entrypoint | 3 + .../image/bin/kernel-mcp-verify | 58 ++++--- .../kernel-mcp-local/image/bin/mcp-call-log | 82 +++++++++ .../kernel-mcp-local/image/layer.Dockerfile | 4 +- toolsets/kernel-mcp-local/toolset.py | 56 +++++-- 25 files changed, 1175 insertions(+), 52 deletions(-) create mode 100644 .github/workflows/retail-cart.yml create mode 100644 arms/codex-repl-site-config.toml create mode 100644 arms/codex-repl.toml create mode 100644 benchmarks/retail-cart.toml create mode 100644 suites/retail-cart/image/Dockerfile create mode 100755 suites/retail-cart/image/bench/finalize-mcp create mode 100755 suites/retail-cart/image/bench/open-start-url create mode 100644 suites/retail-cart/image/bench/score create mode 100644 suites/retail-cart/suite.py create mode 100644 suites/retail-cart/tasks.toml create mode 100644 tests/test_retail_cart.py create mode 100644 tests/test_run.py create mode 100755 toolsets/kernel-mcp-local/image/bin/mcp-call-log diff --git a/.github/workflows/_run.yml b/.github/workflows/_run.yml index 79768d7..e0ee4b5 100644 --- a/.github/workflows/_run.yml +++ b/.github/workflows/_run.yml @@ -27,6 +27,21 @@ on: required: false default: "" type: string + suite_args: + description: Space-separated KEY=VALUE overrides of the benchmark's suite_args + required: false + default: "" + type: string + toolset_args: + description: Space-separated KEY=VALUE overrides of the arms' toolset_args + required: false + default: "" + type: string + attempts: + description: Trials per task and arm + required: false + default: "1" + type: string permissions: contents: read @@ -79,14 +94,19 @@ jobs: TASKS: ${{ inputs.tasks }} CONCURRENCY: ${{ inputs.concurrency }} TOOLSET_REF: ${{ inputs.toolset_ref }} + SUITE_ARGS: ${{ inputs.suite_args }} + TOOLSET_ARGS: ${{ inputs.toolset_args }} + ATTEMPTS: ${{ inputs.attempts }} run: | run_id="$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" jobs_dir="$RUNNER_TEMP/jobs" echo "run_file=$jobs_dir/run-$run_id.json" >>"$GITHUB_OUTPUT" echo "experiment=$BENCHMARK_NAME-$run_id" >>"$GITHUB_OUTPUT" - args=(--tasks "$TASKS" --concurrency "$CONCURRENCY" --jobs-dir "$jobs_dir" --run-id "$run_id") + args=(--tasks "$TASKS" --concurrency "$CONCURRENCY" --attempts "$ATTEMPTS" --jobs-dir "$jobs_dir" --run-id "$run_id") [[ -n "$ARMS" ]] && args+=(--arms "$ARMS") [[ -n "$TOOLSET_REF" ]] && args+=(--toolset-ref "$TOOLSET_REF") + for pair in $SUITE_ARGS; do args+=(--suite-arg "$pair"); done + for pair in $TOOLSET_ARGS; do args+=(--toolset-arg "$pair"); done uv run bench run "$BENCHMARK_NAME" "${args[@]}" - name: Report diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 12666a3..72ff36b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,6 +24,8 @@ jobs: - run: uv run pytest -q - name: Prepare every static arm for one task run: uv run bench prepare kernel-mcp-clawbench --arms codex-kernel-mcp,claude-kernel-mcp --tasks v2-1134-chapter-finder-redcross --out "$RUNNER_TEMP/prepared" + - name: Prepare the static retail-cart arm + run: uv run bench prepare retail-cart --arms codex-repl --out "$RUNNER_TEMP/prepared-retail-cart" report: runs-on: ubuntu-latest diff --git a/.github/workflows/retail-cart.yml b/.github/workflows/retail-cart.yml new file mode 100644 index 0000000..1019679 --- /dev/null +++ b/.github/workflows/retail-cart.yml @@ -0,0 +1,45 @@ +name: Retail cart flows with the browser REPL + +on: + workflow_dispatch: + inputs: + arms: + description: Comma-separated arm names (empty runs every arm in benchmarks/retail-cart.toml) + required: false + default: "" + type: string + tasks: + description: Comma-separated task IDs, or all + required: false + default: all + type: string + attempts: + description: Trials per task and arm + required: false + default: "3" + type: string + concurrency: + description: Concurrent trials per arm + required: false + default: "16" + type: string + kernel_mcp_server_ref: + description: kernel/kernel-mcp-server git ref to build (empty builds main) + required: false + default: "" + type: string + +permissions: + contents: read + +jobs: + benchmark: + uses: ./.github/workflows/_run.yml + secrets: inherit + with: + benchmark: retail-cart + arms: ${{ inputs.arms }} + tasks: ${{ inputs.tasks }} + attempts: ${{ inputs.attempts }} + concurrency: ${{ inputs.concurrency }} + toolset_ref: ${{ inputs.kernel_mcp_server_ref }} diff --git a/README.md b/README.md index ccd7b5f..5049000 100644 --- a/README.md +++ b/README.md @@ -119,6 +119,21 @@ The Kernel MCP server on ClawBench. ClawBench's primary score is the lenient reward. `kernel_mcp_valid` records whether the agent called Kernel MCP's `execute_playwright_code` and used its assigned browser on every call, so a failed task can be told apart from a trial that never used the toolset. +### `retail-cart` + +Guest add-to-cart flows on 16 retail and industrial-supply sites with heavy bot protection. Each task starts on the site's home page and asks the agent to search for one item, open its product page, add it to the cart without logging in, and report the cart. + +| Arm | Agent | Browser | +|---|---|---| +| `codex-repl` | Codex, `gpt-6-luna`, Kernel MCP's `browser_repl` only | Stealth | +| `codex-repl-site-config` | Same | From Kernel's config registry per site, plus its guidance | + +The agent ends each task by calling the `bench` MCP server's `finalize` tool with its evidence and an outcome: `success`, `failed`, `blocked` (bot protection stopped it), `website_error` or `kernel_error`. `reward` is 1 for a reported success, and `outcome_` records the failure, so bot blocks are counted apart from the agent's own failures. The verifier also saves the browser's final URL and title. Run several attempts per task to get a pass rate per site: + +```bash +uv run bench run retail-cart --arms codex-repl --attempts 3 +``` + ## Extending ### Add an arm @@ -140,6 +155,8 @@ model = "anthropic/claude-haiku-5-5" steering = true ``` +`toolset_args` are passed to the toolset; `kernel-mcp-local` takes `toolsets`, the Kernel MCP toolsets to enable (`playwright`, the default, or `repl`). `agent_kwargs` are passed to the Harbor agent. + `config` takes any [`POST /browsers`](https://kernel.sh/docs/api-reference/browsers/create-a-browser-session) parameter, such as `stealth`, `proxy`, `region`, `extensions`, or `viewport`. Browsers are always headful. ### Add a configurator diff --git a/arms/codex-repl-site-config.toml b/arms/codex-repl-site-config.toml new file mode 100644 index 0000000..e99c8f3 --- /dev/null +++ b/arms/codex-repl-site-config.toml @@ -0,0 +1,21 @@ +# Same as codex-repl, but each task's browser and proxy come from Kernel's +# config registry for the task's site, and the registry's guidance for that +# site is added to the instruction. +agent = "codex" +model = "gpt-6-luna" +agent_version = "0.163.0" +toolset = "kernel-mcp-local" + +[agent_kwargs] +disable_web_search = true + +[toolset_args] +toolsets = ["repl"] + +[browser] +config = { stealth = true } +configurator = "site-config" + +[browser.configurator_args] +model = "anthropic/claude-haiku-5-5" +steering = true diff --git a/arms/codex-repl.toml b/arms/codex-repl.toml new file mode 100644 index 0000000..0817a68 --- /dev/null +++ b/arms/codex-repl.toml @@ -0,0 +1,16 @@ +# Codex driving the browser only through Kernel MCP's browser REPL, with a +# fixed stealth browser. +agent = "codex" +model = "gpt-6-luna" +agent_version = "0.163.0" +toolset = "kernel-mcp-local" + +# Codex's own web tool would let the agent answer without the browser. +[agent_kwargs] +disable_web_search = true + +[toolset_args] +toolsets = ["repl"] + +[browser] +config = { stealth = true } diff --git a/benchmarks/retail-cart.toml b/benchmarks/retail-cart.toml new file mode 100644 index 0000000..4d486a6 --- /dev/null +++ b/benchmarks/retail-cart.toml @@ -0,0 +1,5 @@ +# Guest add-to-cart flows on bot-protected retail and industrial-supply sites, +# driven through the browser REPL, with a stealth browser and with the config +# registry's per-site settings. +suite = "retail-cart" +arms = ["codex-repl", "codex-repl-site-config"] diff --git a/harness/cli.py b/harness/cli.py index ceb1b87..171f982 100644 --- a/harness/cli.py +++ b/harness/cli.py @@ -16,6 +16,16 @@ def _split(value: str | None) -> list[str] | None: return [item.strip() for item in value.split(",") if item.strip()] +def _pairs(values: list[str]) -> dict[str, str]: + pairs = {} + for value in values: + key, separator, item = value.partition("=") + if not separator or not key: + raise SystemExit(f"Expected key=value, got {value!r}") + pairs[key] = item + return pairs + + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(prog="bench") commands = parser.add_subparsers(dest="command", required=True) @@ -28,6 +38,20 @@ def main(argv: list[str] | None = None) -> int: ) command.add_argument("--tasks", default="all", help="comma-separated task IDs, or all") command.add_argument("--toolset-ref", help="git ref for toolsets built from source") + command.add_argument( + "--suite-arg", + action="append", + default=[], + metavar="KEY=VALUE", + help="override one of the benchmark's suite_args, e.g. split=validation", + ) + command.add_argument( + "--toolset-arg", + action="append", + default=[], + metavar="KEY=VALUE", + help="override a toolset_args key in the arms that set it, e.g. toolsets=repl", + ) commands.choices["prepare"].add_argument("--out", type=Path, required=True) run_command = commands.choices["run"] @@ -35,14 +59,20 @@ def main(argv: list[str] | None = None) -> int: run_command.add_argument( "--concurrency", type=int, default=10, help="concurrent trials per arm" ) + run_command.add_argument("--attempts", type=int, default=1, help="trials per task and arm") run_command.add_argument("--run-id", default=datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")) args = parser.parse_args(argv) arms = _split(args.arms) tasks = _split(args.tasks) + suite_args = _pairs(args.suite_arg) + toolset_args = _pairs(args.toolset_arg) if args.command == "prepare": - for item in prepare(args.benchmark, arms, tasks, args.out, args.toolset_ref): + prepared = prepare( + args.benchmark, arms, tasks, args.out, args.toolset_ref, suite_args, toolset_args + ) + for item in prepared: print(f"{item.arm.name}: {item.dataset}") return 0 @@ -56,6 +86,9 @@ def main(argv: list[str] | None = None) -> int: args.run_id, args.concurrency, args.toolset_ref, + suite_args, + toolset_args, + args.attempts, ) for arm, status in statuses.items(): print(f"{arm}: harbor exited {status} -> {args.jobs_dir / f'{arm}-{args.run_id}'}") diff --git a/harness/definitions.py b/harness/definitions.py index a6f1fd8..57e12d4 100644 --- a/harness/definitions.py +++ b/harness/definitions.py @@ -3,6 +3,7 @@ from __future__ import annotations import importlib.util +import sys import tomllib from dataclasses import dataclass, field from pathlib import Path @@ -26,7 +27,9 @@ class Arm: model: str toolset: str agent_version: str | None = None + agent_kwargs: dict[str, Any] = field(default_factory=dict) browser: Browser = field(default_factory=Browser) + toolset_args: dict[str, Any] = field(default_factory=dict) @dataclass(frozen=True) @@ -34,6 +37,7 @@ class Benchmark: name: str suite: str arms: list[str] + suite_args: dict[str, Any] = field(default_factory=dict) def _read(path: Path) -> dict[str, Any]: @@ -44,7 +48,12 @@ def _read(path: Path) -> dict[str, Any]: def load_benchmark(name: str) -> Benchmark: data = _read(ROOT / "benchmarks" / f"{name}.toml") - return Benchmark(name=name, suite=data["suite"], arms=list(data["arms"])) + return Benchmark( + name=name, + suite=data["suite"], + arms=list(data["arms"]), + suite_args=data.get("suite_args", {}), + ) def load_arm(name: str) -> Arm: @@ -69,5 +78,7 @@ def _load_module(path: Path) -> ModuleType: ) assert spec is not None and spec.loader is not None module = importlib.util.module_from_spec(spec) + # Registered before it runs, as an import would be, so dataclasses work. + sys.modules[spec.name] = module spec.loader.exec_module(module) return module diff --git a/harness/run.py b/harness/run.py index 57861c4..c4c082e 100644 --- a/harness/run.py +++ b/harness/run.py @@ -8,6 +8,7 @@ import sys from dataclasses import dataclass from pathlib import Path +from typing import Any from harness import preflight from harness.configure import configure_tasks @@ -38,13 +39,26 @@ def prepare( task_ids: list[str] | None, work_dir: Path, toolset_ref: str | None = None, + suite_args: dict[str, Any] | None = None, + toolset_args: dict[str, Any] | None = None, ) -> list[PreparedArm]: + """Generate the suite's tasks and apply each arm's browser settings and toolset. + + `suite_args` overrides the benchmark's `suite_args`. `toolset_args` + overrides the keys an arm's `toolset_args` already sets, so an arm whose + toolset does not take an argument keeps running without it. A key no arm + of the benchmark sets is refused as a likely typo. + """ benchmark = load_benchmark(benchmark_name) arms = [load_arm(name) for name in (arm_names or benchmark.arms)] + known = {key for arm in [*arms, *map(load_arm, benchmark.arms)] for key in arm.toolset_args} + unknown = set(toolset_args or {}) - known + if unknown: + raise SystemExit(f"No arm takes toolset args: {', '.join(sorted(unknown))}") suite = load_suite(benchmark.suite) base = work_dir / "tasks" - task_dirs = suite.generate(task_ids, base) + task_dirs = suite.generate(task_ids, base, **{**benchmark.suite_args, **(suite_args or {})}) if not task_dirs: raise SystemExit("The suite generated no tasks") @@ -54,7 +68,10 @@ def prepare( shutil.copytree(base, dataset) arm_tasks = sorted(path for path in dataset.iterdir() if path.is_dir()) configurations = configure_tasks(arm_tasks, arm.browser) - load_toolset(arm.toolset).apply(arm_tasks, ref=toolset_ref) + args = { + key: (toolset_args or {}).get(key, value) for key, value in arm.toolset_args.items() + } + load_toolset(arm.toolset).apply(arm_tasks, ref=toolset_ref, **args) prepared.append(PreparedArm(arm, dataset, configurations)) return prepared @@ -68,6 +85,9 @@ def run( run_id: str, concurrency: int, toolset_ref: str | None = None, + suite_args: dict[str, Any] | None = None, + toolset_args: dict[str, Any] | None = None, + attempts: int = 1, ) -> dict[str, int]: """Run every arm in parallel and return each arm's Harbor exit status. @@ -78,7 +98,9 @@ def run( """ preflight.check_kernel_project() load_suite(load_benchmark(benchmark_name).suite).preflight() - prepared = prepare(benchmark_name, arm_names, task_ids, work_dir, toolset_ref) + prepared = prepare( + benchmark_name, arm_names, task_ids, work_dir, toolset_ref, suite_args, toolset_args + ) jobs_dir.mkdir(parents=True, exist_ok=True) manifest_path = jobs_dir / f"run-{run_id}.json" @@ -97,7 +119,7 @@ def run( job_name = f"{item.arm.name}-{run_id}" with (jobs_dir / f"{job_name}.log").open("w") as log: processes[item.arm.name] = subprocess.Popen( - harbor_command(item, jobs_dir, job_name, concurrency), + harbor_command(item, jobs_dir, job_name, concurrency, attempts), stdout=log, stderr=subprocess.STDOUT, ) @@ -107,7 +129,9 @@ def run( return {name: entry["status"] for name, entry in manifest.items()} -def harbor_command(item: PreparedArm, jobs_dir: Path, job_name: str, concurrency: int) -> list[str]: +def harbor_command( + item: PreparedArm, jobs_dir: Path, job_name: str, concurrency: int, attempts: int = 1 +) -> list[str]: arm = item.arm command = [ str(Path(sys.executable).parent / "harbor"), @@ -119,12 +143,15 @@ def harbor_command(item: PreparedArm, jobs_dir: Path, job_name: str, concurrency "--job-name", job_name, "--jobs-dir", str(jobs_dir), "--n-concurrent", str(concurrency), + "--n-attempts", str(attempts), "--max-retries", "5", "--delete", "--yes", ] # fmt: skip if arm.agent_version: command += ["--agent-kwarg", f"version={arm.agent_version}"] + for key, value in arm.agent_kwargs.items(): + command += ["--agent-kwarg", f"{key}={json.dumps(value)}"] for exception in RETRY_EXCEPTIONS: command += ["--retry-include", exception] return command diff --git a/suites/retail-cart/image/Dockerfile b/suites/retail-cart/image/Dockerfile new file mode 100644 index 0000000..4362d77 --- /dev/null +++ b/suites/retail-cart/image/Dockerfile @@ -0,0 +1,13 @@ +# Task image for retail-cart. The browser is the Kernel VM this container runs +# in, so the image only holds the finalize tool, the start-URL setup and the +# scorer. Toolsets append their own layers. +FROM node:22-bookworm-slim + +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates curl git python3 \ + && rm -rf /var/lib/apt/lists/* + +COPY bench/ /opt/bench/ +RUN chmod +x /opt/bench/finalize-mcp /opt/bench/open-start-url /opt/bench/score + +WORKDIR /app diff --git a/suites/retail-cart/image/bench/finalize-mcp b/suites/retail-cart/image/bench/finalize-mcp new file mode 100755 index 0000000..9339d7a --- /dev/null +++ b/suites/retail-cart/image/bench/finalize-mcp @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +"""stdio MCP server with one tool, `finalize`, the agent's report of the task. + +The first call writes the report to /logs/agent/finalize.json, where the +verifier reads it; later calls are refused, so the first report stands. +`evidence` and `reasoning` come before `outcome` in the schema so the agent +writes down what it saw and checks it before it gives a verdict. + +`outcome` separates the agent's own failures from failures it could not +control: the website's bot protection, other website errors, and Kernel's. +""" + +from __future__ import annotations + +import json +import sys +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +REPORT = Path("/logs/agent/finalize.json") + +OUTCOMES = { + "success": "every requirement of the task is met and the evidence shows it.", + "failed": ( + "the task is not done, including when what the task asks for does not exist or is " + "unavailable on the site." + ), + "blocked": ( + "the website's bot protection stopped you: an access-denied or 403 page, a CAPTCHA or " + "press-and-hold challenge you could not pass, or a rate-limit page." + ), + "website_error": ( + "the website kept you from finishing for another reason: it showed an error or failed " + "to load." + ), + "kernel_error": ( + "the Kernel browser or its tools kept you from finishing: the browser REPL or the " + "browser itself errored, hung, or crashed." + ), +} + +TOOL = { + "name": "finalize", + "description": ( + "Report the outcome of the task. Call this exactly once, when the task is done or you " + "are certain it cannot be done, and stop afterwards. Your report is the result of the " + "task." + ), + "inputSchema": { + "type": "object", + "properties": { + "evidence": { + "type": "string", + "description": ( + "What you observed in the browser that shows whether the task is done: the " + "values, text and URLs the task asks for, quoted as the page shows them, " + "including everything the task asks you to report." + ), + }, + "reasoning": { + "type": "string", + "description": ( + "Check the evidence against each requirement of the task in turn and say " + "whether it is met. Name anything you could not verify." + ), + }, + "outcome": { + "type": "string", + "enum": list(OUTCOMES), + "description": "How the task ended. " + + " ".join(f"{name}: {meaning}" for name, meaning in OUTCOMES.items()), + }, + }, + "required": ["evidence", "reasoning", "outcome"], + "additionalProperties": False, + }, + "annotations": {"readOnlyHint": False, "idempotentHint": False}, +} + + +def finalize(arguments: dict[str, Any]) -> tuple[str, bool]: + if REPORT.exists(): + return "The task was already finalized. Stop now.", True + evidence, reasoning, outcome = ( + arguments.get(key) for key in ("evidence", "reasoning", "outcome") + ) + if not isinstance(evidence, str) or not evidence.strip(): + return "evidence must be a non-empty string.", True + if not isinstance(reasoning, str) or not reasoning.strip(): + return "reasoning must be a non-empty string.", True + if outcome not in OUTCOMES: + return f"outcome must be one of {', '.join(OUTCOMES)}.", True + REPORT.parent.mkdir(parents=True, exist_ok=True) + report = { + "outcome": outcome, + "evidence": evidence, + "reasoning": reasoning, + "finalized_at": datetime.now(UTC).isoformat(), + } + REPORT.write_text(json.dumps(report, indent=2)) + return "Recorded. The task is finalized; stop now.", False + + +def handle(message: dict[str, Any]) -> dict[str, Any] | None: + method = message.get("method") + if "id" not in message: + return None + if method == "initialize": + params = message.get("params") or {} + result: dict[str, Any] = { + "protocolVersion": params.get("protocolVersion", "2025-06-18"), + "capabilities": {"tools": {}}, + "serverInfo": {"name": "bench", "version": "1.0.0"}, + } + elif method == "ping": + result = {} + elif method == "tools/list": + result = {"tools": [TOOL]} + elif method == "tools/call": + params = message.get("params") or {} + if params.get("name") != TOOL["name"]: + text, is_error = f"Unknown tool {params.get('name')!r}.", True + else: + text, is_error = finalize(params.get("arguments") or {}) + result = {"content": [{"type": "text", "text": text}], "isError": is_error} + else: + return { + "jsonrpc": "2.0", + "id": message["id"], + "error": {"code": -32601, "message": f"Method not found: {method}"}, + } + return {"jsonrpc": "2.0", "id": message["id"], "result": result} + + +def main() -> int: + for line in sys.stdin: + if not line.strip(): + continue + try: + message = json.loads(line) + except json.JSONDecodeError: + response: dict[str, Any] | None = { + "jsonrpc": "2.0", + "id": None, + "error": {"code": -32700, "message": "Parse error"}, + } + else: + response = handle(message) if isinstance(message, dict) else None + if response is not None: + sys.stdout.write(json.dumps(response) + "\n") + sys.stdout.flush() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/suites/retail-cart/image/bench/open-start-url b/suites/retail-cart/image/bench/open-start-url new file mode 100755 index 0000000..d6dd7f3 --- /dev/null +++ b/suites/retail-cart/image/bench/open-start-url @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""Open the task's start URL in the trial's browser before the agent starts. + +Both arms start on the same loaded page, so neither spends its own time on +the first navigation. +""" + +from __future__ import annotations + +import json +import os +import sys +import time +import urllib.error +import urllib.request + +CODE = """ +await page.goto(%s, { waitUntil: "domcontentloaded", timeout: 45000 }); +await page.waitForLoadState("load", { timeout: 15000 }).catch(() => {}); +return page.url(); +""" + + +def main(url: str) -> int: + base = (os.environ.get("KERNEL_BASE_URL") or "https://api.onkernel.com").rstrip("/") + request = urllib.request.Request( + f"{base}/browsers/{os.environ['KERNEL_SESSION_ID']}/playwright/execute", + data=json.dumps({"code": CODE % json.dumps(url), "timeout_sec": 90}).encode(), + headers={ + "Authorization": f"Bearer {os.environ['KERNEL_API_KEY']}", + "Content-Type": "application/json", + }, + method="POST", + ) + # The container's DNS can lag its start; retry until the API is reachable. + for attempt in range(5): + try: + with urllib.request.urlopen(request, timeout=120) as response: + result = json.load(response) + break + except urllib.error.HTTPError: + raise + except urllib.error.URLError: + if attempt == 4: + raise + time.sleep(2 * (attempt + 1)) + if not result.get("success"): + print(f"Could not open {url}: {result.get('error')}", file=sys.stderr) + return 1 + print(f"Opened {result.get('result')}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1])) diff --git a/suites/retail-cart/image/bench/score b/suites/retail-cart/image/bench/score new file mode 100644 index 0000000..679a7ce --- /dev/null +++ b/suites/retail-cart/image/bench/score @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +"""Turn the agent's finalize report into the task's rewards. + +`reward` is 1 when the agent reported success, and 0 for any other outcome or +when it never called finalize. `finalized` records whether it called +finalize, and `outcome_` which failure it reported, so bot blocks can be +told apart from other website errors, Kernel errors and the agent's own +failures. + +The browser's final URL and title are saved next to the report, so a reported +success can be checked against the page the agent ended on. +""" + +from __future__ import annotations + +import json +import os +import shutil +import urllib.request +from pathlib import Path + +REPORT = Path("/logs/agent/finalize.json") +TASK = Path("/opt/bench/task.json") +VERIFIER = Path("/logs/verifier") +FAILURES = ("failed", "blocked", "website_error", "kernel_error") + +FINAL_PAGE = "return { url: page.url(), title: await page.title() };" + + +def final_page() -> dict: + base = (os.environ.get("KERNEL_BASE_URL") or "https://api.onkernel.com").rstrip("/") + request = urllib.request.Request( + f"{base}/browsers/{os.environ['KERNEL_SESSION_ID']}/playwright/execute", + data=json.dumps({"code": FINAL_PAGE, "timeout_sec": 30}).encode(), + headers={ + "Authorization": f"Bearer {os.environ['KERNEL_API_KEY']}", + "Content-Type": "application/json", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=60) as response: + result = json.load(response) + except (OSError, ValueError, KeyError) as error: + return {"error": str(error)} + if not result.get("success"): + return {"error": result.get("error")} + return result.get("result") or {} + + +def main() -> int: + VERIFIER.mkdir(parents=True, exist_ok=True) + try: + report = json.loads(REPORT.read_text()) + except (OSError, json.JSONDecodeError): + report = None + if report is not None: + shutil.copy2(REPORT, VERIFIER / "finalize.json") + if TASK.exists(): + shutil.copy2(TASK, VERIFIER / "task.json") + (VERIFIER / "final-page.json").write_text(json.dumps(final_page(), indent=2)) + outcome = report.get("outcome") if report is not None else None + rewards = { + "reward": float(outcome == "success"), + "finalized": float(report is not None), + **{f"outcome_{name}": float(outcome == name) for name in FAILURES}, + } + (VERIFIER / "reward.json").write_text(json.dumps(rewards, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/suites/retail-cart/suite.py b/suites/retail-cart/suite.py new file mode 100644 index 0000000..7ddc17a --- /dev/null +++ b/suites/retail-cart/suite.py @@ -0,0 +1,117 @@ +"""Guest add-to-cart tasks on retail and industrial-supply sites with heavy bot protection. + +Every task starts on a site's home page, opened before the agent starts, and +asks the agent to search for one item, open its product page, add it to the +cart without logging in, and report the cart. The flow crosses the pages +where bot protection usually acts: the first request, search, the product +page and the cart. + +The agent ends each task by calling the `finalize` tool of the `bench` MCP +server with its evidence, its reasoning and an outcome. The verifier turns +that report into the reward and records the outcome, so a bot block is +counted apart from other failures. The browser's final URL and title are +saved for checking reported successes. +""" + +from __future__ import annotations + +import json +import shutil +import tomllib +from dataclasses import dataclass +from pathlib import Path + +import tomlkit + +HERE = Path(__file__).parent +IMAGE = HERE / "image" + +INSTRUCTION = """Add {item} to the {site} cart, then open the cart. Search for it from the site's search box, as a shopper would, open its product page, and add one to the cart. Report the cart line for it with its quantity and price, and the cart subtotal. + +## Rules + +- The browser is already open on {start_url}. Work on that site only. +- Do not log in, sign up, or enter personal or payment details. Continue as a guest. +- Never check out, pay, or place an order. Stop at the cart. +- If the site shows a bot check, CAPTCHA or press-and-hold challenge, try to get past it as a person would. If you cannot, or the site denies access, report the task as blocked. + +## Finishing + +When the task is done, or you are certain it cannot be done, call the `bench` MCP server's `finalize` tool exactly once, then stop. Your `finalize` report is the result of the task: put everything the task asks you to report in `evidence`. +""" + + +@dataclass(frozen=True) +class Task: + id: str + site: str + start_url: str + item: str + timeout_sec: int = 900 + + +def preflight() -> None: + """Nothing to check: tasks need only the Kernel browser Harbor creates.""" + + +def load_tasks() -> list[Task]: + return [Task(**entry) for entry in tomllib.loads((HERE / "tasks.toml").read_text())["tasks"]] + + +def generate(task_ids: list[str] | None, out_dir: Path) -> list[Path]: + tasks = load_tasks() + if task_ids: + known = {task.id: task for task in tasks} + missing = [task_id for task_id in task_ids if task_id not in known] + if missing: + raise SystemExit(f"Unknown tasks: {', '.join(missing)}") + tasks = [known[task_id] for task_id in task_ids] + out_dir.mkdir(parents=True, exist_ok=True) + return [_write(task, out_dir / task.id) for task in tasks] + + +def _write(task: Task, task_dir: Path) -> Path: + if task_dir.exists(): + shutil.rmtree(task_dir) + environment = task_dir / "environment" + shutil.copytree(IMAGE, environment) + (environment / "bench" / "task.json").write_text(json.dumps({"id": task.id, "site": task.site})) + step = task_dir / "steps" / "run" + (step / "tests").mkdir(parents=True) + (step / "workdir").mkdir() + + (step / "instruction.md").write_text( + INSTRUCTION.format(item=task.item, site=task.site, start_url=task.start_url) + ) + (step / "workdir" / "setup.sh").write_text( + f"#!/bin/bash\nset -euo pipefail\n/opt/bench/open-start-url {json.dumps(task.start_url)}\n" + ) + (step / "tests" / "test.sh").write_text("#!/bin/bash\nset -euo pipefail\n/opt/bench/score\n") + + config = tomlkit.document() + config["schema_version"] = "1.3" + config["source"] = "retail-cart" + config["task"] = {"name": f"retail-cart/{task.id}", "keywords": [task.site]} + config["metadata"] = {"site": task.site, "start_url": task.start_url} + config["environment"] = { + "build_timeout_sec": 1200.0, + "network_mode": "public", + "workdir": "/", + "env": {"KERNEL_API_KEY": "${KERNEL_API_KEY}", "KERNEL_BASE_URL": "${KERNEL_BASE_URL:-}"}, + } + servers = tomlkit.aot() + servers.append( + {"name": "bench", "transport": "stdio", "command": "/opt/bench/finalize-mcp", "args": []} + ) + config["environment"]["mcp_servers"] = servers + steps = tomlkit.aot() + steps.append( + { + "name": "run", + "agent": {"timeout_sec": float(task.timeout_sec)}, + "verifier": {"timeout_sec": 120.0}, + } + ) + config["steps"] = steps + (task_dir / "task.toml").write_text(tomlkit.dumps(config)) + return task_dir diff --git a/suites/retail-cart/tasks.toml b/suites/retail-cart/tasks.toml new file mode 100644 index 0000000..a6e56c7 --- /dev/null +++ b/suites/retail-cart/tasks.toml @@ -0,0 +1,99 @@ +# One guest add-to-cart task per site. Each starts on the site's home page, +# where most bot protection first sees the browser, and walks home page -> +# search -> product page -> cart, the path a shopping agent takes. + +[[tasks]] +id = "amazon-aa-batteries" +site = "amazon.com" +start_url = "https://www.amazon.com/" +item = "the Amazon Basics 48-pack of AA alkaline batteries" + +[[tasks]] +id = "walmart-paper-towels" +site = "walmart.com" +start_url = "https://www.walmart.com/" +item = "a pack of Bounty Select-a-Size paper towels" + +[[tasks]] +id = "costco-microfiber-towels" +site = "costco.com" +start_url = "https://www.costco.com/" +item = "the Kirkland Signature Ultra Plush microfiber towels, 36-count" + +[[tasks]] +id = "lowes-stanley-kit" +site = "lowes.com" +start_url = "https://www.lowes.com/" +item = "the Stanley 27-pack 0.375-in metal kit" + +[[tasks]] +id = "staples-copy-paper" +site = "staples.com" +start_url = "https://www.staples.com/" +item = "one 500-sheet ream of 8.5 x 11 in multipurpose copy paper" + +[[tasks]] +id = "target-paper-towels" +site = "target.com" +start_url = "https://www.target.com/" +item = "a pack of Make-A-Size paper towels that can be shipped" + +[[tasks]] +id = "homedepot-desk" +site = "homedepot.com" +start_url = "https://www.homedepot.com/" +item = "a computer desk under $150 that can be shipped" + +[[tasks]] +id = "radwell-fuse" +site = "radwell.com" +start_url = "https://www.radwell.com/" +item = "a 20 A, 32 V 3AG fuse" + +[[tasks]] +id = "ebay-usb-c-cable" +site = "ebay.com" +start_url = "https://www.ebay.com/" +item = "a USB-C charging cable from a Buy It Now listing under $15" + +[[tasks]] +id = "harborfreight-utility-knife" +site = "harborfreight.com" +start_url = "https://www.harborfreight.com/" +item = "a folding lock-back utility knife" + +[[tasks]] +id = "dickssportinggoods-water-bottle" +site = "dickssportinggoods.com" +start_url = "https://www.dickssportinggoods.com/" +item = "a HydroJug 32 oz sport bottle that can be shipped" + +[[tasks]] +id = "mouser-usb-c-receptacle" +site = "mouser.com" +start_url = "https://www.mouser.com/" +item = "one GCT USB4105-GF-A USB-C receptacle" + +[[tasks]] +id = "newark-aa-batteries" +site = "newark.com" +start_url = "https://www.newark.com/" +item = "a pack of AA alkaline batteries" + +[[tasks]] +id = "zoro-nitrile-gloves" +site = "zoro.com" +start_url = "https://www.zoro.com/" +item = "a box of disposable nitrile gloves, size large" + +[[tasks]] +id = "guitarcenter-picks" +site = "guitarcenter.com" +start_url = "https://www.guitarcenter.com/" +item = "a 12-pack of Fender 351 celluloid medium guitar picks" + +[[tasks]] +id = "wbmason-sticky-notes" +site = "wbmason.com" +start_url = "https://www.wbmason.com/" +item = "a pack of Post-it 3 x 3 in canary yellow sticky notes" diff --git a/tests/test_kernel_mcp_local.py b/tests/test_kernel_mcp_local.py index 90d1afe..5032901 100644 --- a/tests/test_kernel_mcp_local.py +++ b/tests/test_kernel_mcp_local.py @@ -1,6 +1,9 @@ +import json +import os +import subprocess import tomllib -from harness.definitions import load_toolset +from harness.definitions import ROOT, load_toolset def test_apply_adds_server_image_layer_instruction_and_check(task_dir, monkeypatch): @@ -26,3 +29,120 @@ def fake_fetch(ref, dest): assert config["metadata"]["kernel_mcp_server_sha"] == "abc123" assert "execute_playwright_code" in (task_dir / "steps/run/instruction.md").read_text() assert (task_dir / "steps/run/tests/test.sh").read_text().endswith("kernel-mcp-verify\n") + + +def test_repl_toolset_enables_the_repl_and_keeps_other_servers(task_dir, monkeypatch): + toolset = load_toolset("kernel-mcp-local") + monkeypatch.setattr(toolset, "_fetch_source", lambda ref, dest: dest.mkdir() or "abc123") + (task_dir / "task.toml").write_text( + '[task]\nname = "example"\n\n[environment]\n\n' + '[[environment.mcp_servers]]\nname = "bench"\ntransport = "stdio"\n' + 'command = "/opt/bench/finalize-mcp"\nargs = []\n' + ) + toolset.apply([task_dir], toolsets=["repl"]) + + config = tomllib.loads((task_dir / "task.toml").read_text()) + assert [server["name"] for server in config["environment"]["mcp_servers"]] == [ + "bench", + "kernel", + ] + dockerfile = (task_dir / "environment" / "Dockerfile").read_text() + assert dockerfile.endswith("ENV KERNEL_MCP_ENABLED_TOOLSETS=repl\n") + instruction = (task_dir / "steps/run/instruction.md").read_text() + assert "`browser_repl` tool" in instruction + assert "repl.help()" in instruction + assert "execute_playwright_code" not in instruction + + +def test_unknown_toolset_is_refused(task_dir): + toolset = load_toolset("kernel-mcp-local") + try: + toolset.apply([task_dir], toolsets=["shell"]) + except SystemExit as error: + assert "shell" in str(error) + else: + raise AssertionError("expected SystemExit") + + +def test_verify_reads_the_server_call_log(tmp_path): + logs, verifier = tmp_path / "kernel-mcp", tmp_path / "verifier" + logs.mkdir() + verifier.mkdir() + (verifier / "reward.json").write_text('{"reward": 1.0}') + calls = [ + { + "tool": "browser_repl", + "arguments": {"session_id": "s1", "code": "await page.goto(url)"}, + "duration_ms": 900, + }, + { + "tool": "browser_repl", + "arguments": {"session_id": "s1", "code": "repl.write(1)"}, + "duration_ms": 100, + "is_error": True, + }, + {"tool": "finalize", "arguments": {}}, + ] + (logs / "calls.jsonl").write_text("\n".join(json.dumps(call) for call in calls) + "\n") + script = ( + (ROOT / "toolsets/kernel-mcp-local/image/bin/kernel-mcp-verify") + .read_text() + .replace('Path("/var/log/kernel-mcp")', f"Path({str(logs)!r})") + .replace('Path("/logs/verifier")', f"Path({str(verifier)!r})") + ) + env = {**os.environ, "KERNEL_MCP_ENABLED_TOOLSETS": "repl", "KERNEL_SESSION_ID": "s1"} + subprocess.run(["python3", "-c", script], check=True, env=env) + + assert json.loads((verifier / "reward.json").read_text()) == { + "reward": 1.0, + "kernel_mcp_valid": 1.0, + } + result = json.loads((verifier / "kernel-mcp-result.json").read_text()) + assert result["browser_calls"] == 2 + assert result["browser_call_ms"] == 1000 + assert result["browser_call_errors"] == 1 + + +def test_call_log_passes_messages_through_and_records_calls(tmp_path): + server = tmp_path / "server.py" + server.write_text( + "import json, sys\n" + "for line in sys.stdin:\n" + " m = json.loads(line)\n" + " print(json.dumps({'jsonrpc': '2.0', 'id': m['id'], 'result': {'isError': m['id'] == 2}}), flush=True)\n" + ) + log = tmp_path / "calls.jsonl" + messages = [ + { + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": {"name": "browser_repl", "arguments": {"code": "1"}}, + }, + { + "jsonrpc": "2.0", + "id": 2, + "method": "tools/call", + "params": {"name": "browser_repl", "arguments": {}}, + }, + {"jsonrpc": "2.0", "id": 3, "method": "tools/list"}, + ] + result = subprocess.run( + [ + str(ROOT / "toolsets/kernel-mcp-local/image/bin/mcp-call-log"), + str(log), + "python3", + str(server), + ], + input="".join(json.dumps(m) + "\n" for m in messages), + capture_output=True, + text=True, + check=True, + ) + assert [json.loads(line)["id"] for line in result.stdout.splitlines()] == [1, 2, 3] + logged = [json.loads(line) for line in log.read_text().splitlines()] + assert [(call["tool"], call["is_error"]) for call in logged] == [ + ("browser_repl", False), + ("browser_repl", True), + ] + assert logged[0]["arguments"] == {"code": "1"} diff --git a/tests/test_retail_cart.py b/tests/test_retail_cart.py new file mode 100644 index 0000000..ffdd61c --- /dev/null +++ b/tests/test_retail_cart.py @@ -0,0 +1,98 @@ +import json +import os +import subprocess +import tomllib + +import pytest + +from harness.definitions import ROOT, load_suite + +suite = load_suite("retail-cart") +BENCH = ROOT / "suites/retail-cart/image/bench" + + +def test_every_task_has_a_unique_id_and_an_https_start_url(): + tasks = suite.load_tasks() + assert len({task.id for task in tasks}) == len(tasks) + assert all(task.start_url.startswith("https://") for task in tasks) + + +def test_generate_writes_a_harbor_task(tmp_path): + [task_dir] = suite.generate(["costco-microfiber-towels"], tmp_path) + + config = tomllib.loads((task_dir / "task.toml").read_text()) + assert config["metadata"]["site"] == "costco.com" + assert [server["name"] for server in config["environment"]["mcp_servers"]] == ["bench"] + instruction = (task_dir / "steps/run/instruction.md").read_text() + assert "Kirkland Signature" in instruction + assert "https://www.costco.com/" in instruction + setup = (task_dir / "steps/run/workdir/setup.sh").read_text() + assert 'open-start-url "https://www.costco.com/"' in setup + assert (task_dir / "environment/bench/finalize-mcp").is_file() + + +def test_unknown_task_is_refused(tmp_path): + with pytest.raises(SystemExit, match="nope"): + suite.generate(["nope"], tmp_path) + + +def _call(script, messages, tmp_path): + source = ( + (BENCH / script) + .read_text() + .replace('Path("/logs/agent/finalize.json")', f"Path({str(tmp_path / 'finalize.json')!r})") + ) + output = subprocess.run( + ["python3", "-c", source], + input="\n".join(json.dumps(message) for message in messages) + "\n", + capture_output=True, + text=True, + check=True, + ).stdout + return [json.loads(line) for line in output.splitlines()] + + +def test_finalize_records_the_first_report_only(tmp_path): + report = {"evidence": "Access Denied", "reasoning": "403 on home page", "outcome": "blocked"} + call = {"jsonrpc": "2.0", "method": "tools/call"} + responses = _call( + "finalize-mcp", + [ + {**call, "id": 1, "params": {"name": "finalize", "arguments": report}}, + {**call, "id": 2, "params": {"name": "finalize", "arguments": report}}, + ], + tmp_path, + ) + assert responses[0]["result"]["isError"] is False + assert responses[1]["result"]["isError"] is True + assert json.loads((tmp_path / "finalize.json").read_text())["outcome"] == "blocked" + + +def test_score_counts_a_block_apart_from_other_failures(tmp_path): + verifier = tmp_path / "verifier" + (tmp_path / "finalize.json").write_text(json.dumps({"outcome": "blocked"})) + source = ( + (BENCH / "score") + .read_text() + .replace('Path("/logs/agent/finalize.json")', f"Path({str(tmp_path / 'finalize.json')!r})") + .replace('Path("/logs/verifier")', f"Path({str(verifier)!r})") + .replace('Path("/opt/bench/task.json")', f"Path({str(tmp_path / 'task.json')!r})") + ) + env = { + **os.environ, + "KERNEL_SESSION_ID": "s1", + "KERNEL_API_KEY": "k", + "KERNEL_BASE_URL": "http://127.0.0.1:9", + } + subprocess.run(["python3", "-c", source], check=True, env=env) + + rewards = json.loads((verifier / "reward.json").read_text()) + assert rewards == { + "reward": 0.0, + "finalized": 1.0, + "outcome_failed": 0.0, + "outcome_blocked": 1.0, + "outcome_website_error": 0.0, + "outcome_kernel_error": 0.0, + } + assert "error" in json.loads((verifier / "final-page.json").read_text()) diff --git a/tests/test_run.py b/tests/test_run.py new file mode 100644 index 0000000..e74d2bb --- /dev/null +++ b/tests/test_run.py @@ -0,0 +1,69 @@ +import pytest + +from harness import run +from harness.definitions import Arm, Benchmark + + +def test_toolset_args_override_only_the_keys_an_arm_sets(tmp_path, monkeypatch): + applied = {} + + class Suite: + @staticmethod + def generate(task_ids, out_dir, split="test"): + (out_dir / f"task-{split}").mkdir(parents=True) + return [out_dir / f"task-{split}"] + + class Toolset: + def __init__(self, name): + self.name = name + + def apply(self, task_dirs, ref=None, **args): + applied[self.name] = ({path.name for path in task_dirs}, args) + + arms = { + "control": Arm(name="control", agent="codex", model="m", toolset="plain"), + "treatment": Arm( + name="treatment", + agent="codex", + model="m", + toolset="extra", + toolset_args={"source_ref": "main"}, + ), + } + monkeypatch.setattr( + run, + "load_benchmark", + lambda name: Benchmark(name=name, suite="s", arms=list(arms), suite_args={"split": "test"}), + ) + monkeypatch.setattr(run, "load_arm", lambda name: arms[name]) + monkeypatch.setattr(run, "load_suite", lambda name: Suite) + monkeypatch.setattr(run, "load_toolset", Toolset) + monkeypatch.setattr(run, "configure_tasks", lambda task_dirs, browser: {}) + + run.prepare("b", None, None, tmp_path, None, {"split": "train"}, {"source_ref": "branch"}) + assert applied == { + "plain": ({"task-train"}, {}), + "extra": ({"task-train"}, {"source_ref": "branch"}), + } + + applied.clear() + run.prepare("b", ["control"], None, tmp_path / "control", None, None, {"source_ref": "x"}) + assert applied == {"plain": ({"task-test"}, {})} + + with pytest.raises(SystemExit, match="unknown_arg"): + run.prepare("b", None, None, tmp_path / "again", None, None, {"unknown_arg": "x"}) + + +def test_harbor_command_passes_agent_kwargs_as_json(tmp_path): + arm = Arm( + name="a", + agent="codex", + model="gpt-6-luna", + toolset="t", + agent_version="0.163.0", + agent_kwargs={"disable_web_search": True}, + ) + command = run.harbor_command(run.PreparedArm(arm, tmp_path, {}), tmp_path, "job", 2, 3) + assert "version=0.163.0" in command + assert "disable_web_search=true" in command + assert command[command.index("--n-attempts") + 1] == "3" diff --git a/toolsets/kernel-mcp-local/image/bin/kernel-mcp b/toolsets/kernel-mcp-local/image/bin/kernel-mcp index 8965ec9..a871e74 100755 --- a/toolsets/kernel-mcp-local/image/bin/kernel-mcp +++ b/toolsets/kernel-mcp-local/image/bin/kernel-mcp @@ -1,7 +1,8 @@ #!/bin/sh # stdio MCP server for the agent: waits for the local Kernel MCP server and -# proxies to it with the project API key. Nothing may be written to stdout -# before mcp-remote starts, because stdout carries the MCP protocol. +# proxies to it with the project API key, logging each tool call for +# kernel-mcp-verify. Nothing may be written to stdout before mcp-remote +# starts, because stdout carries the MCP protocol. set -eu # mcp-remote expands ${KERNEL_MCP_AUTHORIZATION} itself, which keeps the key @@ -10,7 +11,8 @@ KERNEL_MCP_AUTHORIZATION="Bearer ${KERNEL_API_KEY:-$(cat /run/kernel-mcp/api-key export KERNEL_MCP_AUTHORIZATION for _ in $(seq 1 120); do if curl -s -o /dev/null http://127.0.0.1:3002/; then - exec mcp-remote http://127.0.0.1:3002/mcp \ + exec mcp-call-log /var/log/kernel-mcp/calls.jsonl \ + mcp-remote http://127.0.0.1:3002/mcp \ --header 'Authorization:${KERNEL_MCP_AUTHORIZATION}' fi sleep 1 diff --git a/toolsets/kernel-mcp-local/image/bin/kernel-mcp-entrypoint b/toolsets/kernel-mcp-local/image/bin/kernel-mcp-entrypoint index ceee214..f6d91e3 100755 --- a/toolsets/kernel-mcp-local/image/bin/kernel-mcp-entrypoint +++ b/toolsets/kernel-mcp-local/image/bin/kernel-mcp-entrypoint @@ -8,6 +8,9 @@ set -eu state=/run/kernel-mcp logs=/var/log/kernel-mcp mkdir -p "$state" "$logs" +# The agent may run as another user; its MCP servers log calls here. +touch "$logs/calls.jsonl" +chmod 0666 "$logs/calls.jsonl" printf '%s' "${KERNEL_API_KEY:-}" >"$state/api-key" chmod 0600 "$state/api-key" printf '%s\n' "${KERNEL_SESSION_ID:-}" >"$state/session-id" diff --git a/toolsets/kernel-mcp-local/image/bin/kernel-mcp-verify b/toolsets/kernel-mcp-local/image/bin/kernel-mcp-verify index 2db4c1a..1b202c0 100755 --- a/toolsets/kernel-mcp-local/image/bin/kernel-mcp-verify +++ b/toolsets/kernel-mcp-local/image/bin/kernel-mcp-verify @@ -2,10 +2,14 @@ """Add `kernel_mcp_valid` to the task's rewards. The suite's verifier owns the task reward. This check records whether the -agent called Kernel MCP's `execute_playwright_code`, and whether every call -used the browser Harbor assigned to the trial, so a failed task can be told -apart from a trial that never exercised the toolset. It does not see browser -control outside Kernel MCP, such as direct CDP connections. +agent called the Kernel MCP browser tools its arm enables +(`execute_playwright_code`, `browser_repl`), and whether every call used the +browser Harbor assigned to the trial, so a failed task can be told apart from +a trial that never exercised the toolset. It reads the calls mcp-call-log +recorded at the server, which every agent harness goes through, and does not +see browser control outside Kernel MCP, such as direct CDP connections. + +kernel-mcp-result.json also counts the browser calls and their total time. """ from __future__ import annotations @@ -16,35 +20,33 @@ import shutil from pathlib import Path from typing import Any -LOGS = Path("/logs") -VERIFIER = LOGS / "verifier" -PLAYWRIGHT_TOOLS = { - "execute_playwright_code", - "kernel__execute_playwright_code", - "mcp__kernel__execute_playwright_code", -} +LOGS = Path("/var/log/kernel-mcp") +VERIFIER = Path("/logs/verifier") +TOOLS = {"playwright": "execute_playwright_code", "repl": "browser_repl"} -def read_object(path: Path) -> dict[str, Any]: +def read_calls(path: Path) -> list[dict[str, Any]]: try: - value = json.loads(path.read_text()) - except (OSError, json.JSONDecodeError): - return {} - return value if isinstance(value, dict) else {} + lines = path.read_text().splitlines() + except OSError: + return [] + calls = [] + for line in lines: + try: + calls.append(json.loads(line)) + except json.JSONDecodeError: + continue + return calls def main() -> int: VERIFIER.mkdir(parents=True, exist_ok=True) - if Path("/var/log/kernel-mcp").is_dir(): - shutil.copytree("/var/log/kernel-mcp", VERIFIER / "kernel-mcp", dirs_exist_ok=True) + if LOGS.is_dir(): + shutil.copytree(LOGS, VERIFIER / "kernel-mcp", dirs_exist_ok=True) - trajectory = read_object(LOGS / "agent" / "trajectory.json") - calls = [ - call - for step in trajectory.get("steps") or [] - for call in step.get("tool_calls") or [] - if call.get("function_name") in PLAYWRIGHT_TOOLS - ] + enabled = os.environ.get("KERNEL_MCP_ENABLED_TOOLSETS", "playwright").split(",") + browser_tools = {TOOLS[name] for name in enabled if name in TOOLS} + calls = [call for call in read_calls(LOGS / "calls.jsonl") if call.get("tool") in browser_tools] called_sessions = {(call.get("arguments") or {}).get("session_id") for call in calls} expected_session = os.environ.get("KERNEL_SESSION_ID") checks = { @@ -57,8 +59,12 @@ def main() -> int: result = { "valid": valid, "checks": checks, + "enabled_toolsets": enabled, "expected_session_id": expected_session, "called_session_ids": sorted(str(value) for value in called_sessions), + "browser_calls": len(calls), + "browser_call_ms": sum(call.get("duration_ms") or 0 for call in calls), + "browser_call_errors": sum(1 for call in calls if call.get("is_error")), "kernel_mcp_server_sha": sha_file.read_text().strip() if sha_file.exists() else None, } (VERIFIER / "kernel-mcp-result.json").write_text(json.dumps(result, indent=2)) @@ -67,7 +73,7 @@ def main() -> int: # reward.txt would hide the task reward. reward_path = VERIFIER / "reward.json" if reward_path.exists(): - rewards = read_object(reward_path) + rewards = json.loads(reward_path.read_text()) rewards["kernel_mcp_valid"] = float(valid) reward_path.write_text(json.dumps(rewards, indent=2)) return 0 diff --git a/toolsets/kernel-mcp-local/image/bin/mcp-call-log b/toolsets/kernel-mcp-local/image/bin/mcp-call-log new file mode 100755 index 0000000..c245737 --- /dev/null +++ b/toolsets/kernel-mcp-local/image/bin/mcp-call-log @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +"""Run a stdio MCP server and log every tools/call the agent makes to it. + +Usage: mcp-call-log [args...] + +Messages pass through unchanged. Each completed call appends one line with the +tool name, its arguments, when it started, how long it took and whether it +failed. Agents wrap MCP calls differently (Codex can nest them in a code-mode +cell), so this log, not the agent's trajectory, is the record of what reached +the server. +""" + +from __future__ import annotations + +import json +import subprocess +import sys +import threading +import time +from datetime import UTC, datetime +from typing import IO, Any + + +def parse(line: bytes) -> dict[str, Any]: + try: + message = json.loads(line) + except ValueError: + return {} + return message if isinstance(message, dict) else {} + + +def main() -> int: + log_path, command = sys.argv[1], sys.argv[2:] + try: + log: IO[str] | None = open(log_path, "a") # noqa: SIM115 + except OSError as error: + print(f"mcp-call-log: not logging calls: {error}", file=sys.stderr) + log = None + lock = threading.Lock() + pending: dict[Any, tuple[float, str, dict[str, Any]]] = {} + child = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE) + assert child.stdin is not None and child.stdout is not None + + def to_server() -> None: + for line in sys.stdin.buffer: + message = parse(line) + if message.get("method") == "tools/call" and "id" in message: + with lock: + pending[message["id"]] = ( + time.monotonic(), + datetime.now(UTC).isoformat(), + message.get("params") or {}, + ) + child.stdin.write(line) + child.stdin.flush() + child.stdin.close() + + threading.Thread(target=to_server, daemon=True).start() + for line in child.stdout: + message = parse(line) + with lock: + call = pending.pop(message.get("id"), None) if message else None + if call is not None and log is not None: + started, started_at, params = call + result = message.get("result") or {} + entry = { + "tool": params.get("name"), + "arguments": params.get("arguments"), + "started_at": started_at, + "duration_ms": round((time.monotonic() - started) * 1000), + "is_error": "error" in message or result.get("isError") is True, + } + with lock: + log.write(json.dumps(entry) + "\n") + log.flush() + sys.stdout.buffer.write(line) + sys.stdout.buffer.flush() + return child.wait() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/toolsets/kernel-mcp-local/image/layer.Dockerfile b/toolsets/kernel-mcp-local/image/layer.Dockerfile index fcd089d..4e66a1d 100644 --- a/toolsets/kernel-mcp-local/image/layer.Dockerfile +++ b/toolsets/kernel-mcp-local/image/layer.Dockerfile @@ -1,5 +1,6 @@ # kernel-mcp-local toolset. Appended to the suite's Dockerfile, which must be -# Debian-based with Node.js and Python 3. +# Debian-based with Node.js and Python 3. The toolset appends the +# KERNEL_MCP_ENABLED_TOOLSETS line for the arm. RUN apt-get update \ && apt-get install -y --no-install-recommends redis-server \ && rm -rf /var/lib/apt/lists/* \ @@ -15,7 +16,6 @@ ENV CLERK_SECRET_KEY=sk_test_kernel_mcp_benchmark_local_only \ KERNEL_CLI_PROD_CLIENT_ID=kernel-mcp-benchmark \ KERNEL_CLI_STAGING_CLIENT_ID=kernel-mcp-benchmark \ KERNEL_CLI_DEV_CLIENT_ID=kernel-mcp-benchmark \ - KERNEL_MCP_ENABLED_TOOLSETS=playwright \ REDIS_URL=redis://127.0.0.1:6379 \ NEXT_TELEMETRY_DISABLED=1 RUN bun run build diff --git a/toolsets/kernel-mcp-local/toolset.py b/toolsets/kernel-mcp-local/toolset.py index a1d8093..a492b2e 100644 --- a/toolsets/kernel-mcp-local/toolset.py +++ b/toolsets/kernel-mcp-local/toolset.py @@ -1,8 +1,13 @@ """Kernel MCP server, built from source and run inside the task container. -The agent connects over stdio and drives the trial's browser with -`execute_playwright_code`. Browser lifecycle tools are disabled, so the agent -can only use the browser Harbor created for the trial. +The agent connects over stdio and drives the trial's browser with the tools of +the Kernel MCP toolsets the arm enables: `execute_playwright_code` (the +default `playwright` toolset) or `browser_repl` (`repl`). Browser lifecycle +tools are disabled, so the agent can only use the browser Harbor created for +the trial. + +Arm args: + toolsets: Kernel MCP toolsets to enable, default ["playwright"]. """ from __future__ import annotations @@ -10,6 +15,7 @@ import shutil import subprocess import tempfile +from collections.abc import Sequence from pathlib import Path import tomlkit @@ -19,22 +25,39 @@ REPOSITORY = "https://github.com/kernel/kernel-mcp-server" IMAGE = Path(__file__).with_name("image") +# Tool each supported Kernel MCP toolset gives the agent, and how to use it. +TOOLS = { + "playwright": ( + "execute_playwright_code", + "- Interact through page navigation and DOM/UI actions. Do not call `fetch`, `XMLHttpRequest`, Playwright request APIs, or other HTTP clients inside `execute_playwright_code`.", + ), + "repl": ( + "browser_repl", + "- `browser_repl` runs JavaScript in a persistent REPL attached to your browser. If you have not used it before, start with `repl.help()`.", + ), +} + INSTRUCTION = """ ## Browser tools -- Use the `kernel` MCP server's `execute_playwright_code` tool for all browser interaction. Wait for the server to finish initializing before starting; in Claude Code, call `WaitForMcpServers` if it is still pending. +- Use the `kernel` MCP server's {tools} for all browser interaction. Wait for the server to finish initializing before starting; in Claude Code, call `WaitForMcpServers` if it is still pending. - Before your first browser call, read your browser's session ID from the file `/run/kernel-mcp/session-id` and pass it as `session_id` on every call. - Do not create, list, update, or delete browsers, and do not use Kernel managed auth. -- Interact through page navigation and DOM/UI actions. Do not call `fetch`, `XMLHttpRequest`, Playwright request APIs, or other HTTP clients inside `execute_playwright_code`. +{guidance} """ -def apply(task_dirs: list[Path], ref: str | None = None) -> None: +def apply( + task_dirs: list[Path], ref: str | None = None, toolsets: Sequence[str] = ("playwright",) +) -> None: + unknown = [name for name in toolsets if name not in TOOLS] + if unknown or not toolsets: + raise SystemExit(f"kernel-mcp-local supports toolsets {sorted(TOOLS)}, got {toolsets}") with tempfile.TemporaryDirectory() as tmp: source = Path(tmp) / "source" sha = _fetch_source(ref or "main", source) for task_dir in task_dirs: - _apply(task_dir, source, sha) + _apply(task_dir, source, sha, list(toolsets)) def _fetch_source(ref: str, dest: Path) -> str: @@ -54,13 +77,16 @@ def git(*args: str) -> str: return sha -def _apply(task_dir: Path, source: Path, sha: str) -> None: +def _apply(task_dir: Path, source: Path, sha: str, toolsets: list[str]) -> None: context = task_dir / "environment" / "kernel-mcp-local" shutil.copytree(IMAGE / "bin", context / "bin") shutil.copytree(source, context / "source") dockerfile = task_dir / "environment" / "Dockerfile" dockerfile.write_text( - dockerfile.read_text().rstrip() + "\n\n" + (IMAGE / "layer.Dockerfile").read_text() + dockerfile.read_text().rstrip() + + "\n\n" + + (IMAGE / "layer.Dockerfile").read_text() + + f"ENV KERNEL_MCP_ENABLED_TOOLSETS={','.join(toolsets)}\n" ) config = read_config(task_dir) @@ -68,11 +94,17 @@ def _apply(task_dir: Path, source: Path, sha: str) -> None: server.update( {"name": "kernel", "transport": "stdio", "command": "/usr/local/bin/kernel-mcp", "args": []} ) - servers = tomlkit.aot() + servers = config["environment"].setdefault("mcp_servers", tomlkit.aot()) servers.append(server) - config["environment"]["mcp_servers"] = servers config.setdefault("metadata", tomlkit.table())["kernel_mcp_server_sha"] = sha write_config(task_dir, config) - append_instruction(task_dir, INSTRUCTION) + tools = " and ".join(f"`{TOOLS[name][0]}`" for name in toolsets) + guidance = "\n".join(TOOLS[name][1] for name in toolsets) + append_instruction( + task_dir, + INSTRUCTION.format( + tools=f"{tools} tool{'s' if len(toolsets) > 1 else ''}", guidance=guidance + ), + ) append_test_command(task_dir, "kernel-mcp-verify") From 7d5f8cf65bef13d625016adebd1dc8cd80c01af9 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sat, 31 Oct 2026 18:38:00 +0000 Subject: [PATCH 2/3] Create a proxy per trial in site-config and delete it when the trial ends --- README.md | 4 +- configurators/site-config/configure | 29 ++++-------- harness/environment.py | 55 +++++++++++++++++++++- harness/run.py | 9 +++- tests/test_environment.py | 72 +++++++++++++++++++++++++++++ 5 files changed, 145 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 5049000..3153740 100644 --- a/README.md +++ b/README.md @@ -168,12 +168,12 @@ stdin: {"task": {"config": "", "instruction": ""}, "arg stdout: {"browser": {"stealth": true}, "steering": "advice for the agent" | null, "provenance": {...}} ``` -It never sees the verifier's files. `steering` is appended to the task instruction under "Site guidance". Each task's result is recorded in `jobs/run-.json`. Configurators may be nondeterministic, e.g. when they call an LLM. +`browser` may include `proxy_create`, a [`POST /proxies`](https://kernel.sh/docs/proxies/overview) body: each trial then gets its own proxy, deleted when the trial ends. It never sees the verifier's files. `steering` is appended to the task instruction under "Site guidance". Each task's result is recorded in `jobs/run-.json`. Configurators may be nondeterministic, e.g. when they call an LLM. `site-config` works like this: 1. A model names the task's primary website. 2. The [config registry](https://kernel.sh/docs/config-registry) looks it up. The registry is Kernel's record of which browser and proxy settings have reached a site, and what its bot protection reacted to. -3. A model translates the recommended proxy recipe into a Kernel proxy, which is created once and reused. +3. A model translates the recommended proxy recipe into a Kernel proxy. Each trial creates its own proxy before its browser and deletes it afterwards, so trials never share an ISP IP; `bench run` deletes any a crashed trial left behind. 4. With `steering = true`, the registry's guidance for the site is added to the instruction. ### Add a toolset diff --git a/configurators/site-config/configure b/configurators/site-config/configure index 1bf67f7..25b2e9c 100755 --- a/configurators/site-config/configure +++ b/configurators/site-config/configure @@ -5,8 +5,10 @@ website the task runs on, if there is one. 2. Kernel's config registry looks that site up and returns a recommended browser and proxy recipe, plus guidance for navigating the site. -3. An LLM translates the proxy recipe into a Kernel proxy, which is created - once per distinct recipe and then reused by name. +3. An LLM translates the proxy recipe into Kernel proxy settings. A recipe + that needs a proxy becomes `proxy_create`, the POST /proxies body, which + harness.environment creates for each trial and deletes when it ends, so + no two trials share a proxy's IP. 4. The guidance becomes steering for the agent when `steering` is true. stdin: {"task": {"config": str, "instruction": str}, "args": {"model": str, "steering": bool}} @@ -15,7 +17,6 @@ stdout: {"browser": {...}, "steering": str | null, "provenance": {...}} from __future__ import annotations -import hashlib import json import re import sys @@ -94,23 +95,11 @@ def parse_proxy(answer: dict[str, Any]) -> dict[str, Any]: return {"mode": answer["mode"]} -def proxy_settings(client: Kernel, model: str, recipe: dict[str, Any]) -> dict[str, Any]: +def proxy_settings(model: str, recipe: dict[str, Any]) -> dict[str, Any]: proxy = ask(model, PROXY_PROMPT.format(recipe=json.dumps(recipe, indent=2)), parse_proxy) - if "create" not in proxy: - return proxy - body = proxy["create"] - digest = hashlib.sha256(json.dumps(body, sort_keys=True).encode()).hexdigest()[:12] - name = f"site-config-{body['type']}-{digest}" - for existing in client.proxies.list(): - if existing.name == name: - return {"name": name} - try: - client.proxies.create(**body, name=name) - except Exception: - # Another task may have created the same proxy concurrently. - if not any(existing.name == name for existing in client.proxies.list()): - raise - return {"name": name} + if "create" in proxy: + return {"proxy_create": proxy["create"]} + return {"proxy": proxy} def main() -> int: @@ -138,7 +127,7 @@ def main() -> int: ) if recommendation.get("type") == "recommendation": output["browser"] = dict(recommendation["browser"]) - output["browser"]["proxy"] = proxy_settings(client, model, recommendation["proxy"]) + output["browser"].update(proxy_settings(model, recommendation["proxy"])) if args.get("steering", False): output["steering"] = lookup.get("guidance") json.dump(output, sys.stdout) diff --git a/harness/environment.py b/harness/environment.py index 8c832f2..643eb33 100644 --- a/harness/environment.py +++ b/harness/environment.py @@ -4,22 +4,75 @@ `environment/kernel-config.json`. Harbor applies one set of environment kwargs to a whole job, so this subclass reads the file per trial and passes it to the provider as `browser_create_overrides`. + +A `proxy_create` entry in the file is a POST /proxies body rather than a +browser param. Each trial creates its own proxy from it before the browser, +and deletes it after the browser, so no two trials share a proxy's IP. The +proxy's name starts with `proxy_prefix(run_id)` when `BENCH_RUN_ID` is set, +so `delete_run_proxies` can remove any a crashed trial left behind. """ from __future__ import annotations import json +import os +import uuid from pathlib import Path from typing import Any from harbor.environments.kernel import KernelEnvironment +from kernel import AsyncKernel, Kernel CONFIG_FILE = "kernel-config.json" +PROXY_CREATE = "proxy_create" +RUN_ID_ENV = "BENCH_RUN_ID" + + +def proxy_prefix(run_id: str) -> str: + return f"bench-{run_id}-" + + +def delete_run_proxies(run_id: str) -> list[str]: + """Delete the proxies trials of `run_id` created and did not delete themselves.""" + client = Kernel() + prefix = proxy_prefix(run_id) + leftovers = [ + proxy for proxy in client.proxies.list(query=prefix) if proxy.name.startswith(prefix) + ] + for proxy in leftovers: + client.proxies.delete(proxy.id) + return [proxy.name for proxy in leftovers] class KernelTaskEnvironment(KernelEnvironment): def __init__(self, environment_dir: Path, *args: Any, **kwargs: Any) -> None: + self._proxy_create: dict[str, Any] | None = None + self._proxy_id: str | None = None config_path = Path(environment_dir) / CONFIG_FILE if config_path.is_file(): - kwargs["browser_create_overrides"] = json.loads(config_path.read_text()) + overrides = json.loads(config_path.read_text()) + self._proxy_create = overrides.pop(PROXY_CREATE, None) + kwargs["browser_create_overrides"] = overrides super().__init__(environment_dir, *args, **kwargs) + + async def start(self, force_build: bool) -> None: + if self._proxy_create is not None: + run_id = os.environ.get(RUN_ID_ENV) or "local" + name = f"{proxy_prefix(run_id)}{uuid.uuid4().hex[:12]}" + async with AsyncKernel(api_key=self._api_key) as client: + proxy = await client.proxies.create(**self._proxy_create, name=name) + self._proxy_id = proxy.id + self._browser_create_overrides["proxy_id"] = proxy.id + await super().start(force_build) + + async def stop(self, delete: bool) -> None: + try: + await super().stop(delete) + finally: + if self._proxy_id is not None: + try: + async with AsyncKernel(api_key=self._api_key) as client: + await client.proxies.delete(self._proxy_id) + self._proxy_id = None + except Exception as error: + self.logger.error(f"Failed to delete Kernel proxy {self._proxy_id}: {error}") diff --git a/harness/run.py b/harness/run.py index c4c082e..f18b53f 100644 --- a/harness/run.py +++ b/harness/run.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import os import shutil import subprocess import sys @@ -13,6 +14,7 @@ from harness import preflight from harness.configure import configure_tasks from harness.definitions import Arm, load_arm, load_benchmark, load_suite, load_toolset +from harness.environment import RUN_ID_ENV, delete_run_proxies # Transient failures worth retrying as a whole trial: Kernel API hiccups and # slow agent installs. Image build and task setup failures are not retried. @@ -94,7 +96,8 @@ def run( `jobs_dir/run-.json` maps each arm to its Harbor job directory, browser configurations, and exit status. It is written before Harbor starts and updated as each arm finishes, so an interrupted run can still - be reported. + be reported. Proxies that trials created and could not delete are + removed once every arm has finished. """ preflight.check_kernel_project() load_suite(load_benchmark(benchmark_name).suite).preflight() @@ -114,6 +117,7 @@ def run( } manifest_path.write_text(json.dumps(manifest, indent=2)) + env = {**os.environ, RUN_ID_ENV: run_id} processes = {} for item in prepared: job_name = f"{item.arm.name}-{run_id}" @@ -122,10 +126,13 @@ def run( harbor_command(item, jobs_dir, job_name, concurrency, attempts), stdout=log, stderr=subprocess.STDOUT, + env=env, ) for name, process in processes.items(): manifest[name]["status"] = process.wait() manifest_path.write_text(json.dumps(manifest, indent=2)) + for name in delete_run_proxies(run_id): + print(f"Deleted leftover proxy {name}") return {name: entry["status"] for name, entry in manifest.items()} diff --git a/tests/test_environment.py b/tests/test_environment.py index 557e31f..9f53611 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -1,7 +1,9 @@ +import asyncio import json from harbor.environments.kernel import KernelEnvironment +from harness import environment from harness.environment import KernelTaskEnvironment @@ -17,3 +19,73 @@ def test_task_browser_config_becomes_create_overrides(tmp_path, monkeypatch): KernelTaskEnvironment(environment_dir=tmp_path, session_id="s") assert captured == {"session_id": "s", "browser_create_overrides": {"stealth": False}} + + +class FakeProxies: + def __init__(self, calls): + self.calls = calls + + async def create(self, **body): + self.calls.append(("create", body)) + return type("Proxy", (), {"id": "p1"})() + + async def delete(self, proxy_id): + self.calls.append(("delete", proxy_id)) + + +class FakeKernel: + calls: list = [] + + def __init__(self, **kwargs): + self.proxies = FakeProxies(FakeKernel.calls) + + async def __aenter__(self): + return self + + async def __aexit__(self, *exc): + return False + + +def _environment(tmp_path, monkeypatch, calls): + def init(self, environment_dir, **kwargs): + self._api_key = "k" + self._browser_create_overrides = kwargs["browser_create_overrides"] + + async def start(self, force_build): + calls.append(("browser", dict(self._browser_create_overrides))) + + async def stop(self, delete): + calls.append(("stop browser", None)) + + monkeypatch.setattr(KernelEnvironment, "__init__", init) + monkeypatch.setattr(KernelEnvironment, "start", start) + monkeypatch.setattr(KernelEnvironment, "stop", stop) + monkeypatch.setattr(environment, "AsyncKernel", FakeKernel) + FakeKernel.calls = calls + config = {"stealth": True, "proxy_create": {"type": "isp", "config": {"country": "US"}}} + (tmp_path / "kernel-config.json").write_text(json.dumps(config)) + return KernelTaskEnvironment(environment_dir=tmp_path) + + +def test_each_trial_creates_its_own_proxy_and_deletes_it_after_the_browser(tmp_path, monkeypatch): + calls = [] + monkeypatch.setenv("BENCH_RUN_ID", "r1") + env = _environment(tmp_path, monkeypatch, calls) + + asyncio.run(env.start(force_build=False)) + asyncio.run(env.stop(delete=True)) + + (kind, body), browser, stop, delete = calls + assert kind == "create" + assert body["type"] == "isp" and body["name"].startswith("bench-r1-") + assert browser == ("browser", {"stealth": True, "proxy_id": "p1"}) + assert stop == ("stop browser", None) + assert delete == ("delete", "p1") + + +def test_proxy_names_differ_between_trials(tmp_path, monkeypatch): + calls = [] + for _ in range(2): + asyncio.run(_environment(tmp_path, monkeypatch, calls).start(force_build=False)) + names = [body["name"] for kind, body in calls if kind == "create"] + assert len(set(names)) == 2 From 156fcbed14713a9cfe7793116036ff826cc002e5 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sat, 31 Oct 2026 18:44:44 +0000 Subject: [PATCH 3/3] Report trials whose proxy could not be created as ProxyCreationError; run 10 attempts by default --- .github/workflows/retail-cart.yml | 2 +- README.md | 6 ++--- arms/codex-repl-site-config.toml | 2 +- harness/environment.py | 17 ++++++++++--- report/publish-braintrust.ts | 1 + report/report.ts | 17 +++++++++++++ report/results.test.ts | 40 ++++++++++++++++++++++++++++++- report/results.ts | 26 ++++++++++++++++++++ tests/test_environment.py | 17 +++++++++++++ 9 files changed, 119 insertions(+), 9 deletions(-) diff --git a/.github/workflows/retail-cart.yml b/.github/workflows/retail-cart.yml index 1019679..83183ff 100644 --- a/.github/workflows/retail-cart.yml +++ b/.github/workflows/retail-cart.yml @@ -16,7 +16,7 @@ on: attempts: description: Trials per task and arm required: false - default: "3" + default: "10" type: string concurrency: description: Concurrent trials per arm diff --git a/README.md b/README.md index 3153740..cc7b4ed 100644 --- a/README.md +++ b/README.md @@ -126,12 +126,12 @@ Guest add-to-cart flows on 16 retail and industrial-supply sites with heavy bot | Arm | Agent | Browser | |---|---|---| | `codex-repl` | Codex, `gpt-6-luna`, Kernel MCP's `browser_repl` only | Stealth | -| `codex-repl-site-config` | Same | From Kernel's config registry per site, plus its guidance | +| `codex-repl-site-config` | Same; `gpt-6-luna` also picks the site and proxy for the configurator | From Kernel's config registry per site, plus its guidance. Each trial gets its own proxy. | -The agent ends each task by calling the `bench` MCP server's `finalize` tool with its evidence and an outcome: `success`, `failed`, `blocked` (bot protection stopped it), `website_error` or `kernel_error`. `reward` is 1 for a reported success, and `outcome_` records the failure, so bot blocks are counted apart from the agent's own failures. The verifier also saves the browser's final URL and title. Run several attempts per task to get a pass rate per site: +The agent ends each task by calling the `bench` MCP server's `finalize` tool with its evidence and an outcome: `success`, `failed`, `blocked` (bot protection stopped it), `website_error` or `kernel_error`. `reward` is 1 for a reported success, and `outcome_` records the failure, so bot blocks are counted apart from the agent's own failures. A trial whose proxy cannot be created fails with `ProxyCreationError`, listed under infrastructure failures in the report. The verifier also saves the browser's final URL and title. Run several attempts per task to get a pass rate per site: ```bash -uv run bench run retail-cart --arms codex-repl --attempts 3 +uv run bench run retail-cart --attempts 10 ``` ## Extending diff --git a/arms/codex-repl-site-config.toml b/arms/codex-repl-site-config.toml index e99c8f3..cf6d092 100644 --- a/arms/codex-repl-site-config.toml +++ b/arms/codex-repl-site-config.toml @@ -17,5 +17,5 @@ config = { stealth = true } configurator = "site-config" [browser.configurator_args] -model = "anthropic/claude-haiku-5-5" +model = "openai/gpt-6-luna" steering = true diff --git a/harness/environment.py b/harness/environment.py index 643eb33..9a33116 100644 --- a/harness/environment.py +++ b/harness/environment.py @@ -9,7 +9,9 @@ browser param. Each trial creates its own proxy from it before the browser, and deletes it after the browser, so no two trials share a proxy's IP. The proxy's name starts with `proxy_prefix(run_id)` when `BENCH_RUN_ID` is set, -so `delete_run_proxies` can remove any a crashed trial left behind. +so `delete_run_proxies` can remove any a crashed trial left behind. A +trial whose proxy cannot be created fails with `ProxyCreationError`, so +reports can count those trials apart from other infrastructure failures. """ from __future__ import annotations @@ -28,6 +30,10 @@ RUN_ID_ENV = "BENCH_RUN_ID" +class ProxyCreationError(RuntimeError): + """The trial's proxy could not be created, so no browser was started.""" + + def proxy_prefix(run_id: str) -> str: return f"bench-{run_id}-" @@ -59,8 +65,13 @@ async def start(self, force_build: bool) -> None: if self._proxy_create is not None: run_id = os.environ.get(RUN_ID_ENV) or "local" name = f"{proxy_prefix(run_id)}{uuid.uuid4().hex[:12]}" - async with AsyncKernel(api_key=self._api_key) as client: - proxy = await client.proxies.create(**self._proxy_create, name=name) + try: + async with AsyncKernel(api_key=self._api_key) as client: + proxy = await client.proxies.create(**self._proxy_create, name=name) + except Exception as error: + raise ProxyCreationError( + f"Could not create {self._proxy_create.get('type')} proxy: {error}" + ) from error self._proxy_id = proxy.id self._browser_create_overrides["proxy_id"] = proxy.id await super().start(force_build) diff --git a/report/publish-braintrust.ts b/report/publish-braintrust.ts index 66e1597..b66a043 100644 --- a/report/publish-braintrust.ts +++ b/report/publish-braintrust.ts @@ -203,6 +203,7 @@ function trialMetadata(trial: BenchmarkTrial): Record { model: trial.model, kernelMcpSha: trial.kernelMcpSha, errorClass: trial.errorClass, + errorType: trial.errorType, rewards: trial.rewards, durationMs: trial.metrics.durationMs, }; diff --git a/report/report.ts b/report/report.ts index 9db9679..b605c81 100644 --- a/report/report.ts +++ b/report/report.ts @@ -114,6 +114,23 @@ export function renderMarkdown( ); } + const failures = summaries.flatMap((summary) => + summary.infraFailures.map( + (failure) => + `| ${summary.arm} | ${failure.errorType} | ${failure.taskName} | ${failure.count} |`, + ), + ); + if (failures.length > 0) { + lines.push( + "", + "### Infrastructure failures", + "", + "| Arm | Error | Task | Trials |", + "|---|---|---|---:|", + ...failures, + ); + } + if (typeof publication?.url === "string") { lines.push("", `[Open the Braintrust experiment](${publication.url})`); } diff --git a/report/results.test.ts b/report/results.test.ts index d7c6522..63c18b3 100644 --- a/report/results.test.ts +++ b/report/results.test.ts @@ -10,7 +10,12 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { buildExperimentEvents, publishBenchmark } from "./publish-braintrust"; import { renderMarkdown } from "./report"; -import { readBenchmarkArm, selectPrimaryReward, summarizeArm } from "./results"; +import { + infraFailures, + readBenchmarkArm, + selectPrimaryReward, + summarizeArm, +} from "./results"; import { assertSafeToPublish, collectSensitiveValues, @@ -478,6 +483,39 @@ describe("Harbor result ingestion", () => { }, ]); expect(infra).toContain("candidate had 1 infrastructure failure"); + + const proxy = renderMarkdown("test", [ + { + ...summary, + infraFailures: [ + { errorType: "ProxyCreationError", taskName: "task-two", count: 3 }, + ], + }, + ]); + expect(proxy).toContain( + "| candidate | ProxyCreationError | task-two | 3 |", + ); + }); + + test("counts infrastructure failures by exception type and task", () => { + const root = fixture(); + const failedPath = join(root, "task-two__def", "result.json"); + const failed = JSON.parse(readFileSync(failedPath, "utf8")) as { + exception_info: unknown; + }; + failed.exception_info = { + exception_type: "ProxyCreationError", + exception_message: "Could not create mobile proxy", + }; + writeJson(failedPath, failed); + + const arm = readBenchmarkArm({ name: "candidate", path: root }); + expect(arm.trials.find((trial) => trial.error)?.errorType).toBe( + "ProxyCreationError", + ); + expect(infraFailures(arm)).toEqual([ + { errorType: "ProxyCreationError", taskName: "v2-task-two", count: 1 }, + ]); }); test("keeps full errors until redaction and clamps derived scores", () => { diff --git a/report/results.ts b/report/results.ts index 7910222..8f3d77a 100644 --- a/report/results.ts +++ b/report/results.ts @@ -42,6 +42,7 @@ export interface BenchmarkTrial { scores: Record; error?: string; errorClass?: "infra"; + errorType?: string; metrics: BenchmarkMetrics; phases: { environmentSetup?: BenchmarkPhase; @@ -80,6 +81,7 @@ export interface ArmSummary { strict?: number; strictScored: number; infraErrors: number; + infraFailures: { errorType: string; taskName: string; count: number }[]; retries: number; ungraded: number; complete: boolean; @@ -258,6 +260,7 @@ function parseTrial(arm: string, trialDir: string): BenchmarkTrial { Object.keys(rewards).length === 0 ? steps.map((step) => object(step).exception_info).find(Boolean) : undefined; + const exceptionInfo = result.exception_info ?? stepException; const error = errorText( result.exception_info ?? stepException ?? @@ -296,6 +299,10 @@ function parseTrial(arm: string, trialDir: string): BenchmarkTrial { scores: trialScores(rewards, error !== undefined), error, errorClass: error === undefined ? undefined : "infra", + errorType: + error === undefined + ? undefined + : string(object(exceptionInfo).exception_type), metrics: { inputTokens: sumStepMetric(steps, "n_input_tokens"), cacheTokens: sumStepMetric(steps, "n_cache_tokens"), @@ -444,6 +451,7 @@ export function summarizeArm(arm: BenchmarkArm): ArmSummary { ), strictScored: strict.length, infraErrors, + infraFailures: infraFailures(arm), retries: arm.nRetries, ungraded, complete: incompleteReasons.length === 0, @@ -480,3 +488,21 @@ export function summarizeArm(arm: BenchmarkArm): ArmSummary { : `mixed: ${configurations.join(", ")}`, }; } + +/** Count each arm's infrastructure failures by exception type and task. */ +export function infraFailures( + arm: BenchmarkArm, +): { errorType: string; taskName: string; count: number }[] { + const counts = new Map(); + for (const trial of arm.trials) { + if (trial.errorClass !== "infra") continue; + const key = JSON.stringify([trial.errorType ?? "unknown", trial.taskName]); + counts.set(key, (counts.get(key) ?? 0) + 1); + } + return [...counts] + .map(([key, count]) => { + const [errorType, taskName] = JSON.parse(key) as [string, string]; + return { errorType, taskName, count }; + }) + .sort((a, b) => b.count - a.count || a.taskName.localeCompare(b.taskName)); +} diff --git a/tests/test_environment.py b/tests/test_environment.py index 9f53611..92e564e 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -1,6 +1,7 @@ import asyncio import json +import pytest from harbor.environments.kernel import KernelEnvironment from harness import environment @@ -89,3 +90,19 @@ def test_proxy_names_differ_between_trials(tmp_path, monkeypatch): asyncio.run(_environment(tmp_path, monkeypatch, calls).start(force_build=False)) names = [body["name"] for kind, body in calls if kind == "create"] assert len(set(names)) == 2 + + +def test_a_proxy_that_cannot_be_created_fails_the_trial_with_proxy_creation_error( + tmp_path, monkeypatch +): + calls = [] + env = _environment(tmp_path, monkeypatch, calls) + + async def refuse(self, **body): + raise ValueError("403") + + monkeypatch.setattr(FakeProxies, "create", refuse) + + with pytest.raises(environment.ProxyCreationError, match="isp proxy: 403"): + asyncio.run(env.start(force_build=False)) + assert calls == []