From 92feba165f004c410387906107ad76e880ca075c Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Fri, 21 Aug 2026 17:11:35 +0000 Subject: [PATCH 01/29] Add Harbor MCP benchmark harness --- .dockerignore | 8 + .gitignore | 8 + benchmarks/harbor/README.md | 49 ++++++ benchmarks/harbor/bin/kernel-mcp-local | 13 ++ benchmarks/harbor/bin/mcp-telemetry-proxy.mjs | 112 ++++++++++++++ benchmarks/harbor/bin/start-kernel-mcp-server | 63 ++++++++ benchmarks/harbor/bin/verify-smoke.py | 146 ++++++++++++++++++ benchmarks/harbor/build-image.sh | 65 ++++++++ benchmarks/harbor/image/Dockerfile | 34 ++++ benchmarks/harbor/mcp/kernel.json | 7 + benchmarks/harbor/prepare-task.py | 38 +++++ benchmarks/harbor/run-smoke.sh | 84 ++++++++++ benchmarks/harbor/smoke/environment/.gitkeep | 0 .../harbor/smoke/steps/run/instruction.md | 17 ++ .../harbor/smoke/steps/run/tests/test.sh | 4 + .../harbor/smoke/steps/run/workdir/setup.sh | 12 ++ benchmarks/harbor/smoke/task.toml | 51 ++++++ 17 files changed, 711 insertions(+) create mode 100644 .dockerignore create mode 100644 benchmarks/harbor/README.md create mode 100755 benchmarks/harbor/bin/kernel-mcp-local create mode 100644 benchmarks/harbor/bin/mcp-telemetry-proxy.mjs create mode 100755 benchmarks/harbor/bin/start-kernel-mcp-server create mode 100755 benchmarks/harbor/bin/verify-smoke.py create mode 100755 benchmarks/harbor/build-image.sh create mode 100644 benchmarks/harbor/image/Dockerfile create mode 100644 benchmarks/harbor/mcp/kernel.json create mode 100755 benchmarks/harbor/prepare-task.py create mode 100755 benchmarks/harbor/run-smoke.sh create mode 100644 benchmarks/harbor/smoke/environment/.gitkeep create mode 100644 benchmarks/harbor/smoke/steps/run/instruction.md create mode 100755 benchmarks/harbor/smoke/steps/run/tests/test.sh create mode 100755 benchmarks/harbor/smoke/steps/run/workdir/setup.sh create mode 100644 benchmarks/harbor/smoke/task.toml diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..b58cdc1 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,8 @@ +.git +.github +.next +node_modules +coverage +.env* +*.log +benchmarks/harbor/.image.env diff --git a/.gitignore b/.gitignore index 4069f2b..4e9fb8f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ # Dependencies node_modules/ +__pycache__/ +*.py[cod] npm-debug.log* yarn-debug.log* yarn-error.log* @@ -107,5 +109,11 @@ Makefile # private key mcp-key.pem +# Harbor benchmark runtime data +benchmarks/harbor/.image.env +benchmarks/harbor/.run.env +benchmarks/harbor/jobs/ +benchmarks/harbor/image/source-sha + # TypeScript incremental build cache tsconfig.tsbuildinfo diff --git a/benchmarks/harbor/README.md b/benchmarks/harbor/README.md new file mode 100644 index 0000000..dcb0ae8 --- /dev/null +++ b/benchmarks/harbor/README.md @@ -0,0 +1,49 @@ +# Harbor MCP benchmarks + +This directory runs stock Harbor agents against a locally built `kernel-mcp-server` in a single Hypeman sandbox. The smoke task makes two read-only calls through the configured stdio MCP server and writes standard Harbor job artifacts. + +## Requirements + +- Harbor 0.21.0 with `harbor_hypeman:HypemanEnvironment` +- `harbor-hypeman` with existing Hypeman image-reference support +- Hypeman CLI and credentials +- `KERNEL_MCP_BENCHMARK_API_KEY` scoped to an isolated evaluation project +- `KERNEL_MCP_BENCHMARK_PROJECT_ID` +- `ANTHROPIC_API_KEY` for Claude Code +- `OPENAI_API_KEY` for Codex + +## Build the image + +```bash +./benchmarks/harbor/build-image.sh +``` + +The build uses the current Git SHA, installs dependencies with Bun, runs the production Next.js build, and writes the resulting image reference to the ignored `.image.env` file. Hypeman can report a failed build before the converted image becomes visible; the script performs a bounded 60-second ready-image check for that case. + +## Run the smoke task + +```bash +export KERNEL_MCP_BENCHMARK_PROJECT_ID=project_id +./benchmarks/harbor/run-smoke.sh claude-code +./benchmarks/harbor/run-smoke.sh codex +``` + +Defaults: + +| Agent | Version | Model | +| ----------- | ------: | ---------------------------- | +| Claude Code | 2.1.110 | `claude-sonnet-4-5-20250929` | +| Codex | 0.120.0 | `gpt-5.3-codex` | + +Override models with `CLAUDE_BENCHMARK_MODEL` or `CODEX_BENCHMARK_MODEL`. Runs have a 10-minute wall-clock limit; change it with `HARBOR_BENCHMARK_TIMEOUT`. + +The output defaults to `/tmp/kernel-mcp-harbor-jobs/`. Each successful trial contains: + +- `steps/run/agent/trajectory.json` in ATIF format +- native agent logs and session data +- `steps/run/artifacts/logs/kernel-mcp/requests.jsonl` with MCP latency and status +- server stdout and stderr +- source SHA and Hypeman identity in `run-manifest.json` +- numeric Harbor rewards plus detailed `smoke-result.json` + +The verifier requires native trajectory calls to `get_connection_context` and `manage_browsers`; direct HTTP or custom MCP-client workarounds do not pass. diff --git a/benchmarks/harbor/bin/kernel-mcp-local b/benchmarks/harbor/bin/kernel-mcp-local new file mode 100755 index 0000000..5b79a8e --- /dev/null +++ b/benchmarks/harbor/bin/kernel-mcp-local @@ -0,0 +1,13 @@ +#!/bin/sh +set -eu + +key_file=/run/kernel-mcp-benchmark/api-key +if [ -z "${KERNEL_API_KEY:-}" ] && [ -r "$key_file" ]; then + KERNEL_API_KEY=$(cat "$key_file") + export KERNEL_API_KEY +fi +: "${KERNEL_API_KEY:?KERNEL_API_KEY is required}" + +exec npx -y mcp-remote@0.1.38 \ + http://127.0.0.1:3002/mcp \ + --header "Authorization: Bearer ${KERNEL_API_KEY}" diff --git a/benchmarks/harbor/bin/mcp-telemetry-proxy.mjs b/benchmarks/harbor/bin/mcp-telemetry-proxy.mjs new file mode 100644 index 0000000..ed1f029 --- /dev/null +++ b/benchmarks/harbor/bin/mcp-telemetry-proxy.mjs @@ -0,0 +1,112 @@ +import fs from "node:fs"; +import http from "node:http"; + +const listenPort = Number(process.env.KERNEL_MCP_PROXY_PORT || 3002); +const upstreamPort = Number(process.env.KERNEL_MCP_SERVER_PORT || 3003); +const logPath = + process.env.KERNEL_MCP_REQUEST_LOG || "/logs/kernel-mcp/requests.jsonl"; + +function requestMetadata(body) { + try { + const payload = JSON.parse(body); + return { + jsonrpc_method: payload.method ?? null, + tool_name: + payload.method === "tools/call" ? (payload.params?.name ?? null) : null, + request_id: payload.id ?? null, + }; + } catch { + return { jsonrpc_method: null, tool_name: null, request_id: null }; + } +} + +function responseSucceeded(statusCode, body) { + if (statusCode < 200 || statusCode >= 300) return false; + const candidates = body + .split("\n") + .filter((line) => line.startsWith("data:")) + .map((line) => line.slice(5).trim()); + if (candidates.length === 0) candidates.push(body.trim()); + + for (const candidate of candidates) { + if (!candidate || candidate === "[DONE]") continue; + try { + const payload = JSON.parse(candidate); + if (payload.error || payload.result?.isError === true) return false; + } catch { + // Non-JSON response bodies are successful when the HTTP status succeeded. + } + } + return true; +} + +function appendLog(entry) { + fs.appendFileSync(logPath, `${JSON.stringify(entry)}\n`, { mode: 0o600 }); +} + +const server = http.createServer((clientRequest, clientResponse) => { + const startedAt = new Date(); + const requestChunks = []; + + clientRequest.on("data", (chunk) => requestChunks.push(chunk)); + clientRequest.on("end", () => { + const requestBody = Buffer.concat(requestChunks); + const metadata = requestMetadata(requestBody.toString("utf8")); + const headers = { ...clientRequest.headers }; + headers.host = `127.0.0.1:${upstreamPort}`; + headers["content-length"] = String(requestBody.length); + + const upstreamRequest = http.request( + { + host: "127.0.0.1", + port: upstreamPort, + method: clientRequest.method, + path: clientRequest.url, + headers, + }, + (upstreamResponse) => { + const responseChunks = []; + upstreamResponse.on("data", (chunk) => responseChunks.push(chunk)); + upstreamResponse.on("end", () => { + const responseBody = Buffer.concat(responseChunks); + const statusCode = upstreamResponse.statusCode ?? 502; + clientResponse.writeHead(statusCode, upstreamResponse.headers); + clientResponse.end(responseBody); + + appendLog({ + started_at: startedAt.toISOString(), + duration_ms: Date.now() - startedAt.getTime(), + http_method: clientRequest.method, + path: clientRequest.url, + http_status: statusCode, + success: responseSucceeded( + statusCode, + responseBody.toString("utf8"), + ), + ...metadata, + }); + }); + }, + ); + + upstreamRequest.on("error", (error) => { + if (!clientResponse.headersSent) clientResponse.writeHead(502); + clientResponse.end("Bad Gateway"); + appendLog({ + started_at: startedAt.toISOString(), + duration_ms: Date.now() - startedAt.getTime(), + http_method: clientRequest.method, + path: clientRequest.url, + http_status: 502, + success: false, + error: error.message, + ...metadata, + }); + }); + upstreamRequest.end(requestBody); + }); +}); + +server.listen(listenPort, "127.0.0.1", () => { + console.log(`MCP telemetry proxy listening on 127.0.0.1:${listenPort}`); +}); diff --git a/benchmarks/harbor/bin/start-kernel-mcp-server b/benchmarks/harbor/bin/start-kernel-mcp-server new file mode 100755 index 0000000..fe299ae --- /dev/null +++ b/benchmarks/harbor/bin/start-kernel-mcp-server @@ -0,0 +1,63 @@ +#!/bin/bash +set -euo pipefail + +: "${KERNEL_API_KEY:?KERNEL_API_KEY is required}" + +log_dir=/logs/kernel-mcp +key_dir=/run/kernel-mcp-benchmark +mkdir -p "$log_dir" /logs/artifacts "$key_dir" +chmod 0777 "$log_dir" /logs/artifacts +chmod 0700 "$key_dir" +printf '%s' "$KERNEL_API_KEY" >"$key_dir/api-key" +chmod 0600 "$key_dir/api-key" + +redis-server --daemonize yes --bind 127.0.0.1 --port 6379 \ + --logfile "$log_dir/redis.log" --dir /tmp + +cd /opt/kernel-mcp-server +nohup ./node_modules/.bin/next start -p 3003 \ + >"$log_dir/server.stdout.log" \ + 2>"$log_dir/server.stderr.log" & +echo $! >"$log_dir/server.pid" + +nohup node /usr/local/lib/mcp-telemetry-proxy.mjs \ + >"$log_dir/proxy.stdout.log" \ + 2>"$log_dir/proxy.stderr.log" & +echo $! >"$log_dir/proxy.pid" + +for _ in $(seq 1 90); do + if curl -fsS -X POST http://127.0.0.1:3002/mcp \ + -H "Authorization: Bearer ${KERNEL_API_KEY}" \ + -H "Content-Type: application/json" \ + -H "Accept: application/json, text/event-stream" \ + --data '{"jsonrpc":"2.0","id":"benchmark-healthcheck","method":"initialize","params":{"protocolVersion":"2025-03-26","capabilities":{},"clientInfo":{"name":"harbor-healthcheck","version":"1.0.0"}}}' \ + >"$log_dir/initialize-response.txt"; then + break + fi + sleep 1 +done + +if [ ! -s "$log_dir/initialize-response.txt" ]; then + echo "Kernel MCP server did not become ready" >&2 + tail -100 "$log_dir/server.stderr.log" >&2 || true + exit 1 +fi + +python3 - <<'PY' +import json +import os +import platform +from datetime import datetime, timezone +from pathlib import Path + +manifest = { + "kernel_mcp_server_sha": Path("/opt/kernel-mcp-server/SOURCE_SHA").read_text().strip(), + "image": os.environ.get("KERNEL_MCP_BENCHMARK_IMAGE", ""), + "hypeman_instance_name": os.environ.get("HYPEMAN_INSTANCE_NAME", ""), + "sandbox_hostname": platform.node(), + "started_at": datetime.now(timezone.utc).isoformat(), +} +Path("/logs/kernel-mcp/run-manifest.json").write_text(json.dumps(manifest, indent=2)) +PY + +printf 'ready\n' >"$log_dir/ready" diff --git a/benchmarks/harbor/bin/verify-smoke.py b/benchmarks/harbor/bin/verify-smoke.py new file mode 100755 index 0000000..cacc600 --- /dev/null +++ b/benchmarks/harbor/bin/verify-smoke.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import json +import os +from pathlib import Path +from typing import Any + +LOGS_DIR = Path(os.environ.get("HARBOR_LOGS_DIR", "/logs")) +VERIFIER_DIR = LOGS_DIR / "verifier" + + +def read_json(path: Path) -> dict[str, Any] | None: + try: + value = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError): + return None + return value if isinstance(value, dict) else None + + +def read_requests(path: Path) -> list[dict[str, Any]]: + requests = [] + try: + lines = path.read_text().splitlines() + except OSError: + return requests + for line in lines: + try: + value = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(value, dict): + requests.append(value) + return requests + + +def successful_call(requests: list[dict[str, Any]], name: str) -> dict[str, Any] | None: + return next( + ( + request + for request in requests + if request.get("jsonrpc_method") == "tools/call" + and request.get("tool_name") == name + and request.get("success") is True + ), + None, + ) + + +def trajectory_tool_names(trajectory: dict[str, Any] | None) -> list[str]: + names = [] + for step in (trajectory or {}).get("steps") or []: + if not isinstance(step, dict): + continue + for call in step.get("tool_calls") or []: + if isinstance(call, dict) and isinstance(call.get("function_name"), str): + names.append(call["function_name"]) + return names + + +def main() -> int: + VERIFIER_DIR.mkdir(parents=True, exist_ok=True) + requests = read_requests(LOGS_DIR / "kernel-mcp/requests.jsonl") + report = read_json(LOGS_DIR / "artifacts/agent-report.json") + trajectory = read_json(LOGS_DIR / "agent/trajectory.json") + manifest = read_json(LOGS_DIR / "kernel-mcp/run-manifest.json") + + context_call = successful_call(requests, "get_connection_context") + browsers_call = successful_call(requests, "manage_browsers") + expected_project_id = os.environ.get("KERNEL_MCP_EXPECTED_PROJECT_ID", "") + report_matches = bool( + report + and report.get("get_connection_context_succeeded") is True + and report.get("manage_browsers_list_succeeded") is True + and report.get("connection_scope_kind") == "project" + and report.get("project_id") == expected_project_id + ) + source_sha_matches = bool( + manifest + and manifest.get("kernel_mcp_server_sha") + == os.environ.get("KERNEL_MCP_SOURCE_SHA") + ) + trajectory_names = trajectory_tool_names(trajectory) + trajectory_has_calls = any( + name.endswith("get_connection_context") for name in trajectory_names + ) and any(name.endswith("manage_browsers") for name in trajectory_names) + hypeman_identity_present = bool( + manifest and manifest.get("hypeman_instance_name") + ) + + checks = { + "get_connection_context": context_call is not None, + "manage_browsers_list": browsers_call is not None, + "agent_report": report_matches, + "source_sha": source_sha_matches, + "hypeman_identity": hypeman_identity_present, + "trajectory": trajectory_has_calls, + "server_stdout": (LOGS_DIR / "kernel-mcp/server.stdout.log").is_file(), + "server_stderr": (LOGS_DIR / "kernel-mcp/server.stderr.log").is_file(), + } + reward = 1.0 if all(checks.values()) else 0.0 + tool_calls = [ + { + "name": call["tool_name"], + "success": call["success"], + "duration_ms": call["duration_ms"], + "http_status": call["http_status"], + } + for call in (context_call, browsers_call) + if call is not None + ] + result = { + "reward": reward, + "checks": checks, + "tool_calls": tool_calls, + "agent_report": report, + "trajectory": { + "present": trajectory is not None, + "schema_version": (trajectory or {}).get("schema_version"), + "agent": (trajectory or {}).get("agent"), + "tool_names": trajectory_names, + }, + "run_manifest": manifest, + } + + (VERIFIER_DIR / "reward.txt").write_text(str(reward)) + (VERIFIER_DIR / "reward.json").write_text( + json.dumps( + { + "reward": reward, + "get_connection_context": float(context_call is not None), + "manage_browsers_list": float(browsers_call is not None), + "agent_report": float(report_matches), + "source_sha": float(source_sha_matches), + "hypeman_identity": float(hypeman_identity_present), + "trajectory": float(trajectory_has_calls), + }, + indent=2, + ) + ) + (VERIFIER_DIR / "smoke-result.json").write_text(json.dumps(result, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/harbor/build-image.sh b/benchmarks/harbor/build-image.sh new file mode 100755 index 0000000..30798b8 --- /dev/null +++ b/benchmarks/harbor/build-image.sh @@ -0,0 +1,65 @@ +#!/bin/bash +set -euo pipefail + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +cd "$repo_root" + +source_sha=$(git rev-parse HEAD) +source_sha_file=benchmarks/harbor/image/source-sha +build_log=$(mktemp) +trap 'rm -f "$source_sha_file" "$build_log"' EXIT + +printf '%s\n' "$source_sha" >"$source_sha_file" + +set +e +hypeman build \ + --file benchmarks/harbor/image/Dockerfile \ + --cpus 4 \ + --memory 8GB \ + --timeout 30m \ + . 2>&1 | tee "$build_log" +build_status=${PIPESTATUS[0]} +set -e + +build_id=$(sed -n 's/^Build ID: //p' "$build_log" | tail -1) +if [[ -z "$build_id" ]]; then + echo "Hypeman did not return a build ID" >&2 + exit 1 +fi + +image_ref="builds/$build_id" +if ((build_status != 0)); then + echo "Build record failed; checking for a delayed ready image for up to 60 seconds" >&2 + image_ready=false + for _ in $(seq 1 12); do + if hypeman --format json image list | python3 -c ' +import json +import sys + +image_ref = sys.argv[1] +expected = {image_ref, f"docker.io/{image_ref}:latest"} +images = json.load(sys.stdin) +raise SystemExit( + 0 + if any(image.get("name") in expected and image.get("status") == "ready" for image in images) + else 1 +) +' "$image_ref" + then + image_ready=true + break + fi + sleep 5 + done + if [[ "$image_ready" != true ]]; then + exit "$build_status" + fi +fi + +cat >benchmarks/harbor/.image.env < int: + parser = argparse.ArgumentParser() + parser.add_argument("output", type=Path) + args = parser.parse_args() + + source = Path(__file__).parent / "smoke" + output = args.output.resolve() + image = os.environ["KERNEL_MCP_BENCHMARK_IMAGE"] + source_sha = os.environ["KERNEL_MCP_SOURCE_SHA"] + + if output.exists(): + shutil.rmtree(output) + shutil.copytree(source, output) + + config_path = output / "task.toml" + config = config_path.read_text() + config = config.replace("${KERNEL_MCP_BENCHMARK_IMAGE}", image) + config = config.replace("${KERNEL_MCP_SOURCE_SHA}", source_sha) + config_path.write_text(config) + + wrapper = Path(__file__).parent / "bin" / "kernel-mcp-local" + runtime_wrapper = output / "steps" / "run" / "workdir" / "kernel-mcp-local" + shutil.copy2(wrapper, runtime_wrapper) + runtime_wrapper.chmod(0o755) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/harbor/run-smoke.sh b/benchmarks/harbor/run-smoke.sh new file mode 100755 index 0000000..bfafd5a --- /dev/null +++ b/benchmarks/harbor/run-smoke.sh @@ -0,0 +1,84 @@ +#!/bin/bash +set -euo pipefail + +usage() { + echo "usage: $0 [job-name] [jobs-dir]" >&2 + exit 2 +} + +agent=${1:-} +[[ "$agent" == "claude-code" || "$agent" == "codex" ]] || usage + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +benchmark_dir="$repo_root/benchmarks/harbor" +image_env="$benchmark_dir/.image.env" +[[ -f "$image_env" ]] || { + echo "Missing $image_env; run benchmarks/harbor/build-image.sh first" >&2 + exit 1 +} + +set -a +source "$image_env" +set +a + +: "${KERNEL_MCP_BENCHMARK_API_KEY:?KERNEL_MCP_BENCHMARK_API_KEY is required}" +: "${KERNEL_MCP_BENCHMARK_PROJECT_ID:?KERNEL_MCP_BENCHMARK_PROJECT_ID is required}" + +case "$agent" in + claude-code) + : "${ANTHROPIC_API_KEY:?ANTHROPIC_API_KEY is required}" + model=${CLAUDE_BENCHMARK_MODEL:-claude-sonnet-4-5-20250929} + version=2.1.110 + ;; + codex) + : "${OPENAI_API_KEY:?OPENAI_API_KEY is required}" + model=${CODEX_BENCHMARK_MODEL:-gpt-5.3-codex} + version=0.120.0 + ;; +esac + +if [[ -n "${HARBOR_BIN:-}" ]]; then + harbor_bin=$HARBOR_BIN +elif command -v harbor >/dev/null 2>&1; then + harbor_bin=$(command -v harbor) +elif [[ -x "$repo_root/../harbor-hypeman/.venv/bin/harbor" ]]; then + harbor_bin="$repo_root/../harbor-hypeman/.venv/bin/harbor" +else + echo "Harbor CLI not found; set HARBOR_BIN" >&2 + exit 1 +fi + +job_name=${2:-${agent}-smoke-$(date -u +%Y%m%dT%H%M%SZ)} +jobs_dir=${3:-${HARBOR_JOBS_DIR:-/tmp/kernel-mcp-harbor-jobs}} +runtime_task=$(mktemp -d) +runtime_env=$(mktemp) +trap 'rm -rf "$runtime_task"; rm -f "$runtime_env"' EXIT + +export KERNEL_MCP_BENCHMARK_IMAGE KERNEL_MCP_SOURCE_SHA +python3 "$benchmark_dir/prepare-task.py" "$runtime_task" + +cat >"$runtime_env" <"$key_dir/api-key" +chmod 0600 "$key_dir/api-key" + +/usr/local/bin/start-kernel-mcp-server +printf 'ready\n' >/logs/kernel-mcp/ready +rm -f /app/setup.sh diff --git a/benchmarks/harbor/smoke/task.toml b/benchmarks/harbor/smoke/task.toml new file mode 100644 index 0000000..d3f95cf --- /dev/null +++ b/benchmarks/harbor/smoke/task.toml @@ -0,0 +1,51 @@ +schema_version = "1.4" +source = "kernel-mcp-benchmarks" +artifacts = ["/logs/kernel-mcp"] +multi_step_reward_strategy = "final" + +[task] +name = "kernel-mcp/local-connection-smoke" +description = "Verify an agent can call a locally running Kernel MCP server" +keywords = ["kernel", "mcp", "harbor", "smoke"] + +[metadata] +benchmark = "kernel-mcp-local-connection" +kernel_mcp_server_sha = "${KERNEL_MCP_SOURCE_SHA}" + +[environment] +docker_image = "${KERNEL_MCP_BENCHMARK_IMAGE}" +network_mode = "public" +workdir = "/app" +build_timeout_sec = 1200.0 +cpus = 2 +memory_mb = 4096 +storage_mb = 8192 + +[environment.env] +KERNEL_API_KEY = "${KERNEL_MCP_BENCHMARK_API_KEY}" +KERNEL_MCP_BENCHMARK_IMAGE = "${KERNEL_MCP_BENCHMARK_IMAGE}" +KERNEL_MCP_SOURCE_SHA = "${KERNEL_MCP_SOURCE_SHA}" +KERNEL_MCP_EXPECTED_PROJECT_ID = "${KERNEL_MCP_BENCHMARK_PROJECT_ID}" +API_BASE_URL = "${KERNEL_API_BASE_URL:-https://api.onkernel.com}" +REDIS_URL = "redis://127.0.0.1:6379" +CLERK_SECRET_KEY = "sk_test_kernel_mcp_benchmark_local_only" +NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY = "pk_test_YmVuY2htYXJrLmNsZXJrLmFjY291bnRzLmRldiQ" +MANAGED_AUTH_APP_ORIGIN = "http://127.0.0.1:3002" +NEXT_TELEMETRY_DISABLED = "1" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 300.0 + +[steps.verifier] +timeout_sec = 60.0 + +[steps.healthcheck] +command = "test -s /logs/kernel-mcp/ready && curl -sS -o /dev/null http://127.0.0.1:3002/mcp && curl -sS -o /dev/null http://127.0.0.1:3003/mcp" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 1.0 +start_interval_sec = 1.0 +retries = 10 From 704d0169740649a90450545d130c9c85881e9cc2 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Fri, 21 Aug 2026 19:07:52 +0000 Subject: [PATCH 02/29] Update benchmark model defaults --- benchmarks/harbor/README.md | 8 ++++---- benchmarks/harbor/run-smoke.sh | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/benchmarks/harbor/README.md b/benchmarks/harbor/README.md index dcb0ae8..2462e92 100644 --- a/benchmarks/harbor/README.md +++ b/benchmarks/harbor/README.md @@ -30,10 +30,10 @@ export KERNEL_MCP_BENCHMARK_PROJECT_ID=project_id Defaults: -| Agent | Version | Model | -| ----------- | ------: | ---------------------------- | -| Claude Code | 2.1.110 | `claude-sonnet-4-5-20250929` | -| Codex | 0.120.0 | `gpt-5.3-codex` | +| Agent | Version | Model | +| ----------- | ------: | ----------------- | +| Claude Code | 2.1.110 | `claude-sonnet-5` | +| Codex | 0.120.0 | `gpt-5.6-terra` | Override models with `CLAUDE_BENCHMARK_MODEL` or `CODEX_BENCHMARK_MODEL`. Runs have a 10-minute wall-clock limit; change it with `HARBOR_BENCHMARK_TIMEOUT`. diff --git a/benchmarks/harbor/run-smoke.sh b/benchmarks/harbor/run-smoke.sh index bfafd5a..47b420d 100755 --- a/benchmarks/harbor/run-smoke.sh +++ b/benchmarks/harbor/run-smoke.sh @@ -27,12 +27,12 @@ set +a case "$agent" in claude-code) : "${ANTHROPIC_API_KEY:?ANTHROPIC_API_KEY is required}" - model=${CLAUDE_BENCHMARK_MODEL:-claude-sonnet-4-5-20250929} + model=${CLAUDE_BENCHMARK_MODEL:-claude-sonnet-5} version=2.1.110 ;; codex) : "${OPENAI_API_KEY:?OPENAI_API_KEY is required}" - model=${CODEX_BENCHMARK_MODEL:-gpt-5.3-codex} + model=${CODEX_BENCHMARK_MODEL:-gpt-5.6-terra} version=0.120.0 ;; esac From fecf5bfc4edfc1dec3a4ed699a90d2f8763fd805 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Fri, 21 Aug 2026 19:35:07 +0000 Subject: [PATCH 03/29] Fix Claude Sonnet Harbor smoke defaults --- benchmarks/harbor/README.md | 8 ++++---- benchmarks/harbor/build-image.sh | 16 ++++++++-------- benchmarks/harbor/prepare-task.py | 2 ++ benchmarks/harbor/run-smoke.sh | 17 +++++++++++++++-- benchmarks/harbor/smoke/task.toml | 1 + 5 files changed, 30 insertions(+), 14 deletions(-) diff --git a/benchmarks/harbor/README.md b/benchmarks/harbor/README.md index 2462e92..6a06329 100644 --- a/benchmarks/harbor/README.md +++ b/benchmarks/harbor/README.md @@ -9,7 +9,7 @@ This directory runs stock Harbor agents against a locally built `kernel-mcp-serv - Hypeman CLI and credentials - `KERNEL_MCP_BENCHMARK_API_KEY` scoped to an isolated evaluation project - `KERNEL_MCP_BENCHMARK_PROJECT_ID` -- `ANTHROPIC_API_KEY` for Claude Code +- `ANTHROPIC_API_KEY` or `CLAUDE_CODE_OAUTH_TOKEN` for Claude Code - `OPENAI_API_KEY` for Codex ## Build the image @@ -18,7 +18,7 @@ This directory runs stock Harbor agents against a locally built `kernel-mcp-serv ./benchmarks/harbor/build-image.sh ``` -The build uses the current Git SHA, installs dependencies with Bun, runs the production Next.js build, and writes the resulting image reference to the ignored `.image.env` file. Hypeman can report a failed build before the converted image becomes visible; the script performs a bounded 60-second ready-image check for that case. +The build uses the current Git SHA, installs dependencies with Bun, runs the production Next.js build, and writes the resulting image reference to the ignored `.image.env` file. Hypeman can report a failed build before the converted image becomes visible; the script performs a bounded 5-minute ready-image check for that case. ## Run the smoke task @@ -32,10 +32,10 @@ Defaults: | Agent | Version | Model | | ----------- | ------: | ----------------- | -| Claude Code | 2.1.110 | `claude-sonnet-5` | +| Claude Code | 2.1.238 | `claude-sonnet-5` | | Codex | 0.120.0 | `gpt-5.6-terra` | -Override models with `CLAUDE_BENCHMARK_MODEL` or `CODEX_BENCHMARK_MODEL`. Runs have a 10-minute wall-clock limit; change it with `HARBOR_BENCHMARK_TIMEOUT`. +Override models with `CLAUDE_BENCHMARK_MODEL` or `CODEX_BENCHMARK_MODEL`, or test a specific Claude Code release with `CLAUDE_BENCHMARK_VERSION`. Runs have a 10-minute wall-clock limit; change it with `HARBOR_BENCHMARK_TIMEOUT`. The output defaults to `/tmp/kernel-mcp-harbor-jobs/`. Each successful trial contains: diff --git a/benchmarks/harbor/build-image.sh b/benchmarks/harbor/build-image.sh index 30798b8..d8319af 100755 --- a/benchmarks/harbor/build-image.sh +++ b/benchmarks/harbor/build-image.sh @@ -15,29 +15,29 @@ set +e hypeman build \ --file benchmarks/harbor/image/Dockerfile \ --cpus 4 \ - --memory 8GB \ - --timeout 30m \ + --memory 8192 \ + --timeout 1800 \ . 2>&1 | tee "$build_log" build_status=${PIPESTATUS[0]} set -e -build_id=$(sed -n 's/^Build ID: //p' "$build_log" | tail -1) +build_id=$(sed -n -E 's/^Build (ID|started): //p' "$build_log" | tail -1) if [[ -z "$build_id" ]]; then echo "Hypeman did not return a build ID" >&2 exit 1 fi -image_ref="builds/$build_id" +image_ref="docker.io/builds/$build_id:latest" if ((build_status != 0)); then - echo "Build record failed; checking for a delayed ready image for up to 60 seconds" >&2 + echo "Build record failed; checking for a delayed ready image for up to 5 minutes" >&2 image_ready=false - for _ in $(seq 1 12); do + for _ in $(seq 1 30); do if hypeman --format json image list | python3 -c ' import json import sys image_ref = sys.argv[1] -expected = {image_ref, f"docker.io/{image_ref}:latest"} +expected = {image_ref, image_ref.removeprefix("docker.io/")} images = json.load(sys.stdin) raise SystemExit( 0 @@ -49,7 +49,7 @@ raise SystemExit( image_ready=true break fi - sleep 5 + sleep 10 done if [[ "$image_ready" != true ]]; then exit "$build_status" diff --git a/benchmarks/harbor/prepare-task.py b/benchmarks/harbor/prepare-task.py index 0b031a1..6ad8107 100755 --- a/benchmarks/harbor/prepare-task.py +++ b/benchmarks/harbor/prepare-task.py @@ -16,6 +16,7 @@ def main() -> int: output = args.output.resolve() image = os.environ["KERNEL_MCP_BENCHMARK_IMAGE"] source_sha = os.environ["KERNEL_MCP_SOURCE_SHA"] + project_id = os.environ["KERNEL_MCP_BENCHMARK_PROJECT_ID"] if output.exists(): shutil.rmtree(output) @@ -25,6 +26,7 @@ def main() -> int: config = config_path.read_text() config = config.replace("${KERNEL_MCP_BENCHMARK_IMAGE}", image) config = config.replace("${KERNEL_MCP_SOURCE_SHA}", source_sha) + config = config.replace("${KERNEL_MCP_BENCHMARK_PROJECT_ID}", project_id) config_path.write_text(config) wrapper = Path(__file__).parent / "bin" / "kernel-mcp-local" diff --git a/benchmarks/harbor/run-smoke.sh b/benchmarks/harbor/run-smoke.sh index 47b420d..d4c447c 100755 --- a/benchmarks/harbor/run-smoke.sh +++ b/benchmarks/harbor/run-smoke.sh @@ -26,9 +26,17 @@ set +a case "$agent" in claude-code) - : "${ANTHROPIC_API_KEY:?ANTHROPIC_API_KEY is required}" + if [[ -z "${ANTHROPIC_API_KEY:-}" && -z "${ANTHROPIC_AUTH_TOKEN:-}" && -z "${CLAUDE_CODE_OAUTH_TOKEN:-}" ]]; then + echo "ANTHROPIC_API_KEY, ANTHROPIC_AUTH_TOKEN, or CLAUDE_CODE_OAUTH_TOKEN is required" >&2 + exit 1 + fi + if [[ -z "${ANTHROPIC_API_KEY:-}" && -z "${ANTHROPIC_AUTH_TOKEN:-}" && -n "${CLAUDE_CODE_OAUTH_TOKEN:-}" ]]; then + ANTHROPIC_AUTH_TOKEN=$CLAUDE_CODE_OAUTH_TOKEN + CLAUDE_FORCE_OAUTH=1 + export ANTHROPIC_AUTH_TOKEN CLAUDE_FORCE_OAUTH + fi model=${CLAUDE_BENCHMARK_MODEL:-claude-sonnet-5} - version=2.1.110 + version=${CLAUDE_BENCHMARK_VERSION:-2.1.238} ;; codex) : "${OPENAI_API_KEY:?OPENAI_API_KEY is required}" @@ -63,6 +71,11 @@ KERNEL_MCP_SOURCE_SHA=$KERNEL_MCP_SOURCE_SHA KERNEL_MCP_BENCHMARK_API_KEY=$KERNEL_MCP_BENCHMARK_API_KEY KERNEL_MCP_BENCHMARK_PROJECT_ID=$KERNEL_MCP_BENCHMARK_PROJECT_ID KERNEL_API_BASE_URL=${KERNEL_API_BASE_URL:-https://api.onkernel.com} +KERNEL_PROJECT=${KERNEL_PROJECT:-} +ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY:-} +ANTHROPIC_AUTH_TOKEN=${ANTHROPIC_AUTH_TOKEN:-} +CLAUDE_CODE_OAUTH_TOKEN=${CLAUDE_CODE_OAUTH_TOKEN:-} +CLAUDE_FORCE_OAUTH=${CLAUDE_FORCE_OAUTH:-} EOF chmod 0600 "$runtime_env" diff --git a/benchmarks/harbor/smoke/task.toml b/benchmarks/harbor/smoke/task.toml index d3f95cf..d2b12cb 100644 --- a/benchmarks/harbor/smoke/task.toml +++ b/benchmarks/harbor/smoke/task.toml @@ -26,6 +26,7 @@ KERNEL_API_KEY = "${KERNEL_MCP_BENCHMARK_API_KEY}" KERNEL_MCP_BENCHMARK_IMAGE = "${KERNEL_MCP_BENCHMARK_IMAGE}" KERNEL_MCP_SOURCE_SHA = "${KERNEL_MCP_SOURCE_SHA}" KERNEL_MCP_EXPECTED_PROJECT_ID = "${KERNEL_MCP_BENCHMARK_PROJECT_ID}" +KERNEL_PROJECT = "${KERNEL_MCP_BENCHMARK_PROJECT_ID}" API_BASE_URL = "${KERNEL_API_BASE_URL:-https://api.onkernel.com}" REDIS_URL = "redis://127.0.0.1:6379" CLERK_SECRET_KEY = "sk_test_kernel_mcp_benchmark_local_only" From aceecb9552d7d48a95f18119d2c7e584f33f5079 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Fri, 21 Aug 2026 19:44:01 +0000 Subject: [PATCH 04/29] Verify Harbor MCP calls from ATIF --- benchmarks/harbor/README.md | 3 +- .../trajectory-error-observation.json | 37 +++ .../trajectory-missing-observation.json | 33 +++ .../bin/fixtures/trajectory-positive.json | 37 +++ benchmarks/harbor/bin/mcp-telemetry-proxy.mjs | 112 --------- benchmarks/harbor/bin/start-kernel-mcp-server | 7 +- benchmarks/harbor/bin/test_verify_smoke.py | 49 ++++ benchmarks/harbor/bin/verify-smoke.py | 214 ++++++++++++------ benchmarks/harbor/image/Dockerfile | 1 - benchmarks/harbor/smoke/task.toml | 2 +- 10 files changed, 300 insertions(+), 195 deletions(-) create mode 100644 benchmarks/harbor/bin/fixtures/trajectory-error-observation.json create mode 100644 benchmarks/harbor/bin/fixtures/trajectory-missing-observation.json create mode 100644 benchmarks/harbor/bin/fixtures/trajectory-positive.json delete mode 100644 benchmarks/harbor/bin/mcp-telemetry-proxy.mjs create mode 100644 benchmarks/harbor/bin/test_verify_smoke.py diff --git a/benchmarks/harbor/README.md b/benchmarks/harbor/README.md index 6a06329..bc08cb8 100644 --- a/benchmarks/harbor/README.md +++ b/benchmarks/harbor/README.md @@ -41,9 +41,8 @@ The output defaults to `/tmp/kernel-mcp-harbor-jobs/`. Each successful - `steps/run/agent/trajectory.json` in ATIF format - native agent logs and session data -- `steps/run/artifacts/logs/kernel-mcp/requests.jsonl` with MCP latency and status - server stdout and stderr - source SHA and Hypeman identity in `run-manifest.json` - numeric Harbor rewards plus detailed `smoke-result.json` -The verifier requires native trajectory calls to `get_connection_context` and `manage_browsers`; direct HTTP or custom MCP-client workarounds do not pass. +The verifier proves local-server use from Harbor's ATIF trajectory: it requires native `mcp__kernel__get_connection_context` and `mcp__kernel__manage_browsers` calls, paired non-error observations, the expected project scope, and the required read-only browser-list arguments. Direct HTTP or custom MCP-client workarounds do not pass. diff --git a/benchmarks/harbor/bin/fixtures/trajectory-error-observation.json b/benchmarks/harbor/bin/fixtures/trajectory-error-observation.json new file mode 100644 index 0000000..99f41ef --- /dev/null +++ b/benchmarks/harbor/bin/fixtures/trajectory-error-observation.json @@ -0,0 +1,37 @@ +{ + "schema_version": "ATIF-v1.7", + "steps": [ + { + "step_id": 1, + "source": "agent", + "tool_calls": [ + { + "tool_call_id": "ctx-1", + "function_name": "mcp__kernel__get_connection_context", + "arguments": {} + }, + { + "tool_call_id": "browsers-1", + "function_name": "mcp__kernel__manage_browsers", + "arguments": { + "action": "list", + "status": "active", + "limit": 1 + } + } + ], + "observation": { + "results": [ + { + "source_call_id": "ctx-1", + "content": "{\"connection_scope\":{\"kind\":\"project\",\"project_id\":\"project-123\"}}" + }, + { + "source_call_id": "browsers-1", + "content": "{\"isError\":true,\"content\":[{\"type\":\"text\",\"text\":\"request failed\"}]}" + } + ] + } + } + ] +} diff --git a/benchmarks/harbor/bin/fixtures/trajectory-missing-observation.json b/benchmarks/harbor/bin/fixtures/trajectory-missing-observation.json new file mode 100644 index 0000000..c506f2e --- /dev/null +++ b/benchmarks/harbor/bin/fixtures/trajectory-missing-observation.json @@ -0,0 +1,33 @@ +{ + "schema_version": "ATIF-v1.7", + "steps": [ + { + "step_id": 1, + "source": "agent", + "tool_calls": [ + { + "tool_call_id": "ctx-1", + "function_name": "mcp__kernel__get_connection_context", + "arguments": {} + }, + { + "tool_call_id": "browsers-1", + "function_name": "mcp__kernel__manage_browsers", + "arguments": { + "action": "list", + "status": "active", + "limit": 1 + } + } + ], + "observation": { + "results": [ + { + "source_call_id": "ctx-1", + "content": "{\"connection_scope\":{\"kind\":\"project\",\"project_id\":\"project-123\"}}" + } + ] + } + } + ] +} diff --git a/benchmarks/harbor/bin/fixtures/trajectory-positive.json b/benchmarks/harbor/bin/fixtures/trajectory-positive.json new file mode 100644 index 0000000..a00aa56 --- /dev/null +++ b/benchmarks/harbor/bin/fixtures/trajectory-positive.json @@ -0,0 +1,37 @@ +{ + "schema_version": "ATIF-v1.7", + "steps": [ + { + "step_id": 1, + "source": "agent", + "tool_calls": [ + { + "tool_call_id": "ctx-1", + "function_name": "mcp__kernel__get_connection_context", + "arguments": {} + }, + { + "tool_call_id": "browsers-1", + "function_name": "mcp__kernel__manage_browsers", + "arguments": { + "action": "list", + "status": "active", + "limit": 1 + } + } + ], + "observation": { + "results": [ + { + "source_call_id": "ctx-1", + "content": "{\"type\":\"text\",\"text\":\"{\\\"connection_scope\\\":{\\\"kind\\\":\\\"project\\\",\\\"project_id\\\":\\\"project-123\\\"}}\"}" + }, + { + "source_call_id": "browsers-1", + "content": "{\"type\":\"text\",\"text\":\"{\\\"items\\\":[],\\\"has_more\\\":false}\"}" + } + ] + } + } + ] +} diff --git a/benchmarks/harbor/bin/mcp-telemetry-proxy.mjs b/benchmarks/harbor/bin/mcp-telemetry-proxy.mjs deleted file mode 100644 index ed1f029..0000000 --- a/benchmarks/harbor/bin/mcp-telemetry-proxy.mjs +++ /dev/null @@ -1,112 +0,0 @@ -import fs from "node:fs"; -import http from "node:http"; - -const listenPort = Number(process.env.KERNEL_MCP_PROXY_PORT || 3002); -const upstreamPort = Number(process.env.KERNEL_MCP_SERVER_PORT || 3003); -const logPath = - process.env.KERNEL_MCP_REQUEST_LOG || "/logs/kernel-mcp/requests.jsonl"; - -function requestMetadata(body) { - try { - const payload = JSON.parse(body); - return { - jsonrpc_method: payload.method ?? null, - tool_name: - payload.method === "tools/call" ? (payload.params?.name ?? null) : null, - request_id: payload.id ?? null, - }; - } catch { - return { jsonrpc_method: null, tool_name: null, request_id: null }; - } -} - -function responseSucceeded(statusCode, body) { - if (statusCode < 200 || statusCode >= 300) return false; - const candidates = body - .split("\n") - .filter((line) => line.startsWith("data:")) - .map((line) => line.slice(5).trim()); - if (candidates.length === 0) candidates.push(body.trim()); - - for (const candidate of candidates) { - if (!candidate || candidate === "[DONE]") continue; - try { - const payload = JSON.parse(candidate); - if (payload.error || payload.result?.isError === true) return false; - } catch { - // Non-JSON response bodies are successful when the HTTP status succeeded. - } - } - return true; -} - -function appendLog(entry) { - fs.appendFileSync(logPath, `${JSON.stringify(entry)}\n`, { mode: 0o600 }); -} - -const server = http.createServer((clientRequest, clientResponse) => { - const startedAt = new Date(); - const requestChunks = []; - - clientRequest.on("data", (chunk) => requestChunks.push(chunk)); - clientRequest.on("end", () => { - const requestBody = Buffer.concat(requestChunks); - const metadata = requestMetadata(requestBody.toString("utf8")); - const headers = { ...clientRequest.headers }; - headers.host = `127.0.0.1:${upstreamPort}`; - headers["content-length"] = String(requestBody.length); - - const upstreamRequest = http.request( - { - host: "127.0.0.1", - port: upstreamPort, - method: clientRequest.method, - path: clientRequest.url, - headers, - }, - (upstreamResponse) => { - const responseChunks = []; - upstreamResponse.on("data", (chunk) => responseChunks.push(chunk)); - upstreamResponse.on("end", () => { - const responseBody = Buffer.concat(responseChunks); - const statusCode = upstreamResponse.statusCode ?? 502; - clientResponse.writeHead(statusCode, upstreamResponse.headers); - clientResponse.end(responseBody); - - appendLog({ - started_at: startedAt.toISOString(), - duration_ms: Date.now() - startedAt.getTime(), - http_method: clientRequest.method, - path: clientRequest.url, - http_status: statusCode, - success: responseSucceeded( - statusCode, - responseBody.toString("utf8"), - ), - ...metadata, - }); - }); - }, - ); - - upstreamRequest.on("error", (error) => { - if (!clientResponse.headersSent) clientResponse.writeHead(502); - clientResponse.end("Bad Gateway"); - appendLog({ - started_at: startedAt.toISOString(), - duration_ms: Date.now() - startedAt.getTime(), - http_method: clientRequest.method, - path: clientRequest.url, - http_status: 502, - success: false, - error: error.message, - ...metadata, - }); - }); - upstreamRequest.end(requestBody); - }); -}); - -server.listen(listenPort, "127.0.0.1", () => { - console.log(`MCP telemetry proxy listening on 127.0.0.1:${listenPort}`); -}); diff --git a/benchmarks/harbor/bin/start-kernel-mcp-server b/benchmarks/harbor/bin/start-kernel-mcp-server index fe299ae..08e7601 100755 --- a/benchmarks/harbor/bin/start-kernel-mcp-server +++ b/benchmarks/harbor/bin/start-kernel-mcp-server @@ -15,16 +15,11 @@ redis-server --daemonize yes --bind 127.0.0.1 --port 6379 \ --logfile "$log_dir/redis.log" --dir /tmp cd /opt/kernel-mcp-server -nohup ./node_modules/.bin/next start -p 3003 \ +nohup ./node_modules/.bin/next start -p 3002 \ >"$log_dir/server.stdout.log" \ 2>"$log_dir/server.stderr.log" & echo $! >"$log_dir/server.pid" -nohup node /usr/local/lib/mcp-telemetry-proxy.mjs \ - >"$log_dir/proxy.stdout.log" \ - 2>"$log_dir/proxy.stderr.log" & -echo $! >"$log_dir/proxy.pid" - for _ in $(seq 1 90); do if curl -fsS -X POST http://127.0.0.1:3002/mcp \ -H "Authorization: Bearer ${KERNEL_API_KEY}" \ diff --git a/benchmarks/harbor/bin/test_verify_smoke.py b/benchmarks/harbor/bin/test_verify_smoke.py new file mode 100644 index 0000000..18d53be --- /dev/null +++ b/benchmarks/harbor/bin/test_verify_smoke.py @@ -0,0 +1,49 @@ +import importlib.util +import json +from pathlib import Path +import unittest + + +MODULE_PATH = Path(__file__).with_name("verify-smoke.py") +SPEC = importlib.util.spec_from_file_location("verify_smoke", MODULE_PATH) +assert SPEC and SPEC.loader +verify_smoke = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(verify_smoke) + +FIXTURES = Path(__file__).with_name("fixtures") + + +def load_fixture(name: str) -> dict: + return json.loads((FIXTURES / name).read_text()) + + +class VerifySmokeTest(unittest.TestCase): + def test_accepts_native_calls_with_paired_observations(self) -> None: + proof = verify_smoke.validate_trajectory( + load_fixture("trajectory-positive.json"), "project-123" + ) + + self.assertTrue(proof["native_calls_present"]) + self.assertTrue(proof["observations_valid"]) + self.assertTrue(proof["context_scope_valid"]) + self.assertTrue(proof["manage_browsers_arguments_valid"]) + + def test_rejects_missing_observation(self) -> None: + proof = verify_smoke.validate_trajectory( + load_fixture("trajectory-missing-observation.json"), "project-123" + ) + + self.assertFalse(proof["observations_valid"]) + self.assertEqual(proof["missing_observations"], ["browsers-1"]) + + def test_rejects_error_observation(self) -> None: + proof = verify_smoke.validate_trajectory( + load_fixture("trajectory-error-observation.json"), "project-123" + ) + + self.assertFalse(proof["observations_valid"]) + self.assertEqual(proof["error_observations"], ["browsers-1"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/benchmarks/harbor/bin/verify-smoke.py b/benchmarks/harbor/bin/verify-smoke.py index cacc600..5b0de04 100755 --- a/benchmarks/harbor/bin/verify-smoke.py +++ b/benchmarks/harbor/bin/verify-smoke.py @@ -8,6 +8,10 @@ LOGS_DIR = Path(os.environ.get("HARBOR_LOGS_DIR", "/logs")) VERIFIER_DIR = LOGS_DIR / "verifier" +REQUIRED_TOOLS = { + "mcp__kernel__get_connection_context", + "mcp__kernel__manage_browsers", +} def read_json(path: Path) -> dict[str, Any] | None: @@ -18,107 +22,179 @@ def read_json(path: Path) -> dict[str, Any] | None: return value if isinstance(value, dict) else None -def read_requests(path: Path) -> list[dict[str, Any]]: - requests = [] - try: - lines = path.read_text().splitlines() - except OSError: - return requests - for line in lines: - try: - value = json.loads(line) - except json.JSONDecodeError: +def _decode_tool_result_content(content: Any) -> Any: + value = content + for _ in range(4): + if isinstance(value, str): + try: + value = json.loads(value) + except json.JSONDecodeError: + return value + continue + if isinstance(value, dict) and value.get("type") == "text": + value = value.get("text") + continue + return value + return value + + +def _contains_error(value: Any) -> bool: + if isinstance(value, dict): + if value.get("is_error") is True or value.get("isError") is True: + return True + if "error" in value and value["error"] not in (None, False, ""): + return True + return any(_contains_error(item) for item in value.values()) + if isinstance(value, list): + return any(_contains_error(item) for item in value) + if isinstance(value, str): + text = value.lstrip().lower() + return text.startswith(("[error]", "error:", "error in ")) + return False + + +def _observation_result_map(trajectory: dict[str, Any]) -> dict[str, list[dict[str, Any]]]: + results: dict[str, list[dict[str, Any]]] = {} + for step in trajectory.get("steps") or []: + if not isinstance(step, dict): + continue + observation = step.get("observation") + if not isinstance(observation, dict): continue - if isinstance(value, dict): - requests.append(value) - return requests - - -def successful_call(requests: list[dict[str, Any]], name: str) -> dict[str, Any] | None: - return next( - ( - request - for request in requests - if request.get("jsonrpc_method") == "tools/call" - and request.get("tool_name") == name - and request.get("success") is True - ), - None, + for result in observation.get("results") or []: + if not isinstance(result, dict): + continue + source_call_id = result.get("source_call_id") + if isinstance(source_call_id, str): + results.setdefault(source_call_id, []).append(result) + return results + + +def _native_tool_calls(trajectory: dict[str, Any]) -> list[dict[str, Any]]: + calls = [] + for step in trajectory.get("steps") or []: + if not isinstance(step, dict): + continue + for call in step.get("tool_calls") or []: + if not isinstance(call, dict): + continue + if call.get("function_name") in REQUIRED_TOOLS: + calls.append(call) + return calls + + +def _manage_browsers_arguments_valid(call: dict[str, Any]) -> bool: + arguments = call.get("arguments") + return ( + isinstance(arguments, dict) + and arguments.get("action") == "list" + and arguments.get("status") == "active" + and arguments.get("limit") == 1 ) -def trajectory_tool_names(trajectory: dict[str, Any] | None) -> list[str]: - names = [] - for step in (trajectory or {}).get("steps") or []: - if not isinstance(step, dict): +def validate_trajectory( + trajectory: dict[str, Any] | None, expected_project_id: str +) -> dict[str, Any]: + calls = _native_tool_calls(trajectory or {}) + calls_by_name = { + name: [call for call in calls if call.get("function_name") == name] + for name in REQUIRED_TOOLS + } + result_map = _observation_result_map(trajectory or {}) + missing_observations = [] + duplicate_observations = [] + error_observations = [] + context_scope_valid = True + browser_arguments_valid = True + + for call in calls: + call_id = call.get("tool_call_id") + results = result_map.get(call_id, []) if isinstance(call_id, str) else [] + if len(results) == 0: + missing_observations.append(call_id) continue - for call in step.get("tool_calls") or []: - if isinstance(call, dict) and isinstance(call.get("function_name"), str): - names.append(call["function_name"]) - return names + if len(results) != 1: + duplicate_observations.append(call_id) + continue + result = results[0] + decoded = _decode_tool_result_content(result.get("content")) + if _contains_error(decoded) or _contains_error(result.get("extra")): + error_observations.append(call_id) + continue + if call.get("function_name") == "mcp__kernel__get_connection_context": + scope = decoded.get("connection_scope") if isinstance(decoded, dict) else None + context_scope_valid = context_scope_valid and ( + isinstance(scope, dict) + and scope.get("kind") == "project" + and scope.get("project_id") == expected_project_id + ) + elif call.get("function_name") == "mcp__kernel__manage_browsers": + browser_arguments_valid = ( + browser_arguments_valid and _manage_browsers_arguments_valid(call) + ) + + native_calls_present = all(calls_by_name[name] for name in REQUIRED_TOOLS) + observations_valid = bool(calls) and not ( + missing_observations or duplicate_observations or error_observations + ) + return { + "native_calls_present": native_calls_present, + "observations_valid": observations_valid, + "context_scope_valid": context_scope_valid + and bool(calls_by_name["mcp__kernel__get_connection_context"]), + "manage_browsers_arguments_valid": browser_arguments_valid + and bool(calls_by_name["mcp__kernel__manage_browsers"]), + "missing_observations": missing_observations, + "duplicate_observations": duplicate_observations, + "error_observations": error_observations, + "tool_calls": [ + { + "tool_call_id": call.get("tool_call_id"), + "name": call.get("function_name"), + "arguments": call.get("arguments"), + } + for call in calls + ], + } def main() -> int: VERIFIER_DIR.mkdir(parents=True, exist_ok=True) - requests = read_requests(LOGS_DIR / "kernel-mcp/requests.jsonl") report = read_json(LOGS_DIR / "artifacts/agent-report.json") trajectory = read_json(LOGS_DIR / "agent/trajectory.json") manifest = read_json(LOGS_DIR / "kernel-mcp/run-manifest.json") - - context_call = successful_call(requests, "get_connection_context") - browsers_call = successful_call(requests, "manage_browsers") expected_project_id = os.environ.get("KERNEL_MCP_EXPECTED_PROJECT_ID", "") - report_matches = bool( - report - and report.get("get_connection_context_succeeded") is True - and report.get("manage_browsers_list_succeeded") is True - and report.get("connection_scope_kind") == "project" - and report.get("project_id") == expected_project_id - ) + atif = validate_trajectory(trajectory, expected_project_id) source_sha_matches = bool( manifest and manifest.get("kernel_mcp_server_sha") == os.environ.get("KERNEL_MCP_SOURCE_SHA") ) - trajectory_names = trajectory_tool_names(trajectory) - trajectory_has_calls = any( - name.endswith("get_connection_context") for name in trajectory_names - ) and any(name.endswith("manage_browsers") for name in trajectory_names) hypeman_identity_present = bool( manifest and manifest.get("hypeman_instance_name") ) checks = { - "get_connection_context": context_call is not None, - "manage_browsers_list": browsers_call is not None, - "agent_report": report_matches, + "native_mcp_calls": atif["native_calls_present"], + "tool_observations": atif["observations_valid"], + "context_scope": atif["context_scope_valid"], + "manage_browsers_arguments": atif["manage_browsers_arguments_valid"], "source_sha": source_sha_matches, "hypeman_identity": hypeman_identity_present, - "trajectory": trajectory_has_calls, "server_stdout": (LOGS_DIR / "kernel-mcp/server.stdout.log").is_file(), "server_stderr": (LOGS_DIR / "kernel-mcp/server.stderr.log").is_file(), } reward = 1.0 if all(checks.values()) else 0.0 - tool_calls = [ - { - "name": call["tool_name"], - "success": call["success"], - "duration_ms": call["duration_ms"], - "http_status": call["http_status"], - } - for call in (context_call, browsers_call) - if call is not None - ] result = { "reward": reward, "checks": checks, - "tool_calls": tool_calls, + "atif": atif, "agent_report": report, "trajectory": { "present": trajectory is not None, "schema_version": (trajectory or {}).get("schema_version"), "agent": (trajectory or {}).get("agent"), - "tool_names": trajectory_names, }, "run_manifest": manifest, } @@ -126,15 +202,7 @@ def main() -> int: (VERIFIER_DIR / "reward.txt").write_text(str(reward)) (VERIFIER_DIR / "reward.json").write_text( json.dumps( - { - "reward": reward, - "get_connection_context": float(context_call is not None), - "manage_browsers_list": float(browsers_call is not None), - "agent_report": float(report_matches), - "source_sha": float(source_sha_matches), - "hypeman_identity": float(hypeman_identity_present), - "trajectory": float(trajectory_has_calls), - }, + {"reward": reward, **{name: float(value) for name, value in checks.items()}}, indent=2, ) ) diff --git a/benchmarks/harbor/image/Dockerfile b/benchmarks/harbor/image/Dockerfile index a96e3fe..b2aaa0f 100644 --- a/benchmarks/harbor/image/Dockerfile +++ b/benchmarks/harbor/image/Dockerfile @@ -27,7 +27,6 @@ RUN KERNEL_CLI_PROD_CLIENT_ID=kernel-mcp-benchmark \ && install -m 0755 benchmarks/harbor/bin/start-kernel-mcp-server /usr/local/bin/start-kernel-mcp-server \ && install -m 0755 benchmarks/harbor/bin/kernel-mcp-local /usr/local/bin/kernel-mcp-local \ && install -m 0755 benchmarks/harbor/bin/verify-smoke.py /usr/local/bin/verify-kernel-mcp-smoke \ - && install -m 0644 benchmarks/harbor/bin/mcp-telemetry-proxy.mjs /usr/local/lib/mcp-telemetry-proxy.mjs \ && install -m 0644 benchmarks/harbor/image/source-sha /opt/kernel-mcp-server/SOURCE_SHA ENV NEXT_TELEMETRY_DISABLED=1 diff --git a/benchmarks/harbor/smoke/task.toml b/benchmarks/harbor/smoke/task.toml index d2b12cb..055bd16 100644 --- a/benchmarks/harbor/smoke/task.toml +++ b/benchmarks/harbor/smoke/task.toml @@ -44,7 +44,7 @@ timeout_sec = 300.0 timeout_sec = 60.0 [steps.healthcheck] -command = "test -s /logs/kernel-mcp/ready && curl -sS -o /dev/null http://127.0.0.1:3002/mcp && curl -sS -o /dev/null http://127.0.0.1:3003/mcp" +command = "test -s /logs/kernel-mcp/ready && curl -sS -o /dev/null http://127.0.0.1:3002/mcp" interval_sec = 2.0 timeout_sec = 5.0 start_period_sec = 1.0 From 82865cacb263acbec369f2bd66d31565d17f8be7 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Fri, 21 Aug 2026 19:50:09 +0000 Subject: [PATCH 05/29] Support Codex ATIF observations --- .../trajectory-codex-observation.json | 37 +++++++++++++++++++ benchmarks/harbor/bin/test_verify_smoke.py | 10 +++++ benchmarks/harbor/bin/verify-smoke.py | 21 +++++++++-- 3 files changed, 64 insertions(+), 4 deletions(-) create mode 100644 benchmarks/harbor/bin/fixtures/trajectory-codex-observation.json diff --git a/benchmarks/harbor/bin/fixtures/trajectory-codex-observation.json b/benchmarks/harbor/bin/fixtures/trajectory-codex-observation.json new file mode 100644 index 0000000..d280135 --- /dev/null +++ b/benchmarks/harbor/bin/fixtures/trajectory-codex-observation.json @@ -0,0 +1,37 @@ +{ + "schema_version": "ATIF-v1.7", + "steps": [ + { + "step_id": 1, + "source": "agent", + "tool_calls": [ + { + "tool_call_id": "ctx-1", + "function_name": "mcp__kernel__get_connection_context", + "arguments": {} + }, + { + "tool_call_id": "browsers-1", + "function_name": "mcp__kernel__manage_browsers", + "arguments": { + "action": "list", + "status": "active", + "limit": 1 + } + } + ], + "observation": { + "results": [ + { + "source_call_id": "ctx-1", + "content": "[{'type': 'text', 'text': '{\"connection_scope\":{\"kind\":\"project\",\"project_id\":\"project-123\"}}'}]" + }, + { + "source_call_id": "browsers-1", + "content": "[{'type': 'text', 'text': '{\"items\":[],\"has_more\":false}'}]" + } + ] + } + } + ] +} diff --git a/benchmarks/harbor/bin/test_verify_smoke.py b/benchmarks/harbor/bin/test_verify_smoke.py index 18d53be..707468b 100644 --- a/benchmarks/harbor/bin/test_verify_smoke.py +++ b/benchmarks/harbor/bin/test_verify_smoke.py @@ -28,6 +28,16 @@ def test_accepts_native_calls_with_paired_observations(self) -> None: self.assertTrue(proof["context_scope_valid"]) self.assertTrue(proof["manage_browsers_arguments_valid"]) + def test_accepts_codex_serialized_observations(self) -> None: + proof = verify_smoke.validate_trajectory( + load_fixture("trajectory-codex-observation.json"), "project-123" + ) + + self.assertTrue(proof["native_calls_present"]) + self.assertTrue(proof["observations_valid"]) + self.assertTrue(proof["context_scope_valid"]) + self.assertTrue(proof["manage_browsers_arguments_valid"]) + def test_rejects_missing_observation(self) -> None: proof = verify_smoke.validate_trajectory( load_fixture("trajectory-missing-observation.json"), "project-123" diff --git a/benchmarks/harbor/bin/verify-smoke.py b/benchmarks/harbor/bin/verify-smoke.py index 5b0de04..a3a9e85 100755 --- a/benchmarks/harbor/bin/verify-smoke.py +++ b/benchmarks/harbor/bin/verify-smoke.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 from __future__ import annotations +import ast import json import os from pathlib import Path @@ -24,16 +25,27 @@ def read_json(path: Path) -> dict[str, Any] | None: def _decode_tool_result_content(content: Any) -> Any: value = content - for _ in range(4): + for _ in range(6): if isinstance(value, str): try: value = json.loads(value) except json.JSONDecodeError: - return value + try: + value = ast.literal_eval(value) + except (SyntaxError, ValueError): + return value continue if isinstance(value, dict) and value.get("type") == "text": value = value.get("text") continue + if ( + isinstance(value, list) + and len(value) == 1 + and isinstance(value[0], dict) + and value[0].get("type") == "text" + ): + value = value[0].get("text") + continue return value return value @@ -119,13 +131,14 @@ def validate_trajectory( continue result = results[0] decoded = _decode_tool_result_content(result.get("content")) - if _contains_error(decoded) or _contains_error(result.get("extra")): + if _contains_error(decoded) or _contains_error(result): error_observations.append(call_id) continue if call.get("function_name") == "mcp__kernel__get_connection_context": scope = decoded.get("connection_scope") if isinstance(decoded, dict) else None context_scope_valid = context_scope_valid and ( - isinstance(scope, dict) + bool(expected_project_id) + and isinstance(scope, dict) and scope.get("kind") == "project" and scope.get("project_id") == expected_project_id ) From ed65feddab128066745225556a0851a89dd79dbb Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Fri, 21 Aug 2026 19:57:05 +0000 Subject: [PATCH 06/29] Pin harbor-hypeman benchmark release --- benchmarks/harbor/README.md | 14 +++++++++++--- benchmarks/harbor/run-smoke.sh | 17 ++++++++++------- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/benchmarks/harbor/README.md b/benchmarks/harbor/README.md index bc08cb8..11c73d6 100644 --- a/benchmarks/harbor/README.md +++ b/benchmarks/harbor/README.md @@ -4,14 +4,22 @@ This directory runs stock Harbor agents against a locally built `kernel-mcp-serv ## Requirements -- Harbor 0.21.0 with `harbor_hypeman:HypemanEnvironment` -- `harbor-hypeman` with existing Hypeman image-reference support -- Hypeman CLI and credentials +- Harbor 0.21.0 +- `harbor-hypeman` 0.1.1 +- [uv](https://docs.astral.sh/uv/) and Hypeman CLI credentials - `KERNEL_MCP_BENCHMARK_API_KEY` scoped to an isolated evaluation project - `KERNEL_MCP_BENCHMARK_PROJECT_ID` - `ANTHROPIC_API_KEY` or `CLAUDE_CODE_OAUTH_TOKEN` for Claude Code - `OPENAI_API_KEY` for Codex +`run-smoke.sh` launches the pinned Harbor packages through `uvx`. To install the same versions as a persistent tool instead: + +```bash +uv tool install 'harbor==0.21.0' --with 'harbor-hypeman==0.1.1' +``` + +Set `HARBOR_BIN` only when intentionally testing a different Harbor installation. + ## Build the image ```bash diff --git a/benchmarks/harbor/run-smoke.sh b/benchmarks/harbor/run-smoke.sh index d4c447c..1618978 100755 --- a/benchmarks/harbor/run-smoke.sh +++ b/benchmarks/harbor/run-smoke.sh @@ -46,13 +46,16 @@ case "$agent" in esac if [[ -n "${HARBOR_BIN:-}" ]]; then - harbor_bin=$HARBOR_BIN -elif command -v harbor >/dev/null 2>&1; then - harbor_bin=$(command -v harbor) -elif [[ -x "$repo_root/../harbor-hypeman/.venv/bin/harbor" ]]; then - harbor_bin="$repo_root/../harbor-hypeman/.venv/bin/harbor" + harbor_command=("$HARBOR_BIN") +elif command -v uvx >/dev/null 2>&1; then + harbor_command=( + uvx + --from "harbor==0.21.0" + --with "harbor-hypeman==0.1.1" + harbor + ) else - echo "Harbor CLI not found; set HARBOR_BIN" >&2 + echo "uvx not found; install uv or set HARBOR_BIN" >&2 exit 1 fi @@ -81,7 +84,7 @@ chmod 0600 "$runtime_env" mkdir -p "$jobs_dir" timeout --signal=INT --kill-after=30s "${HARBOR_BENCHMARK_TIMEOUT:-10m}" \ - "$harbor_bin" run \ + "${harbor_command[@]}" run \ --path "$runtime_task" \ --agent "$agent" \ --model "$model" \ From 16278ee1453abe5d402f2eb3d98305c6fe7106aa Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Fri, 21 Aug 2026 20:04:51 +0000 Subject: [PATCH 07/29] Forward Codex credentials to benchmark --- benchmarks/harbor/run-smoke.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/benchmarks/harbor/run-smoke.sh b/benchmarks/harbor/run-smoke.sh index 1618978..7e1571a 100755 --- a/benchmarks/harbor/run-smoke.sh +++ b/benchmarks/harbor/run-smoke.sh @@ -79,6 +79,7 @@ ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY:-} ANTHROPIC_AUTH_TOKEN=${ANTHROPIC_AUTH_TOKEN:-} CLAUDE_CODE_OAUTH_TOKEN=${CLAUDE_CODE_OAUTH_TOKEN:-} CLAUDE_FORCE_OAUTH=${CLAUDE_FORCE_OAUTH:-} +OPENAI_API_KEY=${OPENAI_API_KEY:-} EOF chmod 0600 "$runtime_env" From 2b9901ef56f06b474eefd63c379e6325af96d0a1 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sat, 22 Aug 2026 12:27:04 +0000 Subject: [PATCH 08/29] Add ClawBench Kernel MCP control arm --- README.md | 2 +- benchmarks/harbor/README.md | 20 ++ benchmarks/harbor/bin/start-kernel-mcp-server | 10 + .../harbor/clawbench/prepare-control.py | 175 +++++++++++++ benchmarks/harbor/clawbench/run-control.sh | 131 ++++++++++ benchmarks/harbor/clawbench/test_control.py | 184 +++++++++++++ benchmarks/harbor/clawbench/verify-control.py | 245 ++++++++++++++++++ benchmarks/harbor/image/Dockerfile | 2 + src/lib/mcp/register.test.ts | 31 +++ src/lib/mcp/register.ts | 41 ++- 10 files changed, 837 insertions(+), 4 deletions(-) create mode 100755 benchmarks/harbor/clawbench/prepare-control.py create mode 100755 benchmarks/harbor/clawbench/run-control.sh create mode 100644 benchmarks/harbor/clawbench/test_control.py create mode 100755 benchmarks/harbor/clawbench/verify-control.py diff --git a/README.md b/README.md index 21be43e..062e5a1 100644 --- a/README.md +++ b/README.md @@ -298,7 +298,7 @@ Each Kernel feature has a single `manage_*` tool with an `action` parameter, kee One additional Managed Auth helper (`begin_auth_login`) is marked app-only (`_meta.ui.visibility: ["app"]`); it refuses to execute on hosts that do not declare MCP Apps support. The App forwards the server-issued signed flow checkpoint to the shared `manage_auth_connections` `wait` action, so flow identity and terminal-state decisions stay on the server. -Self-hosted deployments can hide sensitive tool families by setting `KERNEL_MCP_DISABLED_TOOLSETS` to a comma-separated list. For example, `KERNEL_MCP_DISABLED_TOOLSETS=api_keys` prevents `manage_api_keys` from being registered. +Self-hosted deployments can select tool families with `KERNEL_MCP_ENABLED_TOOLSETS` or hide them with `KERNEL_MCP_DISABLED_TOOLSETS`. Both accept comma- or space-separated toolset names and standalone aliases. For example, `KERNEL_MCP_ENABLED_TOOLSETS="playwright computer"` exposes browser-control tools without browser lifecycle or managed-auth tools, while `KERNEL_MCP_DISABLED_TOOLSETS=api_keys` only removes `manage_api_keys`. `get_connection_context` remains available in either mode. Call `get_connection_context` before deciding whether to create or select a project. Its canonical `connection_scope` reports whether the connection is organization-wide or fixed to a project. Project-scoped tools advertise an optional `project` (name or ID) and a deprecated `project_id`: organization-wide connections may omit them to preserve organization-wide reads and API default-project behavior, while fixed-project connections may omit them or pass the matching project. Project resources use project-qualified `kernel://orgs/{organizationId}/projects/{projectId}/...` URIs. Authorization remains enforced by the Kernel API; selecting a project never grants access to it. diff --git a/benchmarks/harbor/README.md b/benchmarks/harbor/README.md index 11c73d6..5ff8de6 100644 --- a/benchmarks/harbor/README.md +++ b/benchmarks/harbor/README.md @@ -54,3 +54,23 @@ The output defaults to `/tmp/kernel-mcp-harbor-jobs/`. Each successful - numeric Harbor rewards plus detailed `smoke-result.json` The verifier proves local-server use from Harbor's ATIF trajectory: it requires native `mcp__kernel__get_connection_context` and `mcp__kernel__manage_browsers` calls, paired non-error observations, the expected project scope, and the required read-only browser-list arguments. Direct HTTP or custom MCP-client workarounds do not pass. + +## Run the ClawBench Kernel MCP arm + +The ClawBench arm starts from the Kernel-backed Harbor task produced by `clawbench-harbor-adapt`, replaces Playwright MCP with the local source-pinned Kernel MCP server, and keeps ClawBench attached to the same pre-created browser. + +```bash +export CLAWBENCH_REPO=../ClawBench +./benchmarks/harbor/clawbench/run-control.sh claude-code \ + v2-1134-chapter-finder-redcross +``` + +The ClawBench checkout must contain commit `6efb04e`, from `kernel/ClawBench` PR #1. The generated task: + +- exposes `get_connection_context`, `execute_playwright_code`, and `computer_action` +- disables browser lifecycle and managed-auth toolsets +- instructs the agent to read `./my-info/kernel_browser.json` and use that session ID +- instructs account tasks to use the supplied PurelyMail credentials instead of managed auth +- verifies ATIF observations, project scope, exact session reuse, ClawBench interception, replay finalization, and browser deletion + +Outputs use the normal Harbor job directory and add `kernel-mcp-control-result.json`, Kernel MCP logs, source manifests, and same-session metrics to the ClawBench verifier artifacts. diff --git a/benchmarks/harbor/bin/start-kernel-mcp-server b/benchmarks/harbor/bin/start-kernel-mcp-server index 08e7601..727c95d 100755 --- a/benchmarks/harbor/bin/start-kernel-mcp-server +++ b/benchmarks/harbor/bin/start-kernel-mcp-server @@ -45,8 +45,18 @@ import platform from datetime import datetime, timezone from pathlib import Path +browser_path = Path("/my-info/kernel_browser.json") +try: + browser = json.loads(browser_path.read_text()) +except (OSError, json.JSONDecodeError): + browser = {} + manifest = { "kernel_mcp_server_sha": Path("/opt/kernel-mcp-server/SOURCE_SHA").read_text().strip(), + "clawbench_source_sha": os.environ.get("CLAWBENCH_SOURCE_SHA", ""), + "browser_session_id": browser.get("session_id"), + "enabled_toolsets": os.environ.get("KERNEL_MCP_ENABLED_TOOLSETS", ""), + "disabled_toolsets": os.environ.get("KERNEL_MCP_DISABLED_TOOLSETS", ""), "image": os.environ.get("KERNEL_MCP_BENCHMARK_IMAGE", ""), "hypeman_instance_name": os.environ.get("HYPEMAN_INSTANCE_NAME", ""), "sandbox_hostname": platform.node(), diff --git a/benchmarks/harbor/clawbench/prepare-control.py b/benchmarks/harbor/clawbench/prepare-control.py new file mode 100755 index 0000000..50b58b6 --- /dev/null +++ b/benchmarks/harbor/clawbench/prepare-control.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import shutil +from pathlib import Path + +ENABLED_TOOLSETS = "playwright computer" + + +def _drop_mcp_servers(task_toml: str) -> str: + lines = task_toml.splitlines() + output: list[str] = [] + dropping = False + for line in lines: + if line.strip() == "[[environment.mcp_servers]]": + dropping = True + continue + if dropping and line.startswith("["): + dropping = False + if not dropping: + output.append(line) + return "\n".join(output).rstrip() + "\n" + + +def _add_environment(task_toml: str, *, image: str, server_sha: str, clawbench_sha: str) -> str: + lines = task_toml.splitlines() + output: list[str] = [] + inserted_image = False + inserted_env = False + for line in lines: + output.append(line) + if line.strip() == "[environment]": + output.append(f"docker_image = {json.dumps(image)}") + inserted_image = True + elif line.strip() == "[environment.env]": + output.extend( + [ + f"KERNEL_MCP_BENCHMARK_IMAGE = {json.dumps(image)}", + f"KERNEL_MCP_SOURCE_SHA = {json.dumps(server_sha)}", + f"CLAWBENCH_SOURCE_SHA = {json.dumps(clawbench_sha)}", + f"KERNEL_MCP_ENABLED_TOOLSETS = {json.dumps(ENABLED_TOOLSETS)}", + 'KERNEL_MCP_EXPECTED_PROJECT_ID = "${KERNEL_MCP_BENCHMARK_PROJECT_ID}"', + 'KERNEL_API_BASE_URL = "${KERNEL_API_BASE_URL:-}"', + ] + ) + inserted_env = True + if not inserted_image or not inserted_env: + raise ValueError("generated task is missing Harbor environment sections") + output.extend( + [ + "", + "[[environment.mcp_servers]]", + 'name = "kernel"', + 'transport = "stdio"', + 'command = "/usr/local/bin/kernel-mcp-local"', + "args = []", + ] + ) + return "\n".join(output).rstrip() + "\n" + + +def _patch_setup(setup: str) -> str: + install = """install_clawbench_runtime() { + mkdir -p /app/src + rm -rf /app/src/runtime-server /app/src/chrome-extension /app/src/shared /app/src/harbor + cp -a /runtime-server /app/src/runtime-server + cp -a /chrome-extension /app/src/chrome-extension + cp -a /shared /app/src/shared + cp -a /harbor /app/src/harbor + chmod +x /app/src/harbor/*.sh /app/src/harbor/*.py + cd /app/src/runtime-server + UV_PYTHON_PREFERENCE=only-system uv sync --frozen + uv pip install --python .venv/bin/python fpdf2 + cd / +} + +install_clawbench_runtime +""" + marker = "mkdir -p /data /logs/verifier /extra_info\n" + if marker not in setup: + raise ValueError("generated setup script is missing directory initialization") + setup = setup.replace(marker, marker + "\n" + install, 1) + runtime_marker = "/app/src/harbor/start-runtime.sh\n" + if runtime_marker not in setup: + raise ValueError("generated setup script is missing runtime startup") + return setup.replace( + runtime_marker, + runtime_marker + "\nstart-kernel-mcp-server\n", + 1, + ) + + +def _patch_verifier(test_script: str) -> str: + verify_marker = "/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py\n" + if verify_marker not in test_script: + raise ValueError("generated verifier script is missing ClawBench verification") + return test_script.replace( + verify_marker, + verify_marker + + "mkdir -p /data/kernel-mcp\n" + + "cp -a /logs/kernel-mcp/. /data/kernel-mcp/\n" + + "/app/src/runtime-server/.venv/bin/python " + + "/app/src/harbor/verify-kernel-mcp-control.py\n", + 1, + ) + + +def _patch_instruction(instruction: str) -> str: + return instruction.rstrip() + """ + +--- +Kernel MCP benchmark arm: +- Call `get_connection_context` once before taking any browser action. +- Read `./my-info/kernel_browser.json` and use its existing `session_id` for every `execute_playwright_code` or `computer_action` call. +- Do not create, list, update, or delete browsers. Browser lifecycle tools are intentionally unavailable. +- Use Kernel MCP for all browser interaction. Do not use Playwright MCP or a direct CDP client. +- Use the PurelyMail-backed credentials already provided under `./my-info/` when the task requires an account. +- Do not use Kernel managed auth, create an auth connection, or start a hosted login flow. +- Complete and submit the task through the existing browser, then stop. +""" + + +def transform_task(task_dir: Path, *, image: str, server_sha: str, clawbench_sha: str) -> None: + dockerfile = task_dir / "environment" / "Dockerfile" + dockerfile.unlink(missing_ok=True) + + task_toml_path = task_dir / "task.toml" + task_toml = _drop_mcp_servers(task_toml_path.read_text()) + task_toml_path.write_text( + _add_environment( + task_toml, + image=image, + server_sha=server_sha, + clawbench_sha=clawbench_sha, + ) + ) + + step_dir = task_dir / "steps" / "run" + setup_path = step_dir / "workdir" / "setup.sh" + setup_path.write_text(_patch_setup(setup_path.read_text())) + setup_path.chmod(0o755) + + test_path = step_dir / "tests" / "test.sh" + test_path.write_text(_patch_verifier(test_path.read_text())) + test_path.chmod(0o755) + + instruction_path = step_dir / "instruction.md" + instruction_path.write_text(_patch_instruction(instruction_path.read_text())) + + verifier_source = Path(__file__).with_name("verify-control.py") + verifier_target = task_dir / "environment" / "harbor" / "verify-kernel-mcp-control.py" + shutil.copy2(verifier_source, verifier_target) + verifier_target.chmod(0o755) + + +def main() -> int: + parser = argparse.ArgumentParser(description="Convert a Kernel-backed ClawBench Harbor task to the Kernel MCP arm") + parser.add_argument("task_dir", type=Path) + parser.add_argument("--image", required=True) + parser.add_argument("--server-sha", required=True) + parser.add_argument("--clawbench-sha", required=True) + args = parser.parse_args() + transform_task( + args.task_dir, + image=args.image, + server_sha=args.server_sha, + clawbench_sha=args.clawbench_sha, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/harbor/clawbench/run-control.sh b/benchmarks/harbor/clawbench/run-control.sh new file mode 100755 index 0000000..db8a3f2 --- /dev/null +++ b/benchmarks/harbor/clawbench/run-control.sh @@ -0,0 +1,131 @@ +#!/bin/bash +set -euo pipefail + +usage() { + echo "usage: $0 [task-id] [job-name] [jobs-dir]" >&2 + exit 2 +} + +agent=${1:-} +[[ "$agent" == "claude-code" || "$agent" == "codex" ]] || usage +task_id=${2:-v2-1134-chapter-finder-redcross} + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd) +benchmark_dir="$repo_root/benchmarks/harbor" +image_env="$benchmark_dir/.image.env" +clawbench_repo=${CLAWBENCH_REPO:-$repo_root/../ClawBench} +clawbench_ref=${CLAWBENCH_REF:-6efb04e49efc44f36fa03c8be3bcdb3ef091434f} + +[[ -f "$image_env" ]] || { + echo "Missing $image_env; run benchmarks/harbor/build-image.sh first" >&2 + exit 1 +} +[[ -d "$clawbench_repo/.git" || -f "$clawbench_repo/.git" ]] || { + echo "ClawBench checkout not found at $clawbench_repo" >&2 + exit 1 +} +git -C "$clawbench_repo" merge-base --is-ancestor "$clawbench_ref" HEAD || { + echo "ClawBench checkout must contain $clawbench_ref" >&2 + exit 1 +} + +if [[ -f "$clawbench_repo/.env" ]]; then + set -a + source "$clawbench_repo/.env" + set +a +fi +set -a +source "$image_env" +set +a + +: "${KERNEL_MCP_BENCHMARK_API_KEY:?KERNEL_MCP_BENCHMARK_API_KEY is required}" +: "${KERNEL_MCP_BENCHMARK_PROJECT_ID:?KERNEL_MCP_BENCHMARK_PROJECT_ID is required}" +: "${PURELY_MAIL_API_KEY:?PURELY_MAIL_API_KEY is required}" +: "${PURELY_MAIL_DOMAIN:?PURELY_MAIL_DOMAIN is required}" + +case "$agent" in + claude-code) + if [[ -z "${ANTHROPIC_API_KEY:-}" && -z "${ANTHROPIC_AUTH_TOKEN:-}" && -z "${CLAUDE_CODE_OAUTH_TOKEN:-}" ]]; then + echo "ANTHROPIC_API_KEY, ANTHROPIC_AUTH_TOKEN, or CLAUDE_CODE_OAUTH_TOKEN is required" >&2 + exit 1 + fi + if [[ -z "${ANTHROPIC_API_KEY:-}" && -z "${ANTHROPIC_AUTH_TOKEN:-}" ]]; then + ANTHROPIC_AUTH_TOKEN=$CLAUDE_CODE_OAUTH_TOKEN + CLAUDE_FORCE_OAUTH=1 + export ANTHROPIC_AUTH_TOKEN CLAUDE_FORCE_OAUTH + fi + model=${CLAUDE_BENCHMARK_MODEL:-claude-sonnet-5} + version=${CLAUDE_BENCHMARK_VERSION:-2.1.238} + ;; + codex) + : "${OPENAI_API_KEY:?OPENAI_API_KEY is required}" + model=${CODEX_BENCHMARK_MODEL:-gpt-5.6-terra} + version=${CODEX_BENCHMARK_VERSION:-0.120.0} + ;; +esac + +runtime_root=$(mktemp -d) +runtime_env=$(mktemp) +trap 'rm -rf "$runtime_root"; rm -f "$runtime_env"' EXIT + +dataset="$runtime_root/dataset" +uv --directory "$clawbench_repo" run clawbench-harbor-adapt \ + --output-dir "$dataset" \ + --task-ids "$task_id" \ + --browser-runtime kernel \ + --browser-runtime-options '{"stealth": false}' \ + --overwrite + +task_dir=$(find "$dataset" -mindepth 1 -maxdepth 1 -type d | head -1) +[[ -n "$task_dir" ]] || { + echo "ClawBench did not generate task $task_id" >&2 + exit 1 +} + +python3 "$benchmark_dir/clawbench/prepare-control.py" "$task_dir" \ + --image "$KERNEL_MCP_BENCHMARK_IMAGE" \ + --server-sha "$KERNEL_MCP_SOURCE_SHA" \ + --clawbench-sha "$clawbench_ref" + +export KERNEL_API_KEY=$KERNEL_MCP_BENCHMARK_API_KEY +export KERNEL_BASE_URL=${KERNEL_BASE_URL:-https://api.onkernel.com} +export KERNEL_API_BASE_URL=${KERNEL_API_BASE_URL:-$KERNEL_BASE_URL} +export KERNEL_MCP_BENCHMARK_PROJECT_ID + +cat >"$runtime_env" < ModuleType: + spec = importlib.util.spec_from_file_location(name, HERE / filename) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +prepare = load_module("prepare_control", "prepare-control.py") +verify = load_module("verify_control", "verify-control.py") + + +class PrepareControlTest(unittest.TestCase): + def test_transforms_playwright_control_into_kernel_mcp_arm(self) -> None: + with tempfile.TemporaryDirectory() as temp: + task = Path(temp) + environment = task / "environment" + step = task / "steps" / "run" + (step / "workdir").mkdir(parents=True) + (step / "tests").mkdir() + (step / "instruction.md").write_text("Complete the browser task.\n") + (environment / "harbor").mkdir(parents=True) + (environment / "Dockerfile").write_text("FROM python:3.11-slim\n") + (task / "task.toml").write_text( + """[environment] +workdir = "/" + +[environment.env] +KERNEL_API_KEY = "${KERNEL_API_KEY}" + +[[steps]] +name = "run" + +[[environment.mcp_servers]] +name = "playwright" +transport = "stdio" +command = "npx" +args = ["-y", "@playwright/mcp@0.0.79"] +""" + ) + (step / "workdir" / "setup.sh").write_text( + "#!/bin/bash\nmkdir -p /data /logs/verifier /extra_info\n" + "/app/src/harbor/start-runtime.sh\n" + ) + (step / "tests" / "test.sh").write_text( + "#!/bin/bash\n" + "/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py\n" + ) + + prepare.transform_task( + task, + image="docker.io/builds/image:latest", + server_sha="server-sha", + clawbench_sha="clawbench-sha", + ) + + task_toml = (task / "task.toml").read_text() + self.assertFalse((environment / "Dockerfile").exists()) + self.assertIn('docker_image = "docker.io/builds/image:latest"', task_toml) + self.assertIn('name = "kernel"', task_toml) + self.assertIn('command = "/usr/local/bin/kernel-mcp-local"', task_toml) + self.assertNotIn("@playwright/mcp", task_toml) + self.assertIn( + 'KERNEL_MCP_ENABLED_TOOLSETS = "playwright computer"', task_toml + ) + self.assertNotIn("KERNEL_MCP_DISABLED_TOOLSETS", task_toml) + + setup = (step / "workdir" / "setup.sh").read_text() + self.assertIn("install_clawbench_runtime", setup) + self.assertIn("start-kernel-mcp-server", setup) + test_script = (step / "tests" / "test.sh").read_text() + self.assertIn("verify-kernel-mcp-control.py", test_script) + self.assertIn("/data/kernel-mcp", test_script) + instruction = (step / "instruction.md").read_text() + self.assertIn("existing `session_id`", instruction) + self.assertIn("PurelyMail-backed credentials", instruction) + self.assertIn("Do not use Kernel managed auth", instruction) + self.assertTrue((environment / "harbor" / "verify-kernel-mcp-control.py").is_file()) + + +class VerifyControlTest(unittest.TestCase): + def trajectory(self, session_id: str = "session-123") -> dict: + return { + "steps": [ + { + "tool_calls": [ + { + "tool_call_id": "context-1", + "function_name": "mcp__kernel__get_connection_context", + "arguments": {}, + }, + { + "tool_call_id": "playwright-1", + "function_name": "mcp__kernel__execute_playwright_code", + "arguments": { + "session_id": session_id, + "code": "await page.goto('https://example.com')", + }, + }, + ], + "observation": { + "results": [ + { + "source_call_id": "context-1", + "content": { + "connection_scope": { + "kind": "project", + "project_id": "project-123", + } + }, + }, + { + "source_call_id": "playwright-1", + "content": [{"type": "text", "text": "{\"ok\": true}"}], + }, + ] + }, + } + ] + } + + def test_accepts_successful_calls_on_precreated_session(self) -> None: + result = verify.validate_control( + self.trajectory(), + expected_session_id="session-123", + expected_project_id="project-123", + ) + for key in ( + "context_called", + "browser_control_called", + "observations_valid", + "context_scope_valid", + "same_session", + "no_playwright_mcp", + "no_forbidden_kernel_tools", + ): + self.assertTrue(result[key], key) + + def test_rejects_another_session(self) -> None: + result = verify.validate_control( + self.trajectory("session-other"), + expected_session_id="session-123", + expected_project_id="project-123", + ) + self.assertFalse(result["same_session"]) + + def test_rejects_playwright_mcp_and_lifecycle_tools(self) -> None: + trajectory = self.trajectory() + trajectory["steps"][0]["tool_calls"].extend( + [ + { + "tool_call_id": "direct-playwright", + "function_name": "mcp__playwright__browser_navigate", + "arguments": {}, + }, + { + "tool_call_id": "browser-list", + "function_name": "mcp__kernel__manage_browsers", + "arguments": {"action": "list"}, + }, + ] + ) + result = verify.validate_control( + trajectory, + expected_session_id="session-123", + expected_project_id="project-123", + ) + self.assertFalse(result["no_playwright_mcp"]) + self.assertFalse(result["no_forbidden_kernel_tools"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/benchmarks/harbor/clawbench/verify-control.py b/benchmarks/harbor/clawbench/verify-control.py new file mode 100755 index 0000000..681dac8 --- /dev/null +++ b/benchmarks/harbor/clawbench/verify-control.py @@ -0,0 +1,245 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import ast +import json +import os +from pathlib import Path +from typing import Any + +LOGS_DIR = Path(os.environ.get("HARBOR_LOGS_DIR", "/logs")) +VERIFIER_DIR = LOGS_DIR / "verifier" +CONTEXT_TOOL = "mcp__kernel__get_connection_context" +BROWSER_TOOLS = { + "mcp__kernel__execute_playwright_code", + "mcp__kernel__computer_action", +} +FORBIDDEN_KERNEL_TOOLS = { + "mcp__kernel__manage_browsers", + "mcp__kernel__manage_auth_connections", + "mcp__kernel__open_auth_login", +} + + +def read_json(path: Path) -> dict[str, Any] | None: + try: + value = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError): + return None + return value if isinstance(value, dict) else None + + +def _decode_content(content: Any) -> Any: + value = content + for _ in range(6): + if isinstance(value, str): + try: + value = json.loads(value) + except json.JSONDecodeError: + try: + value = ast.literal_eval(value) + except (SyntaxError, ValueError): + return value + continue + if isinstance(value, dict) and value.get("type") == "text": + value = value.get("text") + continue + if ( + isinstance(value, list) + and len(value) == 1 + and isinstance(value[0], dict) + and value[0].get("type") == "text" + ): + value = value[0].get("text") + continue + return value + return value + + +def _contains_error(value: Any) -> bool: + if isinstance(value, dict): + if value.get("is_error") is True or value.get("isError") is True: + return True + if value.get("error") not in (None, False, ""): + return True + return any(_contains_error(item) for item in value.values()) + if isinstance(value, list): + return any(_contains_error(item) for item in value) + if isinstance(value, str): + return value.lstrip().lower().startswith(("[error]", "error:", "error in ")) + return False + + +def _calls(trajectory: dict[str, Any]) -> list[dict[str, Any]]: + calls: list[dict[str, Any]] = [] + for step in trajectory.get("steps") or []: + if not isinstance(step, dict): + continue + calls.extend(call for call in step.get("tool_calls") or [] if isinstance(call, dict)) + return calls + + +def _results(trajectory: dict[str, Any]) -> dict[str, list[dict[str, Any]]]: + results: dict[str, list[dict[str, Any]]] = {} + for step in trajectory.get("steps") or []: + if not isinstance(step, dict): + continue + observation = step.get("observation") + if not isinstance(observation, dict): + continue + for result in observation.get("results") or []: + if not isinstance(result, dict): + continue + call_id = result.get("source_call_id") + if isinstance(call_id, str): + results.setdefault(call_id, []).append(result) + return results + + +def validate_control( + trajectory: dict[str, Any] | None, + *, + expected_session_id: str, + expected_project_id: str, +) -> dict[str, Any]: + trajectory = trajectory or {} + calls = _calls(trajectory) + result_map = _results(trajectory) + kernel_calls = [ + call for call in calls if str(call.get("function_name", "")).startswith("mcp__kernel__") + ] + context_calls = [call for call in kernel_calls if call.get("function_name") == CONTEXT_TOOL] + browser_calls = [call for call in kernel_calls if call.get("function_name") in BROWSER_TOOLS] + playwright_calls = [ + call for call in calls if str(call.get("function_name", "")).startswith("mcp__playwright__") + ] + forbidden_calls = [ + call for call in kernel_calls if call.get("function_name") in FORBIDDEN_KERNEL_TOOLS + ] + + missing_observations: list[Any] = [] + duplicate_observations: list[Any] = [] + error_observations: list[Any] = [] + context_scope_valid = bool(expected_project_id) + same_session = bool(expected_session_id and browser_calls) + + for call in context_calls + browser_calls: + call_id = call.get("tool_call_id") + observations = result_map.get(call_id, []) if isinstance(call_id, str) else [] + if not observations: + missing_observations.append(call_id) + continue + if len(observations) != 1: + duplicate_observations.append(call_id) + continue + decoded = _decode_content(observations[0].get("content")) + if _contains_error(decoded) or _contains_error(observations[0]): + error_observations.append(call_id) + continue + if call.get("function_name") == CONTEXT_TOOL: + scope = decoded.get("connection_scope") if isinstance(decoded, dict) else None + context_scope_valid = context_scope_valid and ( + isinstance(scope, dict) + and scope.get("kind") == "project" + and scope.get("project_id") == expected_project_id + ) + elif call.get("function_name") in BROWSER_TOOLS: + arguments = call.get("arguments") + same_session = same_session and ( + isinstance(arguments, dict) + and arguments.get("session_id") == expected_session_id + ) + + observations_valid = bool(context_calls and browser_calls) and not ( + missing_observations or duplicate_observations or error_observations + ) + return { + "context_called": bool(context_calls), + "browser_control_called": bool(browser_calls), + "observations_valid": observations_valid, + "context_scope_valid": context_scope_valid and bool(context_calls), + "same_session": same_session, + "no_playwright_mcp": not playwright_calls, + "no_forbidden_kernel_tools": not forbidden_calls, + "missing_observations": missing_observations, + "duplicate_observations": duplicate_observations, + "error_observations": error_observations, + "kernel_tool_calls": [ + { + "tool_call_id": call.get("tool_call_id"), + "name": call.get("function_name"), + "arguments": call.get("arguments"), + } + for call in kernel_calls + ], + } + + +def main() -> int: + VERIFIER_DIR.mkdir(parents=True, exist_ok=True) + trajectory = read_json(LOGS_DIR / "agent" / "trajectory.json") + browser = read_json(Path("/my-info/kernel_browser.json")) + lifecycle = read_json(Path("/data/kernel-browser-lifecycle.json")) + manifest = read_json(LOGS_DIR / "kernel-mcp" / "run-manifest.json") + clawbench_result = read_json(VERIFIER_DIR / "clawbench-result.json") + reward_path = VERIFIER_DIR / "reward.json" + reward_metrics = read_json(reward_path) or {} + + session_id = str((browser or {}).get("session_id") or "") + expected_project_id = os.environ.get("KERNEL_MCP_EXPECTED_PROJECT_ID", "") + atif = validate_control( + trajectory, + expected_session_id=session_id, + expected_project_id=expected_project_id, + ) + checks = { + "kernel_mcp_context": atif["context_called"], + "kernel_mcp_browser_control": atif["browser_control_called"], + "kernel_mcp_observations": atif["observations_valid"], + "kernel_mcp_project_scope": atif["context_scope_valid"], + "kernel_mcp_same_session": atif["same_session"], + "no_playwright_mcp": atif["no_playwright_mcp"], + "no_forbidden_kernel_tools": atif["no_forbidden_kernel_tools"], + "kernel_mcp_source_sha": bool( + manifest + and manifest.get("kernel_mcp_server_sha") == os.environ.get("KERNEL_MCP_SOURCE_SHA") + ), + "kernel_mcp_manifest_session": bool( + manifest and manifest.get("browser_session_id") == session_id + ), + "kernel_mcp_toolset_allowlist": bool( + manifest + and set(str(manifest.get("enabled_toolsets", "")).split()) + == {"playwright", "computer"} + ), + "hypeman_identity": bool(manifest and manifest.get("hypeman_instance_name")), + "browser_deleted": bool( + lifecycle + and lifecycle.get("status") == "deleted" + and lifecycle.get("deletion_verified") is True + ), + "clawbench_intercepted": bool( + reward_metrics.get("intercepted") == 1 + or (clawbench_result or {}).get("intercepted") is True + ), + } + infra_ok = all(value for name, value in checks.items() if name != "clawbench_intercepted") + checks["infra_ok"] = infra_ok + + reward_metrics.update({name: float(value) for name, value in checks.items()}) + reward_path.write_text(json.dumps(reward_metrics, indent=2)) + result = { + "checks": checks, + "session_id": session_id, + "expected_project_id": expected_project_id, + "atif": atif, + "run_manifest": manifest, + "browser_lifecycle": lifecycle, + "clawbench_result": clawbench_result, + } + (VERIFIER_DIR / "kernel-mcp-control-result.json").write_text(json.dumps(result, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/harbor/image/Dockerfile b/benchmarks/harbor/image/Dockerfile index b2aaa0f..be85f3a 100644 --- a/benchmarks/harbor/image/Dockerfile +++ b/benchmarks/harbor/image/Dockerfile @@ -12,6 +12,8 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* \ && npm install --global bun@1.3.3 +COPY --from=ghcr.io/astral-sh/uv:0.11.6 /uv /usr/local/bin/uv + WORKDIR /opt/kernel-mcp-server COPY package.json bun.lock ./ diff --git a/src/lib/mcp/register.test.ts b/src/lib/mcp/register.test.ts index 52061de..a6b0c94 100644 --- a/src/lib/mcp/register.test.ts +++ b/src/lib/mcp/register.test.ts @@ -83,6 +83,37 @@ describe("MCP Apps additive registration", () => { }); }); +describe("MCP toolset allowlist", () => { + test("keeps connection context and only the selected browser controls", () => { + const previousEnabled = process.env.KERNEL_MCP_ENABLED_TOOLSETS; + const previousDisabled = process.env.KERNEL_MCP_DISABLED_TOOLSETS; + process.env.KERNEL_MCP_ENABLED_TOOLSETS = + "execute_playwright_code computer_action"; + delete process.env.KERNEL_MCP_DISABLED_TOOLSETS; + try { + const registration = captureRegistration(false); + expect(registration.legacyTools).toEqual([ + "get_connection_context", + "computer_action", + "execute_playwright_code", + ]); + expect(registration.appTools).toEqual([]); + expect(registration.resources).toEqual([]); + } finally { + if (previousEnabled === undefined) { + delete process.env.KERNEL_MCP_ENABLED_TOOLSETS; + } else { + process.env.KERNEL_MCP_ENABLED_TOOLSETS = previousEnabled; + } + if (previousDisabled === undefined) { + delete process.env.KERNEL_MCP_DISABLED_TOOLSETS; + } else { + process.env.KERNEL_MCP_DISABLED_TOOLSETS = previousDisabled; + } + } + }); +}); + describe("project selection registration", () => { const projectScopedTools = [ "manage_profiles", diff --git a/src/lib/mcp/register.ts b/src/lib/mcp/register.ts index 59db2cb..712be6c 100644 --- a/src/lib/mcp/register.ts +++ b/src/lib/mcp/register.ts @@ -90,6 +90,33 @@ function normalizeMcpToolset(value: string): McpToolset | undefined { return undefined; } +function enabledMcpToolsetsFromEnv() { + const raw = process.env.KERNEL_MCP_ENABLED_TOOLSETS; + if (!raw?.trim()) return undefined; + + const enabled = new Set(); + const unknown: string[] = []; + for (const value of raw.split(/[,\s]+/)) { + const token = value.trim().toLowerCase(); + if (!token || token === "none") continue; + if (token === "all") return new Set(mcpToolsets); + + const toolset = normalizeMcpToolset(token); + if (toolset) { + enabled.add(toolset); + } else { + unknown.push(value); + } + } + + if (unknown.length > 0) { + throw new Error( + `Unknown KERNEL_MCP_ENABLED_TOOLSETS value(s): ${unknown.join(", ")}. Supported toolsets: ${mcpToolsets.join(", ")}.`, + ); + } + return enabled; +} + function disabledMcpToolsetsFromEnv() { const raw = process.env.KERNEL_MCP_DISABLED_TOOLSETS; if (!raw?.trim()) return new Set(); @@ -126,10 +153,14 @@ function disabledMcpToolsetsFromEnv() { } function toolsetEnabled( + enabledToolsets: Set | undefined, disabledToolsets: Set, toolset: McpToolset, ) { - return !disabledToolsets.has(toolset); + return ( + (enabledToolsets === undefined || enabledToolsets.has(toolset)) && + !disabledToolsets.has(toolset) + ); } export function registerMcpCapabilities( @@ -139,6 +170,7 @@ export function registerMcpCapabilities( dependencies = defaultMcpDependencies, }: McpRegistrationOptions = {}, ) { + const enabledToolsets = enabledMcpToolsetsFromEnv(); const disabledToolsets = disabledMcpToolsetsFromEnv(); registerKernelPrompts(server); @@ -147,7 +179,7 @@ export function registerMcpCapabilities( registerConnectionContextTool(server); for (const [toolset, registerToolset] of mcpToolRegistrations) { - if (toolsetEnabled(disabledToolsets, toolset)) { + if (toolsetEnabled(enabledToolsets, disabledToolsets, toolset)) { registerToolset(server, dependencies); } } @@ -155,7 +187,10 @@ export function registerMcpCapabilities( // Managed Auth remains fully programmatic for every client. MCP Apps support // adds one interactive launcher (plus its app-only implementation tools and // resource) without replacing or narrowing manage_auth_connections. - if (mcpApps && toolsetEnabled(disabledToolsets, "auth_connections")) { + if ( + mcpApps && + toolsetEnabled(enabledToolsets, disabledToolsets, "auth_connections") + ) { registerAuthLoginApp(server); } } From 0187c781a9a681f2636bb078e69e383a89f6f546 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sat, 22 Aug 2026 12:38:00 +0000 Subject: [PATCH 09/29] Fix benchmark runtime configuration --- benchmarks/harbor/bin/start-kernel-mcp-server | 3 +++ benchmarks/harbor/clawbench/run-control.sh | 1 + 2 files changed, 4 insertions(+) diff --git a/benchmarks/harbor/bin/start-kernel-mcp-server b/benchmarks/harbor/bin/start-kernel-mcp-server index 727c95d..2096588 100755 --- a/benchmarks/harbor/bin/start-kernel-mcp-server +++ b/benchmarks/harbor/bin/start-kernel-mcp-server @@ -14,6 +14,9 @@ chmod 0600 "$key_dir/api-key" redis-server --daemonize yes --bind 127.0.0.1 --port 6379 \ --logfile "$log_dir/redis.log" --dir /tmp +export CLERK_SECRET_KEY=${CLERK_SECRET_KEY:-sk_test_kernel_mcp_benchmark_local_only} +export NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY=${NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY:-pk_test_YmVuY2htYXJrLmNsZXJrLmFjY291bnRzLmRldiQ} + cd /opt/kernel-mcp-server nohup ./node_modules/.bin/next start -p 3002 \ >"$log_dir/server.stdout.log" \ diff --git a/benchmarks/harbor/clawbench/run-control.sh b/benchmarks/harbor/clawbench/run-control.sh index db8a3f2..69ce746 100755 --- a/benchmarks/harbor/clawbench/run-control.sh +++ b/benchmarks/harbor/clawbench/run-control.sh @@ -105,6 +105,7 @@ CLAWBENCH_JUDGE_MODEL=${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro} CLAWBENCH_JUDGE_API_TYPE=${CLAWBENCH_JUDGE_API_TYPE:-openai-completions} ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY:-} ANTHROPIC_AUTH_TOKEN=${ANTHROPIC_AUTH_TOKEN:-} +ANTHROPIC_BASE_URL=${ANTHROPIC_BASE_URL:-} CLAUDE_CODE_OAUTH_TOKEN=${CLAUDE_CODE_OAUTH_TOKEN:-} CLAUDE_FORCE_OAUTH=${CLAUDE_FORCE_OAUTH:-} OPENAI_API_KEY=${OPENAI_API_KEY:-} From 5fa321a7224abd1dda680661bd8d2981edb57120 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sat, 22 Aug 2026 12:50:27 +0000 Subject: [PATCH 10/29] Make benchmark MCP startup deterministic --- benchmarks/harbor/bin/kernel-mcp-local | 2 +- benchmarks/harbor/clawbench/prepare-control.py | 10 ++++++++-- benchmarks/harbor/clawbench/run-control.sh | 2 +- benchmarks/harbor/clawbench/test_control.py | 9 +++++++-- benchmarks/harbor/image/Dockerfile | 2 +- benchmarks/harbor/run-smoke.sh | 2 +- 6 files changed, 19 insertions(+), 8 deletions(-) diff --git a/benchmarks/harbor/bin/kernel-mcp-local b/benchmarks/harbor/bin/kernel-mcp-local index 5b79a8e..9cb8c06 100755 --- a/benchmarks/harbor/bin/kernel-mcp-local +++ b/benchmarks/harbor/bin/kernel-mcp-local @@ -8,6 +8,6 @@ if [ -z "${KERNEL_API_KEY:-}" ] && [ -r "$key_file" ]; then fi : "${KERNEL_API_KEY:?KERNEL_API_KEY is required}" -exec npx -y mcp-remote@0.1.38 \ +exec mcp-remote \ http://127.0.0.1:3002/mcp \ --header "Authorization: Bearer ${KERNEL_API_KEY}" diff --git a/benchmarks/harbor/clawbench/prepare-control.py b/benchmarks/harbor/clawbench/prepare-control.py index 50b58b6..a57dd50 100755 --- a/benchmarks/harbor/clawbench/prepare-control.py +++ b/benchmarks/harbor/clawbench/prepare-control.py @@ -43,6 +43,7 @@ def _add_environment(task_toml: str, *, image: str, server_sha: str, clawbench_s f"KERNEL_MCP_ENABLED_TOOLSETS = {json.dumps(ENABLED_TOOLSETS)}", 'KERNEL_MCP_EXPECTED_PROJECT_ID = "${KERNEL_MCP_BENCHMARK_PROJECT_ID}"', 'KERNEL_API_BASE_URL = "${KERNEL_API_BASE_URL:-}"', + 'REDIS_URL = "redis://127.0.0.1:6379"', ] ) inserted_env = True @@ -99,8 +100,8 @@ def _patch_verifier(test_script: str) -> str: return test_script.replace( verify_marker, verify_marker - + "mkdir -p /data/kernel-mcp\n" - + "cp -a /logs/kernel-mcp/. /data/kernel-mcp/\n" + + "mkdir -p /logs/verifier/kernel-mcp\n" + + "cp -a /logs/kernel-mcp/. /logs/verifier/kernel-mcp/\n" + "/app/src/runtime-server/.venv/bin/python " + "/app/src/harbor/verify-kernel-mcp-control.py\n", 1, @@ -108,10 +109,15 @@ def _patch_verifier(test_script: str) -> str: def _patch_instruction(instruction: str) -> str: + instruction = instruction.replace( + "Use only Playwright MCP browser tools plus reading files", + "Use only Kernel MCP browser-control tools plus reading files", + ) return instruction.rstrip() + """ --- Kernel MCP benchmark arm: +- Wait for the `kernel` MCP server to finish initializing before starting. In Claude Code, call `WaitForMcpServers` if it is still pending; do not conclude that the tools are unavailable while it initializes. - Call `get_connection_context` once before taking any browser action. - Read `./my-info/kernel_browser.json` and use its existing `session_id` for every `execute_playwright_code` or `computer_action` call. - Do not create, list, update, or delete browsers. Browser lifecycle tools are intentionally unavailable. diff --git a/benchmarks/harbor/clawbench/run-control.sh b/benchmarks/harbor/clawbench/run-control.sh index 69ce746..8bf985d 100755 --- a/benchmarks/harbor/clawbench/run-control.sh +++ b/benchmarks/harbor/clawbench/run-control.sh @@ -107,7 +107,7 @@ ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY:-} ANTHROPIC_AUTH_TOKEN=${ANTHROPIC_AUTH_TOKEN:-} ANTHROPIC_BASE_URL=${ANTHROPIC_BASE_URL:-} CLAUDE_CODE_OAUTH_TOKEN=${CLAUDE_CODE_OAUTH_TOKEN:-} -CLAUDE_FORCE_OAUTH=${CLAUDE_FORCE_OAUTH:-} +CLAUDE_FORCE_OAUTH=${CLAUDE_FORCE_OAUTH:-false} OPENAI_API_KEY=${OPENAI_API_KEY:-} EOF chmod 0600 "$runtime_env" diff --git a/benchmarks/harbor/clawbench/test_control.py b/benchmarks/harbor/clawbench/test_control.py index d60a0d1..cf374c0 100644 --- a/benchmarks/harbor/clawbench/test_control.py +++ b/benchmarks/harbor/clawbench/test_control.py @@ -29,7 +29,9 @@ def test_transforms_playwright_control_into_kernel_mcp_arm(self) -> None: step = task / "steps" / "run" (step / "workdir").mkdir(parents=True) (step / "tests").mkdir() - (step / "instruction.md").write_text("Complete the browser task.\n") + (step / "instruction.md").write_text( + "Use only Playwright MCP browser tools plus reading files under ./my-info/.\n" + ) (environment / "harbor").mkdir(parents=True) (environment / "Dockerfile").write_text("FROM python:3.11-slim\n") (task / "task.toml").write_text( @@ -75,15 +77,18 @@ def test_transforms_playwright_control_into_kernel_mcp_arm(self) -> None: 'KERNEL_MCP_ENABLED_TOOLSETS = "playwright computer"', task_toml ) self.assertNotIn("KERNEL_MCP_DISABLED_TOOLSETS", task_toml) + self.assertIn('REDIS_URL = "redis://127.0.0.1:6379"', task_toml) setup = (step / "workdir" / "setup.sh").read_text() self.assertIn("install_clawbench_runtime", setup) self.assertIn("start-kernel-mcp-server", setup) test_script = (step / "tests" / "test.sh").read_text() self.assertIn("verify-kernel-mcp-control.py", test_script) - self.assertIn("/data/kernel-mcp", test_script) + self.assertIn("/logs/verifier/kernel-mcp", test_script) instruction = (step / "instruction.md").read_text() + self.assertIn("WaitForMcpServers", instruction) self.assertIn("existing `session_id`", instruction) + self.assertIn("Use only Kernel MCP browser-control tools", instruction) self.assertIn("PurelyMail-backed credentials", instruction) self.assertIn("Do not use Kernel managed auth", instruction) self.assertTrue((environment / "harbor" / "verify-kernel-mcp-control.py").is_file()) diff --git a/benchmarks/harbor/image/Dockerfile b/benchmarks/harbor/image/Dockerfile index be85f3a..dfc0d5b 100644 --- a/benchmarks/harbor/image/Dockerfile +++ b/benchmarks/harbor/image/Dockerfile @@ -10,7 +10,7 @@ RUN apt-get update \ python3 \ redis-server \ && rm -rf /var/lib/apt/lists/* \ - && npm install --global bun@1.3.3 + && npm install --global bun@1.3.3 mcp-remote@0.1.38 COPY --from=ghcr.io/astral-sh/uv:0.11.6 /uv /usr/local/bin/uv diff --git a/benchmarks/harbor/run-smoke.sh b/benchmarks/harbor/run-smoke.sh index 7e1571a..c011da6 100755 --- a/benchmarks/harbor/run-smoke.sh +++ b/benchmarks/harbor/run-smoke.sh @@ -78,7 +78,7 @@ KERNEL_PROJECT=${KERNEL_PROJECT:-} ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY:-} ANTHROPIC_AUTH_TOKEN=${ANTHROPIC_AUTH_TOKEN:-} CLAUDE_CODE_OAUTH_TOKEN=${CLAUDE_CODE_OAUTH_TOKEN:-} -CLAUDE_FORCE_OAUTH=${CLAUDE_FORCE_OAUTH:-} +CLAUDE_FORCE_OAUTH=${CLAUDE_FORCE_OAUTH:-false} OPENAI_API_KEY=${OPENAI_API_KEY:-} EOF chmod 0600 "$runtime_env" From eb1ffd8e0c57ec10e823c8b4dc2c977f661603b7 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sat, 22 Aug 2026 13:11:01 +0000 Subject: [PATCH 11/29] Verify ClawBench same-session execution --- .../harbor/clawbench/prepare-control.py | 1 + benchmarks/harbor/clawbench/test_control.py | 14 ++++++++ benchmarks/harbor/clawbench/verify-control.py | 36 ++++++++++++++----- 3 files changed, 43 insertions(+), 8 deletions(-) diff --git a/benchmarks/harbor/clawbench/prepare-control.py b/benchmarks/harbor/clawbench/prepare-control.py index a57dd50..98ef4b2 100755 --- a/benchmarks/harbor/clawbench/prepare-control.py +++ b/benchmarks/harbor/clawbench/prepare-control.py @@ -122,6 +122,7 @@ def _patch_instruction(instruction: str) -> str: - Read `./my-info/kernel_browser.json` and use its existing `session_id` for every `execute_playwright_code` or `computer_action` call. - Do not create, list, update, or delete browsers. Browser lifecycle tools are intentionally unavailable. - Use Kernel MCP for all browser interaction. Do not use Playwright MCP or a direct CDP client. +- Interact through visible page navigation and DOM/UI actions. Do not call `fetch`, `XMLHttpRequest`, Playwright request APIs, or other direct HTTP clients inside `execute_playwright_code`. - Use the PurelyMail-backed credentials already provided under `./my-info/` when the task requires an account. - Do not use Kernel managed auth, create an auth connection, or start a hosted login flow. - Complete and submit the task through the existing browser, then stop. diff --git a/benchmarks/harbor/clawbench/test_control.py b/benchmarks/harbor/clawbench/test_control.py index cf374c0..f2f84f8 100644 --- a/benchmarks/harbor/clawbench/test_control.py +++ b/benchmarks/harbor/clawbench/test_control.py @@ -91,6 +91,7 @@ def test_transforms_playwright_control_into_kernel_mcp_arm(self) -> None: self.assertIn("Use only Kernel MCP browser-control tools", instruction) self.assertIn("PurelyMail-backed credentials", instruction) self.assertIn("Do not use Kernel managed auth", instruction) + self.assertIn("Do not call `fetch`", instruction) self.assertTrue((environment / "harbor" / "verify-kernel-mcp-control.py").is_file()) @@ -149,6 +150,7 @@ def test_accepts_successful_calls_on_precreated_session(self) -> None: "same_session", "no_playwright_mcp", "no_forbidden_kernel_tools", + "no_direct_http_automation", ): self.assertTrue(result[key], key) @@ -160,6 +162,18 @@ def test_rejects_another_session(self) -> None: ) self.assertFalse(result["same_session"]) + def test_rejects_direct_http_inside_playwright_code(self) -> None: + trajectory = self.trajectory() + trajectory["steps"][0]["tool_calls"][1]["arguments"]["code"] = ( + "return await page.evaluate(() => fetch('/api'))" + ) + result = verify.validate_control( + trajectory, + expected_session_id="session-123", + expected_project_id="project-123", + ) + self.assertFalse(result["no_direct_http_automation"]) + def test_rejects_playwright_mcp_and_lifecycle_tools(self) -> None: trajectory = self.trajectory() trajectory["steps"][0]["tool_calls"].extend( diff --git a/benchmarks/harbor/clawbench/verify-control.py b/benchmarks/harbor/clawbench/verify-control.py index 681dac8..395133d 100755 --- a/benchmarks/harbor/clawbench/verify-control.py +++ b/benchmarks/harbor/clawbench/verify-control.py @@ -4,6 +4,7 @@ import ast import json import os +import re from pathlib import Path from typing import Any @@ -121,7 +122,13 @@ def validate_control( duplicate_observations: list[Any] = [] error_observations: list[Any] = [] context_scope_valid = bool(expected_project_id) - same_session = bool(expected_session_id and browser_calls) + same_session = bool(expected_session_id and browser_calls) and all( + isinstance(call.get("arguments"), dict) + and call["arguments"].get("session_id") == expected_session_id + for call in browser_calls + ) + successful_context_calls = 0 + successful_browser_calls = 0 for call in context_calls + browser_calls: call_id = call.get("tool_call_id") @@ -143,16 +150,26 @@ def validate_control( and scope.get("kind") == "project" and scope.get("project_id") == expected_project_id ) + successful_context_calls += 1 elif call.get("function_name") in BROWSER_TOOLS: - arguments = call.get("arguments") - same_session = same_session and ( - isinstance(arguments, dict) - and arguments.get("session_id") == expected_session_id - ) + successful_browser_calls += 1 - observations_valid = bool(context_calls and browser_calls) and not ( - missing_observations or duplicate_observations or error_observations + observations_valid = ( + successful_context_calls > 0 + and successful_browser_calls > 0 + and not (missing_observations or duplicate_observations) ) + direct_http_patterns = re.compile( + r"\bfetch\s*\(|\bXMLHttpRequest\b|\b(?:page|context)\.request\b|\brequest\.(?:get|post|put|patch|delete)\s*\(", + re.IGNORECASE, + ) + direct_http_calls = [ + call + for call in browser_calls + if call.get("function_name") == "mcp__kernel__execute_playwright_code" + and isinstance(call.get("arguments"), dict) + and direct_http_patterns.search(str(call["arguments"].get("code", ""))) + ] return { "context_called": bool(context_calls), "browser_control_called": bool(browser_calls), @@ -161,9 +178,11 @@ def validate_control( "same_session": same_session, "no_playwright_mcp": not playwright_calls, "no_forbidden_kernel_tools": not forbidden_calls, + "no_direct_http_automation": not direct_http_calls, "missing_observations": missing_observations, "duplicate_observations": duplicate_observations, "error_observations": error_observations, + "direct_http_calls": [call.get("tool_call_id") for call in direct_http_calls], "kernel_tool_calls": [ { "tool_call_id": call.get("tool_call_id"), @@ -200,6 +219,7 @@ def main() -> int: "kernel_mcp_same_session": atif["same_session"], "no_playwright_mcp": atif["no_playwright_mcp"], "no_forbidden_kernel_tools": atif["no_forbidden_kernel_tools"], + "no_direct_http_automation": atif["no_direct_http_automation"], "kernel_mcp_source_sha": bool( manifest and manifest.get("kernel_mcp_server_sha") == os.environ.get("KERNEL_MCP_SOURCE_SHA") From 6ecdcb29d461a2a949c9ee44cb9b1aa9aba11dc3 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sat, 22 Aug 2026 13:14:52 +0000 Subject: [PATCH 12/29] Forward benchmark API base URL --- benchmarks/harbor/clawbench/prepare-control.py | 2 +- benchmarks/harbor/clawbench/test_control.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/benchmarks/harbor/clawbench/prepare-control.py b/benchmarks/harbor/clawbench/prepare-control.py index 98ef4b2..040a9d5 100755 --- a/benchmarks/harbor/clawbench/prepare-control.py +++ b/benchmarks/harbor/clawbench/prepare-control.py @@ -42,7 +42,7 @@ def _add_environment(task_toml: str, *, image: str, server_sha: str, clawbench_s f"CLAWBENCH_SOURCE_SHA = {json.dumps(clawbench_sha)}", f"KERNEL_MCP_ENABLED_TOOLSETS = {json.dumps(ENABLED_TOOLSETS)}", 'KERNEL_MCP_EXPECTED_PROJECT_ID = "${KERNEL_MCP_BENCHMARK_PROJECT_ID}"', - 'KERNEL_API_BASE_URL = "${KERNEL_API_BASE_URL:-}"', + 'API_BASE_URL = "${KERNEL_API_BASE_URL:-}"', 'REDIS_URL = "redis://127.0.0.1:6379"', ] ) diff --git a/benchmarks/harbor/clawbench/test_control.py b/benchmarks/harbor/clawbench/test_control.py index f2f84f8..4f1bf82 100644 --- a/benchmarks/harbor/clawbench/test_control.py +++ b/benchmarks/harbor/clawbench/test_control.py @@ -77,6 +77,8 @@ def test_transforms_playwright_control_into_kernel_mcp_arm(self) -> None: 'KERNEL_MCP_ENABLED_TOOLSETS = "playwright computer"', task_toml ) self.assertNotIn("KERNEL_MCP_DISABLED_TOOLSETS", task_toml) + self.assertIn('API_BASE_URL = "${KERNEL_API_BASE_URL:-}"', task_toml) + self.assertNotIn("KERNEL_API_BASE_URL =", task_toml) self.assertIn('REDIS_URL = "redis://127.0.0.1:6379"', task_toml) setup = (step / "workdir" / "setup.sh").read_text() From fa1f8f5ceaa8a3d9ca4b59211144524b9c109c45 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sat, 22 Aug 2026 18:21:45 +0000 Subject: [PATCH 13/29] Enable stealth for ClawBench control --- benchmarks/harbor/clawbench/run-control.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/harbor/clawbench/run-control.sh b/benchmarks/harbor/clawbench/run-control.sh index 8bf985d..61dfd12 100755 --- a/benchmarks/harbor/clawbench/run-control.sh +++ b/benchmarks/harbor/clawbench/run-control.sh @@ -73,7 +73,7 @@ uv --directory "$clawbench_repo" run clawbench-harbor-adapt \ --output-dir "$dataset" \ --task-ids "$task_id" \ --browser-runtime kernel \ - --browser-runtime-options '{"stealth": false}' \ + --browser-runtime-options '{"stealth": true}' \ --overwrite task_dir=$(find "$dataset" -mindepth 1 -maxdepth 1 -type d | head -1) From 405564e49a5939b3f3daf7412ce9d46c49f734b5 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sat, 22 Aug 2026 18:22:59 +0000 Subject: [PATCH 14/29] Pin corrected ClawBench evaluator --- benchmarks/harbor/README.md | 2 +- benchmarks/harbor/clawbench/run-control.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/harbor/README.md b/benchmarks/harbor/README.md index 5ff8de6..e5bd2a4 100644 --- a/benchmarks/harbor/README.md +++ b/benchmarks/harbor/README.md @@ -65,7 +65,7 @@ export CLAWBENCH_REPO=../ClawBench v2-1134-chapter-finder-redcross ``` -The ClawBench checkout must contain commit `6efb04e`, from `kernel/ClawBench` PR #1. The generated task: +The ClawBench checkout must contain commit `bf6d1ff`, from `kernel/ClawBench` PR #1. The generated task: - exposes `get_connection_context`, `execute_playwright_code`, and `computer_action` - disables browser lifecycle and managed-auth toolsets diff --git a/benchmarks/harbor/clawbench/run-control.sh b/benchmarks/harbor/clawbench/run-control.sh index 61dfd12..784a5c8 100755 --- a/benchmarks/harbor/clawbench/run-control.sh +++ b/benchmarks/harbor/clawbench/run-control.sh @@ -14,7 +14,7 @@ repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd) benchmark_dir="$repo_root/benchmarks/harbor" image_env="$benchmark_dir/.image.env" clawbench_repo=${CLAWBENCH_REPO:-$repo_root/../ClawBench} -clawbench_ref=${CLAWBENCH_REF:-6efb04e49efc44f36fa03c8be3bcdb3ef091434f} +clawbench_ref=${CLAWBENCH_REF:-bf6d1ff822c80c3cbb086208955b78fe7c9e9e9d} [[ -f "$image_env" ]] || { echo "Missing $image_env; run benchmarks/harbor/build-image.sh first" >&2 From a233743bf567095c776b574aa79644f5a613df60 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sat, 22 Aug 2026 21:17:57 +0000 Subject: [PATCH 15/29] Pin stop-aware ClawBench runtime --- benchmarks/harbor/README.md | 2 +- benchmarks/harbor/clawbench/run-control.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/harbor/README.md b/benchmarks/harbor/README.md index e5bd2a4..6c442b3 100644 --- a/benchmarks/harbor/README.md +++ b/benchmarks/harbor/README.md @@ -65,7 +65,7 @@ export CLAWBENCH_REPO=../ClawBench v2-1134-chapter-finder-redcross ``` -The ClawBench checkout must contain commit `bf6d1ff`, from `kernel/ClawBench` PR #1. The generated task: +The ClawBench checkout must contain commit `4f39b26`, from `kernel/ClawBench` PR #1. The generated task: - exposes `get_connection_context`, `execute_playwright_code`, and `computer_action` - disables browser lifecycle and managed-auth toolsets diff --git a/benchmarks/harbor/clawbench/run-control.sh b/benchmarks/harbor/clawbench/run-control.sh index 784a5c8..5427ac6 100755 --- a/benchmarks/harbor/clawbench/run-control.sh +++ b/benchmarks/harbor/clawbench/run-control.sh @@ -14,7 +14,7 @@ repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd) benchmark_dir="$repo_root/benchmarks/harbor" image_env="$benchmark_dir/.image.env" clawbench_repo=${CLAWBENCH_REPO:-$repo_root/../ClawBench} -clawbench_ref=${CLAWBENCH_REF:-bf6d1ff822c80c3cbb086208955b78fe7c9e9e9d} +clawbench_ref=${CLAWBENCH_REF:-4f39b269abaab26cb886b643c0cfe6dde1b78698} [[ -f "$image_env" ]] || { echo "Missing $image_env; run benchmarks/harbor/build-image.sh first" >&2 From 4f850caf59f8c814c26f557f52ba80aeec7d5fdb Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sat, 22 Aug 2026 21:24:21 +0000 Subject: [PATCH 16/29] Accept intercepted terminal tool calls --- benchmarks/harbor/clawbench/test_control.py | 24 +++++++++++++ benchmarks/harbor/clawbench/verify-control.py | 35 ++++++++++++++++++- 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/benchmarks/harbor/clawbench/test_control.py b/benchmarks/harbor/clawbench/test_control.py index 4f1bf82..c2f2eb5 100644 --- a/benchmarks/harbor/clawbench/test_control.py +++ b/benchmarks/harbor/clawbench/test_control.py @@ -156,6 +156,30 @@ def test_accepts_successful_calls_on_precreated_session(self) -> None: ): self.assertTrue(result[key], key) + def test_accepts_missing_terminal_observation_after_interception(self) -> None: + trajectory = self.trajectory() + trajectory["steps"][0]["tool_calls"].append( + { + "tool_call_id": "computer-final", + "function_name": "mcp__kernel__computer_action", + "arguments": { + "session_id": "session-123", + "actions": [{"type": "click_mouse", "x": 10, "y": 10}], + }, + } + ) + result = verify.validate_control( + trajectory, + expected_session_id="session-123", + expected_project_id="project-123", + allowed_missing_observation_ids={"computer-final"}, + ) + self.assertTrue(result["observations_valid"]) + self.assertEqual( + result["expected_interrupted_observations"], ["computer-final"] + ) + self.assertEqual(result["unexpected_missing_observations"], []) + def test_rejects_another_session(self) -> None: result = verify.validate_control( self.trajectory("session-other"), diff --git a/benchmarks/harbor/clawbench/verify-control.py b/benchmarks/harbor/clawbench/verify-control.py index 395133d..db476a5 100755 --- a/benchmarks/harbor/clawbench/verify-control.py +++ b/benchmarks/harbor/clawbench/verify-control.py @@ -102,6 +102,7 @@ def validate_control( *, expected_session_id: str, expected_project_id: str, + allowed_missing_observation_ids: set[str] | None = None, ) -> dict[str, Any]: trajectory = trajectory or {} calls = _calls(trajectory) @@ -154,10 +155,14 @@ def validate_control( elif call.get("function_name") in BROWSER_TOOLS: successful_browser_calls += 1 + allowed_missing = allowed_missing_observation_ids or set() + unexpected_missing_observations = [ + call_id for call_id in missing_observations if call_id not in allowed_missing + ] observations_valid = ( successful_context_calls > 0 and successful_browser_calls > 0 - and not (missing_observations or duplicate_observations) + and not (unexpected_missing_observations or duplicate_observations) ) direct_http_patterns = re.compile( r"\bfetch\s*\(|\bXMLHttpRequest\b|\b(?:page|context)\.request\b|\brequest\.(?:get|post|put|patch|delete)\s*\(", @@ -180,6 +185,10 @@ def validate_control( "no_forbidden_kernel_tools": not forbidden_calls, "no_direct_http_automation": not direct_http_calls, "missing_observations": missing_observations, + "expected_interrupted_observations": [ + call_id for call_id in missing_observations if call_id in allowed_missing + ], + "unexpected_missing_observations": unexpected_missing_observations, "duplicate_observations": duplicate_observations, "error_observations": error_observations, "direct_http_calls": [call.get("tool_call_id") for call in direct_http_calls], @@ -201,15 +210,33 @@ def main() -> int: lifecycle = read_json(Path("/data/kernel-browser-lifecycle.json")) manifest = read_json(LOGS_DIR / "kernel-mcp" / "run-manifest.json") clawbench_result = read_json(VERIFIER_DIR / "clawbench-result.json") + interception = read_json(Path("/data/interception.json")) + agent_stop = read_json(Path("/data/agent-stop.json")) reward_path = VERIFIER_DIR / "reward.json" reward_metrics = read_json(reward_path) or {} session_id = str((browser or {}).get("session_id") or "") expected_project_id = os.environ.get("KERNEL_MCP_EXPECTED_PROJECT_ID", "") + all_calls = _calls(trajectory or {}) + browser_calls = [call for call in all_calls if call.get("function_name") in BROWSER_TOOLS] + terminal_call_id = browser_calls[-1].get("tool_call_id") if browser_calls else None + stop_detected_at = (agent_stop or {}).get("stop_detected_at") + intercepted_at = (interception or {}).get("intercepted_at") + stopped_after_interception = bool( + isinstance(stop_detected_at, (int, float)) + and isinstance(intercepted_at, (int, float)) + and 0 <= stop_detected_at - intercepted_at <= 5 + ) + allowed_missing = ( + {terminal_call_id} + if stopped_after_interception and isinstance(terminal_call_id, str) + else set() + ) atif = validate_control( trajectory, expected_session_id=session_id, expected_project_id=expected_project_id, + allowed_missing_observation_ids=allowed_missing, ) checks = { "kernel_mcp_context": atif["context_called"], @@ -242,6 +269,7 @@ def main() -> int: reward_metrics.get("intercepted") == 1 or (clawbench_result or {}).get("intercepted") is True ), + "agent_stopped_after_interception": stopped_after_interception, } infra_ok = all(value for name, value in checks.items() if name != "clawbench_intercepted") checks["infra_ok"] = infra_ok @@ -256,6 +284,11 @@ def main() -> int: "run_manifest": manifest, "browser_lifecycle": lifecycle, "clawbench_result": clawbench_result, + "interception": interception, + "agent_stop": agent_stop, + "stop_latency_seconds": ( + stop_detected_at - intercepted_at if stopped_after_interception else None + ), } (VERIFIER_DIR / "kernel-mcp-control-result.json").write_text(json.dumps(result, indent=2)) return 0 From 99709a92c5571df15e90144e37b682bbe0fe101b Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sat, 22 Aug 2026 21:24:42 +0000 Subject: [PATCH 17/29] Pin stop timing fix --- benchmarks/harbor/README.md | 2 +- benchmarks/harbor/clawbench/run-control.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/harbor/README.md b/benchmarks/harbor/README.md index 6c442b3..832d2c5 100644 --- a/benchmarks/harbor/README.md +++ b/benchmarks/harbor/README.md @@ -65,7 +65,7 @@ export CLAWBENCH_REPO=../ClawBench v2-1134-chapter-finder-redcross ``` -The ClawBench checkout must contain commit `4f39b26`, from `kernel/ClawBench` PR #1. The generated task: +The ClawBench checkout must contain commit `6cf9dc5`, from `kernel/ClawBench` PR #1. The generated task: - exposes `get_connection_context`, `execute_playwright_code`, and `computer_action` - disables browser lifecycle and managed-auth toolsets diff --git a/benchmarks/harbor/clawbench/run-control.sh b/benchmarks/harbor/clawbench/run-control.sh index 5427ac6..faa6a2e 100755 --- a/benchmarks/harbor/clawbench/run-control.sh +++ b/benchmarks/harbor/clawbench/run-control.sh @@ -14,7 +14,7 @@ repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd) benchmark_dir="$repo_root/benchmarks/harbor" image_env="$benchmark_dir/.image.env" clawbench_repo=${CLAWBENCH_REPO:-$repo_root/../ClawBench} -clawbench_ref=${CLAWBENCH_REF:-4f39b269abaab26cb886b643c0cfe6dde1b78698} +clawbench_ref=${CLAWBENCH_REF:-6cf9dc5c4d5b0ee9ae7d17bb8984691cdfad1796} [[ -f "$image_env" ]] || { echo "Missing $image_env; run benchmarks/harbor/build-image.sh first" >&2 From 7648623e982234cb78e3be9f4752e6ec6dc2451e Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sat, 22 Aug 2026 21:49:35 +0000 Subject: [PATCH 18/29] Keep infrastructure health task-independent --- benchmarks/harbor/clawbench/verify-control.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/benchmarks/harbor/clawbench/verify-control.py b/benchmarks/harbor/clawbench/verify-control.py index db476a5..b5db624 100755 --- a/benchmarks/harbor/clawbench/verify-control.py +++ b/benchmarks/harbor/clawbench/verify-control.py @@ -271,7 +271,11 @@ def main() -> int: ), "agent_stopped_after_interception": stopped_after_interception, } - infra_ok = all(value for name, value in checks.items() if name != "clawbench_intercepted") + infra_ok = all( + value + for name, value in checks.items() + if name not in {"clawbench_intercepted", "agent_stopped_after_interception"} + ) checks["infra_ok"] = infra_ok reward_metrics.update({name: float(value) for name, value in checks.items()}) From 7458bc8288aa221de9273113a6225ac8385388fc Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sun, 23 Aug 2026 13:22:29 +0000 Subject: [PATCH 19/29] Encourage useful Playwright state returns --- src/lib/mcp/tools/playwright.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/mcp/tools/playwright.ts b/src/lib/mcp/tools/playwright.ts index 585bff5..3fd0120 100644 --- a/src/lib/mcp/tools/playwright.ts +++ b/src/lib/mcp/tools/playwright.ts @@ -31,7 +31,7 @@ export function registerPlaywrightTool( code: z .string() .describe( - "Playwright/TypeScript code with `page`, `context`, and `browser` objects in scope; the value you `return` is sent back. Example: `await page.goto('https://example.com'); return await page.title();` Return only what you need — prefer a targeted selector (e.g. `await page.locator('h1').innerText()`) or a region-scoped snapshot (e.g. `await page.locator('main').ariaSnapshot()`) rather than dumping the whole page.", + "Playwright/TypeScript code with `page`, `context`, and `browser` objects in scope; the value you `return` is sent back. Every invocation should return useful page state. After navigation or interaction, return a condensed accessibility snapshot of the relevant region, e.g. `await page.goto('https://example.com'); return await page.locator('main').ariaSnapshot();` or `await page.getByRole('button', { name: 'Submit' }).click(); return await page.locator('main').ariaSnapshot();`. For targeted reads, return a compact value or object. Do not dump the full DOM or body text.", ), session_id: z .string() From cb1a2c54398daa35cf4d2a562d0478585983ec95 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sun, 23 Aug 2026 23:10:24 +0000 Subject: [PATCH 20/29] Strip smoke-task scaffolding, keep only the ClawBench Kernel MCP arm The smoke task duplicated what the ClawBench arm already proves. Drop its task definition, runner, verifier, fixtures, and MCP config, and drop stale ignore entries nothing writes. Document only the ClawBench flow. --- .gitignore | 2 - benchmarks/harbor/README.md | 49 +--- .../trajectory-codex-observation.json | 37 --- .../trajectory-error-observation.json | 37 --- .../trajectory-missing-observation.json | 33 --- .../bin/fixtures/trajectory-positive.json | 37 --- benchmarks/harbor/bin/test_verify_smoke.py | 59 ----- benchmarks/harbor/bin/verify-smoke.py | 227 ------------------ benchmarks/harbor/image/Dockerfile | 1 - benchmarks/harbor/mcp/kernel.json | 7 - benchmarks/harbor/prepare-task.py | 40 --- benchmarks/harbor/run-smoke.sh | 101 -------- benchmarks/harbor/smoke/environment/.gitkeep | 0 .../harbor/smoke/steps/run/instruction.md | 17 -- .../harbor/smoke/steps/run/tests/test.sh | 4 - .../harbor/smoke/steps/run/workdir/setup.sh | 12 - benchmarks/harbor/smoke/task.toml | 52 ---- 17 files changed, 11 insertions(+), 704 deletions(-) delete mode 100644 benchmarks/harbor/bin/fixtures/trajectory-codex-observation.json delete mode 100644 benchmarks/harbor/bin/fixtures/trajectory-error-observation.json delete mode 100644 benchmarks/harbor/bin/fixtures/trajectory-missing-observation.json delete mode 100644 benchmarks/harbor/bin/fixtures/trajectory-positive.json delete mode 100644 benchmarks/harbor/bin/test_verify_smoke.py delete mode 100755 benchmarks/harbor/bin/verify-smoke.py delete mode 100644 benchmarks/harbor/mcp/kernel.json delete mode 100755 benchmarks/harbor/prepare-task.py delete mode 100755 benchmarks/harbor/run-smoke.sh delete mode 100644 benchmarks/harbor/smoke/environment/.gitkeep delete mode 100644 benchmarks/harbor/smoke/steps/run/instruction.md delete mode 100755 benchmarks/harbor/smoke/steps/run/tests/test.sh delete mode 100755 benchmarks/harbor/smoke/steps/run/workdir/setup.sh delete mode 100644 benchmarks/harbor/smoke/task.toml diff --git a/.gitignore b/.gitignore index 4e9fb8f..0240eaf 100644 --- a/.gitignore +++ b/.gitignore @@ -111,8 +111,6 @@ mcp-key.pem # Harbor benchmark runtime data benchmarks/harbor/.image.env -benchmarks/harbor/.run.env -benchmarks/harbor/jobs/ benchmarks/harbor/image/source-sha # TypeScript incremental build cache diff --git a/benchmarks/harbor/README.md b/benchmarks/harbor/README.md index 832d2c5..8646ea2 100644 --- a/benchmarks/harbor/README.md +++ b/benchmarks/harbor/README.md @@ -1,25 +1,18 @@ -# Harbor MCP benchmarks +# Harbor ClawBench benchmark -This directory runs stock Harbor agents against a locally built `kernel-mcp-server` in a single Hypeman sandbox. The smoke task makes two read-only calls through the configured stdio MCP server and writes standard Harbor job artifacts. +This directory runs stock Harbor agents (Claude Code, Codex) against a locally built `kernel-mcp-server` on a ClawBench task in a single Hypeman sandbox. The task starts from the Kernel-backed ClawBench Harbor adaptation, replaces Playwright MCP with the local source-pinned Kernel MCP server, and keeps ClawBench attached to the same pre-created browser. ## Requirements -- Harbor 0.21.0 -- `harbor-hypeman` 0.1.1 +- Harbor 0.21.0 with `harbor-hypeman` 0.1.1 (launched through `uvx`) - [uv](https://docs.astral.sh/uv/) and Hypeman CLI credentials +- A ClawBench checkout containing commit `6cf9dc5` (`kernel/ClawBench` PR #1) - `KERNEL_MCP_BENCHMARK_API_KEY` scoped to an isolated evaluation project - `KERNEL_MCP_BENCHMARK_PROJECT_ID` +- `PURELY_MAIL_API_KEY` and `PURELY_MAIL_DOMAIN` for account-task credentials - `ANTHROPIC_API_KEY` or `CLAUDE_CODE_OAUTH_TOKEN` for Claude Code - `OPENAI_API_KEY` for Codex -`run-smoke.sh` launches the pinned Harbor packages through `uvx`. To install the same versions as a persistent tool instead: - -```bash -uv tool install 'harbor==0.21.0' --with 'harbor-hypeman==0.1.1' -``` - -Set `HARBOR_BIN` only when intentionally testing a different Harbor installation. - ## Build the image ```bash @@ -28,12 +21,12 @@ Set `HARBOR_BIN` only when intentionally testing a different Harbor installation The build uses the current Git SHA, installs dependencies with Bun, runs the production Next.js build, and writes the resulting image reference to the ignored `.image.env` file. Hypeman can report a failed build before the converted image becomes visible; the script performs a bounded 5-minute ready-image check for that case. -## Run the smoke task +## Run the ClawBench Kernel MCP arm ```bash -export KERNEL_MCP_BENCHMARK_PROJECT_ID=project_id -./benchmarks/harbor/run-smoke.sh claude-code -./benchmarks/harbor/run-smoke.sh codex +export CLAWBENCH_REPO=../ClawBench +./benchmarks/harbor/clawbench/run-control.sh claude-code \ + v2-1134-chapter-finder-redcross ``` Defaults: @@ -43,29 +36,9 @@ Defaults: | Claude Code | 2.1.238 | `claude-sonnet-5` | | Codex | 0.120.0 | `gpt-5.6-terra` | -Override models with `CLAUDE_BENCHMARK_MODEL` or `CODEX_BENCHMARK_MODEL`, or test a specific Claude Code release with `CLAUDE_BENCHMARK_VERSION`. Runs have a 10-minute wall-clock limit; change it with `HARBOR_BENCHMARK_TIMEOUT`. - -The output defaults to `/tmp/kernel-mcp-harbor-jobs/`. Each successful trial contains: - -- `steps/run/agent/trajectory.json` in ATIF format -- native agent logs and session data -- server stdout and stderr -- source SHA and Hypeman identity in `run-manifest.json` -- numeric Harbor rewards plus detailed `smoke-result.json` - -The verifier proves local-server use from Harbor's ATIF trajectory: it requires native `mcp__kernel__get_connection_context` and `mcp__kernel__manage_browsers` calls, paired non-error observations, the expected project scope, and the required read-only browser-list arguments. Direct HTTP or custom MCP-client workarounds do not pass. - -## Run the ClawBench Kernel MCP arm - -The ClawBench arm starts from the Kernel-backed Harbor task produced by `clawbench-harbor-adapt`, replaces Playwright MCP with the local source-pinned Kernel MCP server, and keeps ClawBench attached to the same pre-created browser. - -```bash -export CLAWBENCH_REPO=../ClawBench -./benchmarks/harbor/clawbench/run-control.sh claude-code \ - v2-1134-chapter-finder-redcross -``` +Override models with `CLAUDE_BENCHMARK_MODEL` or `CODEX_BENCHMARK_MODEL`. Runs have a 40-minute wall-clock limit; change it with `HARBOR_BENCHMARK_TIMEOUT`. -The ClawBench checkout must contain commit `6cf9dc5`, from `kernel/ClawBench` PR #1. The generated task: +`run-control.sh` adapts one ClawBench task with `clawbench-harbor-adapt`, converts it with `clawbench/prepare-control.py`, and runs it under Harbor. The generated task: - exposes `get_connection_context`, `execute_playwright_code`, and `computer_action` - disables browser lifecycle and managed-auth toolsets diff --git a/benchmarks/harbor/bin/fixtures/trajectory-codex-observation.json b/benchmarks/harbor/bin/fixtures/trajectory-codex-observation.json deleted file mode 100644 index d280135..0000000 --- a/benchmarks/harbor/bin/fixtures/trajectory-codex-observation.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "schema_version": "ATIF-v1.7", - "steps": [ - { - "step_id": 1, - "source": "agent", - "tool_calls": [ - { - "tool_call_id": "ctx-1", - "function_name": "mcp__kernel__get_connection_context", - "arguments": {} - }, - { - "tool_call_id": "browsers-1", - "function_name": "mcp__kernel__manage_browsers", - "arguments": { - "action": "list", - "status": "active", - "limit": 1 - } - } - ], - "observation": { - "results": [ - { - "source_call_id": "ctx-1", - "content": "[{'type': 'text', 'text': '{\"connection_scope\":{\"kind\":\"project\",\"project_id\":\"project-123\"}}'}]" - }, - { - "source_call_id": "browsers-1", - "content": "[{'type': 'text', 'text': '{\"items\":[],\"has_more\":false}'}]" - } - ] - } - } - ] -} diff --git a/benchmarks/harbor/bin/fixtures/trajectory-error-observation.json b/benchmarks/harbor/bin/fixtures/trajectory-error-observation.json deleted file mode 100644 index 99f41ef..0000000 --- a/benchmarks/harbor/bin/fixtures/trajectory-error-observation.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "schema_version": "ATIF-v1.7", - "steps": [ - { - "step_id": 1, - "source": "agent", - "tool_calls": [ - { - "tool_call_id": "ctx-1", - "function_name": "mcp__kernel__get_connection_context", - "arguments": {} - }, - { - "tool_call_id": "browsers-1", - "function_name": "mcp__kernel__manage_browsers", - "arguments": { - "action": "list", - "status": "active", - "limit": 1 - } - } - ], - "observation": { - "results": [ - { - "source_call_id": "ctx-1", - "content": "{\"connection_scope\":{\"kind\":\"project\",\"project_id\":\"project-123\"}}" - }, - { - "source_call_id": "browsers-1", - "content": "{\"isError\":true,\"content\":[{\"type\":\"text\",\"text\":\"request failed\"}]}" - } - ] - } - } - ] -} diff --git a/benchmarks/harbor/bin/fixtures/trajectory-missing-observation.json b/benchmarks/harbor/bin/fixtures/trajectory-missing-observation.json deleted file mode 100644 index c506f2e..0000000 --- a/benchmarks/harbor/bin/fixtures/trajectory-missing-observation.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "schema_version": "ATIF-v1.7", - "steps": [ - { - "step_id": 1, - "source": "agent", - "tool_calls": [ - { - "tool_call_id": "ctx-1", - "function_name": "mcp__kernel__get_connection_context", - "arguments": {} - }, - { - "tool_call_id": "browsers-1", - "function_name": "mcp__kernel__manage_browsers", - "arguments": { - "action": "list", - "status": "active", - "limit": 1 - } - } - ], - "observation": { - "results": [ - { - "source_call_id": "ctx-1", - "content": "{\"connection_scope\":{\"kind\":\"project\",\"project_id\":\"project-123\"}}" - } - ] - } - } - ] -} diff --git a/benchmarks/harbor/bin/fixtures/trajectory-positive.json b/benchmarks/harbor/bin/fixtures/trajectory-positive.json deleted file mode 100644 index a00aa56..0000000 --- a/benchmarks/harbor/bin/fixtures/trajectory-positive.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "schema_version": "ATIF-v1.7", - "steps": [ - { - "step_id": 1, - "source": "agent", - "tool_calls": [ - { - "tool_call_id": "ctx-1", - "function_name": "mcp__kernel__get_connection_context", - "arguments": {} - }, - { - "tool_call_id": "browsers-1", - "function_name": "mcp__kernel__manage_browsers", - "arguments": { - "action": "list", - "status": "active", - "limit": 1 - } - } - ], - "observation": { - "results": [ - { - "source_call_id": "ctx-1", - "content": "{\"type\":\"text\",\"text\":\"{\\\"connection_scope\\\":{\\\"kind\\\":\\\"project\\\",\\\"project_id\\\":\\\"project-123\\\"}}\"}" - }, - { - "source_call_id": "browsers-1", - "content": "{\"type\":\"text\",\"text\":\"{\\\"items\\\":[],\\\"has_more\\\":false}\"}" - } - ] - } - } - ] -} diff --git a/benchmarks/harbor/bin/test_verify_smoke.py b/benchmarks/harbor/bin/test_verify_smoke.py deleted file mode 100644 index 707468b..0000000 --- a/benchmarks/harbor/bin/test_verify_smoke.py +++ /dev/null @@ -1,59 +0,0 @@ -import importlib.util -import json -from pathlib import Path -import unittest - - -MODULE_PATH = Path(__file__).with_name("verify-smoke.py") -SPEC = importlib.util.spec_from_file_location("verify_smoke", MODULE_PATH) -assert SPEC and SPEC.loader -verify_smoke = importlib.util.module_from_spec(SPEC) -SPEC.loader.exec_module(verify_smoke) - -FIXTURES = Path(__file__).with_name("fixtures") - - -def load_fixture(name: str) -> dict: - return json.loads((FIXTURES / name).read_text()) - - -class VerifySmokeTest(unittest.TestCase): - def test_accepts_native_calls_with_paired_observations(self) -> None: - proof = verify_smoke.validate_trajectory( - load_fixture("trajectory-positive.json"), "project-123" - ) - - self.assertTrue(proof["native_calls_present"]) - self.assertTrue(proof["observations_valid"]) - self.assertTrue(proof["context_scope_valid"]) - self.assertTrue(proof["manage_browsers_arguments_valid"]) - - def test_accepts_codex_serialized_observations(self) -> None: - proof = verify_smoke.validate_trajectory( - load_fixture("trajectory-codex-observation.json"), "project-123" - ) - - self.assertTrue(proof["native_calls_present"]) - self.assertTrue(proof["observations_valid"]) - self.assertTrue(proof["context_scope_valid"]) - self.assertTrue(proof["manage_browsers_arguments_valid"]) - - def test_rejects_missing_observation(self) -> None: - proof = verify_smoke.validate_trajectory( - load_fixture("trajectory-missing-observation.json"), "project-123" - ) - - self.assertFalse(proof["observations_valid"]) - self.assertEqual(proof["missing_observations"], ["browsers-1"]) - - def test_rejects_error_observation(self) -> None: - proof = verify_smoke.validate_trajectory( - load_fixture("trajectory-error-observation.json"), "project-123" - ) - - self.assertFalse(proof["observations_valid"]) - self.assertEqual(proof["error_observations"], ["browsers-1"]) - - -if __name__ == "__main__": - unittest.main() diff --git a/benchmarks/harbor/bin/verify-smoke.py b/benchmarks/harbor/bin/verify-smoke.py deleted file mode 100755 index a3a9e85..0000000 --- a/benchmarks/harbor/bin/verify-smoke.py +++ /dev/null @@ -1,227 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import ast -import json -import os -from pathlib import Path -from typing import Any - -LOGS_DIR = Path(os.environ.get("HARBOR_LOGS_DIR", "/logs")) -VERIFIER_DIR = LOGS_DIR / "verifier" -REQUIRED_TOOLS = { - "mcp__kernel__get_connection_context", - "mcp__kernel__manage_browsers", -} - - -def read_json(path: Path) -> dict[str, Any] | None: - try: - value = json.loads(path.read_text()) - except (OSError, json.JSONDecodeError): - return None - return value if isinstance(value, dict) else None - - -def _decode_tool_result_content(content: Any) -> Any: - value = content - for _ in range(6): - if isinstance(value, str): - try: - value = json.loads(value) - except json.JSONDecodeError: - try: - value = ast.literal_eval(value) - except (SyntaxError, ValueError): - return value - continue - if isinstance(value, dict) and value.get("type") == "text": - value = value.get("text") - continue - if ( - isinstance(value, list) - and len(value) == 1 - and isinstance(value[0], dict) - and value[0].get("type") == "text" - ): - value = value[0].get("text") - continue - return value - return value - - -def _contains_error(value: Any) -> bool: - if isinstance(value, dict): - if value.get("is_error") is True or value.get("isError") is True: - return True - if "error" in value and value["error"] not in (None, False, ""): - return True - return any(_contains_error(item) for item in value.values()) - if isinstance(value, list): - return any(_contains_error(item) for item in value) - if isinstance(value, str): - text = value.lstrip().lower() - return text.startswith(("[error]", "error:", "error in ")) - return False - - -def _observation_result_map(trajectory: dict[str, Any]) -> dict[str, list[dict[str, Any]]]: - results: dict[str, list[dict[str, Any]]] = {} - for step in trajectory.get("steps") or []: - if not isinstance(step, dict): - continue - observation = step.get("observation") - if not isinstance(observation, dict): - continue - for result in observation.get("results") or []: - if not isinstance(result, dict): - continue - source_call_id = result.get("source_call_id") - if isinstance(source_call_id, str): - results.setdefault(source_call_id, []).append(result) - return results - - -def _native_tool_calls(trajectory: dict[str, Any]) -> list[dict[str, Any]]: - calls = [] - for step in trajectory.get("steps") or []: - if not isinstance(step, dict): - continue - for call in step.get("tool_calls") or []: - if not isinstance(call, dict): - continue - if call.get("function_name") in REQUIRED_TOOLS: - calls.append(call) - return calls - - -def _manage_browsers_arguments_valid(call: dict[str, Any]) -> bool: - arguments = call.get("arguments") - return ( - isinstance(arguments, dict) - and arguments.get("action") == "list" - and arguments.get("status") == "active" - and arguments.get("limit") == 1 - ) - - -def validate_trajectory( - trajectory: dict[str, Any] | None, expected_project_id: str -) -> dict[str, Any]: - calls = _native_tool_calls(trajectory or {}) - calls_by_name = { - name: [call for call in calls if call.get("function_name") == name] - for name in REQUIRED_TOOLS - } - result_map = _observation_result_map(trajectory or {}) - missing_observations = [] - duplicate_observations = [] - error_observations = [] - context_scope_valid = True - browser_arguments_valid = True - - for call in calls: - call_id = call.get("tool_call_id") - results = result_map.get(call_id, []) if isinstance(call_id, str) else [] - if len(results) == 0: - missing_observations.append(call_id) - continue - if len(results) != 1: - duplicate_observations.append(call_id) - continue - result = results[0] - decoded = _decode_tool_result_content(result.get("content")) - if _contains_error(decoded) or _contains_error(result): - error_observations.append(call_id) - continue - if call.get("function_name") == "mcp__kernel__get_connection_context": - scope = decoded.get("connection_scope") if isinstance(decoded, dict) else None - context_scope_valid = context_scope_valid and ( - bool(expected_project_id) - and isinstance(scope, dict) - and scope.get("kind") == "project" - and scope.get("project_id") == expected_project_id - ) - elif call.get("function_name") == "mcp__kernel__manage_browsers": - browser_arguments_valid = ( - browser_arguments_valid and _manage_browsers_arguments_valid(call) - ) - - native_calls_present = all(calls_by_name[name] for name in REQUIRED_TOOLS) - observations_valid = bool(calls) and not ( - missing_observations or duplicate_observations or error_observations - ) - return { - "native_calls_present": native_calls_present, - "observations_valid": observations_valid, - "context_scope_valid": context_scope_valid - and bool(calls_by_name["mcp__kernel__get_connection_context"]), - "manage_browsers_arguments_valid": browser_arguments_valid - and bool(calls_by_name["mcp__kernel__manage_browsers"]), - "missing_observations": missing_observations, - "duplicate_observations": duplicate_observations, - "error_observations": error_observations, - "tool_calls": [ - { - "tool_call_id": call.get("tool_call_id"), - "name": call.get("function_name"), - "arguments": call.get("arguments"), - } - for call in calls - ], - } - - -def main() -> int: - VERIFIER_DIR.mkdir(parents=True, exist_ok=True) - report = read_json(LOGS_DIR / "artifacts/agent-report.json") - trajectory = read_json(LOGS_DIR / "agent/trajectory.json") - manifest = read_json(LOGS_DIR / "kernel-mcp/run-manifest.json") - expected_project_id = os.environ.get("KERNEL_MCP_EXPECTED_PROJECT_ID", "") - atif = validate_trajectory(trajectory, expected_project_id) - source_sha_matches = bool( - manifest - and manifest.get("kernel_mcp_server_sha") - == os.environ.get("KERNEL_MCP_SOURCE_SHA") - ) - hypeman_identity_present = bool( - manifest and manifest.get("hypeman_instance_name") - ) - - checks = { - "native_mcp_calls": atif["native_calls_present"], - "tool_observations": atif["observations_valid"], - "context_scope": atif["context_scope_valid"], - "manage_browsers_arguments": atif["manage_browsers_arguments_valid"], - "source_sha": source_sha_matches, - "hypeman_identity": hypeman_identity_present, - "server_stdout": (LOGS_DIR / "kernel-mcp/server.stdout.log").is_file(), - "server_stderr": (LOGS_DIR / "kernel-mcp/server.stderr.log").is_file(), - } - reward = 1.0 if all(checks.values()) else 0.0 - result = { - "reward": reward, - "checks": checks, - "atif": atif, - "agent_report": report, - "trajectory": { - "present": trajectory is not None, - "schema_version": (trajectory or {}).get("schema_version"), - "agent": (trajectory or {}).get("agent"), - }, - "run_manifest": manifest, - } - - (VERIFIER_DIR / "reward.txt").write_text(str(reward)) - (VERIFIER_DIR / "reward.json").write_text( - json.dumps( - {"reward": reward, **{name: float(value) for name, value in checks.items()}}, - indent=2, - ) - ) - (VERIFIER_DIR / "smoke-result.json").write_text(json.dumps(result, indent=2)) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/benchmarks/harbor/image/Dockerfile b/benchmarks/harbor/image/Dockerfile index dfc0d5b..9bbc1c8 100644 --- a/benchmarks/harbor/image/Dockerfile +++ b/benchmarks/harbor/image/Dockerfile @@ -28,7 +28,6 @@ RUN KERNEL_CLI_PROD_CLIENT_ID=kernel-mcp-benchmark \ bun run build \ && install -m 0755 benchmarks/harbor/bin/start-kernel-mcp-server /usr/local/bin/start-kernel-mcp-server \ && install -m 0755 benchmarks/harbor/bin/kernel-mcp-local /usr/local/bin/kernel-mcp-local \ - && install -m 0755 benchmarks/harbor/bin/verify-smoke.py /usr/local/bin/verify-kernel-mcp-smoke \ && install -m 0644 benchmarks/harbor/image/source-sha /opt/kernel-mcp-server/SOURCE_SHA ENV NEXT_TELEMETRY_DISABLED=1 diff --git a/benchmarks/harbor/mcp/kernel.json b/benchmarks/harbor/mcp/kernel.json deleted file mode 100644 index 0ce2a9c..0000000 --- a/benchmarks/harbor/mcp/kernel.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "mcpServers": { - "kernel": { - "command": "/app/kernel-mcp-local" - } - } -} diff --git a/benchmarks/harbor/prepare-task.py b/benchmarks/harbor/prepare-task.py deleted file mode 100755 index 6ad8107..0000000 --- a/benchmarks/harbor/prepare-task.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import argparse -import os -import shutil -from pathlib import Path - - -def main() -> int: - parser = argparse.ArgumentParser() - parser.add_argument("output", type=Path) - args = parser.parse_args() - - source = Path(__file__).parent / "smoke" - output = args.output.resolve() - image = os.environ["KERNEL_MCP_BENCHMARK_IMAGE"] - source_sha = os.environ["KERNEL_MCP_SOURCE_SHA"] - project_id = os.environ["KERNEL_MCP_BENCHMARK_PROJECT_ID"] - - if output.exists(): - shutil.rmtree(output) - shutil.copytree(source, output) - - config_path = output / "task.toml" - config = config_path.read_text() - config = config.replace("${KERNEL_MCP_BENCHMARK_IMAGE}", image) - config = config.replace("${KERNEL_MCP_SOURCE_SHA}", source_sha) - config = config.replace("${KERNEL_MCP_BENCHMARK_PROJECT_ID}", project_id) - config_path.write_text(config) - - wrapper = Path(__file__).parent / "bin" / "kernel-mcp-local" - runtime_wrapper = output / "steps" / "run" / "workdir" / "kernel-mcp-local" - shutil.copy2(wrapper, runtime_wrapper) - runtime_wrapper.chmod(0o755) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/benchmarks/harbor/run-smoke.sh b/benchmarks/harbor/run-smoke.sh deleted file mode 100755 index c011da6..0000000 --- a/benchmarks/harbor/run-smoke.sh +++ /dev/null @@ -1,101 +0,0 @@ -#!/bin/bash -set -euo pipefail - -usage() { - echo "usage: $0 [job-name] [jobs-dir]" >&2 - exit 2 -} - -agent=${1:-} -[[ "$agent" == "claude-code" || "$agent" == "codex" ]] || usage - -repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) -benchmark_dir="$repo_root/benchmarks/harbor" -image_env="$benchmark_dir/.image.env" -[[ -f "$image_env" ]] || { - echo "Missing $image_env; run benchmarks/harbor/build-image.sh first" >&2 - exit 1 -} - -set -a -source "$image_env" -set +a - -: "${KERNEL_MCP_BENCHMARK_API_KEY:?KERNEL_MCP_BENCHMARK_API_KEY is required}" -: "${KERNEL_MCP_BENCHMARK_PROJECT_ID:?KERNEL_MCP_BENCHMARK_PROJECT_ID is required}" - -case "$agent" in - claude-code) - if [[ -z "${ANTHROPIC_API_KEY:-}" && -z "${ANTHROPIC_AUTH_TOKEN:-}" && -z "${CLAUDE_CODE_OAUTH_TOKEN:-}" ]]; then - echo "ANTHROPIC_API_KEY, ANTHROPIC_AUTH_TOKEN, or CLAUDE_CODE_OAUTH_TOKEN is required" >&2 - exit 1 - fi - if [[ -z "${ANTHROPIC_API_KEY:-}" && -z "${ANTHROPIC_AUTH_TOKEN:-}" && -n "${CLAUDE_CODE_OAUTH_TOKEN:-}" ]]; then - ANTHROPIC_AUTH_TOKEN=$CLAUDE_CODE_OAUTH_TOKEN - CLAUDE_FORCE_OAUTH=1 - export ANTHROPIC_AUTH_TOKEN CLAUDE_FORCE_OAUTH - fi - model=${CLAUDE_BENCHMARK_MODEL:-claude-sonnet-5} - version=${CLAUDE_BENCHMARK_VERSION:-2.1.238} - ;; - codex) - : "${OPENAI_API_KEY:?OPENAI_API_KEY is required}" - model=${CODEX_BENCHMARK_MODEL:-gpt-5.6-terra} - version=0.120.0 - ;; -esac - -if [[ -n "${HARBOR_BIN:-}" ]]; then - harbor_command=("$HARBOR_BIN") -elif command -v uvx >/dev/null 2>&1; then - harbor_command=( - uvx - --from "harbor==0.21.0" - --with "harbor-hypeman==0.1.1" - harbor - ) -else - echo "uvx not found; install uv or set HARBOR_BIN" >&2 - exit 1 -fi - -job_name=${2:-${agent}-smoke-$(date -u +%Y%m%dT%H%M%SZ)} -jobs_dir=${3:-${HARBOR_JOBS_DIR:-/tmp/kernel-mcp-harbor-jobs}} -runtime_task=$(mktemp -d) -runtime_env=$(mktemp) -trap 'rm -rf "$runtime_task"; rm -f "$runtime_env"' EXIT - -export KERNEL_MCP_BENCHMARK_IMAGE KERNEL_MCP_SOURCE_SHA -python3 "$benchmark_dir/prepare-task.py" "$runtime_task" - -cat >"$runtime_env" <"$key_dir/api-key" -chmod 0600 "$key_dir/api-key" - -/usr/local/bin/start-kernel-mcp-server -printf 'ready\n' >/logs/kernel-mcp/ready -rm -f /app/setup.sh diff --git a/benchmarks/harbor/smoke/task.toml b/benchmarks/harbor/smoke/task.toml deleted file mode 100644 index 055bd16..0000000 --- a/benchmarks/harbor/smoke/task.toml +++ /dev/null @@ -1,52 +0,0 @@ -schema_version = "1.4" -source = "kernel-mcp-benchmarks" -artifacts = ["/logs/kernel-mcp"] -multi_step_reward_strategy = "final" - -[task] -name = "kernel-mcp/local-connection-smoke" -description = "Verify an agent can call a locally running Kernel MCP server" -keywords = ["kernel", "mcp", "harbor", "smoke"] - -[metadata] -benchmark = "kernel-mcp-local-connection" -kernel_mcp_server_sha = "${KERNEL_MCP_SOURCE_SHA}" - -[environment] -docker_image = "${KERNEL_MCP_BENCHMARK_IMAGE}" -network_mode = "public" -workdir = "/app" -build_timeout_sec = 1200.0 -cpus = 2 -memory_mb = 4096 -storage_mb = 8192 - -[environment.env] -KERNEL_API_KEY = "${KERNEL_MCP_BENCHMARK_API_KEY}" -KERNEL_MCP_BENCHMARK_IMAGE = "${KERNEL_MCP_BENCHMARK_IMAGE}" -KERNEL_MCP_SOURCE_SHA = "${KERNEL_MCP_SOURCE_SHA}" -KERNEL_MCP_EXPECTED_PROJECT_ID = "${KERNEL_MCP_BENCHMARK_PROJECT_ID}" -KERNEL_PROJECT = "${KERNEL_MCP_BENCHMARK_PROJECT_ID}" -API_BASE_URL = "${KERNEL_API_BASE_URL:-https://api.onkernel.com}" -REDIS_URL = "redis://127.0.0.1:6379" -CLERK_SECRET_KEY = "sk_test_kernel_mcp_benchmark_local_only" -NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY = "pk_test_YmVuY2htYXJrLmNsZXJrLmFjY291bnRzLmRldiQ" -MANAGED_AUTH_APP_ORIGIN = "http://127.0.0.1:3002" -NEXT_TELEMETRY_DISABLED = "1" - -[[steps]] -name = "run" - -[steps.agent] -timeout_sec = 300.0 - -[steps.verifier] -timeout_sec = 60.0 - -[steps.healthcheck] -command = "test -s /logs/kernel-mcp/ready && curl -sS -o /dev/null http://127.0.0.1:3002/mcp" -interval_sec = 2.0 -timeout_sec = 5.0 -start_period_sec = 1.0 -start_interval_sec = 1.0 -retries = 10 From 0bb84c426205d4c47709955e7c60cc82d33055a4 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sun, 23 Aug 2026 23:56:21 +0000 Subject: [PATCH 21/29] Make ClawBench benchmark DOM-only --- benchmarks/harbor/README.md | 16 +++++--- .../harbor/clawbench/prepare-control.py | 8 ++-- benchmarks/harbor/clawbench/run-control.sh | 38 +++++++++++-------- benchmarks/harbor/clawbench/test_control.py | 21 +++++----- benchmarks/harbor/clawbench/verify-control.py | 8 ++-- 5 files changed, 52 insertions(+), 39 deletions(-) diff --git a/benchmarks/harbor/README.md b/benchmarks/harbor/README.md index 8646ea2..1453451 100644 --- a/benchmarks/harbor/README.md +++ b/benchmarks/harbor/README.md @@ -6,7 +6,7 @@ This directory runs stock Harbor agents (Claude Code, Codex) against a locally b - Harbor 0.21.0 with `harbor-hypeman` 0.1.1 (launched through `uvx`) - [uv](https://docs.astral.sh/uv/) and Hypeman CLI credentials -- A ClawBench checkout containing commit `6cf9dc5` (`kernel/ClawBench` PR #1) +- A ClawBench checkout containing commit `df6743f` (`kernel/ClawBench` PR #1) - `KERNEL_MCP_BENCHMARK_API_KEY` scoped to an isolated evaluation project - `KERNEL_MCP_BENCHMARK_PROJECT_ID` - `PURELY_MAIL_API_KEY` and `PURELY_MAIL_DOMAIN` for account-task credentials @@ -34,14 +34,20 @@ Defaults: | Agent | Version | Model | | ----------- | ------: | ----------------- | | Claude Code | 2.1.238 | `claude-sonnet-5` | -| Codex | 0.120.0 | `gpt-5.6-terra` | +| Codex | 0.120.0 | `gpt-5.6-luna` | Override models with `CLAUDE_BENCHMARK_MODEL` or `CODEX_BENCHMARK_MODEL`. Runs have a 40-minute wall-clock limit; change it with `HARBOR_BENCHMARK_TIMEOUT`. -`run-control.sh` adapts one ClawBench task with `clawbench-harbor-adapt`, converts it with `clawbench/prepare-control.py`, and runs it under Harbor. The generated task: +Pass `all` instead of a task ID to run the complete suite, and set `HARBOR_N_CONCURRENT` to control parallelism: -- exposes `get_connection_context`, `execute_playwright_code`, and `computer_action` -- disables browser lifecycle and managed-auth toolsets +```bash +HARBOR_N_CONCURRENT=10 ./benchmarks/harbor/clawbench/run-control.sh codex all +``` + +`run-control.sh` adapts the selected ClawBench tasks with `clawbench-harbor-adapt`, converts them with `clawbench/prepare-control.py`, and runs them under Harbor. Each generated task: + +- exposes `get_connection_context` and `execute_playwright_code` +- disables coordinate-based computer actions, browser lifecycle, and managed-auth toolsets - instructs the agent to read `./my-info/kernel_browser.json` and use that session ID - instructs account tasks to use the supplied PurelyMail credentials instead of managed auth - verifies ATIF observations, project scope, exact session reuse, ClawBench interception, replay finalization, and browser deletion diff --git a/benchmarks/harbor/clawbench/prepare-control.py b/benchmarks/harbor/clawbench/prepare-control.py index 040a9d5..26fd412 100755 --- a/benchmarks/harbor/clawbench/prepare-control.py +++ b/benchmarks/harbor/clawbench/prepare-control.py @@ -6,7 +6,7 @@ import shutil from pathlib import Path -ENABLED_TOOLSETS = "playwright computer" +ENABLED_TOOLSETS = "playwright" def _drop_mcp_servers(task_toml: str) -> str: @@ -119,9 +119,9 @@ def _patch_instruction(instruction: str) -> str: Kernel MCP benchmark arm: - Wait for the `kernel` MCP server to finish initializing before starting. In Claude Code, call `WaitForMcpServers` if it is still pending; do not conclude that the tools are unavailable while it initializes. - Call `get_connection_context` once before taking any browser action. -- Read `./my-info/kernel_browser.json` and use its existing `session_id` for every `execute_playwright_code` or `computer_action` call. -- Do not create, list, update, or delete browsers. Browser lifecycle tools are intentionally unavailable. -- Use Kernel MCP for all browser interaction. Do not use Playwright MCP or a direct CDP client. +- Read `./my-info/kernel_browser.json` and use its existing `session_id` for every `execute_playwright_code` call. +- Do not create, list, update, or delete browsers. Browser lifecycle tools and `computer_action` are intentionally unavailable. +- Use Kernel MCP `execute_playwright_code` for all browser interaction. Do not use Playwright MCP or a direct CDP client. - Interact through visible page navigation and DOM/UI actions. Do not call `fetch`, `XMLHttpRequest`, Playwright request APIs, or other direct HTTP clients inside `execute_playwright_code`. - Use the PurelyMail-backed credentials already provided under `./my-info/` when the task requires an account. - Do not use Kernel managed auth, create an auth connection, or start a hosted login flow. diff --git a/benchmarks/harbor/clawbench/run-control.sh b/benchmarks/harbor/clawbench/run-control.sh index faa6a2e..3dbd888 100755 --- a/benchmarks/harbor/clawbench/run-control.sh +++ b/benchmarks/harbor/clawbench/run-control.sh @@ -2,7 +2,7 @@ set -euo pipefail usage() { - echo "usage: $0 [task-id] [job-name] [jobs-dir]" >&2 + echo "usage: $0 [task-id|all] [job-name] [jobs-dir]" >&2 exit 2 } @@ -14,7 +14,7 @@ repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd) benchmark_dir="$repo_root/benchmarks/harbor" image_env="$benchmark_dir/.image.env" clawbench_repo=${CLAWBENCH_REPO:-$repo_root/../ClawBench} -clawbench_ref=${CLAWBENCH_REF:-6cf9dc5c4d5b0ee9ae7d17bb8984691cdfad1796} +clawbench_ref=${CLAWBENCH_REF:-df6743fd8abcd09cb7636ef8c310dd4db016162c} [[ -f "$image_env" ]] || { echo "Missing $image_env; run benchmarks/harbor/build-image.sh first" >&2 @@ -59,7 +59,7 @@ case "$agent" in ;; codex) : "${OPENAI_API_KEY:?OPENAI_API_KEY is required}" - model=${CODEX_BENCHMARK_MODEL:-gpt-5.6-terra} + model=${CODEX_BENCHMARK_MODEL:-gpt-5.6-luna} version=${CODEX_BENCHMARK_VERSION:-0.120.0} ;; esac @@ -69,23 +69,29 @@ runtime_env=$(mktemp) trap 'rm -rf "$runtime_root"; rm -f "$runtime_env"' EXIT dataset="$runtime_root/dataset" -uv --directory "$clawbench_repo" run clawbench-harbor-adapt \ - --output-dir "$dataset" \ - --task-ids "$task_id" \ - --browser-runtime kernel \ - --browser-runtime-options '{"stealth": true}' \ +adapt_args=( + --output-dir "$dataset" + --browser-runtime kernel + --browser-runtime-options '{"stealth": true}' --overwrite +) +if [[ "$task_id" != "all" ]]; then + adapt_args+=(--task-ids "$task_id") +fi +uv --directory "$clawbench_repo" run clawbench-harbor-adapt "${adapt_args[@]}" -task_dir=$(find "$dataset" -mindepth 1 -maxdepth 1 -type d | head -1) -[[ -n "$task_dir" ]] || { - echo "ClawBench did not generate task $task_id" >&2 +mapfile -t task_dirs < <(find "$dataset" -mindepth 1 -maxdepth 1 -type d | sort) +((${#task_dirs[@]} > 0)) || { + echo "ClawBench did not generate tasks for $task_id" >&2 exit 1 } -python3 "$benchmark_dir/clawbench/prepare-control.py" "$task_dir" \ - --image "$KERNEL_MCP_BENCHMARK_IMAGE" \ - --server-sha "$KERNEL_MCP_SOURCE_SHA" \ - --clawbench-sha "$clawbench_ref" +for task_dir in "${task_dirs[@]}"; do + python3 "$benchmark_dir/clawbench/prepare-control.py" "$task_dir" \ + --image "$KERNEL_MCP_BENCHMARK_IMAGE" \ + --server-sha "$KERNEL_MCP_SOURCE_SHA" \ + --clawbench-sha "$clawbench_ref" +done export KERNEL_API_KEY=$KERNEL_MCP_BENCHMARK_API_KEY export KERNEL_BASE_URL=${KERNEL_BASE_URL:-https://api.onkernel.com} @@ -126,7 +132,7 @@ timeout --signal=INT --kill-after=30s "${HARBOR_BENCHMARK_TIMEOUT:-40m}" \ --env-file "$runtime_env" \ --job-name "$job_name" \ --jobs-dir "$jobs_dir" \ - --n-concurrent 1 \ + --n-concurrent "${HARBOR_N_CONCURRENT:-1}" \ --max-retries 0 \ --delete \ --yes diff --git a/benchmarks/harbor/clawbench/test_control.py b/benchmarks/harbor/clawbench/test_control.py index c2f2eb5..bb6941b 100644 --- a/benchmarks/harbor/clawbench/test_control.py +++ b/benchmarks/harbor/clawbench/test_control.py @@ -73,9 +73,7 @@ def test_transforms_playwright_control_into_kernel_mcp_arm(self) -> None: self.assertIn('name = "kernel"', task_toml) self.assertIn('command = "/usr/local/bin/kernel-mcp-local"', task_toml) self.assertNotIn("@playwright/mcp", task_toml) - self.assertIn( - 'KERNEL_MCP_ENABLED_TOOLSETS = "playwright computer"', task_toml - ) + self.assertIn('KERNEL_MCP_ENABLED_TOOLSETS = "playwright"', task_toml) self.assertNotIn("KERNEL_MCP_DISABLED_TOOLSETS", task_toml) self.assertIn('API_BASE_URL = "${KERNEL_API_BASE_URL:-}"', task_toml) self.assertNotIn("KERNEL_API_BASE_URL =", task_toml) @@ -90,7 +88,7 @@ def test_transforms_playwright_control_into_kernel_mcp_arm(self) -> None: instruction = (step / "instruction.md").read_text() self.assertIn("WaitForMcpServers", instruction) self.assertIn("existing `session_id`", instruction) - self.assertIn("Use only Kernel MCP browser-control tools", instruction) + self.assertIn("Use Kernel MCP `execute_playwright_code`", instruction) self.assertIn("PurelyMail-backed credentials", instruction) self.assertIn("Do not use Kernel managed auth", instruction) self.assertIn("Do not call `fetch`", instruction) @@ -160,11 +158,11 @@ def test_accepts_missing_terminal_observation_after_interception(self) -> None: trajectory = self.trajectory() trajectory["steps"][0]["tool_calls"].append( { - "tool_call_id": "computer-final", - "function_name": "mcp__kernel__computer_action", + "tool_call_id": "playwright-final", + "function_name": "mcp__kernel__execute_playwright_code", "arguments": { "session_id": "session-123", - "actions": [{"type": "click_mouse", "x": 10, "y": 10}], + "code": "await page.getByRole('button').click()", }, } ) @@ -172,11 +170,11 @@ def test_accepts_missing_terminal_observation_after_interception(self) -> None: trajectory, expected_session_id="session-123", expected_project_id="project-123", - allowed_missing_observation_ids={"computer-final"}, + allowed_missing_observation_ids={"playwright-final"}, ) self.assertTrue(result["observations_valid"]) self.assertEqual( - result["expected_interrupted_observations"], ["computer-final"] + result["expected_interrupted_observations"], ["playwright-final"] ) self.assertEqual(result["unexpected_missing_observations"], []) @@ -214,6 +212,11 @@ def test_rejects_playwright_mcp_and_lifecycle_tools(self) -> None: "function_name": "mcp__kernel__manage_browsers", "arguments": {"action": "list"}, }, + { + "tool_call_id": "computer-action", + "function_name": "mcp__kernel__computer_action", + "arguments": {"session_id": "session-123", "actions": []}, + }, ] ) result = verify.validate_control( diff --git a/benchmarks/harbor/clawbench/verify-control.py b/benchmarks/harbor/clawbench/verify-control.py index b5db624..7660925 100755 --- a/benchmarks/harbor/clawbench/verify-control.py +++ b/benchmarks/harbor/clawbench/verify-control.py @@ -11,11 +11,9 @@ LOGS_DIR = Path(os.environ.get("HARBOR_LOGS_DIR", "/logs")) VERIFIER_DIR = LOGS_DIR / "verifier" CONTEXT_TOOL = "mcp__kernel__get_connection_context" -BROWSER_TOOLS = { - "mcp__kernel__execute_playwright_code", - "mcp__kernel__computer_action", -} +BROWSER_TOOLS = {"mcp__kernel__execute_playwright_code"} FORBIDDEN_KERNEL_TOOLS = { + "mcp__kernel__computer_action", "mcp__kernel__manage_browsers", "mcp__kernel__manage_auth_connections", "mcp__kernel__open_auth_login", @@ -257,7 +255,7 @@ def main() -> int: "kernel_mcp_toolset_allowlist": bool( manifest and set(str(manifest.get("enabled_toolsets", "")).split()) - == {"playwright", "computer"} + == {"playwright"} ), "hypeman_identity": bool(manifest and manifest.get("hypeman_instance_name")), "browser_deleted": bool( From 230331b957709618b8e3d952028c922684f20ba0 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Mon, 24 Aug 2026 00:01:15 +0000 Subject: [PATCH 22/29] Allow full ClawBench suite to finish --- benchmarks/harbor/README.md | 2 +- benchmarks/harbor/clawbench/run-control.sh | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/benchmarks/harbor/README.md b/benchmarks/harbor/README.md index 1453451..a30fed5 100644 --- a/benchmarks/harbor/README.md +++ b/benchmarks/harbor/README.md @@ -36,7 +36,7 @@ Defaults: | Claude Code | 2.1.238 | `claude-sonnet-5` | | Codex | 0.120.0 | `gpt-5.6-luna` | -Override models with `CLAUDE_BENCHMARK_MODEL` or `CODEX_BENCHMARK_MODEL`. Runs have a 40-minute wall-clock limit; change it with `HARBOR_BENCHMARK_TIMEOUT`. +Override models with `CLAUDE_BENCHMARK_MODEL` or `CODEX_BENCHMARK_MODEL`. Single-task runs have a 40-minute wall-clock limit; full-suite runs default to 6 hours. Change either with `HARBOR_BENCHMARK_TIMEOUT`. Pass `all` instead of a task ID to run the complete suite, and set `HARBOR_N_CONCURRENT` to control parallelism: diff --git a/benchmarks/harbor/clawbench/run-control.sh b/benchmarks/harbor/clawbench/run-control.sh index 3dbd888..0406273 100755 --- a/benchmarks/harbor/clawbench/run-control.sh +++ b/benchmarks/harbor/clawbench/run-control.sh @@ -122,7 +122,13 @@ job_name=${3:-kernel-mcp-${agent}-${task_id}-$(date -u +%Y%m%dT%H%M%SZ)} jobs_dir=${4:-${HARBOR_JOBS_DIR:-/tmp/kernel-mcp-clawbench-jobs}} mkdir -p "$jobs_dir" -timeout --signal=INT --kill-after=30s "${HARBOR_BENCHMARK_TIMEOUT:-40m}" \ +if [[ "$task_id" == "all" ]]; then + default_timeout=6h +else + default_timeout=40m +fi + +timeout --signal=INT --kill-after=30s "${HARBOR_BENCHMARK_TIMEOUT:-$default_timeout}" \ uvx --from "harbor==0.21.0" --with "harbor-hypeman==0.1.1" harbor run \ --path "$dataset" \ --agent "$agent" \ From 8cba88ec862a6ffea17fa3985b90ad12b62a3dda Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Mon, 24 Aug 2026 11:49:03 +0000 Subject: [PATCH 23/29] Simplify ClawBench benchmark harness --- benchmarks/harbor/README.md | 73 +++-- benchmarks/harbor/bin/kernel-mcp-local | 5 + benchmarks/harbor/bin/start-kernel-mcp-server | 10 +- .../{prepare-control.py => prepare-task.py} | 37 ++- .../clawbench/{run-control.sh => run.sh} | 10 +- benchmarks/harbor/clawbench/test_control.py | 232 -------------- benchmarks/harbor/clawbench/verify-control.py | 300 ------------------ benchmarks/harbor/clawbench/verify-task.py | 86 +++++ 8 files changed, 170 insertions(+), 583 deletions(-) rename benchmarks/harbor/clawbench/{prepare-control.py => prepare-task.py} (85%) rename benchmarks/harbor/clawbench/{run-control.sh => run.sh} (92%) delete mode 100644 benchmarks/harbor/clawbench/test_control.py delete mode 100755 benchmarks/harbor/clawbench/verify-control.py create mode 100755 benchmarks/harbor/clawbench/verify-task.py diff --git a/benchmarks/harbor/README.md b/benchmarks/harbor/README.md index a30fed5..3748955 100644 --- a/benchmarks/harbor/README.md +++ b/benchmarks/harbor/README.md @@ -1,55 +1,66 @@ -# Harbor ClawBench benchmark +# Benchmark Kernel MCP with ClawBench -This directory runs stock Harbor agents (Claude Code, Codex) against a locally built `kernel-mcp-server` on a ClawBench task in a single Hypeman sandbox. The task starts from the Kernel-backed ClawBench Harbor adaptation, replaces Playwright MCP with the local source-pinned Kernel MCP server, and keeps ClawBench attached to the same pre-created browser. +[ClawBench](https://github.com/TIGER-AI-Lab/ClawBench) is a suite of browser tasks. Each task describes work to complete on a real website and an evaluator that watches for the network request representing completion. ClawBench then judges the submitted request parameters. + +[Harbor](https://github.com/laude-institute/harbor) runs those tasks as reproducible agent trials. For each trial, Harbor creates an isolated environment, installs a stock agent such as Codex or Claude Code, gives it the task's MCP tools and instruction, runs the verifier, and writes the reward and ATIF trajectory to a job directory. + +This benchmark uses `harbor_hypeman:HypemanEnvironment` as Harbor's execution backend. Hypeman starts one environment from this repository's benchmark image for every trial. Everything for that trial runs inside the same environment: + +1. ClawBench creates one stealth Kernel browser and attaches its request evaluator. +2. The task setup starts Redis and the locally built `kernel-mcp-server` on port 3002. +3. Harbor starts the stock agent with a stdio MCP command that connects to that local server. +4. The agent controls ClawBench's existing browser through `execute_playwright_code`; it cannot create browsers or use managed auth. +5. ClawBench scores the intercepted request, downloads the replay, and deletes the browser. + +The image records the current Git SHA, and the generated task records the ClawBench SHA and browser session ID. The additional `kernel_mcp_valid` result confirms that the agent called the local server with the browser ClawBench created. Task reward still comes directly from ClawBench. ## Requirements -- Harbor 0.21.0 with `harbor-hypeman` 0.1.1 (launched through `uvx`) -- [uv](https://docs.astral.sh/uv/) and Hypeman CLI credentials -- A ClawBench checkout containing commit `df6743f` (`kernel/ClawBench` PR #1) +- `uv`, Harbor 0.21.0, and `harbor-hypeman` 0.1.1 +- Hypeman CLI credentials +- a ClawBench checkout containing `df6743f` from `kernel/ClawBench` PR #1 - `KERNEL_MCP_BENCHMARK_API_KEY` scoped to an isolated evaluation project -- `KERNEL_MCP_BENCHMARK_PROJECT_ID` -- `PURELY_MAIL_API_KEY` and `PURELY_MAIL_DOMAIN` for account-task credentials -- `ANTHROPIC_API_KEY` or `CLAUDE_CODE_OAUTH_TOKEN` for Claude Code -- `OPENAI_API_KEY` for Codex +- `PURELY_MAIL_API_KEY` and `PURELY_MAIL_DOMAIN` for ClawBench account tasks +- `OPENAI_API_KEY` for Codex, or Anthropic credentials for Claude Code +- the ClawBench judge variables when using a hosted judge: `CLAWBENCH_JUDGE_BASE_URL`, `CLAWBENCH_JUDGE_API_KEY`, `CLAWBENCH_JUDGE_MODEL`, and `CLAWBENCH_JUDGE_API_TYPE` + +## Build the trial image -## Build the image +From the `kernel-mcp-server` checkout: ```bash ./benchmarks/harbor/build-image.sh ``` -The build uses the current Git SHA, installs dependencies with Bun, runs the production Next.js build, and writes the resulting image reference to the ignored `.image.env` file. Hypeman can report a failed build before the converted image becomes visible; the script performs a bounded 5-minute ready-image check for that case. +This builds the current checkout with Bun and writes the image reference and Git SHA to the ignored `benchmarks/harbor/.image.env` file. -## Run the ClawBench Kernel MCP arm +## Run one task ```bash export CLAWBENCH_REPO=../ClawBench -./benchmarks/harbor/clawbench/run-control.sh claude-code \ +./benchmarks/harbor/clawbench/run.sh codex \ v2-1134-chapter-finder-redcross ``` -Defaults: - -| Agent | Version | Model | -| ----------- | ------: | ----------------- | -| Claude Code | 2.1.238 | `claude-sonnet-5` | -| Codex | 0.120.0 | `gpt-5.6-luna` | - -Override models with `CLAUDE_BENCHMARK_MODEL` or `CODEX_BENCHMARK_MODEL`. Single-task runs have a 40-minute wall-clock limit; full-suite runs default to 6 hours. Change either with `HARBOR_BENCHMARK_TIMEOUT`. - -Pass `all` instead of a task ID to run the complete suite, and set `HARBOR_N_CONCURRENT` to control parallelism: +## Run the full suite ```bash -HARBOR_N_CONCURRENT=10 ./benchmarks/harbor/clawbench/run-control.sh codex all +export CLAWBENCH_REPO=../ClawBench +HARBOR_N_CONCURRENT=10 \ + ./benchmarks/harbor/clawbench/run.sh codex all ``` -`run-control.sh` adapts the selected ClawBench tasks with `clawbench-harbor-adapt`, converts them with `clawbench/prepare-control.py`, and runs them under Harbor. Each generated task: +Codex defaults to version `0.120.0` with `gpt-5.6-luna`. Claude Code defaults to version `2.1.238` with `claude-sonnet-5`. Override these with `CODEX_BENCHMARK_MODEL`, `CODEX_BENCHMARK_VERSION`, `CLAUDE_BENCHMARK_MODEL`, or `CLAUDE_BENCHMARK_VERSION`. + +Single-task runs have a 40-minute wall-clock limit. Full-suite runs default to six hours. Set `HARBOR_BENCHMARK_TIMEOUT` to override either limit. Set `HARBOR_JOBS_DIR` to choose where Harbor writes results. + +## Results -- exposes `get_connection_context` and `execute_playwright_code` -- disables coordinate-based computer actions, browser lifecycle, and managed-auth toolsets -- instructs the agent to read `./my-info/kernel_browser.json` and use that session ID -- instructs account tasks to use the supplied PurelyMail credentials instead of managed auth -- verifies ATIF observations, project scope, exact session reuse, ClawBench interception, replay finalization, and browser deletion +Harbor writes its normal job directory, including: -Outputs use the normal Harbor job directory and add `kernel-mcp-control-result.json`, Kernel MCP logs, source manifests, and same-session metrics to the ClawBench verifier artifacts. +- `trajectory.json`: the agent's ATIF messages and tool calls +- `reward.json`: ClawBench's reward plus the `kernel_mcp_valid` diagnostic +- `clawbench-result.json`: evaluator details +- `kernel-mcp-result.json`: local-source and same-browser wiring details +- `recording.mp4`: the finalized Kernel replay +- `kernel-mcp/`: local server logs and the source/session manifest diff --git a/benchmarks/harbor/bin/kernel-mcp-local b/benchmarks/harbor/bin/kernel-mcp-local index 9cb8c06..301c209 100755 --- a/benchmarks/harbor/bin/kernel-mcp-local +++ b/benchmarks/harbor/bin/kernel-mcp-local @@ -1,4 +1,9 @@ #!/bin/sh +# Harbor launches MCP servers as stdio subprocesses of the benchmark agent. +# This wrapper turns that stdio connection into an authenticated connection to +# the kernel-mcp-server HTTP endpoint running locally in the same Hypeman task. +# The setup script writes the project-scoped key to /run so it never appears in +# the generated agent configuration. set -eu key_file=/run/kernel-mcp-benchmark/api-key diff --git a/benchmarks/harbor/bin/start-kernel-mcp-server b/benchmarks/harbor/bin/start-kernel-mcp-server index 2096588..d1a2ba0 100755 --- a/benchmarks/harbor/bin/start-kernel-mcp-server +++ b/benchmarks/harbor/bin/start-kernel-mcp-server @@ -1,4 +1,8 @@ #!/bin/bash +# Start the services used by the source-pinned Kernel MCP build inside a Harbor +# trial. ClawBench has already created the browser; this script starts Redis and +# Next.js, stores the project-scoped key for the stdio wrapper, waits for MCP to +# accept connections, and records which source build and browser the trial used. set -euo pipefail : "${KERNEL_API_KEY:?KERNEL_API_KEY is required}" @@ -44,8 +48,6 @@ fi python3 - <<'PY' import json import os -import platform -from datetime import datetime, timezone from pathlib import Path browser_path = Path("/my-info/kernel_browser.json") @@ -59,11 +61,7 @@ manifest = { "clawbench_source_sha": os.environ.get("CLAWBENCH_SOURCE_SHA", ""), "browser_session_id": browser.get("session_id"), "enabled_toolsets": os.environ.get("KERNEL_MCP_ENABLED_TOOLSETS", ""), - "disabled_toolsets": os.environ.get("KERNEL_MCP_DISABLED_TOOLSETS", ""), "image": os.environ.get("KERNEL_MCP_BENCHMARK_IMAGE", ""), - "hypeman_instance_name": os.environ.get("HYPEMAN_INSTANCE_NAME", ""), - "sandbox_hostname": platform.node(), - "started_at": datetime.now(timezone.utc).isoformat(), } Path("/logs/kernel-mcp/run-manifest.json").write_text(json.dumps(manifest, indent=2)) PY diff --git a/benchmarks/harbor/clawbench/prepare-control.py b/benchmarks/harbor/clawbench/prepare-task.py similarity index 85% rename from benchmarks/harbor/clawbench/prepare-control.py rename to benchmarks/harbor/clawbench/prepare-task.py index 26fd412..67bdec2 100755 --- a/benchmarks/harbor/clawbench/prepare-control.py +++ b/benchmarks/harbor/clawbench/prepare-task.py @@ -1,4 +1,12 @@ #!/usr/bin/env python3 +"""Turn a ClawBench Harbor task into the Kernel MCP benchmark arm. + +ClawBench generates a complete Harbor task that normally gives the agent +Playwright MCP. This script keeps ClawBench's instruction, evaluator, browser, +and cleanup lifecycle, but swaps the agent-facing MCP server for the local +source build and starts that server during task setup. +""" + from __future__ import annotations import argparse @@ -24,7 +32,9 @@ def _drop_mcp_servers(task_toml: str) -> str: return "\n".join(output).rstrip() + "\n" -def _add_environment(task_toml: str, *, image: str, server_sha: str, clawbench_sha: str) -> str: +def _add_environment( + task_toml: str, *, image: str, server_sha: str, clawbench_sha: str +) -> str: lines = task_toml.splitlines() output: list[str] = [] inserted_image = False @@ -41,7 +51,6 @@ def _add_environment(task_toml: str, *, image: str, server_sha: str, clawbench_s f"KERNEL_MCP_SOURCE_SHA = {json.dumps(server_sha)}", f"CLAWBENCH_SOURCE_SHA = {json.dumps(clawbench_sha)}", f"KERNEL_MCP_ENABLED_TOOLSETS = {json.dumps(ENABLED_TOOLSETS)}", - 'KERNEL_MCP_EXPECTED_PROJECT_ID = "${KERNEL_MCP_BENCHMARK_PROJECT_ID}"', 'API_BASE_URL = "${KERNEL_API_BASE_URL:-}"', 'REDIS_URL = "redis://127.0.0.1:6379"', ] @@ -94,7 +103,9 @@ def _patch_setup(setup: str) -> str: def _patch_verifier(test_script: str) -> str: - verify_marker = "/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py\n" + verify_marker = ( + "/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py\n" + ) if verify_marker not in test_script: raise ValueError("generated verifier script is missing ClawBench verification") return test_script.replace( @@ -103,7 +114,7 @@ def _patch_verifier(test_script: str) -> str: + "mkdir -p /logs/verifier/kernel-mcp\n" + "cp -a /logs/kernel-mcp/. /logs/verifier/kernel-mcp/\n" + "/app/src/runtime-server/.venv/bin/python " - + "/app/src/harbor/verify-kernel-mcp-control.py\n", + + "/app/src/harbor/verify-kernel-mcp-task.py\n", 1, ) @@ -113,12 +124,13 @@ def _patch_instruction(instruction: str) -> str: "Use only Playwright MCP browser tools plus reading files", "Use only Kernel MCP browser-control tools plus reading files", ) - return instruction.rstrip() + """ + return ( + instruction.rstrip() + + """ --- Kernel MCP benchmark arm: - Wait for the `kernel` MCP server to finish initializing before starting. In Claude Code, call `WaitForMcpServers` if it is still pending; do not conclude that the tools are unavailable while it initializes. -- Call `get_connection_context` once before taking any browser action. - Read `./my-info/kernel_browser.json` and use its existing `session_id` for every `execute_playwright_code` call. - Do not create, list, update, or delete browsers. Browser lifecycle tools and `computer_action` are intentionally unavailable. - Use Kernel MCP `execute_playwright_code` for all browser interaction. Do not use Playwright MCP or a direct CDP client. @@ -127,9 +139,12 @@ def _patch_instruction(instruction: str) -> str: - Do not use Kernel managed auth, create an auth connection, or start a hosted login flow. - Complete and submit the task through the existing browser, then stop. """ + ) -def transform_task(task_dir: Path, *, image: str, server_sha: str, clawbench_sha: str) -> None: +def transform_task( + task_dir: Path, *, image: str, server_sha: str, clawbench_sha: str +) -> None: dockerfile = task_dir / "environment" / "Dockerfile" dockerfile.unlink(missing_ok=True) @@ -156,14 +171,16 @@ def transform_task(task_dir: Path, *, image: str, server_sha: str, clawbench_sha instruction_path = step_dir / "instruction.md" instruction_path.write_text(_patch_instruction(instruction_path.read_text())) - verifier_source = Path(__file__).with_name("verify-control.py") - verifier_target = task_dir / "environment" / "harbor" / "verify-kernel-mcp-control.py" + verifier_source = Path(__file__).with_name("verify-task.py") + verifier_target = task_dir / "environment" / "harbor" / "verify-kernel-mcp-task.py" shutil.copy2(verifier_source, verifier_target) verifier_target.chmod(0o755) def main() -> int: - parser = argparse.ArgumentParser(description="Convert a Kernel-backed ClawBench Harbor task to the Kernel MCP arm") + parser = argparse.ArgumentParser( + description="Replace a generated ClawBench task's Playwright MCP server with the local Kernel MCP build" + ) parser.add_argument("task_dir", type=Path) parser.add_argument("--image", required=True) parser.add_argument("--server-sha", required=True) diff --git a/benchmarks/harbor/clawbench/run-control.sh b/benchmarks/harbor/clawbench/run.sh similarity index 92% rename from benchmarks/harbor/clawbench/run-control.sh rename to benchmarks/harbor/clawbench/run.sh index 0406273..bf59f24 100755 --- a/benchmarks/harbor/clawbench/run-control.sh +++ b/benchmarks/harbor/clawbench/run.sh @@ -1,4 +1,9 @@ #!/bin/bash +# Run one ClawBench task or the full suite through the local Kernel MCP build. +# +# The script asks ClawBench to generate ordinary Harbor tasks, rewrites those +# tasks with prepare-task.py, then lets Harbor create one isolated Hypeman +# environment per trial and install the selected stock agent inside it. set -euo pipefail usage() { @@ -39,7 +44,6 @@ source "$image_env" set +a : "${KERNEL_MCP_BENCHMARK_API_KEY:?KERNEL_MCP_BENCHMARK_API_KEY is required}" -: "${KERNEL_MCP_BENCHMARK_PROJECT_ID:?KERNEL_MCP_BENCHMARK_PROJECT_ID is required}" : "${PURELY_MAIL_API_KEY:?PURELY_MAIL_API_KEY is required}" : "${PURELY_MAIL_DOMAIN:?PURELY_MAIL_DOMAIN is required}" @@ -87,7 +91,7 @@ mapfile -t task_dirs < <(find "$dataset" -mindepth 1 -maxdepth 1 -type d | sort) } for task_dir in "${task_dirs[@]}"; do - python3 "$benchmark_dir/clawbench/prepare-control.py" "$task_dir" \ + python3 "$benchmark_dir/clawbench/prepare-task.py" "$task_dir" \ --image "$KERNEL_MCP_BENCHMARK_IMAGE" \ --server-sha "$KERNEL_MCP_SOURCE_SHA" \ --clawbench-sha "$clawbench_ref" @@ -96,13 +100,11 @@ done export KERNEL_API_KEY=$KERNEL_MCP_BENCHMARK_API_KEY export KERNEL_BASE_URL=${KERNEL_BASE_URL:-https://api.onkernel.com} export KERNEL_API_BASE_URL=${KERNEL_API_BASE_URL:-$KERNEL_BASE_URL} -export KERNEL_MCP_BENCHMARK_PROJECT_ID cat >"$runtime_env" < ModuleType: - spec = importlib.util.spec_from_file_location(name, HERE / filename) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - -prepare = load_module("prepare_control", "prepare-control.py") -verify = load_module("verify_control", "verify-control.py") - - -class PrepareControlTest(unittest.TestCase): - def test_transforms_playwright_control_into_kernel_mcp_arm(self) -> None: - with tempfile.TemporaryDirectory() as temp: - task = Path(temp) - environment = task / "environment" - step = task / "steps" / "run" - (step / "workdir").mkdir(parents=True) - (step / "tests").mkdir() - (step / "instruction.md").write_text( - "Use only Playwright MCP browser tools plus reading files under ./my-info/.\n" - ) - (environment / "harbor").mkdir(parents=True) - (environment / "Dockerfile").write_text("FROM python:3.11-slim\n") - (task / "task.toml").write_text( - """[environment] -workdir = "/" - -[environment.env] -KERNEL_API_KEY = "${KERNEL_API_KEY}" - -[[steps]] -name = "run" - -[[environment.mcp_servers]] -name = "playwright" -transport = "stdio" -command = "npx" -args = ["-y", "@playwright/mcp@0.0.79"] -""" - ) - (step / "workdir" / "setup.sh").write_text( - "#!/bin/bash\nmkdir -p /data /logs/verifier /extra_info\n" - "/app/src/harbor/start-runtime.sh\n" - ) - (step / "tests" / "test.sh").write_text( - "#!/bin/bash\n" - "/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py\n" - ) - - prepare.transform_task( - task, - image="docker.io/builds/image:latest", - server_sha="server-sha", - clawbench_sha="clawbench-sha", - ) - - task_toml = (task / "task.toml").read_text() - self.assertFalse((environment / "Dockerfile").exists()) - self.assertIn('docker_image = "docker.io/builds/image:latest"', task_toml) - self.assertIn('name = "kernel"', task_toml) - self.assertIn('command = "/usr/local/bin/kernel-mcp-local"', task_toml) - self.assertNotIn("@playwright/mcp", task_toml) - self.assertIn('KERNEL_MCP_ENABLED_TOOLSETS = "playwright"', task_toml) - self.assertNotIn("KERNEL_MCP_DISABLED_TOOLSETS", task_toml) - self.assertIn('API_BASE_URL = "${KERNEL_API_BASE_URL:-}"', task_toml) - self.assertNotIn("KERNEL_API_BASE_URL =", task_toml) - self.assertIn('REDIS_URL = "redis://127.0.0.1:6379"', task_toml) - - setup = (step / "workdir" / "setup.sh").read_text() - self.assertIn("install_clawbench_runtime", setup) - self.assertIn("start-kernel-mcp-server", setup) - test_script = (step / "tests" / "test.sh").read_text() - self.assertIn("verify-kernel-mcp-control.py", test_script) - self.assertIn("/logs/verifier/kernel-mcp", test_script) - instruction = (step / "instruction.md").read_text() - self.assertIn("WaitForMcpServers", instruction) - self.assertIn("existing `session_id`", instruction) - self.assertIn("Use Kernel MCP `execute_playwright_code`", instruction) - self.assertIn("PurelyMail-backed credentials", instruction) - self.assertIn("Do not use Kernel managed auth", instruction) - self.assertIn("Do not call `fetch`", instruction) - self.assertTrue((environment / "harbor" / "verify-kernel-mcp-control.py").is_file()) - - -class VerifyControlTest(unittest.TestCase): - def trajectory(self, session_id: str = "session-123") -> dict: - return { - "steps": [ - { - "tool_calls": [ - { - "tool_call_id": "context-1", - "function_name": "mcp__kernel__get_connection_context", - "arguments": {}, - }, - { - "tool_call_id": "playwright-1", - "function_name": "mcp__kernel__execute_playwright_code", - "arguments": { - "session_id": session_id, - "code": "await page.goto('https://example.com')", - }, - }, - ], - "observation": { - "results": [ - { - "source_call_id": "context-1", - "content": { - "connection_scope": { - "kind": "project", - "project_id": "project-123", - } - }, - }, - { - "source_call_id": "playwright-1", - "content": [{"type": "text", "text": "{\"ok\": true}"}], - }, - ] - }, - } - ] - } - - def test_accepts_successful_calls_on_precreated_session(self) -> None: - result = verify.validate_control( - self.trajectory(), - expected_session_id="session-123", - expected_project_id="project-123", - ) - for key in ( - "context_called", - "browser_control_called", - "observations_valid", - "context_scope_valid", - "same_session", - "no_playwright_mcp", - "no_forbidden_kernel_tools", - "no_direct_http_automation", - ): - self.assertTrue(result[key], key) - - def test_accepts_missing_terminal_observation_after_interception(self) -> None: - trajectory = self.trajectory() - trajectory["steps"][0]["tool_calls"].append( - { - "tool_call_id": "playwright-final", - "function_name": "mcp__kernel__execute_playwright_code", - "arguments": { - "session_id": "session-123", - "code": "await page.getByRole('button').click()", - }, - } - ) - result = verify.validate_control( - trajectory, - expected_session_id="session-123", - expected_project_id="project-123", - allowed_missing_observation_ids={"playwright-final"}, - ) - self.assertTrue(result["observations_valid"]) - self.assertEqual( - result["expected_interrupted_observations"], ["playwright-final"] - ) - self.assertEqual(result["unexpected_missing_observations"], []) - - def test_rejects_another_session(self) -> None: - result = verify.validate_control( - self.trajectory("session-other"), - expected_session_id="session-123", - expected_project_id="project-123", - ) - self.assertFalse(result["same_session"]) - - def test_rejects_direct_http_inside_playwright_code(self) -> None: - trajectory = self.trajectory() - trajectory["steps"][0]["tool_calls"][1]["arguments"]["code"] = ( - "return await page.evaluate(() => fetch('/api'))" - ) - result = verify.validate_control( - trajectory, - expected_session_id="session-123", - expected_project_id="project-123", - ) - self.assertFalse(result["no_direct_http_automation"]) - - def test_rejects_playwright_mcp_and_lifecycle_tools(self) -> None: - trajectory = self.trajectory() - trajectory["steps"][0]["tool_calls"].extend( - [ - { - "tool_call_id": "direct-playwright", - "function_name": "mcp__playwright__browser_navigate", - "arguments": {}, - }, - { - "tool_call_id": "browser-list", - "function_name": "mcp__kernel__manage_browsers", - "arguments": {"action": "list"}, - }, - { - "tool_call_id": "computer-action", - "function_name": "mcp__kernel__computer_action", - "arguments": {"session_id": "session-123", "actions": []}, - }, - ] - ) - result = verify.validate_control( - trajectory, - expected_session_id="session-123", - expected_project_id="project-123", - ) - self.assertFalse(result["no_playwright_mcp"]) - self.assertFalse(result["no_forbidden_kernel_tools"]) - - -if __name__ == "__main__": - unittest.main() diff --git a/benchmarks/harbor/clawbench/verify-control.py b/benchmarks/harbor/clawbench/verify-control.py deleted file mode 100755 index 7660925..0000000 --- a/benchmarks/harbor/clawbench/verify-control.py +++ /dev/null @@ -1,300 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import ast -import json -import os -import re -from pathlib import Path -from typing import Any - -LOGS_DIR = Path(os.environ.get("HARBOR_LOGS_DIR", "/logs")) -VERIFIER_DIR = LOGS_DIR / "verifier" -CONTEXT_TOOL = "mcp__kernel__get_connection_context" -BROWSER_TOOLS = {"mcp__kernel__execute_playwright_code"} -FORBIDDEN_KERNEL_TOOLS = { - "mcp__kernel__computer_action", - "mcp__kernel__manage_browsers", - "mcp__kernel__manage_auth_connections", - "mcp__kernel__open_auth_login", -} - - -def read_json(path: Path) -> dict[str, Any] | None: - try: - value = json.loads(path.read_text()) - except (OSError, json.JSONDecodeError): - return None - return value if isinstance(value, dict) else None - - -def _decode_content(content: Any) -> Any: - value = content - for _ in range(6): - if isinstance(value, str): - try: - value = json.loads(value) - except json.JSONDecodeError: - try: - value = ast.literal_eval(value) - except (SyntaxError, ValueError): - return value - continue - if isinstance(value, dict) and value.get("type") == "text": - value = value.get("text") - continue - if ( - isinstance(value, list) - and len(value) == 1 - and isinstance(value[0], dict) - and value[0].get("type") == "text" - ): - value = value[0].get("text") - continue - return value - return value - - -def _contains_error(value: Any) -> bool: - if isinstance(value, dict): - if value.get("is_error") is True or value.get("isError") is True: - return True - if value.get("error") not in (None, False, ""): - return True - return any(_contains_error(item) for item in value.values()) - if isinstance(value, list): - return any(_contains_error(item) for item in value) - if isinstance(value, str): - return value.lstrip().lower().startswith(("[error]", "error:", "error in ")) - return False - - -def _calls(trajectory: dict[str, Any]) -> list[dict[str, Any]]: - calls: list[dict[str, Any]] = [] - for step in trajectory.get("steps") or []: - if not isinstance(step, dict): - continue - calls.extend(call for call in step.get("tool_calls") or [] if isinstance(call, dict)) - return calls - - -def _results(trajectory: dict[str, Any]) -> dict[str, list[dict[str, Any]]]: - results: dict[str, list[dict[str, Any]]] = {} - for step in trajectory.get("steps") or []: - if not isinstance(step, dict): - continue - observation = step.get("observation") - if not isinstance(observation, dict): - continue - for result in observation.get("results") or []: - if not isinstance(result, dict): - continue - call_id = result.get("source_call_id") - if isinstance(call_id, str): - results.setdefault(call_id, []).append(result) - return results - - -def validate_control( - trajectory: dict[str, Any] | None, - *, - expected_session_id: str, - expected_project_id: str, - allowed_missing_observation_ids: set[str] | None = None, -) -> dict[str, Any]: - trajectory = trajectory or {} - calls = _calls(trajectory) - result_map = _results(trajectory) - kernel_calls = [ - call for call in calls if str(call.get("function_name", "")).startswith("mcp__kernel__") - ] - context_calls = [call for call in kernel_calls if call.get("function_name") == CONTEXT_TOOL] - browser_calls = [call for call in kernel_calls if call.get("function_name") in BROWSER_TOOLS] - playwright_calls = [ - call for call in calls if str(call.get("function_name", "")).startswith("mcp__playwright__") - ] - forbidden_calls = [ - call for call in kernel_calls if call.get("function_name") in FORBIDDEN_KERNEL_TOOLS - ] - - missing_observations: list[Any] = [] - duplicate_observations: list[Any] = [] - error_observations: list[Any] = [] - context_scope_valid = bool(expected_project_id) - same_session = bool(expected_session_id and browser_calls) and all( - isinstance(call.get("arguments"), dict) - and call["arguments"].get("session_id") == expected_session_id - for call in browser_calls - ) - successful_context_calls = 0 - successful_browser_calls = 0 - - for call in context_calls + browser_calls: - call_id = call.get("tool_call_id") - observations = result_map.get(call_id, []) if isinstance(call_id, str) else [] - if not observations: - missing_observations.append(call_id) - continue - if len(observations) != 1: - duplicate_observations.append(call_id) - continue - decoded = _decode_content(observations[0].get("content")) - if _contains_error(decoded) or _contains_error(observations[0]): - error_observations.append(call_id) - continue - if call.get("function_name") == CONTEXT_TOOL: - scope = decoded.get("connection_scope") if isinstance(decoded, dict) else None - context_scope_valid = context_scope_valid and ( - isinstance(scope, dict) - and scope.get("kind") == "project" - and scope.get("project_id") == expected_project_id - ) - successful_context_calls += 1 - elif call.get("function_name") in BROWSER_TOOLS: - successful_browser_calls += 1 - - allowed_missing = allowed_missing_observation_ids or set() - unexpected_missing_observations = [ - call_id for call_id in missing_observations if call_id not in allowed_missing - ] - observations_valid = ( - successful_context_calls > 0 - and successful_browser_calls > 0 - and not (unexpected_missing_observations or duplicate_observations) - ) - direct_http_patterns = re.compile( - r"\bfetch\s*\(|\bXMLHttpRequest\b|\b(?:page|context)\.request\b|\brequest\.(?:get|post|put|patch|delete)\s*\(", - re.IGNORECASE, - ) - direct_http_calls = [ - call - for call in browser_calls - if call.get("function_name") == "mcp__kernel__execute_playwright_code" - and isinstance(call.get("arguments"), dict) - and direct_http_patterns.search(str(call["arguments"].get("code", ""))) - ] - return { - "context_called": bool(context_calls), - "browser_control_called": bool(browser_calls), - "observations_valid": observations_valid, - "context_scope_valid": context_scope_valid and bool(context_calls), - "same_session": same_session, - "no_playwright_mcp": not playwright_calls, - "no_forbidden_kernel_tools": not forbidden_calls, - "no_direct_http_automation": not direct_http_calls, - "missing_observations": missing_observations, - "expected_interrupted_observations": [ - call_id for call_id in missing_observations if call_id in allowed_missing - ], - "unexpected_missing_observations": unexpected_missing_observations, - "duplicate_observations": duplicate_observations, - "error_observations": error_observations, - "direct_http_calls": [call.get("tool_call_id") for call in direct_http_calls], - "kernel_tool_calls": [ - { - "tool_call_id": call.get("tool_call_id"), - "name": call.get("function_name"), - "arguments": call.get("arguments"), - } - for call in kernel_calls - ], - } - - -def main() -> int: - VERIFIER_DIR.mkdir(parents=True, exist_ok=True) - trajectory = read_json(LOGS_DIR / "agent" / "trajectory.json") - browser = read_json(Path("/my-info/kernel_browser.json")) - lifecycle = read_json(Path("/data/kernel-browser-lifecycle.json")) - manifest = read_json(LOGS_DIR / "kernel-mcp" / "run-manifest.json") - clawbench_result = read_json(VERIFIER_DIR / "clawbench-result.json") - interception = read_json(Path("/data/interception.json")) - agent_stop = read_json(Path("/data/agent-stop.json")) - reward_path = VERIFIER_DIR / "reward.json" - reward_metrics = read_json(reward_path) or {} - - session_id = str((browser or {}).get("session_id") or "") - expected_project_id = os.environ.get("KERNEL_MCP_EXPECTED_PROJECT_ID", "") - all_calls = _calls(trajectory or {}) - browser_calls = [call for call in all_calls if call.get("function_name") in BROWSER_TOOLS] - terminal_call_id = browser_calls[-1].get("tool_call_id") if browser_calls else None - stop_detected_at = (agent_stop or {}).get("stop_detected_at") - intercepted_at = (interception or {}).get("intercepted_at") - stopped_after_interception = bool( - isinstance(stop_detected_at, (int, float)) - and isinstance(intercepted_at, (int, float)) - and 0 <= stop_detected_at - intercepted_at <= 5 - ) - allowed_missing = ( - {terminal_call_id} - if stopped_after_interception and isinstance(terminal_call_id, str) - else set() - ) - atif = validate_control( - trajectory, - expected_session_id=session_id, - expected_project_id=expected_project_id, - allowed_missing_observation_ids=allowed_missing, - ) - checks = { - "kernel_mcp_context": atif["context_called"], - "kernel_mcp_browser_control": atif["browser_control_called"], - "kernel_mcp_observations": atif["observations_valid"], - "kernel_mcp_project_scope": atif["context_scope_valid"], - "kernel_mcp_same_session": atif["same_session"], - "no_playwright_mcp": atif["no_playwright_mcp"], - "no_forbidden_kernel_tools": atif["no_forbidden_kernel_tools"], - "no_direct_http_automation": atif["no_direct_http_automation"], - "kernel_mcp_source_sha": bool( - manifest - and manifest.get("kernel_mcp_server_sha") == os.environ.get("KERNEL_MCP_SOURCE_SHA") - ), - "kernel_mcp_manifest_session": bool( - manifest and manifest.get("browser_session_id") == session_id - ), - "kernel_mcp_toolset_allowlist": bool( - manifest - and set(str(manifest.get("enabled_toolsets", "")).split()) - == {"playwright"} - ), - "hypeman_identity": bool(manifest and manifest.get("hypeman_instance_name")), - "browser_deleted": bool( - lifecycle - and lifecycle.get("status") == "deleted" - and lifecycle.get("deletion_verified") is True - ), - "clawbench_intercepted": bool( - reward_metrics.get("intercepted") == 1 - or (clawbench_result or {}).get("intercepted") is True - ), - "agent_stopped_after_interception": stopped_after_interception, - } - infra_ok = all( - value - for name, value in checks.items() - if name not in {"clawbench_intercepted", "agent_stopped_after_interception"} - ) - checks["infra_ok"] = infra_ok - - reward_metrics.update({name: float(value) for name, value in checks.items()}) - reward_path.write_text(json.dumps(reward_metrics, indent=2)) - result = { - "checks": checks, - "session_id": session_id, - "expected_project_id": expected_project_id, - "atif": atif, - "run_manifest": manifest, - "browser_lifecycle": lifecycle, - "clawbench_result": clawbench_result, - "interception": interception, - "agent_stop": agent_stop, - "stop_latency_seconds": ( - stop_detected_at - intercepted_at if stopped_after_interception else None - ), - } - (VERIFIER_DIR / "kernel-mcp-control-result.json").write_text(json.dumps(result, indent=2)) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/benchmarks/harbor/clawbench/verify-task.py b/benchmarks/harbor/clawbench/verify-task.py new file mode 100755 index 0000000..466432a --- /dev/null +++ b/benchmarks/harbor/clawbench/verify-task.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +"""Record whether a ClawBench trial used the intended Kernel MCP setup. + +ClawBench's verifier remains responsible for the task reward, request +interception, replay download, and browser cleanup. This script adds one +`kernel_mcp_valid` diagnostic metric so benchmark results can distinguish a +failed task from a trial that never exercised the local server correctly. +""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +from typing import Any + +LOGS_DIR = Path(os.environ.get("HARBOR_LOGS_DIR", "/logs")) +VERIFIER_DIR = LOGS_DIR / "verifier" +PLAYWRIGHT_TOOL = "mcp__kernel__execute_playwright_code" + + +def read_object(path: Path) -> dict[str, Any]: + try: + value = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError): + return {} + return value if isinstance(value, dict) else {} + + +def tool_calls(trajectory: dict[str, Any]) -> list[dict[str, Any]]: + calls: list[dict[str, Any]] = [] + for step in trajectory.get("steps") or []: + if isinstance(step, dict): + calls.extend( + call for call in step.get("tool_calls") or [] if isinstance(call, dict) + ) + return calls + + +def main() -> int: + VERIFIER_DIR.mkdir(parents=True, exist_ok=True) + browser = read_object(Path("/my-info/kernel_browser.json")) + manifest = read_object(LOGS_DIR / "kernel-mcp" / "run-manifest.json") + trajectory = read_object(LOGS_DIR / "agent" / "trajectory.json") + + expected_session = browser.get("session_id") + calls = [ + call + for call in tool_calls(trajectory) + if call.get("function_name") == PLAYWRIGHT_TOOL + ] + called_sessions = { + arguments.get("session_id") + for call in calls + if isinstance((arguments := call.get("arguments")), dict) + } + + expected_source = os.environ.get("KERNEL_MCP_SOURCE_SHA") + checks = { + "used_kernel_mcp": bool(calls), + "used_clawbench_browser": bool(expected_session) + and called_sessions == {expected_session}, + "used_expected_source": bool(expected_source) + and manifest.get("kernel_mcp_server_sha") == expected_source, + } + valid = all(checks.values()) + + result = { + "valid": valid, + "checks": checks, + "expected_session_id": expected_session, + "called_session_ids": sorted(str(value) for value in called_sessions), + "expected_source_sha": expected_source, + "actual_source_sha": manifest.get("kernel_mcp_server_sha"), + } + (VERIFIER_DIR / "kernel-mcp-result.json").write_text(json.dumps(result, indent=2)) + + reward_path = VERIFIER_DIR / "reward.json" + rewards = read_object(reward_path) + rewards["kernel_mcp_valid"] = float(valid) + reward_path.write_text(json.dumps(rewards, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From a20d7aa8da791480c856e6164e9059f6af58988e Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Mon, 24 Aug 2026 11:53:27 +0000 Subject: [PATCH 24/29] Clarify Hypeman trial lifecycle --- benchmarks/harbor/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/harbor/README.md b/benchmarks/harbor/README.md index 3748955..bde47d6 100644 --- a/benchmarks/harbor/README.md +++ b/benchmarks/harbor/README.md @@ -4,7 +4,7 @@ [Harbor](https://github.com/laude-institute/harbor) runs those tasks as reproducible agent trials. For each trial, Harbor creates an isolated environment, installs a stock agent such as Codex or Claude Code, gives it the task's MCP tools and instruction, runs the verifier, and writes the reward and ATIF trajectory to a job directory. -This benchmark uses `harbor_hypeman:HypemanEnvironment` as Harbor's execution backend. Hypeman starts one environment from this repository's benchmark image for every trial. Everything for that trial runs inside the same environment: +This benchmark uses `harbor_hypeman:HypemanEnvironment` as Harbor's execution backend. For every trial, Harbor asks Hypeman to start an isolated VM from this repository's benchmark image. Everything for that trial runs inside that VM: 1. ClawBench creates one stealth Kernel browser and attaches its request evaluator. 2. The task setup starts Redis and the locally built `kernel-mcp-server` on port 3002. From 950a576595b6ac877ed5b60e05688c44ad7399fd Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Fri, 28 Aug 2026 15:56:41 +0000 Subject: [PATCH 25/29] Add ClawBench benchmark automation --- .github/workflows/benchmark-clawbench.yml | 462 ++++++++++++++++++++ benchmarks/harbor/README.md | 39 +- benchmarks/harbor/clawbench/prepare-task.py | 1 + benchmarks/harbor/clawbench/run.sh | 4 +- benchmarks/harbor/publish-braintrust.ts | 411 +++++++++++++++++ benchmarks/harbor/redact.ts | 53 +++ benchmarks/harbor/report.ts | 151 +++++++ benchmarks/harbor/results.test.ts | 252 +++++++++++ benchmarks/harbor/results.ts | 388 ++++++++++++++++ package.json | 2 + 10 files changed, 1760 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/benchmark-clawbench.yml create mode 100644 benchmarks/harbor/publish-braintrust.ts create mode 100644 benchmarks/harbor/redact.ts create mode 100644 benchmarks/harbor/report.ts create mode 100644 benchmarks/harbor/results.test.ts create mode 100644 benchmarks/harbor/results.ts diff --git a/.github/workflows/benchmark-clawbench.yml b/.github/workflows/benchmark-clawbench.yml new file mode 100644 index 0000000..524d57b --- /dev/null +++ b/.github/workflows/benchmark-clawbench.yml @@ -0,0 +1,462 @@ +name: Benchmark ClawBench + +on: + issue_comment: + types: [created] + schedule: + - cron: "0 13 * * 1" + workflow_dispatch: + inputs: + pr_number: + description: "Same-repository PR to benchmark" + required: false + type: string + ref: + description: "Ref to benchmark when no PR is supplied" + required: false + default: "main" + type: string + task: + description: "ClawBench task ID or all" + required: true + default: "all" + type: string + agent: + description: "Stock Harbor agent" + required: true + default: "codex" + type: choice + options: + - codex + - claude-code + concurrency: + description: "Concurrent Harbor trials per arm" + required: true + default: "30" + type: string + compare_to_base: + description: "Run the PR/base or ref/main comparison arm" + required: true + default: true + type: boolean + +permissions: {} + +jobs: + resolve: + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + outputs: + enabled: ${{ steps.resolve.outputs.enabled }} + head_sha: ${{ steps.resolve.outputs.head_sha }} + base_sha: ${{ steps.resolve.outputs.base_sha }} + compare: ${{ steps.resolve.outputs.compare }} + pr_number: ${{ steps.resolve.outputs.pr_number }} + task: ${{ steps.resolve.outputs.task }} + agent: ${{ steps.resolve.outputs.agent }} + concurrency: ${{ steps.resolve.outputs.concurrency }} + experiment: ${{ steps.resolve.outputs.experiment }} + title: ${{ steps.resolve.outputs.title }} + steps: + - id: resolve + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 + env: + INPUT_PR_NUMBER: ${{ inputs.pr_number }} + INPUT_REF: ${{ inputs.ref }} + INPUT_TASK: ${{ inputs.task }} + INPUT_AGENT: ${{ inputs.agent }} + INPUT_CONCURRENCY: ${{ inputs.concurrency }} + INPUT_COMPARE: ${{ inputs.compare_to_base }} + with: + script: | + const owner = context.repo.owner; + const repo = context.repo.repo; + const defaultBranch = context.payload.repository.default_branch; + const event = context.eventName; + let headSha; + let baseSha; + let prNumber = ""; + let compare = false; + let task = "all"; + let agent = "codex"; + let concurrency = "30"; + + if (event === "issue_comment") { + if ((context.payload.comment.body || "").trim() !== "/benchmark clawbench") { + core.setOutput("enabled", "false"); + return; + } + if (!context.payload.issue.pull_request) { + core.setFailed("/benchmark clawbench can only be used on a pull request"); + return; + } + const allowed = new Set(["OWNER", "MEMBER", "COLLABORATOR"]); + if (!allowed.has(context.payload.comment.author_association)) { + core.setFailed("Only repository collaborators can run benchmarks"); + return; + } + const pull = (await github.rest.pulls.get({ + owner, + repo, + pull_number: context.payload.issue.number, + })).data; + if (pull.head.repo?.full_name !== `${owner}/${repo}`) { + core.setFailed("Benchmarks cannot run code from fork pull requests"); + return; + } + prNumber = String(pull.number); + headSha = pull.head.sha; + baseSha = pull.base.sha; + compare = true; + } else if (event === "workflow_dispatch") { + if (process.env.GITHUB_REF_NAME !== defaultBranch) { + core.setFailed(`Run this workflow from ${defaultBranch}; choose the target with the inputs`); + return; + } + task = process.env.INPUT_TASK || "all"; + agent = process.env.INPUT_AGENT || "codex"; + concurrency = process.env.INPUT_CONCURRENCY || "30"; + compare = process.env.INPUT_COMPARE === "true"; + if (process.env.INPUT_PR_NUMBER) { + const number = Number(process.env.INPUT_PR_NUMBER); + if (!Number.isInteger(number) || number <= 0) { + core.setFailed("pr_number must be a positive integer"); + return; + } + const pull = (await github.rest.pulls.get({ owner, repo, pull_number: number })).data; + if (pull.head.repo?.full_name !== `${owner}/${repo}`) { + core.setFailed("Benchmarks cannot run code from fork pull requests"); + return; + } + prNumber = String(number); + headSha = pull.head.sha; + baseSha = pull.base.sha; + } else { + headSha = (await github.rest.repos.getCommit({ + owner, + repo, + ref: process.env.INPUT_REF || defaultBranch, + })).data.sha; + baseSha = (await github.rest.repos.getCommit({ + owner, + repo, + ref: defaultBranch, + })).data.sha; + } + } else { + headSha = (await github.rest.repos.getCommit({ owner, repo, ref: defaultBranch })).data.sha; + baseSha = headSha; + compare = false; + } + + if (!/^(all|v2-[a-z0-9-]+)$/.test(task)) { + core.setFailed("task must be all or a ClawBench v2 task ID"); + return; + } + if (!new Set(["codex", "claude-code"]).has(agent)) { + core.setFailed("agent must be codex or claude-code"); + return; + } + const concurrencyNumber = Number(concurrency); + if (!Number.isInteger(concurrencyNumber) || concurrencyNumber < 1 || concurrencyNumber > 30) { + core.setFailed("concurrency must be an integer from 1 through 30"); + return; + } + if (headSha === baseSha) compare = false; + + const shortHead = headSha.slice(0, 7); + const attempt = Number(process.env.GITHUB_RUN_ATTEMPT || "1"); + const attemptSuffix = attempt > 1 ? `-attempt${attempt}` : ""; + const subject = prNumber ? `pr-${prNumber}` : compare ? "ref" : "main"; + const experiment = `${subject}-${shortHead}-${process.env.GITHUB_RUN_ID}${attemptSuffix}`; + const candidate = prNumber ? `PR #${prNumber} (${shortHead})` : shortHead; + const title = compare + ? `ClawBench · ${candidate} vs base (${baseSha.slice(0, 7)})` + : `ClawBench · ${candidate}`; + + core.setOutput("enabled", "true"); + core.setOutput("head_sha", headSha); + core.setOutput("base_sha", baseSha); + core.setOutput("compare", String(compare)); + core.setOutput("pr_number", prNumber); + core.setOutput("task", task); + core.setOutput("agent", agent); + core.setOutput("concurrency", String(concurrencyNumber)); + core.setOutput("experiment", experiment); + core.setOutput("title", title); + + benchmark: + needs: resolve + if: needs.resolve.outputs.enabled == 'true' + runs-on: ubuntu-latest + timeout-minutes: 480 + environment: benchmarks + permissions: + contents: read + issues: write + concurrency: + group: benchmark-clawbench-${{ needs.resolve.outputs.pr_number || needs.resolve.outputs.head_sha }} + cancel-in-progress: false + env: + HYPEMAN_API_KEY: ${{ secrets.HYPEMAN_API_KEY }} + HYPEMAN_BASE_URL: ${{ vars.HYPEMAN_BASE_URL }} + KERNEL_MCP_BENCHMARK_API_KEY: ${{ secrets.KERNEL_MCP_BENCHMARK_API_KEY }} + KERNEL_PROJECT: ${{ vars.KERNEL_PROJECT }} + PURELY_MAIL_API_KEY: ${{ secrets.PURELY_MAIL_API_KEY }} + PURELY_MAIL_DOMAIN: ${{ vars.PURELY_MAIL_DOMAIN }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + CLAWBENCH_JUDGE_BASE_URL: ${{ vars.CLAWBENCH_JUDGE_BASE_URL }} + CLAWBENCH_JUDGE_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + CLAWBENCH_JUDGE_MODEL: ${{ vars.CLAWBENCH_JUDGE_MODEL }} + CLAWBENCH_JUDGE_API_TYPE: ${{ vars.CLAWBENCH_JUDGE_API_TYPE }} + BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }} + BRAINTRUST_PROJECT: ${{ vars.BRAINTRUST_PROJECT }} + HARBOR_N_CONCURRENT: ${{ needs.resolve.outputs.concurrency }} + BENCHMARK_PR_NUMBER: ${{ needs.resolve.outputs.pr_number }} + BENCHMARK_HEAD_SHA: ${{ needs.resolve.outputs.head_sha }} + steps: + - name: Mark the PR benchmark as running + if: needs.resolve.outputs.pr_number != '' + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 + env: + PR_NUMBER: ${{ needs.resolve.outputs.pr_number }} + BENCHMARK_TITLE: ${{ needs.resolve.outputs.title }} + with: + script: | + const marker = ""; + const body = `${marker}\n## ${process.env.BENCHMARK_TITLE}\n\nBenchmark running: ${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`; + const comments = await github.paginate(github.rest.issues.listComments, { + ...context.repo, + issue_number: Number(process.env.PR_NUMBER), + per_page: 100, + }); + const existing = comments.find((comment) => + comment.user?.type === "Bot" && comment.body?.includes(marker) + ); + if (existing) { + await github.rest.issues.updateComment({ ...context.repo, comment_id: existing.id, body }); + } else { + await github.rest.issues.createComment({ + ...context.repo, + issue_number: Number(process.env.PR_NUMBER), + body, + }); + } + + - name: Check out trusted benchmark tooling + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + ref: ${{ github.event.repository.default_branch }} + path: harness + persist-credentials: false + + - name: Check out candidate + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + ref: ${{ needs.resolve.outputs.head_sha }} + path: candidate + persist-credentials: false + + - name: Check out base + if: needs.resolve.outputs.compare == 'true' + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + ref: ${{ needs.resolve.outputs.base_sha }} + path: baseline + persist-credentials: false + + - name: Check out pinned ClawBench + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + repository: kernel/ClawBench + ref: 45a71c4b0c78186851c94cfc77bfe619c9e01387 + path: clawbench + persist-credentials: false + + - uses: oven-sh/setup-bun@3d267786b128fe76c2f16a390aa2448b815359f3 # v2.1.2 + with: + bun-version: "1.3.3" + + - uses: astral-sh/setup-uv@b75a909f75acd358c2196fb9a5f1299a9a8868a4 # v6.7.0 + with: + version: "0.8.17" + + - name: Install benchmark tools + run: | + bun install --cwd harness --frozen-lockfile + archive="$RUNNER_TEMP/hypeman_0.18.0_linux_amd64.tar.gz" + checksums="$RUNNER_TEMP/hypeman_0.18.0_checksums.txt" + curl --fail --location --silent --show-error \ + https://github.com/kernel/hypeman-cli/releases/download/v0.18.0/hypeman_0.18.0_linux_amd64.tar.gz \ + --output "$archive" + curl --fail --location --silent --show-error \ + https://github.com/kernel/hypeman-cli/releases/download/v0.18.0/hypeman_0.18.0_checksums.txt \ + --output "$checksums" + (cd "$RUNNER_TEMP" && grep 'hypeman_0.18.0_linux_amd64.tar.gz$' "$checksums" | sha256sum --check --strict) + tar -xzf "$archive" -C "$RUNNER_TEMP" hypeman + sudo install "$RUNNER_TEMP/hypeman" /usr/local/bin/hypeman + uv sync --directory clawbench --frozen + + - name: Build candidate image + working-directory: candidate + run: ./benchmarks/harbor/build-image.sh + + - name: Build base image + if: needs.resolve.outputs.compare == 'true' + working-directory: baseline + run: ./benchmarks/harbor/build-image.sh + + - name: Run synchronized benchmark arms + id: run + shell: bash + env: + CLAWBENCH_REPO: ${{ github.workspace }}/clawbench + CLAWBENCH_REF: 45a71c4b0c78186851c94cfc77bfe619c9e01387 + HARBOR_BENCHMARK_TIMEOUT: 7h + BENCHMARK_AGENT: ${{ needs.resolve.outputs.agent }} + BENCHMARK_TASK: ${{ needs.resolve.outputs.task }} + BENCHMARK_COMPARE: ${{ needs.resolve.outputs.compare }} + BENCHMARK_EXPERIMENT: ${{ needs.resolve.outputs.experiment }} + run: | + set -u + jobs_root="$RUNNER_TEMP/harbor-jobs" + mkdir -p "$jobs_root" + candidate_job="candidate-$BENCHMARK_EXPERIMENT" + baseline_job="baseline-$BENCHMARK_EXPERIMENT" + + run_arm() { + local checkout=$1 arm=$2 job_name=$3 + set +e + "$checkout/benchmarks/harbor/clawbench/run.sh" \ + "$BENCHMARK_AGENT" \ + "$BENCHMARK_TASK" \ + "$job_name" \ + "$jobs_root/$arm" \ + >"$RUNNER_TEMP/$arm.log" 2>&1 + echo $? >"$RUNNER_TEMP/$arm.status" + } + + run_arm "$GITHUB_WORKSPACE/candidate" candidate "$candidate_job" & + candidate_pid=$! + if [[ "$BENCHMARK_COMPARE" == "true" ]]; then + run_arm "$GITHUB_WORKSPACE/baseline" baseline "$baseline_job" & + baseline_pid=$! + fi + + while kill -0 "$candidate_pid" 2>/dev/null || { [[ -n "${baseline_pid:-}" ]] && kill -0 "$baseline_pid" 2>/dev/null; }; do + echo "ClawBench is still running at $(date -u +%Y-%m-%dT%H:%M:%SZ)" + sleep 60 + done + wait "$candidate_pid" + if [[ -n "${baseline_pid:-}" ]]; then wait "$baseline_pid"; fi + + candidate_status=$(cat "$RUNNER_TEMP/candidate.status") + baseline_status=0 + if [[ -f "$RUNNER_TEMP/baseline.status" ]]; then + baseline_status=$(cat "$RUNNER_TEMP/baseline.status") + fi + echo "candidate_status=$candidate_status" >>"$GITHUB_OUTPUT" + echo "baseline_status=$baseline_status" >>"$GITHUB_OUTPUT" + echo "candidate_dir=$jobs_root/candidate/$candidate_job" >>"$GITHUB_OUTPUT" + echo "baseline_dir=$jobs_root/baseline/$baseline_job" >>"$GITHUB_OUTPUT" + + if ((candidate_status != 0)); then tail -100 "$RUNNER_TEMP/candidate.log" >&2; fi + if ((baseline_status != 0)); then tail -100 "$RUNNER_TEMP/baseline.log" >&2; fi + + - name: Publish Braintrust experiment + id: publish + continue-on-error: true + shell: bash + env: + CANDIDATE_DIR: ${{ steps.run.outputs.candidate_dir }} + BASELINE_DIR: ${{ steps.run.outputs.baseline_dir }} + BENCHMARK_EXPERIMENT: ${{ needs.resolve.outputs.experiment }} + run: | + args=() + [[ -f "$CANDIDATE_DIR/result.json" ]] && \ + args+=(--arm "candidate=$CANDIDATE_DIR") + [[ -f "$BASELINE_DIR/result.json" ]] && \ + args+=(--arm "baseline=$BASELINE_DIR") + ((${#args[@]} > 0)) || { echo "No completed Harbor job to publish" >&2; exit 1; } + bun harness/benchmarks/harbor/publish-braintrust.ts \ + --experiment "$BENCHMARK_EXPERIMENT" \ + --output "$RUNNER_TEMP/publication.json" \ + "${args[@]}" + + - name: Render benchmark report + id: report + if: always() + continue-on-error: true + shell: bash + env: + CANDIDATE_DIR: ${{ steps.run.outputs.candidate_dir }} + BASELINE_DIR: ${{ steps.run.outputs.baseline_dir }} + BENCHMARK_TITLE: ${{ needs.resolve.outputs.title }} + run: | + args=() + [[ -f "$CANDIDATE_DIR/result.json" ]] && \ + args+=(--arm "candidate=$CANDIDATE_DIR") + [[ -f "$BASELINE_DIR/result.json" ]] && \ + args+=(--arm "baseline=$BASELINE_DIR") + ((${#args[@]} > 0)) || { echo "No completed Harbor job to report" >&2; exit 1; } + publication=() + [[ -f "$RUNNER_TEMP/publication.json" ]] && \ + publication=(--publication "$RUNNER_TEMP/publication.json") + bun harness/benchmarks/harbor/report.ts \ + --title "$BENCHMARK_TITLE" \ + --json "$RUNNER_TEMP/benchmark-summary.json" \ + --markdown "$RUNNER_TEMP/benchmark-summary.md" \ + "${publication[@]}" \ + "${args[@]}" + cat "$RUNNER_TEMP/benchmark-summary.md" >>"$GITHUB_STEP_SUMMARY" + + - name: Update PR benchmark comment + if: always() && needs.resolve.outputs.pr_number != '' + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 + env: + PR_NUMBER: ${{ needs.resolve.outputs.pr_number }} + BENCHMARK_TITLE: ${{ needs.resolve.outputs.title }} + REPORT_OUTCOME: ${{ steps.report.outcome }} + with: + script: | + const fs = require("fs"); + const marker = ""; + const reportPath = `${process.env.RUNNER_TEMP}/benchmark-summary.md`; + const body = fs.existsSync(reportPath) + ? fs.readFileSync(reportPath, "utf8") + : `${marker}\n## ${process.env.BENCHMARK_TITLE}\n\nBenchmark failed before a report was produced. [Open the workflow run](${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}).`; + const comments = await github.paginate(github.rest.issues.listComments, { + ...context.repo, + issue_number: Number(process.env.PR_NUMBER), + per_page: 100, + }); + const existing = comments.find((comment) => + comment.user?.type === "Bot" && comment.body?.includes(marker) + ); + if (existing) { + await github.rest.issues.updateComment({ ...context.repo, comment_id: existing.id, body }); + } else { + await github.rest.issues.createComment({ + ...context.repo, + issue_number: Number(process.env.PR_NUMBER), + body, + }); + } + + - name: Check benchmark execution + if: always() + shell: bash + env: + CANDIDATE_STATUS: ${{ steps.run.outputs.candidate_status }} + BASELINE_STATUS: ${{ steps.run.outputs.baseline_status }} + PUBLISH_OUTCOME: ${{ steps.publish.outcome }} + REPORT_OUTCOME: ${{ steps.report.outcome }} + run: | + [[ "$CANDIDATE_STATUS" == "0" ]] + [[ "$BASELINE_STATUS" == "0" ]] + [[ "$PUBLISH_OUTCOME" == "success" ]] + [[ "$REPORT_OUTCOME" == "success" ]] diff --git a/benchmarks/harbor/README.md b/benchmarks/harbor/README.md index bde47d6..ba5754b 100644 --- a/benchmarks/harbor/README.md +++ b/benchmarks/harbor/README.md @@ -18,11 +18,12 @@ The image records the current Git SHA, and the generated task records the ClawBe - `uv`, Harbor 0.21.0, and `harbor-hypeman` 0.1.1 - Hypeman CLI credentials -- a ClawBench checkout containing `df6743f` from `kernel/ClawBench` PR #1 -- `KERNEL_MCP_BENCHMARK_API_KEY` scoped to an isolated evaluation project +- a ClawBench checkout containing pinned commit `45a71c4` +- `KERNEL_MCP_BENCHMARK_API_KEY` scoped to an isolated evaluation project, plus its `KERNEL_PROJECT` name - `PURELY_MAIL_API_KEY` and `PURELY_MAIL_DOMAIN` for ClawBench account tasks - `OPENAI_API_KEY` for Codex, or Anthropic credentials for Claude Code - the ClawBench judge variables when using a hosted judge: `CLAWBENCH_JUDGE_BASE_URL`, `CLAWBENCH_JUDGE_API_KEY`, `CLAWBENCH_JUDGE_MODEL`, and `CLAWBENCH_JUDGE_API_TYPE` +- `BRAINTRUST_API_KEY` and `BRAINTRUST_PROJECT` when publishing results ## Build the trial image @@ -54,6 +55,18 @@ Codex defaults to version `0.120.0` with `gpt-5.6-luna`. Claude Code defaults to Single-task runs have a 40-minute wall-clock limit. Full-suite runs default to six hours. Set `HARBOR_BENCHMARK_TIMEOUT` to override either limit. Set `HARBOR_JOBS_DIR` to choose where Harbor writes results. +## GitHub Actions + +The `Benchmark ClawBench` workflow runs the complete suite weekly and on demand. Select it from the Actions tab and provide either a same-repository PR number or a ref. PR runs compare the candidate SHA with its base SHA by default. + +A repository collaborator can also start the full PR comparison by commenting this exact command on a same-repository pull request: + +```text +/benchmark clawbench +``` + +The command parser does not execute comment text. It accepts only the exact command, rejects fork pull requests and non-collaborators, and resolves the candidate and base SHAs through GitHub's pull-request API. The workflow uses the `benchmarks` environment for credentials, updates one benchmark comment on the pull request, and publishes the same results to Braintrust. + ## Results Harbor writes its normal job directory, including: @@ -64,3 +77,25 @@ Harbor writes its normal job directory, including: - `kernel-mcp-result.json`: local-source and same-browser wiring details - `recording.mp4`: the finalized Kernel replay - `kernel-mcp/`: local server logs and the source/session manifest + +Generate a redacted summary from one or more completed jobs: + +```bash +bun run benchmark:report -- \ + --arm candidate=/path/to/candidate-job \ + --arm baseline=/path/to/baseline-job \ + --json /tmp/benchmark-summary.json \ + --markdown /tmp/benchmark-summary.md +``` + +Publish those arms as one idempotent Braintrust experiment: + +```bash +BRAINTRUST_PROJECT=kernel-mcp-server-benchmarks \ + bun run benchmark:publish -- \ + --experiment pr-162-a60c518-example \ + --arm candidate=/path/to/candidate-job \ + --arm baseline=/path/to/baseline-job +``` + +The experiment name and deterministic row/span IDs make it safe to publish the same job directories again. Rows contain task identity, numeric rewards, provenance, bounded errors, timing, token, call, and cost metrics. ATIF agent/tool activity is attached as child spans after secret redaction. Task instructions, ground truth, browser session URLs, and recordings are not placed on experiment rows or public pull-request comments. diff --git a/benchmarks/harbor/clawbench/prepare-task.py b/benchmarks/harbor/clawbench/prepare-task.py index 67bdec2..b724f23 100755 --- a/benchmarks/harbor/clawbench/prepare-task.py +++ b/benchmarks/harbor/clawbench/prepare-task.py @@ -52,6 +52,7 @@ def _add_environment( f"CLAWBENCH_SOURCE_SHA = {json.dumps(clawbench_sha)}", f"KERNEL_MCP_ENABLED_TOOLSETS = {json.dumps(ENABLED_TOOLSETS)}", 'API_BASE_URL = "${KERNEL_API_BASE_URL:-}"', + 'KERNEL_PROJECT = "${KERNEL_PROJECT:-}"', 'REDIS_URL = "redis://127.0.0.1:6379"', ] ) diff --git a/benchmarks/harbor/clawbench/run.sh b/benchmarks/harbor/clawbench/run.sh index bf59f24..847c85f 100755 --- a/benchmarks/harbor/clawbench/run.sh +++ b/benchmarks/harbor/clawbench/run.sh @@ -19,7 +19,7 @@ repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd) benchmark_dir="$repo_root/benchmarks/harbor" image_env="$benchmark_dir/.image.env" clawbench_repo=${CLAWBENCH_REPO:-$repo_root/../ClawBench} -clawbench_ref=${CLAWBENCH_REF:-df6743fd8abcd09cb7636ef8c310dd4db016162c} +clawbench_ref=${CLAWBENCH_REF:-45a71c4b0c78186851c94cfc77bfe619c9e01387} [[ -f "$image_env" ]] || { echo "Missing $image_env; run benchmarks/harbor/build-image.sh first" >&2 @@ -105,6 +105,8 @@ cat >"$runtime_env" <; + observation?: { + results?: Array<{ source_call_id?: string; content?: unknown }>; + }; + metrics?: Record; +} + +interface BraintrustEvent { + id: string; + span_id: string; + root_span_id: string; + span_parents: string[]; + span_attributes: { name: string; type: "eval" | "llm" | "tool" }; + created?: string; + input?: unknown; + output?: unknown; + expected?: unknown; + error?: string; + scores?: Record; + metadata?: Record; + metrics?: Record; + _is_merge: false; +} + +interface BraintrustProject { + id: string; + org_id: string; + name: string; +} + +interface BraintrustExperiment { + id: string; + project_id: string; + name: string; +} + +function parseArgs(args: string[]): CliOptions { + const options: CliOptions = { arms: [] }; + for (let index = 0; index < args.length; index += 1) { + const flag = args[index]; + const value = args[index + 1]; + if (!value || !flag.startsWith("--")) + throw new Error(`Missing value for ${flag}`); + index += 1; + switch (flag) { + case "--arm": + options.arms.push(value); + break; + case "--experiment": + options.experiment = value; + break; + case "--project": + options.project = value; + break; + case "--output": + options.output = value; + break; + default: + throw new Error(`Unknown argument ${flag}`); + } + } + if (options.arms.length === 0) + throw new Error("At least one --arm name=/job/path is required"); + return options; +} + +function uuidV5(name: string): string { + const namespace = Buffer.from("cf5141b9e00051a9b55482e0567b5c88", "hex"); + const digest = createHash("sha1") + .update(namespace) + .update(name) + .digest() + .subarray(0, 16); + digest[6] = (digest[6] & 0x0f) | 0x50; + digest[8] = (digest[8] & 0x3f) | 0x80; + const hex = digest.toString("hex"); + return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`; +} + +function number(value: unknown): number | undefined { + return typeof value === "number" && Number.isFinite(value) + ? value + : undefined; +} + +function trajectorySteps(trial: BenchmarkTrial): AtifStep[] { + if (!trial.trajectoryPath) return []; + const trajectory = JSON.parse(readFileSync(trial.trajectoryPath, "utf8")) as { + steps?: AtifStep[]; + }; + return Array.isArray(trajectory.steps) ? trajectory.steps : []; +} + +function metricRecord(trial: BenchmarkTrial): Record { + return Object.fromEntries( + Object.entries({ + start: trial.metrics.start, + end: trial.metrics.end, + input_tokens: trial.metrics.inputTokens, + cached_tokens: trial.metrics.cacheTokens, + output_tokens: trial.metrics.outputTokens, + cost_usd: trial.metrics.costUsd, + duration_ms: trial.metrics.durationMs, + tool_calls: trial.metrics.toolCalls, + }).filter((entry): entry is [string, number] => entry[1] !== undefined), + ); +} + +function trialMetadata(trial: BenchmarkTrial): Record { + return { + trialName: trial.trialName, + arm: trial.arm, + verdict: + trial.errorClass === "infra" + ? "error" + : trial.scores.ungraded_rate === 1 + ? "ungraded" + : trial.scores.accuracy === 1 + ? "correct" + : "false_negative", + agent: trial.agent, + agentVersion: trial.agentVersion, + agentConfigHash: trial.agentConfigHash, + model: trial.model, + kernelMcpSha: trial.kernelMcpSha, + clawbenchSha: trial.clawbenchSha, + errorClass: trial.errorClass, + rewards: trial.rewards, + durationMs: trial.metrics.durationMs, + }; +} + +function atifEvents(trial: BenchmarkTrial, rowId: string): BraintrustEvent[] { + const events: BraintrustEvent[] = []; + for (const step of trajectorySteps(trial).filter( + (candidate) => candidate.source === "agent", + )) { + const stepId = step.step_id ?? 0; + const llmId = uuidV5(`${rowId}:llm:${stepId}`); + const start = step.timestamp + ? Date.parse(step.timestamp) / 1000 + : undefined; + const llmMetrics = Object.fromEntries( + Object.entries({ + start, + end: start, + prompt_tokens: number(step.metrics?.prompt_tokens), + completion_tokens: number(step.metrics?.completion_tokens), + cost_usd: number(step.metrics?.cost_usd), + }).filter((entry): entry is [string, number] => entry[1] !== undefined), + ); + events.push({ + id: llmId, + span_id: llmId, + root_span_id: rowId, + span_parents: [rowId], + span_attributes: { name: "agent", type: "llm" }, + created: step.timestamp, + output: redactValue(step.message), + metadata: { + phase: "agent_execution", + stepId, + model: step.model_name ?? trial.model, + }, + metrics: llmMetrics, + _is_merge: false, + }); + + for (const [toolIndex, call] of (step.tool_calls ?? []).entries()) { + const toolId = uuidV5( + `${rowId}:tool:${stepId}:${call.tool_call_id ?? toolIndex}`, + ); + const observation = step.observation?.results?.find( + (result) => result.source_call_id === call.tool_call_id, + ); + events.push({ + id: toolId, + span_id: toolId, + root_span_id: rowId, + span_parents: [llmId], + span_attributes: { name: call.function_name ?? "tool", type: "tool" }, + created: step.timestamp, + input: redactValue(call.arguments), + output: redactValue(observation?.content), + metadata: { + phase: "agent_execution", + stepId, + toolCallId: call.tool_call_id, + }, + metrics: start === undefined ? undefined : { start: start, end: start }, + _is_merge: false, + }); + } + } + return events; +} + +export function buildExperimentEvents( + arms: BenchmarkArm[], + experimentName: string, +): BraintrustEvent[] { + const events: BraintrustEvent[] = []; + for (const arm of arms) { + for (const trial of arm.trials) { + const rowId = uuidV5(`${experimentName}:${arm.name}:${trial.id}`); + const reward = + trial.errorClass === "infra" + ? undefined + : (trial.rewards.reward ?? trial.rewards.reward_lenient); + events.push({ + id: rowId, + span_id: rowId, + root_span_id: rowId, + span_parents: [], + span_attributes: { name: trial.taskName, type: "eval" }, + created: trial.startedAt, + input: { source: trial.source, taskName: trial.taskName }, + output: { + reward, + rewardKey: reward === undefined ? undefined : "reward", + error: trial.error ? redactString(trial.error, 400) : undefined, + }, + expected: { reward: 1 }, + error: trial.error ? redactString(trial.error, 400) : undefined, + scores: trial.scores, + metadata: trialMetadata(trial), + metrics: metricRecord(trial), + _is_merge: false, + }); + events.push(...atifEvents(trial, rowId)); + } + } + return events; +} + +function experimentMetadata(arms: BenchmarkArm[]): Record { + return { + product: "kernel-mcp-server", + execution_mode: "harbor", + benchmark: "clawbench", + harborVersion: "0.21.0", + environment: "ci", + gitSha: process.env.BENCHMARK_HEAD_SHA ?? process.env.GITHUB_SHA, + githubRunId: process.env.GITHUB_RUN_ID, + githubRunAttempt: process.env.GITHUB_RUN_ATTEMPT, + githubEvent: process.env.GITHUB_EVENT_NAME, + pullRequest: process.env.BENCHMARK_PR_NUMBER || undefined, + concurrency: process.env.HARBOR_N_CONCURRENT, + benchByArm: Object.fromEntries( + arms.map((arm) => [arm.name, summarizeArm(arm)]), + ), + sources: Object.fromEntries( + arms.map((arm) => [ + arm.name, + { + jobId: arm.jobId, + jobName: arm.jobName, + startedAt: arm.startedAt, + finishedAt: arm.finishedAt, + stats: { + nTotalTrials: arm.nTotalTrials, + nCompletedTrials: arm.nCompletedTrials, + nErroredTrials: arm.nErroredTrials, + nCancelledTrials: arm.nCancelledTrials, + nRetries: arm.nRetries, + }, + kernelMcpShas: [ + ...new Set(arm.trials.flatMap((trial) => trial.kernelMcpSha ?? [])), + ], + clawbenchShas: [ + ...new Set(arm.trials.flatMap((trial) => trial.clawbenchSha ?? [])), + ], + }, + ]), + ), + }; +} + +class BraintrustApi { + private readonly baseUrl = + process.env.BRAINTRUST_API_URL ?? "https://api.braintrust.dev"; + + constructor(private readonly apiKey: string) {} + + async request(path: string, method = "GET", body?: unknown): Promise { + const response = await fetch(`${this.baseUrl}${path}`, { + method, + headers: { + Authorization: `Bearer ${this.apiKey}`, + "Content-Type": "application/json", + }, + body: body === undefined ? undefined : JSON.stringify(body), + }); + if (!response.ok) { + throw new Error( + `Braintrust ${method} ${path} returned ${response.status}: ${redactString(await response.text(), 400)}`, + ); + } + return (await response.json()) as T; + } +} + +async function insertEvents( + api: BraintrustApi, + experimentId: string, + events: BraintrustEvent[], +): Promise { + const batchSize = 100; + for (let offset = 0; offset < events.length; offset += batchSize) { + await api.request(`/v1/experiment/${experimentId}/insert`, "POST", { + events: events.slice(offset, offset + batchSize), + }); + } +} + +export async function publishBenchmark( + arms: BenchmarkArm[], + projectName: string, + experimentName: string, + apiKey: string, +): Promise> { + const api = new BraintrustApi(apiKey); + const project = await api.request("/v1/project", "POST", { + name: projectName, + }); + const experiment = await api.request( + "/v1/experiment", + "POST", + { + project_id: project.id, + name: experimentName, + public: false, + metadata: experimentMetadata(arms), + }, + ); + const events = buildExperimentEvents(arms, experimentName); + await insertEvents(api, experiment.id, events); + const organization = await api.request<{ name: string }>( + `/v1/organization/${project.org_id}`, + ); + const url = `https://www.braintrust.dev/app/${encodeURIComponent(organization.name)}/p/${encodeURIComponent(project.name)}/experiments/${encodeURIComponent(experiment.name)}`; + return { + project: project.name, + experiment: experiment.name, + experimentId: experiment.id, + url, + rows: arms.reduce((total, arm) => total + arm.trials.length, 0), + spans: events.length, + }; +} + +async function main(): Promise { + const options = parseArgs(process.argv.slice(2)); + const project = options.project ?? process.env.BRAINTRUST_PROJECT; + const experimentName = options.experiment; + const apiKey = process.env.BRAINTRUST_API_KEY; + if (!project) throw new Error("BRAINTRUST_PROJECT or --project is required"); + if (!experimentName) throw new Error("--experiment is required"); + if (!apiKey) throw new Error("BRAINTRUST_API_KEY is required"); + + const arms = options.arms.map(parseArmSpec).map(readBenchmarkArm); + const publication = await publishBenchmark( + arms, + project, + experimentName, + apiKey, + ); + const serialized = `${JSON.stringify(publication, undefined, 2)}\n`; + if (options.output) { + mkdirSync(dirname(options.output), { recursive: true }); + writeFileSync(options.output, serialized); + } + process.stdout.write(serialized); +} + +if (import.meta.main) { + main().catch((error) => { + console.error(error instanceof Error ? error.message : error); + process.exitCode = 1; + }); +} diff --git a/benchmarks/harbor/redact.ts b/benchmarks/harbor/redact.ts new file mode 100644 index 0000000..7b5d551 --- /dev/null +++ b/benchmarks/harbor/redact.ts @@ -0,0 +1,53 @@ +const SECRET_NAME = /(API_KEY|TOKEN|SECRET|PASSWORD|PRIVATE_KEY|CREDENTIAL)/i; +const REDACTED = "[REDACTED]"; + +function secretValues(): string[] { + return Object.entries(process.env) + .filter( + ([name, value]) => SECRET_NAME.test(name) && value && value.length >= 6, + ) + .map(([, value]) => value as string) + .sort((left, right) => right.length - left.length); +} + +export function redactString(value: string, maxLength = 20_000): string { + let redacted = value; + for (const secret of secretValues()) { + redacted = redacted.split(secret).join(REDACTED); + } + + redacted = redacted + .replace(/\bBearer\s+[A-Za-z0-9._~+/=-]+/gi, `Bearer ${REDACTED}`) + .replace(/\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b/g, REDACTED) + .replace(/\b(?:sk|pk|bt|kapi|whsec)[-_][A-Za-z0-9_-]{12,}\b/gi, REDACTED) + .replace( + /(["']?(?:api[_-]?key|access[_-]?token|credential|password|secret)["']?\s*[:=]\s*["']?)[^"'\s,}&]+/gi, + `$1${REDACTED}`, + ) + .replace( + /([?&](?:api[_-]?key|access[_-]?token|auth|code|credential|password|secret|session[_-]?token)=)[^&#\s]+/gi, + `$1${REDACTED}`, + ) + .replace(/(wss?:\/\/)[^/@\s]+@/gi, `$1${REDACTED}@`) + .replace(/\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi, "[REDACTED_EMAIL]"); + + return redacted.length > maxLength + ? `${redacted.slice(0, maxLength)}…` + : redacted; +} + +export function redactValue(value: unknown, maxStringLength = 20_000): unknown { + if (typeof value === "string") return redactString(value, maxStringLength); + if (Array.isArray(value)) { + return value.map((entry) => redactValue(entry, maxStringLength)); + } + if (value !== null && typeof value === "object") { + return Object.fromEntries( + Object.entries(value as Record).map(([key, entry]) => [ + key, + SECRET_NAME.test(key) ? REDACTED : redactValue(entry, maxStringLength), + ]), + ); + } + return value; +} diff --git a/benchmarks/harbor/report.ts b/benchmarks/harbor/report.ts new file mode 100644 index 0000000..0e24f49 --- /dev/null +++ b/benchmarks/harbor/report.ts @@ -0,0 +1,151 @@ +#!/usr/bin/env bun +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { dirname } from "node:path"; +import { + type ArmSummary, + parseArmSpec, + readBenchmarkArm, + summarizeArm, +} from "./results"; + +interface Options { + arms: string[]; + json?: string; + markdown?: string; + publication?: string; + title: string; +} + +function parseArgs(args: string[]): Options { + const options: Options = { arms: [], title: "ClawBench benchmark" }; + for (let index = 0; index < args.length; index += 1) { + const flag = args[index]; + const value = args[index + 1]; + if (!value || !flag.startsWith("--")) + throw new Error(`Missing value for ${flag}`); + index += 1; + switch (flag) { + case "--arm": + options.arms.push(value); + break; + case "--json": + options.json = value; + break; + case "--markdown": + options.markdown = value; + break; + case "--publication": + options.publication = value; + break; + case "--title": + options.title = value; + break; + default: + throw new Error(`Unknown argument ${flag}`); + } + } + if (options.arms.length === 0) + throw new Error("At least one --arm name=/job/path is required"); + return options; +} + +function ratio(value: number | undefined, denominator: number): string { + return value === undefined ? "—" : `${value}/${denominator}`; +} + +function duration(value: number | undefined): string { + return value === undefined ? "—" : `${Math.round(value / 1000)}s`; +} + +function cost(value: number | undefined): string { + return value === undefined ? "—" : `$${value.toFixed(4)}`; +} + +function markdown( + title: string, + summaries: ArmSummary[], + publication?: Record, +): string { + const lines = [ + "", + `## ${title}`, + "", + "| Arm | Lenient | Strict | Intercepted | Infra | Ungraded | Kernel MCP valid | Median calls | Median duration | Cost |", + "|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|", + ]; + for (const summary of summaries) { + lines.push( + `| ${summary.arm} | ${ratio(summary.lenient, summary.trials)} | ${ratio(summary.strict, summary.trials)} | ${ratio(summary.intercepted, summary.trials)} | ${summary.infraErrors} | ${summary.ungraded} | ${ratio(summary.kernelMcpValid, summary.kernelMcpChecked)} | ${summary.medianCalls ?? "—"} | ${duration(summary.medianDurationMs)} | ${cost(summary.totalCostUsd)} |`, + ); + } + + const candidate = summaries.find((summary) => summary.arm === "candidate"); + const baseline = summaries.find((summary) => summary.arm === "baseline"); + if (candidate && baseline) { + const signed = (value: number) => `${value >= 0 ? "+" : ""}${value}`; + const deltas = [ + `**${signed(candidate.lenient - baseline.lenient)} lenient**`, + candidate.strict !== undefined && baseline.strict !== undefined + ? `**${signed(candidate.strict - baseline.strict)} strict**` + : undefined, + `**${signed(candidate.intercepted - baseline.intercepted)} intercepted**`, + ].filter(Boolean); + lines.push("", `Candidate minus baseline: ${deltas.join(", ")}.`); + } + if (typeof publication?.url === "string") { + lines.push("", `[Open the Braintrust experiment](${publication.url})`); + } + if ( + process.env.GITHUB_SERVER_URL && + process.env.GITHUB_REPOSITORY && + process.env.GITHUB_RUN_ID + ) { + lines.push( + "", + `[Open the GitHub Actions run](${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID})`, + ); + } + lines.push( + "", + "Lenient reward is the primary ClawBench score. Infrastructure failures remain in the intended-task denominator.", + "", + ); + return lines.join("\n"); +} + +function write(path: string | undefined, content: string): void { + if (!path) return; + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, content); +} + +function main(): void { + const options = parseArgs(process.argv.slice(2)); + const arms = options.arms.map(parseArmSpec).map(readBenchmarkArm); + const summaries = arms.map(summarizeArm); + const publication = options.publication + ? (JSON.parse(readFileSync(options.publication, "utf8")) as Record< + string, + unknown + >) + : undefined; + const result = { + benchmark: "clawbench", + generatedAt: new Date().toISOString(), + arms: summaries, + publication, + }; + const rendered = markdown(options.title, summaries, publication); + write(options.json, `${JSON.stringify(result, undefined, 2)}\n`); + write(options.markdown, rendered); + process.stdout.write(rendered); +} + +if (import.meta.main) { + try { + main(); + } catch (error) { + console.error(error instanceof Error ? error.message : error); + process.exitCode = 1; + } +} diff --git a/benchmarks/harbor/results.test.ts b/benchmarks/harbor/results.test.ts new file mode 100644 index 0000000..e2ac705 --- /dev/null +++ b/benchmarks/harbor/results.test.ts @@ -0,0 +1,252 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { buildExperimentEvents, publishBenchmark } from "./publish-braintrust"; +import { readBenchmarkArm, summarizeArm } from "./results"; +import { redactString, redactValue } from "./redact"; + +const temporaryDirectories: string[] = []; + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) { + rmSync(directory, { recursive: true, force: true }); + } +}); + +function writeJson(path: string, value: unknown): void { + mkdirSync(join(path, ".."), { recursive: true }); + writeFileSync(path, JSON.stringify(value)); +} + +function fixture(): string { + const root = mkdtempSync(join(tmpdir(), "harbor-results-")); + temporaryDirectories.push(root); + writeJson(join(root, "config.json"), { job_name: "test-job" }); + writeJson(join(root, "result.json"), { + id: "job-id", + n_total_trials: 2, + stats: { + n_completed_trials: 1, + n_errored_trials: 1, + n_cancelled_trials: 0, + n_retries: 0, + }, + }); + + const success = join(root, "task-one__abc"); + writeJson(join(success, "result.json"), { + id: "trial-one", + task_name: "clawbench/v2-task-one", + trial_name: "task-one__abc", + source: "clawbench-v2", + config: { + agent: { + name: "codex", + model_name: "gpt-5.6-luna", + kwargs: { version: "0.120.0" }, + }, + }, + verifier_result: { + rewards: { + reward: 1, + reward_lenient: 1, + reward_strict: 0, + intercepted: 1, + kernel_mcp_valid: 1, + }, + }, + started_at: "2026-01-01T00:00:00Z", + finished_at: "2026-01-01T00:01:00Z", + step_results: [ + { + agent_result: { + n_input_tokens: 100, + n_cache_tokens: 80, + n_output_tokens: 20, + cost_usd: 0.01, + }, + }, + ], + }); + writeJson(join(success, "steps/run/agent/trajectory.json"), { + steps: [ + { + step_id: 1, + source: "agent", + timestamp: "2026-01-01T00:00:01Z", + message: "working", + tool_calls: [ + { + tool_call_id: "call-1", + function_name: "execute_playwright_code", + arguments: { code: "return 'done'" }, + }, + ], + observation: { + results: [{ source_call_id: "call-1", content: "done" }], + }, + }, + ], + }); + writeJson(join(success, "steps/run/verifier/kernel-mcp/run-manifest.json"), { + kernel_mcp_server_sha: "server-sha", + clawbench_source_sha: "clawbench-sha", + }); + + const failed = join(root, "task-two__def"); + writeJson(join(failed, "result.json"), { + id: "trial-two", + task_name: "clawbench/v2-task-two", + trial_name: "task-two__def", + config: { agent: { name: "codex", model_name: "gpt-5.6-luna" } }, + exception_info: { type: "ExecProtocolError", message: "setup failed" }, + verifier_result: { rewards: { reward: 0, intercepted: 0 } }, + }); + return root; +} + +describe("Harbor result ingestion", () => { + test("keeps infrastructure errors out of task-quality scores", () => { + const arm = readBenchmarkArm({ name: "candidate", path: fixture() }); + expect(arm.trials).toHaveLength(2); + expect(arm.trials[0].scores).toEqual({ + accuracy: 1, + false_positive_rate: 0, + false_negative_rate: 0, + ungraded_rate: 0, + reward: 1, + reward_lenient: 1, + reward_strict: 0, + intercepted: 1, + kernel_mcp_valid: 1, + }); + expect(arm.trials[1].scores).toEqual({ + infra_error_rate: 1, + ungraded_rate: 1, + }); + }); + + test("summarizes against the intended task denominator", () => { + const summary = summarizeArm( + readBenchmarkArm({ name: "candidate", path: fixture() }), + ); + expect(summary).toMatchObject({ + trials: 2, + lenient: 1, + strict: 0, + intercepted: 1, + infraErrors: 1, + ungraded: 1, + kernelMcpValid: 1, + medianCalls: 1, + totalCostUsd: 0.01, + }); + }); + + test("builds deterministic root, llm, and tool spans", () => { + const arm = readBenchmarkArm({ name: "candidate", path: fixture() }); + const first = buildExperimentEvents([arm], "test-experiment"); + const second = buildExperimentEvents([arm], "test-experiment"); + expect(first).toEqual(second); + expect( + first.filter((event) => event.span_attributes.type === "eval"), + ).toHaveLength(2); + expect( + first.filter((event) => event.span_attributes.type === "llm"), + ).toHaveLength(1); + expect( + first.filter((event) => event.span_attributes.type === "tool"), + ).toHaveLength(1); + const root = first.find((event) => event.span_attributes.type === "eval"); + expect(root?.input).toEqual({ + source: "clawbench-v2", + taskName: "v2-task-one", + }); + expect(root?.span_parents).toEqual([]); + const infra = first.find( + (event) => + event.span_attributes.type === "eval" && + (event.output as { error?: string }).error, + ); + expect(infra?.scores).toEqual({ infra_error_rate: 1, ungraded_rate: 1 }); + expect(infra?.output).not.toHaveProperty("reward", 0); + }); + + test("re-publishes the same rows and spans by deterministic ID", async () => { + const arm = readBenchmarkArm({ name: "candidate", path: fixture() }); + const originalFetch = globalThis.fetch; + const inserts: string[][] = []; + globalThis.fetch = (async (request, init) => { + const url = String(request); + if (url.endsWith("/v1/project")) { + return Response.json({ + id: "project-id", + org_id: "org-id", + name: "project name", + }); + } + if (url.endsWith("/v1/experiment")) { + return Response.json({ + id: "experiment-id", + project_id: "project-id", + name: "experiment name", + }); + } + if (url.includes("/insert")) { + const body = JSON.parse(String(init?.body)) as { + events: Array<{ id: string }>; + }; + inserts.push(body.events.map((event) => event.id)); + return Response.json({ row_ids: body.events.map((event) => event.id) }); + } + if (url.endsWith("/v1/organization/org-id")) { + return Response.json({ name: "Kernel" }); + } + return new Response("not found", { status: 404 }); + }) as typeof fetch; + + try { + const first = await publishBenchmark( + [arm], + "project name", + "experiment name", + "test-key", + ); + const second = await publishBenchmark( + [arm], + "project name", + "experiment name", + "test-key", + ); + expect(first).toEqual(second); + expect(inserts).toHaveLength(2); + expect(inserts[0]).toEqual(inserts[1]); + expect(first.url).toBe( + "https://www.braintrust.dev/app/Kernel/p/project%20name/experiments/experiment%20name", + ); + } finally { + globalThis.fetch = originalFetch; + } + }); +}); + +describe("Braintrust redaction", () => { + test("redacts configured secrets and credential-shaped strings", () => { + process.env.TEST_API_KEY = "super-secret-value"; + expect( + redactString( + 'Bearer super-secret-value sk-proj-abcdefghijklmnop?access_token=visible "password":"generated-password" user@example.com', + ), + ).toBe( + 'Bearer [REDACTED] [REDACTED]?access_token=[REDACTED] "password":"[REDACTED]" [REDACTED_EMAIL]', + ); + expect( + redactValue({ api_key: "visible", nested: ["bt-abcdefghijklmnop"] }), + ).toEqual({ + api_key: "[REDACTED]", + nested: ["[REDACTED]"], + }); + delete process.env.TEST_API_KEY; + }); +}); diff --git a/benchmarks/harbor/results.ts b/benchmarks/harbor/results.ts new file mode 100644 index 0000000..1a27396 --- /dev/null +++ b/benchmarks/harbor/results.ts @@ -0,0 +1,388 @@ +import { createHash } from "node:crypto"; +import { existsSync, readdirSync, readFileSync } from "node:fs"; +import { basename, join, resolve } from "node:path"; + +export type JsonObject = Record; + +export interface ArmInput { + name: string; + path: string; +} + +export interface BenchmarkMetrics { + inputTokens?: number; + cacheTokens?: number; + outputTokens?: number; + costUsd?: number; + durationMs?: number; + toolCalls?: number; + start?: number; + end?: number; +} + +export interface BenchmarkTrial { + arm: string; + id: string; + taskName: string; + trialName: string; + source: string; + agent: string; + agentVersion?: string; + agentConfigHash: string; + model?: string; + rewards: Record; + scores: Record; + error?: string; + errorClass?: "infra"; + metrics: BenchmarkMetrics; + kernelMcpSha?: string; + clawbenchSha?: string; + trajectoryPath?: string; + startedAt?: string; + finishedAt?: string; +} + +export interface BenchmarkArm { + name: string; + path: string; + jobId: string; + jobName: string; + startedAt?: string; + finishedAt?: string; + nTotalTrials: number; + nCompletedTrials: number; + nErroredTrials: number; + nCancelledTrials: number; + nRetries: number; + trials: BenchmarkTrial[]; +} + +export interface ArmSummary { + arm: string; + trials: number; + scored: number; + intercepted: number; + lenient: number; + strict?: number; + strictScored: number; + infraErrors: number; + ungraded: number; + kernelMcpValid?: number; + kernelMcpChecked: number; + medianCalls?: number; + medianDurationMs?: number; + totalCostUsd?: number; +} + +function object(value: unknown): JsonObject { + return value !== null && typeof value === "object" && !Array.isArray(value) + ? (value as JsonObject) + : {}; +} + +function array(value: unknown): unknown[] { + return Array.isArray(value) ? value : []; +} + +function string(value: unknown): string | undefined { + return typeof value === "string" && value.length > 0 ? value : undefined; +} + +function number(value: unknown): number | undefined { + return typeof value === "number" && Number.isFinite(value) + ? value + : undefined; +} + +function readJson(path: string): JsonObject { + return object(JSON.parse(readFileSync(path, "utf8"))); +} + +function readJsonIfPresent(path: string): JsonObject { + return existsSync(path) ? readJson(path) : {}; +} + +function boundedError(value: unknown): string | undefined { + if (value === null || value === undefined) return undefined; + const text = + typeof value === "string" ? value : JSON.stringify(value, undefined, 2); + return text.replace(/\s+/g, " ").trim().slice(0, 400) || undefined; +} + +function numericRecord(value: unknown): Record { + return Object.fromEntries( + Object.entries(object(value)).flatMap(([key, raw]) => { + const parsed = number(raw); + return parsed === undefined ? [] : [[key, parsed]]; + }), + ); +} + +function isoSeconds(value: unknown): number | undefined { + const timestamp = string(value); + if (!timestamp) return undefined; + const millis = Date.parse(timestamp); + return Number.isFinite(millis) ? millis / 1000 : undefined; +} + +function durationMs( + startedAt?: string, + finishedAt?: string, +): number | undefined { + if (!startedAt || !finishedAt) return undefined; + const duration = Date.parse(finishedAt) - Date.parse(startedAt); + return Number.isFinite(duration) && duration >= 0 ? duration : undefined; +} + +function sumStepMetric( + stepResults: unknown[], + key: string, +): number | undefined { + const values = stepResults + .map((step) => number(object(object(step).agent_result)[key])) + .filter((value): value is number => value !== undefined); + return values.length > 0 + ? values.reduce((total, value) => total + value, 0) + : undefined; +} + +function trajectoryMetrics(path: string): Pick { + if (!existsSync(path)) return {}; + const trajectory = readJson(path); + const toolCalls = array(trajectory.steps) + .map((step) => array(object(step).tool_calls).length) + .reduce((total, count) => total + count, 0); + return { toolCalls }; +} + +function trialRewards( + result: JsonObject, + trialDir: string, +): Record { + const direct = numericRecord(object(result.verifier_result).rewards); + if (Object.keys(direct).length > 0) return direct; + + const steps = array(result.step_results); + const lastStep = object(steps.at(-1)); + const fromStep = numericRecord(object(lastStep.verifier_result).rewards); + if (Object.keys(fromStep).length > 0) return fromStep; + + return numericRecord( + readJsonIfPresent(join(trialDir, "steps/run/verifier/reward.json")), + ); +} + +function trialScores( + rewards: Record, + hasInfraError: boolean, +): Record { + if (hasInfraError) { + return { infra_error_rate: 1, ungraded_rate: 1 }; + } + + const reward = rewards.reward ?? rewards.reward_lenient; + if (reward === undefined) return { ungraded_rate: 1 }; + + const scores: Record = { + accuracy: reward, + false_positive_rate: 0, + false_negative_rate: 1 - reward, + ungraded_rate: 0, + }; + for (const [key, value] of Object.entries(rewards)) { + if (value >= 0 && value <= 1) scores[key] = value; + } + return scores; +} + +function parseTrial(arm: string, trialDir: string): BenchmarkTrial { + const result = readJson(join(trialDir, "result.json")); + const config = object(result.config); + const agentConfig = object(config.agent); + const agentInfo = object(result.agent_info); + const modelInfo = object(agentInfo.model_info); + const steps = array(result.step_results); + const exception = result.exception_info; + const exceptionFile = join(trialDir, "exception.txt"); + const error = boundedError( + exception ?? + (existsSync(exceptionFile) + ? readFileSync(exceptionFile, "utf8") + : undefined), + ); + const rewards = trialRewards(result, trialDir); + const startedAt = string(result.started_at); + const finishedAt = string(result.finished_at); + const trajectoryPath = join(trialDir, "steps/run/agent/trajectory.json"); + const runManifest = readJsonIfPresent( + join(trialDir, "steps/run/verifier/kernel-mcp/run-manifest.json"), + ); + + return { + arm, + id: string(result.id) ?? basename(trialDir), + taskName: + string(result.task_name)?.replace(/^clawbench\//, "") ?? + string(object(config.task).name) ?? + basename(trialDir).split("__", 1)[0], + trialName: string(result.trial_name) ?? basename(trialDir), + source: + string(result.source) ?? + string(object(config.task).source) ?? + "clawbench", + agent: string(agentInfo.name) ?? string(agentConfig.name) ?? "unknown", + agentVersion: + string(agentInfo.version) ?? string(object(agentConfig.kwargs).version), + agentConfigHash: createHash("sha256") + .update(JSON.stringify(agentConfig)) + .digest("hex") + .slice(0, 8), + model: string(modelInfo.name) ?? string(agentConfig.model_name), + rewards, + scores: trialScores(rewards, error !== undefined), + error, + errorClass: error === undefined ? undefined : "infra", + metrics: { + inputTokens: sumStepMetric(steps, "n_input_tokens"), + cacheTokens: sumStepMetric(steps, "n_cache_tokens"), + outputTokens: sumStepMetric(steps, "n_output_tokens"), + costUsd: sumStepMetric(steps, "cost_usd"), + durationMs: durationMs(startedAt, finishedAt), + start: isoSeconds(startedAt), + end: isoSeconds(finishedAt), + ...trajectoryMetrics(trajectoryPath), + }, + kernelMcpSha: string(runManifest.kernel_mcp_server_sha), + clawbenchSha: string(runManifest.clawbench_source_sha), + trajectoryPath: existsSync(trajectoryPath) ? trajectoryPath : undefined, + startedAt, + finishedAt, + }; +} + +export function parseArmSpec(spec: string): ArmInput { + const separator = spec.indexOf("="); + if (separator <= 0 || separator === spec.length - 1) { + throw new Error( + `Invalid --arm ${JSON.stringify(spec)}; expected name=/job/path`, + ); + } + return { + name: spec.slice(0, separator), + path: resolve(spec.slice(separator + 1)), + }; +} + +export function readBenchmarkArm(input: ArmInput): BenchmarkArm { + const configPath = join(input.path, "config.json"); + const resultPath = join(input.path, "result.json"); + if (!existsSync(configPath) || !existsSync(resultPath)) { + throw new Error(`${input.path} is not a completed Harbor job directory`); + } + + const config = readJson(configPath); + const result = readJson(resultPath); + const stats = object(result.stats); + const trials = readdirSync(input.path, { withFileTypes: true }) + .filter( + (entry) => + entry.isDirectory() && + existsSync(join(input.path, entry.name, "result.json")), + ) + .map((entry) => parseTrial(input.name, join(input.path, entry.name))) + .sort((left, right) => left.taskName.localeCompare(right.taskName)); + + return { + name: input.name, + path: input.path, + jobId: + string(result.id) ?? + createHash("sha256").update(input.path).digest("hex").slice(0, 16), + jobName: string(config.job_name) ?? basename(input.path), + startedAt: string(result.started_at), + finishedAt: string(result.finished_at), + nTotalTrials: number(result.n_total_trials) ?? trials.length, + nCompletedTrials: number(stats.n_completed_trials) ?? trials.length, + nErroredTrials: + number(stats.n_errored_trials) ?? + trials.filter((trial) => trial.error).length, + nCancelledTrials: number(stats.n_cancelled_trials) ?? 0, + nRetries: number(stats.n_retries) ?? 0, + trials, + }; +} + +function median(values: number[]): number | undefined { + if (values.length === 0) return undefined; + const sorted = [...values].sort((left, right) => left - right); + const middle = Math.floor(sorted.length / 2); + return sorted.length % 2 === 0 + ? (sorted[middle - 1] + sorted[middle]) / 2 + : sorted[middle]; +} + +export function summarizeArm(arm: BenchmarkArm): ArmSummary { + const numeric = (key: string) => + arm.trials.filter( + (trial) => + trial.errorClass !== "infra" && trial.rewards[key] !== undefined, + ); + const lenient = numeric("reward_lenient"); + const primary = lenient.length > 0 ? lenient : numeric("reward"); + const strict = numeric("reward_strict"); + const validity = numeric("kernel_mcp_valid"); + const costs = arm.trials.flatMap((trial) => + trial.metrics.costUsd === undefined ? [] : [trial.metrics.costUsd], + ); + + return { + arm: arm.name, + trials: arm.nTotalTrials, + scored: primary.length, + intercepted: numeric("intercepted").reduce( + (total, trial) => total + trial.rewards.intercepted, + 0, + ), + lenient: primary.reduce( + (total, trial) => + total + (trial.rewards.reward_lenient ?? trial.rewards.reward), + 0, + ), + strict: + strict.length === 0 + ? undefined + : strict.reduce( + (total, trial) => total + trial.rewards.reward_strict, + 0, + ), + strictScored: strict.length, + infraErrors: arm.trials.filter((trial) => trial.errorClass === "infra") + .length, + ungraded: arm.trials.filter((trial) => trial.scores.ungraded_rate === 1) + .length, + kernelMcpValid: + validity.length === 0 + ? undefined + : validity.reduce( + (total, trial) => total + trial.rewards.kernel_mcp_valid, + 0, + ), + kernelMcpChecked: validity.length, + medianCalls: median( + arm.trials.flatMap((trial) => + trial.metrics.toolCalls === undefined ? [] : [trial.metrics.toolCalls], + ), + ), + medianDurationMs: median( + arm.trials.flatMap((trial) => + trial.metrics.durationMs === undefined + ? [] + : [trial.metrics.durationMs], + ), + ), + totalCostUsd: + costs.length === 0 + ? undefined + : costs.reduce((total, cost) => total + cost, 0), + }; +} diff --git a/package.json b/package.json index 62b06e5..73061f5 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,8 @@ "check:managed-auth-app": "bun scripts/build-managed-auth-app.mjs --check", "build": "bun run check:managed-auth-app && next build", "start": "next start -p 3002", + "benchmark:report": "bun benchmarks/harbor/report.ts", + "benchmark:publish": "bun benchmarks/harbor/publish-braintrust.ts", "lint": "next lint", "test": "bun test", "record:oauth-redis-contract": "bun scripts/record-oauth-redis-contract.ts", From 75b36ffaceeb2580b32874a8673f458dbaa40c19 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Fri, 28 Aug 2026 16:10:43 +0000 Subject: [PATCH 26/29] Bound ClawBench workflow runtime --- .github/workflows/benchmark-clawbench.yml | 4 ++-- benchmarks/harbor/clawbench/prepare-task.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/benchmark-clawbench.yml b/.github/workflows/benchmark-clawbench.yml index 524d57b..e1f9a1f 100644 --- a/.github/workflows/benchmark-clawbench.yml +++ b/.github/workflows/benchmark-clawbench.yml @@ -191,7 +191,7 @@ jobs: needs: resolve if: needs.resolve.outputs.enabled == 'true' runs-on: ubuntu-latest - timeout-minutes: 480 + timeout-minutes: 360 environment: benchmarks permissions: contents: read @@ -316,7 +316,7 @@ jobs: env: CLAWBENCH_REPO: ${{ github.workspace }}/clawbench CLAWBENCH_REF: 45a71c4b0c78186851c94cfc77bfe619c9e01387 - HARBOR_BENCHMARK_TIMEOUT: 7h + HARBOR_BENCHMARK_TIMEOUT: 4h BENCHMARK_AGENT: ${{ needs.resolve.outputs.agent }} BENCHMARK_TASK: ${{ needs.resolve.outputs.task }} BENCHMARK_COMPARE: ${{ needs.resolve.outputs.compare }} diff --git a/benchmarks/harbor/clawbench/prepare-task.py b/benchmarks/harbor/clawbench/prepare-task.py index b724f23..5dc0bc8 100755 --- a/benchmarks/harbor/clawbench/prepare-task.py +++ b/benchmarks/harbor/clawbench/prepare-task.py @@ -132,7 +132,7 @@ def _patch_instruction(instruction: str) -> str: --- Kernel MCP benchmark arm: - Wait for the `kernel` MCP server to finish initializing before starting. In Claude Code, call `WaitForMcpServers` if it is still pending; do not conclude that the tools are unavailable while it initializes. -- Read `./my-info/kernel_browser.json` and use its existing `session_id` for every `execute_playwright_code` call. +- Read `/my-info/kernel_browser.json` and use its existing `session_id` for every `execute_playwright_code` call. - Do not create, list, update, or delete browsers. Browser lifecycle tools and `computer_action` are intentionally unavailable. - Use Kernel MCP `execute_playwright_code` for all browser interaction. Do not use Playwright MCP or a direct CDP client. - Interact through visible page navigation and DOM/UI actions. Do not call `fetch`, `XMLHttpRequest`, Playwright request APIs, or other direct HTTP clients inside `execute_playwright_code`. From c38a95c8ff3bc69dd7bfe4a0c0e1805eb12966a9 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Fri, 28 Aug 2026 16:32:34 +0000 Subject: [PATCH 27/29] Keep benchmark PR scoped to infrastructure --- src/lib/mcp/tools/playwright.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/mcp/tools/playwright.ts b/src/lib/mcp/tools/playwright.ts index 3fd0120..585bff5 100644 --- a/src/lib/mcp/tools/playwright.ts +++ b/src/lib/mcp/tools/playwright.ts @@ -31,7 +31,7 @@ export function registerPlaywrightTool( code: z .string() .describe( - "Playwright/TypeScript code with `page`, `context`, and `browser` objects in scope; the value you `return` is sent back. Every invocation should return useful page state. After navigation or interaction, return a condensed accessibility snapshot of the relevant region, e.g. `await page.goto('https://example.com'); return await page.locator('main').ariaSnapshot();` or `await page.getByRole('button', { name: 'Submit' }).click(); return await page.locator('main').ariaSnapshot();`. For targeted reads, return a compact value or object. Do not dump the full DOM or body text.", + "Playwright/TypeScript code with `page`, `context`, and `browser` objects in scope; the value you `return` is sent back. Example: `await page.goto('https://example.com'); return await page.title();` Return only what you need — prefer a targeted selector (e.g. `await page.locator('h1').innerText()`) or a region-scoped snapshot (e.g. `await page.locator('main').ariaSnapshot()`) rather than dumping the whole page.", ), session_id: z .string() From 17dc7a73cbcb6c7cfbf1695a45f4d9066a6d09ea Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Fri, 28 Aug 2026 23:33:59 +0000 Subject: [PATCH 28/29] Address benchmark review feedback --- .dockerignore | 1 + .github/workflows/benchmark-clawbench.yml | 31 ++++- benchmarks/harbor/README.md | 8 +- benchmarks/harbor/clawbench/run.sh | 26 +++- benchmarks/harbor/publish-braintrust.ts | 27 ++-- benchmarks/harbor/redact.ts | 12 +- benchmarks/harbor/report.ts | 60 ++++++-- benchmarks/harbor/results.test.ts | 159 +++++++++++++++++++++- benchmarks/harbor/results.ts | 62 +++++++-- 9 files changed, 323 insertions(+), 63 deletions(-) diff --git a/.dockerignore b/.dockerignore index b58cdc1..9a0a41a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,4 +5,5 @@ node_modules coverage .env* *.log +*.pem benchmarks/harbor/.image.env diff --git a/.github/workflows/benchmark-clawbench.yml b/.github/workflows/benchmark-clawbench.yml index e1f9a1f..4b8f6ca 100644 --- a/.github/workflows/benchmark-clawbench.yml +++ b/.github/workflows/benchmark-clawbench.yml @@ -35,7 +35,7 @@ on: default: "30" type: string compare_to_base: - description: "Run the PR/base or ref/main comparison arm" + description: "Compare the candidate with its merge base" required: true default: true type: boolean @@ -75,6 +75,9 @@ jobs: const repo = context.repo.repo; const defaultBranch = context.payload.repository.default_branch; const event = context.eventName; + const mergeBase = async (base, head) => + (await github.rest.repos.compareCommits({ owner, repo, base, head })) + .data.merge_base_commit.sha; let headSha; let baseSha; let prNumber = ""; @@ -94,7 +97,7 @@ jobs: } const allowed = new Set(["OWNER", "MEMBER", "COLLABORATOR"]); if (!allowed.has(context.payload.comment.author_association)) { - core.setFailed("Only repository collaborators can run benchmarks"); + core.setFailed("Only organization members or repository collaborators can run benchmarks"); return; } const pull = (await github.rest.pulls.get({ @@ -108,7 +111,7 @@ jobs: } prNumber = String(pull.number); headSha = pull.head.sha; - baseSha = pull.base.sha; + baseSha = await mergeBase(pull.base.sha, headSha); compare = true; } else if (event === "workflow_dispatch") { if (process.env.GITHUB_REF_NAME !== defaultBranch) { @@ -132,18 +135,19 @@ jobs: } prNumber = String(number); headSha = pull.head.sha; - baseSha = pull.base.sha; + baseSha = await mergeBase(pull.base.sha, headSha); } else { headSha = (await github.rest.repos.getCommit({ owner, repo, ref: process.env.INPUT_REF || defaultBranch, })).data.sha; - baseSha = (await github.rest.repos.getCommit({ + const defaultSha = (await github.rest.repos.getCommit({ owner, repo, ref: defaultBranch, })).data.sha; + baseSha = await mergeBase(defaultSha, headSha); } } else { headSha = (await github.rest.repos.getCommit({ owner, repo, ref: defaultBranch })).data.sha; @@ -173,7 +177,7 @@ jobs: const experiment = `${subject}-${shortHead}-${process.env.GITHUB_RUN_ID}${attemptSuffix}`; const candidate = prNumber ? `PR #${prNumber} (${shortHead})` : shortHead; const title = compare - ? `ClawBench · ${candidate} vs base (${baseSha.slice(0, 7)})` + ? `ClawBench · ${candidate} vs merge base (${baseSha.slice(0, 7)})` : `ClawBench · ${candidate}`; core.setOutput("enabled", "true"); @@ -215,6 +219,12 @@ jobs: CLAWBENCH_JUDGE_API_TYPE: ${{ vars.CLAWBENCH_JUDGE_API_TYPE }} BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }} BRAINTRUST_PROJECT: ${{ vars.BRAINTRUST_PROJECT }} + HARBOR_VERSION: "0.21.0" + HARBOR_HYPEMAN_VERSION: "0.1.1" + CODEX_BENCHMARK_MODEL: gpt-5.6-luna + CODEX_BENCHMARK_VERSION: "0.120.0" + CLAUDE_BENCHMARK_MODEL: claude-sonnet-5 + CLAUDE_BENCHMARK_VERSION: "2.1.238" HARBOR_N_CONCURRENT: ${{ needs.resolve.outputs.concurrency }} BENCHMARK_PR_NUMBER: ${{ needs.resolve.outputs.pr_number }} BENCHMARK_HEAD_SHA: ${{ needs.resolve.outputs.head_sha }} @@ -395,9 +405,16 @@ jobs: env: CANDIDATE_DIR: ${{ steps.run.outputs.candidate_dir }} BASELINE_DIR: ${{ steps.run.outputs.baseline_dir }} + CANDIDATE_STATUS: ${{ steps.run.outputs.candidate_status }} + BASELINE_STATUS: ${{ steps.run.outputs.baseline_status }} + BENCHMARK_COMPARE: ${{ needs.resolve.outputs.compare }} BENCHMARK_TITLE: ${{ needs.resolve.outputs.title }} run: | args=() + statuses=(--status "candidate=${CANDIDATE_STATUS:-1}") + if [[ "$BENCHMARK_COMPARE" == "true" ]]; then + statuses+=(--status "baseline=${BASELINE_STATUS:-1}") + fi [[ -f "$CANDIDATE_DIR/result.json" ]] && \ args+=(--arm "candidate=$CANDIDATE_DIR") [[ -f "$BASELINE_DIR/result.json" ]] && \ @@ -411,6 +428,7 @@ jobs: --json "$RUNNER_TEMP/benchmark-summary.json" \ --markdown "$RUNNER_TEMP/benchmark-summary.md" \ "${publication[@]}" \ + "${statuses[@]}" \ "${args[@]}" cat "$RUNNER_TEMP/benchmark-summary.md" >>"$GITHUB_STEP_SUMMARY" @@ -420,7 +438,6 @@ jobs: env: PR_NUMBER: ${{ needs.resolve.outputs.pr_number }} BENCHMARK_TITLE: ${{ needs.resolve.outputs.title }} - REPORT_OUTCOME: ${{ steps.report.outcome }} with: script: | const fs = require("fs"); diff --git a/benchmarks/harbor/README.md b/benchmarks/harbor/README.md index ba5754b..b2d8c1f 100644 --- a/benchmarks/harbor/README.md +++ b/benchmarks/harbor/README.md @@ -57,15 +57,15 @@ Single-task runs have a 40-minute wall-clock limit. Full-suite runs default to s ## GitHub Actions -The `Benchmark ClawBench` workflow runs the complete suite weekly and on demand. Select it from the Actions tab and provide either a same-repository PR number or a ref. PR runs compare the candidate SHA with its base SHA by default. +The `Benchmark ClawBench` workflow runs the complete suite weekly and on demand. Select it from the Actions tab and provide either a same-repository PR number or a ref. Comparison runs benchmark the candidate SHA against its merge base so unrelated changes on the target branch do not affect the delta. Harbor, Hypeman, agent, and model versions are pinned by the workflow and each arm's observed agent configuration appears in the report. -A repository collaborator can also start the full PR comparison by commenting this exact command on a same-repository pull request: +An organization member or repository collaborator can also start the full PR comparison by commenting this exact command on a same-repository pull request: ```text /benchmark clawbench ``` -The command parser does not execute comment text. It accepts only the exact command, rejects fork pull requests and non-collaborators, and resolves the candidate and base SHAs through GitHub's pull-request API. The workflow uses the `benchmarks` environment for credentials, updates one benchmark comment on the pull request, and publishes the same results to Braintrust. +The command parser does not execute comment text. It accepts only the exact command, rejects fork pull requests and untrusted commenters, and resolves the candidate and merge-base SHAs through GitHub's API. The workflow uses the `benchmarks` environment for credentials, updates one benchmark comment on the pull request, and publishes the same results to Braintrust. ## Results @@ -98,4 +98,4 @@ BRAINTRUST_PROJECT=kernel-mcp-server-benchmarks \ --arm baseline=/path/to/baseline-job ``` -The experiment name and deterministic row/span IDs make it safe to publish the same job directories again. Rows contain task identity, numeric rewards, provenance, bounded errors, timing, token, call, and cost metrics. ATIF agent/tool activity is attached as child spans after secret redaction. Task instructions, ground truth, browser session URLs, and recordings are not placed on experiment rows or public pull-request comments. +The experiment name and deterministic row/span IDs make it safe to publish the same job directories again. Re-publication replaces the rows and refreshes experiment metadata. Rows contain task identity, numeric rewards, provenance, bounded errors, timing, token, call, and cost metrics. ATIF agent/tool activity is attached as child spans after secret redaction. Task instructions, ground truth, browser session URLs, and recordings are not placed on experiment rows or public pull-request comments. diff --git a/benchmarks/harbor/clawbench/run.sh b/benchmarks/harbor/clawbench/run.sh index 847c85f..960be1e 100755 --- a/benchmarks/harbor/clawbench/run.sh +++ b/benchmarks/harbor/clawbench/run.sh @@ -68,6 +68,9 @@ case "$agent" in ;; esac +harbor_version=${HARBOR_VERSION:-0.21.0} +harbor_hypeman_version=${HARBOR_HYPEMAN_VERSION:-0.1.1} + runtime_root=$(mktemp -d) runtime_env=$(mktemp) trap 'rm -rf "$runtime_root"; rm -f "$runtime_env"' EXIT @@ -113,13 +116,22 @@ CLAWBENCH_JUDGE_BASE_URL=${CLAWBENCH_JUDGE_BASE_URL:-} CLAWBENCH_JUDGE_API_KEY=${CLAWBENCH_JUDGE_API_KEY:-} CLAWBENCH_JUDGE_MODEL=${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro} CLAWBENCH_JUDGE_API_TYPE=${CLAWBENCH_JUDGE_API_TYPE:-openai-completions} -ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY:-} -ANTHROPIC_AUTH_TOKEN=${ANTHROPIC_AUTH_TOKEN:-} -ANTHROPIC_BASE_URL=${ANTHROPIC_BASE_URL:-} -CLAUDE_CODE_OAUTH_TOKEN=${CLAUDE_CODE_OAUTH_TOKEN:-} -CLAUDE_FORCE_OAUTH=${CLAUDE_FORCE_OAUTH:-false} -OPENAI_API_KEY=${OPENAI_API_KEY:-} EOF + +case "$agent" in + claude-code) + { + printf 'ANTHROPIC_API_KEY=%s\n' "${ANTHROPIC_API_KEY:-}" + printf 'ANTHROPIC_AUTH_TOKEN=%s\n' "${ANTHROPIC_AUTH_TOKEN:-}" + printf 'ANTHROPIC_BASE_URL=%s\n' "${ANTHROPIC_BASE_URL:-}" + printf 'CLAUDE_CODE_OAUTH_TOKEN=%s\n' "${CLAUDE_CODE_OAUTH_TOKEN:-}" + printf 'CLAUDE_FORCE_OAUTH=%s\n' "${CLAUDE_FORCE_OAUTH:-false}" + } >>"$runtime_env" + ;; + codex) + printf 'OPENAI_API_KEY=%s\n' "$OPENAI_API_KEY" >>"$runtime_env" + ;; +esac chmod 0600 "$runtime_env" job_name=${3:-kernel-mcp-${agent}-${task_id}-$(date -u +%Y%m%dT%H%M%SZ)} @@ -133,7 +145,7 @@ else fi timeout --signal=INT --kill-after=30s "${HARBOR_BENCHMARK_TIMEOUT:-$default_timeout}" \ - uvx --from "harbor==0.21.0" --with "harbor-hypeman==0.1.1" harbor run \ + uvx --from "harbor==$harbor_version" --with "harbor-hypeman==$harbor_hypeman_version" harbor run \ --path "$dataset" \ --agent "$agent" \ --model "$model" \ diff --git a/benchmarks/harbor/publish-braintrust.ts b/benchmarks/harbor/publish-braintrust.ts index 6a50e18..51b032c 100644 --- a/benchmarks/harbor/publish-braintrust.ts +++ b/benchmarks/harbor/publish-braintrust.ts @@ -7,6 +7,7 @@ import { type BenchmarkTrial, parseArmSpec, readBenchmarkArm, + selectPrimaryReward, summarizeArm, } from "./results"; import { redactString, redactValue } from "./redact"; @@ -162,11 +163,13 @@ function trialMetadata(trial: BenchmarkTrial): Record { function atifEvents(trial: BenchmarkTrial, rowId: string): BraintrustEvent[] { const events: BraintrustEvent[] = []; - for (const step of trajectorySteps(trial).filter( + const agentSteps = trajectorySteps(trial).filter( (candidate) => candidate.source === "agent", - )) { - const stepId = step.step_id ?? 0; - const llmId = uuidV5(`${rowId}:llm:${stepId}`); + ); + for (const [stepIndex, step] of agentSteps.entries()) { + const stepId = step.step_id; + const stepKey = `${stepId ?? "missing"}:${stepIndex}`; + const llmId = uuidV5(`${rowId}:llm:${stepKey}`); const start = step.timestamp ? Date.parse(step.timestamp) / 1000 : undefined; @@ -190,6 +193,7 @@ function atifEvents(trial: BenchmarkTrial, rowId: string): BraintrustEvent[] { metadata: { phase: "agent_execution", stepId, + stepIndex, model: step.model_name ?? trial.model, }, metrics: llmMetrics, @@ -198,7 +202,7 @@ function atifEvents(trial: BenchmarkTrial, rowId: string): BraintrustEvent[] { for (const [toolIndex, call] of (step.tool_calls ?? []).entries()) { const toolId = uuidV5( - `${rowId}:tool:${stepId}:${call.tool_call_id ?? toolIndex}`, + `${rowId}:tool:${stepKey}:${call.tool_call_id ?? toolIndex}`, ); const observation = step.observation?.results?.find( (result) => result.source_call_id === call.tool_call_id, @@ -215,6 +219,7 @@ function atifEvents(trial: BenchmarkTrial, rowId: string): BraintrustEvent[] { metadata: { phase: "agent_execution", stepId, + stepIndex, toolCallId: call.tool_call_id, }, metrics: start === undefined ? undefined : { start: start, end: start }, @@ -233,10 +238,10 @@ export function buildExperimentEvents( for (const arm of arms) { for (const trial of arm.trials) { const rowId = uuidV5(`${experimentName}:${arm.name}:${trial.id}`); - const reward = + const primaryReward = trial.errorClass === "infra" ? undefined - : (trial.rewards.reward ?? trial.rewards.reward_lenient); + : selectPrimaryReward(trial.rewards); events.push({ id: rowId, span_id: rowId, @@ -246,8 +251,8 @@ export function buildExperimentEvents( created: trial.startedAt, input: { source: trial.source, taskName: trial.taskName }, output: { - reward, - rewardKey: reward === undefined ? undefined : "reward", + reward: primaryReward?.value, + rewardKey: primaryReward?.key, error: trial.error ? redactString(trial.error, 400) : undefined, }, expected: { reward: 1 }, @@ -353,6 +358,7 @@ export async function publishBenchmark( const project = await api.request("/v1/project", "POST", { name: projectName, }); + const metadata = experimentMetadata(arms); const experiment = await api.request( "/v1/experiment", "POST", @@ -360,9 +366,10 @@ export async function publishBenchmark( project_id: project.id, name: experimentName, public: false, - metadata: experimentMetadata(arms), + metadata, }, ); + await api.request(`/v1/experiment/${experiment.id}`, "PATCH", { metadata }); const events = buildExperimentEvents(arms, experimentName); await insertEvents(api, experiment.id, events); const organization = await api.request<{ name: string }>( diff --git a/benchmarks/harbor/redact.ts b/benchmarks/harbor/redact.ts index 7b5d551..851c792 100644 --- a/benchmarks/harbor/redact.ts +++ b/benchmarks/harbor/redact.ts @@ -1,4 +1,6 @@ const SECRET_NAME = /(API_KEY|TOKEN|SECRET|PASSWORD|PRIVATE_KEY|CREDENTIAL)/i; +const SENSITIVE_FIELD = + /(API_KEY|TOKEN|JWT|SECRET|PASSWORD|PRIVATE_KEY|CREDENTIAL|^COOKIE$|^SET-COOKIE$)/i; const REDACTED = "[REDACTED]"; function secretValues(): string[] { @@ -21,13 +23,15 @@ export function redactString(value: string, maxLength = 20_000): string { .replace(/\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b/g, REDACTED) .replace(/\b(?:sk|pk|bt|kapi|whsec)[-_][A-Za-z0-9_-]{12,}\b/gi, REDACTED) .replace( - /(["']?(?:api[_-]?key|access[_-]?token|credential|password|secret)["']?\s*[:=]\s*["']?)[^"'\s,}&]+/gi, + /(["']?(?:api[_-]?key|access[_-]?token|credential|jwt|password|secret|token)["']?\s*[:=]\s*["']?)[^"'\s,}&]+/gi, `$1${REDACTED}`, ) .replace( - /([?&](?:api[_-]?key|access[_-]?token|auth|code|credential|password|secret|session[_-]?token)=)[^&#\s]+/gi, + /([?&](?:api[_-]?key|access[_-]?token|auth|code|credential|jwt|password|secret|session[_-]?token|token)=)[^&#\s]+/gi, `$1${REDACTED}`, ) + .replace(/(\b(?:cookie|set-cookie)\s*:\s*)[^\r\n]+/gi, `$1${REDACTED}`) + .replace(/(\/browser\/live\/)[^/?#\s]+/gi, `$1${REDACTED}`) .replace(/(wss?:\/\/)[^/@\s]+@/gi, `$1${REDACTED}@`) .replace(/\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi, "[REDACTED_EMAIL]"); @@ -45,7 +49,9 @@ export function redactValue(value: unknown, maxStringLength = 20_000): unknown { return Object.fromEntries( Object.entries(value as Record).map(([key, entry]) => [ key, - SECRET_NAME.test(key) ? REDACTED : redactValue(entry, maxStringLength), + SENSITIVE_FIELD.test(key) + ? REDACTED + : redactValue(entry, maxStringLength), ]), ); } diff --git a/benchmarks/harbor/report.ts b/benchmarks/harbor/report.ts index 0e24f49..55eb752 100644 --- a/benchmarks/harbor/report.ts +++ b/benchmarks/harbor/report.ts @@ -10,6 +10,7 @@ import { interface Options { arms: string[]; + statuses: Record; json?: string; markdown?: string; publication?: string; @@ -17,7 +18,11 @@ interface Options { } function parseArgs(args: string[]): Options { - const options: Options = { arms: [], title: "ClawBench benchmark" }; + const options: Options = { + arms: [], + statuses: {}, + title: "ClawBench benchmark", + }; for (let index = 0; index < args.length; index += 1) { const flag = args[index]; const value = args[index + 1]; @@ -28,6 +33,21 @@ function parseArgs(args: string[]): Options { case "--arm": options.arms.push(value); break; + case "--status": { + const separator = value.indexOf("="); + const status = Number(value.slice(separator + 1)); + if ( + separator <= 0 || + separator === value.length - 1 || + !Number.isInteger(status) + ) { + throw new Error( + `Invalid --status ${JSON.stringify(value)}; expected name=exit-code`, + ); + } + options.statuses[value.slice(0, separator)] = status; + break; + } case "--json": options.json = value; break; @@ -61,28 +81,38 @@ function cost(value: number | undefined): string { return value === undefined ? "—" : `$${value.toFixed(4)}`; } -function markdown( +export function renderMarkdown( title: string, summaries: ArmSummary[], publication?: Record, + statuses: Record = {}, ): string { - const lines = [ - "", - `## ${title}`, + const lines = ["", `## ${title}`]; + const failed = Object.entries(statuses).filter(([, status]) => status !== 0); + if (failed.length > 0) { + lines.push( + "", + `> [!WARNING]\n> Incomplete benchmark: ${failed.map(([arm, status]) => `${arm} exited ${status}`).join(", ")}. Scores below include only completed Harbor results and are not a complete comparison.`, + ); + } + lines.push( "", - "| Arm | Lenient | Strict | Intercepted | Infra | Ungraded | Kernel MCP valid | Median calls | Median duration | Cost |", - "|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|", - ]; + "| Arm | Configuration | Lenient | Strict | Intercepted | Infra | Ungraded | Kernel MCP valid | Median calls | Median duration | Cost |", + "|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|", + ); for (const summary of summaries) { lines.push( - `| ${summary.arm} | ${ratio(summary.lenient, summary.trials)} | ${ratio(summary.strict, summary.trials)} | ${ratio(summary.intercepted, summary.trials)} | ${summary.infraErrors} | ${summary.ungraded} | ${ratio(summary.kernelMcpValid, summary.kernelMcpChecked)} | ${summary.medianCalls ?? "—"} | ${duration(summary.medianDurationMs)} | ${cost(summary.totalCostUsd)} |`, + `| ${summary.arm} | ${summary.configuration ?? "—"} | ${ratio(summary.lenient, summary.trials)} | ${ratio(summary.strict, summary.trials)} | ${ratio(summary.intercepted, summary.trials)} | ${summary.infraErrors} | ${summary.ungraded} | ${ratio(summary.kernelMcpValid, summary.kernelMcpChecked)} | ${summary.medianCalls ?? "—"} | ${duration(summary.medianDurationMs)} | ${cost(summary.totalCostUsd)} |`, ); } const candidate = summaries.find((summary) => summary.arm === "candidate"); const baseline = summaries.find((summary) => summary.arm === "baseline"); - if (candidate && baseline) { - const signed = (value: number) => `${value >= 0 ? "+" : ""}${value}`; + if (candidate && baseline && failed.length === 0) { + const signed = (value: number) => { + const rounded = Number(value.toFixed(3)); + return `${rounded >= 0 ? "+" : ""}${rounded}`; + }; const deltas = [ `**${signed(candidate.lenient - baseline.lenient)} lenient**`, candidate.strict !== undefined && baseline.strict !== undefined @@ -133,9 +163,15 @@ function main(): void { benchmark: "clawbench", generatedAt: new Date().toISOString(), arms: summaries, + statuses: options.statuses, publication, }; - const rendered = markdown(options.title, summaries, publication); + const rendered = renderMarkdown( + options.title, + summaries, + publication, + options.statuses, + ); write(options.json, `${JSON.stringify(result, undefined, 2)}\n`); write(options.markdown, rendered); process.stdout.write(rendered); diff --git a/benchmarks/harbor/results.test.ts b/benchmarks/harbor/results.test.ts index e2ac705..f266934 100644 --- a/benchmarks/harbor/results.test.ts +++ b/benchmarks/harbor/results.test.ts @@ -1,9 +1,16 @@ import { afterEach, describe, expect, test } from "bun:test"; -import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { + mkdtempSync, + mkdirSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { buildExperimentEvents, publishBenchmark } from "./publish-braintrust"; -import { readBenchmarkArm, summarizeArm } from "./results"; +import { renderMarkdown } from "./report"; +import { readBenchmarkArm, selectPrimaryReward, summarizeArm } from "./results"; import { redactString, redactValue } from "./redact"; const temporaryDirectories: string[] = []; @@ -114,6 +121,7 @@ describe("Harbor result ingestion", () => { accuracy: 1, false_positive_rate: 0, false_negative_rate: 0, + infra_error_rate: 0, ungraded_rate: 0, reward: 1, reward_lenient: 1, @@ -137,7 +145,7 @@ describe("Harbor result ingestion", () => { strict: 0, intercepted: 1, infraErrors: 1, - ungraded: 1, + ungraded: 0, kernelMcpValid: 1, medianCalls: 1, totalCostUsd: 0.01, @@ -171,12 +179,22 @@ describe("Harbor result ingestion", () => { ); expect(infra?.scores).toEqual({ infra_error_rate: 1, ungraded_rate: 1 }); expect(infra?.output).not.toHaveProperty("reward", 0); + const success = first.find( + (event) => + event.span_attributes.type === "eval" && + (event.output as { reward?: number }).reward === 1, + ); + expect(success?.output).toMatchObject({ + reward: 1, + rewardKey: "reward_lenient", + }); }); test("re-publishes the same rows and spans by deterministic ID", async () => { const arm = readBenchmarkArm({ name: "candidate", path: fixture() }); const originalFetch = globalThis.fetch; const inserts: string[][] = []; + const metadataUpdates: unknown[] = []; globalThis.fetch = (async (request, init) => { const url = String(request); if (url.endsWith("/v1/project")) { @@ -193,6 +211,13 @@ describe("Harbor result ingestion", () => { name: "experiment name", }); } + if ( + url.endsWith("/v1/experiment/experiment-id") && + init?.method === "PATCH" + ) { + metadataUpdates.push(JSON.parse(String(init.body))); + return Response.json({ id: "experiment-id" }); + } if (url.includes("/insert")) { const body = JSON.parse(String(init?.body)) as { events: Array<{ id: string }>; @@ -222,6 +247,8 @@ describe("Harbor result ingestion", () => { expect(first).toEqual(second); expect(inserts).toHaveLength(2); expect(inserts[0]).toEqual(inserts[1]); + expect(metadataUpdates).toHaveLength(2); + expect(metadataUpdates[0]).toEqual(metadataUpdates[1]); expect(first.url).toBe( "https://www.braintrust.dev/app/Kernel/p/project%20name/experiments/experiment%20name", ); @@ -229,6 +256,82 @@ describe("Harbor result ingestion", () => { globalThis.fetch = originalFetch; } }); + + test("uses the lenient reward per trial and reports incomplete arms", () => { + expect(selectPrimaryReward({ reward: 0, reward_lenient: 1 })).toEqual({ + key: "reward_lenient", + value: 1, + }); + const arm = readBenchmarkArm({ name: "candidate", path: fixture() }); + const second = arm.trials[1]; + second.error = undefined; + second.errorClass = undefined; + second.rewards = { reward: 0 }; + second.scores = { + accuracy: 0, + false_positive_rate: 0, + false_negative_rate: 1, + infra_error_rate: 0, + reward: 0, + ungraded_rate: 0, + }; + const summary = summarizeArm(arm); + expect(summary.scored).toBe(2); + expect(summary.lenient).toBe(1); + expect(summary.configuration).toContain("codex"); + expect( + renderMarkdown("test", [summary], undefined, { candidate: 124 }), + ).toContain("Incomplete benchmark: candidate exited 124"); + expect( + renderMarkdown("test", [ + { ...summary, arm: "candidate", lenient: 0.3 }, + { ...summary, arm: "baseline", lenient: 0.2 }, + ]), + ).toContain("+0.1 lenient"); + }); + + test("keeps full errors until redaction and clamps derived scores", () => { + const root = fixture(); + const successPath = join(root, "task-one__abc", "result.json"); + const result = JSON.parse(readFileSync(successPath, "utf8")) as { + verifier_result: { rewards: Record }; + exception_info?: string; + }; + result.verifier_result.rewards.reward_lenient = 2; + writeJson(successPath, result); + + const failedPath = join(root, "task-two__def", "result.json"); + const failed = JSON.parse(readFileSync(failedPath, "utf8")) as { + exception_info: unknown; + }; + failed.exception_info = `${"x".repeat(395)}secret-value-after-boundary`; + writeJson(failedPath, failed); + + const arm = readBenchmarkArm({ name: "candidate", path: root }); + expect(arm.trials[0].scores.accuracy).toBe(1); + expect(arm.trials[1].error?.length).toBeGreaterThan(400); + + process.env.TEST_SECRET = "secret-value-after-boundary"; + const events = buildExperimentEvents([arm], "redaction-boundary"); + expect(JSON.stringify(events)).not.toContain("secret-value-after-boundary"); + expect(JSON.stringify(events)).not.toContain(`${"x".repeat(395)}secre`); + delete process.env.TEST_SECRET; + }); + + test("assigns unique span IDs when ATIF step IDs are absent", () => { + const root = fixture(); + writeJson(join(root, "task-one__abc", "steps/run/agent/trajectory.json"), { + steps: [ + { source: "agent", message: "first" }, + { source: "agent", message: "second" }, + ], + }); + const events = buildExperimentEvents( + [readBenchmarkArm({ name: "candidate", path: root })], + "missing-step-ids", + ).filter((event) => event.span_attributes.type === "llm"); + expect(new Set(events.map((event) => event.id)).size).toBe(2); + }); }); describe("Braintrust redaction", () => { @@ -236,17 +339,61 @@ describe("Braintrust redaction", () => { process.env.TEST_API_KEY = "super-secret-value"; expect( redactString( - 'Bearer super-secret-value sk-proj-abcdefghijklmnop?access_token=visible "password":"generated-password" user@example.com', + 'Bearer super-secret-value sk-proj-abcdefghijklmnop?access_token=visible&token=plain&jwt=opaque "password":"generated-password" Cookie: session=visible\nhttps://example.com/browser/live/replay-slug user@example.com', ), ).toBe( - 'Bearer [REDACTED] [REDACTED]?access_token=[REDACTED] "password":"[REDACTED]" [REDACTED_EMAIL]', + 'Bearer [REDACTED] [REDACTED]?access_token=[REDACTED]&token=[REDACTED]&jwt=[REDACTED] "password":"[REDACTED]" Cookie: [REDACTED]\nhttps://example.com/browser/live/[REDACTED] [REDACTED_EMAIL]', ); expect( - redactValue({ api_key: "visible", nested: ["bt-abcdefghijklmnop"] }), + redactValue({ + api_key: "visible", + Cookie: "session=visible", + nested: ["bt-abcdefghijklmnop"], + }), ).toEqual({ api_key: "[REDACTED]", + Cookie: "[REDACTED]", nested: ["[REDACTED]"], }); delete process.env.TEST_API_KEY; }); }); + +describe("benchmark workflow hardening", () => { + test("uses merge-base comparisons, fixed configs, and arm statuses", () => { + const workflow = readFileSync( + join(process.cwd(), ".github/workflows/benchmark-clawbench.yml"), + "utf8", + ); + expect(workflow).toContain("github.rest.repos.compareCommits"); + expect(workflow).not.toContain("baseSha = pull.base.sha"); + expect(workflow).toContain('HARBOR_VERSION: "0.21.0"'); + expect(workflow).toContain('CODEX_BENCHMARK_VERSION: "0.120.0"'); + expect(workflow).toContain( + 'statuses=(--status "candidate=${CANDIDATE_STATUS:-1}")', + ); + }); + + test("excludes private keys and forwards only the selected provider", () => { + const dockerignore = readFileSync( + join(process.cwd(), ".dockerignore"), + "utf8", + ); + const runner = readFileSync( + join(process.cwd(), "benchmarks/harbor/clawbench/run.sh"), + "utf8", + ); + expect(dockerignore.split("\n")).toContain("*.pem"); + const heredocStart = runner.indexOf('cat >"$runtime_env"'); + const providerCaseStart = runner.indexOf('case "$agent" in', heredocStart); + const providerCase = runner.slice( + providerCaseStart, + runner.indexOf('chmod 0600 "$runtime_env"'), + ); + expect(providerCase).toContain("ANTHROPIC_API_KEY"); + expect(providerCase).toContain("OPENAI_API_KEY"); + const commonEnvironment = runner.slice(heredocStart, providerCaseStart); + expect(commonEnvironment).not.toContain("OPENAI_API_KEY"); + expect(commonEnvironment).not.toContain("ANTHROPIC_API_KEY"); + }); +}); diff --git a/benchmarks/harbor/results.ts b/benchmarks/harbor/results.ts index 1a27396..dc24bab 100644 --- a/benchmarks/harbor/results.ts +++ b/benchmarks/harbor/results.ts @@ -72,6 +72,7 @@ export interface ArmSummary { medianCalls?: number; medianDurationMs?: number; totalCostUsd?: number; + configuration?: string; } function object(value: unknown): JsonObject { @@ -102,11 +103,11 @@ function readJsonIfPresent(path: string): JsonObject { return existsSync(path) ? readJson(path) : {}; } -function boundedError(value: unknown): string | undefined { +function errorText(value: unknown): string | undefined { if (value === null || value === undefined) return undefined; const text = typeof value === "string" ? value : JSON.stringify(value, undefined, 2); - return text.replace(/\s+/g, " ").trim().slice(0, 400) || undefined; + return text.replace(/\s+/g, " ").trim() || undefined; } function numericRecord(value: unknown): Record { @@ -118,6 +119,22 @@ function numericRecord(value: unknown): Record { ); } +export function selectPrimaryReward( + rewards: Record, +): { key: "reward_lenient" | "reward"; value: number } | undefined { + if (rewards.reward_lenient !== undefined) { + return { key: "reward_lenient", value: rewards.reward_lenient }; + } + if (rewards.reward !== undefined) { + return { key: "reward", value: rewards.reward }; + } + return undefined; +} + +function clampScore(value: number): number { + return Math.min(1, Math.max(0, value)); +} + function isoSeconds(value: unknown): number | undefined { const timestamp = string(value); if (!timestamp) return undefined; @@ -180,13 +197,15 @@ function trialScores( return { infra_error_rate: 1, ungraded_rate: 1 }; } - const reward = rewards.reward ?? rewards.reward_lenient; - if (reward === undefined) return { ungraded_rate: 1 }; + const primary = selectPrimaryReward(rewards); + if (!primary) return { infra_error_rate: 0, ungraded_rate: 1 }; + const reward = clampScore(primary.value); const scores: Record = { accuracy: reward, false_positive_rate: 0, false_negative_rate: 1 - reward, + infra_error_rate: 0, ungraded_rate: 0, }; for (const [key, value] of Object.entries(rewards)) { @@ -204,7 +223,7 @@ function parseTrial(arm: string, trialDir: string): BenchmarkTrial { const steps = array(result.step_results); const exception = result.exception_info; const exceptionFile = join(trialDir, "exception.txt"); - const error = boundedError( + const error = errorText( exception ?? (existsSync(exceptionFile) ? readFileSync(exceptionFile, "utf8") @@ -327,13 +346,24 @@ export function summarizeArm(arm: BenchmarkArm): ArmSummary { (trial) => trial.errorClass !== "infra" && trial.rewards[key] !== undefined, ); - const lenient = numeric("reward_lenient"); - const primary = lenient.length > 0 ? lenient : numeric("reward"); + const primary = arm.trials.flatMap((trial) => { + if (trial.errorClass === "infra") return []; + const reward = selectPrimaryReward(trial.rewards); + return reward ? [{ trial, reward }] : []; + }); const strict = numeric("reward_strict"); const validity = numeric("kernel_mcp_valid"); const costs = arm.trials.flatMap((trial) => trial.metrics.costUsd === undefined ? [] : [trial.metrics.costUsd], ); + const configurations = [ + ...new Set( + arm.trials.map( + (trial) => + `${trial.agent}${trial.agentVersion ? `@${trial.agentVersion}` : ""}${trial.model ? ` · ${trial.model}` : ""} · config ${trial.agentConfigHash}`, + ), + ), + ]; return { arm: arm.name, @@ -343,11 +373,7 @@ export function summarizeArm(arm: BenchmarkArm): ArmSummary { (total, trial) => total + trial.rewards.intercepted, 0, ), - lenient: primary.reduce( - (total, trial) => - total + (trial.rewards.reward_lenient ?? trial.rewards.reward), - 0, - ), + lenient: primary.reduce((total, entry) => total + entry.reward.value, 0), strict: strict.length === 0 ? undefined @@ -358,8 +384,10 @@ export function summarizeArm(arm: BenchmarkArm): ArmSummary { strictScored: strict.length, infraErrors: arm.trials.filter((trial) => trial.errorClass === "infra") .length, - ungraded: arm.trials.filter((trial) => trial.scores.ungraded_rate === 1) - .length, + ungraded: arm.trials.filter( + (trial) => + trial.errorClass !== "infra" && trial.scores.ungraded_rate === 1, + ).length, kernelMcpValid: validity.length === 0 ? undefined @@ -384,5 +412,11 @@ export function summarizeArm(arm: BenchmarkArm): ArmSummary { costs.length === 0 ? undefined : costs.reduce((total, cost) => total + cost, 0), + configuration: + configurations.length === 0 + ? undefined + : configurations.length === 1 + ? configurations[0] + : `mixed: ${configurations.join(", ")}`, }; } From 685e66aa4589e809129315c844e551c930db6d2e Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sat, 29 Aug 2026 00:35:26 +0000 Subject: [PATCH 29/29] Harden benchmark runtime validation --- benchmarks/harbor/clawbench/run.sh | 26 +++++++++++----------- benchmarks/harbor/clawbench/verify-task.py | 8 +++++-- benchmarks/harbor/results.test.ts | 15 ++++++++++--- 3 files changed, 31 insertions(+), 18 deletions(-) diff --git a/benchmarks/harbor/clawbench/run.sh b/benchmarks/harbor/clawbench/run.sh index 960be1e..4f59be5 100755 --- a/benchmarks/harbor/clawbench/run.sh +++ b/benchmarks/harbor/clawbench/run.sh @@ -104,19 +104,19 @@ export KERNEL_API_KEY=$KERNEL_MCP_BENCHMARK_API_KEY export KERNEL_BASE_URL=${KERNEL_BASE_URL:-https://api.onkernel.com} export KERNEL_API_BASE_URL=${KERNEL_API_BASE_URL:-$KERNEL_BASE_URL} -cat >"$runtime_env" <"$runtime_env" case "$agent" in claude-code) diff --git a/benchmarks/harbor/clawbench/verify-task.py b/benchmarks/harbor/clawbench/verify-task.py index 466432a..a314bef 100755 --- a/benchmarks/harbor/clawbench/verify-task.py +++ b/benchmarks/harbor/clawbench/verify-task.py @@ -16,7 +16,11 @@ LOGS_DIR = Path(os.environ.get("HARBOR_LOGS_DIR", "/logs")) VERIFIER_DIR = LOGS_DIR / "verifier" -PLAYWRIGHT_TOOL = "mcp__kernel__execute_playwright_code" +PLAYWRIGHT_TOOLS = { + "execute_playwright_code", + "kernel__execute_playwright_code", + "mcp__kernel__execute_playwright_code", +} def read_object(path: Path) -> dict[str, Any]: @@ -47,7 +51,7 @@ def main() -> int: calls = [ call for call in tool_calls(trajectory) - if call.get("function_name") == PLAYWRIGHT_TOOL + if call.get("function_name") in PLAYWRIGHT_TOOLS ] called_sessions = { arguments.get("session_id") diff --git a/benchmarks/harbor/results.test.ts b/benchmarks/harbor/results.test.ts index f266934..dbe80fe 100644 --- a/benchmarks/harbor/results.test.ts +++ b/benchmarks/harbor/results.test.ts @@ -383,17 +383,26 @@ describe("benchmark workflow hardening", () => { join(process.cwd(), "benchmarks/harbor/clawbench/run.sh"), "utf8", ); + const verifier = readFileSync( + join(process.cwd(), "benchmarks/harbor/clawbench/verify-task.py"), + "utf8", + ); expect(dockerignore.split("\n")).toContain("*.pem"); - const heredocStart = runner.indexOf('cat >"$runtime_env"'); - const providerCaseStart = runner.indexOf('case "$agent" in', heredocStart); + expect(verifier).toContain('"mcp__kernel__execute_playwright_code"'); + expect(verifier).toContain('"kernel__execute_playwright_code"'); + expect(verifier).toContain('"execute_playwright_code"'); + const commonStart = runner.indexOf("printf 'KERNEL_API_KEY=%s\\n'"); + const providerCaseStart = runner.indexOf('case "$agent" in', commonStart); const providerCase = runner.slice( providerCaseStart, runner.indexOf('chmod 0600 "$runtime_env"'), ); expect(providerCase).toContain("ANTHROPIC_API_KEY"); expect(providerCase).toContain("OPENAI_API_KEY"); - const commonEnvironment = runner.slice(heredocStart, providerCaseStart); + const commonEnvironment = runner.slice(commonStart, providerCaseStart); + expect(commonEnvironment).toContain("CLAWBENCH_JUDGE_API_KEY"); expect(commonEnvironment).not.toContain("OPENAI_API_KEY"); expect(commonEnvironment).not.toContain("ANTHROPIC_API_KEY"); + expect(runner).not.toContain("<