diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..9a0a41a --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +.git +.github +.next +node_modules +coverage +.env* +*.log +*.pem +benchmarks/harbor/.image.env diff --git a/.github/workflows/benchmark-clawbench.yml b/.github/workflows/benchmark-clawbench.yml new file mode 100644 index 0000000..4b8f6ca --- /dev/null +++ b/.github/workflows/benchmark-clawbench.yml @@ -0,0 +1,479 @@ +name: Benchmark ClawBench + +on: + issue_comment: + types: [created] + schedule: + - cron: "0 13 * * 1" + workflow_dispatch: + inputs: + pr_number: + description: "Same-repository PR to benchmark" + required: false + type: string + ref: + description: "Ref to benchmark when no PR is supplied" + required: false + default: "main" + type: string + task: + description: "ClawBench task ID or all" + required: true + default: "all" + type: string + agent: + description: "Stock Harbor agent" + required: true + default: "codex" + type: choice + options: + - codex + - claude-code + concurrency: + description: "Concurrent Harbor trials per arm" + required: true + default: "30" + type: string + compare_to_base: + description: "Compare the candidate with its merge base" + required: true + default: true + type: boolean + +permissions: {} + +jobs: + resolve: + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + outputs: + enabled: ${{ steps.resolve.outputs.enabled }} + head_sha: ${{ steps.resolve.outputs.head_sha }} + base_sha: ${{ steps.resolve.outputs.base_sha }} + compare: ${{ steps.resolve.outputs.compare }} + pr_number: ${{ steps.resolve.outputs.pr_number }} + task: ${{ steps.resolve.outputs.task }} + agent: ${{ steps.resolve.outputs.agent }} + concurrency: ${{ steps.resolve.outputs.concurrency }} + experiment: ${{ steps.resolve.outputs.experiment }} + title: ${{ steps.resolve.outputs.title }} + steps: + - id: resolve + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 + env: + INPUT_PR_NUMBER: ${{ inputs.pr_number }} + INPUT_REF: ${{ inputs.ref }} + INPUT_TASK: ${{ inputs.task }} + INPUT_AGENT: ${{ inputs.agent }} + INPUT_CONCURRENCY: ${{ inputs.concurrency }} + INPUT_COMPARE: ${{ inputs.compare_to_base }} + with: + script: | + const owner = context.repo.owner; + const repo = context.repo.repo; + const defaultBranch = context.payload.repository.default_branch; + const event = context.eventName; + const mergeBase = async (base, head) => + (await github.rest.repos.compareCommits({ owner, repo, base, head })) + .data.merge_base_commit.sha; + let headSha; + let baseSha; + let prNumber = ""; + let compare = false; + let task = "all"; + let agent = "codex"; + let concurrency = "30"; + + if (event === "issue_comment") { + if ((context.payload.comment.body || "").trim() !== "/benchmark clawbench") { + core.setOutput("enabled", "false"); + return; + } + if (!context.payload.issue.pull_request) { + core.setFailed("/benchmark clawbench can only be used on a pull request"); + return; + } + const allowed = new Set(["OWNER", "MEMBER", "COLLABORATOR"]); + if (!allowed.has(context.payload.comment.author_association)) { + core.setFailed("Only organization members or repository collaborators can run benchmarks"); + return; + } + const pull = (await github.rest.pulls.get({ + owner, + repo, + pull_number: context.payload.issue.number, + })).data; + if (pull.head.repo?.full_name !== `${owner}/${repo}`) { + core.setFailed("Benchmarks cannot run code from fork pull requests"); + return; + } + prNumber = String(pull.number); + headSha = pull.head.sha; + baseSha = await mergeBase(pull.base.sha, headSha); + compare = true; + } else if (event === "workflow_dispatch") { + if (process.env.GITHUB_REF_NAME !== defaultBranch) { + core.setFailed(`Run this workflow from ${defaultBranch}; choose the target with the inputs`); + return; + } + task = process.env.INPUT_TASK || "all"; + agent = process.env.INPUT_AGENT || "codex"; + concurrency = process.env.INPUT_CONCURRENCY || "30"; + compare = process.env.INPUT_COMPARE === "true"; + if (process.env.INPUT_PR_NUMBER) { + const number = Number(process.env.INPUT_PR_NUMBER); + if (!Number.isInteger(number) || number <= 0) { + core.setFailed("pr_number must be a positive integer"); + return; + } + const pull = (await github.rest.pulls.get({ owner, repo, pull_number: number })).data; + if (pull.head.repo?.full_name !== `${owner}/${repo}`) { + core.setFailed("Benchmarks cannot run code from fork pull requests"); + return; + } + prNumber = String(number); + headSha = pull.head.sha; + baseSha = await mergeBase(pull.base.sha, headSha); + } else { + headSha = (await github.rest.repos.getCommit({ + owner, + repo, + ref: process.env.INPUT_REF || defaultBranch, + })).data.sha; + const defaultSha = (await github.rest.repos.getCommit({ + owner, + repo, + ref: defaultBranch, + })).data.sha; + baseSha = await mergeBase(defaultSha, headSha); + } + } else { + headSha = (await github.rest.repos.getCommit({ owner, repo, ref: defaultBranch })).data.sha; + baseSha = headSha; + compare = false; + } + + if (!/^(all|v2-[a-z0-9-]+)$/.test(task)) { + core.setFailed("task must be all or a ClawBench v2 task ID"); + return; + } + if (!new Set(["codex", "claude-code"]).has(agent)) { + core.setFailed("agent must be codex or claude-code"); + return; + } + const concurrencyNumber = Number(concurrency); + if (!Number.isInteger(concurrencyNumber) || concurrencyNumber < 1 || concurrencyNumber > 30) { + core.setFailed("concurrency must be an integer from 1 through 30"); + return; + } + if (headSha === baseSha) compare = false; + + const shortHead = headSha.slice(0, 7); + const attempt = Number(process.env.GITHUB_RUN_ATTEMPT || "1"); + const attemptSuffix = attempt > 1 ? `-attempt${attempt}` : ""; + const subject = prNumber ? `pr-${prNumber}` : compare ? "ref" : "main"; + const experiment = `${subject}-${shortHead}-${process.env.GITHUB_RUN_ID}${attemptSuffix}`; + const candidate = prNumber ? `PR #${prNumber} (${shortHead})` : shortHead; + const title = compare + ? `ClawBench · ${candidate} vs merge base (${baseSha.slice(0, 7)})` + : `ClawBench · ${candidate}`; + + core.setOutput("enabled", "true"); + core.setOutput("head_sha", headSha); + core.setOutput("base_sha", baseSha); + core.setOutput("compare", String(compare)); + core.setOutput("pr_number", prNumber); + core.setOutput("task", task); + core.setOutput("agent", agent); + core.setOutput("concurrency", String(concurrencyNumber)); + core.setOutput("experiment", experiment); + core.setOutput("title", title); + + benchmark: + needs: resolve + if: needs.resolve.outputs.enabled == 'true' + runs-on: ubuntu-latest + timeout-minutes: 360 + environment: benchmarks + permissions: + contents: read + issues: write + concurrency: + group: benchmark-clawbench-${{ needs.resolve.outputs.pr_number || needs.resolve.outputs.head_sha }} + cancel-in-progress: false + env: + HYPEMAN_API_KEY: ${{ secrets.HYPEMAN_API_KEY }} + HYPEMAN_BASE_URL: ${{ vars.HYPEMAN_BASE_URL }} + KERNEL_MCP_BENCHMARK_API_KEY: ${{ secrets.KERNEL_MCP_BENCHMARK_API_KEY }} + KERNEL_PROJECT: ${{ vars.KERNEL_PROJECT }} + PURELY_MAIL_API_KEY: ${{ secrets.PURELY_MAIL_API_KEY }} + PURELY_MAIL_DOMAIN: ${{ vars.PURELY_MAIL_DOMAIN }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + CLAWBENCH_JUDGE_BASE_URL: ${{ vars.CLAWBENCH_JUDGE_BASE_URL }} + CLAWBENCH_JUDGE_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + CLAWBENCH_JUDGE_MODEL: ${{ vars.CLAWBENCH_JUDGE_MODEL }} + CLAWBENCH_JUDGE_API_TYPE: ${{ vars.CLAWBENCH_JUDGE_API_TYPE }} + BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }} + BRAINTRUST_PROJECT: ${{ vars.BRAINTRUST_PROJECT }} + HARBOR_VERSION: "0.21.0" + HARBOR_HYPEMAN_VERSION: "0.1.1" + CODEX_BENCHMARK_MODEL: gpt-5.6-luna + CODEX_BENCHMARK_VERSION: "0.120.0" + CLAUDE_BENCHMARK_MODEL: claude-sonnet-5 + CLAUDE_BENCHMARK_VERSION: "2.1.238" + HARBOR_N_CONCURRENT: ${{ needs.resolve.outputs.concurrency }} + BENCHMARK_PR_NUMBER: ${{ needs.resolve.outputs.pr_number }} + BENCHMARK_HEAD_SHA: ${{ needs.resolve.outputs.head_sha }} + steps: + - name: Mark the PR benchmark as running + if: needs.resolve.outputs.pr_number != '' + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 + env: + PR_NUMBER: ${{ needs.resolve.outputs.pr_number }} + BENCHMARK_TITLE: ${{ needs.resolve.outputs.title }} + with: + script: | + const marker = ""; + const body = `${marker}\n## ${process.env.BENCHMARK_TITLE}\n\nBenchmark running: ${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`; + const comments = await github.paginate(github.rest.issues.listComments, { + ...context.repo, + issue_number: Number(process.env.PR_NUMBER), + per_page: 100, + }); + const existing = comments.find((comment) => + comment.user?.type === "Bot" && comment.body?.includes(marker) + ); + if (existing) { + await github.rest.issues.updateComment({ ...context.repo, comment_id: existing.id, body }); + } else { + await github.rest.issues.createComment({ + ...context.repo, + issue_number: Number(process.env.PR_NUMBER), + body, + }); + } + + - name: Check out trusted benchmark tooling + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + ref: ${{ github.event.repository.default_branch }} + path: harness + persist-credentials: false + + - name: Check out candidate + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + ref: ${{ needs.resolve.outputs.head_sha }} + path: candidate + persist-credentials: false + + - name: Check out base + if: needs.resolve.outputs.compare == 'true' + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + ref: ${{ needs.resolve.outputs.base_sha }} + path: baseline + persist-credentials: false + + - name: Check out pinned ClawBench + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + repository: kernel/ClawBench + ref: 45a71c4b0c78186851c94cfc77bfe619c9e01387 + path: clawbench + persist-credentials: false + + - uses: oven-sh/setup-bun@3d267786b128fe76c2f16a390aa2448b815359f3 # v2.1.2 + with: + bun-version: "1.3.3" + + - uses: astral-sh/setup-uv@b75a909f75acd358c2196fb9a5f1299a9a8868a4 # v6.7.0 + with: + version: "0.8.17" + + - name: Install benchmark tools + run: | + bun install --cwd harness --frozen-lockfile + archive="$RUNNER_TEMP/hypeman_0.18.0_linux_amd64.tar.gz" + checksums="$RUNNER_TEMP/hypeman_0.18.0_checksums.txt" + curl --fail --location --silent --show-error \ + https://github.com/kernel/hypeman-cli/releases/download/v0.18.0/hypeman_0.18.0_linux_amd64.tar.gz \ + --output "$archive" + curl --fail --location --silent --show-error \ + https://github.com/kernel/hypeman-cli/releases/download/v0.18.0/hypeman_0.18.0_checksums.txt \ + --output "$checksums" + (cd "$RUNNER_TEMP" && grep 'hypeman_0.18.0_linux_amd64.tar.gz$' "$checksums" | sha256sum --check --strict) + tar -xzf "$archive" -C "$RUNNER_TEMP" hypeman + sudo install "$RUNNER_TEMP/hypeman" /usr/local/bin/hypeman + uv sync --directory clawbench --frozen + + - name: Build candidate image + working-directory: candidate + run: ./benchmarks/harbor/build-image.sh + + - name: Build base image + if: needs.resolve.outputs.compare == 'true' + working-directory: baseline + run: ./benchmarks/harbor/build-image.sh + + - name: Run synchronized benchmark arms + id: run + shell: bash + env: + CLAWBENCH_REPO: ${{ github.workspace }}/clawbench + CLAWBENCH_REF: 45a71c4b0c78186851c94cfc77bfe619c9e01387 + HARBOR_BENCHMARK_TIMEOUT: 4h + BENCHMARK_AGENT: ${{ needs.resolve.outputs.agent }} + BENCHMARK_TASK: ${{ needs.resolve.outputs.task }} + BENCHMARK_COMPARE: ${{ needs.resolve.outputs.compare }} + BENCHMARK_EXPERIMENT: ${{ needs.resolve.outputs.experiment }} + run: | + set -u + jobs_root="$RUNNER_TEMP/harbor-jobs" + mkdir -p "$jobs_root" + candidate_job="candidate-$BENCHMARK_EXPERIMENT" + baseline_job="baseline-$BENCHMARK_EXPERIMENT" + + run_arm() { + local checkout=$1 arm=$2 job_name=$3 + set +e + "$checkout/benchmarks/harbor/clawbench/run.sh" \ + "$BENCHMARK_AGENT" \ + "$BENCHMARK_TASK" \ + "$job_name" \ + "$jobs_root/$arm" \ + >"$RUNNER_TEMP/$arm.log" 2>&1 + echo $? >"$RUNNER_TEMP/$arm.status" + } + + run_arm "$GITHUB_WORKSPACE/candidate" candidate "$candidate_job" & + candidate_pid=$! + if [[ "$BENCHMARK_COMPARE" == "true" ]]; then + run_arm "$GITHUB_WORKSPACE/baseline" baseline "$baseline_job" & + baseline_pid=$! + fi + + while kill -0 "$candidate_pid" 2>/dev/null || { [[ -n "${baseline_pid:-}" ]] && kill -0 "$baseline_pid" 2>/dev/null; }; do + echo "ClawBench is still running at $(date -u +%Y-%m-%dT%H:%M:%SZ)" + sleep 60 + done + wait "$candidate_pid" + if [[ -n "${baseline_pid:-}" ]]; then wait "$baseline_pid"; fi + + candidate_status=$(cat "$RUNNER_TEMP/candidate.status") + baseline_status=0 + if [[ -f "$RUNNER_TEMP/baseline.status" ]]; then + baseline_status=$(cat "$RUNNER_TEMP/baseline.status") + fi + echo "candidate_status=$candidate_status" >>"$GITHUB_OUTPUT" + echo "baseline_status=$baseline_status" >>"$GITHUB_OUTPUT" + echo "candidate_dir=$jobs_root/candidate/$candidate_job" >>"$GITHUB_OUTPUT" + echo "baseline_dir=$jobs_root/baseline/$baseline_job" >>"$GITHUB_OUTPUT" + + if ((candidate_status != 0)); then tail -100 "$RUNNER_TEMP/candidate.log" >&2; fi + if ((baseline_status != 0)); then tail -100 "$RUNNER_TEMP/baseline.log" >&2; fi + + - name: Publish Braintrust experiment + id: publish + continue-on-error: true + shell: bash + env: + CANDIDATE_DIR: ${{ steps.run.outputs.candidate_dir }} + BASELINE_DIR: ${{ steps.run.outputs.baseline_dir }} + BENCHMARK_EXPERIMENT: ${{ needs.resolve.outputs.experiment }} + run: | + args=() + [[ -f "$CANDIDATE_DIR/result.json" ]] && \ + args+=(--arm "candidate=$CANDIDATE_DIR") + [[ -f "$BASELINE_DIR/result.json" ]] && \ + args+=(--arm "baseline=$BASELINE_DIR") + ((${#args[@]} > 0)) || { echo "No completed Harbor job to publish" >&2; exit 1; } + bun harness/benchmarks/harbor/publish-braintrust.ts \ + --experiment "$BENCHMARK_EXPERIMENT" \ + --output "$RUNNER_TEMP/publication.json" \ + "${args[@]}" + + - name: Render benchmark report + id: report + if: always() + continue-on-error: true + shell: bash + env: + CANDIDATE_DIR: ${{ steps.run.outputs.candidate_dir }} + BASELINE_DIR: ${{ steps.run.outputs.baseline_dir }} + CANDIDATE_STATUS: ${{ steps.run.outputs.candidate_status }} + BASELINE_STATUS: ${{ steps.run.outputs.baseline_status }} + BENCHMARK_COMPARE: ${{ needs.resolve.outputs.compare }} + BENCHMARK_TITLE: ${{ needs.resolve.outputs.title }} + run: | + args=() + statuses=(--status "candidate=${CANDIDATE_STATUS:-1}") + if [[ "$BENCHMARK_COMPARE" == "true" ]]; then + statuses+=(--status "baseline=${BASELINE_STATUS:-1}") + fi + [[ -f "$CANDIDATE_DIR/result.json" ]] && \ + args+=(--arm "candidate=$CANDIDATE_DIR") + [[ -f "$BASELINE_DIR/result.json" ]] && \ + args+=(--arm "baseline=$BASELINE_DIR") + ((${#args[@]} > 0)) || { echo "No completed Harbor job to report" >&2; exit 1; } + publication=() + [[ -f "$RUNNER_TEMP/publication.json" ]] && \ + publication=(--publication "$RUNNER_TEMP/publication.json") + bun harness/benchmarks/harbor/report.ts \ + --title "$BENCHMARK_TITLE" \ + --json "$RUNNER_TEMP/benchmark-summary.json" \ + --markdown "$RUNNER_TEMP/benchmark-summary.md" \ + "${publication[@]}" \ + "${statuses[@]}" \ + "${args[@]}" + cat "$RUNNER_TEMP/benchmark-summary.md" >>"$GITHUB_STEP_SUMMARY" + + - name: Update PR benchmark comment + if: always() && needs.resolve.outputs.pr_number != '' + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 + env: + PR_NUMBER: ${{ needs.resolve.outputs.pr_number }} + BENCHMARK_TITLE: ${{ needs.resolve.outputs.title }} + with: + script: | + const fs = require("fs"); + const marker = ""; + const reportPath = `${process.env.RUNNER_TEMP}/benchmark-summary.md`; + const body = fs.existsSync(reportPath) + ? fs.readFileSync(reportPath, "utf8") + : `${marker}\n## ${process.env.BENCHMARK_TITLE}\n\nBenchmark failed before a report was produced. [Open the workflow run](${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}).`; + const comments = await github.paginate(github.rest.issues.listComments, { + ...context.repo, + issue_number: Number(process.env.PR_NUMBER), + per_page: 100, + }); + const existing = comments.find((comment) => + comment.user?.type === "Bot" && comment.body?.includes(marker) + ); + if (existing) { + await github.rest.issues.updateComment({ ...context.repo, comment_id: existing.id, body }); + } else { + await github.rest.issues.createComment({ + ...context.repo, + issue_number: Number(process.env.PR_NUMBER), + body, + }); + } + + - name: Check benchmark execution + if: always() + shell: bash + env: + CANDIDATE_STATUS: ${{ steps.run.outputs.candidate_status }} + BASELINE_STATUS: ${{ steps.run.outputs.baseline_status }} + PUBLISH_OUTCOME: ${{ steps.publish.outcome }} + REPORT_OUTCOME: ${{ steps.report.outcome }} + run: | + [[ "$CANDIDATE_STATUS" == "0" ]] + [[ "$BASELINE_STATUS" == "0" ]] + [[ "$PUBLISH_OUTCOME" == "success" ]] + [[ "$REPORT_OUTCOME" == "success" ]] diff --git a/.gitignore b/.gitignore index 4069f2b..0240eaf 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ # Dependencies node_modules/ +__pycache__/ +*.py[cod] npm-debug.log* yarn-debug.log* yarn-error.log* @@ -107,5 +109,9 @@ Makefile # private key mcp-key.pem +# Harbor benchmark runtime data +benchmarks/harbor/.image.env +benchmarks/harbor/image/source-sha + # TypeScript incremental build cache tsconfig.tsbuildinfo diff --git a/README.md b/README.md index 21be43e..062e5a1 100644 --- a/README.md +++ b/README.md @@ -298,7 +298,7 @@ Each Kernel feature has a single `manage_*` tool with an `action` parameter, kee One additional Managed Auth helper (`begin_auth_login`) is marked app-only (`_meta.ui.visibility: ["app"]`); it refuses to execute on hosts that do not declare MCP Apps support. The App forwards the server-issued signed flow checkpoint to the shared `manage_auth_connections` `wait` action, so flow identity and terminal-state decisions stay on the server. -Self-hosted deployments can hide sensitive tool families by setting `KERNEL_MCP_DISABLED_TOOLSETS` to a comma-separated list. For example, `KERNEL_MCP_DISABLED_TOOLSETS=api_keys` prevents `manage_api_keys` from being registered. +Self-hosted deployments can select tool families with `KERNEL_MCP_ENABLED_TOOLSETS` or hide them with `KERNEL_MCP_DISABLED_TOOLSETS`. Both accept comma- or space-separated toolset names and standalone aliases. For example, `KERNEL_MCP_ENABLED_TOOLSETS="playwright computer"` exposes browser-control tools without browser lifecycle or managed-auth tools, while `KERNEL_MCP_DISABLED_TOOLSETS=api_keys` only removes `manage_api_keys`. `get_connection_context` remains available in either mode. Call `get_connection_context` before deciding whether to create or select a project. Its canonical `connection_scope` reports whether the connection is organization-wide or fixed to a project. Project-scoped tools advertise an optional `project` (name or ID) and a deprecated `project_id`: organization-wide connections may omit them to preserve organization-wide reads and API default-project behavior, while fixed-project connections may omit them or pass the matching project. Project resources use project-qualified `kernel://orgs/{organizationId}/projects/{projectId}/...` URIs. Authorization remains enforced by the Kernel API; selecting a project never grants access to it. diff --git a/benchmarks/harbor/README.md b/benchmarks/harbor/README.md new file mode 100644 index 0000000..b2d8c1f --- /dev/null +++ b/benchmarks/harbor/README.md @@ -0,0 +1,101 @@ +# Benchmark Kernel MCP with ClawBench + +[ClawBench](https://github.com/TIGER-AI-Lab/ClawBench) is a suite of browser tasks. Each task describes work to complete on a real website and an evaluator that watches for the network request representing completion. ClawBench then judges the submitted request parameters. + +[Harbor](https://github.com/laude-institute/harbor) runs those tasks as reproducible agent trials. For each trial, Harbor creates an isolated environment, installs a stock agent such as Codex or Claude Code, gives it the task's MCP tools and instruction, runs the verifier, and writes the reward and ATIF trajectory to a job directory. + +This benchmark uses `harbor_hypeman:HypemanEnvironment` as Harbor's execution backend. For every trial, Harbor asks Hypeman to start an isolated VM from this repository's benchmark image. Everything for that trial runs inside that VM: + +1. ClawBench creates one stealth Kernel browser and attaches its request evaluator. +2. The task setup starts Redis and the locally built `kernel-mcp-server` on port 3002. +3. Harbor starts the stock agent with a stdio MCP command that connects to that local server. +4. The agent controls ClawBench's existing browser through `execute_playwright_code`; it cannot create browsers or use managed auth. +5. ClawBench scores the intercepted request, downloads the replay, and deletes the browser. + +The image records the current Git SHA, and the generated task records the ClawBench SHA and browser session ID. The additional `kernel_mcp_valid` result confirms that the agent called the local server with the browser ClawBench created. Task reward still comes directly from ClawBench. + +## Requirements + +- `uv`, Harbor 0.21.0, and `harbor-hypeman` 0.1.1 +- Hypeman CLI credentials +- a ClawBench checkout containing pinned commit `45a71c4` +- `KERNEL_MCP_BENCHMARK_API_KEY` scoped to an isolated evaluation project, plus its `KERNEL_PROJECT` name +- `PURELY_MAIL_API_KEY` and `PURELY_MAIL_DOMAIN` for ClawBench account tasks +- `OPENAI_API_KEY` for Codex, or Anthropic credentials for Claude Code +- the ClawBench judge variables when using a hosted judge: `CLAWBENCH_JUDGE_BASE_URL`, `CLAWBENCH_JUDGE_API_KEY`, `CLAWBENCH_JUDGE_MODEL`, and `CLAWBENCH_JUDGE_API_TYPE` +- `BRAINTRUST_API_KEY` and `BRAINTRUST_PROJECT` when publishing results + +## Build the trial image + +From the `kernel-mcp-server` checkout: + +```bash +./benchmarks/harbor/build-image.sh +``` + +This builds the current checkout with Bun and writes the image reference and Git SHA to the ignored `benchmarks/harbor/.image.env` file. + +## Run one task + +```bash +export CLAWBENCH_REPO=../ClawBench +./benchmarks/harbor/clawbench/run.sh codex \ + v2-1134-chapter-finder-redcross +``` + +## Run the full suite + +```bash +export CLAWBENCH_REPO=../ClawBench +HARBOR_N_CONCURRENT=10 \ + ./benchmarks/harbor/clawbench/run.sh codex all +``` + +Codex defaults to version `0.120.0` with `gpt-5.6-luna`. Claude Code defaults to version `2.1.238` with `claude-sonnet-5`. Override these with `CODEX_BENCHMARK_MODEL`, `CODEX_BENCHMARK_VERSION`, `CLAUDE_BENCHMARK_MODEL`, or `CLAUDE_BENCHMARK_VERSION`. + +Single-task runs have a 40-minute wall-clock limit. Full-suite runs default to six hours. Set `HARBOR_BENCHMARK_TIMEOUT` to override either limit. Set `HARBOR_JOBS_DIR` to choose where Harbor writes results. + +## GitHub Actions + +The `Benchmark ClawBench` workflow runs the complete suite weekly and on demand. Select it from the Actions tab and provide either a same-repository PR number or a ref. Comparison runs benchmark the candidate SHA against its merge base so unrelated changes on the target branch do not affect the delta. Harbor, Hypeman, agent, and model versions are pinned by the workflow and each arm's observed agent configuration appears in the report. + +An organization member or repository collaborator can also start the full PR comparison by commenting this exact command on a same-repository pull request: + +```text +/benchmark clawbench +``` + +The command parser does not execute comment text. It accepts only the exact command, rejects fork pull requests and untrusted commenters, and resolves the candidate and merge-base SHAs through GitHub's API. The workflow uses the `benchmarks` environment for credentials, updates one benchmark comment on the pull request, and publishes the same results to Braintrust. + +## Results + +Harbor writes its normal job directory, including: + +- `trajectory.json`: the agent's ATIF messages and tool calls +- `reward.json`: ClawBench's reward plus the `kernel_mcp_valid` diagnostic +- `clawbench-result.json`: evaluator details +- `kernel-mcp-result.json`: local-source and same-browser wiring details +- `recording.mp4`: the finalized Kernel replay +- `kernel-mcp/`: local server logs and the source/session manifest + +Generate a redacted summary from one or more completed jobs: + +```bash +bun run benchmark:report -- \ + --arm candidate=/path/to/candidate-job \ + --arm baseline=/path/to/baseline-job \ + --json /tmp/benchmark-summary.json \ + --markdown /tmp/benchmark-summary.md +``` + +Publish those arms as one idempotent Braintrust experiment: + +```bash +BRAINTRUST_PROJECT=kernel-mcp-server-benchmarks \ + bun run benchmark:publish -- \ + --experiment pr-162-a60c518-example \ + --arm candidate=/path/to/candidate-job \ + --arm baseline=/path/to/baseline-job +``` + +The experiment name and deterministic row/span IDs make it safe to publish the same job directories again. Re-publication replaces the rows and refreshes experiment metadata. Rows contain task identity, numeric rewards, provenance, bounded errors, timing, token, call, and cost metrics. ATIF agent/tool activity is attached as child spans after secret redaction. Task instructions, ground truth, browser session URLs, and recordings are not placed on experiment rows or public pull-request comments. diff --git a/benchmarks/harbor/bin/kernel-mcp-local b/benchmarks/harbor/bin/kernel-mcp-local new file mode 100755 index 0000000..301c209 --- /dev/null +++ b/benchmarks/harbor/bin/kernel-mcp-local @@ -0,0 +1,18 @@ +#!/bin/sh +# Harbor launches MCP servers as stdio subprocesses of the benchmark agent. +# This wrapper turns that stdio connection into an authenticated connection to +# the kernel-mcp-server HTTP endpoint running locally in the same Hypeman task. +# The setup script writes the project-scoped key to /run so it never appears in +# the generated agent configuration. +set -eu + +key_file=/run/kernel-mcp-benchmark/api-key +if [ -z "${KERNEL_API_KEY:-}" ] && [ -r "$key_file" ]; then + KERNEL_API_KEY=$(cat "$key_file") + export KERNEL_API_KEY +fi +: "${KERNEL_API_KEY:?KERNEL_API_KEY is required}" + +exec mcp-remote \ + http://127.0.0.1:3002/mcp \ + --header "Authorization: Bearer ${KERNEL_API_KEY}" diff --git a/benchmarks/harbor/bin/start-kernel-mcp-server b/benchmarks/harbor/bin/start-kernel-mcp-server new file mode 100755 index 0000000..d1a2ba0 --- /dev/null +++ b/benchmarks/harbor/bin/start-kernel-mcp-server @@ -0,0 +1,69 @@ +#!/bin/bash +# Start the services used by the source-pinned Kernel MCP build inside a Harbor +# trial. ClawBench has already created the browser; this script starts Redis and +# Next.js, stores the project-scoped key for the stdio wrapper, waits for MCP to +# accept connections, and records which source build and browser the trial used. +set -euo pipefail + +: "${KERNEL_API_KEY:?KERNEL_API_KEY is required}" + +log_dir=/logs/kernel-mcp +key_dir=/run/kernel-mcp-benchmark +mkdir -p "$log_dir" /logs/artifacts "$key_dir" +chmod 0777 "$log_dir" /logs/artifacts +chmod 0700 "$key_dir" +printf '%s' "$KERNEL_API_KEY" >"$key_dir/api-key" +chmod 0600 "$key_dir/api-key" + +redis-server --daemonize yes --bind 127.0.0.1 --port 6379 \ + --logfile "$log_dir/redis.log" --dir /tmp + +export CLERK_SECRET_KEY=${CLERK_SECRET_KEY:-sk_test_kernel_mcp_benchmark_local_only} +export NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY=${NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY:-pk_test_YmVuY2htYXJrLmNsZXJrLmFjY291bnRzLmRldiQ} + +cd /opt/kernel-mcp-server +nohup ./node_modules/.bin/next start -p 3002 \ + >"$log_dir/server.stdout.log" \ + 2>"$log_dir/server.stderr.log" & +echo $! >"$log_dir/server.pid" + +for _ in $(seq 1 90); do + if curl -fsS -X POST http://127.0.0.1:3002/mcp \ + -H "Authorization: Bearer ${KERNEL_API_KEY}" \ + -H "Content-Type: application/json" \ + -H "Accept: application/json, text/event-stream" \ + --data '{"jsonrpc":"2.0","id":"benchmark-healthcheck","method":"initialize","params":{"protocolVersion":"2025-03-26","capabilities":{},"clientInfo":{"name":"harbor-healthcheck","version":"1.0.0"}}}' \ + >"$log_dir/initialize-response.txt"; then + break + fi + sleep 1 +done + +if [ ! -s "$log_dir/initialize-response.txt" ]; then + echo "Kernel MCP server did not become ready" >&2 + tail -100 "$log_dir/server.stderr.log" >&2 || true + exit 1 +fi + +python3 - <<'PY' +import json +import os +from pathlib import Path + +browser_path = Path("/my-info/kernel_browser.json") +try: + browser = json.loads(browser_path.read_text()) +except (OSError, json.JSONDecodeError): + browser = {} + +manifest = { + "kernel_mcp_server_sha": Path("/opt/kernel-mcp-server/SOURCE_SHA").read_text().strip(), + "clawbench_source_sha": os.environ.get("CLAWBENCH_SOURCE_SHA", ""), + "browser_session_id": browser.get("session_id"), + "enabled_toolsets": os.environ.get("KERNEL_MCP_ENABLED_TOOLSETS", ""), + "image": os.environ.get("KERNEL_MCP_BENCHMARK_IMAGE", ""), +} +Path("/logs/kernel-mcp/run-manifest.json").write_text(json.dumps(manifest, indent=2)) +PY + +printf 'ready\n' >"$log_dir/ready" diff --git a/benchmarks/harbor/build-image.sh b/benchmarks/harbor/build-image.sh new file mode 100755 index 0000000..d8319af --- /dev/null +++ b/benchmarks/harbor/build-image.sh @@ -0,0 +1,65 @@ +#!/bin/bash +set -euo pipefail + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +cd "$repo_root" + +source_sha=$(git rev-parse HEAD) +source_sha_file=benchmarks/harbor/image/source-sha +build_log=$(mktemp) +trap 'rm -f "$source_sha_file" "$build_log"' EXIT + +printf '%s\n' "$source_sha" >"$source_sha_file" + +set +e +hypeman build \ + --file benchmarks/harbor/image/Dockerfile \ + --cpus 4 \ + --memory 8192 \ + --timeout 1800 \ + . 2>&1 | tee "$build_log" +build_status=${PIPESTATUS[0]} +set -e + +build_id=$(sed -n -E 's/^Build (ID|started): //p' "$build_log" | tail -1) +if [[ -z "$build_id" ]]; then + echo "Hypeman did not return a build ID" >&2 + exit 1 +fi + +image_ref="docker.io/builds/$build_id:latest" +if ((build_status != 0)); then + echo "Build record failed; checking for a delayed ready image for up to 5 minutes" >&2 + image_ready=false + for _ in $(seq 1 30); do + if hypeman --format json image list | python3 -c ' +import json +import sys + +image_ref = sys.argv[1] +expected = {image_ref, image_ref.removeprefix("docker.io/")} +images = json.load(sys.stdin) +raise SystemExit( + 0 + if any(image.get("name") in expected and image.get("status") == "ready" for image in images) + else 1 +) +' "$image_ref" + then + image_ready=true + break + fi + sleep 10 + done + if [[ "$image_ready" != true ]]; then + exit "$build_status" + fi +fi + +cat >benchmarks/harbor/.image.env < str: + lines = task_toml.splitlines() + output: list[str] = [] + dropping = False + for line in lines: + if line.strip() == "[[environment.mcp_servers]]": + dropping = True + continue + if dropping and line.startswith("["): + dropping = False + if not dropping: + output.append(line) + return "\n".join(output).rstrip() + "\n" + + +def _add_environment( + task_toml: str, *, image: str, server_sha: str, clawbench_sha: str +) -> str: + lines = task_toml.splitlines() + output: list[str] = [] + inserted_image = False + inserted_env = False + for line in lines: + output.append(line) + if line.strip() == "[environment]": + output.append(f"docker_image = {json.dumps(image)}") + inserted_image = True + elif line.strip() == "[environment.env]": + output.extend( + [ + f"KERNEL_MCP_BENCHMARK_IMAGE = {json.dumps(image)}", + f"KERNEL_MCP_SOURCE_SHA = {json.dumps(server_sha)}", + f"CLAWBENCH_SOURCE_SHA = {json.dumps(clawbench_sha)}", + f"KERNEL_MCP_ENABLED_TOOLSETS = {json.dumps(ENABLED_TOOLSETS)}", + 'API_BASE_URL = "${KERNEL_API_BASE_URL:-}"', + 'KERNEL_PROJECT = "${KERNEL_PROJECT:-}"', + 'REDIS_URL = "redis://127.0.0.1:6379"', + ] + ) + inserted_env = True + if not inserted_image or not inserted_env: + raise ValueError("generated task is missing Harbor environment sections") + output.extend( + [ + "", + "[[environment.mcp_servers]]", + 'name = "kernel"', + 'transport = "stdio"', + 'command = "/usr/local/bin/kernel-mcp-local"', + "args = []", + ] + ) + return "\n".join(output).rstrip() + "\n" + + +def _patch_setup(setup: str) -> str: + install = """install_clawbench_runtime() { + mkdir -p /app/src + rm -rf /app/src/runtime-server /app/src/chrome-extension /app/src/shared /app/src/harbor + cp -a /runtime-server /app/src/runtime-server + cp -a /chrome-extension /app/src/chrome-extension + cp -a /shared /app/src/shared + cp -a /harbor /app/src/harbor + chmod +x /app/src/harbor/*.sh /app/src/harbor/*.py + cd /app/src/runtime-server + UV_PYTHON_PREFERENCE=only-system uv sync --frozen + uv pip install --python .venv/bin/python fpdf2 + cd / +} + +install_clawbench_runtime +""" + marker = "mkdir -p /data /logs/verifier /extra_info\n" + if marker not in setup: + raise ValueError("generated setup script is missing directory initialization") + setup = setup.replace(marker, marker + "\n" + install, 1) + runtime_marker = "/app/src/harbor/start-runtime.sh\n" + if runtime_marker not in setup: + raise ValueError("generated setup script is missing runtime startup") + return setup.replace( + runtime_marker, + runtime_marker + "\nstart-kernel-mcp-server\n", + 1, + ) + + +def _patch_verifier(test_script: str) -> str: + verify_marker = ( + "/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py\n" + ) + if verify_marker not in test_script: + raise ValueError("generated verifier script is missing ClawBench verification") + return test_script.replace( + verify_marker, + verify_marker + + "mkdir -p /logs/verifier/kernel-mcp\n" + + "cp -a /logs/kernel-mcp/. /logs/verifier/kernel-mcp/\n" + + "/app/src/runtime-server/.venv/bin/python " + + "/app/src/harbor/verify-kernel-mcp-task.py\n", + 1, + ) + + +def _patch_instruction(instruction: str) -> str: + instruction = instruction.replace( + "Use only Playwright MCP browser tools plus reading files", + "Use only Kernel MCP browser-control tools plus reading files", + ) + return ( + instruction.rstrip() + + """ + +--- +Kernel MCP benchmark arm: +- Wait for the `kernel` MCP server to finish initializing before starting. In Claude Code, call `WaitForMcpServers` if it is still pending; do not conclude that the tools are unavailable while it initializes. +- Read `/my-info/kernel_browser.json` and use its existing `session_id` for every `execute_playwright_code` call. +- Do not create, list, update, or delete browsers. Browser lifecycle tools and `computer_action` are intentionally unavailable. +- Use Kernel MCP `execute_playwright_code` for all browser interaction. Do not use Playwright MCP or a direct CDP client. +- Interact through visible page navigation and DOM/UI actions. Do not call `fetch`, `XMLHttpRequest`, Playwright request APIs, or other direct HTTP clients inside `execute_playwright_code`. +- Use the PurelyMail-backed credentials already provided under `./my-info/` when the task requires an account. +- Do not use Kernel managed auth, create an auth connection, or start a hosted login flow. +- Complete and submit the task through the existing browser, then stop. +""" + ) + + +def transform_task( + task_dir: Path, *, image: str, server_sha: str, clawbench_sha: str +) -> None: + dockerfile = task_dir / "environment" / "Dockerfile" + dockerfile.unlink(missing_ok=True) + + task_toml_path = task_dir / "task.toml" + task_toml = _drop_mcp_servers(task_toml_path.read_text()) + task_toml_path.write_text( + _add_environment( + task_toml, + image=image, + server_sha=server_sha, + clawbench_sha=clawbench_sha, + ) + ) + + step_dir = task_dir / "steps" / "run" + setup_path = step_dir / "workdir" / "setup.sh" + setup_path.write_text(_patch_setup(setup_path.read_text())) + setup_path.chmod(0o755) + + test_path = step_dir / "tests" / "test.sh" + test_path.write_text(_patch_verifier(test_path.read_text())) + test_path.chmod(0o755) + + instruction_path = step_dir / "instruction.md" + instruction_path.write_text(_patch_instruction(instruction_path.read_text())) + + verifier_source = Path(__file__).with_name("verify-task.py") + verifier_target = task_dir / "environment" / "harbor" / "verify-kernel-mcp-task.py" + shutil.copy2(verifier_source, verifier_target) + verifier_target.chmod(0o755) + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Replace a generated ClawBench task's Playwright MCP server with the local Kernel MCP build" + ) + parser.add_argument("task_dir", type=Path) + parser.add_argument("--image", required=True) + parser.add_argument("--server-sha", required=True) + parser.add_argument("--clawbench-sha", required=True) + args = parser.parse_args() + transform_task( + args.task_dir, + image=args.image, + server_sha=args.server_sha, + clawbench_sha=args.clawbench_sha, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/harbor/clawbench/run.sh b/benchmarks/harbor/clawbench/run.sh new file mode 100755 index 0000000..4f59be5 --- /dev/null +++ b/benchmarks/harbor/clawbench/run.sh @@ -0,0 +1,160 @@ +#!/bin/bash +# Run one ClawBench task or the full suite through the local Kernel MCP build. +# +# The script asks ClawBench to generate ordinary Harbor tasks, rewrites those +# tasks with prepare-task.py, then lets Harbor create one isolated Hypeman +# environment per trial and install the selected stock agent inside it. +set -euo pipefail + +usage() { + echo "usage: $0 [task-id|all] [job-name] [jobs-dir]" >&2 + exit 2 +} + +agent=${1:-} +[[ "$agent" == "claude-code" || "$agent" == "codex" ]] || usage +task_id=${2:-v2-1134-chapter-finder-redcross} + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd) +benchmark_dir="$repo_root/benchmarks/harbor" +image_env="$benchmark_dir/.image.env" +clawbench_repo=${CLAWBENCH_REPO:-$repo_root/../ClawBench} +clawbench_ref=${CLAWBENCH_REF:-45a71c4b0c78186851c94cfc77bfe619c9e01387} + +[[ -f "$image_env" ]] || { + echo "Missing $image_env; run benchmarks/harbor/build-image.sh first" >&2 + exit 1 +} +[[ -d "$clawbench_repo/.git" || -f "$clawbench_repo/.git" ]] || { + echo "ClawBench checkout not found at $clawbench_repo" >&2 + exit 1 +} +git -C "$clawbench_repo" merge-base --is-ancestor "$clawbench_ref" HEAD || { + echo "ClawBench checkout must contain $clawbench_ref" >&2 + exit 1 +} + +if [[ -f "$clawbench_repo/.env" ]]; then + set -a + source "$clawbench_repo/.env" + set +a +fi +set -a +source "$image_env" +set +a + +: "${KERNEL_MCP_BENCHMARK_API_KEY:?KERNEL_MCP_BENCHMARK_API_KEY is required}" +: "${PURELY_MAIL_API_KEY:?PURELY_MAIL_API_KEY is required}" +: "${PURELY_MAIL_DOMAIN:?PURELY_MAIL_DOMAIN is required}" + +case "$agent" in + claude-code) + if [[ -z "${ANTHROPIC_API_KEY:-}" && -z "${ANTHROPIC_AUTH_TOKEN:-}" && -z "${CLAUDE_CODE_OAUTH_TOKEN:-}" ]]; then + echo "ANTHROPIC_API_KEY, ANTHROPIC_AUTH_TOKEN, or CLAUDE_CODE_OAUTH_TOKEN is required" >&2 + exit 1 + fi + if [[ -z "${ANTHROPIC_API_KEY:-}" && -z "${ANTHROPIC_AUTH_TOKEN:-}" ]]; then + ANTHROPIC_AUTH_TOKEN=$CLAUDE_CODE_OAUTH_TOKEN + CLAUDE_FORCE_OAUTH=1 + export ANTHROPIC_AUTH_TOKEN CLAUDE_FORCE_OAUTH + fi + model=${CLAUDE_BENCHMARK_MODEL:-claude-sonnet-5} + version=${CLAUDE_BENCHMARK_VERSION:-2.1.238} + ;; + codex) + : "${OPENAI_API_KEY:?OPENAI_API_KEY is required}" + model=${CODEX_BENCHMARK_MODEL:-gpt-5.6-luna} + version=${CODEX_BENCHMARK_VERSION:-0.120.0} + ;; +esac + +harbor_version=${HARBOR_VERSION:-0.21.0} +harbor_hypeman_version=${HARBOR_HYPEMAN_VERSION:-0.1.1} + +runtime_root=$(mktemp -d) +runtime_env=$(mktemp) +trap 'rm -rf "$runtime_root"; rm -f "$runtime_env"' EXIT + +dataset="$runtime_root/dataset" +adapt_args=( + --output-dir "$dataset" + --browser-runtime kernel + --browser-runtime-options '{"stealth": true}' + --overwrite +) +if [[ "$task_id" != "all" ]]; then + adapt_args+=(--task-ids "$task_id") +fi +uv --directory "$clawbench_repo" run clawbench-harbor-adapt "${adapt_args[@]}" + +mapfile -t task_dirs < <(find "$dataset" -mindepth 1 -maxdepth 1 -type d | sort) +((${#task_dirs[@]} > 0)) || { + echo "ClawBench did not generate tasks for $task_id" >&2 + exit 1 +} + +for task_dir in "${task_dirs[@]}"; do + python3 "$benchmark_dir/clawbench/prepare-task.py" "$task_dir" \ + --image "$KERNEL_MCP_BENCHMARK_IMAGE" \ + --server-sha "$KERNEL_MCP_SOURCE_SHA" \ + --clawbench-sha "$clawbench_ref" +done + +export KERNEL_API_KEY=$KERNEL_MCP_BENCHMARK_API_KEY +export KERNEL_BASE_URL=${KERNEL_BASE_URL:-https://api.onkernel.com} +export KERNEL_API_BASE_URL=${KERNEL_API_BASE_URL:-$KERNEL_BASE_URL} + +{ + printf 'KERNEL_API_KEY=%s\n' "$KERNEL_API_KEY" + printf 'KERNEL_BASE_URL=%s\n' "$KERNEL_BASE_URL" + printf 'KERNEL_API_BASE_URL=%s\n' "$KERNEL_API_BASE_URL" + printf 'API_BASE_URL=%s\n' "$KERNEL_API_BASE_URL" + printf 'KERNEL_PROJECT=%s\n' "${KERNEL_PROJECT:-}" + printf 'PURELY_MAIL_API_KEY=%s\n' "$PURELY_MAIL_API_KEY" + printf 'PURELY_MAIL_DOMAIN=%s\n' "$PURELY_MAIL_DOMAIN" + printf 'CLAWBENCH_JUDGE_BASE_URL=%s\n' "${CLAWBENCH_JUDGE_BASE_URL:-}" + printf 'CLAWBENCH_JUDGE_API_KEY=%s\n' "${CLAWBENCH_JUDGE_API_KEY:-}" + printf 'CLAWBENCH_JUDGE_MODEL=%s\n' "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" + printf 'CLAWBENCH_JUDGE_API_TYPE=%s\n' "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" +} >"$runtime_env" + +case "$agent" in + claude-code) + { + printf 'ANTHROPIC_API_KEY=%s\n' "${ANTHROPIC_API_KEY:-}" + printf 'ANTHROPIC_AUTH_TOKEN=%s\n' "${ANTHROPIC_AUTH_TOKEN:-}" + printf 'ANTHROPIC_BASE_URL=%s\n' "${ANTHROPIC_BASE_URL:-}" + printf 'CLAUDE_CODE_OAUTH_TOKEN=%s\n' "${CLAUDE_CODE_OAUTH_TOKEN:-}" + printf 'CLAUDE_FORCE_OAUTH=%s\n' "${CLAUDE_FORCE_OAUTH:-false}" + } >>"$runtime_env" + ;; + codex) + printf 'OPENAI_API_KEY=%s\n' "$OPENAI_API_KEY" >>"$runtime_env" + ;; +esac +chmod 0600 "$runtime_env" + +job_name=${3:-kernel-mcp-${agent}-${task_id}-$(date -u +%Y%m%dT%H%M%SZ)} +jobs_dir=${4:-${HARBOR_JOBS_DIR:-/tmp/kernel-mcp-clawbench-jobs}} +mkdir -p "$jobs_dir" + +if [[ "$task_id" == "all" ]]; then + default_timeout=6h +else + default_timeout=40m +fi + +timeout --signal=INT --kill-after=30s "${HARBOR_BENCHMARK_TIMEOUT:-$default_timeout}" \ + uvx --from "harbor==$harbor_version" --with "harbor-hypeman==$harbor_hypeman_version" harbor run \ + --path "$dataset" \ + --agent "$agent" \ + --model "$model" \ + --agent-kwarg "version=$version" \ + --env harbor_hypeman:HypemanEnvironment \ + --env-file "$runtime_env" \ + --job-name "$job_name" \ + --jobs-dir "$jobs_dir" \ + --n-concurrent "${HARBOR_N_CONCURRENT:-1}" \ + --max-retries 0 \ + --delete \ + --yes diff --git a/benchmarks/harbor/clawbench/verify-task.py b/benchmarks/harbor/clawbench/verify-task.py new file mode 100755 index 0000000..a314bef --- /dev/null +++ b/benchmarks/harbor/clawbench/verify-task.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +"""Record whether a ClawBench trial used the intended Kernel MCP setup. + +ClawBench's verifier remains responsible for the task reward, request +interception, replay download, and browser cleanup. This script adds one +`kernel_mcp_valid` diagnostic metric so benchmark results can distinguish a +failed task from a trial that never exercised the local server correctly. +""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +from typing import Any + +LOGS_DIR = Path(os.environ.get("HARBOR_LOGS_DIR", "/logs")) +VERIFIER_DIR = LOGS_DIR / "verifier" +PLAYWRIGHT_TOOLS = { + "execute_playwright_code", + "kernel__execute_playwright_code", + "mcp__kernel__execute_playwright_code", +} + + +def read_object(path: Path) -> dict[str, Any]: + try: + value = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError): + return {} + return value if isinstance(value, dict) else {} + + +def tool_calls(trajectory: dict[str, Any]) -> list[dict[str, Any]]: + calls: list[dict[str, Any]] = [] + for step in trajectory.get("steps") or []: + if isinstance(step, dict): + calls.extend( + call for call in step.get("tool_calls") or [] if isinstance(call, dict) + ) + return calls + + +def main() -> int: + VERIFIER_DIR.mkdir(parents=True, exist_ok=True) + browser = read_object(Path("/my-info/kernel_browser.json")) + manifest = read_object(LOGS_DIR / "kernel-mcp" / "run-manifest.json") + trajectory = read_object(LOGS_DIR / "agent" / "trajectory.json") + + expected_session = browser.get("session_id") + calls = [ + call + for call in tool_calls(trajectory) + if call.get("function_name") in PLAYWRIGHT_TOOLS + ] + called_sessions = { + arguments.get("session_id") + for call in calls + if isinstance((arguments := call.get("arguments")), dict) + } + + expected_source = os.environ.get("KERNEL_MCP_SOURCE_SHA") + checks = { + "used_kernel_mcp": bool(calls), + "used_clawbench_browser": bool(expected_session) + and called_sessions == {expected_session}, + "used_expected_source": bool(expected_source) + and manifest.get("kernel_mcp_server_sha") == expected_source, + } + valid = all(checks.values()) + + result = { + "valid": valid, + "checks": checks, + "expected_session_id": expected_session, + "called_session_ids": sorted(str(value) for value in called_sessions), + "expected_source_sha": expected_source, + "actual_source_sha": manifest.get("kernel_mcp_server_sha"), + } + (VERIFIER_DIR / "kernel-mcp-result.json").write_text(json.dumps(result, indent=2)) + + reward_path = VERIFIER_DIR / "reward.json" + rewards = read_object(reward_path) + rewards["kernel_mcp_valid"] = float(valid) + reward_path.write_text(json.dumps(rewards, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/harbor/image/Dockerfile b/benchmarks/harbor/image/Dockerfile new file mode 100644 index 0000000..9bbc1c8 --- /dev/null +++ b/benchmarks/harbor/image/Dockerfile @@ -0,0 +1,34 @@ +FROM node:22-bookworm-slim + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + git \ + jq \ + procps \ + python3 \ + redis-server \ + && rm -rf /var/lib/apt/lists/* \ + && npm install --global bun@1.3.3 mcp-remote@0.1.38 + +COPY --from=ghcr.io/astral-sh/uv:0.11.6 /uv /usr/local/bin/uv + +WORKDIR /opt/kernel-mcp-server + +COPY package.json bun.lock ./ +RUN bun install --frozen-lockfile + +COPY . . +RUN KERNEL_CLI_PROD_CLIENT_ID=kernel-mcp-benchmark \ + KERNEL_CLI_STAGING_CLIENT_ID=kernel-mcp-benchmark \ + KERNEL_CLI_DEV_CLIENT_ID=kernel-mcp-benchmark \ + CLERK_SECRET_KEY=sk_test_kernel_mcp_benchmark_local_only \ + NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY=pk_test_YmVuY2htYXJrLmNsZXJrLmFjY291bnRzLmRldiQ \ + bun run build \ + && install -m 0755 benchmarks/harbor/bin/start-kernel-mcp-server /usr/local/bin/start-kernel-mcp-server \ + && install -m 0755 benchmarks/harbor/bin/kernel-mcp-local /usr/local/bin/kernel-mcp-local \ + && install -m 0644 benchmarks/harbor/image/source-sha /opt/kernel-mcp-server/SOURCE_SHA + +ENV NEXT_TELEMETRY_DISABLED=1 +WORKDIR /app diff --git a/benchmarks/harbor/publish-braintrust.ts b/benchmarks/harbor/publish-braintrust.ts new file mode 100644 index 0000000..51b032c --- /dev/null +++ b/benchmarks/harbor/publish-braintrust.ts @@ -0,0 +1,418 @@ +#!/usr/bin/env bun +import { createHash } from "node:crypto"; +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { dirname } from "node:path"; +import { + type BenchmarkArm, + type BenchmarkTrial, + parseArmSpec, + readBenchmarkArm, + selectPrimaryReward, + summarizeArm, +} from "./results"; +import { redactString, redactValue } from "./redact"; + +interface CliOptions { + arms: string[]; + experiment?: string; + project?: string; + output?: string; +} + +interface AtifStep { + step_id?: number; + timestamp?: string; + source?: string; + model_name?: string; + message?: unknown; + tool_calls?: Array<{ + tool_call_id?: string; + function_name?: string; + arguments?: unknown; + }>; + observation?: { + results?: Array<{ source_call_id?: string; content?: unknown }>; + }; + metrics?: Record; +} + +interface BraintrustEvent { + id: string; + span_id: string; + root_span_id: string; + span_parents: string[]; + span_attributes: { name: string; type: "eval" | "llm" | "tool" }; + created?: string; + input?: unknown; + output?: unknown; + expected?: unknown; + error?: string; + scores?: Record; + metadata?: Record; + metrics?: Record; + _is_merge: false; +} + +interface BraintrustProject { + id: string; + org_id: string; + name: string; +} + +interface BraintrustExperiment { + id: string; + project_id: string; + name: string; +} + +function parseArgs(args: string[]): CliOptions { + const options: CliOptions = { arms: [] }; + for (let index = 0; index < args.length; index += 1) { + const flag = args[index]; + const value = args[index + 1]; + if (!value || !flag.startsWith("--")) + throw new Error(`Missing value for ${flag}`); + index += 1; + switch (flag) { + case "--arm": + options.arms.push(value); + break; + case "--experiment": + options.experiment = value; + break; + case "--project": + options.project = value; + break; + case "--output": + options.output = value; + break; + default: + throw new Error(`Unknown argument ${flag}`); + } + } + if (options.arms.length === 0) + throw new Error("At least one --arm name=/job/path is required"); + return options; +} + +function uuidV5(name: string): string { + const namespace = Buffer.from("cf5141b9e00051a9b55482e0567b5c88", "hex"); + const digest = createHash("sha1") + .update(namespace) + .update(name) + .digest() + .subarray(0, 16); + digest[6] = (digest[6] & 0x0f) | 0x50; + digest[8] = (digest[8] & 0x3f) | 0x80; + const hex = digest.toString("hex"); + return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`; +} + +function number(value: unknown): number | undefined { + return typeof value === "number" && Number.isFinite(value) + ? value + : undefined; +} + +function trajectorySteps(trial: BenchmarkTrial): AtifStep[] { + if (!trial.trajectoryPath) return []; + const trajectory = JSON.parse(readFileSync(trial.trajectoryPath, "utf8")) as { + steps?: AtifStep[]; + }; + return Array.isArray(trajectory.steps) ? trajectory.steps : []; +} + +function metricRecord(trial: BenchmarkTrial): Record { + return Object.fromEntries( + Object.entries({ + start: trial.metrics.start, + end: trial.metrics.end, + input_tokens: trial.metrics.inputTokens, + cached_tokens: trial.metrics.cacheTokens, + output_tokens: trial.metrics.outputTokens, + cost_usd: trial.metrics.costUsd, + duration_ms: trial.metrics.durationMs, + tool_calls: trial.metrics.toolCalls, + }).filter((entry): entry is [string, number] => entry[1] !== undefined), + ); +} + +function trialMetadata(trial: BenchmarkTrial): Record { + return { + trialName: trial.trialName, + arm: trial.arm, + verdict: + trial.errorClass === "infra" + ? "error" + : trial.scores.ungraded_rate === 1 + ? "ungraded" + : trial.scores.accuracy === 1 + ? "correct" + : "false_negative", + agent: trial.agent, + agentVersion: trial.agentVersion, + agentConfigHash: trial.agentConfigHash, + model: trial.model, + kernelMcpSha: trial.kernelMcpSha, + clawbenchSha: trial.clawbenchSha, + errorClass: trial.errorClass, + rewards: trial.rewards, + durationMs: trial.metrics.durationMs, + }; +} + +function atifEvents(trial: BenchmarkTrial, rowId: string): BraintrustEvent[] { + const events: BraintrustEvent[] = []; + const agentSteps = trajectorySteps(trial).filter( + (candidate) => candidate.source === "agent", + ); + for (const [stepIndex, step] of agentSteps.entries()) { + const stepId = step.step_id; + const stepKey = `${stepId ?? "missing"}:${stepIndex}`; + const llmId = uuidV5(`${rowId}:llm:${stepKey}`); + const start = step.timestamp + ? Date.parse(step.timestamp) / 1000 + : undefined; + const llmMetrics = Object.fromEntries( + Object.entries({ + start, + end: start, + prompt_tokens: number(step.metrics?.prompt_tokens), + completion_tokens: number(step.metrics?.completion_tokens), + cost_usd: number(step.metrics?.cost_usd), + }).filter((entry): entry is [string, number] => entry[1] !== undefined), + ); + events.push({ + id: llmId, + span_id: llmId, + root_span_id: rowId, + span_parents: [rowId], + span_attributes: { name: "agent", type: "llm" }, + created: step.timestamp, + output: redactValue(step.message), + metadata: { + phase: "agent_execution", + stepId, + stepIndex, + model: step.model_name ?? trial.model, + }, + metrics: llmMetrics, + _is_merge: false, + }); + + for (const [toolIndex, call] of (step.tool_calls ?? []).entries()) { + const toolId = uuidV5( + `${rowId}:tool:${stepKey}:${call.tool_call_id ?? toolIndex}`, + ); + const observation = step.observation?.results?.find( + (result) => result.source_call_id === call.tool_call_id, + ); + events.push({ + id: toolId, + span_id: toolId, + root_span_id: rowId, + span_parents: [llmId], + span_attributes: { name: call.function_name ?? "tool", type: "tool" }, + created: step.timestamp, + input: redactValue(call.arguments), + output: redactValue(observation?.content), + metadata: { + phase: "agent_execution", + stepId, + stepIndex, + toolCallId: call.tool_call_id, + }, + metrics: start === undefined ? undefined : { start: start, end: start }, + _is_merge: false, + }); + } + } + return events; +} + +export function buildExperimentEvents( + arms: BenchmarkArm[], + experimentName: string, +): BraintrustEvent[] { + const events: BraintrustEvent[] = []; + for (const arm of arms) { + for (const trial of arm.trials) { + const rowId = uuidV5(`${experimentName}:${arm.name}:${trial.id}`); + const primaryReward = + trial.errorClass === "infra" + ? undefined + : selectPrimaryReward(trial.rewards); + events.push({ + id: rowId, + span_id: rowId, + root_span_id: rowId, + span_parents: [], + span_attributes: { name: trial.taskName, type: "eval" }, + created: trial.startedAt, + input: { source: trial.source, taskName: trial.taskName }, + output: { + reward: primaryReward?.value, + rewardKey: primaryReward?.key, + error: trial.error ? redactString(trial.error, 400) : undefined, + }, + expected: { reward: 1 }, + error: trial.error ? redactString(trial.error, 400) : undefined, + scores: trial.scores, + metadata: trialMetadata(trial), + metrics: metricRecord(trial), + _is_merge: false, + }); + events.push(...atifEvents(trial, rowId)); + } + } + return events; +} + +function experimentMetadata(arms: BenchmarkArm[]): Record { + return { + product: "kernel-mcp-server", + execution_mode: "harbor", + benchmark: "clawbench", + harborVersion: "0.21.0", + environment: "ci", + gitSha: process.env.BENCHMARK_HEAD_SHA ?? process.env.GITHUB_SHA, + githubRunId: process.env.GITHUB_RUN_ID, + githubRunAttempt: process.env.GITHUB_RUN_ATTEMPT, + githubEvent: process.env.GITHUB_EVENT_NAME, + pullRequest: process.env.BENCHMARK_PR_NUMBER || undefined, + concurrency: process.env.HARBOR_N_CONCURRENT, + benchByArm: Object.fromEntries( + arms.map((arm) => [arm.name, summarizeArm(arm)]), + ), + sources: Object.fromEntries( + arms.map((arm) => [ + arm.name, + { + jobId: arm.jobId, + jobName: arm.jobName, + startedAt: arm.startedAt, + finishedAt: arm.finishedAt, + stats: { + nTotalTrials: arm.nTotalTrials, + nCompletedTrials: arm.nCompletedTrials, + nErroredTrials: arm.nErroredTrials, + nCancelledTrials: arm.nCancelledTrials, + nRetries: arm.nRetries, + }, + kernelMcpShas: [ + ...new Set(arm.trials.flatMap((trial) => trial.kernelMcpSha ?? [])), + ], + clawbenchShas: [ + ...new Set(arm.trials.flatMap((trial) => trial.clawbenchSha ?? [])), + ], + }, + ]), + ), + }; +} + +class BraintrustApi { + private readonly baseUrl = + process.env.BRAINTRUST_API_URL ?? "https://api.braintrust.dev"; + + constructor(private readonly apiKey: string) {} + + async request(path: string, method = "GET", body?: unknown): Promise { + const response = await fetch(`${this.baseUrl}${path}`, { + method, + headers: { + Authorization: `Bearer ${this.apiKey}`, + "Content-Type": "application/json", + }, + body: body === undefined ? undefined : JSON.stringify(body), + }); + if (!response.ok) { + throw new Error( + `Braintrust ${method} ${path} returned ${response.status}: ${redactString(await response.text(), 400)}`, + ); + } + return (await response.json()) as T; + } +} + +async function insertEvents( + api: BraintrustApi, + experimentId: string, + events: BraintrustEvent[], +): Promise { + const batchSize = 100; + for (let offset = 0; offset < events.length; offset += batchSize) { + await api.request(`/v1/experiment/${experimentId}/insert`, "POST", { + events: events.slice(offset, offset + batchSize), + }); + } +} + +export async function publishBenchmark( + arms: BenchmarkArm[], + projectName: string, + experimentName: string, + apiKey: string, +): Promise> { + const api = new BraintrustApi(apiKey); + const project = await api.request("/v1/project", "POST", { + name: projectName, + }); + const metadata = experimentMetadata(arms); + const experiment = await api.request( + "/v1/experiment", + "POST", + { + project_id: project.id, + name: experimentName, + public: false, + metadata, + }, + ); + await api.request(`/v1/experiment/${experiment.id}`, "PATCH", { metadata }); + const events = buildExperimentEvents(arms, experimentName); + await insertEvents(api, experiment.id, events); + const organization = await api.request<{ name: string }>( + `/v1/organization/${project.org_id}`, + ); + const url = `https://www.braintrust.dev/app/${encodeURIComponent(organization.name)}/p/${encodeURIComponent(project.name)}/experiments/${encodeURIComponent(experiment.name)}`; + return { + project: project.name, + experiment: experiment.name, + experimentId: experiment.id, + url, + rows: arms.reduce((total, arm) => total + arm.trials.length, 0), + spans: events.length, + }; +} + +async function main(): Promise { + const options = parseArgs(process.argv.slice(2)); + const project = options.project ?? process.env.BRAINTRUST_PROJECT; + const experimentName = options.experiment; + const apiKey = process.env.BRAINTRUST_API_KEY; + if (!project) throw new Error("BRAINTRUST_PROJECT or --project is required"); + if (!experimentName) throw new Error("--experiment is required"); + if (!apiKey) throw new Error("BRAINTRUST_API_KEY is required"); + + const arms = options.arms.map(parseArmSpec).map(readBenchmarkArm); + const publication = await publishBenchmark( + arms, + project, + experimentName, + apiKey, + ); + const serialized = `${JSON.stringify(publication, undefined, 2)}\n`; + if (options.output) { + mkdirSync(dirname(options.output), { recursive: true }); + writeFileSync(options.output, serialized); + } + process.stdout.write(serialized); +} + +if (import.meta.main) { + main().catch((error) => { + console.error(error instanceof Error ? error.message : error); + process.exitCode = 1; + }); +} diff --git a/benchmarks/harbor/redact.ts b/benchmarks/harbor/redact.ts new file mode 100644 index 0000000..851c792 --- /dev/null +++ b/benchmarks/harbor/redact.ts @@ -0,0 +1,59 @@ +const SECRET_NAME = /(API_KEY|TOKEN|SECRET|PASSWORD|PRIVATE_KEY|CREDENTIAL)/i; +const SENSITIVE_FIELD = + /(API_KEY|TOKEN|JWT|SECRET|PASSWORD|PRIVATE_KEY|CREDENTIAL|^COOKIE$|^SET-COOKIE$)/i; +const REDACTED = "[REDACTED]"; + +function secretValues(): string[] { + return Object.entries(process.env) + .filter( + ([name, value]) => SECRET_NAME.test(name) && value && value.length >= 6, + ) + .map(([, value]) => value as string) + .sort((left, right) => right.length - left.length); +} + +export function redactString(value: string, maxLength = 20_000): string { + let redacted = value; + for (const secret of secretValues()) { + redacted = redacted.split(secret).join(REDACTED); + } + + redacted = redacted + .replace(/\bBearer\s+[A-Za-z0-9._~+/=-]+/gi, `Bearer ${REDACTED}`) + .replace(/\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b/g, REDACTED) + .replace(/\b(?:sk|pk|bt|kapi|whsec)[-_][A-Za-z0-9_-]{12,}\b/gi, REDACTED) + .replace( + /(["']?(?:api[_-]?key|access[_-]?token|credential|jwt|password|secret|token)["']?\s*[:=]\s*["']?)[^"'\s,}&]+/gi, + `$1${REDACTED}`, + ) + .replace( + /([?&](?:api[_-]?key|access[_-]?token|auth|code|credential|jwt|password|secret|session[_-]?token|token)=)[^&#\s]+/gi, + `$1${REDACTED}`, + ) + .replace(/(\b(?:cookie|set-cookie)\s*:\s*)[^\r\n]+/gi, `$1${REDACTED}`) + .replace(/(\/browser\/live\/)[^/?#\s]+/gi, `$1${REDACTED}`) + .replace(/(wss?:\/\/)[^/@\s]+@/gi, `$1${REDACTED}@`) + .replace(/\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi, "[REDACTED_EMAIL]"); + + return redacted.length > maxLength + ? `${redacted.slice(0, maxLength)}…` + : redacted; +} + +export function redactValue(value: unknown, maxStringLength = 20_000): unknown { + if (typeof value === "string") return redactString(value, maxStringLength); + if (Array.isArray(value)) { + return value.map((entry) => redactValue(entry, maxStringLength)); + } + if (value !== null && typeof value === "object") { + return Object.fromEntries( + Object.entries(value as Record).map(([key, entry]) => [ + key, + SENSITIVE_FIELD.test(key) + ? REDACTED + : redactValue(entry, maxStringLength), + ]), + ); + } + return value; +} diff --git a/benchmarks/harbor/report.ts b/benchmarks/harbor/report.ts new file mode 100644 index 0000000..55eb752 --- /dev/null +++ b/benchmarks/harbor/report.ts @@ -0,0 +1,187 @@ +#!/usr/bin/env bun +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { dirname } from "node:path"; +import { + type ArmSummary, + parseArmSpec, + readBenchmarkArm, + summarizeArm, +} from "./results"; + +interface Options { + arms: string[]; + statuses: Record; + json?: string; + markdown?: string; + publication?: string; + title: string; +} + +function parseArgs(args: string[]): Options { + const options: Options = { + arms: [], + statuses: {}, + title: "ClawBench benchmark", + }; + for (let index = 0; index < args.length; index += 1) { + const flag = args[index]; + const value = args[index + 1]; + if (!value || !flag.startsWith("--")) + throw new Error(`Missing value for ${flag}`); + index += 1; + switch (flag) { + case "--arm": + options.arms.push(value); + break; + case "--status": { + const separator = value.indexOf("="); + const status = Number(value.slice(separator + 1)); + if ( + separator <= 0 || + separator === value.length - 1 || + !Number.isInteger(status) + ) { + throw new Error( + `Invalid --status ${JSON.stringify(value)}; expected name=exit-code`, + ); + } + options.statuses[value.slice(0, separator)] = status; + break; + } + case "--json": + options.json = value; + break; + case "--markdown": + options.markdown = value; + break; + case "--publication": + options.publication = value; + break; + case "--title": + options.title = value; + break; + default: + throw new Error(`Unknown argument ${flag}`); + } + } + if (options.arms.length === 0) + throw new Error("At least one --arm name=/job/path is required"); + return options; +} + +function ratio(value: number | undefined, denominator: number): string { + return value === undefined ? "—" : `${value}/${denominator}`; +} + +function duration(value: number | undefined): string { + return value === undefined ? "—" : `${Math.round(value / 1000)}s`; +} + +function cost(value: number | undefined): string { + return value === undefined ? "—" : `$${value.toFixed(4)}`; +} + +export function renderMarkdown( + title: string, + summaries: ArmSummary[], + publication?: Record, + statuses: Record = {}, +): string { + const lines = ["", `## ${title}`]; + const failed = Object.entries(statuses).filter(([, status]) => status !== 0); + if (failed.length > 0) { + lines.push( + "", + `> [!WARNING]\n> Incomplete benchmark: ${failed.map(([arm, status]) => `${arm} exited ${status}`).join(", ")}. Scores below include only completed Harbor results and are not a complete comparison.`, + ); + } + lines.push( + "", + "| Arm | Configuration | Lenient | Strict | Intercepted | Infra | Ungraded | Kernel MCP valid | Median calls | Median duration | Cost |", + "|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|", + ); + for (const summary of summaries) { + lines.push( + `| ${summary.arm} | ${summary.configuration ?? "—"} | ${ratio(summary.lenient, summary.trials)} | ${ratio(summary.strict, summary.trials)} | ${ratio(summary.intercepted, summary.trials)} | ${summary.infraErrors} | ${summary.ungraded} | ${ratio(summary.kernelMcpValid, summary.kernelMcpChecked)} | ${summary.medianCalls ?? "—"} | ${duration(summary.medianDurationMs)} | ${cost(summary.totalCostUsd)} |`, + ); + } + + const candidate = summaries.find((summary) => summary.arm === "candidate"); + const baseline = summaries.find((summary) => summary.arm === "baseline"); + if (candidate && baseline && failed.length === 0) { + const signed = (value: number) => { + const rounded = Number(value.toFixed(3)); + return `${rounded >= 0 ? "+" : ""}${rounded}`; + }; + const deltas = [ + `**${signed(candidate.lenient - baseline.lenient)} lenient**`, + candidate.strict !== undefined && baseline.strict !== undefined + ? `**${signed(candidate.strict - baseline.strict)} strict**` + : undefined, + `**${signed(candidate.intercepted - baseline.intercepted)} intercepted**`, + ].filter(Boolean); + lines.push("", `Candidate minus baseline: ${deltas.join(", ")}.`); + } + if (typeof publication?.url === "string") { + lines.push("", `[Open the Braintrust experiment](${publication.url})`); + } + if ( + process.env.GITHUB_SERVER_URL && + process.env.GITHUB_REPOSITORY && + process.env.GITHUB_RUN_ID + ) { + lines.push( + "", + `[Open the GitHub Actions run](${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID})`, + ); + } + lines.push( + "", + "Lenient reward is the primary ClawBench score. Infrastructure failures remain in the intended-task denominator.", + "", + ); + return lines.join("\n"); +} + +function write(path: string | undefined, content: string): void { + if (!path) return; + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, content); +} + +function main(): void { + const options = parseArgs(process.argv.slice(2)); + const arms = options.arms.map(parseArmSpec).map(readBenchmarkArm); + const summaries = arms.map(summarizeArm); + const publication = options.publication + ? (JSON.parse(readFileSync(options.publication, "utf8")) as Record< + string, + unknown + >) + : undefined; + const result = { + benchmark: "clawbench", + generatedAt: new Date().toISOString(), + arms: summaries, + statuses: options.statuses, + publication, + }; + const rendered = renderMarkdown( + options.title, + summaries, + publication, + options.statuses, + ); + write(options.json, `${JSON.stringify(result, undefined, 2)}\n`); + write(options.markdown, rendered); + process.stdout.write(rendered); +} + +if (import.meta.main) { + try { + main(); + } catch (error) { + console.error(error instanceof Error ? error.message : error); + process.exitCode = 1; + } +} diff --git a/benchmarks/harbor/results.test.ts b/benchmarks/harbor/results.test.ts new file mode 100644 index 0000000..dbe80fe --- /dev/null +++ b/benchmarks/harbor/results.test.ts @@ -0,0 +1,408 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { + mkdtempSync, + mkdirSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { buildExperimentEvents, publishBenchmark } from "./publish-braintrust"; +import { renderMarkdown } from "./report"; +import { readBenchmarkArm, selectPrimaryReward, summarizeArm } from "./results"; +import { redactString, redactValue } from "./redact"; + +const temporaryDirectories: string[] = []; + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) { + rmSync(directory, { recursive: true, force: true }); + } +}); + +function writeJson(path: string, value: unknown): void { + mkdirSync(join(path, ".."), { recursive: true }); + writeFileSync(path, JSON.stringify(value)); +} + +function fixture(): string { + const root = mkdtempSync(join(tmpdir(), "harbor-results-")); + temporaryDirectories.push(root); + writeJson(join(root, "config.json"), { job_name: "test-job" }); + writeJson(join(root, "result.json"), { + id: "job-id", + n_total_trials: 2, + stats: { + n_completed_trials: 1, + n_errored_trials: 1, + n_cancelled_trials: 0, + n_retries: 0, + }, + }); + + const success = join(root, "task-one__abc"); + writeJson(join(success, "result.json"), { + id: "trial-one", + task_name: "clawbench/v2-task-one", + trial_name: "task-one__abc", + source: "clawbench-v2", + config: { + agent: { + name: "codex", + model_name: "gpt-5.6-luna", + kwargs: { version: "0.120.0" }, + }, + }, + verifier_result: { + rewards: { + reward: 1, + reward_lenient: 1, + reward_strict: 0, + intercepted: 1, + kernel_mcp_valid: 1, + }, + }, + started_at: "2026-01-01T00:00:00Z", + finished_at: "2026-01-01T00:01:00Z", + step_results: [ + { + agent_result: { + n_input_tokens: 100, + n_cache_tokens: 80, + n_output_tokens: 20, + cost_usd: 0.01, + }, + }, + ], + }); + writeJson(join(success, "steps/run/agent/trajectory.json"), { + steps: [ + { + step_id: 1, + source: "agent", + timestamp: "2026-01-01T00:00:01Z", + message: "working", + tool_calls: [ + { + tool_call_id: "call-1", + function_name: "execute_playwright_code", + arguments: { code: "return 'done'" }, + }, + ], + observation: { + results: [{ source_call_id: "call-1", content: "done" }], + }, + }, + ], + }); + writeJson(join(success, "steps/run/verifier/kernel-mcp/run-manifest.json"), { + kernel_mcp_server_sha: "server-sha", + clawbench_source_sha: "clawbench-sha", + }); + + const failed = join(root, "task-two__def"); + writeJson(join(failed, "result.json"), { + id: "trial-two", + task_name: "clawbench/v2-task-two", + trial_name: "task-two__def", + config: { agent: { name: "codex", model_name: "gpt-5.6-luna" } }, + exception_info: { type: "ExecProtocolError", message: "setup failed" }, + verifier_result: { rewards: { reward: 0, intercepted: 0 } }, + }); + return root; +} + +describe("Harbor result ingestion", () => { + test("keeps infrastructure errors out of task-quality scores", () => { + const arm = readBenchmarkArm({ name: "candidate", path: fixture() }); + expect(arm.trials).toHaveLength(2); + expect(arm.trials[0].scores).toEqual({ + accuracy: 1, + false_positive_rate: 0, + false_negative_rate: 0, + infra_error_rate: 0, + ungraded_rate: 0, + reward: 1, + reward_lenient: 1, + reward_strict: 0, + intercepted: 1, + kernel_mcp_valid: 1, + }); + expect(arm.trials[1].scores).toEqual({ + infra_error_rate: 1, + ungraded_rate: 1, + }); + }); + + test("summarizes against the intended task denominator", () => { + const summary = summarizeArm( + readBenchmarkArm({ name: "candidate", path: fixture() }), + ); + expect(summary).toMatchObject({ + trials: 2, + lenient: 1, + strict: 0, + intercepted: 1, + infraErrors: 1, + ungraded: 0, + kernelMcpValid: 1, + medianCalls: 1, + totalCostUsd: 0.01, + }); + }); + + test("builds deterministic root, llm, and tool spans", () => { + const arm = readBenchmarkArm({ name: "candidate", path: fixture() }); + const first = buildExperimentEvents([arm], "test-experiment"); + const second = buildExperimentEvents([arm], "test-experiment"); + expect(first).toEqual(second); + expect( + first.filter((event) => event.span_attributes.type === "eval"), + ).toHaveLength(2); + expect( + first.filter((event) => event.span_attributes.type === "llm"), + ).toHaveLength(1); + expect( + first.filter((event) => event.span_attributes.type === "tool"), + ).toHaveLength(1); + const root = first.find((event) => event.span_attributes.type === "eval"); + expect(root?.input).toEqual({ + source: "clawbench-v2", + taskName: "v2-task-one", + }); + expect(root?.span_parents).toEqual([]); + const infra = first.find( + (event) => + event.span_attributes.type === "eval" && + (event.output as { error?: string }).error, + ); + expect(infra?.scores).toEqual({ infra_error_rate: 1, ungraded_rate: 1 }); + expect(infra?.output).not.toHaveProperty("reward", 0); + const success = first.find( + (event) => + event.span_attributes.type === "eval" && + (event.output as { reward?: number }).reward === 1, + ); + expect(success?.output).toMatchObject({ + reward: 1, + rewardKey: "reward_lenient", + }); + }); + + test("re-publishes the same rows and spans by deterministic ID", async () => { + const arm = readBenchmarkArm({ name: "candidate", path: fixture() }); + const originalFetch = globalThis.fetch; + const inserts: string[][] = []; + const metadataUpdates: unknown[] = []; + globalThis.fetch = (async (request, init) => { + const url = String(request); + if (url.endsWith("/v1/project")) { + return Response.json({ + id: "project-id", + org_id: "org-id", + name: "project name", + }); + } + if (url.endsWith("/v1/experiment")) { + return Response.json({ + id: "experiment-id", + project_id: "project-id", + name: "experiment name", + }); + } + if ( + url.endsWith("/v1/experiment/experiment-id") && + init?.method === "PATCH" + ) { + metadataUpdates.push(JSON.parse(String(init.body))); + return Response.json({ id: "experiment-id" }); + } + if (url.includes("/insert")) { + const body = JSON.parse(String(init?.body)) as { + events: Array<{ id: string }>; + }; + inserts.push(body.events.map((event) => event.id)); + return Response.json({ row_ids: body.events.map((event) => event.id) }); + } + if (url.endsWith("/v1/organization/org-id")) { + return Response.json({ name: "Kernel" }); + } + return new Response("not found", { status: 404 }); + }) as typeof fetch; + + try { + const first = await publishBenchmark( + [arm], + "project name", + "experiment name", + "test-key", + ); + const second = await publishBenchmark( + [arm], + "project name", + "experiment name", + "test-key", + ); + expect(first).toEqual(second); + expect(inserts).toHaveLength(2); + expect(inserts[0]).toEqual(inserts[1]); + expect(metadataUpdates).toHaveLength(2); + expect(metadataUpdates[0]).toEqual(metadataUpdates[1]); + expect(first.url).toBe( + "https://www.braintrust.dev/app/Kernel/p/project%20name/experiments/experiment%20name", + ); + } finally { + globalThis.fetch = originalFetch; + } + }); + + test("uses the lenient reward per trial and reports incomplete arms", () => { + expect(selectPrimaryReward({ reward: 0, reward_lenient: 1 })).toEqual({ + key: "reward_lenient", + value: 1, + }); + const arm = readBenchmarkArm({ name: "candidate", path: fixture() }); + const second = arm.trials[1]; + second.error = undefined; + second.errorClass = undefined; + second.rewards = { reward: 0 }; + second.scores = { + accuracy: 0, + false_positive_rate: 0, + false_negative_rate: 1, + infra_error_rate: 0, + reward: 0, + ungraded_rate: 0, + }; + const summary = summarizeArm(arm); + expect(summary.scored).toBe(2); + expect(summary.lenient).toBe(1); + expect(summary.configuration).toContain("codex"); + expect( + renderMarkdown("test", [summary], undefined, { candidate: 124 }), + ).toContain("Incomplete benchmark: candidate exited 124"); + expect( + renderMarkdown("test", [ + { ...summary, arm: "candidate", lenient: 0.3 }, + { ...summary, arm: "baseline", lenient: 0.2 }, + ]), + ).toContain("+0.1 lenient"); + }); + + test("keeps full errors until redaction and clamps derived scores", () => { + const root = fixture(); + const successPath = join(root, "task-one__abc", "result.json"); + const result = JSON.parse(readFileSync(successPath, "utf8")) as { + verifier_result: { rewards: Record }; + exception_info?: string; + }; + result.verifier_result.rewards.reward_lenient = 2; + writeJson(successPath, result); + + const failedPath = join(root, "task-two__def", "result.json"); + const failed = JSON.parse(readFileSync(failedPath, "utf8")) as { + exception_info: unknown; + }; + failed.exception_info = `${"x".repeat(395)}secret-value-after-boundary`; + writeJson(failedPath, failed); + + const arm = readBenchmarkArm({ name: "candidate", path: root }); + expect(arm.trials[0].scores.accuracy).toBe(1); + expect(arm.trials[1].error?.length).toBeGreaterThan(400); + + process.env.TEST_SECRET = "secret-value-after-boundary"; + const events = buildExperimentEvents([arm], "redaction-boundary"); + expect(JSON.stringify(events)).not.toContain("secret-value-after-boundary"); + expect(JSON.stringify(events)).not.toContain(`${"x".repeat(395)}secre`); + delete process.env.TEST_SECRET; + }); + + test("assigns unique span IDs when ATIF step IDs are absent", () => { + const root = fixture(); + writeJson(join(root, "task-one__abc", "steps/run/agent/trajectory.json"), { + steps: [ + { source: "agent", message: "first" }, + { source: "agent", message: "second" }, + ], + }); + const events = buildExperimentEvents( + [readBenchmarkArm({ name: "candidate", path: root })], + "missing-step-ids", + ).filter((event) => event.span_attributes.type === "llm"); + expect(new Set(events.map((event) => event.id)).size).toBe(2); + }); +}); + +describe("Braintrust redaction", () => { + test("redacts configured secrets and credential-shaped strings", () => { + process.env.TEST_API_KEY = "super-secret-value"; + expect( + redactString( + 'Bearer super-secret-value sk-proj-abcdefghijklmnop?access_token=visible&token=plain&jwt=opaque "password":"generated-password" Cookie: session=visible\nhttps://example.com/browser/live/replay-slug user@example.com', + ), + ).toBe( + 'Bearer [REDACTED] [REDACTED]?access_token=[REDACTED]&token=[REDACTED]&jwt=[REDACTED] "password":"[REDACTED]" Cookie: [REDACTED]\nhttps://example.com/browser/live/[REDACTED] [REDACTED_EMAIL]', + ); + expect( + redactValue({ + api_key: "visible", + Cookie: "session=visible", + nested: ["bt-abcdefghijklmnop"], + }), + ).toEqual({ + api_key: "[REDACTED]", + Cookie: "[REDACTED]", + nested: ["[REDACTED]"], + }); + delete process.env.TEST_API_KEY; + }); +}); + +describe("benchmark workflow hardening", () => { + test("uses merge-base comparisons, fixed configs, and arm statuses", () => { + const workflow = readFileSync( + join(process.cwd(), ".github/workflows/benchmark-clawbench.yml"), + "utf8", + ); + expect(workflow).toContain("github.rest.repos.compareCommits"); + expect(workflow).not.toContain("baseSha = pull.base.sha"); + expect(workflow).toContain('HARBOR_VERSION: "0.21.0"'); + expect(workflow).toContain('CODEX_BENCHMARK_VERSION: "0.120.0"'); + expect(workflow).toContain( + 'statuses=(--status "candidate=${CANDIDATE_STATUS:-1}")', + ); + }); + + test("excludes private keys and forwards only the selected provider", () => { + const dockerignore = readFileSync( + join(process.cwd(), ".dockerignore"), + "utf8", + ); + const runner = readFileSync( + join(process.cwd(), "benchmarks/harbor/clawbench/run.sh"), + "utf8", + ); + const verifier = readFileSync( + join(process.cwd(), "benchmarks/harbor/clawbench/verify-task.py"), + "utf8", + ); + expect(dockerignore.split("\n")).toContain("*.pem"); + expect(verifier).toContain('"mcp__kernel__execute_playwright_code"'); + expect(verifier).toContain('"kernel__execute_playwright_code"'); + expect(verifier).toContain('"execute_playwright_code"'); + const commonStart = runner.indexOf("printf 'KERNEL_API_KEY=%s\\n'"); + const providerCaseStart = runner.indexOf('case "$agent" in', commonStart); + const providerCase = runner.slice( + providerCaseStart, + runner.indexOf('chmod 0600 "$runtime_env"'), + ); + expect(providerCase).toContain("ANTHROPIC_API_KEY"); + expect(providerCase).toContain("OPENAI_API_KEY"); + const commonEnvironment = runner.slice(commonStart, providerCaseStart); + expect(commonEnvironment).toContain("CLAWBENCH_JUDGE_API_KEY"); + expect(commonEnvironment).not.toContain("OPENAI_API_KEY"); + expect(commonEnvironment).not.toContain("ANTHROPIC_API_KEY"); + expect(runner).not.toContain("<; + +export interface ArmInput { + name: string; + path: string; +} + +export interface BenchmarkMetrics { + inputTokens?: number; + cacheTokens?: number; + outputTokens?: number; + costUsd?: number; + durationMs?: number; + toolCalls?: number; + start?: number; + end?: number; +} + +export interface BenchmarkTrial { + arm: string; + id: string; + taskName: string; + trialName: string; + source: string; + agent: string; + agentVersion?: string; + agentConfigHash: string; + model?: string; + rewards: Record; + scores: Record; + error?: string; + errorClass?: "infra"; + metrics: BenchmarkMetrics; + kernelMcpSha?: string; + clawbenchSha?: string; + trajectoryPath?: string; + startedAt?: string; + finishedAt?: string; +} + +export interface BenchmarkArm { + name: string; + path: string; + jobId: string; + jobName: string; + startedAt?: string; + finishedAt?: string; + nTotalTrials: number; + nCompletedTrials: number; + nErroredTrials: number; + nCancelledTrials: number; + nRetries: number; + trials: BenchmarkTrial[]; +} + +export interface ArmSummary { + arm: string; + trials: number; + scored: number; + intercepted: number; + lenient: number; + strict?: number; + strictScored: number; + infraErrors: number; + ungraded: number; + kernelMcpValid?: number; + kernelMcpChecked: number; + medianCalls?: number; + medianDurationMs?: number; + totalCostUsd?: number; + configuration?: string; +} + +function object(value: unknown): JsonObject { + return value !== null && typeof value === "object" && !Array.isArray(value) + ? (value as JsonObject) + : {}; +} + +function array(value: unknown): unknown[] { + return Array.isArray(value) ? value : []; +} + +function string(value: unknown): string | undefined { + return typeof value === "string" && value.length > 0 ? value : undefined; +} + +function number(value: unknown): number | undefined { + return typeof value === "number" && Number.isFinite(value) + ? value + : undefined; +} + +function readJson(path: string): JsonObject { + return object(JSON.parse(readFileSync(path, "utf8"))); +} + +function readJsonIfPresent(path: string): JsonObject { + return existsSync(path) ? readJson(path) : {}; +} + +function errorText(value: unknown): string | undefined { + if (value === null || value === undefined) return undefined; + const text = + typeof value === "string" ? value : JSON.stringify(value, undefined, 2); + return text.replace(/\s+/g, " ").trim() || undefined; +} + +function numericRecord(value: unknown): Record { + return Object.fromEntries( + Object.entries(object(value)).flatMap(([key, raw]) => { + const parsed = number(raw); + return parsed === undefined ? [] : [[key, parsed]]; + }), + ); +} + +export function selectPrimaryReward( + rewards: Record, +): { key: "reward_lenient" | "reward"; value: number } | undefined { + if (rewards.reward_lenient !== undefined) { + return { key: "reward_lenient", value: rewards.reward_lenient }; + } + if (rewards.reward !== undefined) { + return { key: "reward", value: rewards.reward }; + } + return undefined; +} + +function clampScore(value: number): number { + return Math.min(1, Math.max(0, value)); +} + +function isoSeconds(value: unknown): number | undefined { + const timestamp = string(value); + if (!timestamp) return undefined; + const millis = Date.parse(timestamp); + return Number.isFinite(millis) ? millis / 1000 : undefined; +} + +function durationMs( + startedAt?: string, + finishedAt?: string, +): number | undefined { + if (!startedAt || !finishedAt) return undefined; + const duration = Date.parse(finishedAt) - Date.parse(startedAt); + return Number.isFinite(duration) && duration >= 0 ? duration : undefined; +} + +function sumStepMetric( + stepResults: unknown[], + key: string, +): number | undefined { + const values = stepResults + .map((step) => number(object(object(step).agent_result)[key])) + .filter((value): value is number => value !== undefined); + return values.length > 0 + ? values.reduce((total, value) => total + value, 0) + : undefined; +} + +function trajectoryMetrics(path: string): Pick { + if (!existsSync(path)) return {}; + const trajectory = readJson(path); + const toolCalls = array(trajectory.steps) + .map((step) => array(object(step).tool_calls).length) + .reduce((total, count) => total + count, 0); + return { toolCalls }; +} + +function trialRewards( + result: JsonObject, + trialDir: string, +): Record { + const direct = numericRecord(object(result.verifier_result).rewards); + if (Object.keys(direct).length > 0) return direct; + + const steps = array(result.step_results); + const lastStep = object(steps.at(-1)); + const fromStep = numericRecord(object(lastStep.verifier_result).rewards); + if (Object.keys(fromStep).length > 0) return fromStep; + + return numericRecord( + readJsonIfPresent(join(trialDir, "steps/run/verifier/reward.json")), + ); +} + +function trialScores( + rewards: Record, + hasInfraError: boolean, +): Record { + if (hasInfraError) { + return { infra_error_rate: 1, ungraded_rate: 1 }; + } + + const primary = selectPrimaryReward(rewards); + if (!primary) return { infra_error_rate: 0, ungraded_rate: 1 }; + + const reward = clampScore(primary.value); + const scores: Record = { + accuracy: reward, + false_positive_rate: 0, + false_negative_rate: 1 - reward, + infra_error_rate: 0, + ungraded_rate: 0, + }; + for (const [key, value] of Object.entries(rewards)) { + if (value >= 0 && value <= 1) scores[key] = value; + } + return scores; +} + +function parseTrial(arm: string, trialDir: string): BenchmarkTrial { + const result = readJson(join(trialDir, "result.json")); + const config = object(result.config); + const agentConfig = object(config.agent); + const agentInfo = object(result.agent_info); + const modelInfo = object(agentInfo.model_info); + const steps = array(result.step_results); + const exception = result.exception_info; + const exceptionFile = join(trialDir, "exception.txt"); + const error = errorText( + exception ?? + (existsSync(exceptionFile) + ? readFileSync(exceptionFile, "utf8") + : undefined), + ); + const rewards = trialRewards(result, trialDir); + const startedAt = string(result.started_at); + const finishedAt = string(result.finished_at); + const trajectoryPath = join(trialDir, "steps/run/agent/trajectory.json"); + const runManifest = readJsonIfPresent( + join(trialDir, "steps/run/verifier/kernel-mcp/run-manifest.json"), + ); + + return { + arm, + id: string(result.id) ?? basename(trialDir), + taskName: + string(result.task_name)?.replace(/^clawbench\//, "") ?? + string(object(config.task).name) ?? + basename(trialDir).split("__", 1)[0], + trialName: string(result.trial_name) ?? basename(trialDir), + source: + string(result.source) ?? + string(object(config.task).source) ?? + "clawbench", + agent: string(agentInfo.name) ?? string(agentConfig.name) ?? "unknown", + agentVersion: + string(agentInfo.version) ?? string(object(agentConfig.kwargs).version), + agentConfigHash: createHash("sha256") + .update(JSON.stringify(agentConfig)) + .digest("hex") + .slice(0, 8), + model: string(modelInfo.name) ?? string(agentConfig.model_name), + rewards, + scores: trialScores(rewards, error !== undefined), + error, + errorClass: error === undefined ? undefined : "infra", + metrics: { + inputTokens: sumStepMetric(steps, "n_input_tokens"), + cacheTokens: sumStepMetric(steps, "n_cache_tokens"), + outputTokens: sumStepMetric(steps, "n_output_tokens"), + costUsd: sumStepMetric(steps, "cost_usd"), + durationMs: durationMs(startedAt, finishedAt), + start: isoSeconds(startedAt), + end: isoSeconds(finishedAt), + ...trajectoryMetrics(trajectoryPath), + }, + kernelMcpSha: string(runManifest.kernel_mcp_server_sha), + clawbenchSha: string(runManifest.clawbench_source_sha), + trajectoryPath: existsSync(trajectoryPath) ? trajectoryPath : undefined, + startedAt, + finishedAt, + }; +} + +export function parseArmSpec(spec: string): ArmInput { + const separator = spec.indexOf("="); + if (separator <= 0 || separator === spec.length - 1) { + throw new Error( + `Invalid --arm ${JSON.stringify(spec)}; expected name=/job/path`, + ); + } + return { + name: spec.slice(0, separator), + path: resolve(spec.slice(separator + 1)), + }; +} + +export function readBenchmarkArm(input: ArmInput): BenchmarkArm { + const configPath = join(input.path, "config.json"); + const resultPath = join(input.path, "result.json"); + if (!existsSync(configPath) || !existsSync(resultPath)) { + throw new Error(`${input.path} is not a completed Harbor job directory`); + } + + const config = readJson(configPath); + const result = readJson(resultPath); + const stats = object(result.stats); + const trials = readdirSync(input.path, { withFileTypes: true }) + .filter( + (entry) => + entry.isDirectory() && + existsSync(join(input.path, entry.name, "result.json")), + ) + .map((entry) => parseTrial(input.name, join(input.path, entry.name))) + .sort((left, right) => left.taskName.localeCompare(right.taskName)); + + return { + name: input.name, + path: input.path, + jobId: + string(result.id) ?? + createHash("sha256").update(input.path).digest("hex").slice(0, 16), + jobName: string(config.job_name) ?? basename(input.path), + startedAt: string(result.started_at), + finishedAt: string(result.finished_at), + nTotalTrials: number(result.n_total_trials) ?? trials.length, + nCompletedTrials: number(stats.n_completed_trials) ?? trials.length, + nErroredTrials: + number(stats.n_errored_trials) ?? + trials.filter((trial) => trial.error).length, + nCancelledTrials: number(stats.n_cancelled_trials) ?? 0, + nRetries: number(stats.n_retries) ?? 0, + trials, + }; +} + +function median(values: number[]): number | undefined { + if (values.length === 0) return undefined; + const sorted = [...values].sort((left, right) => left - right); + const middle = Math.floor(sorted.length / 2); + return sorted.length % 2 === 0 + ? (sorted[middle - 1] + sorted[middle]) / 2 + : sorted[middle]; +} + +export function summarizeArm(arm: BenchmarkArm): ArmSummary { + const numeric = (key: string) => + arm.trials.filter( + (trial) => + trial.errorClass !== "infra" && trial.rewards[key] !== undefined, + ); + const primary = arm.trials.flatMap((trial) => { + if (trial.errorClass === "infra") return []; + const reward = selectPrimaryReward(trial.rewards); + return reward ? [{ trial, reward }] : []; + }); + const strict = numeric("reward_strict"); + const validity = numeric("kernel_mcp_valid"); + const costs = arm.trials.flatMap((trial) => + trial.metrics.costUsd === undefined ? [] : [trial.metrics.costUsd], + ); + const configurations = [ + ...new Set( + arm.trials.map( + (trial) => + `${trial.agent}${trial.agentVersion ? `@${trial.agentVersion}` : ""}${trial.model ? ` · ${trial.model}` : ""} · config ${trial.agentConfigHash}`, + ), + ), + ]; + + return { + arm: arm.name, + trials: arm.nTotalTrials, + scored: primary.length, + intercepted: numeric("intercepted").reduce( + (total, trial) => total + trial.rewards.intercepted, + 0, + ), + lenient: primary.reduce((total, entry) => total + entry.reward.value, 0), + strict: + strict.length === 0 + ? undefined + : strict.reduce( + (total, trial) => total + trial.rewards.reward_strict, + 0, + ), + strictScored: strict.length, + infraErrors: arm.trials.filter((trial) => trial.errorClass === "infra") + .length, + ungraded: arm.trials.filter( + (trial) => + trial.errorClass !== "infra" && trial.scores.ungraded_rate === 1, + ).length, + kernelMcpValid: + validity.length === 0 + ? undefined + : validity.reduce( + (total, trial) => total + trial.rewards.kernel_mcp_valid, + 0, + ), + kernelMcpChecked: validity.length, + medianCalls: median( + arm.trials.flatMap((trial) => + trial.metrics.toolCalls === undefined ? [] : [trial.metrics.toolCalls], + ), + ), + medianDurationMs: median( + arm.trials.flatMap((trial) => + trial.metrics.durationMs === undefined + ? [] + : [trial.metrics.durationMs], + ), + ), + totalCostUsd: + costs.length === 0 + ? undefined + : costs.reduce((total, cost) => total + cost, 0), + configuration: + configurations.length === 0 + ? undefined + : configurations.length === 1 + ? configurations[0] + : `mixed: ${configurations.join(", ")}`, + }; +} diff --git a/package.json b/package.json index 62b06e5..73061f5 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,8 @@ "check:managed-auth-app": "bun scripts/build-managed-auth-app.mjs --check", "build": "bun run check:managed-auth-app && next build", "start": "next start -p 3002", + "benchmark:report": "bun benchmarks/harbor/report.ts", + "benchmark:publish": "bun benchmarks/harbor/publish-braintrust.ts", "lint": "next lint", "test": "bun test", "record:oauth-redis-contract": "bun scripts/record-oauth-redis-contract.ts", diff --git a/src/lib/mcp/register.test.ts b/src/lib/mcp/register.test.ts index 52061de..a6b0c94 100644 --- a/src/lib/mcp/register.test.ts +++ b/src/lib/mcp/register.test.ts @@ -83,6 +83,37 @@ describe("MCP Apps additive registration", () => { }); }); +describe("MCP toolset allowlist", () => { + test("keeps connection context and only the selected browser controls", () => { + const previousEnabled = process.env.KERNEL_MCP_ENABLED_TOOLSETS; + const previousDisabled = process.env.KERNEL_MCP_DISABLED_TOOLSETS; + process.env.KERNEL_MCP_ENABLED_TOOLSETS = + "execute_playwright_code computer_action"; + delete process.env.KERNEL_MCP_DISABLED_TOOLSETS; + try { + const registration = captureRegistration(false); + expect(registration.legacyTools).toEqual([ + "get_connection_context", + "computer_action", + "execute_playwright_code", + ]); + expect(registration.appTools).toEqual([]); + expect(registration.resources).toEqual([]); + } finally { + if (previousEnabled === undefined) { + delete process.env.KERNEL_MCP_ENABLED_TOOLSETS; + } else { + process.env.KERNEL_MCP_ENABLED_TOOLSETS = previousEnabled; + } + if (previousDisabled === undefined) { + delete process.env.KERNEL_MCP_DISABLED_TOOLSETS; + } else { + process.env.KERNEL_MCP_DISABLED_TOOLSETS = previousDisabled; + } + } + }); +}); + describe("project selection registration", () => { const projectScopedTools = [ "manage_profiles", diff --git a/src/lib/mcp/register.ts b/src/lib/mcp/register.ts index 59db2cb..712be6c 100644 --- a/src/lib/mcp/register.ts +++ b/src/lib/mcp/register.ts @@ -90,6 +90,33 @@ function normalizeMcpToolset(value: string): McpToolset | undefined { return undefined; } +function enabledMcpToolsetsFromEnv() { + const raw = process.env.KERNEL_MCP_ENABLED_TOOLSETS; + if (!raw?.trim()) return undefined; + + const enabled = new Set(); + const unknown: string[] = []; + for (const value of raw.split(/[,\s]+/)) { + const token = value.trim().toLowerCase(); + if (!token || token === "none") continue; + if (token === "all") return new Set(mcpToolsets); + + const toolset = normalizeMcpToolset(token); + if (toolset) { + enabled.add(toolset); + } else { + unknown.push(value); + } + } + + if (unknown.length > 0) { + throw new Error( + `Unknown KERNEL_MCP_ENABLED_TOOLSETS value(s): ${unknown.join(", ")}. Supported toolsets: ${mcpToolsets.join(", ")}.`, + ); + } + return enabled; +} + function disabledMcpToolsetsFromEnv() { const raw = process.env.KERNEL_MCP_DISABLED_TOOLSETS; if (!raw?.trim()) return new Set(); @@ -126,10 +153,14 @@ function disabledMcpToolsetsFromEnv() { } function toolsetEnabled( + enabledToolsets: Set | undefined, disabledToolsets: Set, toolset: McpToolset, ) { - return !disabledToolsets.has(toolset); + return ( + (enabledToolsets === undefined || enabledToolsets.has(toolset)) && + !disabledToolsets.has(toolset) + ); } export function registerMcpCapabilities( @@ -139,6 +170,7 @@ export function registerMcpCapabilities( dependencies = defaultMcpDependencies, }: McpRegistrationOptions = {}, ) { + const enabledToolsets = enabledMcpToolsetsFromEnv(); const disabledToolsets = disabledMcpToolsetsFromEnv(); registerKernelPrompts(server); @@ -147,7 +179,7 @@ export function registerMcpCapabilities( registerConnectionContextTool(server); for (const [toolset, registerToolset] of mcpToolRegistrations) { - if (toolsetEnabled(disabledToolsets, toolset)) { + if (toolsetEnabled(enabledToolsets, disabledToolsets, toolset)) { registerToolset(server, dependencies); } } @@ -155,7 +187,10 @@ export function registerMcpCapabilities( // Managed Auth remains fully programmatic for every client. MCP Apps support // adds one interactive launcher (plus its app-only implementation tools and // resource) without replacing or narrowing manage_auth_connections. - if (mcpApps && toolsetEnabled(disabledToolsets, "auth_connections")) { + if ( + mcpApps && + toolsetEnabled(enabledToolsets, disabledToolsets, "auth_connections") + ) { registerAuthLoginApp(server); } }