| Agent | +Model | +mean (95% CI) | +pass^5 | +Avg Time | +Version | +
|---|---|---|---|---|---|
| {{ agg.agent_name }} | +{{ agg.model }} | +{{ agg.average | times: 100.0 | round: 1 }}%{% if agg.ci_low %} ({{ agg.ci_low | times: 100.0 | round: 1 }}-{{ agg.ci_high | times: 100.0 | round: 1 }}%){% endif %} | +{% if agg.pass_hat_5 %}{{ agg.pass_hat_5 | times: 100.0 | round: 1 }}%{% endif %} | +{{ agg.average_duration | round: 1 }}s | +{{ agg.benchmark_version }} | +
No results available yet. Check back soon!
+{% endif %} + +## BC MCP Experiment + +Comparing runs that enable the **Business Central Data Query MCP tools** (`bc-mcp`) against the +matching no-tooling **Default** baseline for the same model. + +{% if site.data.data-query.aggregate %} +{%- assign mcp_models = "" -%} +{%- for agg in site.data.data-query.aggregate -%} + {%- if agg.experiment and agg.experiment.mcp_servers.size > 0 -%} + {%- assign mcp_models = mcp_models | append: "|" | append: agg.model | append: "|" -%} + {%- endif -%} +{%- endfor -%} +| Model | +MCP Servers | +Skills | +mean (95% CI) | +pass^5 | +Avg Time | +Ver | +
|---|---|---|---|---|---|---|
| {{ agg.model }} | +{% if is_mcp %}{{ agg.experiment.mcp_servers | join: ", " }}{% else %}Default{% endif %} | +{% if is_mcp and agg.experiment.skills_enabled %}✓{% else %}—{% endif %} | +{{ agg.average | times: 100.0 | round: 1 }}%{% if agg.ci_low %} ({{ agg.ci_low | times: 100.0 | round: 1 }}-{{ agg.ci_high | times: 100.0 | round: 1 }}%){% endif %} | +{% if agg.pass_hat_5 %}{{ agg.pass_hat_5 | times: 100.0 | round: 1 }}%{% endif %} | +{{ agg.average_duration | round: 1 }}s | +{{ agg.benchmark_version }} | +
No results available yet. Check back soon!
+{% endif %} + +[← Back to Home](index.md) diff --git a/docs/index.md b/docs/index.md index a34539ef0..7a95dca7b 100644 --- a/docs/index.md +++ b/docs/index.md @@ -12,6 +12,7 @@ A benchmark for evaluating AI coding agents on real-world **Business Central (AL | [Bug Fixing](bug-fix.md) | Follows [SWE-Bench](https://www.swebench.com/) methodology to evaluate bug fixing in AL code | | [Test Generation](test-generation.md) | "Reverses" SWE-Bench: Generates reproduction tests (TDD) instead of fixes | | [Code Review](code-review.md) | Reviews AL pull requests; scored with Precision / Recall / F1 against gold findings | +| [Data Query](data-query.md) | Retrieves data from a live BC environment to answer natural-language questions; scored deterministically against a gold query's result set (baseline vs. BC MCP Data Query tools) | ## Diagnostics diff --git a/scripts/BCBenchUtils.psm1 b/scripts/BCBenchUtils.psm1 index 8965af9ef..e46518a55 100644 --- a/scripts/BCBenchUtils.psm1 +++ b/scripts/BCBenchUtils.psm1 @@ -490,7 +490,7 @@ function Get-BCBenchDatasetPath { param( [Parameter(Mandatory = $true)] # Category validation lives only here: every caller resolves the dataset path through this function, so there's no need to duplicate ValidateSet on each caller. - [ValidateSet("bug-fix", "test-generation", "code-review", "nl2al", "extensibility-request-advisor", "extensibility-request-implement", "extensibility-request-triage")] + [ValidateSet("bug-fix", "test-generation", "code-review", "nl2al", "data-query", "extensibility-request-advisor", "extensibility-request-implement", "extensibility-request-triage")] [string] $Category ) @@ -499,6 +499,7 @@ function Get-BCBenchDatasetPath { "test-generation" { $DatasetName = "bcbench.jsonl" } "code-review" { $DatasetName = "codereview.jsonl" } "nl2al" { $DatasetName = "nl2al.jsonl" } + "data-query" { $DatasetName = "dataquery.jsonl" } "extensibility-request-advisor" { $DatasetName = "extensibility_request_advisor.jsonl" } "extensibility-request-implement" { $DatasetName = "extensibility_request_implement.jsonl" } "extensibility-request-triage" { $DatasetName = "extensibility_request_triage.jsonl" } diff --git a/scripts/BCContainerManagement.psm1 b/scripts/BCContainerManagement.psm1 index ef8aaad51..ebb72db75 100644 --- a/scripts/BCContainerManagement.psm1 +++ b/scripts/BCContainerManagement.psm1 @@ -302,6 +302,8 @@ function New-BCContainerSync { shortcuts = 'None' memoryLimit = "16G" isolation = "hyperv" + # TEMPORARY (remove once BC 29 is GA): required to build a container from an insider (BC 29) artifact. + accept_insiderEula = $true } if ($AcceptEula) { @@ -349,4 +351,85 @@ function New-BCCompilerFolderSync { Write-Log "Compiler folder created at: $compilerFolder" -Level Success } -Export-ModuleMember -Function Test-Database, Set-AppVersion, Move-AppIntoDevScope, Initialize-ContainerForDevelopment, Test-ContainerExists, New-BCContainerSync, New-BCCompilerFolderSync +function Publish-MCPConfigApp { + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [string]$ContainerName, + + [Parameter(Mandatory = $true)] + [string]$Version, + + [Parameter(Mandatory = $true)] + [PSCredential]$Credential, + + [Parameter(Mandatory = $true)] + [string]$BuildRoot + ) + + Import-Module "$PSScriptRoot\AppUtils.psm1" -Force -DisableNameChecking + + [int]$major = ([System.Version]$Version).Major + [string]$sourceFolder = Join-Path $PSScriptRoot "al\mcp-config-setup" + # Build inside BuildRoot: Compile-AppInBcContainer only accepts a project folder that is shared with + # the container, and BuildRoot ($RepoPath) is the folder mounted into it. Cleaned up after publish so + # nothing leaks into the agent's workspace. + [string]$buildFolder = Join-Path $BuildRoot ".bcbench-mcp-config-app" + + if (Test-Path $buildFolder) { + Remove-Item -Path $buildFolder -Recurse -Force + } + Copy-Item -Path $sourceFolder -Destination $buildFolder -Recurse -Force + + # The source keeps a placeholder so the same app builds against any evaluated BC version; + # pin the app/platform dependency to the container's major version at publish time. + [string]$appJsonPath = Join-Path $buildFolder "app.json" + (Get-Content -Path $appJsonPath -Raw).Replace("__APP_VERSION__", "$major.0.0.0") | Set-Content -Path $appJsonPath -Encoding UTF8 + + try { + Write-Log "Publishing BC-Bench MCP config app to provision the MCP configuration..." -Level Info + Invoke-AppBuildAndPublish -containerName $ContainerName -appProjectFolder $buildFolder -credential $Credential -skipVerification -useDevEndpoint + Write-Log "BC MCP configuration 'BCBench' provisioned and activated." -Level Success + } + finally { + Remove-Item -Path $buildFolder -Recurse -Force -ErrorAction SilentlyContinue + } +} + +function Get-BCMCPConnectionInfo { + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [string]$ContainerName + ) + + # The evaluated agent runs on the host, not inside the container, and the runner does not update + # its hosts file -- so reach the BC web listener by the container's IP address. + [string]$ip = Get-BcContainerIpAddress -containerName $ContainerName + if (-not $ip) { + throw "Could not resolve IP address for container $ContainerName; BC MCP endpoint is unreachable from the host." + } + + # BcContainerHelper containers always serve the 'BC' instance on port 7048; the MCP endpoint hangs + # off the same base as the API endpoint the query harness already uses (base + '/mcp'). + [string]$baseUrl = "http://${ip}:7048/BC" + + $companies = @(Get-CompanyInBcContainer -containerName $ContainerName) + $evaluationCompany = $companies | Where-Object { $_.EvaluationCompany } | Select-Object -First 1 + if (-not $evaluationCompany) { + $evaluationCompany = $companies | Select-Object -First 1 + } + + # BcContainerHelper has exposed the company name as either CompanyName or Name across versions. + [string]$company = $evaluationCompany.CompanyName + if (-not $company) { + $company = $evaluationCompany.Name + } + + return [PSCustomObject]@{ + BaseUrl = $baseUrl + Company = $company + } +} + +Export-ModuleMember -Function Test-Database, Set-AppVersion, Move-AppIntoDevScope, Initialize-ContainerForDevelopment, Test-ContainerExists, New-BCContainerSync, New-BCCompilerFolderSync, Publish-MCPConfigApp, Get-BCMCPConnectionInfo diff --git a/scripts/Setup-ContainerAndRepository.ps1 b/scripts/Setup-ContainerAndRepository.ps1 index efeffd1f3..66efcffb3 100644 --- a/scripts/Setup-ContainerAndRepository.ps1 +++ b/scripts/Setup-ContainerAndRepository.ps1 @@ -91,8 +91,9 @@ if (-not $SkipContainer) { Write-Log "Creating container $ContainerName for version $Version..." -Level Info - # Get BC artifact URL - [string] $url = Get-BCArtifactUrl -version $Version -Country $Country + # TEMPORARY (remove once BC 29 is GA on the public feed): the Data Query tools only exist in + # BC 29, so until it is GA pull the sandbox artifact from the insider feed. + [string] $url = Get-BCArtifactUrl -version $Version -Country $Country -select Latest -storageAccount bcinsider -accept_insiderEula Write-Log "Retrieved artifact URL: $url" -Level Info # Create container synchronously with NAV folder shared @@ -102,6 +103,21 @@ if (-not $SkipContainer) { New-BCCompilerFolderSync -ContainerName $ContainerName -ArtifactUrl $url Initialize-ContainerForDevelopment -ContainerName $ContainerName -RepoVersion ([System.Version]$Version) + + # data-query benchmarks the agent WITH the BC MCP server as a feedback loop. Publish the install + # app that provisions and activates the 'BCBench' MCP configuration, then expose the endpoint and + # company to the agent step so it can point its MCP client at the container. + if ($Category -eq 'data-query') { + Publish-MCPConfigApp -ContainerName $ContainerName -Version $Version -Credential $credential -BuildRoot $RepoPath + + $mcpInfo = Get-BCMCPConnectionInfo -ContainerName $ContainerName + Write-Log "BC MCP base URL: $($mcpInfo.BaseUrl) (company '$($mcpInfo.Company)')" -Level Info + + if ($env:GITHUB_ENV) { + "BC_MCP_URL=$($mcpInfo.BaseUrl)" | Out-File -FilePath $env:GITHUB_ENV -Append + "BC_MCP_COMPANY=$($mcpInfo.Company)" | Out-File -FilePath $env:GITHUB_ENV -Append + } + } } else { Write-Log "Skipping BC container setup (SkipContainer flag set)" -Level Info diff --git a/scripts/al/mcp-config-setup/MCPConfigSetup.Codeunit.al b/scripts/al/mcp-config-setup/MCPConfigSetup.Codeunit.al new file mode 100644 index 000000000..66226140b --- /dev/null +++ b/scripts/al/mcp-config-setup/MCPConfigSetup.Codeunit.al @@ -0,0 +1,34 @@ +namespace BCBench.MCP; + +using System.MCP; + +// Installed at container-setup time (not part of the benchmarked workspace). Provisions and activates +// the MCP configuration the evaluated agent connects to over the BC MCP server; this app is the single +// place that decides which server capabilities the eval exposes. Idempotent: re-installs reuse the +// existing configuration by name. +codeunit 50150 "BCBench MCP Config Setup" +{ + Subtype = Install; + + trigger OnInstallAppPerCompany() + begin + EnsureConfiguration(); + end; + + local procedure EnsureConfiguration() + var + MCPConfig: Codeunit "MCP Config"; + ConfigId: Guid; + begin + ConfigId := MCPConfig.GetConfigurationIdByName(ConfigNameTok); + if IsNullGuid(ConfigId) then + ConfigId := MCPConfig.CreateConfiguration(ConfigNameTok, ConfigDescriptionTok); + + MCPConfig.EnableDataQueryTools(ConfigId, true); + MCPConfig.ActivateConfiguration(ConfigId, true); + end; + + var + ConfigNameTok: Label 'BCBench', Locked = true; + ConfigDescriptionTok: Label 'BC-Bench evaluation', Locked = true; +} diff --git a/scripts/al/mcp-config-setup/app.json b/scripts/al/mcp-config-setup/app.json new file mode 100644 index 000000000..f8ad27ed0 --- /dev/null +++ b/scripts/al/mcp-config-setup/app.json @@ -0,0 +1,19 @@ +{ + "id": "9f3d6b6e-4c2a-4d3f-9b7a-2f1e8c5a7d10", + "name": "BC-Bench MCP Config Setup", + "publisher": "BC-Bench", + "version": "1.0.0.0", + "brief": "Provisions the BC MCP configuration for BC-Bench evaluation.", + "description": "Installed at container-setup time to provision and activate the 'BCBench' Business Central MCP configuration the evaluated agent connects to.", + "application": "__APP_VERSION__", + "platform": "__APP_VERSION__", + "idRanges": [ + { + "from": 50150, + "to": 50159 + } + ], + "runtime": "13.0", + "target": "OnPrem", + "dependencies": [] +} diff --git a/src/bcbench/agent/claude/agent.py b/src/bcbench/agent/claude/agent.py index 1bc7bceb2..21d8b1396 100644 --- a/src/bcbench/agent/claude/agent.py +++ b/src/bcbench/agent/claude/agent.py @@ -1,13 +1,11 @@ -import json -import os import shutil import subprocess from pathlib import Path import yaml -from bcbench.agent.claude.metrics import parse_metrics -from bcbench.agent.shared import build_al_lsp_plugin, build_mcp_config, build_prompt, parse_tool_usage_from_hooks, resolve_config_plugins +from bcbench.agent.claude.metrics import parse_stream_output +from bcbench.agent.shared import agent_subprocess_env, build_al_lsp_plugin, build_mcp_config, build_prompt, parse_tool_usage_from_hooks, resolve_config_plugins, start_bc_mcp_gateway from bcbench.config import get_config from bcbench.dataset import BaseDatasetEntry from bcbench.exceptions import AgentError, AgentTimeoutError @@ -27,6 +25,8 @@ def run_claude_code( output_dir: Path, al_mcp: bool = False, al_lsp: bool = False, + bc_mcp: bool = False, + skills: bool = False, container_name: str = "bcbench", ) -> tuple[AgentMetrics | None, ExperimentConfiguration]: """Run Claude Code on a single dataset entry. @@ -44,10 +44,19 @@ def run_claude_code( logger.info(f"Running Claude Code on: {entry.instance_id}") prompt: str = build_prompt(entry, repo_path, claude_config, category, al_mcp=al_mcp) - mcp_config_json, mcp_server_names = build_mcp_config(claude_config, entry, repo_path, al_mcp=al_mcp, container_name=container_name) + bc_gateway = start_bc_mcp_gateway(bc_mcp) + mcp_config_json, mcp_server_names = build_mcp_config( + claude_config, + entry, + repo_path, + al_mcp=al_mcp, + bc_mcp=bc_mcp, + container_name=container_name, + bc_mcp_gateway_url=bc_gateway.base_url if bc_gateway else None, + ) lsp_plugin_dir: Path | None = build_al_lsp_plugin(entry, category, repo_path, AgentHarness.CLAUDE, al_lsp=al_lsp, container_name=container_name) instructions_enabled: bool = setup_instructions_from_config(claude_config, entry, repo_path, harness=AgentHarness.CLAUDE) - skills_enabled: bool = setup_agent_skills(claude_config, entry, repo_path, harness=AgentHarness.CLAUDE) + skills_enabled: bool = setup_agent_skills(claude_config, entry, repo_path, harness=AgentHarness.CLAUDE, skills_enabled_override=skills) custom_agent: str | None = setup_custom_agent(claude_config, entry, repo_path, harness=AgentHarness.CLAUDE) tool_log_path: Path = setup_hooks(repo_path, AgentHarness.CLAUDE, output_dir) plugins: list[tuple[PluginConfig, Path]] = resolve_config_plugins(claude_config, allow_copilot_manifest=False) @@ -67,7 +76,8 @@ def run_claude_code( try: cmd_args = [ claude_cmd, - "--output-format=json", + "--output-format=stream-json", # emit every event (incl. tool_use, session init) as JSONL + "--verbose", # required for stream-json in --print mode "--strict-mcp-config", # Only use MCP servers from --mcp-config, ignoring all other MCP configurations "--setting-sources=project,local", f"--model={model}", @@ -101,10 +111,17 @@ def run_claude_code( result = subprocess.run( cmd_args, cwd=str(repo_path), - env={ - **os.environ, - "CLAUDE_CODE_DISABLE_AUTO_MEMORY": "1", - }, + env=agent_subprocess_env( + { + "CLAUDE_CODE_DISABLE_AUTO_MEMORY": "1", + # BC MCP's first tools/list compiles the tool catalog and can take ~45s on a cold + # container, well past Claude's 30s default MCP startup timeout -> the server is + # marked "failed" and its tools never register. Raise both the connection and tool + # execution timeouts so the slow first response is tolerated. + "MCP_TIMEOUT": "180000", + "MCP_TOOL_TIMEOUT": "180000", + } + ), timeout=_config.timeout.agent_execution, check=True, capture_output=True, @@ -113,21 +130,14 @@ def run_claude_code( stdout: str = result.stdout.decode("utf-8", errors="replace") if result.stdout else "" logger.debug(f"Claude Code raw output: {stdout}") - metrics = None - for line in stdout.splitlines(): - striped_line: str = line.strip() - if striped_line: - try: - data = json.loads(striped_line) - if "result" in data: - logger.info(data["result"]) - metrics = parse_metrics(data) - except json.JSONDecodeError: - logger.warning(f"Skipping non-JSON line: {striped_line}") - - tool_usage: dict[str, int] | None = parse_tool_usage_from_hooks(tool_log_path) - if metrics and tool_usage: - metrics = metrics.model_copy(update={"tool_usage": tool_usage}) + metrics, final_response = parse_stream_output(stdout.splitlines()) + if final_response: + logger.info(final_response) + + # The stream's tool_use events capture sub-agent and MCP tool calls; fall back to the pre-tool-use + # hook only when the stream carried none. + if metrics and not metrics.tool_usage and (hook_tool_usage := parse_tool_usage_from_hooks(tool_log_path)): + metrics = metrics.model_copy(update={"tool_usage": hook_tool_usage}) except subprocess.TimeoutExpired: logger.exception(f"Claude Code timed out after {_config.timeout.agent_execution} seconds") metrics = AgentMetrics(execution_time=_config.timeout.agent_execution) @@ -140,3 +150,6 @@ def run_claude_code( raise else: return metrics, config + finally: + if bc_gateway is not None: + bc_gateway.stop() diff --git a/src/bcbench/agent/claude/metrics.py b/src/bcbench/agent/claude/metrics.py index f4ee6a3d7..eca66bc3b 100644 --- a/src/bcbench/agent/claude/metrics.py +++ b/src/bcbench/agent/claude/metrics.py @@ -1,3 +1,7 @@ +import json +from collections import Counter +from collections.abc import Sequence + from bcbench.logger import get_logger from bcbench.types import AgentMetrics @@ -41,3 +45,53 @@ def parse_metrics(data: dict) -> AgentMetrics | None: logger.warning("No metrics found in Claude Code output") return None + + +def parse_stream_output(output_lines: Sequence[str]) -> tuple[AgentMetrics | None, str | None]: + """Parse metrics + final response from `claude --output-format=stream-json --verbose` (JSONL) stdout. + + Event shapes (Claude Code): + assistant: ``message.content`` holds ``tool_use`` blocks whose ``name`` (e.g. + ``mcp__bcmcp__bc_data_query``) captures sub-agent and MCP tool calls the pre-tool-use hook + never sees. + result: terminal event carrying duration/turns/usage and the final ``result`` text. + + Returns: + The parsed metrics (with tool usage from the stream) and the agent's final response text. + """ + tool_usage: Counter[str] = Counter() + final_response: str | None = None + metrics: AgentMetrics | None = None + + for line_number, line in enumerate(output_lines, start=1): + if not line.strip(): + continue + + try: + event = json.loads(line) + except json.JSONDecodeError as error: + logger.warning(f"Skipping invalid JSON from Claude Code output at line {line_number}: {error}") + continue + + if not isinstance(event, dict): + continue + + match event.get("type"): + case "assistant": + message = event.get("message") + if isinstance(message, dict): + for block in message.get("content", []): + if isinstance(block, dict) and block.get("type") == "tool_use": + name = block.get("name") + if isinstance(name, str) and name: + tool_usage[name] += 1 + case "result": + metrics = parse_metrics(event) + result_text = event.get("result") + if isinstance(result_text, str) and result_text: + final_response = result_text + + if tool_usage: + metrics = (metrics or AgentMetrics()).model_copy(update={"tool_usage": dict(tool_usage)}) + + return metrics, final_response diff --git a/src/bcbench/agent/copilot/agent.py b/src/bcbench/agent/copilot/agent.py index e7c63be85..e5d2c8b74 100644 --- a/src/bcbench/agent/copilot/agent.py +++ b/src/bcbench/agent/copilot/agent.py @@ -1,6 +1,5 @@ """GitHub Copilot CLI Agent implementation.""" -import os import subprocess import sys from pathlib import Path @@ -8,7 +7,7 @@ import yaml from bcbench.agent.copilot.metrics import parse_output -from bcbench.agent.shared import build_al_lsp_plugin, build_mcp_config, build_prompt, parse_tool_usage_from_hooks, resolve_config_plugins +from bcbench.agent.shared import agent_subprocess_env, build_al_lsp_plugin, build_mcp_config, build_prompt, parse_tool_usage_from_hooks, resolve_config_plugins, start_bc_mcp_gateway from bcbench.config import get_config from bcbench.copilot_cli import find_copilot from bcbench.dataset import BaseDatasetEntry @@ -29,6 +28,8 @@ def run_copilot_agent( output_dir: Path, al_mcp: bool = False, al_lsp: bool = False, + bc_mcp: bool = False, + skills: bool = False, container_name: str = "bcbench", ) -> tuple[AgentMetrics | None, ExperimentConfiguration]: """Run GitHub Copilot CLI agent on a single dataset entry. @@ -48,10 +49,19 @@ def run_copilot_agent( logger.info(f"Running GitHub Copilot CLI on: {entry.instance_id}") prompt: str = build_prompt(entry, repo_path, copilot_config, category, al_mcp=al_mcp) - mcp_config_json, mcp_server_names = build_mcp_config(copilot_config, entry, repo_path, al_mcp=al_mcp, container_name=container_name) + bc_gateway = start_bc_mcp_gateway(bc_mcp) + mcp_config_json, mcp_server_names = build_mcp_config( + copilot_config, + entry, + repo_path, + al_mcp=al_mcp, + bc_mcp=bc_mcp, + container_name=container_name, + bc_mcp_gateway_url=bc_gateway.base_url if bc_gateway else None, + ) lsp_plugin_dir: Path | None = build_al_lsp_plugin(entry, category, repo_path, AgentHarness.COPILOT, al_lsp=al_lsp, container_name=container_name) instructions_enabled: bool = setup_instructions_from_config(copilot_config, entry, repo_path, harness=AgentHarness.COPILOT) - skills_enabled: bool = setup_agent_skills(copilot_config, entry, repo_path, harness=AgentHarness.COPILOT) + skills_enabled: bool = setup_agent_skills(copilot_config, entry, repo_path, harness=AgentHarness.COPILOT, skills_enabled_override=skills) custom_agent: str | None = setup_custom_agent(copilot_config, entry, repo_path, harness=AgentHarness.COPILOT) tool_log_path: Path = setup_hooks(repo_path, AgentHarness.COPILOT, output_dir) plugins: list[tuple[PluginConfig, Path]] = resolve_config_plugins(copilot_config, allow_copilot_manifest=True) @@ -99,11 +109,12 @@ def run_copilot_agent( result = subprocess.run( cmd_args, cwd=str(repo_path), - env={ - **os.environ, - "GITHUB_COPILOT_PROMPT_MODE_REPO_HOOKS": "true", - "GITHUB_COPILOT_PROMPT_MODE_WORKSPACE_MCP": "true", - }, + env=agent_subprocess_env( + { + "GITHUB_COPILOT_PROMPT_MODE_REPO_HOOKS": "true", + "GITHUB_COPILOT_PROMPT_MODE_WORKSPACE_MCP": "true", + } + ), capture_output=True, timeout=_config.timeout.agent_execution, check=True, @@ -119,9 +130,11 @@ def run_copilot_agent( if final_response: logger.info(final_response) - tool_usage: dict[str, int] | None = parse_tool_usage_from_hooks(tool_log_path) - if metrics and tool_usage: - metrics = metrics.model_copy(update={"tool_usage": tool_usage}) + # Tool usage now comes from the JSON event stream (tool.execution_start), which — unlike the + # pre-tool-use hook — also captures sub-agent and MCP tool calls. Fall back to the hook only if + # the stream carried none. + if metrics and not metrics.tool_usage and (hook_tool_usage := parse_tool_usage_from_hooks(tool_log_path)): + metrics = metrics.model_copy(update={"tool_usage": hook_tool_usage}) except subprocess.TimeoutExpired: logger.exception(f"Copilot CLI timed out after {_config.timeout.agent_execution} seconds") metrics = AgentMetrics(execution_time=_config.timeout.agent_execution) @@ -134,3 +147,6 @@ def run_copilot_agent( raise else: return metrics, config + finally: + if bc_gateway is not None: + bc_gateway.stop() diff --git a/src/bcbench/agent/copilot/metrics.py b/src/bcbench/agent/copilot/metrics.py index e268335c8..677cb0bb6 100644 --- a/src/bcbench/agent/copilot/metrics.py +++ b/src/bcbench/agent/copilot/metrics.py @@ -1,4 +1,5 @@ import json +from collections import Counter from collections.abc import Sequence from bcbench.logger import get_logger @@ -21,11 +22,25 @@ def _milliseconds_to_seconds(value: object) -> float | None: return None if milliseconds is None else milliseconds / 1000.0 +def _tool_label(data: dict) -> str | None: + """Tool name for a tool.execution_start event, sub-labelling LSP ops (lsp: