Add data-query category: AL query-generation benchmark

.github/workflows/claude-evaluation.yml

@@ -24,6 +24,7 @@ on:
           - "bug-fix"
           - "test-generation"
           - "code-review"
+          - "data-query"
           - "extensibility-request-advisor"
           - "extensibility-request-implement"
           - "extensibility-request-triage"
@@ -42,6 +43,16 @@ on:
         required: false
         default: false
         type: boolean
+      bc-mcp:
+        description: "Enable the Business Central MCP server"
+        required: false
+        default: false
+        type: boolean
+      skills:
+        description: "Enable agent skills"
+        required: false
+        default: false
+        type: boolean
       repeat:
         description: "Number of times to run sequentially (ignored for test runs)"
         required: false
@@ -152,7 +163,9 @@ jobs:
             --repo-path "${{ steps.setup-env.outputs.repo_path }}" `
             --output-dir "${{ env.EVALUATION_RESULTS_DIR }}" `
             ${{ inputs.al-mcp && '--al-mcp' || '' }} `
-            ${{ inputs.al-lsp && '--al-lsp' || '' }}
+            ${{ inputs.al-lsp && '--al-lsp' || '' }} `
+            ${{ inputs.bc-mcp && '--bc-mcp' || '' }} `
+            ${{ inputs.skills && '--skills' || '' }}
 
       - name: Upload evaluation results
         uses: actions/upload-artifact@v6
@@ -189,4 +202,4 @@ jobs:
       repeat: ${{ inputs.repeat }}
       existing-tag: ${{ needs.pin-commit.outputs.tag-name }}
       workflow-inputs: |
-        {"model": "${{ inputs.model }}", "category": "${{ inputs.category }}", "test-run": "${{ inputs.test-run }}", "al-mcp": "${{ inputs.al-mcp }}", "al-lsp": "${{ inputs.al-lsp }}", "git-ref": "${{ inputs.git-ref || github.ref_name }}"}
+        {"model": "${{ inputs.model }}", "category": "${{ inputs.category }}", "test-run": "${{ inputs.test-run }}", "al-mcp": "${{ inputs.al-mcp }}", "al-lsp": "${{ inputs.al-lsp }}", "bc-mcp": "${{ inputs.bc-mcp }}", "skills": "${{ inputs.skills }}", "git-ref": "${{ inputs.git-ref || github.ref_name }}"}

.github/workflows/copilot-evaluation.yml

@@ -29,6 +29,7 @@ on:
           - "bug-fix"
           - "test-generation"
           - "code-review"
+          - "data-query"
           - "extensibility-request-advisor"
           - "extensibility-request-implement"
           - "extensibility-request-triage"
@@ -47,6 +48,16 @@ on:
         required: false
         default: false
         type: boolean
+      bc-mcp:
+        description: "Enable the Business Central MCP server"
+        required: false
+        default: false
+        type: boolean
+      skills:
+        description: "Enable agent skills"
+        required: false
+        default: false
+        type: boolean
       repeat:
         description: "Number of times to run sequentially (ignored for test runs)"
         required: false
@@ -144,7 +155,12 @@ jobs:
         timeout-minutes: 120
         shell: pwsh
         env:
-          COPILOT_GITHUB_TOKEN: ${{ github.token }}
+          # Copilot CLI must authenticate MCP with a USER token: the Actions github.token (a ghs_
+          # installation token) gets 403 from GET /copilot/mcp_registry and fails closed, blocking ALL
+          # custom MCP servers (github/copilot-cli#4346). The org policy is already allow_all, so a
+          # Copilot-licensed user PAT lets the registry fetch succeed and the BC/MS-Learn MCP load.
+          # Falls back to github.token when the secret is absent (MCP off, but completions still work).
+          COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_CLI_TOKEN || github.token }}
           GH_TOKEN: ${{ github.token }}
         run: |
           Write-Output "::add-mask::$env:COPILOT_GITHUB_TOKEN"
@@ -155,7 +171,9 @@ jobs:
             --repo-path "${{ steps.setup-env.outputs.repo_path }}" `
             --output-dir "${{ env.EVALUATION_RESULTS_DIR }}" `
             ${{ inputs.al-mcp && '--al-mcp' || '' }} `
-            ${{ inputs.al-lsp && '--al-lsp' || '' }}
+            ${{ inputs.al-lsp && '--al-lsp' || '' }} `
+            ${{ inputs.bc-mcp && '--bc-mcp' || '' }} `
+            ${{ inputs.skills && '--skills' || '' }}
 
       - name: Upload evaluation results
         uses: actions/upload-artifact@v6
@@ -192,4 +210,4 @@ jobs:
       repeat: ${{ inputs.repeat }}
       existing-tag: ${{ needs.pin-commit.outputs.tag-name }}
       workflow-inputs: |
-        {"model": "${{ inputs.model }}", "category": "${{ inputs.category }}", "test-run": "${{ inputs.test-run }}", "al-mcp": "${{ inputs.al-mcp }}", "al-lsp": "${{ inputs.al-lsp }}", "git-ref": "${{ inputs.git-ref || github.ref_name }}"}
+        {"model": "${{ inputs.model }}", "category": "${{ inputs.category }}", "test-run": "${{ inputs.test-run }}", "al-mcp": "${{ inputs.al-mcp }}", "al-lsp": "${{ inputs.al-lsp }}", "bc-mcp": "${{ inputs.bc-mcp }}", "skills": "${{ inputs.skills }}", "git-ref": "${{ inputs.git-ref || github.ref_name }}"}

.gitignore

@@ -283,3 +283,6 @@ docs/Gemfile
 docs/Gemfile.lock
 
 *.orig
+
+# Local diagnostic artifact download folders (temp, leading underscore)
+_*/

dataset/dataquery.jsonl

@@ -0,0 +1,11 @@
+{"instance_id": "dataquery__outstanding-sales-value-by-customer-1", "environment_setup_version": "29.0", "created_at": "2026-07-10", "metadata": {"area": "sales"}, "nl_prompt": "For each customer that has at least one open sales order line, return the customer's number, the customer's name, and their total outstanding amount. Use open sales order lines (Sales Line records whose document type is Order) and sum the line 'Outstanding Amount' field (which is net of VAT). Customers with no open sales order lines must not appear.", "ordered": false, "gold_query": "query 50100 OutstandingSalesByCustomer\n{\n    QueryType = Normal;\n\n    elements\n    {\n        dataitem(Customer; Customer)\n        {\n            column(No; \"No.\") { }\n            column(Name; Name) { }\n            dataitem(SalesLine; \"Sales Line\")\n            {\n                DataItemLink = \"Sell-to Customer No.\" = Customer.\"No.\";\n                DataItemTableFilter = \"Document Type\" = const(Order);\n                column(TotalOutstanding; \"Outstanding Amount\") { Method = Sum; }\n            }\n        }\n    }\n}"}
+{"instance_id": "dataquery__total-sold-quantity-by-item-1", "environment_setup_version": "29.0", "created_at": "2026-07-10", "metadata": {"area": "inventory"}, "nl_prompt": "Across all posted sales invoice lines whose type is Item, return each item's number together with the total quantity sold (the sum of the line Quantity). Produce one row per item number.", "ordered": false, "gold_query": "query 50100 SoldQuantityByItem\n{\n    QueryType = Normal;\n\n    elements\n    {\n        dataitem(SalesInvoiceLine; \"Sales Invoice Line\")\n        {\n            DataItemTableFilter = Type = const(Item);\n            column(ItemNo; \"No.\") { }\n            column(TotalQuantity; Quantity) { Method = Sum; }\n        }\n    }\n}"}
+{"instance_id": "dataquery__avg-invoice-amount-by-country-1", "environment_setup_version": "29.0", "created_at": "2026-07-10", "metadata": {"area": "finance"}, "nl_prompt": "For each country/region, return the country/region code and the average posted sales invoice line amount. Average the 'Amount' field (net of VAT) over all posted sales invoice lines, grouping the lines by their bill-to customer's Country/Region Code.", "ordered": false, "gold_query": "query 50100 AvgInvoiceAmountByCountry\n{\n    QueryType = Normal;\n\n    elements\n    {\n        dataitem(Customer; Customer)\n        {\n            column(CountryRegionCode; \"Country/Region Code\") { }\n            dataitem(SalesInvoiceHeader; \"Sales Invoice Header\")\n            {\n                DataItemLink = \"Bill-to Customer No.\" = Customer.\"No.\";\n                dataitem(SalesInvoiceLine; \"Sales Invoice Line\")\n                {\n                    DataItemLink = \"Document No.\" = SalesInvoiceHeader.\"No.\";\n                    column(AvgAmount; Amount) { Method = Average; }\n                }\n            }\n        }\n    }\n}"}
+{"instance_id": "dataquery__total-purchase-amount-by-vendor-1", "environment_setup_version": "29.0", "created_at": "2026-07-10", "metadata": {"area": "purchasing"}, "nl_prompt": "For each vendor that has at least one posted purchase invoice line, return the vendor's number, the vendor's name, and their total posted purchase amount. Sum the 'Amount' field (net of VAT) from posted purchase invoice lines. Vendors with no posted purchase invoice lines must not appear.", "ordered": false, "gold_query": "query 50100 PurchaseAmountByVendor\n{\n    QueryType = Normal;\n\n    elements\n    {\n        dataitem(Vendor; Vendor)\n        {\n            column(No; \"No.\") { }\n            column(Name; Name) { }\n            dataitem(PurchInvLine; \"Purch. Inv. Line\")\n            {\n                DataItemLink = \"Buy-from Vendor No.\" = Vendor.\"No.\";\n                column(TotalAmount; Amount) { Method = Sum; }\n            }\n        }\n    }\n}"}
+{"instance_id": "dataquery__open-sales-order-count-by-customer-1", "environment_setup_version": "29.0", "created_at": "2026-07-13", "metadata": {"area": "sales"}, "nl_prompt": "For each customer that has at least one open sales order, return the customer's number, the customer's name, and the number of open sales orders they have. An open sales order is a sales document whose document type is Order; count the order documents (headers), not the order lines. Customers with no open sales orders must not appear.", "ordered": false, "gold_query": "query 50100 OpenSalesOrdersByCustomer\n{\n    QueryType = Normal;\n\n    elements\n    {\n        dataitem(Customer; Customer)\n        {\n            column(No; \"No.\") { }\n            column(Name; Name) { }\n            dataitem(SalesHeader; \"Sales Header\")\n            {\n                DataItemLink = \"Sell-to Customer No.\" = Customer.\"No.\";\n                DataItemTableFilter = \"Document Type\" = const(Order);\n                column(OrderCount) { Method = Count; }\n            }\n        }\n    }\n}"}
+{"instance_id": "dataquery__opportunity-count-by-status-1", "environment_setup_version": "29.0", "created_at": "2026-07-10", "metadata": {"area": "crm"}, "nl_prompt": "Return the number of CRM opportunities in each status. Group the opportunities by their Status field and, for each status value that occurs, output the status and the count of opportunities with that status.", "ordered": false, "gold_query": "query 50100 OpportunityCountByStatus\n{\n    QueryType = Normal;\n\n    elements\n    {\n        dataitem(Opportunity; Opportunity)\n        {\n            column(Status; Status) { }\n            column(OpportunityCount) { Method = Count; }\n        }\n    }\n}"}
+{"instance_id": "dataquery__customer-count-by-country-1", "environment_setup_version": "29.0", "created_at": "2026-07-28", "metadata": {"area": "sales"}, "nl_prompt": "Group all customers by their Country/Region Code and return, for each distinct code, the Country/Region Code and the number of customers that have it. Include customers whose Country/Region Code is blank as their own group. Count the Customer records (one row per distinct Country/Region Code).", "ordered": false, "gold_query": "query 50100 CustomerCountByCountry\n{\n    QueryType = Normal;\n\n    elements\n    {\n        dataitem(Customer; Customer)\n        {\n            column(CountryRegionCode; \"Country/Region Code\") { }\n            column(CustomerCount) { Method = Count; }\n        }\n    }\n}"}
+{"instance_id": "dataquery__outstanding-purchase-value-by-vendor-1", "environment_setup_version": "29.0", "created_at": "2026-07-28", "metadata": {"area": "purchase"}, "nl_prompt": "For each vendor that has at least one open purchase order line, return the vendor's number, the vendor's name, and their total outstanding amount. Use open purchase order lines (Purchase Line records whose Document Type is Order) and sum the line 'Outstanding Amount' field (net of VAT). Vendors with no open purchase order lines must not appear.", "ordered": false, "gold_query": "query 50100 OutstandingPurchaseByVendor\n{\n    QueryType = Normal;\n\n    elements\n    {\n        dataitem(Vendor; Vendor)\n        {\n            column(No; \"No.\") { }\n            column(Name; Name) { }\n            dataitem(PurchaseLine; \"Purchase Line\")\n            {\n                DataItemLink = \"Buy-from Vendor No.\" = Vendor.\"No.\";\n                DataItemTableFilter = \"Document Type\" = const(Order);\n                column(TotalOutstanding; \"Outstanding Amount\") { Method = Sum; }\n            }\n        }\n    }\n}"}
+{"instance_id": "dataquery__total-posted-sales-amount-by-customer-1", "environment_setup_version": "29.0", "created_at": "2026-07-28", "metadata": {"area": "sales"}, "nl_prompt": "For each customer that has at least one posted sales invoice line, return the customer's number, the customer's name, and their total posted sales amount. Join posted sales invoice headers to their lines via Document No., group by the header's Bill-to Customer No., and sum the line 'Amount' field (net of VAT). Customers with no posted sales invoice lines must not appear.", "ordered": false, "gold_query": "query 50100 PostedSalesAmountByCustomer\n{\n    QueryType = Normal;\n\n    elements\n    {\n        dataitem(Customer; Customer)\n        {\n            column(No; \"No.\") { }\n            column(Name; Name) { }\n            dataitem(SalesInvoiceHeader; \"Sales Invoice Header\")\n            {\n                DataItemLink = \"Bill-to Customer No.\" = Customer.\"No.\";\n                dataitem(SalesInvoiceLine; \"Sales Invoice Line\")\n                {\n                    DataItemLink = \"Document No.\" = SalesInvoiceHeader.\"No.\";\n                    column(TotalAmount; Amount) { Method = Sum; }\n                }\n            }\n        }\n    }\n}"}
+{"instance_id": "dataquery__line-count-per-open-sales-order-1", "environment_setup_version": "29.0", "created_at": "2026-07-28", "metadata": {"area": "sales"}, "nl_prompt": "For each open sales order, return the order's document number and the number of lines it has. Use Sales Line records whose Document Type is Order, group them by Document No., and count the lines per order (one row per order document number).", "ordered": false, "gold_query": "query 50100 LineCountPerOpenSalesOrder\n{\n    QueryType = Normal;\n\n    elements\n    {\n        dataitem(SalesLine; \"Sales Line\")\n        {\n            DataItemTableFilter = \"Document Type\" = const(Order);\n            column(DocumentNo; \"Document No.\") { }\n            column(LineCount) { Method = Count; }\n        }\n    }\n}"}
+{"instance_id": "dataquery__total-purchased-quantity-by-item-1", "environment_setup_version": "29.0", "created_at": "2026-07-28", "metadata": {"area": "purchase"}, "nl_prompt": "Across all posted purchase invoice lines whose Type is Item, return each item's number together with the total purchased quantity (the sum of the line Quantity). Produce one row per item number.", "ordered": false, "gold_query": "query 50100 PurchasedQuantityByItem\n{\n    QueryType = Normal;\n\n    elements\n    {\n        dataitem(PurchInvLine; \"Purch. Inv. Line\")\n        {\n            DataItemTableFilter = Type = const(Item);\n            column(ItemNo; \"No.\") { }\n            column(TotalQuantity; Quantity) { Method = Sum; }\n        }\n    }\n}"}

docs/data-query.md

@@ -0,0 +1,111 @@
+---
+layout: default
+title: Data Query - BC-Bench
+---
+
+# Data Query
+
+This category evaluates an **agent harness and model (or MCP Host)** on its ability to **retrieve data
+from a live Business Central environment** to answer a natural-language data question. It is
+**execution-based** (no LLM judge): the agent reports the rows it retrieved (`answer.json`), and the
+run is **resolved** when those rows match the result set of a hidden gold AL query run against the same
+Cronus/Contoso demo data (values compared normalized; order ignored unless the entry is `ordered`).
+
+The point of the category is to compare **how the data is retrieved**:
+
+- **Baseline** — no data tooling. The agent has to reach the answer on its own (e.g. authoring an AL
+  query from knowledge of the schema), which is hard on a low-resource domain language.
+- **BC MCP experiment** — the agent is given Business Central's **Data Query MCP tools**
+  (`bc_data_find_tables`, `bc_data_get_table_schema`, `bc_data_get_table_relations`, `bc_data_query`)
+  so it can discover tables, inspect schemas and relations, and compile/run read-only AL queries
+  against the live environment. The agent is isolated so the MCP endpoint is its only route to the
+  data, which keeps the comparison honest.
+
+## Baseline Leaderboard
+
+{% if site.data.data-query.aggregate %}
+<table>
+  <thead>
+    <tr>
+      <th>Agent</th>
+      <th>Model</th>
+      <th>mean (95% CI)</th>
+      <th>pass^5</th>
+      <th>Avg Time</th>
+      <th>Version</th>
+    </tr>
+  </thead>
+  <tbody>
+    {% assign sorted_results = site.data.data-query.aggregate | sort: "average" | reverse %}
+    {% for agg in sorted_results %}
+      {% if agg.experiment == null %}
+    <tr>
+      <td>{{ agg.agent_name }}</td>
+      <td>{{ agg.model }}</td>
+      <td>{{ agg.average | times: 100.0 | round: 1 }}%{% if agg.ci_low %} ({{ agg.ci_low | times: 100.0 | round: 1 }}-{{ agg.ci_high | times: 100.0 | round: 1 }}%){% endif %}</td>
+      <td>{% if agg.pass_hat_5 %}{{ agg.pass_hat_5 | times: 100.0 | round: 1 }}%{% endif %}</td>
+      <td>{{ agg.average_duration | round: 1 }}s</td>
+      <td><a href="https://github.com/microsoft/BC-Bench/releases/tag/v{{ agg.benchmark_version }}" target="_blank">{{ agg.benchmark_version }}</a></td>
+    </tr>
+      {% endif %}
+    {% endfor %}
+  </tbody>
+</table>
+{% else %}
+<p><em>No results available yet. Check back soon!</em></p>
+{% endif %}
+
+## BC MCP Experiment
+
+Comparing runs that enable the **Business Central Data Query MCP tools** (`bc-mcp`) against the
+matching no-tooling **Default** baseline for the same model.
+
+{% if site.data.data-query.aggregate %}
+{%- assign mcp_models = "" -%}
+{%- for agg in site.data.data-query.aggregate -%}
+  {%- if agg.experiment and agg.experiment.mcp_servers.size > 0 -%}
+    {%- assign mcp_models = mcp_models | append: "|" | append: agg.model | append: "|" -%}
+  {%- endif -%}
+{%- endfor -%}
+<table>
+  <thead>
+    <tr>
+      <th>Model</th>
+      <th>MCP Servers</th>
+      <th>Skills</th>
+      <th>mean (95% CI)</th>
+      <th>pass^5</th>
+      <th>Avg Time</th>
+      <th>Ver</th>
+    </tr>
+  </thead>
+  <tbody>
+    {%- assign sorted_results = site.data.data-query.aggregate | sort: "average" | reverse -%}
+    {%- for agg in sorted_results -%}
+      {%- assign is_mcp = false -%}
+      {%- assign show_row = false -%}
+      {%- if agg.experiment -%}
+        {%- if agg.experiment.mcp_servers.size > 0 %}{% assign is_mcp = true %}{% assign show_row = true %}{% endif -%}
+      {%- else -%}
+        {%- assign model_key = agg.model | prepend: "|" | append: "|" -%}
+        {%- if mcp_models contains model_key %}{% assign show_row = true %}{% endif -%}
+      {%- endif -%}
+      {%- if show_row %}
+    <tr>
+      <td>{{ agg.model }}</td>
+      <td>{% if is_mcp %}{{ agg.experiment.mcp_servers | join: ", " }}{% else %}<em>Default</em>{% endif %}</td>
+      <td>{% if is_mcp and agg.experiment.skills_enabled %}✓{% else %}—{% endif %}</td>
+      <td>{{ agg.average | times: 100.0 | round: 1 }}%{% if agg.ci_low %} ({{ agg.ci_low | times: 100.0 | round: 1 }}-{{ agg.ci_high | times: 100.0 | round: 1 }}%){% endif %}</td>
+      <td>{% if agg.pass_hat_5 %}{{ agg.pass_hat_5 | times: 100.0 | round: 1 }}%{% endif %}</td>
+      <td>{{ agg.average_duration | round: 1 }}s</td>
+      <td><a href="https://github.com/microsoft/BC-Bench/releases/tag/v{{ agg.benchmark_version }}" target="_blank">{{ agg.benchmark_version }}</a></td>
+    </tr>
+      {%- endif -%}
+    {%- endfor %}
+  </tbody>
+</table>
+{% else %}
+<p><em>No results available yet. Check back soon!</em></p>
+{% endif %}
+
+[← Back to Home](index.md)