From b511b84f60ac637d54a2345b7cb37a92c03f60ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?BLC=20Core=20Studio=20=E2=80=94=20Developer=20Tools?= <207100624+BLCCoreStudio@users.noreply.github.com> Date: Thu, 3 Sep 2026 05:06:31 +0300 Subject: [PATCH 1/5] bench: add AgentContextMap vs WhichRules benchmark --- .github/workflows/benchmark-competitive.yml | 124 ++++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 .github/workflows/benchmark-competitive.yml diff --git a/.github/workflows/benchmark-competitive.yml b/.github/workflows/benchmark-competitive.yml new file mode 100644 index 0000000..fdbff94 --- /dev/null +++ b/.github/workflows/benchmark-competitive.yml @@ -0,0 +1,124 @@ +name: Competitive benchmark + +on: + pull_request: + paths: + - 'src/**' + - 'Cargo.toml' + - 'Cargo.lock' + - '.github/workflows/benchmark-competitive.yml' + workflow_dispatch: + +permissions: + contents: read + +jobs: + instruction-map: + name: AgentContextMap vs WhichRules + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: Install benchmark tools + run: | + sudo apt-get update + sudo apt-get install --yes hyperfine time + npm install --global whichrules@0.2.0 + + - name: Build AgentContextMap + run: cargo build --release --locked + + - name: Create common-subset fixture + shell: bash + run: | + set -euo pipefail + rm -rf /tmp/agent-rules-fixture + mkdir -p /tmp/agent-rules-fixture/src /tmp/agent-rules-fixture/.github /tmp/agent-rules-fixture/.cursor/rules + cat >/tmp/agent-rules-fixture/AGENTS.md <<'EOF' + Always use pnpm for dependency management. + Use single quotes in TypeScript. + EOF + cat >/tmp/agent-rules-fixture/CLAUDE.md <<'EOF' + Use npm for all package scripts. + Use double quotes in TypeScript. + EOF + cat >/tmp/agent-rules-fixture/GEMINI.md <<'EOF' + Run tests before committing changes. + EOF + cat >/tmp/agent-rules-fixture/.github/copilot-instructions.md <<'EOF' + Keep changes focused and add tests for behavior changes. + EOF + cat >/tmp/agent-rules-fixture/.cursor/rules/typescript.mdc <<'EOF' + --- + globs: "**/*.ts" + alwaysApply: false + --- + Prefer explicit return types for exported functions. + EOF + printf 'export const value: number = 1;\n' >/tmp/agent-rules-fixture/src/api.ts + + - name: Verify shared instruction-discovery task + shell: bash + run: | + set -euo pipefail + ./target/release/agentcontext /tmp/agent-rules-fixture --target src/api.ts --json >/tmp/acm.json + (cd /tmp/agent-rules-fixture && whichrules src/api.ts --agent all --json) >/tmp/whichrules.json + for file in AGENTS.md CLAUDE.md GEMINI.md copilot-instructions.md typescript.mdc; do + grep -q "$file" /tmp/acm.json + grep -q "$file" /tmp/whichrules.json + done + printf 'Both tools discovered the same five common-subset instruction sources.\n' + + - name: Benchmark one-target instruction mapping + shell: bash + run: | + set -euo pipefail + mkdir -p benchmark-results + { + echo "workload=map instruction sources affecting src/api.ts across the five shared agent ecosystems" + echo "fixture=AGENTS.md + CLAUDE.md + GEMINI.md + Copilot instructions + Cursor MDC" + echo "runner=ubuntu-24.04" + echo "kernel=$(uname -srmo)" + echo "cpu=$(lscpu | awk -F: '/Model name/{gsub(/^[ \t]+/,"",$2); print $2; exit}')" + echo "agentcontext=$(./target/release/agentcontext --version)" + echo "whichrules=$(whichrules --version 2>&1 | head -n1 || true)" + echo "node=$(node --version)" + echo "hyperfine=$(hyperfine --version | head -n1)" + } > benchmark-results/environment.txt + + hyperfine --warmup 5 --runs 30 \ + --export-json benchmark-results/instruction-map.json \ + --export-markdown benchmark-results/instruction-map.md \ + --command-name 'AgentContextMap' './target/release/agentcontext /tmp/agent-rules-fixture --target src/api.ts --json >/dev/null' \ + --command-name 'WhichRules 0.2.0' 'cd /tmp/agent-rules-fixture && whichrules src/api.ts --agent all --json >/dev/null' + + /usr/bin/time -v ./target/release/agentcontext /tmp/agent-rules-fixture --target src/api.ts --json >/dev/null 2>benchmark-results/agentcontext-time-v.txt + /usr/bin/time -v bash -lc 'cd /tmp/agent-rules-fixture && whichrules src/api.ts --agent all --json >/dev/null' 2>benchmark-results/whichrules-time-v.txt + cp /tmp/acm.json benchmark-results/agentcontext-output.json + cp /tmp/whichrules.json benchmark-results/whichrules-output.json + cat benchmark-results/instruction-map.md + + - name: Publish summary + if: always() + run: | + if [[ -f benchmark-results/environment.txt ]]; then + echo '## Environment' >> "$GITHUB_STEP_SUMMARY" + echo '```text' >> "$GITHUB_STEP_SUMMARY" + cat benchmark-results/environment.txt >> "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" + fi + if [[ -f benchmark-results/instruction-map.md ]]; then + echo '## Common-subset instruction mapping' >> "$GITHUB_STEP_SUMMARY" + cat benchmark-results/instruction-map.md >> "$GITHUB_STEP_SUMMARY" + fi + + - name: Upload raw evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: agentcontextmap-competitive-benchmark-${{ github.sha }} + path: benchmark-results/ + if-no-files-found: error + retention-days: 90 From b76e33dea74f508012d14f7c183c27b91420fb02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?BLC=20Core=20Studio=20=E2=80=94=20Developer=20Tools?= <207100624+BLCCoreStudio@users.noreply.github.com> Date: Thu, 3 Sep 2026 05:10:49 +0300 Subject: [PATCH 2/5] bench: pin WhichRules from GitHub source --- .github/workflows/benchmark-competitive.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmark-competitive.yml b/.github/workflows/benchmark-competitive.yml index fdbff94..921ee88 100644 --- a/.github/workflows/benchmark-competitive.yml +++ b/.github/workflows/benchmark-competitive.yml @@ -24,8 +24,8 @@ jobs: - name: Install benchmark tools run: | sudo apt-get update - sudo apt-get install --yes hyperfine time - npm install --global whichrules@0.2.0 + sudo apt-get install --yes hyperfine time git + npm install --global 'git+https://github.com/z35068037/whichrules.git#413f62a3583538636d378dee2441e75a0b4c4981' - name: Build AgentContextMap run: cargo build --release --locked @@ -79,6 +79,7 @@ jobs: { echo "workload=map instruction sources affecting src/api.ts across the five shared agent ecosystems" echo "fixture=AGENTS.md + CLAUDE.md + GEMINI.md + Copilot instructions + Cursor MDC" + echo "whichrules_commit=413f62a3583538636d378dee2441e75a0b4c4981" echo "runner=ubuntu-24.04" echo "kernel=$(uname -srmo)" echo "cpu=$(lscpu | awk -F: '/Model name/{gsub(/^[ \t]+/,"",$2); print $2; exit}')" From 2729ea69862b63f28135db15ac5c7a5494d633a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?BLC=20Core=20Studio=20=E2=80=94=20Developer=20Tools?= <207100624+BLCCoreStudio@users.noreply.github.com> Date: Thu, 3 Sep 2026 05:14:26 +0300 Subject: [PATCH 3/5] bench: make WhichRules root and command explicit --- .github/workflows/benchmark-competitive.yml | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/.github/workflows/benchmark-competitive.yml b/.github/workflows/benchmark-competitive.yml index 921ee88..d126bb4 100644 --- a/.github/workflows/benchmark-competitive.yml +++ b/.github/workflows/benchmark-competitive.yml @@ -63,11 +63,16 @@ jobs: shell: bash run: | set -euo pipefail - ./target/release/agentcontext /tmp/agent-rules-fixture --target src/api.ts --json >/tmp/acm.json - (cd /tmp/agent-rules-fixture && whichrules src/api.ts --agent all --json) >/tmp/whichrules.json + mkdir -p benchmark-results + ./target/release/agentcontext /tmp/agent-rules-fixture --target src/api.ts --json >benchmark-results/agentcontext-output.json + whichrules explain src/api.ts --root /tmp/agent-rules-fixture --agent all --json >benchmark-results/whichrules-output.json + echo '--- AgentContextMap output ---' + cat benchmark-results/agentcontext-output.json + echo '--- WhichRules output ---' + cat benchmark-results/whichrules-output.json for file in AGENTS.md CLAUDE.md GEMINI.md copilot-instructions.md typescript.mdc; do - grep -q "$file" /tmp/acm.json - grep -q "$file" /tmp/whichrules.json + grep -q "$file" benchmark-results/agentcontext-output.json + grep -q "$file" benchmark-results/whichrules-output.json done printf 'Both tools discovered the same five common-subset instruction sources.\n' @@ -75,7 +80,6 @@ jobs: shell: bash run: | set -euo pipefail - mkdir -p benchmark-results { echo "workload=map instruction sources affecting src/api.ts across the five shared agent ecosystems" echo "fixture=AGENTS.md + CLAUDE.md + GEMINI.md + Copilot instructions + Cursor MDC" @@ -84,7 +88,6 @@ jobs: echo "kernel=$(uname -srmo)" echo "cpu=$(lscpu | awk -F: '/Model name/{gsub(/^[ \t]+/,"",$2); print $2; exit}')" echo "agentcontext=$(./target/release/agentcontext --version)" - echo "whichrules=$(whichrules --version 2>&1 | head -n1 || true)" echo "node=$(node --version)" echo "hyperfine=$(hyperfine --version | head -n1)" } > benchmark-results/environment.txt @@ -93,12 +96,10 @@ jobs: --export-json benchmark-results/instruction-map.json \ --export-markdown benchmark-results/instruction-map.md \ --command-name 'AgentContextMap' './target/release/agentcontext /tmp/agent-rules-fixture --target src/api.ts --json >/dev/null' \ - --command-name 'WhichRules 0.2.0' 'cd /tmp/agent-rules-fixture && whichrules src/api.ts --agent all --json >/dev/null' + --command-name 'WhichRules 0.2.0' 'whichrules explain src/api.ts --root /tmp/agent-rules-fixture --agent all --json >/dev/null' /usr/bin/time -v ./target/release/agentcontext /tmp/agent-rules-fixture --target src/api.ts --json >/dev/null 2>benchmark-results/agentcontext-time-v.txt - /usr/bin/time -v bash -lc 'cd /tmp/agent-rules-fixture && whichrules src/api.ts --agent all --json >/dev/null' 2>benchmark-results/whichrules-time-v.txt - cp /tmp/acm.json benchmark-results/agentcontext-output.json - cp /tmp/whichrules.json benchmark-results/whichrules-output.json + /usr/bin/time -v whichrules explain src/api.ts --root /tmp/agent-rules-fixture --agent all --json >/dev/null 2>benchmark-results/whichrules-time-v.txt cat benchmark-results/instruction-map.md - name: Publish summary From 4372d763b1a425e71247b6353c678947c5b6139f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?BLC=20Core=20Studio=20=E2=80=94=20Developer=20Tools?= <207100624+BLCCoreStudio@users.noreply.github.com> Date: Thu, 3 Sep 2026 05:19:04 +0300 Subject: [PATCH 4/5] bench: use absolute WhichRules target path --- .github/workflows/benchmark-competitive.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/benchmark-competitive.yml b/.github/workflows/benchmark-competitive.yml index d126bb4..deda6ce 100644 --- a/.github/workflows/benchmark-competitive.yml +++ b/.github/workflows/benchmark-competitive.yml @@ -65,7 +65,7 @@ jobs: set -euo pipefail mkdir -p benchmark-results ./target/release/agentcontext /tmp/agent-rules-fixture --target src/api.ts --json >benchmark-results/agentcontext-output.json - whichrules explain src/api.ts --root /tmp/agent-rules-fixture --agent all --json >benchmark-results/whichrules-output.json + whichrules explain /tmp/agent-rules-fixture/src/api.ts --root /tmp/agent-rules-fixture --agent all --json >benchmark-results/whichrules-output.json echo '--- AgentContextMap output ---' cat benchmark-results/agentcontext-output.json echo '--- WhichRules output ---' @@ -96,10 +96,10 @@ jobs: --export-json benchmark-results/instruction-map.json \ --export-markdown benchmark-results/instruction-map.md \ --command-name 'AgentContextMap' './target/release/agentcontext /tmp/agent-rules-fixture --target src/api.ts --json >/dev/null' \ - --command-name 'WhichRules 0.2.0' 'whichrules explain src/api.ts --root /tmp/agent-rules-fixture --agent all --json >/dev/null' + --command-name 'WhichRules 0.2.0' 'whichrules explain /tmp/agent-rules-fixture/src/api.ts --root /tmp/agent-rules-fixture --agent all --json >/dev/null' /usr/bin/time -v ./target/release/agentcontext /tmp/agent-rules-fixture --target src/api.ts --json >/dev/null 2>benchmark-results/agentcontext-time-v.txt - /usr/bin/time -v whichrules explain src/api.ts --root /tmp/agent-rules-fixture --agent all --json >/dev/null 2>benchmark-results/whichrules-time-v.txt + /usr/bin/time -v whichrules explain /tmp/agent-rules-fixture/src/api.ts --root /tmp/agent-rules-fixture --agent all --json >/dev/null 2>benchmark-results/whichrules-time-v.txt cat benchmark-results/instruction-map.md - name: Publish summary From 3d1d8b12763c39d4f1153eae9c1aef3f69b9c44d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?BLC=20Core=20Studio=20=E2=80=94=20Developer=20Tools?= <207100624+BLCCoreStudio@users.noreply.github.com> Date: Thu, 3 Sep 2026 05:22:02 +0300 Subject: [PATCH 5/5] bench: aggregate instruction mapping for stable timing --- .github/workflows/benchmark-competitive.yml | 34 ++++++++++++++------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/.github/workflows/benchmark-competitive.yml b/.github/workflows/benchmark-competitive.yml index deda6ce..032289e 100644 --- a/.github/workflows/benchmark-competitive.yml +++ b/.github/workflows/benchmark-competitive.yml @@ -66,22 +66,34 @@ jobs: mkdir -p benchmark-results ./target/release/agentcontext /tmp/agent-rules-fixture --target src/api.ts --json >benchmark-results/agentcontext-output.json whichrules explain /tmp/agent-rules-fixture/src/api.ts --root /tmp/agent-rules-fixture --agent all --json >benchmark-results/whichrules-output.json - echo '--- AgentContextMap output ---' - cat benchmark-results/agentcontext-output.json - echo '--- WhichRules output ---' - cat benchmark-results/whichrules-output.json for file in AGENTS.md CLAUDE.md GEMINI.md copilot-instructions.md typescript.mdc; do grep -q "$file" benchmark-results/agentcontext-output.json grep -q "$file" benchmark-results/whichrules-output.json done printf 'Both tools discovered the same five common-subset instruction sources.\n' - - name: Benchmark one-target instruction mapping + - name: Benchmark aggregated instruction mapping shell: bash run: | set -euo pipefail + cat >/tmp/run-agentcontext-100.sh <<'EOF' + #!/usr/bin/env bash + set -euo pipefail + for _ in $(seq 1 100); do + ./target/release/agentcontext /tmp/agent-rules-fixture --target src/api.ts --json >/dev/null + done + EOF + cat >/tmp/run-whichrules-100.sh <<'EOF' + #!/usr/bin/env bash + set -euo pipefail + for _ in $(seq 1 100); do + whichrules explain /tmp/agent-rules-fixture/src/api.ts --root /tmp/agent-rules-fixture --agent all --json >/dev/null + done + EOF + chmod +x /tmp/run-agentcontext-100.sh /tmp/run-whichrules-100.sh + { - echo "workload=map instruction sources affecting src/api.ts across the five shared agent ecosystems" + echo "workload=100 sequential mappings of instruction sources affecting src/api.ts" echo "fixture=AGENTS.md + CLAUDE.md + GEMINI.md + Copilot instructions + Cursor MDC" echo "whichrules_commit=413f62a3583538636d378dee2441e75a0b4c4981" echo "runner=ubuntu-24.04" @@ -92,14 +104,14 @@ jobs: echo "hyperfine=$(hyperfine --version | head -n1)" } > benchmark-results/environment.txt - hyperfine --warmup 5 --runs 30 \ + hyperfine --warmup 2 --runs 10 \ --export-json benchmark-results/instruction-map.json \ --export-markdown benchmark-results/instruction-map.md \ - --command-name 'AgentContextMap' './target/release/agentcontext /tmp/agent-rules-fixture --target src/api.ts --json >/dev/null' \ - --command-name 'WhichRules 0.2.0' 'whichrules explain /tmp/agent-rules-fixture/src/api.ts --root /tmp/agent-rules-fixture --agent all --json >/dev/null' + --command-name 'AgentContextMap x100' '/tmp/run-agentcontext-100.sh' \ + --command-name 'WhichRules 0.2.0 x100' '/tmp/run-whichrules-100.sh' - /usr/bin/time -v ./target/release/agentcontext /tmp/agent-rules-fixture --target src/api.ts --json >/dev/null 2>benchmark-results/agentcontext-time-v.txt - /usr/bin/time -v whichrules explain /tmp/agent-rules-fixture/src/api.ts --root /tmp/agent-rules-fixture --agent all --json >/dev/null 2>benchmark-results/whichrules-time-v.txt + /usr/bin/time -v /tmp/run-agentcontext-100.sh >/dev/null 2>benchmark-results/agentcontext-time-v.txt + /usr/bin/time -v /tmp/run-whichrules-100.sh >/dev/null 2>benchmark-results/whichrules-time-v.txt cat benchmark-results/instruction-map.md - name: Publish summary