diff --git a/.github/workflows/benchmark-competitive.yml b/.github/workflows/benchmark-competitive.yml new file mode 100644 index 0000000..032289e --- /dev/null +++ b/.github/workflows/benchmark-competitive.yml @@ -0,0 +1,138 @@ +name: Competitive benchmark + +on: + pull_request: + paths: + - 'src/**' + - 'Cargo.toml' + - 'Cargo.lock' + - '.github/workflows/benchmark-competitive.yml' + workflow_dispatch: + +permissions: + contents: read + +jobs: + instruction-map: + name: AgentContextMap vs WhichRules + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: Install benchmark tools + run: | + sudo apt-get update + sudo apt-get install --yes hyperfine time git + npm install --global 'git+https://github.com/z35068037/whichrules.git#413f62a3583538636d378dee2441e75a0b4c4981' + + - name: Build AgentContextMap + run: cargo build --release --locked + + - name: Create common-subset fixture + shell: bash + run: | + set -euo pipefail + rm -rf /tmp/agent-rules-fixture + mkdir -p /tmp/agent-rules-fixture/src /tmp/agent-rules-fixture/.github /tmp/agent-rules-fixture/.cursor/rules + cat >/tmp/agent-rules-fixture/AGENTS.md <<'EOF' + Always use pnpm for dependency management. + Use single quotes in TypeScript. + EOF + cat >/tmp/agent-rules-fixture/CLAUDE.md <<'EOF' + Use npm for all package scripts. + Use double quotes in TypeScript. + EOF + cat >/tmp/agent-rules-fixture/GEMINI.md <<'EOF' + Run tests before committing changes. + EOF + cat >/tmp/agent-rules-fixture/.github/copilot-instructions.md <<'EOF' + Keep changes focused and add tests for behavior changes. + EOF + cat >/tmp/agent-rules-fixture/.cursor/rules/typescript.mdc <<'EOF' + --- + globs: "**/*.ts" + alwaysApply: false + --- + Prefer explicit return types for exported functions. + EOF + printf 'export const value: number = 1;\n' >/tmp/agent-rules-fixture/src/api.ts + + - name: Verify shared instruction-discovery task + shell: bash + run: | + set -euo pipefail + mkdir -p benchmark-results + ./target/release/agentcontext /tmp/agent-rules-fixture --target src/api.ts --json >benchmark-results/agentcontext-output.json + whichrules explain /tmp/agent-rules-fixture/src/api.ts --root /tmp/agent-rules-fixture --agent all --json >benchmark-results/whichrules-output.json + for file in AGENTS.md CLAUDE.md GEMINI.md copilot-instructions.md typescript.mdc; do + grep -q "$file" benchmark-results/agentcontext-output.json + grep -q "$file" benchmark-results/whichrules-output.json + done + printf 'Both tools discovered the same five common-subset instruction sources.\n' + + - name: Benchmark aggregated instruction mapping + shell: bash + run: | + set -euo pipefail + cat >/tmp/run-agentcontext-100.sh <<'EOF' + #!/usr/bin/env bash + set -euo pipefail + for _ in $(seq 1 100); do + ./target/release/agentcontext /tmp/agent-rules-fixture --target src/api.ts --json >/dev/null + done + EOF + cat >/tmp/run-whichrules-100.sh <<'EOF' + #!/usr/bin/env bash + set -euo pipefail + for _ in $(seq 1 100); do + whichrules explain /tmp/agent-rules-fixture/src/api.ts --root /tmp/agent-rules-fixture --agent all --json >/dev/null + done + EOF + chmod +x /tmp/run-agentcontext-100.sh /tmp/run-whichrules-100.sh + + { + echo "workload=100 sequential mappings of instruction sources affecting src/api.ts" + echo "fixture=AGENTS.md + CLAUDE.md + GEMINI.md + Copilot instructions + Cursor MDC" + echo "whichrules_commit=413f62a3583538636d378dee2441e75a0b4c4981" + echo "runner=ubuntu-24.04" + echo "kernel=$(uname -srmo)" + echo "cpu=$(lscpu | awk -F: '/Model name/{gsub(/^[ \t]+/,"",$2); print $2; exit}')" + echo "agentcontext=$(./target/release/agentcontext --version)" + echo "node=$(node --version)" + echo "hyperfine=$(hyperfine --version | head -n1)" + } > benchmark-results/environment.txt + + hyperfine --warmup 2 --runs 10 \ + --export-json benchmark-results/instruction-map.json \ + --export-markdown benchmark-results/instruction-map.md \ + --command-name 'AgentContextMap x100' '/tmp/run-agentcontext-100.sh' \ + --command-name 'WhichRules 0.2.0 x100' '/tmp/run-whichrules-100.sh' + + /usr/bin/time -v /tmp/run-agentcontext-100.sh >/dev/null 2>benchmark-results/agentcontext-time-v.txt + /usr/bin/time -v /tmp/run-whichrules-100.sh >/dev/null 2>benchmark-results/whichrules-time-v.txt + cat benchmark-results/instruction-map.md + + - name: Publish summary + if: always() + run: | + if [[ -f benchmark-results/environment.txt ]]; then + echo '## Environment' >> "$GITHUB_STEP_SUMMARY" + echo '```text' >> "$GITHUB_STEP_SUMMARY" + cat benchmark-results/environment.txt >> "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" + fi + if [[ -f benchmark-results/instruction-map.md ]]; then + echo '## Common-subset instruction mapping' >> "$GITHUB_STEP_SUMMARY" + cat benchmark-results/instruction-map.md >> "$GITHUB_STEP_SUMMARY" + fi + + - name: Upload raw evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: agentcontextmap-competitive-benchmark-${{ github.sha }} + path: benchmark-results/ + if-no-files-found: error + retention-days: 90