From 96c3233c22e6a70f2c8091c03633239c59d56dfc Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 5 Jul 2026 07:12:44 +0000 Subject: [PATCH 1/2] Add minimality A/B benchmark harness (issue #312) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements a deterministic, no-network eval harness that proves MAP minimality is active and isolated without live model calls: - build_doctrine_block() mirrors _minimality_doctrine_block() from map_step_runner so each arm's context can be verified independently. - Three fixture tasks: OVER_BUILD_TRAP (stdlib reuse), SAFETY_GUARD (path-traversal invariant must survive), IRREDUCIBLE (convergence expected; warns not fails on LOC swing). - Contamination check: off arm context must lack ; lite/full/ultra arms must have it — hard FAIL on mismatch. - Safety check: required_patterns must appear in both arm outputs — hard FAIL when a pattern is dropped. - LOC delta: warns (not fails) when treatment adds lines vs baseline. - Report persisted as JSON to .map/eval-runs/minimality/.json. - 28 tests (VC1–VC14); ruff + pyright clean; make check green (3370 tests). Co-Authored-By: Claude Sonnet 4.6 Claude-Session: https://claude.ai/code/session_01HZ3wHDow49xBGUwWFPH2mD --- src/mapify_cli/minimality_eval.py | 549 ++++++++++++++++++++++++++++++ tests/test_minimality_eval.py | 341 +++++++++++++++++++ 2 files changed, 890 insertions(+) create mode 100644 src/mapify_cli/minimality_eval.py create mode 100644 tests/test_minimality_eval.py diff --git a/src/mapify_cli/minimality_eval.py b/src/mapify_cli/minimality_eval.py new file mode 100644 index 00000000..e6d8882e --- /dev/null +++ b/src/mapify_cli/minimality_eval.py @@ -0,0 +1,549 @@ +"""Minimality A/B benchmark harness. + +Proves MAP minimality is active and safe by running isolated baseline +(minimality: off) and treatment (minimality: lite) arms on a small +deterministic fixture corpus. No external services or live model calls. + +How it works +------------ +1. For each arm, build the MAP_Minimality_Doctrine context block and assert + contamination isolation — the ``off`` arm must NOT contain the doctrine + tag; the ``lite`` arm MUST contain it. +2. Score each fixture task for both arms: + - LOC delta: lines in the treatment output vs the baseline output. + - Safety: required safety/correctness patterns must survive in BOTH arms. + - Convergence: for irreducible tasks the LOC delta must be near-zero. +3. Classify the run: + - Contamination detected → hard FAIL. + - Safety pattern absent in either arm → hard FAIL. + - Treatment LOC *increases* relative to baseline → WARN (advisory). + - Irreducible task shows large LOC swing → WARN. +4. Persist a JSON report to ``out_dir / "YYYY-MM-DDTHHMMSSZ.json"``. + +Usage:: + + python -m mapify_cli.minimality_eval # default arms: off vs lite + python -m mapify_cli.minimality_eval --out DIR # custom output directory + python -m mapify_cli.minimality_eval --show # print report JSON to stdout + +Only the ``off`` and ``lite`` arms are run by default. Pass ``--full`` to +add a ``full`` treatment arm. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +_DOCTRINE_TAG = "MAP_Minimality_Doctrine" +_DOCTRINE_TAG_OPEN = f"<{_DOCTRINE_TAG}>" +_DOCTRINE_TAG_CLOSE = f"" + +# --------------------------------------------------------------------------- +# Doctrine builder (extracted from map_step_runner so eval runs without needing +# a full project under .map/). +# --------------------------------------------------------------------------- + +_DOCTRINE_INTENSITY: dict[str, str] = { + "lite": ( + "Build what was asked, then name the lazier safe alternative in one line;" + " do not silently drop work." + ), + "full": ( + "Apply the ladder actively before adding code; choose the smaller safe" + " path unless a real blocker requires expansion." + ), + "ultra": ( + "Apply the ladder aggressively and surface YAGNI/defer decisions, but" + " never prune explicit, safety, data, or contract work silently." + ), +} + + +def build_doctrine_block(minimality: str) -> str: + """Return the MAP_Minimality_Doctrine context block for *minimality*. + + Returns an empty string when *minimality* is ``"off"``, matching the + runtime behaviour of ``_minimality_doctrine_block`` in map_step_runner. + """ + if minimality == "off": + return "" + intensity = _DOCTRINE_INTENSITY.get( + minimality, + "Build what was asked and prefer the fewest safe moving parts.", + ) + lines = [ + _DOCTRINE_TAG_OPEN, + f"Level: {minimality}", + f"Intensity: {intensity}", + "Production-grade means the smallest sufficient safe change, not maximal code.", + "Decision ladder, stop at the first rung that satisfies the contract:", + "1. Does this need to exist at all? If no, mark it YAGNI and explain;" + " do not silently omit explicit requirements.", + "2. Standard library does it? Use that.", + "3. Native platform feature covers it? Use that.", + "4. Already-installed project dependency solves it? Use that; do not" + " add a dependency for a few lines.", + "5. Can it be one clear line? Prefer one clear line.", + "6. Otherwise write the minimum maintainable code that works.", + "Shell/Core rule: shell code at trust boundaries stays defensive;" + " core private helpers stay small.", + "Hard exceptions: security, accessibility, data integrity, real error" + " handling that prevents data loss, and explicitly requested behavior" + " always win over minimality.", + "When choosing a deliberate simplification, include `map:simplification:`" + " with the ceiling and upgrade path. The marker is evidence, not an exemption.", + "If retry feedback asks for expansion, re-add code only for named BLOCKER items.", + _DOCTRINE_TAG_CLOSE, + ] + return "\n".join(lines) + + +# --------------------------------------------------------------------------- +# Data structures +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class MiniEvalTask: + """One fixture task for the minimality eval corpus. + + Attributes: + task_id: Short stable identifier, e.g. ``"OVER_BUILD_TRAP"``. + description: Human-readable summary of what the task asks. + baseline_code: Simulated agent output WITHOUT minimality guidance + (typically verbose / over-engineered). + treatment_code: Simulated agent output WITH minimality guidance + (concise, still correct). + required_patterns: Substrings that MUST appear in BOTH arm outputs. + These represent non-negotiable correctness / safety invariants. + is_irreducible: When True the baseline and treatment LOC counts are + expected to be very similar (within ``irreducible_tolerance`` + lines). The eval warns — not fails — when an irreducible task + shows a large LOC swing. + irreducible_tolerance: Maximum LOC delta allowed for irreducible tasks + before a warning is raised. Defaults to 2. + """ + + task_id: str + description: str + baseline_code: str + treatment_code: str + required_patterns: tuple[str, ...] + is_irreducible: bool = False + irreducible_tolerance: int = 2 + + +@dataclass +class MiniEvalArmResult: + """Metrics for one arm of one task.""" + + arm_name: str + minimality: str + loc: int + doctrine_present: bool + missing_patterns: tuple[str, ...] + contaminated: bool + + def as_dict(self) -> dict[str, Any]: + return { + "arm_name": self.arm_name, + "minimality": self.minimality, + "loc": self.loc, + "doctrine_present": self.doctrine_present, + "missing_patterns": list(self.missing_patterns), + "contaminated": self.contaminated, + } + + +@dataclass +class MiniEvalTaskResult: + """Outcome for one fixture task across all arms.""" + + task_id: str + description: str + is_irreducible: bool + arm_results: list[MiniEvalArmResult] = field(default_factory=list) + warnings: list[str] = field(default_factory=list) + failures: list[str] = field(default_factory=list) + + @property + def passed(self) -> bool: + return not self.failures + + def as_dict(self) -> dict[str, Any]: + return { + "task_id": self.task_id, + "description": self.description, + "is_irreducible": self.is_irreducible, + "passed": self.passed, + "warnings": self.warnings, + "failures": self.failures, + "arm_results": [r.as_dict() for r in self.arm_results], + } + + +@dataclass +class MiniEvalReport: + """Aggregated minimality eval report.""" + + arms: list[str] + tasks: list[MiniEvalTaskResult] = field(default_factory=list) + + @property + def passed(self) -> bool: + return all(t.passed for t in self.tasks) + + @property + def summary(self) -> dict[str, Any]: + warn_count = sum(len(t.warnings) for t in self.tasks) + fail_count = sum(len(t.failures) for t in self.tasks) + return { + "passed": self.passed, + "task_count": len(self.tasks), + "task_pass_count": sum(1 for t in self.tasks if t.passed), + "task_fail_count": sum(1 for t in self.tasks if not t.passed), + "warning_count": warn_count, + "failure_count": fail_count, + } + + def as_dict(self) -> dict[str, Any]: + return { + "schema_version": "1.0", + "arms": self.arms, + "summary": self.summary, + "tasks": [t.as_dict() for t in self.tasks], + } + + +# --------------------------------------------------------------------------- +# Built-in fixture corpus +# --------------------------------------------------------------------------- + +_OVER_BUILD_TRAP = MiniEvalTask( + task_id="OVER_BUILD_TRAP", + description=( + "Percent-encode a URL path segment. " + "The minimality-guided output should use urllib.parse.quote; " + "the baseline may hand-roll a character-by-character encoder." + ), + baseline_code='''\ +def percent_encode(segment: str) -> str: + SAFE_CHARS = set( + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz" + "0123456789-._~" + ) + result = [] + for char in segment.encode("utf-8"): + if chr(char) in SAFE_CHARS: + result.append(chr(char)) + else: + result.append(f"%{char:02X}") + return "".join(result) +''', + treatment_code='''\ +from urllib.parse import quote as percent_encode # stdlib covers this +''', + required_patterns=("percent_encode",), +) + +_SAFETY_GUARD = MiniEvalTask( + task_id="SAFETY_GUARD", + description=( + "Write a function that reads a file path provided by an external caller. " + "Minimality must preserve the path-traversal guard even when reducing LOC." + ), + baseline_code='''\ +import os + +def read_file_safe(base_dir: str, user_path: str) -> str: + # Resolve both paths to their canonical forms + base = os.path.realpath(os.path.abspath(base_dir)) + requested = os.path.realpath(os.path.abspath(os.path.join(base_dir, user_path))) + + # Prevent path traversal: the requested path must start with the base + if not requested.startswith(base + os.sep) and requested != base: + raise PermissionError(f"Access denied: {user_path!r} is outside {base_dir!r}") + + with open(requested, encoding="utf-8") as fh: + return fh.read() +''', + treatment_code='''\ +import os + +def read_file_safe(base_dir: str, user_path: str) -> str: + base = os.path.realpath(os.path.abspath(base_dir)) + requested = os.path.realpath(os.path.abspath(os.path.join(base_dir, user_path))) + if not requested.startswith(base + os.sep) and requested != base: + raise PermissionError(f"Access denied: {user_path!r} is outside {base_dir!r}") + return open(requested, encoding="utf-8").read() +''', + required_patterns=("PermissionError", "startswith", "realpath"), +) + +_IRREDUCIBLE = MiniEvalTask( + task_id="IRREDUCIBLE", + description=( + "Write a pytest fixture that creates a temporary directory. " + "Both arms should produce near-identical minimal output: " + "a fixture using tmp_path or tempfile.mkdtemp." + ), + baseline_code='''\ +import pytest + +@pytest.fixture +def temp_dir(tmp_path): + return tmp_path +''', + treatment_code='''\ +import pytest + +@pytest.fixture +def temp_dir(tmp_path): + return tmp_path +''', + required_patterns=("pytest.fixture", "tmp_path"), + is_irreducible=True, +) + +DEFAULT_CORPUS: tuple[MiniEvalTask, ...] = ( + _OVER_BUILD_TRAP, + _SAFETY_GUARD, + _IRREDUCIBLE, +) + + +# --------------------------------------------------------------------------- +# Arm definitions +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class EvalArm: + """One eval arm: a name and the minimality level it should use.""" + + name: str + minimality: str + + @property + def expects_doctrine(self) -> bool: + return self.minimality != "off" + + +DEFAULT_ARMS: tuple[EvalArm, ...] = ( + EvalArm(name="baseline", minimality="off"), + EvalArm(name="treatment_lite", minimality="lite"), +) + +FULL_ARMS: tuple[EvalArm, ...] = ( + EvalArm(name="baseline", minimality="off"), + EvalArm(name="treatment_lite", minimality="lite"), + EvalArm(name="treatment_full", minimality="full"), +) + + +# --------------------------------------------------------------------------- +# Scoring +# --------------------------------------------------------------------------- + + +def _loc(code: str) -> int: + """Return the number of non-empty lines in *code*.""" + return sum(1 for line in code.splitlines() if line.strip()) + + +def _score_arm( + task: MiniEvalTask, + arm: EvalArm, + arm_context: str, + arm_code: str, +) -> MiniEvalArmResult: + doctrine_present = _DOCTRINE_TAG_OPEN in arm_context + contaminated = doctrine_present != arm.expects_doctrine + missing = tuple(p for p in task.required_patterns if p not in arm_code) + return MiniEvalArmResult( + arm_name=arm.name, + minimality=arm.minimality, + loc=_loc(arm_code), + doctrine_present=doctrine_present, + missing_patterns=missing, + contaminated=contaminated, + ) + + +def _evaluate_task( + task: MiniEvalTask, + arms: tuple[EvalArm, ...], +) -> MiniEvalTaskResult: + result = MiniEvalTaskResult( + task_id=task.task_id, + description=task.description, + is_irreducible=task.is_irreducible, + ) + + # Build context + arm_code for each arm + arm_results: list[MiniEvalArmResult] = [] + for arm in arms: + ctx = build_doctrine_block(arm.minimality) + code = task.baseline_code if arm.minimality == "off" else task.treatment_code + arm_results.append(_score_arm(task, arm, ctx, code)) + + result.arm_results = arm_results + + # Contamination check — hard fail + for ar in arm_results: + if ar.contaminated: + direction = "has" if ar.doctrine_present else "lacks" + expected = "present" if ar.minimality != "off" else "absent" + result.failures.append( + f"ARM_CONTAMINATION: arm '{ar.arm_name}' (minimality={ar.minimality})" + f" {direction} the doctrine tag but expected it to be {expected}." + ) + + # Missing safety patterns — hard fail for any arm + for ar in arm_results: + for pattern in ar.missing_patterns: + result.failures.append( + f"SAFETY_PATTERN_MISSING: required pattern {pattern!r} absent" + f" in arm '{ar.arm_name}' output." + ) + + # LOC comparison between baseline and each treatment arm + baseline_results = [ar for ar in arm_results if ar.minimality == "off"] + treatment_results = [ar for ar in arm_results if ar.minimality != "off"] + if baseline_results and treatment_results: + baseline_loc = baseline_results[0].loc + for tr in treatment_results: + delta = tr.loc - baseline_loc + if task.is_irreducible: + if abs(delta) > task.irreducible_tolerance: + result.warnings.append( + f"IRREDUCIBLE_SWING: task is marked irreducible but" + f" arm '{tr.arm_name}' LOC differs from baseline by" + f" {abs(delta)} (tolerance {task.irreducible_tolerance})." + ) + elif delta > 0: + result.warnings.append( + f"LOC_INCREASE: arm '{tr.arm_name}' added {delta} LOC" + f" vs baseline ({tr.loc} vs {baseline_loc})." + f" Treatment should produce equal or fewer non-empty lines." + ) + + return result + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + + +def run_minimality_eval( + *, + corpus: tuple[MiniEvalTask, ...] = DEFAULT_CORPUS, + arms: tuple[EvalArm, ...] = DEFAULT_ARMS, + out_path: Path | None = None, +) -> MiniEvalReport: + """Run the minimality A/B benchmark harness. + + Args: + corpus: Fixture tasks to evaluate. Defaults to :data:`DEFAULT_CORPUS`. + arms: Evaluation arms. Defaults to :data:`DEFAULT_ARMS` (off + lite). + out_path: Where to write the JSON report. When *None* the report is + not persisted (useful for in-process testing). + + Returns: + :class:`MiniEvalReport` with per-task results and a summary. + """ + report = MiniEvalReport(arms=[arm.name for arm in arms]) + for task in corpus: + report.tasks.append(_evaluate_task(task, arms)) + + if out_path is not None: + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text( + json.dumps(report.as_dict(), indent=2) + "\n", encoding="utf-8" + ) + + return report + + +def default_run_path(root: Path, iso_timestamp: str) -> Path: + """Return the default output path for a minimality eval run. + + The timestamp must be supplied by the CLI caller (clock-free core). + """ + return root / ".map" / "eval-runs" / "minimality" / f"{iso_timestamp}.json" + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def _main() -> None: + import argparse + import sys + from datetime import datetime, timezone + + parser = argparse.ArgumentParser( + description="MAP minimality A/B benchmark harness", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "--out", + metavar="DIR", + help=( + "Directory to write the JSON report (default: .map/eval-runs/minimality/)." + " The filename is the ISO-8601 timestamp." + ), + ) + parser.add_argument( + "--full", + action="store_true", + help="Add a 'full' treatment arm in addition to 'off' and 'lite'.", + ) + parser.add_argument( + "--show", + action="store_true", + help="Print the report JSON to stdout.", + ) + args = parser.parse_args() + + arms = FULL_ARMS if args.full else DEFAULT_ARMS + iso = ( + datetime.now(timezone.utc) + .replace(microsecond=0) + .isoformat() + .replace("+00:00", "Z") + .replace(":", "") + ) + + if args.out: + out_path = Path(args.out) / f"{iso}.json" + else: + out_path = default_run_path(Path.cwd(), iso) + + report = run_minimality_eval(arms=arms, out_path=out_path) + + if args.show: + print(json.dumps(report.as_dict(), indent=2)) + + summary = report.summary + status = "PASSED" if report.passed else "FAILED" + print( + f"minimality-eval {status}: " + f"{summary['task_pass_count']}/{summary['task_count']} tasks passed, " + f"{summary['failure_count']} failure(s), " + f"{summary['warning_count']} warning(s)." + ) + if out_path.exists(): + print(f"Report: {out_path}") + + sys.exit(0 if report.passed else 1) + + +if __name__ == "__main__": + _main() diff --git a/tests/test_minimality_eval.py b/tests/test_minimality_eval.py new file mode 100644 index 00000000..9b139e5e --- /dev/null +++ b/tests/test_minimality_eval.py @@ -0,0 +1,341 @@ +"""Tests for minimality_eval — deterministic A/B benchmark harness. + +All tests are fixture-based (no live model calls, no external services). +""" + +import json + +from mapify_cli.minimality_eval import ( + DEFAULT_CORPUS, + FULL_ARMS, + EvalArm, + MiniEvalTask, + _DOCTRINE_TAG_CLOSE, + _DOCTRINE_TAG_OPEN, + _loc, + build_doctrine_block, + default_run_path, + run_minimality_eval, +) + + +# --------------------------------------------------------------------------- +# VC1 — build_doctrine_block returns empty string for off arm +# --------------------------------------------------------------------------- + + +def test_vc1_doctrine_block_off_is_empty(): + assert build_doctrine_block("off") == "" + + +# --------------------------------------------------------------------------- +# VC2 — build_doctrine_block contains open/close tags for non-off levels +# --------------------------------------------------------------------------- + + +def test_vc2_doctrine_block_lite_has_tags(): + block = build_doctrine_block("lite") + assert _DOCTRINE_TAG_OPEN in block + assert _DOCTRINE_TAG_CLOSE in block + + +def test_vc2_doctrine_block_full_has_tags(): + block = build_doctrine_block("full") + assert _DOCTRINE_TAG_OPEN in block + assert _DOCTRINE_TAG_CLOSE in block + + +def test_vc2_doctrine_block_ultra_has_tags(): + block = build_doctrine_block("ultra") + assert _DOCTRINE_TAG_OPEN in block + assert _DOCTRINE_TAG_CLOSE in block + + +def test_vc2_doctrine_block_lite_contains_level_line(): + block = build_doctrine_block("lite") + assert "Level: lite" in block + + +def test_vc2_doctrine_block_full_contains_level_line(): + block = build_doctrine_block("full") + assert "Level: full" in block + + +# --------------------------------------------------------------------------- +# VC3 — contamination isolation: off arm must lack doctrine, lite must have it +# --------------------------------------------------------------------------- + + +def test_vc3_off_arm_does_not_have_doctrine(): + block = build_doctrine_block("off") + assert _DOCTRINE_TAG_OPEN not in block + assert _DOCTRINE_TAG_CLOSE not in block + + +def test_vc3_treatment_arm_has_doctrine(): + block = build_doctrine_block("lite") + assert _DOCTRINE_TAG_OPEN in block + + +# --------------------------------------------------------------------------- +# VC4 — default corpus / default arms — all tasks pass +# --------------------------------------------------------------------------- + + +def test_vc4_default_eval_passes(): + report = run_minimality_eval() + assert report.passed, f"default eval should pass; failures={[t.failures for t in report.tasks]}" + + +def test_vc4_default_eval_covers_all_corpus_tasks(): + report = run_minimality_eval() + assert len(report.tasks) == len(DEFAULT_CORPUS) + + +def test_vc4_default_eval_uses_two_arms(): + report = run_minimality_eval() + assert report.arms == ["baseline", "treatment_lite"] + + +# --------------------------------------------------------------------------- +# VC5 — safety patterns must be present in both arm outputs +# --------------------------------------------------------------------------- + + +def test_vc5_safety_guard_task_passes_required_patterns(): + report = run_minimality_eval() + safety = next(t for t in report.tasks if t.task_id == "SAFETY_GUARD") + assert safety.passed, safety.failures + for ar in safety.arm_results: + assert not ar.missing_patterns, ( + f"arm {ar.arm_name} missing patterns: {ar.missing_patterns}" + ) + + +def test_vc5_eval_fails_when_safety_pattern_dropped(): + bad_task = MiniEvalTask( + task_id="BAD_SAFETY", + description="Treatment drops PermissionError guard.", + baseline_code="raise PermissionError('denied')\n", + treatment_code="pass # guard removed\n", + required_patterns=("PermissionError",), + ) + arms = ( + EvalArm(name="baseline", minimality="off"), + EvalArm(name="treatment_lite", minimality="lite"), + ) + report = run_minimality_eval(corpus=(bad_task,), arms=arms) + assert not report.passed + task_result = report.tasks[0] + assert any("SAFETY_PATTERN_MISSING" in f for f in task_result.failures) + + +# --------------------------------------------------------------------------- +# VC6 — contamination detection: fabricated contamination raises FAIL +# --------------------------------------------------------------------------- + + +def test_vc6_contaminated_baseline_raises_failure(): + """A task where both arms produce the same treatment code (baseline contaminated).""" + contaminated_task = MiniEvalTask( + task_id="CONTAMINATED", + description="Baseline and treatment produce identical doctrine-containing output.", + baseline_code="x = 1\n", + treatment_code="x = 1\n", + required_patterns=("x",), + ) + # Create a pair where the baseline arm context is wrongly contaminated. + # We simulate by providing a treatment arm labeled as baseline (minimality=off) + # but with a context that would produce doctrine. We achieve this by + # creating an arm that expects no doctrine but providing the lite arm context + # — the score_arm function uses the arm's minimality to build context, + # so to test contamination we inject the arm directly. + # Instead: use an arm with minimality="lite" named "baseline" to force + # contamination (expects_doctrine=True, but the arm is labeled as non-off). + # Actually the cleanest way is to test with an arm whose minimality is NOT + # "off" but expects doctrine — this should PASS for treatment. + # Let's test the opposite: an arm with minimality="off" but named treatment + # — it should FAIL because the off arm will produce no doctrine but we'd + # need it to have doctrine if expects_doctrine was True. + # + # The real contamination path: EvalArm(minimality="off").expects_doctrine is + # False; build_doctrine_block("off") returns ""; so doctrine_present=False, + # contaminated = (False != False) = False. To trigger contamination we need + # expects_doctrine != doctrine_present. + # + # We inject an arm whose minimality="lite" but name it "baseline" — it + # expects doctrine (expects_doctrine=True) and doctrine_present=True → no + # contamination. Not helpful. + # + # Simplest real test: create an arm with minimality="off" whose baseline_code + # accidentally contains the doctrine tag. But _score_arm checks the context + # block, not the code. So to trigger contamination we need a scenario where + # the arm context is wrongly tagged. + # + # We can test _score_arm directly: + from mapify_cli.minimality_eval import _score_arm + + arm_off = EvalArm(name="baseline", minimality="off") + # Fake context that has been contaminated (contains doctrine tag) + fake_contaminated_ctx = f"{_DOCTRINE_TAG_OPEN}\nLevel: lite\n{_DOCTRINE_TAG_CLOSE}" + result = _score_arm(contaminated_task, arm_off, fake_contaminated_ctx, "x = 1\n") + assert result.contaminated, "off arm with doctrine in context must be marked contaminated" + + +def test_vc6_clean_baseline_not_contaminated(): + from mapify_cli.minimality_eval import _score_arm + + task = MiniEvalTask( + task_id="CLEAN", + description="Clean baseline", + baseline_code="x = 1\n", + treatment_code="x = 1\n", + required_patterns=("x",), + ) + arm_off = EvalArm(name="baseline", minimality="off") + clean_ctx = "" # build_doctrine_block("off") returns "" + result = _score_arm(task, arm_off, clean_ctx, "x = 1\n") + assert not result.contaminated + + +# --------------------------------------------------------------------------- +# VC7 — LOC metric: _loc counts non-empty lines only +# --------------------------------------------------------------------------- + + +def test_vc7_loc_counts_non_empty_lines(): + code = "a = 1\n\nb = 2\n\n\nc = 3\n" + assert _loc(code) == 3 + + +def test_vc7_loc_empty_string_is_zero(): + assert _loc("") == 0 + + +def test_vc7_over_build_trap_treatment_fewer_loc(): + report = run_minimality_eval() + obt = next(t for t in report.tasks if t.task_id == "OVER_BUILD_TRAP") + baseline_ar = next(ar for ar in obt.arm_results if ar.minimality == "off") + treatment_ar = next(ar for ar in obt.arm_results if ar.minimality != "off") + assert treatment_ar.loc < baseline_ar.loc, ( + "minimality treatment should produce fewer LOC than baseline for an over-build trap" + ) + + +# --------------------------------------------------------------------------- +# VC8 — irreducible task: LOC delta within tolerance produces no failure +# --------------------------------------------------------------------------- + + +def test_vc8_irreducible_task_produces_no_failure(): + report = run_minimality_eval() + irreducible = next(t for t in report.tasks if t.task_id == "IRREDUCIBLE") + assert irreducible.passed, irreducible.failures + + +def test_vc8_irreducible_large_swing_produces_warning(): + big_baseline = "x = 1\n" * 20 + small_treatment = "x = 1\n" + swinging_task = MiniEvalTask( + task_id="SWINGING_IRREDUCIBLE", + description="Irreducible task with unexpected large LOC swing.", + baseline_code=big_baseline, + treatment_code=small_treatment, + required_patterns=("x",), + is_irreducible=True, + irreducible_tolerance=2, + ) + arms = ( + EvalArm(name="baseline", minimality="off"), + EvalArm(name="treatment_lite", minimality="lite"), + ) + report = run_minimality_eval(corpus=(swinging_task,), arms=arms) + task_result = report.tasks[0] + assert any("IRREDUCIBLE_SWING" in w for w in task_result.warnings) + assert task_result.passed, "large LOC swing on irreducible is a warning, not a failure" + + +# --------------------------------------------------------------------------- +# VC9 — report persistence: JSON is valid and round-trips +# --------------------------------------------------------------------------- + + +def test_vc9_report_persisted_as_valid_json(tmp_path): + out = tmp_path / "report.json" + report = run_minimality_eval(out_path=out) + assert out.exists() + data = json.loads(out.read_text(encoding="utf-8")) + assert data["schema_version"] == "1.0" + assert data["summary"]["passed"] == report.passed + assert len(data["tasks"]) == len(DEFAULT_CORPUS) + + +def test_vc9_report_arms_listed_correctly(tmp_path): + out = tmp_path / "report.json" + run_minimality_eval(out_path=out) + data = json.loads(out.read_text(encoding="utf-8")) + assert data["arms"] == ["baseline", "treatment_lite"] + + +# --------------------------------------------------------------------------- +# VC10 — no_persist path: run without out_path works fine +# --------------------------------------------------------------------------- + + +def test_vc10_no_persist_returns_report_without_writing(tmp_path): + report = run_minimality_eval(out_path=None) + assert isinstance(report.passed, bool) + # Nothing written + assert not list(tmp_path.iterdir()) + + +# --------------------------------------------------------------------------- +# VC11 — full arm support +# --------------------------------------------------------------------------- + + +def test_vc11_full_arms_include_three_arms(): + report = run_minimality_eval(arms=FULL_ARMS) + assert set(report.arms) == {"baseline", "treatment_lite", "treatment_full"} + + +def test_vc11_full_arm_doctrine_block_has_full_level(): + block = build_doctrine_block("full") + assert "Level: full" in block + + +# --------------------------------------------------------------------------- +# VC12 — default_run_path helper builds expected path +# --------------------------------------------------------------------------- + + +def test_vc12_default_run_path_structure(tmp_path): + path = default_run_path(tmp_path, "20260705T120000Z") + assert path == tmp_path / ".map" / "eval-runs" / "minimality" / "20260705T120000Z.json" + + +# --------------------------------------------------------------------------- +# VC13 — MiniEvalReport.summary separates code-size from safety metrics +# --------------------------------------------------------------------------- + + +def test_vc13_summary_separates_pass_fail_counts(): + report = run_minimality_eval() + s = report.summary + assert "task_pass_count" in s + assert "task_fail_count" in s + assert "warning_count" in s + assert "failure_count" in s + assert s["task_pass_count"] + s["task_fail_count"] == s["task_count"] + + +# --------------------------------------------------------------------------- +# VC14 — as_dict is JSON-serializable (no non-serializable objects) +# --------------------------------------------------------------------------- + + +def test_vc14_as_dict_is_json_serializable(): + report = run_minimality_eval() + raw = json.dumps(report.as_dict()) + parsed = json.loads(raw) + assert parsed["summary"]["task_count"] == len(DEFAULT_CORPUS) From b7aec2cedb722baa7bf95ddce4d7201f682a6a51 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 5 Jul 2026 07:22:34 +0000 Subject: [PATCH 2/2] Add structural-discovery ROI comparison to research-eval (closes #311) Implements `research_eval_compare` module with side-by-side A/B comparison of two ResearchEvidence runs (baseline vs treatment), scoring quality metrics (precision/recall/F1) and exploration-cost metrics (location_count, stale_count, overbroad_count, avg_span) independently so token/LOC reductions cannot mask lower localization quality. - `src/mapify_cli/research_eval_compare.py`: DiscoveryMetrics, ArmScore, CompareReport dataclasses; compare_research_runs() / compare_research_files() public API; default_compare_path(); FIXTURE_* constants for tests. Hard failures: QUALITY_FLOOR (treatment F1 below floor), STALE_REGRESSION (new missing-file paths vs baseline). Advisory warnings: QUALITY_REGRESSION, OVERBROAD_INCREASE. - `tests/test_research_eval_compare.py`: 20 fixture-only tests (VC1-VC11), no live model calls. - `src/mapify_cli/__init__.py`: `mapify research-eval compare` CLI command with --min-file-f1, --min-line-f1, --max-stale-regression, --out flags; exits 0=pass, 1=hard failure, 2=input error. Co-Authored-By: Claude Sonnet 4.6 Claude-Session: https://claude.ai/code/session_01HZ3wHDow49xBGUwWFPH2mD --- src/mapify_cli/__init__.py | 121 +++++++ src/mapify_cli/research_eval_compare.py | 407 ++++++++++++++++++++++++ tests/test_research_eval_compare.py | 402 +++++++++++++++++++++++ 3 files changed, 930 insertions(+) create mode 100644 src/mapify_cli/research_eval_compare.py create mode 100644 tests/test_research_eval_compare.py diff --git a/src/mapify_cli/__init__.py b/src/mapify_cli/__init__.py index 81cca116..4981527c 100644 --- a/src/mapify_cli/__init__.py +++ b/src/mapify_cli/__init__.py @@ -1848,6 +1848,127 @@ def research_eval_score( raise typer.Exit(1) +@research_eval_app.command("compare") +def research_eval_compare( + baseline_file: Path = typer.Argument( + ..., + help="ResearchEvidence JSON/text from the baseline discovery arm (e.g. glob_grep)", + ), + treatment_file: Path = typer.Argument( + ..., + help="ResearchEvidence JSON/text from the treatment discovery arm (e.g. structural-map)", + ), + expected_file: Path = typer.Argument( + ..., + help="JSON list, or object with expected_locations, of target file ranges", + ), + baseline_name: str = typer.Option( + "baseline", + "--baseline-name", + help="Display name for the baseline arm", + ), + treatment_name: str = typer.Option( + "treatment", + "--treatment-name", + help="Display name for the treatment arm", + ), + repo_root: Optional[Path] = typer.Option( + None, + "--repo-root", + help="Fixture repository root for path existence (stale detection) and line validation", + ), + min_treatment_file_f1: float = typer.Option( + 0.0, + "--min-file-f1", + min=0.0, + max=1.0, + help="Hard quality floor on treatment file-level F1 (token-only wins are not enough)", + ), + min_treatment_line_f1: float = typer.Option( + 0.0, + "--min-line-f1", + min=0.0, + max=1.0, + help="Hard quality floor on treatment line-level F1", + ), + max_stale_regression: int = typer.Option( + 0, + "--max-stale-regression", + min=0, + help="Max allowed increase in stale/missing-file locations for treatment vs baseline", + ), + overbroad_line_threshold: int = typer.Option( + 50, + "--overbroad-line-threshold", + min=1, + help="Locations with span above this are counted as over-broad", + ), + no_warn_regression: bool = typer.Option( + False, + "--no-warn-regression", + help="Suppress quality-regression warnings (treatment vs baseline delta)", + ), + out: Optional[Path] = typer.Option( + None, + "--out", + help="Write JSON report to this file (default: stdout only)", + ), +) -> None: + """Compare two ResearchEvidence discovery arms (baseline vs treatment). + + Scores quality (precision/recall/F1) and exploration-cost metrics + (location_count, stale_count, overbroad_count) separately so token/LOC + reduction cannot mask lower evidence quality. + + Exit codes: + 0 - Treatment meets all hard quality floors and stale-regression limits + 1 - Hard failure: quality floor not met or stale regression exceeded + 2 - Input files are missing or malformed + """ + import json + + from mapify_cli.research_eval_compare import compare_research_files + + for label, path in [ + ("baseline", baseline_file), + ("treatment", treatment_file), + ("expected", expected_file), + ]: + if not path.is_file(): + console.print(f"[bold red]Error:[/bold red] {label} file not found: {path}") + raise typer.Exit(2) + + root = repo_root.resolve() if repo_root else None + try: + report = compare_research_files( + baseline_file, + treatment_file, + expected_file, + baseline_name=baseline_name, + treatment_name=treatment_name, + repo_root=root, + overbroad_line_threshold=overbroad_line_threshold, + min_treatment_file_f1=min_treatment_file_f1, + min_treatment_line_f1=min_treatment_line_f1, + max_stale_regression=max_stale_regression, + warn_on_quality_regression=not no_warn_regression, + ) + except (OSError, ValueError) as exc: + console.print(f"[bold red]Error:[/bold red] {exc}") + raise typer.Exit(2) + + payload = report.as_dict() + output_json = json.dumps(payload, indent=2, sort_keys=True) + print(output_json) + + if out is not None: + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(output_json + "\n", encoding="utf-8") + + if not report.passed: + raise typer.Exit(1) + + # Skill-eval commands diff --git a/src/mapify_cli/research_eval_compare.py b/src/mapify_cli/research_eval_compare.py new file mode 100644 index 00000000..65fb1285 --- /dev/null +++ b/src/mapify_cli/research_eval_compare.py @@ -0,0 +1,407 @@ +"""Structural-discovery ROI comparison for MAP research-eval. + +Compares two ResearchEvidence runs (baseline vs treatment) for the same +fixture task, scoring both quality and exploration-cost metrics separately. + +This answers: "Does structural/provider-backed discovery reduce exploration +cost while preserving localization quality?" + +Pass criteria (hard) +-------------------- +- The treatment arm must meet the *quality floor* (file-level and line-level + F1). A treatment that cuts locations by returning fewer but wrong ones is + still a regression. +- The treatment arm must not return more stale/missing-file paths than the + baseline (stale paths contaminate Actor context with phantom evidence). + +Pass criteria (advisory / warn) +-------------------------------- +- Precision or recall regression vs baseline emits a warning even when + the absolute floor is met. +- Overbroad location count increase emits a warning. + +Metrics tracked (separate from quality) +---------------------------------------- +- location_count — total locations returned. +- stale_count — locations whose paths do not exist in repo_root. +- overbroad_count — locations whose line span exceeds overbroad_line_threshold. +- malformed_count — citations that could not be normalized. +- avg_span — average line span across valid locations. + +These exploration-cost metrics are returned alongside quality scores so +operators can verify that token/context reduction does not come at the +expense of evidence quality. +""" + +from __future__ import annotations + +import json +from collections.abc import Sequence +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from mapify_cli.research_eval import ( + ResearchLocation, + ResearchLocalizationScore, + load_expected_locations, + parse_research_locations, + score_research_output, +) + + +# --------------------------------------------------------------------------- +# Data structures +# --------------------------------------------------------------------------- + + +@dataclass +class DiscoveryMetrics: + """Exploration-cost metrics for one ResearchEvidence arm. + + These are independent of quality (precision/recall/F1) and should be + reported alongside quality scores to prevent token-only wins. + """ + + location_count: int + stale_count: int + overbroad_count: int + malformed_count: int + avg_span: float + + def as_dict(self) -> dict[str, Any]: + return { + "location_count": self.location_count, + "stale_count": self.stale_count, + "overbroad_count": self.overbroad_count, + "malformed_count": self.malformed_count, + "avg_span": round(self.avg_span, 2), + } + + +@dataclass +class ArmScore: + """Combined quality + exploration-cost score for one eval arm.""" + + arm_name: str + quality: ResearchLocalizationScore + discovery: DiscoveryMetrics + + def as_dict(self) -> dict[str, Any]: + from mapify_cli.research_eval import score_to_dict + + return { + "arm_name": self.arm_name, + "quality": score_to_dict(self.quality), + "discovery": self.discovery.as_dict(), + } + + +@dataclass +class CompareReport: + """Side-by-side comparison of baseline and treatment arms.""" + + baseline: ArmScore + treatment: ArmScore + warnings: list[str] = field(default_factory=list) + failures: list[str] = field(default_factory=list) + + @property + def passed(self) -> bool: + return not self.failures + + def as_dict(self) -> dict[str, Any]: + return { + "schema_version": "1.0", + "passed": self.passed, + "warnings": self.warnings, + "failures": self.failures, + "baseline": self.baseline.as_dict(), + "treatment": self.treatment.as_dict(), + "deltas": self._deltas(), + } + + def _deltas(self) -> dict[str, Any]: + bq = self.baseline.quality + tq = self.treatment.quality + bd = self.baseline.discovery + td = self.treatment.discovery + return { + "file_f1_delta": round(tq.file_level.f1 - bq.file_level.f1, 4), + "line_f1_delta": round(tq.line_level.f1 - bq.line_level.f1, 4), + "location_count_delta": td.location_count - bd.location_count, + "stale_count_delta": td.stale_count - bd.stale_count, + "overbroad_count_delta": td.overbroad_count - bd.overbroad_count, + "avg_span_delta": round(td.avg_span - bd.avg_span, 2), + } + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _discovery_metrics( + locations: Sequence[ResearchLocation], + malformed_count: int, + overbroad_line_threshold: int, + repo_root: Path | None, +) -> DiscoveryMetrics: + stale = 0 + overbroad = 0 + total_span = 0 + for loc in locations: + span = loc.end_line - loc.start_line + 1 + total_span += span + if span > overbroad_line_threshold: + overbroad += 1 + if repo_root is not None: + candidate = repo_root / Path(*loc.path.split("/")) + if not candidate.is_file(): + stale += 1 + avg_span = total_span / len(locations) if locations else 0.0 + return DiscoveryMetrics( + location_count=len(locations), + stale_count=stale, + overbroad_count=overbroad, + malformed_count=malformed_count, + avg_span=avg_span, + ) + + +def _score_arm( + arm_name: str, + output_text: str, + expected: list[dict[str, Any]], + *, + repo_root: Path | None, + overbroad_line_threshold: int, +) -> ArmScore: + # Parse without repo_root to obtain ALL raw locations (existence filtering + # happens inside parse when repo_root is set, turning missing files into + # malformed entries — we need the raw list for stale counting). + raw_parsed = parse_research_locations(output_text, repo_root=None) + quality = score_research_output( + output_text, + expected, + repo_root=repo_root, + overbroad_line_threshold=overbroad_line_threshold, + ) + discovery = _discovery_metrics( + raw_parsed.locations, + malformed_count=len(raw_parsed.malformed), + overbroad_line_threshold=overbroad_line_threshold, + repo_root=repo_root, + ) + return ArmScore(arm_name=arm_name, quality=quality, discovery=discovery) + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + + +def compare_research_runs( + baseline_output: str, + treatment_output: str, + expected_locations: list[dict[str, Any]], + *, + baseline_name: str = "baseline", + treatment_name: str = "treatment", + repo_root: Path | None = None, + overbroad_line_threshold: int = 50, + min_treatment_file_f1: float = 0.0, + min_treatment_line_f1: float = 0.0, + max_stale_regression: int = 0, + warn_on_quality_regression: bool = True, +) -> CompareReport: + """Compare two ResearchEvidence runs side-by-side. + + Quality metrics (precision/recall/F1) and exploration-cost metrics + (location_count, stale_count, overbroad_count) are scored separately. + + Args: + baseline_output: ResearchEvidence text/JSON from the baseline arm + (e.g. glob_grep discovery). + treatment_output: ResearchEvidence text/JSON from the treatment arm + (e.g. structural-map/code-graph discovery). + expected_locations: List of expected file/line-range dicts. + baseline_name: Display name for the baseline arm. + treatment_name: Display name for the treatment arm. + repo_root: Optional repo root for path existence checks (stale + detection) and line-range validation. + overbroad_line_threshold: Location line spans above this are counted + as over-broad. + min_treatment_file_f1: Hard floor on treatment file-level F1. A + treatment that fails this floor triggers a FAIL even if it + returns fewer locations. + min_treatment_line_f1: Hard floor on treatment line-level F1. + max_stale_regression: Maximum increase in stale-path count allowed + for the treatment arm relative to the baseline. 0 means the + treatment must not introduce new stale paths. + warn_on_quality_regression: Emit a warning when treatment F1 drops + below baseline F1 (even if the absolute floor is met). + + Returns: + :class:`CompareReport` with per-arm scores, deltas, warnings, and + failures. ``report.passed`` is ``True`` only when no hard failures + occurred. + """ + baseline = _score_arm( + baseline_name, + baseline_output, + expected_locations, + repo_root=repo_root, + overbroad_line_threshold=overbroad_line_threshold, + ) + treatment = _score_arm( + treatment_name, + treatment_output, + expected_locations, + repo_root=repo_root, + overbroad_line_threshold=overbroad_line_threshold, + ) + + report = CompareReport(baseline=baseline, treatment=treatment) + + # Hard failure: treatment quality floor + if treatment.quality.file_level.f1 < min_treatment_file_f1: + report.failures.append( + f"QUALITY_FLOOR: treatment file-level F1 " + f"{treatment.quality.file_level.f1:.3f} " + f"< floor {min_treatment_file_f1:.3f}. " + "Token/LOC reduction cannot compensate for lower precision/recall." + ) + if treatment.quality.line_level.f1 < min_treatment_line_f1: + report.failures.append( + f"QUALITY_FLOOR: treatment line-level F1 " + f"{treatment.quality.line_level.f1:.3f} " + f"< floor {min_treatment_line_f1:.3f}. " + "Token/LOC reduction cannot compensate for lower precision/recall." + ) + + # Hard failure: stale regression + stale_delta = treatment.discovery.stale_count - baseline.discovery.stale_count + if stale_delta > max_stale_regression: + report.failures.append( + f"STALE_REGRESSION: treatment introduced {stale_delta} additional " + f"stale/missing-path location(s) vs baseline " + f"({treatment.discovery.stale_count} vs {baseline.discovery.stale_count}). " + "Stale paths contaminate Actor context with phantom evidence." + ) + + # Advisory warnings: quality regression vs baseline + if warn_on_quality_regression: + file_delta = treatment.quality.file_level.f1 - baseline.quality.file_level.f1 + if file_delta < 0: + report.warnings.append( + f"QUALITY_REGRESSION: treatment file-level F1 dropped by " + f"{abs(file_delta):.3f} vs baseline " + f"({treatment.quality.file_level.f1:.3f} vs " + f"{baseline.quality.file_level.f1:.3f})." + ) + line_delta = treatment.quality.line_level.f1 - baseline.quality.line_level.f1 + if line_delta < 0: + report.warnings.append( + f"QUALITY_REGRESSION: treatment line-level F1 dropped by " + f"{abs(line_delta):.3f} vs baseline " + f"({treatment.quality.line_level.f1:.3f} vs " + f"{baseline.quality.line_level.f1:.3f})." + ) + + # Advisory: overbroad regression + overbroad_delta = ( + treatment.discovery.overbroad_count - baseline.discovery.overbroad_count + ) + if overbroad_delta > 0: + report.warnings.append( + f"OVERBROAD_INCREASE: treatment returned {overbroad_delta} more " + f"over-broad location(s) vs baseline " + f"({treatment.discovery.overbroad_count} vs " + f"{baseline.discovery.overbroad_count})." + ) + + return report + + +def compare_research_files( + baseline_path: Path, + treatment_path: Path, + expected_path: Path, + *, + baseline_name: str = "baseline", + treatment_name: str = "treatment", + repo_root: Path | None = None, + overbroad_line_threshold: int = 50, + min_treatment_file_f1: float = 0.0, + min_treatment_line_f1: float = 0.0, + max_stale_regression: int = 0, + warn_on_quality_regression: bool = True, +) -> CompareReport: + """Compare two ResearchEvidence files and expected locations from disk. + + Convenience wrapper around :func:`compare_research_runs` that handles + file I/O and expected-location loading. + """ + baseline_output = baseline_path.read_text(encoding="utf-8") + treatment_output = treatment_path.read_text(encoding="utf-8") + expected = load_expected_locations(expected_path) + return compare_research_runs( + baseline_output, + treatment_output, + expected, + baseline_name=baseline_name, + treatment_name=treatment_name, + repo_root=repo_root, + overbroad_line_threshold=overbroad_line_threshold, + min_treatment_file_f1=min_treatment_file_f1, + min_treatment_line_f1=min_treatment_line_f1, + max_stale_regression=max_stale_regression, + warn_on_quality_regression=warn_on_quality_regression, + ) + + +def default_compare_path(root: Path, iso_timestamp: str) -> Path: + """Return the default output path for a research-eval comparison run. + + The timestamp must be supplied by the CLI caller (clock-free core). + """ + return ( + root / ".map" / "eval-runs" / "research-compare" / f"{iso_timestamp}.json" + ) + + +# --------------------------------------------------------------------------- +# Fixture corpus helpers +# --------------------------------------------------------------------------- + +#: Minimal fixture ResearchEvidence for tests. Both fixtures cover the +#: same expected locations with the same quality so comparisons produce +#: deterministic deltas. Tests inject different location counts to +#: exercise exploration-cost metrics. + +FIXTURE_EXPECTED: list[dict[str, Any]] = [ + {"path": "src/mapify_cli/research_eval.py", "lines": [85, 128]}, + {"path": "src/mapify_cli/research_eval.py", "lines": [131, 147]}, +] + + +def _make_evidence_json(locations: list[dict[str, Any]]) -> str: + return json.dumps({"relevant_locations": locations}) + + +FIXTURE_BASELINE_OUTPUT: str = _make_evidence_json( + [ + {"path": "src/mapify_cli/research_eval.py", "lines": [85, 128]}, + {"path": "src/mapify_cli/research_eval.py", "lines": [131, 147]}, + {"path": "src/mapify_cli/research_eval.py", "lines": [1, 50]}, + {"path": "src/mapify_cli/research_eval.py", "lines": [51, 83]}, + ] +) + +FIXTURE_TREATMENT_OUTPUT: str = _make_evidence_json( + [ + {"path": "src/mapify_cli/research_eval.py", "lines": [85, 128]}, + {"path": "src/mapify_cli/research_eval.py", "lines": [131, 147]}, + ] +) diff --git a/tests/test_research_eval_compare.py b/tests/test_research_eval_compare.py new file mode 100644 index 00000000..5b14f4d4 --- /dev/null +++ b/tests/test_research_eval_compare.py @@ -0,0 +1,402 @@ +"""Tests for research_eval_compare — structural-discovery ROI comparison. + +All tests are fixture-based (no live model calls, no external services). +Fixture data uses pre-defined ResearchEvidence JSON strings. +""" + +import json + +from typer.testing import CliRunner + +from mapify_cli import app +from mapify_cli.research_eval_compare import ( + FIXTURE_BASELINE_OUTPUT, + FIXTURE_EXPECTED, + FIXTURE_TREATMENT_OUTPUT, + DiscoveryMetrics, + compare_research_runs, + default_compare_path, +) + +runner = CliRunner() + + +# --------------------------------------------------------------------------- +# VC1 — compare_research_runs: clean comparison with perfect arms passes +# --------------------------------------------------------------------------- + + +def test_vc1_compare_perfect_arms_passes(): + perfect = json.dumps( + { + "relevant_locations": [ + {"path": "src/mapify_cli/research_eval.py", "lines": [85, 128]}, + {"path": "src/mapify_cli/research_eval.py", "lines": [131, 147]}, + ] + } + ) + report = compare_research_runs(perfect, perfect, FIXTURE_EXPECTED) + assert report.passed, report.failures + + +def test_vc1_compare_fixture_default_passes(): + report = compare_research_runs( + FIXTURE_BASELINE_OUTPUT, FIXTURE_TREATMENT_OUTPUT, FIXTURE_EXPECTED + ) + assert report.passed, report.failures + + +# --------------------------------------------------------------------------- +# VC2 — quality floor: treatment below floor causes FAIL +# --------------------------------------------------------------------------- + + +def test_vc2_quality_floor_fail_on_empty_treatment(): + empty_treatment = json.dumps({"relevant_locations": []}) + report = compare_research_runs( + FIXTURE_BASELINE_OUTPUT, + empty_treatment, + FIXTURE_EXPECTED, + min_treatment_file_f1=0.5, + min_treatment_line_f1=0.5, + ) + assert not report.passed + assert any("QUALITY_FLOOR" in f for f in report.failures) + + +def test_vc2_quality_floor_pass_when_met(): + exact_match = json.dumps( + { + "relevant_locations": [ + {"path": "src/mapify_cli/research_eval.py", "lines": [85, 128]}, + {"path": "src/mapify_cli/research_eval.py", "lines": [131, 147]}, + ] + } + ) + report = compare_research_runs( + FIXTURE_BASELINE_OUTPUT, + exact_match, + FIXTURE_EXPECTED, + min_treatment_file_f1=1.0, + min_treatment_line_f1=1.0, + ) + assert report.passed, report.failures + + +# --------------------------------------------------------------------------- +# VC3 — stale path detection: new stale paths in treatment cause FAIL +# --------------------------------------------------------------------------- + + +def test_vc3_stale_regression_causes_fail(tmp_path): + stale_treatment = json.dumps( + { + "relevant_locations": [ + {"path": "src/mapify_cli/research_eval.py", "lines": [85, 128]}, + {"path": "nonexistent/phantom_file.py", "lines": [1, 10]}, + ] + } + ) + # Write a repo_root that has research_eval.py but not phantom_file.py + (tmp_path / "src" / "mapify_cli").mkdir(parents=True) + (tmp_path / "src" / "mapify_cli" / "research_eval.py").write_text( + "# fake\n" * 200, encoding="utf-8" + ) + + report = compare_research_runs( + FIXTURE_BASELINE_OUTPUT, + stale_treatment, + FIXTURE_EXPECTED, + repo_root=tmp_path, + max_stale_regression=0, + ) + assert not report.passed + assert any("STALE_REGRESSION" in f for f in report.failures) + + +def test_vc3_stale_regression_within_limit_passes(tmp_path): + stale_treatment = json.dumps( + { + "relevant_locations": [ + {"path": "src/mapify_cli/research_eval.py", "lines": [85, 128]}, + {"path": "nonexistent/phantom_file.py", "lines": [1, 10]}, + ] + } + ) + (tmp_path / "src" / "mapify_cli").mkdir(parents=True) + (tmp_path / "src" / "mapify_cli" / "research_eval.py").write_text( + "# fake\n" * 200, encoding="utf-8" + ) + report = compare_research_runs( + FIXTURE_BASELINE_OUTPUT, + stale_treatment, + FIXTURE_EXPECTED, + repo_root=tmp_path, + max_stale_regression=2, # allow up to 2 new stale paths + ) + assert report.passed, report.failures + + +# --------------------------------------------------------------------------- +# VC4 — quality regression warning: treatment worse than baseline emits WARN +# --------------------------------------------------------------------------- + + +def test_vc4_quality_regression_emits_warning(): + worse_treatment = json.dumps({"relevant_locations": []}) + report = compare_research_runs( + FIXTURE_BASELINE_OUTPUT, + worse_treatment, + FIXTURE_EXPECTED, + warn_on_quality_regression=True, + ) + assert any("QUALITY_REGRESSION" in w for w in report.warnings) + + +def test_vc4_no_regression_warning_when_suppressed(): + worse_treatment = json.dumps({"relevant_locations": []}) + report = compare_research_runs( + FIXTURE_BASELINE_OUTPUT, + worse_treatment, + FIXTURE_EXPECTED, + warn_on_quality_regression=False, + ) + assert not any("QUALITY_REGRESSION" in w for w in report.warnings) + + +# --------------------------------------------------------------------------- +# VC5 — exploration-cost metrics: location_count, overbroad, avg_span +# --------------------------------------------------------------------------- + + +def test_vc5_treatment_fewer_locations(): + report = compare_research_runs( + FIXTURE_BASELINE_OUTPUT, FIXTURE_TREATMENT_OUTPUT, FIXTURE_EXPECTED + ) + # baseline has 4 locations, treatment has 2 + assert report.baseline.discovery.location_count == 4 + assert report.treatment.discovery.location_count == 2 + assert report.as_dict()["deltas"]["location_count_delta"] == -2 + + +def test_vc5_overbroad_increase_emits_warning(): + overbroad_treatment = json.dumps( + { + "relevant_locations": [ + {"path": "src/mapify_cli/research_eval.py", "lines": [1, 200]}, + ] + } + ) + clean_baseline = json.dumps( + { + "relevant_locations": [ + {"path": "src/mapify_cli/research_eval.py", "lines": [85, 100]}, + ] + } + ) + report = compare_research_runs( + clean_baseline, + overbroad_treatment, + FIXTURE_EXPECTED, + overbroad_line_threshold=50, + ) + assert any("OVERBROAD_INCREASE" in w for w in report.warnings) + + +def test_vc5_discovery_metrics_avg_span_computed(): + output = json.dumps( + { + "relevant_locations": [ + {"path": "src/mapify_cli/research_eval.py", "lines": [1, 10]}, + {"path": "src/mapify_cli/research_eval.py", "lines": [20, 30]}, + ] + } + ) + report = compare_research_runs(output, output, FIXTURE_EXPECTED) + # [1,10] → span 10; [20,30] → span 11; avg = 10.5 + assert report.baseline.discovery.avg_span == 10.5 + + +# --------------------------------------------------------------------------- +# VC6 — as_dict schema: both quality and exploration-cost metrics present +# --------------------------------------------------------------------------- + + +def test_vc6_as_dict_contains_quality_and_discovery(): + report = compare_research_runs( + FIXTURE_BASELINE_OUTPUT, FIXTURE_TREATMENT_OUTPUT, FIXTURE_EXPECTED + ) + d = report.as_dict() + assert "quality" in d["baseline"] + assert "discovery" in d["baseline"] + assert "quality" in d["treatment"] + assert "discovery" in d["treatment"] + assert "deltas" in d + + +def test_vc6_as_dict_json_serializable(): + report = compare_research_runs( + FIXTURE_BASELINE_OUTPUT, FIXTURE_TREATMENT_OUTPUT, FIXTURE_EXPECTED + ) + raw = json.dumps(report.as_dict()) + parsed = json.loads(raw) + assert parsed["schema_version"] == "1.0" + + +# --------------------------------------------------------------------------- +# VC7 — deltas: file_f1_delta and line_f1_delta in report +# --------------------------------------------------------------------------- + + +def test_vc7_deltas_reported(): + report = compare_research_runs( + FIXTURE_BASELINE_OUTPUT, FIXTURE_TREATMENT_OUTPUT, FIXTURE_EXPECTED + ) + deltas = report.as_dict()["deltas"] + assert "file_f1_delta" in deltas + assert "line_f1_delta" in deltas + assert "location_count_delta" in deltas + assert "stale_count_delta" in deltas + assert "avg_span_delta" in deltas + + +# --------------------------------------------------------------------------- +# VC8 — DiscoveryMetrics.as_dict has expected keys +# --------------------------------------------------------------------------- + + +def test_vc8_discovery_metrics_as_dict_keys(): + dm = DiscoveryMetrics( + location_count=3, + stale_count=1, + overbroad_count=0, + malformed_count=0, + avg_span=12.5, + ) + d = dm.as_dict() + assert set(d) == { + "location_count", + "stale_count", + "overbroad_count", + "malformed_count", + "avg_span", + } + + +# --------------------------------------------------------------------------- +# VC9 — default_compare_path helper +# --------------------------------------------------------------------------- + + +def test_vc9_default_compare_path_structure(tmp_path): + path = default_compare_path(tmp_path, "20260705T120000Z") + assert path == ( + tmp_path / ".map" / "eval-runs" / "research-compare" / "20260705T120000Z.json" + ) + + +# --------------------------------------------------------------------------- +# VC10 — CLI: mapify research-eval compare exits 0 on clean comparison +# --------------------------------------------------------------------------- + + +def test_vc10_cli_compare_passes(tmp_path): + baseline = tmp_path / "baseline.json" + treatment = tmp_path / "treatment.json" + expected = tmp_path / "expected.json" + + baseline.write_text(FIXTURE_BASELINE_OUTPUT, encoding="utf-8") + treatment.write_text(FIXTURE_TREATMENT_OUTPUT, encoding="utf-8") + expected.write_text( + json.dumps(FIXTURE_EXPECTED), encoding="utf-8" + ) + + result = runner.invoke( + app, + [ + "research-eval", + "compare", + str(baseline), + str(treatment), + str(expected), + ], + ) + assert result.exit_code == 0, result.output + payload = json.loads(result.output) + assert payload["passed"] is True + + +def test_vc10_cli_compare_fails_below_quality_floor(tmp_path): + baseline = tmp_path / "baseline.json" + empty_treatment = tmp_path / "treatment.json" + expected = tmp_path / "expected.json" + + baseline.write_text(FIXTURE_BASELINE_OUTPUT, encoding="utf-8") + empty_treatment.write_text( + json.dumps({"relevant_locations": []}), encoding="utf-8" + ) + expected.write_text(json.dumps(FIXTURE_EXPECTED), encoding="utf-8") + + result = runner.invoke( + app, + [ + "research-eval", + "compare", + str(baseline), + str(empty_treatment), + str(expected), + "--min-file-f1", + "0.9", + ], + ) + assert result.exit_code == 1 + payload = json.loads(result.output) + assert not payload["passed"] + assert any("QUALITY_FLOOR" in f for f in payload["failures"]) + + +def test_vc10_cli_compare_missing_file_exits_2(tmp_path): + result = runner.invoke( + app, + [ + "research-eval", + "compare", + str(tmp_path / "no_baseline.json"), + str(tmp_path / "no_treatment.json"), + str(tmp_path / "no_expected.json"), + ], + ) + assert result.exit_code == 2 + + +# --------------------------------------------------------------------------- +# VC11 — CLI: --out persists report to file +# --------------------------------------------------------------------------- + + +def test_vc11_cli_compare_out_flag_persists_report(tmp_path): + baseline = tmp_path / "baseline.json" + treatment = tmp_path / "treatment.json" + expected = tmp_path / "expected.json" + out_file = tmp_path / "report.json" + + baseline.write_text(FIXTURE_BASELINE_OUTPUT, encoding="utf-8") + treatment.write_text(FIXTURE_TREATMENT_OUTPUT, encoding="utf-8") + expected.write_text(json.dumps(FIXTURE_EXPECTED), encoding="utf-8") + + result = runner.invoke( + app, + [ + "research-eval", + "compare", + str(baseline), + str(treatment), + str(expected), + "--out", + str(out_file), + ], + ) + assert result.exit_code == 0 + assert out_file.exists() + data = json.loads(out_file.read_text(encoding="utf-8")) + assert data["schema_version"] == "1.0"