From 73de90504f1852d8fbe13da7e438b7765b1992df Mon Sep 17 00:00:00 2001 From: Kavya Sree Kaitepalli Date: Tue, 25 Aug 2026 09:55:26 +0000 Subject: [PATCH 01/13] feat: implement evaluation task framework with feedback mechanism --- src/microbots/auto_memory/__init__.py | 8 + src/microbots/auto_memory/analyzer.py | 65 ++++++++ src/microbots/auto_memory/orchestrator.py | 124 ++++++++++++++ src/microbots/auto_memory/task.py | 189 ++++++++++++++++++++++ 4 files changed, 386 insertions(+) create mode 100644 src/microbots/auto_memory/__init__.py create mode 100644 src/microbots/auto_memory/analyzer.py create mode 100644 src/microbots/auto_memory/orchestrator.py create mode 100644 src/microbots/auto_memory/task.py diff --git a/src/microbots/auto_memory/__init__.py b/src/microbots/auto_memory/__init__.py new file mode 100644 index 0000000..27f9a4b --- /dev/null +++ b/src/microbots/auto_memory/__init__.py @@ -0,0 +1,8 @@ +"""Train <-> eval loop for repo-learning agents. + +Re-exports the public task, outcome, and orchestrator types used to define +an evaluation task and run it in a loop against a training agent. +""" + +from .task import CallbackResult, EvalOutcome, EvalTask +from .orchestrator import LoopResult, run_train_eval_loop \ No newline at end of file diff --git a/src/microbots/auto_memory/analyzer.py b/src/microbots/auto_memory/analyzer.py new file mode 100644 index 0000000..2589390 --- /dev/null +++ b/src/microbots/auto_memory/analyzer.py @@ -0,0 +1,65 @@ +"""Build feedback text for a failed evaluation round. + +Uses ``LogAnalysisBot`` to analyze the eval callback's raw log and produce +concrete feedback describing what went wrong, to be passed into the +training agent as ``feedback`` for the next round. +""" + +from logging import getLogger + +from microbots.auto_memory.task import EvalOutcome, EvalTask +from microbots.bot.LogAnalysisBot import LogAnalysisBot +from microbots.MicroBot import BotRunResult + +logger = getLogger(__name__) + +def build_feedback( + task: EvalTask, + outcome: EvalOutcome, + repo_path: str, + model: str, +) -> str: + """Analyze a failed eval outcome's log and produce training feedback. + + Parameters + ---------- + task : EvalTask + The eval task that produced ``outcome``. + outcome : EvalOutcome + The failed outcome to analyze, including its ``log_path``. + repo_path : str + Absolute path to the repo the task was evaluated against. + model : str + The model to use, in the format ``/``. + + Returns + ------- + str + Feedback text describing the root cause of the failure and what + the agent's memory notes should cover next time, suitable for + passing as ``feedback`` to ``run_training``. + """ + bot = LogAnalysisBot(model=model, folder_to_mount=repo_path) + result: BotRunResult = bot.run( + file_name=outcome.log_path, + user_prompt=( + "This log was produced while verifying whether an " + "automated agent completed its task correctly. Identify " + "the root cause of the failure and describe concretely " + "what the agent's memory notes should cover next time to " + "avoid this failure." + ), + ) + + if result.status and result.result: + return result.result + + logger.warning( + "LogAnalysisBot failed to analyze failure (%s); falling back to plain feedback", + result.error, + ) + return ( + f"Evaluation failed. Agent output: {outcome.output}\n" + f"Callback reason: {outcome.result.reason}" + ) + \ No newline at end of file diff --git a/src/microbots/auto_memory/orchestrator.py b/src/microbots/auto_memory/orchestrator.py new file mode 100644 index 0000000..7558a51 --- /dev/null +++ b/src/microbots/auto_memory/orchestrator.py @@ -0,0 +1,124 @@ +"""Orchestrates the train <-> eval loop for repo-learning agents. + +Repeatedly runs an ``EvalTask`` against a repo, and on failure builds +feedback and retrains via the training agent, looping until the task +passes or ``max_rounds`` is exhausted. +""" + +from dataclasses import dataclass, field +from logging import getLogger +from pathlib import Path + +from microbots.auto_memory.analyzer import build_feedback +from microbots.auto_memory.task import EvalOutcome, EvalTask +from microbots.auto_memory.training.runner import run_training + +logger = getLogger(__name__) + +@dataclass +class LoopResult: + """Result of running ``run_train_eval_loop``. + + Attributes + ---------- + passed : bool + Whether the task passed within ``max_rounds``. + rounds_run : int + Number of eval rounds actually run. + final_outcome : EvalOutcome + The outcome of the last round run. + outcomes : list[EvalOutcome] + The outcome of every round run, in order. + """ + + passed: bool + rounds_run: int + final_outcome: EvalOutcome + outcomes: list[EvalOutcome] = field(default_factory=list) + +def run_train_eval_loop( + repo_path: str, + memory_dir: str, + model: str, + task: EvalTask, + max_rounds: int = 5, +) -> LoopResult: + """Run an eval task in a loop, retraining on failure until it passes. + + Each round runs ``task.run(...)``. If the task passes, the loop + returns immediately. If it fails, feedback is built from the round's + log and used to retrain via ``run_training`` before the next round. + The round's log file is always deleted before the next round starts. + + Parameters + ---------- + repo_path : str + Absolute path to the repo to evaluate and train against. + memory_dir : str + Directory where the training agent reads/writes memory files. + model : str + The model to use, in the format ``/``. + task : EvalTask + The eval task to run each round. + max_rounds : int + Maximum number of train/eval rounds to attempt. Defaults to 5. + + Returns + ------- + LoopResult + Whether the task passed, how many rounds ran, and every round's + outcome. + """ + outcomes: list[EvalOutcome] = [] + + for round_idx in range(max_rounds): + logger.info( + "run_train_eval_loop: round %d/%d starting", round_idx + 1, max_rounds + ) + outcome = task.run(repo_path, memory_dir, model) + outcomes.append(outcome) + + try: + if outcome.passed: + logger.info( + "run_train_eval_loop: passed on round %d/%d", round_idx + 1, max_rounds + ) + return LoopResult( + passed=True, + rounds_run=round_idx + 1, + final_outcome=outcome, + outcomes=outcomes, + ) + + logger.info( + "run_train_eval_loop: round %d failed (%s), retraining", + round_idx + 1, + outcome.result.reason, + ) + try: + feedback = build_feedback(task, outcome, repo_path, model) + + run_training( + repo_path=repo_path, + feedback=feedback, + memory_dir=memory_dir, + model=model, + ) + except Exception: + logger.exception( + "run_train_eval_loop: round %d failed to build feedback/retrain; " + "continuing to next round without retraining", + round_idx + 1, + ) + finally: + Path(outcome.log_path).unlink(missing_ok=True) + + logger.info( + "run_train_eval_loop: exhausted %d rounds without passing", max_rounds + ) + return LoopResult( + passed=False, + rounds_run=max_rounds, + final_outcome=outcomes[-1], + outcomes=outcomes, + ) \ No newline at end of file diff --git a/src/microbots/auto_memory/task.py b/src/microbots/auto_memory/task.py new file mode 100644 index 0000000..e0e8fef --- /dev/null +++ b/src/microbots/auto_memory/task.py @@ -0,0 +1,189 @@ +"""Defines the abstract eval task interface for the train <-> eval loop. + +An ``EvalTask`` describes one unit of work: how to prepare a repo, what +prompt to give the agent, how to verify the agent's output, and how to +clean up afterward. +""" + +import tempfile +from abc import ABC, abstractmethod +from dataclasses import dataclass +from logging import getLogger +from pathlib import Path + +from microbots.bot.WritingBot import WritingBot +from microbots.tools.MemoryTool import MemoryTool + +logger = getLogger(__name__) + +@dataclass +class CallbackResult: + """Result of verifying whether an eval task was completed correctly. + + Attributes + ---------- + passed : bool + Whether the agent's output satisfies the task's check. + reason : str + A short human-readable explanation of the pass/fail verdict. + """ + + passed: bool + reason: str + +@dataclass +class EvalOutcome: + """Full record of one eval round. + + Attributes + ---------- + passed : bool + Whether the round passed, mirrors ``result.passed``. + output : str | None + The agent's raw output for the round, if any. + result : CallbackResult + The verdict produced by ``EvalTask.check``. + log_path : str + Path to the round's log file, containing the agent output and + any failure/exception details recorded during the round. + """ + + passed: bool + output: str | None + result: CallbackResult + log_path: str + + +class EvalTask(ABC): + """Base class for a single evaluation task in the train <-> eval loop. + + Subclasses must implement ``build_prompt`` and ``check``, and may + override ``setup``, ``teardown``, and ``run`` as needed. + """ + + def setup(self, repo_path: str) -> None: + """Optional. Prepare repo/environment before the agent runs. + + Parameters + ---------- + repo_path : str + Absolute path to the repo to prepare. + """ + pass + + @abstractmethod + def build_prompt(self, repo_path: str) -> str: + """Required. Return the task prompt/instructions for the agent. + + Parameters + ---------- + repo_path : str + Absolute path to the repo the agent will operate on. + + Returns + ------- + str + The prompt/instructions to give the agent. + """ + + @abstractmethod + def check(self, repo_path: str, agent_output: str, log_path: str) -> CallbackResult: + """Required. Verify whether the task was actually completed correctly. + + Parameters + ---------- + repo_path : str + Absolute path to the repo the agent operated on. + agent_output : str + The agent's raw output/result text. + log_path : str + Path to a log file, already created by ``run``, that this + check may append verification details to. + + Returns + ------- + CallbackResult + The pass/fail verdict and its reason. + """ + + def teardown(self, repo_path: str) -> None: + """Optional. Clean up anything setup() created. + + Parameters + ---------- + repo_path : str + Absolute path to the repo that was prepared by ``setup``. + """ + pass + + def run(self, repo_path: str, memory_dir: str, model: str) -> EvalOutcome: + """Default eval iteration: setup -> build_prompt -> WritingBot -> check -> teardown. + Override this entirely if your task needs a different bot type, + additional tools, or custom retry/orchestration logic. + + Parameters + ---------- + repo_path : str + Absolute path to the repo to run the eval round against. + memory_dir : str + Directory containing memory files to give the agent via + ``MemoryTool``. + model : str + The model to use, in the format ``/``. + + Returns + ------- + EvalOutcome + The result of this eval round, including the agent's output, + the check verdict, and the round's log file path. + """ + self.setup(repo_path) + log_path = tempfile.mktemp(suffix=".log") + Path(log_path).write_text("") + + try: + try: + prompt = self.build_prompt(repo_path) + bot = WritingBot( + model=model, + folder_to_mount=repo_path, + additional_tools=[MemoryTool(memory_dir=memory_dir)], + ) + bot_result = bot.run(prompt) + + with open(log_path, "a") as f: + f.write(f"Agent output:\n{bot_result.result}\n") + + if not bot_result.status: + reason = f"Bot run failed: {bot_result.error}" + with open(log_path, "a") as f: + f.write(f"\n{reason}\n") + result = CallbackResult(passed=False, reason=reason) + else: + result = self.check(repo_path, bot_result.result or "", log_path) + + return EvalOutcome( + passed=result.passed, + output=bot_result.result, + result=result, + log_path=log_path, + ) + except Exception as exc: + logger.exception( + "EvalTask.run: iteration raised %s", type(exc).__name__ + ) + with open(log_path, "a") as f: + f.write(f"\nException during eval iteration: {type(exc).__name__}: {exc}\n") + return EvalOutcome( + passed=False, + output=None, + result=CallbackResult( + passed=False, reason=f"{type(exc).__name__}: {exc}" + ), + log_path=log_path, + ) + finally: + try: + self.teardown(repo_path) + except Exception: + logger.exception("EvalTask.run: teardown() raised exception; ignoring") From c9d7b9b27940fff5f4fdb1968ba7d1399732d68b Mon Sep 17 00:00:00 2001 From: Kavya Sree Kaitepalli Date: Wed, 26 Aug 2026 06:30:05 +0000 Subject: [PATCH 02/13] Add SweBenchVerifiedTask for evaluation and improve EvalTask interface --- src/microbots/auto_memory/analyzer.py | 2 +- .../auto_memory/eval_swebenchverified/eval.py | 269 ++++++++++++++++++ src/microbots/auto_memory/task.py | 8 +- 3 files changed, 274 insertions(+), 5 deletions(-) create mode 100644 src/microbots/auto_memory/eval_swebenchverified/eval.py diff --git a/src/microbots/auto_memory/analyzer.py b/src/microbots/auto_memory/analyzer.py index 2589390..dccea6a 100644 --- a/src/microbots/auto_memory/analyzer.py +++ b/src/microbots/auto_memory/analyzer.py @@ -44,7 +44,7 @@ def build_feedback( file_name=outcome.log_path, user_prompt=( "This log was produced while verifying whether an " - "automated agent completed its task correctly. Identify " + "agent completed its task correctly. Identify " "the root cause of the failure and describe concretely " "what the agent's memory notes should cover next time to " "avoid this failure." diff --git a/src/microbots/auto_memory/eval_swebenchverified/eval.py b/src/microbots/auto_memory/eval_swebenchverified/eval.py new file mode 100644 index 0000000..4c3ef15 --- /dev/null +++ b/src/microbots/auto_memory/eval_swebenchverified/eval.py @@ -0,0 +1,269 @@ +"""Minimal SWE-bench-verified eval task. + +Loads instances from the SWE-bench-verified dataset, checks out each +instance's repo at its base commit, has the agent attempt a fix, and +verifies the result via ``swebench.harness.run_evaluation``. +""" + +import json +import shutil +import subprocess +import sys +import tempfile +import uuid +from dataclasses import dataclass +from logging import getLogger +from pathlib import Path + +from datasets import load_dataset +import argparse +from microbots.auto_memory.task import CallbackResult, EvalTask +from microbots.auto_memory.orchestrator import run_train_eval_loop + +logger = getLogger(__name__) + +SWE_BENCH_SUITE = "SWE-bench/SWE-bench_Verified" + + +@dataclass +class SweBenchInstance: + """A single SWE-bench-verified dataset row. + + Attributes + ---------- + instance_id : str + Unique identifier for the instance, e.g. ``"django__django-11099"``. + repo : str + The GitHub repo this instance belongs to, e.g. ``"django/django"``. + base_commit : str + Commit hash representing the repo state before the issue's fix. + problem_statement : str + The GitHub issue title and body describing the bug to fix. + """ + + instance_id: str + repo: str + base_commit: str + problem_statement: str + + +def load_instances_of_repo( + dataset_name: str = SWE_BENCH_SUITE, + repo: str | None = None, +) -> list[SweBenchInstance]: + """Load all dataset instances, optionally filtered to a single repo. + + Parameters + ---------- + dataset_name : str + Hugging Face dataset name to load. Defaults to + ``SWE_BENCH_SUITE``. + repo : str | None + If given, only instances whose ``repo`` matches this value are + returned, e.g. ``"django/django"``. If ``None``, all instances + are returned. + + Returns + ------- + list[SweBenchInstance] + The matching instances. + """ + rows = load_dataset(dataset_name, split="test") + instances = [ + SweBenchInstance( + instance_id=row["instance_id"], + repo=row["repo"], + base_commit=row["base_commit"], + problem_statement=row["problem_statement"], + ) + for row in rows + if repo is None or row["repo"] == repo + ] + return instances + +def load_instance_using_id(instance_id: str, dataset_name: str = SWE_BENCH_SUITE) -> SweBenchInstance: + """Load a single dataset instance by its instance ID. + + Parameters + ---------- + instance_id : str + The instance ID to look up, e.g. ``"django__django-11099"``. + dataset_name : str + Hugging Face dataset name to load. Defaults to + ``SWE_BENCH_SUITE``. + + Returns + ------- + SweBenchInstance + The matching instance. + + Raises + ------ + ValueError + If no instance with the given ``instance_id`` exists in the + dataset. + """ + rows = load_dataset(dataset_name, split="test") + for row in rows: + if row["instance_id"] == instance_id: + return SweBenchInstance( + instance_id=row["instance_id"], + repo=row["repo"], + base_commit=row["base_commit"], + problem_statement=row["problem_statement"], + ) + raise ValueError(f"instance_id not found: {instance_id}") + +class SweBenchVerifiedTask(EvalTask): + """Eval task that verifies a fix against one SWE-bench-verified instance. + + Checks out the instance's repo at its base commit, gives the agent + the issue's problem statement, and verifies the agent's patch using + the official SWE-bench evaluation harness. + + Parameters + ---------- + instance : SweBenchInstance + The dataset instance this task evaluates against. + """ + + def __init__(self, instance: SweBenchInstance): + """Initialize the task for a single dataset instance. + + Parameters + ---------- + instance : SweBenchInstance + The dataset instance this task evaluates against. + """ + self.instance = instance + + def setup(self, repo_path: str) -> None: + """Clone the instance's repo and check out its base commit. + + Parameters + ---------- + repo_path : str + Absolute path to clone the repo into. + """ + subprocess.run( + ["git", "clone", f"https://github.com/{self.instance.repo}.git", repo_path], + check=True, + ) + subprocess.run( + ["git", "checkout", self.instance.base_commit], cwd=repo_path, check=True + ) + + def build_prompt(self, repo_path: str) -> str: + """Return the instance's issue text as the agent's prompt. + + Parameters + ---------- + repo_path : str + Absolute path to the repo the agent will operate on. + + Returns + ------- + str + The instance's ``problem_statement``. + """ + return self.instance.problem_statement + + def check(self, repo_path: str, agent_output: str, log_path: str) -> CallbackResult: + """Verify the agent's patch using the SWE-bench evaluation harness. + + Captures the agent's changes as a git diff, submits it as a + prediction to ``swebench.harness.run_evaluation``, and checks + whether the harness marked this instance as resolved. + + Parameters + ---------- + repo_path : str + Absolute path to the repo the agent operated on. + agent_output : str + The agent's raw output/result text. Unused here, since + verification is based on the repo's git diff, not the + agent's textual output. + log_path : str + Path to a log file to append the harness's output to. + + Returns + ------- + CallbackResult + Whether the harness marked this instance as resolved. + """ + diff = subprocess.run( + ["git", "diff"], cwd=repo_path, capture_output=True, text=True + ).stdout + + run_id = f"microbots-{uuid.uuid4().hex[:8]}" + model_name_or_path = "microbots-eval-agent" + pred_path = Path(tempfile.mktemp(suffix=".json")) + report_dir = Path(tempfile.mkdtemp()) + pred_path.write_text(json.dumps([{ + "instance_id": self.instance.instance_id, + "model_patch": diff, + "model_name_or_path": model_name_or_path, + }])) + + try: + proc = subprocess.run( + [sys.executable, "-m", "swebench.harness.run_evaluation", + "--dataset_name", SWE_BENCH_SUITE, + "--max_workers", "1", + "--predictions_path", str(pred_path), + "--run_id", run_id, + "--report_dir", str(report_dir), + "--instance_ids", self.instance.instance_id], + #can add timeout if needed + capture_output=True, text=True, + cwd=report_dir, + ) + with open(log_path, "a") as f: + f.write(proc.stdout + proc.stderr) + + report_file = report_dir / f"{model_name_or_path}.{run_id}.json" + passed = False + if report_file.exists(): + report = json.loads(report_file.read_text()) + passed = self.instance.instance_id in report.get("resolved_ids", []) + finally: + pred_path.unlink(missing_ok=True) + shutil.rmtree(report_dir, ignore_errors=True) + + return CallbackResult(passed=passed, reason="resolved" if passed else "not resolved") + + def teardown(self, repo_path: str) -> None: + """Remove the cloned repo working directory. + + Parameters + ---------- + repo_path : str + Absolute path to the repo cloned by ``setup``. + """ + subprocess.run(["rm", "-rf", repo_path], check=False) + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser() + parser.add_argument("--repo", help='e.g. "django/django"') + parser.add_argument("--instance-id", help='e.g. "django__django-11099"') + parser.add_argument("--model", default="azure-openai/gpt-5.5") + parser.add_argument("--max-rounds", type=int, default=5) + args = parser.parse_args() + + if args.instance_id: + instances = [load_instance_using_id(args.instance_id)] + else: + instances = load_instances_of_repo(repo=args.repo) + + for instance in instances: + task = SweBenchVerifiedTask(instance) + result = run_train_eval_loop( + repo_path=tempfile.mkdtemp(), + memory_dir="memory", + model=args.model, + task=task, + max_rounds=args.max_rounds, + ) + logger.info("%s: passed=%s", instance.instance_id, result.passed) diff --git a/src/microbots/auto_memory/task.py b/src/microbots/auto_memory/task.py index e0e8fef..1b4417c 100644 --- a/src/microbots/auto_memory/task.py +++ b/src/microbots/auto_memory/task.py @@ -57,19 +57,19 @@ class EvalOutcome: class EvalTask(ABC): """Base class for a single evaluation task in the train <-> eval loop. - Subclasses must implement ``build_prompt`` and ``check``, and may - override ``setup``, ``teardown``, and ``run`` as needed. + Subclasses must implement ``setup``, ``build_prompt``, and ``check``, + and may override ``teardown`` and ``run`` as needed. """ + @abstractmethod def setup(self, repo_path: str) -> None: - """Optional. Prepare repo/environment before the agent runs. + """Required. Prepare repo/environment before the agent runs. Parameters ---------- repo_path : str Absolute path to the repo to prepare. """ - pass @abstractmethod def build_prompt(self, repo_path: str) -> str: From 0070b6110044fcb26fc1acbd0021bb0d4604dbb1 Mon Sep 17 00:00:00 2001 From: Kavya Sree Kaitepalli Date: Wed, 26 Aug 2026 06:54:43 +0000 Subject: [PATCH 03/13] Add unit tests for evaluation tasks and orchestrator --- src/microbots/auto_memory/task.py | 2 +- .../eval_swebenchverified/test_eval.py | 243 ++++++++++++++++++ test/auto_memory/test_analyzer.py | 81 ++++++ test/auto_memory/test_orchestrator.py | 163 ++++++++++++ test/auto_memory/test_task.py | 192 ++++++++++++++ 5 files changed, 680 insertions(+), 1 deletion(-) create mode 100644 test/auto_memory/eval_swebenchverified/test_eval.py create mode 100644 test/auto_memory/test_analyzer.py create mode 100644 test/auto_memory/test_orchestrator.py create mode 100644 test/auto_memory/test_task.py diff --git a/src/microbots/auto_memory/task.py b/src/microbots/auto_memory/task.py index 1b4417c..8a1de46 100644 --- a/src/microbots/auto_memory/task.py +++ b/src/microbots/auto_memory/task.py @@ -12,7 +12,7 @@ from pathlib import Path from microbots.bot.WritingBot import WritingBot -from microbots.tools.MemoryTool import MemoryTool +from microbots.tools.tool_definitions.memory_tool import MemoryTool logger = getLogger(__name__) diff --git a/test/auto_memory/eval_swebenchverified/test_eval.py b/test/auto_memory/eval_swebenchverified/test_eval.py new file mode 100644 index 0000000..4115e45 --- /dev/null +++ b/test/auto_memory/eval_swebenchverified/test_eval.py @@ -0,0 +1,243 @@ +"""Unit tests for microbots.auto_memory.eval_swebenchverified.eval.""" + +import json +import os +import sys +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../src/"))) + +from microbots.auto_memory.eval_swebenchverified.eval import ( + SweBenchInstance, + SweBenchVerifiedTask, + load_instance_using_id, + load_instances_of_repo, +) + +MODULE = "microbots.auto_memory.eval_swebenchverified.eval" + + +def _fake_rows(): + return [ + { + "instance_id": "django__django-1", + "repo": "django/django", + "base_commit": "abc123", + "problem_statement": "fix bug 1", + }, + { + "instance_id": "astropy__astropy-1", + "repo": "astropy/astropy", + "base_commit": "def456", + "problem_statement": "fix bug 2", + }, + { + "instance_id": "django__django-2", + "repo": "django/django", + "base_commit": "ghi789", + "problem_statement": "fix bug 3", + }, + ] + + +# --------------------------------------------------------------------------- +# load_instances_of_repo / load_instance_using_id +# --------------------------------------------------------------------------- + +@pytest.mark.unit +@patch(f"{MODULE}.load_dataset") +def test_load_instances_of_repo_filters_by_repo(mock_load_dataset): + mock_load_dataset.return_value = _fake_rows() + + instances = load_instances_of_repo(repo="django/django") + + assert [i.instance_id for i in instances] == ["django__django-1", "django__django-2"] + assert all(isinstance(i, SweBenchInstance) for i in instances) + + +@pytest.mark.unit +@patch(f"{MODULE}.load_dataset") +def test_load_instances_of_repo_returns_all_when_repo_none(mock_load_dataset): + mock_load_dataset.return_value = _fake_rows() + + instances = load_instances_of_repo(repo=None) + + assert len(instances) == 3 + + +@pytest.mark.unit +@patch(f"{MODULE}.load_dataset") +def test_load_instance_using_id_returns_matching_instance(mock_load_dataset): + mock_load_dataset.return_value = _fake_rows() + + instance = load_instance_using_id("astropy__astropy-1") + + assert instance.repo == "astropy/astropy" + assert instance.problem_statement == "fix bug 2" + + +@pytest.mark.unit +@patch(f"{MODULE}.load_dataset") +def test_load_instance_using_id_raises_when_not_found(mock_load_dataset): + mock_load_dataset.return_value = _fake_rows() + + with pytest.raises(ValueError, match="not found"): + load_instance_using_id("does-not-exist") + + +# --------------------------------------------------------------------------- +# SweBenchVerifiedTask.setup / build_prompt / teardown +# --------------------------------------------------------------------------- + +def _instance(): + return SweBenchInstance( + instance_id="django__django-1", + repo="django/django", + base_commit="abc123", + problem_statement="fix the bug", + ) + + +@pytest.mark.unit +@patch(f"{MODULE}.subprocess.run") +def test_setup_clones_and_checks_out_base_commit(mock_run): + task = SweBenchVerifiedTask(_instance()) + task.setup("/repo") + + clone_call, checkout_call = mock_run.call_args_list + assert clone_call.args[0] == ["git", "clone", "https://github.com/django/django.git", "/repo"] + assert checkout_call.args[0] == ["git", "checkout", "abc123"] + assert checkout_call.kwargs["cwd"] == "/repo" + + +@pytest.mark.unit +def test_build_prompt_returns_problem_statement(): + task = SweBenchVerifiedTask(_instance()) + assert task.build_prompt("/repo") == "fix the bug" + + +@pytest.mark.unit +@patch(f"{MODULE}.subprocess.run") +def test_teardown_removes_repo_path(mock_run): + task = SweBenchVerifiedTask(_instance()) + task.teardown("/repo") + + mock_run.assert_called_once_with(["rm", "-rf", "/repo"], check=False) + + +# --------------------------------------------------------------------------- +# SweBenchVerifiedTask.check +# --------------------------------------------------------------------------- + +def _make_fake_subprocess_run(resolved: bool, raise_on_harness: bool = False): + """Build a subprocess.run stand-in that fakes git diff + the harness call.""" + + def _fake_run(cmd, **kwargs): + if cmd[:2] == ["git", "diff"]: + return MagicMock(stdout="diff --git a/x.py b/x.py\n+fix", stderr="", returncode=0) + if "swebench.harness.run_evaluation" in cmd: + if raise_on_harness: + raise RuntimeError("harness crashed") + run_id = cmd[cmd.index("--run_id") + 1] + report_dir = kwargs["cwd"] + instance_id = cmd[cmd.index("--instance_ids") + 1] + report = {"resolved_ids": [instance_id] if resolved else []} + (Path(report_dir) / f"microbots-eval-agent.{run_id}.json").write_text(json.dumps(report)) + return MagicMock(stdout="harness ran\n", stderr="", returncode=0) + return MagicMock(stdout="", stderr="", returncode=0) + + return _fake_run + + +@pytest.mark.unit +@patch(f"{MODULE}.subprocess.run") +def test_check_passed_true_when_instance_in_resolved_ids(mock_run, tmp_path): + mock_run.side_effect = _make_fake_subprocess_run(resolved=True) + log_path = tmp_path / "check.log" + log_path.write_text("") + + task = SweBenchVerifiedTask(_instance()) + result = task.check("/repo", "agent output", str(log_path)) + + assert result.passed is True + assert result.reason == "resolved" + + +@pytest.mark.unit +@patch(f"{MODULE}.subprocess.run") +def test_check_passed_false_when_instance_not_in_resolved_ids(mock_run, tmp_path): + mock_run.side_effect = _make_fake_subprocess_run(resolved=False) + log_path = tmp_path / "check.log" + log_path.write_text("") + + task = SweBenchVerifiedTask(_instance()) + result = task.check("/repo", "agent output", str(log_path)) + + assert result.passed is False + assert result.reason == "not resolved" + + +@pytest.mark.unit +@patch(f"{MODULE}.subprocess.run") +def test_check_passed_false_when_report_file_never_written(mock_run, tmp_path): + # harness call succeeds but never writes a report file (e.g. it errored internally) + def _fake_run(cmd, **kwargs): + if cmd[:2] == ["git", "diff"]: + return MagicMock(stdout="diff", stderr="", returncode=0) + return MagicMock(stdout="", stderr="", returncode=1) + + mock_run.side_effect = _fake_run + log_path = tmp_path / "check.log" + log_path.write_text("") + + task = SweBenchVerifiedTask(_instance()) + result = task.check("/repo", "agent output", str(log_path)) + + assert result.passed is False + + +@pytest.mark.unit +@patch(f"{MODULE}.subprocess.run") +def test_check_appends_to_log_file_without_truncating_existing_content(mock_run, tmp_path): + mock_run.side_effect = _make_fake_subprocess_run(resolved=True) + log_path = tmp_path / "check.log" + log_path.write_text("Agent output:\nprevious content\n") + + task = SweBenchVerifiedTask(_instance()) + task.check("/repo", "agent output", str(log_path)) + + content = log_path.read_text() + assert "previous content" in content + assert "harness ran" in content + + +@pytest.mark.unit +@patch(f"{MODULE}.shutil.rmtree") +@patch(f"{MODULE}.subprocess.run") +def test_check_cleans_up_pred_path_and_report_dir_on_success(mock_run, mock_rmtree, tmp_path): + mock_run.side_effect = _make_fake_subprocess_run(resolved=True) + log_path = tmp_path / "check.log" + log_path.write_text("") + + task = SweBenchVerifiedTask(_instance()) + task.check("/repo", "agent output", str(log_path)) + + mock_rmtree.assert_called_once() + + +@pytest.mark.unit +@patch(f"{MODULE}.shutil.rmtree") +@patch(f"{MODULE}.subprocess.run") +def test_check_cleans_up_even_when_harness_raises(mock_run, mock_rmtree, tmp_path): + mock_run.side_effect = _make_fake_subprocess_run(resolved=True, raise_on_harness=True) + log_path = tmp_path / "check.log" + log_path.write_text("") + + task = SweBenchVerifiedTask(_instance()) + with pytest.raises(RuntimeError, match="harness crashed"): + task.check("/repo", "agent output", str(log_path)) + + mock_rmtree.assert_called_once() diff --git a/test/auto_memory/test_analyzer.py b/test/auto_memory/test_analyzer.py new file mode 100644 index 0000000..601b044 --- /dev/null +++ b/test/auto_memory/test_analyzer.py @@ -0,0 +1,81 @@ +"""Unit tests for microbots.auto_memory.analyzer.""" + +import os +import sys +from unittest.mock import MagicMock, patch + +import pytest + +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../src/"))) + +from microbots.auto_memory.analyzer import build_feedback +from microbots.auto_memory.task import CallbackResult, EvalOutcome +from microbots.MicroBot import BotRunResult + + +def _make_outcome(reason: str = "tests failed", output: str = "agent output") -> EvalOutcome: + return EvalOutcome( + passed=False, + output=output, + result=CallbackResult(passed=False, reason=reason), + log_path="/tmp/some.log", + ) + + +@pytest.mark.unit +@patch("microbots.auto_memory.analyzer.LogAnalysisBot") +def test_build_feedback_returns_bot_result_on_success(mock_bot_cls): + mock_bot = MagicMock() + mock_bot.run.return_value = BotRunResult( + status=True, result="root cause: missing edge case handling", error=None + ) + mock_bot_cls.return_value = mock_bot + + outcome = _make_outcome() + feedback = build_feedback(task=MagicMock(), outcome=outcome, repo_path="/repo", model="azure-openai/gpt-4o") + + assert feedback == "root cause: missing edge case handling" + mock_bot_cls.assert_called_once_with(model="azure-openai/gpt-4o", folder_to_mount="/repo") + mock_bot.run.assert_called_once() + assert mock_bot.run.call_args.kwargs["file_name"] == outcome.log_path + + +@pytest.mark.unit +@patch("microbots.auto_memory.analyzer.LogAnalysisBot") +def test_build_feedback_falls_back_when_bot_status_false(mock_bot_cls): + mock_bot = MagicMock() + mock_bot.run.return_value = BotRunResult(status=False, result=None, error="bot crashed") + mock_bot_cls.return_value = mock_bot + + outcome = _make_outcome(reason="tests failed", output="some output") + feedback = build_feedback(task=MagicMock(), outcome=outcome, repo_path="/repo", model="azure-openai/gpt-4o") + + assert "some output" in feedback + assert "tests failed" in feedback + + +@pytest.mark.unit +@patch("microbots.auto_memory.analyzer.LogAnalysisBot") +def test_build_feedback_falls_back_when_result_is_empty(mock_bot_cls): + mock_bot = MagicMock() + mock_bot.run.return_value = BotRunResult(status=True, result="", error=None) + mock_bot_cls.return_value = mock_bot + + outcome = _make_outcome(reason="assertion error", output="agent tried X") + feedback = build_feedback(task=MagicMock(), outcome=outcome, repo_path="/repo", model="azure-openai/gpt-4o") + + assert "agent tried X" in feedback + assert "assertion error" in feedback + + +@pytest.mark.unit +@patch("microbots.auto_memory.analyzer.LogAnalysisBot") +def test_build_feedback_falls_back_when_result_is_none(mock_bot_cls): + mock_bot = MagicMock() + mock_bot.run.return_value = BotRunResult(status=True, result=None, error=None) + mock_bot_cls.return_value = mock_bot + + outcome = _make_outcome() + feedback = build_feedback(task=MagicMock(), outcome=outcome, repo_path="/repo", model="azure-openai/gpt-4o") + + assert "Evaluation failed" in feedback diff --git a/test/auto_memory/test_orchestrator.py b/test/auto_memory/test_orchestrator.py new file mode 100644 index 0000000..2bbd2d7 --- /dev/null +++ b/test/auto_memory/test_orchestrator.py @@ -0,0 +1,163 @@ +"""Unit tests for microbots.auto_memory.orchestrator.""" + +import os +import sys +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../src/"))) + +from microbots.auto_memory.orchestrator import LoopResult, run_train_eval_loop +from microbots.auto_memory.task import CallbackResult, EvalOutcome + + +def _make_outcome(passed: bool, log_path: str, reason: str = "reason") -> EvalOutcome: + return EvalOutcome( + passed=passed, + output="agent output", + result=CallbackResult(passed=passed, reason=reason), + log_path=log_path, + ) + + +def _touch(path: str) -> str: + Path(path).write_text("log contents") + return path + + +@pytest.mark.unit +@patch("microbots.auto_memory.orchestrator.run_training") +@patch("microbots.auto_memory.orchestrator.build_feedback") +def test_loop_returns_immediately_when_first_round_passes(mock_build_feedback, mock_run_training, tmp_path): + log_path = _touch(str(tmp_path / "round1.log")) + task = MagicMock() + task.run.return_value = _make_outcome(passed=True, log_path=log_path) + + result = run_train_eval_loop("/repo", "/memory", "azure-openai/gpt-4o", task, max_rounds=5) + + assert isinstance(result, LoopResult) + assert result.passed is True + assert result.rounds_run == 1 + assert task.run.call_count == 1 + mock_build_feedback.assert_not_called() + mock_run_training.assert_not_called() + + +@pytest.mark.unit +@patch("microbots.auto_memory.orchestrator.run_training") +@patch("microbots.auto_memory.orchestrator.build_feedback") +def test_loop_retrains_and_continues_on_failure_then_passes(mock_build_feedback, mock_run_training, tmp_path): + log1 = _touch(str(tmp_path / "round1.log")) + log2 = _touch(str(tmp_path / "round2.log")) + task = MagicMock() + task.run.side_effect = [ + _make_outcome(passed=False, log_path=log1), + _make_outcome(passed=True, log_path=log2), + ] + mock_build_feedback.return_value = "feedback text" + + result = run_train_eval_loop("/repo", "/memory", "azure-openai/gpt-4o", task, max_rounds=5) + + assert result.passed is True + assert result.rounds_run == 2 + mock_build_feedback.assert_called_once() + mock_run_training.assert_called_once_with( + repo_path="/repo", feedback="feedback text", memory_dir="/memory", model="azure-openai/gpt-4o" + ) + + +@pytest.mark.unit +@patch("microbots.auto_memory.orchestrator.run_training") +@patch("microbots.auto_memory.orchestrator.build_feedback") +def test_loop_exhausts_max_rounds_without_passing(mock_build_feedback, mock_run_training, tmp_path): + task = MagicMock() + task.run.side_effect = [ + _make_outcome(passed=False, log_path=_touch(str(tmp_path / f"round{i}.log"))) + for i in range(3) + ] + mock_build_feedback.return_value = "feedback text" + + result = run_train_eval_loop("/repo", "/memory", "azure-openai/gpt-4o", task, max_rounds=3) + + assert result.passed is False + assert result.rounds_run == 3 + assert len(result.outcomes) == 3 + assert result.final_outcome is result.outcomes[-1] + assert mock_build_feedback.call_count == 3 + assert mock_run_training.call_count == 3 + + +@pytest.mark.unit +@patch("microbots.auto_memory.orchestrator.run_training") +@patch("microbots.auto_memory.orchestrator.build_feedback") +def test_log_path_deleted_after_passing_round(mock_build_feedback, mock_run_training, tmp_path): + log_path = _touch(str(tmp_path / "round1.log")) + task = MagicMock() + task.run.return_value = _make_outcome(passed=True, log_path=log_path) + + run_train_eval_loop("/repo", "/memory", "azure-openai/gpt-4o", task, max_rounds=5) + + assert not Path(log_path).exists() + + +@pytest.mark.unit +@patch("microbots.auto_memory.orchestrator.run_training") +@patch("microbots.auto_memory.orchestrator.build_feedback") +def test_log_path_deleted_after_failing_round(mock_build_feedback, mock_run_training, tmp_path): + log1 = _touch(str(tmp_path / "round1.log")) + log2 = _touch(str(tmp_path / "round2.log")) + task = MagicMock() + task.run.side_effect = [ + _make_outcome(passed=False, log_path=log1), + _make_outcome(passed=True, log_path=log2), + ] + mock_build_feedback.return_value = "feedback text" + + run_train_eval_loop("/repo", "/memory", "azure-openai/gpt-4o", task, max_rounds=5) + + assert not Path(log1).exists() + assert not Path(log2).exists() + + +@pytest.mark.unit +@patch("microbots.auto_memory.orchestrator.run_training") +@patch("microbots.auto_memory.orchestrator.build_feedback") +def test_build_feedback_exception_does_not_crash_loop(mock_build_feedback, mock_run_training, tmp_path): + log1 = _touch(str(tmp_path / "round1.log")) + log2 = _touch(str(tmp_path / "round2.log")) + task = MagicMock() + task.run.side_effect = [ + _make_outcome(passed=False, log_path=log1), + _make_outcome(passed=True, log_path=log2), + ] + mock_build_feedback.side_effect = RuntimeError("analysis bot crashed") + + result = run_train_eval_loop("/repo", "/memory", "azure-openai/gpt-4o", task, max_rounds=5) + + assert result.passed is True + assert result.rounds_run == 2 + mock_run_training.assert_not_called() + assert not Path(log1).exists() + + +@pytest.mark.unit +@patch("microbots.auto_memory.orchestrator.run_training") +@patch("microbots.auto_memory.orchestrator.build_feedback") +def test_run_training_exception_does_not_crash_loop(mock_build_feedback, mock_run_training, tmp_path): + log1 = _touch(str(tmp_path / "round1.log")) + log2 = _touch(str(tmp_path / "round2.log")) + task = MagicMock() + task.run.side_effect = [ + _make_outcome(passed=False, log_path=log1), + _make_outcome(passed=True, log_path=log2), + ] + mock_build_feedback.return_value = "feedback text" + mock_run_training.side_effect = RuntimeError("training crashed") + + result = run_train_eval_loop("/repo", "/memory", "azure-openai/gpt-4o", task, max_rounds=5) + + assert result.passed is True + assert result.rounds_run == 2 + assert not Path(log1).exists() diff --git a/test/auto_memory/test_task.py b/test/auto_memory/test_task.py new file mode 100644 index 0000000..bfc2cb6 --- /dev/null +++ b/test/auto_memory/test_task.py @@ -0,0 +1,192 @@ +"""Unit tests for microbots.auto_memory.task.""" + +import os +import sys +from unittest.mock import MagicMock, patch + +import pytest + +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../src/"))) + +from microbots.auto_memory.task import CallbackResult, EvalOutcome, EvalTask +from microbots.MicroBot import BotRunResult + + +class _StubTask(EvalTask): + """A minimal concrete EvalTask used to exercise the base run() logic.""" + + def __init__(self, check_result=None, check_side_effect=None, build_prompt_side_effect=None): + self.setup_calls = [] + self.teardown_calls = [] + self.check_calls = [] + self._check_result = check_result or CallbackResult(passed=True, reason="ok") + self._check_side_effect = check_side_effect + self._build_prompt_side_effect = build_prompt_side_effect + + def setup(self, repo_path): + self.setup_calls.append(repo_path) + + def build_prompt(self, repo_path): + if self._build_prompt_side_effect: + raise self._build_prompt_side_effect + return "do the task" + + def check(self, repo_path, agent_output, log_path): + self.check_calls.append((repo_path, agent_output, log_path)) + if self._check_side_effect: + raise self._check_side_effect + return self._check_result + + def teardown(self, repo_path): + self.teardown_calls.append(repo_path) + + +class _RaisingTeardownTask(_StubTask): + def teardown(self, repo_path): + super().teardown(repo_path) + raise RuntimeError("teardown boom") + + +class _DefaultTeardownTask(EvalTask): + """A task that relies on EvalTask's default no-op teardown.""" + + def setup(self, repo_path): + pass + + def build_prompt(self, repo_path): + return "do the task" + + def check(self, repo_path, agent_output, log_path): + return CallbackResult(passed=True, reason="ok") + + +@pytest.mark.unit +def test_setup_and_check_are_abstract(): + with pytest.raises(TypeError): + EvalTask() + + +@pytest.mark.unit +@patch("microbots.auto_memory.task.MemoryTool") +@patch("microbots.auto_memory.task.WritingBot") +def test_run_calls_setup_build_prompt_check_teardown_in_order(mock_bot_cls, mock_memory_tool): + mock_bot = MagicMock() + mock_bot.run.return_value = BotRunResult(status=True, result="agent did stuff", error=None) + mock_bot_cls.return_value = mock_bot + + task = _StubTask() + outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o") + + assert task.setup_calls == ["/repo"] + assert task.check_calls == [("/repo", "agent did stuff", outcome.log_path)] + assert task.teardown_calls == ["/repo"] + assert outcome.passed is True + assert outcome.output == "agent did stuff" + + +@pytest.mark.unit +@patch("microbots.auto_memory.task.MemoryTool") +@patch("microbots.auto_memory.task.WritingBot") +def test_run_creates_log_file_before_check_is_called(mock_bot_cls, mock_memory_tool): + mock_bot = MagicMock() + mock_bot.run.return_value = BotRunResult(status=True, result="output", error=None) + mock_bot_cls.return_value = mock_bot + + seen_log_exists = {} + + class _CheckingTask(_StubTask): + def check(self, repo_path, agent_output, log_path): + seen_log_exists["exists"] = os.path.exists(log_path) + return super().check(repo_path, agent_output, log_path) + + task = _CheckingTask() + task.run("/repo", "/memory", "azure-openai/gpt-4o") + + assert seen_log_exists["exists"] is True + + +@pytest.mark.unit +@patch("microbots.auto_memory.task.MemoryTool") +@patch("microbots.auto_memory.task.WritingBot") +def test_run_skips_check_when_bot_status_is_false(mock_bot_cls, mock_memory_tool): + mock_bot = MagicMock() + mock_bot.run.return_value = BotRunResult(status=False, result=None, error="bot crashed") + mock_bot_cls.return_value = mock_bot + + task = _StubTask() + outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o") + + assert task.check_calls == [] + assert outcome.passed is False + assert "bot crashed" in outcome.result.reason + + +@pytest.mark.unit +@patch("microbots.auto_memory.task.MemoryTool") +@patch("microbots.auto_memory.task.WritingBot") +def test_run_converts_build_prompt_exception_to_failed_outcome(mock_bot_cls, mock_memory_tool): + task = _StubTask(build_prompt_side_effect=ValueError("bad prompt")) + outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o") + + assert outcome.passed is False + assert "bad prompt" in outcome.result.reason + with open(outcome.log_path) as f: + assert "bad prompt" in f.read() + + +@pytest.mark.unit +@patch("microbots.auto_memory.task.MemoryTool") +@patch("microbots.auto_memory.task.WritingBot") +def test_run_converts_check_exception_to_failed_outcome(mock_bot_cls, mock_memory_tool): + mock_bot = MagicMock() + mock_bot.run.return_value = BotRunResult(status=True, result="output", error=None) + mock_bot_cls.return_value = mock_bot + + task = _StubTask(check_side_effect=RuntimeError("check exploded")) + outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o") + + assert outcome.passed is False + assert "check exploded" in outcome.result.reason + + +@pytest.mark.unit +@patch("microbots.auto_memory.task.MemoryTool") +@patch("microbots.auto_memory.task.WritingBot") +def test_run_still_calls_teardown_when_body_raises(mock_bot_cls, mock_memory_tool): + mock_bot_cls.side_effect = RuntimeError("bot construction failed") + + task = _StubTask() + task.run("/repo", "/memory", "azure-openai/gpt-4o") + + assert task.teardown_calls == ["/repo"] + + +@pytest.mark.unit +@patch("microbots.auto_memory.task.MemoryTool") +@patch("microbots.auto_memory.task.WritingBot") +def test_run_teardown_exception_does_not_clobber_returned_outcome(mock_bot_cls, mock_memory_tool): + mock_bot = MagicMock() + mock_bot.run.return_value = BotRunResult(status=True, result="output", error=None) + mock_bot_cls.return_value = mock_bot + + task = _RaisingTeardownTask() + outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o") + + # teardown() raised, but the already-computed EvalOutcome must still be returned + assert isinstance(outcome, EvalOutcome) + assert outcome.passed is True + + +@pytest.mark.unit +@patch("microbots.auto_memory.task.MemoryTool") +@patch("microbots.auto_memory.task.WritingBot") +def test_run_uses_default_noop_teardown_when_not_overridden(mock_bot_cls, mock_memory_tool): + mock_bot = MagicMock() + mock_bot.run.return_value = BotRunResult(status=True, result="output", error=None) + mock_bot_cls.return_value = mock_bot + + task = _DefaultTeardownTask() + outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o") + + assert outcome.passed is True + From 90ed58656c84a099198e3b26d8d633ca9d63dfd6 Mon Sep 17 00:00:00 2001 From: Kavya Sree Kaitepalli Date: Wed, 26 Aug 2026 09:20:18 +0000 Subject: [PATCH 04/13] refactor: update training function references to use run_training_loop and add training_iterations parameter --- src/microbots/auto_memory/orchestrator.py | 14 +++-- test/auto_memory/test_orchestrator.py | 62 +++++++++++++++-------- 2 files changed, 52 insertions(+), 24 deletions(-) diff --git a/src/microbots/auto_memory/orchestrator.py b/src/microbots/auto_memory/orchestrator.py index 7558a51..8f1dcfb 100644 --- a/src/microbots/auto_memory/orchestrator.py +++ b/src/microbots/auto_memory/orchestrator.py @@ -11,7 +11,7 @@ from microbots.auto_memory.analyzer import build_feedback from microbots.auto_memory.task import EvalOutcome, EvalTask -from microbots.auto_memory.training.runner import run_training +from microbots.auto_memory.training.runner import run_training_loop logger = getLogger(__name__) @@ -42,13 +42,15 @@ def run_train_eval_loop( model: str, task: EvalTask, max_rounds: int = 5, + training_iterations: int = 1, ) -> LoopResult: """Run an eval task in a loop, retraining on failure until it passes. Each round runs ``task.run(...)``. If the task passes, the loop returns immediately. If it fails, feedback is built from the round's - log and used to retrain via ``run_training`` before the next round. - The round's log file is always deleted before the next round starts. + log and used to retrain via ``run_training_loop`` before the next + round. The round's log file is always deleted before the next round + starts. Parameters ---------- @@ -62,6 +64,9 @@ def run_train_eval_loop( The eval task to run each round. max_rounds : int Maximum number of train/eval rounds to attempt. Defaults to 5. + training_iterations : int + Number of training passes to run per retraining round, each + reusing the same ``memory_dir``. Defaults to 1. Returns ------- @@ -98,11 +103,12 @@ def run_train_eval_loop( try: feedback = build_feedback(task, outcome, repo_path, model) - run_training( + run_training_loop( repo_path=repo_path, feedback=feedback, memory_dir=memory_dir, model=model, + iterations=training_iterations, ) except Exception: logger.exception( diff --git a/test/auto_memory/test_orchestrator.py b/test/auto_memory/test_orchestrator.py index 2bbd2d7..ac21773 100644 --- a/test/auto_memory/test_orchestrator.py +++ b/test/auto_memory/test_orchestrator.py @@ -28,9 +28,9 @@ def _touch(path: str) -> str: @pytest.mark.unit -@patch("microbots.auto_memory.orchestrator.run_training") +@patch("microbots.auto_memory.orchestrator.run_training_loop") @patch("microbots.auto_memory.orchestrator.build_feedback") -def test_loop_returns_immediately_when_first_round_passes(mock_build_feedback, mock_run_training, tmp_path): +def test_loop_returns_immediately_when_first_round_passes(mock_build_feedback, mock_run_training_loop, tmp_path): log_path = _touch(str(tmp_path / "round1.log")) task = MagicMock() task.run.return_value = _make_outcome(passed=True, log_path=log_path) @@ -42,13 +42,13 @@ def test_loop_returns_immediately_when_first_round_passes(mock_build_feedback, m assert result.rounds_run == 1 assert task.run.call_count == 1 mock_build_feedback.assert_not_called() - mock_run_training.assert_not_called() + mock_run_training_loop.assert_not_called() @pytest.mark.unit -@patch("microbots.auto_memory.orchestrator.run_training") +@patch("microbots.auto_memory.orchestrator.run_training_loop") @patch("microbots.auto_memory.orchestrator.build_feedback") -def test_loop_retrains_and_continues_on_failure_then_passes(mock_build_feedback, mock_run_training, tmp_path): +def test_loop_retrains_and_continues_on_failure_then_passes(mock_build_feedback, mock_run_training_loop, tmp_path): log1 = _touch(str(tmp_path / "round1.log")) log2 = _touch(str(tmp_path / "round2.log")) task = MagicMock() @@ -63,15 +63,15 @@ def test_loop_retrains_and_continues_on_failure_then_passes(mock_build_feedback, assert result.passed is True assert result.rounds_run == 2 mock_build_feedback.assert_called_once() - mock_run_training.assert_called_once_with( - repo_path="/repo", feedback="feedback text", memory_dir="/memory", model="azure-openai/gpt-4o" + mock_run_training_loop.assert_called_once_with( + repo_path="/repo", feedback="feedback text", memory_dir="/memory", model="azure-openai/gpt-4o", iterations=1 ) @pytest.mark.unit -@patch("microbots.auto_memory.orchestrator.run_training") +@patch("microbots.auto_memory.orchestrator.run_training_loop") @patch("microbots.auto_memory.orchestrator.build_feedback") -def test_loop_exhausts_max_rounds_without_passing(mock_build_feedback, mock_run_training, tmp_path): +def test_loop_exhausts_max_rounds_without_passing(mock_build_feedback, mock_run_training_loop, tmp_path): task = MagicMock() task.run.side_effect = [ _make_outcome(passed=False, log_path=_touch(str(tmp_path / f"round{i}.log"))) @@ -86,13 +86,13 @@ def test_loop_exhausts_max_rounds_without_passing(mock_build_feedback, mock_run_ assert len(result.outcomes) == 3 assert result.final_outcome is result.outcomes[-1] assert mock_build_feedback.call_count == 3 - assert mock_run_training.call_count == 3 + assert mock_run_training_loop.call_count == 3 @pytest.mark.unit -@patch("microbots.auto_memory.orchestrator.run_training") +@patch("microbots.auto_memory.orchestrator.run_training_loop") @patch("microbots.auto_memory.orchestrator.build_feedback") -def test_log_path_deleted_after_passing_round(mock_build_feedback, mock_run_training, tmp_path): +def test_log_path_deleted_after_passing_round(mock_build_feedback, mock_run_training_loop, tmp_path): log_path = _touch(str(tmp_path / "round1.log")) task = MagicMock() task.run.return_value = _make_outcome(passed=True, log_path=log_path) @@ -103,9 +103,9 @@ def test_log_path_deleted_after_passing_round(mock_build_feedback, mock_run_trai @pytest.mark.unit -@patch("microbots.auto_memory.orchestrator.run_training") +@patch("microbots.auto_memory.orchestrator.run_training_loop") @patch("microbots.auto_memory.orchestrator.build_feedback") -def test_log_path_deleted_after_failing_round(mock_build_feedback, mock_run_training, tmp_path): +def test_log_path_deleted_after_failing_round(mock_build_feedback, mock_run_training_loop, tmp_path): log1 = _touch(str(tmp_path / "round1.log")) log2 = _touch(str(tmp_path / "round2.log")) task = MagicMock() @@ -122,9 +122,9 @@ def test_log_path_deleted_after_failing_round(mock_build_feedback, mock_run_trai @pytest.mark.unit -@patch("microbots.auto_memory.orchestrator.run_training") +@patch("microbots.auto_memory.orchestrator.run_training_loop") @patch("microbots.auto_memory.orchestrator.build_feedback") -def test_build_feedback_exception_does_not_crash_loop(mock_build_feedback, mock_run_training, tmp_path): +def test_build_feedback_exception_does_not_crash_loop(mock_build_feedback, mock_run_training_loop, tmp_path): log1 = _touch(str(tmp_path / "round1.log")) log2 = _touch(str(tmp_path / "round2.log")) task = MagicMock() @@ -138,14 +138,14 @@ def test_build_feedback_exception_does_not_crash_loop(mock_build_feedback, mock_ assert result.passed is True assert result.rounds_run == 2 - mock_run_training.assert_not_called() + mock_run_training_loop.assert_not_called() assert not Path(log1).exists() @pytest.mark.unit -@patch("microbots.auto_memory.orchestrator.run_training") +@patch("microbots.auto_memory.orchestrator.run_training_loop") @patch("microbots.auto_memory.orchestrator.build_feedback") -def test_run_training_exception_does_not_crash_loop(mock_build_feedback, mock_run_training, tmp_path): +def test_loop_forwards_training_iterations_to_run_training_loop(mock_build_feedback, mock_run_training_loop, tmp_path): log1 = _touch(str(tmp_path / "round1.log")) log2 = _touch(str(tmp_path / "round2.log")) task = MagicMock() @@ -154,7 +154,29 @@ def test_run_training_exception_does_not_crash_loop(mock_build_feedback, mock_ru _make_outcome(passed=True, log_path=log2), ] mock_build_feedback.return_value = "feedback text" - mock_run_training.side_effect = RuntimeError("training crashed") + + run_train_eval_loop( + "/repo", "/memory", "azure-openai/gpt-4o", task, max_rounds=5, training_iterations=4 + ) + + mock_run_training_loop.assert_called_once_with( + repo_path="/repo", feedback="feedback text", memory_dir="/memory", model="azure-openai/gpt-4o", iterations=4 + ) + + +@pytest.mark.unit +@patch("microbots.auto_memory.orchestrator.run_training_loop") +@patch("microbots.auto_memory.orchestrator.build_feedback") +def test_run_training_exception_does_not_crash_loop(mock_build_feedback, mock_run_training_loop, tmp_path): + log1 = _touch(str(tmp_path / "round1.log")) + log2 = _touch(str(tmp_path / "round2.log")) + task = MagicMock() + task.run.side_effect = [ + _make_outcome(passed=False, log_path=log1), + _make_outcome(passed=True, log_path=log2), + ] + mock_build_feedback.return_value = "feedback text" + mock_run_training_loop.side_effect = RuntimeError("training crashed") result = run_train_eval_loop("/repo", "/memory", "azure-openai/gpt-4o", task, max_rounds=5) From b2b08ff8ca6a569c2ab88eddca67c1f047c06dd5 Mon Sep 17 00:00:00 2001 From: Kavya Sree Kaitepalli Date: Mon, 31 Aug 2026 05:07:34 +0000 Subject: [PATCH 05/13] modify file structure --- src/microbots/auto_memory/analyzer.py | 2 +- .../{eval_swebenchverified/eval.py => eval/swebenchverified.py} | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) rename src/microbots/auto_memory/{eval_swebenchverified/eval.py => eval/swebenchverified.py} (99%) diff --git a/src/microbots/auto_memory/analyzer.py b/src/microbots/auto_memory/analyzer.py index dccea6a..83e75cf 100644 --- a/src/microbots/auto_memory/analyzer.py +++ b/src/microbots/auto_memory/analyzer.py @@ -12,7 +12,7 @@ from microbots.MicroBot import BotRunResult logger = getLogger(__name__) - +#make this abstract def build_feedback( task: EvalTask, outcome: EvalOutcome, diff --git a/src/microbots/auto_memory/eval_swebenchverified/eval.py b/src/microbots/auto_memory/eval/swebenchverified.py similarity index 99% rename from src/microbots/auto_memory/eval_swebenchverified/eval.py rename to src/microbots/auto_memory/eval/swebenchverified.py index 4c3ef15..d83a2e0 100644 --- a/src/microbots/auto_memory/eval_swebenchverified/eval.py +++ b/src/microbots/auto_memory/eval/swebenchverified.py @@ -249,6 +249,7 @@ def teardown(self, repo_path: str) -> None: parser.add_argument("--repo", help='e.g. "django/django"') parser.add_argument("--instance-id", help='e.g. "django__django-11099"') parser.add_argument("--model", default="azure-openai/gpt-5.5") + parser.add_argument("--max-rounds", type=int, default=5) args = parser.parse_args() From a3237ae2babeffe8c46fa863c78d469c8086bd07 Mon Sep 17 00:00:00 2001 From: Kavya Sree Kaitepalli Date: Mon, 31 Aug 2026 11:04:59 +0000 Subject: [PATCH 06/13] Implement task registry for EvalTask instances --- src/microbots/auto_memory/cli.py | 90 ++++ .../auto_memory/eval/swebenchverified.py | 147 ++++-- src/microbots/auto_memory/orchestrator.py | 49 +- src/microbots/auto_memory/task.py | 97 +--- src/microbots/auto_memory/task_registry.py | 92 ++++ .../auto_memory/eval/test_swebenchverified.py | 477 ++++++++++++++++++ .../eval_swebenchverified/test_eval.py | 243 --------- test/auto_memory/test_cli.py | 108 ++++ test/auto_memory/test_orchestrator.py | 36 +- test/auto_memory/test_task.py | 180 +------ test/auto_memory/test_task_registry.py | 104 ++++ 11 files changed, 1119 insertions(+), 504 deletions(-) create mode 100644 src/microbots/auto_memory/cli.py create mode 100644 src/microbots/auto_memory/task_registry.py create mode 100644 test/auto_memory/eval/test_swebenchverified.py delete mode 100644 test/auto_memory/eval_swebenchverified/test_eval.py create mode 100644 test/auto_memory/test_cli.py create mode 100644 test/auto_memory/test_task_registry.py diff --git a/src/microbots/auto_memory/cli.py b/src/microbots/auto_memory/cli.py new file mode 100644 index 0000000..98d3147 --- /dev/null +++ b/src/microbots/auto_memory/cli.py @@ -0,0 +1,90 @@ +"""Command-line entry point for the auto-memory train/eval loop. + +Two modes, selected by ``--task``: + +- ``--task `` given: run the full train <-> eval loop for that + task (via ``run_train_eval_loop``). +- ``--task`` omitted: train only, no eval task (via ``run_training_loop``, + with empty feedback). +""" + +import argparse +import logging + +from microbots.auto_memory.orchestrator import run_train_eval_loop, run_training_loop +from microbots.auto_memory.task_registry import TASK_REGISTRY, discover_tasks + +logger = logging.getLogger(__name__) + +# Import every task module so their @register_task decorators fire. +discover_tasks() + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + """Parse CLI args, including task-specific args when ``--task`` is given. + + Parameters + ---------- + argv : list[str] | None + Args to parse. Defaults to ``sys.argv[1:]`` when ``None``. + + Returns + ------- + argparse.Namespace + The parsed args. + """ + parser = argparse.ArgumentParser(description="Run the auto-memory train/eval loop.") + parser.add_argument("--repo", required=True, help="Absolute path to the repo.") + parser.add_argument("--memory-dir", required=True, help="Directory for memory files.") + parser.add_argument("--model", required=True, help='Model, e.g. "azure-openai/gpt-5.5".') + parser.add_argument( + "--task", + choices=sorted(TASK_REGISTRY), + help="Eval task to run. Omit to only run training, with no eval task.", + ) + parser.add_argument("--max-rounds", type=int, default=5) + parser.add_argument("--training-iterations", type=int, default=1) + + # First pass just to discover --task, so we can register its + # task-specific flags before the real parse. + known_args, _ = parser.parse_known_args(argv) + if known_args.task: + TASK_REGISTRY[known_args.task].add_cli_args(parser) + + return parser.parse_args(argv) + +def main(argv: list[str] | None = None) -> None: + """CLI entry point: run training only, or the full train/eval loop. + + Parameters + ---------- + argv : list[str] | None + Args to parse. Defaults to ``sys.argv[1:]`` when ``None``. + """ + args = parse_args(argv) + + if not args.task: + run_training_loop( + repo_path=args.repo, + feedback="", + memory_dir=args.memory_dir, + model=args.model, + iterations=args.training_iterations, + ) + return + + task_cls = TASK_REGISTRY[args.task] + for task in task_cls.from_cli_args(args): + result = run_train_eval_loop( + repo_path=args.repo, + memory_dir=args.memory_dir, + model=args.model, + task=task, + max_rounds=args.max_rounds, + training_iterations=args.training_iterations, + ) + logger.info( + "task=%s passed=%s rounds_run=%d", args.task, result.passed, result.rounds_run + ) + +if __name__ == "__main__": + main() diff --git a/src/microbots/auto_memory/eval/swebenchverified.py b/src/microbots/auto_memory/eval/swebenchverified.py index d83a2e0..2cd974b 100644 --- a/src/microbots/auto_memory/eval/swebenchverified.py +++ b/src/microbots/auto_memory/eval/swebenchverified.py @@ -1,10 +1,11 @@ -"""Minimal SWE-bench-verified eval task. +"""SWE-bench-verified eval task. Loads instances from the SWE-bench-verified dataset, checks out each instance's repo at its base commit, has the agent attempt a fix, and verifies the result via ``swebench.harness.run_evaluation``. """ +import argparse import json import shutil import subprocess @@ -16,9 +17,10 @@ from pathlib import Path from datasets import load_dataset -import argparse -from microbots.auto_memory.task import CallbackResult, EvalTask -from microbots.auto_memory.orchestrator import run_train_eval_loop +from microbots.auto_memory.task import CallbackResult, EvalOutcome, EvalTask +from microbots.auto_memory.task_registry import register_task +from microbots.bot.WritingBot import WritingBot +from microbots.tools.tool_definitions.memory_tool import MemoryTool logger = getLogger(__name__) @@ -114,6 +116,7 @@ def load_instance_using_id(instance_id: str, dataset_name: str = SWE_BENCH_SUITE ) raise ValueError(f"instance_id not found: {instance_id}") +@register_task("swebenchverified") class SweBenchVerifiedTask(EvalTask): """Eval task that verifies a fix against one SWE-bench-verified instance. @@ -137,6 +140,47 @@ def __init__(self, instance: SweBenchInstance): """ self.instance = instance + @staticmethod + def add_cli_args(parser: argparse.ArgumentParser) -> None: + """Register this task's CLI flags on ``parser``. + + Parameters + ---------- + parser : argparse.ArgumentParser + The CLI's argument parser to add task-specific flags to. + """ + parser.add_argument( + "--instance-id", + help='SWE-bench-verified instance ID, e.g. "django__django-11099".', + ) + parser.add_argument( + "--swebench-repo", + help='Restrict to instances for this repo, e.g. "django/django". ' + "Ignored if --instance-id is given.", + ) + + @classmethod + def from_cli_args(cls, args: argparse.Namespace) -> list["SweBenchVerifiedTask"]: + """Build task(s) from parsed CLI args. + + Parameters + ---------- + args : argparse.Namespace + Parsed CLI args, expected to include ``instance_id`` and/or + ``swebench_repo`` (see ``add_cli_args``). + + Returns + ------- + list[SweBenchVerifiedTask] + One task per matching dataset instance. A single-element + list when ``--instance-id`` is given. + """ + if getattr(args, "instance_id", None): + instances = [load_instance_using_id(args.instance_id)] + else: + instances = load_instances_of_repo(repo=getattr(args, "swebench_repo", None)) + return [cls(instance) for instance in instances] + def setup(self, repo_path: str) -> None: """Clone the instance's repo and check out its base commit. @@ -242,29 +286,74 @@ def teardown(self, repo_path: str) -> None: """ subprocess.run(["rm", "-rf", repo_path], check=False) + def run(self, repo_path: str, memory_dir: str, model: str) -> EvalOutcome: + """Run one eval iteration: setup -> build_prompt -> WritingBot -> check -> teardown. + + Parameters + ---------- + repo_path : str + Absolute path to the repo to run the eval round against. + memory_dir : str + Directory containing memory files to give the agent via + ``MemoryTool``. + model : str + The model to use, in the format ``/``. + + Returns + ------- + EvalOutcome + The result of this eval round, including the agent's output, + the check verdict, and the round's log file path. + """ + self.setup(repo_path) + log_path = tempfile.mktemp(suffix=".log") + Path(log_path).write_text("") + + try: + try: + prompt = self.build_prompt(repo_path) + bot = WritingBot( + model=model, + folder_to_mount=repo_path, + additional_tools=[MemoryTool(memory_dir=memory_dir)], + ) + bot_result = bot.run(prompt) + + with open(log_path, "a") as f: + f.write(f"Agent output:\n{bot_result.result}\n") + + if not bot_result.status: + reason = f"Bot run failed: {bot_result.error}" + with open(log_path, "a") as f: + f.write(f"\n{reason}\n") + result = CallbackResult(passed=False, reason=reason) + else: + result = self.check(repo_path, bot_result.result or "", log_path) + + return EvalOutcome( + passed=result.passed, + output=bot_result.result, + result=result, + log_path=log_path, + ) + except Exception as exc: + logger.exception( + "SweBenchVerifiedTask.run: iteration raised %s", type(exc).__name__ + ) + with open(log_path, "a") as f: + f.write(f"\nException during eval iteration: {type(exc).__name__}: {exc}\n") + return EvalOutcome( + passed=False, + output=None, + result=CallbackResult( + passed=False, reason=f"{type(exc).__name__}: {exc}" + ), + log_path=log_path, + ) + finally: + try: + self.teardown(repo_path) + except Exception: + logger.exception("SweBenchVerifiedTask.run: teardown() raised exception; ignoring") + -if __name__ == "__main__": - - parser = argparse.ArgumentParser() - parser.add_argument("--repo", help='e.g. "django/django"') - parser.add_argument("--instance-id", help='e.g. "django__django-11099"') - parser.add_argument("--model", default="azure-openai/gpt-5.5") - - parser.add_argument("--max-rounds", type=int, default=5) - args = parser.parse_args() - - if args.instance_id: - instances = [load_instance_using_id(args.instance_id)] - else: - instances = load_instances_of_repo(repo=args.repo) - - for instance in instances: - task = SweBenchVerifiedTask(instance) - result = run_train_eval_loop( - repo_path=tempfile.mkdtemp(), - memory_dir="memory", - model=args.model, - task=task, - max_rounds=args.max_rounds, - ) - logger.info("%s: passed=%s", instance.instance_id, result.passed) diff --git a/src/microbots/auto_memory/orchestrator.py b/src/microbots/auto_memory/orchestrator.py index 8f1dcfb..f4ca198 100644 --- a/src/microbots/auto_memory/orchestrator.py +++ b/src/microbots/auto_memory/orchestrator.py @@ -11,7 +11,7 @@ from microbots.auto_memory.analyzer import build_feedback from microbots.auto_memory.task import EvalOutcome, EvalTask -from microbots.auto_memory.training.runner import run_training_loop +from microbots.auto_memory.training.runner import run_training logger = getLogger(__name__) @@ -36,6 +36,45 @@ class LoopResult: final_outcome: EvalOutcome outcomes: list[EvalOutcome] = field(default_factory=list) +def run_training_loop( + repo_path: str, + feedback: str, + memory_dir: str, + model: str, + iterations: int = 1, +) -> None: + """Run ``run_training`` ``iterations`` times, reusing the same memory dir. + + Shared by the eval-loop's retrain step and any training-only entry + point (e.g. a CLI) that needs to run training without an eval task. + + Parameters + ---------- + repo_path : str + Absolute path to the repo to train against. + feedback : str + Feedback from a prior failed eval attempt, or ``""`` if none. + memory_dir : str + Directory where the training agent reads/writes memory files. + model : str + The model to use, in the format ``/``. + iterations : int + Number of training passes to run, each reusing the same + ``memory_dir``. Defaults to 1. + """ + for iteration in range(1, iterations + 1): + logger.info( + "run_training_loop: training iteration %d/%d", + iteration, + iterations, + ) + run_training( + repo_path=repo_path, + feedback=feedback, + memory_dir=memory_dir, + model=model, + ) + def run_train_eval_loop( repo_path: str, memory_dir: str, @@ -48,9 +87,10 @@ def run_train_eval_loop( Each round runs ``task.run(...)``. If the task passes, the loop returns immediately. If it fails, feedback is built from the round's - log and used to retrain via ``run_training_loop`` before the next - round. The round's log file is always deleted before the next round - starts. + log and used to retrain via ``run_training`` (called + ``training_iterations`` times, each pass reusing the same + ``memory_dir``) before the next round. The round's log file is + always deleted before the next round starts. Parameters ---------- @@ -102,7 +142,6 @@ def run_train_eval_loop( ) try: feedback = build_feedback(task, outcome, repo_path, model) - run_training_loop( repo_path=repo_path, feedback=feedback, diff --git a/src/microbots/auto_memory/task.py b/src/microbots/auto_memory/task.py index 8a1de46..9eb00ae 100644 --- a/src/microbots/auto_memory/task.py +++ b/src/microbots/auto_memory/task.py @@ -5,16 +5,8 @@ clean up afterward. """ -import tempfile from abc import ABC, abstractmethod from dataclasses import dataclass -from logging import getLogger -from pathlib import Path - -from microbots.bot.WritingBot import WritingBot -from microbots.tools.tool_definitions.memory_tool import MemoryTool - -logger = getLogger(__name__) @dataclass class CallbackResult: @@ -57,23 +49,31 @@ class EvalOutcome: class EvalTask(ABC): """Base class for a single evaluation task in the train <-> eval loop. - Subclasses must implement ``setup``, ``build_prompt``, and ``check``, - and may override ``teardown`` and ``run`` as needed. + Subclasses must implement ``run``. ``setup``, ``build_prompt``, + ``check``, and ``teardown`` are optional hooks subclasses may use + to structure their own ``run`` implementation (see + ``SweBenchVerifiedTask`` for an example), but nothing in this base + class calls them automatically. """ - @abstractmethod def setup(self, repo_path: str) -> None: - """Required. Prepare repo/environment before the agent runs. + """Optional. Prepare repo/environment before the agent runs. + + Not called automatically; only useful if your ``run`` + implementation calls it. Parameters ---------- repo_path : str Absolute path to the repo to prepare. """ + pass - @abstractmethod def build_prompt(self, repo_path: str) -> str: - """Required. Return the task prompt/instructions for the agent. + """Optional. Return the task prompt/instructions for the agent. + + Not called automatically; only useful if your ``run`` + implementation calls it. Parameters ---------- @@ -83,12 +83,16 @@ def build_prompt(self, repo_path: str) -> str: Returns ------- str - The prompt/instructions to give the agent. + The prompt/instructions to give the agent. Empty string by + default. """ + return "" - @abstractmethod def check(self, repo_path: str, agent_output: str, log_path: str) -> CallbackResult: - """Required. Verify whether the task was actually completed correctly. + """Optional. Verify whether the task was actually completed correctly. + + Not called automatically; only useful if your ``run`` + implementation calls it. Parameters ---------- @@ -103,8 +107,10 @@ def check(self, repo_path: str, agent_output: str, log_path: str) -> CallbackRes Returns ------- CallbackResult - The pass/fail verdict and its reason. + The pass/fail verdict and its reason. Passes by default. """ + return CallbackResult(passed=True, reason="not checked") + def teardown(self, repo_path: str) -> None: """Optional. Clean up anything setup() created. @@ -116,10 +122,9 @@ def teardown(self, repo_path: str) -> None: """ pass + @abstractmethod def run(self, repo_path: str, memory_dir: str, model: str) -> EvalOutcome: - """Default eval iteration: setup -> build_prompt -> WritingBot -> check -> teardown. - Override this entirely if your task needs a different bot type, - additional tools, or custom retry/orchestration logic. + """Required. Run one eval iteration and return its outcome. Parameters ---------- @@ -137,53 +142,3 @@ def run(self, repo_path: str, memory_dir: str, model: str) -> EvalOutcome: The result of this eval round, including the agent's output, the check verdict, and the round's log file path. """ - self.setup(repo_path) - log_path = tempfile.mktemp(suffix=".log") - Path(log_path).write_text("") - - try: - try: - prompt = self.build_prompt(repo_path) - bot = WritingBot( - model=model, - folder_to_mount=repo_path, - additional_tools=[MemoryTool(memory_dir=memory_dir)], - ) - bot_result = bot.run(prompt) - - with open(log_path, "a") as f: - f.write(f"Agent output:\n{bot_result.result}\n") - - if not bot_result.status: - reason = f"Bot run failed: {bot_result.error}" - with open(log_path, "a") as f: - f.write(f"\n{reason}\n") - result = CallbackResult(passed=False, reason=reason) - else: - result = self.check(repo_path, bot_result.result or "", log_path) - - return EvalOutcome( - passed=result.passed, - output=bot_result.result, - result=result, - log_path=log_path, - ) - except Exception as exc: - logger.exception( - "EvalTask.run: iteration raised %s", type(exc).__name__ - ) - with open(log_path, "a") as f: - f.write(f"\nException during eval iteration: {type(exc).__name__}: {exc}\n") - return EvalOutcome( - passed=False, - output=None, - result=CallbackResult( - passed=False, reason=f"{type(exc).__name__}: {exc}" - ), - log_path=log_path, - ) - finally: - try: - self.teardown(repo_path) - except Exception: - logger.exception("EvalTask.run: teardown() raised exception; ignoring") diff --git a/src/microbots/auto_memory/task_registry.py b/src/microbots/auto_memory/task_registry.py new file mode 100644 index 0000000..63367f7 --- /dev/null +++ b/src/microbots/auto_memory/task_registry.py @@ -0,0 +1,92 @@ +"""Registry for constructing ``EvalTask`` instances by name. + +Tasks self-register via the ``@register_task`` decorator, so new task +types can be added without editing a central if/elif factory function. +Callers (e.g. a CLI) look tasks up by name via ``create_task``. +""" + +import importlib +import pkgutil + +from microbots.auto_memory.task import EvalTask + +TASK_REGISTRY: dict[str, type[EvalTask]] = {} + +def register_task(name: str): + """Register an ``EvalTask`` subclass under ``name`` as a class decorator. + + Parameters + ---------- + name : str + The key other code will use to look up this task via + ``create_task``, e.g. ``"swebenchverified"``. + + Returns + ------- + Callable[[type[EvalTask]], type[EvalTask]] + A decorator that registers the class in ``TASK_REGISTRY`` and + returns it unchanged. + """ + + def decorator(task_cls: type[EvalTask]) -> type[EvalTask]: + """Register ``task_cls`` in ``TASK_REGISTRY`` under the enclosing ``name``. + + Parameters + ---------- + task_cls : type[EvalTask] + The ``EvalTask`` subclass to register. + + Returns + ------- + type[EvalTask] + ``task_cls``, unchanged. + """ + TASK_REGISTRY[name] = task_cls + return task_cls + + return decorator + +def create_task(name: str, **kwargs) -> EvalTask: + """Construct a registered ``EvalTask`` by name. + + Parameters + ---------- + name : str + The registered task name, e.g. ``"swebenchverified"``. + **kwargs + Keyword arguments forwarded to the task's constructor. + + Returns + ------- + EvalTask + The constructed task instance. + + Raises + ------ + ValueError + If ``name`` has not been registered via ``register_task``. + """ + try: + task_cls = TASK_REGISTRY[name] + except KeyError: + raise ValueError( + f"Unknown task {name!r}. Registered tasks: {sorted(TASK_REGISTRY)}" + ) from None + return task_cls(**kwargs) + +def discover_tasks(package_name: str = "microbots.auto_memory.eval") -> None: + """Import every module in ``package_name`` so ``@register_task`` fires. + + Adding a new task only requires dropping a new module into this + package (with its own ``@register_task`` decorator) — no other code + needs to change to make it discoverable. + + Parameters + ---------- + package_name : str + Dotted path of the package to scan for task modules. Defaults + to ``"microbots.auto_memory.eval"``. + """ + package = importlib.import_module(package_name) + for module_info in pkgutil.iter_modules(package.__path__): + importlib.import_module(f"{package_name}.{module_info.name}") diff --git a/test/auto_memory/eval/test_swebenchverified.py b/test/auto_memory/eval/test_swebenchverified.py new file mode 100644 index 0000000..8754211 --- /dev/null +++ b/test/auto_memory/eval/test_swebenchverified.py @@ -0,0 +1,477 @@ +"""Unit tests for microbots.auto_memory.eval.swebenchverified.""" + +import json +import os +import sys +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../src/"))) + +from microbots.auto_memory.eval.swebenchverified import ( + SweBenchInstance, + SweBenchVerifiedTask, + load_instance_using_id, + load_instances_of_repo, +) +from microbots.auto_memory.task import CallbackResult + +MODULE = "microbots.auto_memory.eval.swebenchverified" + + +def _fake_rows(): + return [ + { + "instance_id": "django__django-1", + "repo": "django/django", + "base_commit": "abc123", + "problem_statement": "fix bug 1", + }, + { + "instance_id": "astropy__astropy-1", + "repo": "astropy/astropy", + "base_commit": "def456", + "problem_statement": "fix bug 2", + }, + { + "instance_id": "django__django-2", + "repo": "django/django", + "base_commit": "ghi789", + "problem_statement": "fix bug 3", + }, + ] + + +# --------------------------------------------------------------------------- +# load_instances_of_repo / load_instance_using_id +# --------------------------------------------------------------------------- + +@pytest.mark.unit +@patch(f"{MODULE}.load_dataset") +def test_load_instances_of_repo_filters_by_repo(mock_load_dataset): + mock_load_dataset.return_value = _fake_rows() + + instances = load_instances_of_repo(repo="django/django") + + assert [i.instance_id for i in instances] == ["django__django-1", "django__django-2"] + assert all(isinstance(i, SweBenchInstance) for i in instances) + + +@pytest.mark.unit +@patch(f"{MODULE}.load_dataset") +def test_load_instances_of_repo_returns_all_when_repo_none(mock_load_dataset): + mock_load_dataset.return_value = _fake_rows() + + instances = load_instances_of_repo(repo=None) + + assert len(instances) == 3 + + +@pytest.mark.unit +@patch(f"{MODULE}.load_dataset") +def test_load_instance_using_id_returns_matching_instance(mock_load_dataset): + mock_load_dataset.return_value = _fake_rows() + + instance = load_instance_using_id("astropy__astropy-1") + + assert instance.repo == "astropy/astropy" + assert instance.problem_statement == "fix bug 2" + + +@pytest.mark.unit +@patch(f"{MODULE}.load_dataset") +def test_load_instance_using_id_raises_when_not_found(mock_load_dataset): + mock_load_dataset.return_value = _fake_rows() + + with pytest.raises(ValueError, match="not found"): + load_instance_using_id("does-not-exist") + + +# --------------------------------------------------------------------------- +# SweBenchVerifiedTask.setup / build_prompt / teardown +# --------------------------------------------------------------------------- + +def _instance(): + return SweBenchInstance( + instance_id="django__django-1", + repo="django/django", + base_commit="abc123", + problem_statement="fix the bug", + ) + + +@pytest.mark.unit +@patch(f"{MODULE}.subprocess.run") +def test_setup_clones_and_checks_out_base_commit(mock_run): + task = SweBenchVerifiedTask(_instance()) + task.setup("/repo") + + clone_call, checkout_call = mock_run.call_args_list + assert clone_call.args[0] == ["git", "clone", "https://github.com/django/django.git", "/repo"] + assert checkout_call.args[0] == ["git", "checkout", "abc123"] + assert checkout_call.kwargs["cwd"] == "/repo" + + +@pytest.mark.unit +def test_build_prompt_returns_problem_statement(): + task = SweBenchVerifiedTask(_instance()) + assert task.build_prompt("/repo") == "fix the bug" + + +@pytest.mark.unit +@patch(f"{MODULE}.subprocess.run") +def test_teardown_removes_repo_path(mock_run): + task = SweBenchVerifiedTask(_instance()) + task.teardown("/repo") + + mock_run.assert_called_once_with(["rm", "-rf", "/repo"], check=False) + + +# --------------------------------------------------------------------------- +# SweBenchVerifiedTask.check +# --------------------------------------------------------------------------- + +def _make_fake_subprocess_run(resolved: bool, raise_on_harness: bool = False): + """Build a subprocess.run stand-in that fakes git diff + the harness call.""" + + def _fake_run(cmd, **kwargs): + if cmd[:2] == ["git", "diff"]: + return MagicMock(stdout="diff --git a/x.py b/x.py\n+fix", stderr="", returncode=0) + if "swebench.harness.run_evaluation" in cmd: + if raise_on_harness: + raise RuntimeError("harness crashed") + run_id = cmd[cmd.index("--run_id") + 1] + report_dir = kwargs["cwd"] + instance_id = cmd[cmd.index("--instance_ids") + 1] + report = {"resolved_ids": [instance_id] if resolved else []} + (Path(report_dir) / f"microbots-eval-agent.{run_id}.json").write_text(json.dumps(report)) + return MagicMock(stdout="harness ran\n", stderr="", returncode=0) + return MagicMock(stdout="", stderr="", returncode=0) + + return _fake_run + + +@pytest.mark.unit +@patch(f"{MODULE}.subprocess.run") +def test_check_passed_true_when_instance_in_resolved_ids(mock_run, tmp_path): + mock_run.side_effect = _make_fake_subprocess_run(resolved=True) + log_path = tmp_path / "check.log" + log_path.write_text("") + + task = SweBenchVerifiedTask(_instance()) + result = task.check("/repo", "agent output", str(log_path)) + + assert result.passed is True + assert result.reason == "resolved" + + +@pytest.mark.unit +@patch(f"{MODULE}.subprocess.run") +def test_check_passed_false_when_instance_not_in_resolved_ids(mock_run, tmp_path): + mock_run.side_effect = _make_fake_subprocess_run(resolved=False) + log_path = tmp_path / "check.log" + log_path.write_text("") + + task = SweBenchVerifiedTask(_instance()) + result = task.check("/repo", "agent output", str(log_path)) + + assert result.passed is False + assert result.reason == "not resolved" + + +@pytest.mark.unit +@patch(f"{MODULE}.subprocess.run") +def test_check_passed_false_when_report_file_never_written(mock_run, tmp_path): + # harness call succeeds but never writes a report file (e.g. it errored internally) + def _fake_run(cmd, **kwargs): + if cmd[:2] == ["git", "diff"]: + return MagicMock(stdout="diff", stderr="", returncode=0) + return MagicMock(stdout="", stderr="", returncode=1) + + mock_run.side_effect = _fake_run + log_path = tmp_path / "check.log" + log_path.write_text("") + + task = SweBenchVerifiedTask(_instance()) + result = task.check("/repo", "agent output", str(log_path)) + + assert result.passed is False + + +@pytest.mark.unit +@patch(f"{MODULE}.subprocess.run") +def test_check_appends_to_log_file_without_truncating_existing_content(mock_run, tmp_path): + mock_run.side_effect = _make_fake_subprocess_run(resolved=True) + log_path = tmp_path / "check.log" + log_path.write_text("Agent output:\nprevious content\n") + + task = SweBenchVerifiedTask(_instance()) + task.check("/repo", "agent output", str(log_path)) + + content = log_path.read_text() + assert "previous content" in content + assert "harness ran" in content + + +@pytest.mark.unit +@patch(f"{MODULE}.shutil.rmtree") +@patch(f"{MODULE}.subprocess.run") +def test_check_cleans_up_pred_path_and_report_dir_on_success(mock_run, mock_rmtree, tmp_path): + mock_run.side_effect = _make_fake_subprocess_run(resolved=True) + log_path = tmp_path / "check.log" + log_path.write_text("") + + task = SweBenchVerifiedTask(_instance()) + task.check("/repo", "agent output", str(log_path)) + + mock_rmtree.assert_called_once() + + +@pytest.mark.unit +@patch(f"{MODULE}.shutil.rmtree") +@patch(f"{MODULE}.subprocess.run") +def test_check_cleans_up_even_when_harness_raises(mock_run, mock_rmtree, tmp_path): + mock_run.side_effect = _make_fake_subprocess_run(resolved=True, raise_on_harness=True) + log_path = tmp_path / "check.log" + log_path.write_text("") + + task = SweBenchVerifiedTask(_instance()) + with pytest.raises(RuntimeError, match="harness crashed"): + task.check("/repo", "agent output", str(log_path)) + + mock_rmtree.assert_called_once() + + +# --------------------------------------------------------------------------- +# SweBenchVerifiedTask.run +# --------------------------------------------------------------------------- + +@pytest.mark.unit +@patch(f"{MODULE}.MemoryTool") +@patch(f"{MODULE}.WritingBot") +def test_run_calls_setup_build_prompt_check_teardown_in_order(mock_bot_cls, mock_memory_tool): + from microbots.MicroBot import BotRunResult + + mock_bot = MagicMock() + mock_bot.run.return_value = BotRunResult(status=True, result="agent did stuff", error=None) + mock_bot_cls.return_value = mock_bot + + task = SweBenchVerifiedTask(_instance()) + calls = [] + task.setup = lambda repo_path: calls.append(("setup", repo_path)) + task.build_prompt = lambda repo_path: "do the task" + task.check = lambda repo_path, agent_output, log_path: ( + calls.append(("check", repo_path, agent_output, log_path)) + or CallbackResult(passed=True, reason="ok") + ) + task.teardown = lambda repo_path: calls.append(("teardown", repo_path)) + + outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o") + + assert calls[0] == ("setup", "/repo") + assert calls[1] == ("check", "/repo", "agent did stuff", outcome.log_path) + assert calls[2] == ("teardown", "/repo") + assert outcome.passed is True + assert outcome.output == "agent did stuff" + + +@pytest.mark.unit +@patch(f"{MODULE}.MemoryTool") +@patch(f"{MODULE}.WritingBot") +def test_run_creates_log_file_before_check_is_called(mock_bot_cls, mock_memory_tool): + from microbots.MicroBot import BotRunResult + + mock_bot = MagicMock() + mock_bot.run.return_value = BotRunResult(status=True, result="output", error=None) + mock_bot_cls.return_value = mock_bot + + task = SweBenchVerifiedTask(_instance()) + task.setup = lambda repo_path: None + task.build_prompt = lambda repo_path: "do the task" + task.teardown = lambda repo_path: None + seen_log_exists = {} + + def _check(repo_path, agent_output, log_path): + seen_log_exists["exists"] = os.path.exists(log_path) + return CallbackResult(passed=True, reason="ok") + + task.check = _check + + task.run("/repo", "/memory", "azure-openai/gpt-4o") + + assert seen_log_exists["exists"] is True + + +@pytest.mark.unit +@patch(f"{MODULE}.MemoryTool") +@patch(f"{MODULE}.WritingBot") +def test_run_skips_check_when_bot_status_is_false(mock_bot_cls, mock_memory_tool): + from microbots.MicroBot import BotRunResult + + mock_bot = MagicMock() + mock_bot.run.return_value = BotRunResult(status=False, result=None, error="bot crashed") + mock_bot_cls.return_value = mock_bot + + task = SweBenchVerifiedTask(_instance()) + check_calls = [] + task.setup = lambda repo_path: None + task.build_prompt = lambda repo_path: "do the task" + task.check = lambda *a: check_calls.append(a) + task.teardown = lambda repo_path: None + + outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o") + + assert check_calls == [] + assert outcome.passed is False + assert "bot crashed" in outcome.result.reason + + +@pytest.mark.unit +@patch(f"{MODULE}.MemoryTool") +@patch(f"{MODULE}.WritingBot") +def test_run_converts_build_prompt_exception_to_failed_outcome(mock_bot_cls, mock_memory_tool): + task = SweBenchVerifiedTask(_instance()) + task.setup = lambda repo_path: None + task.teardown = lambda repo_path: None + + def _build_prompt(repo_path): + raise ValueError("bad prompt") + + task.build_prompt = _build_prompt + + outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o") + + assert outcome.passed is False + assert "bad prompt" in outcome.result.reason + with open(outcome.log_path) as f: + assert "bad prompt" in f.read() + + +@pytest.mark.unit +@patch(f"{MODULE}.MemoryTool") +@patch(f"{MODULE}.WritingBot") +def test_run_converts_check_exception_to_failed_outcome(mock_bot_cls, mock_memory_tool): + from microbots.MicroBot import BotRunResult + + mock_bot = MagicMock() + mock_bot.run.return_value = BotRunResult(status=True, result="output", error=None) + mock_bot_cls.return_value = mock_bot + + task = SweBenchVerifiedTask(_instance()) + task.setup = lambda repo_path: None + task.build_prompt = lambda repo_path: "do the task" + task.teardown = lambda repo_path: None + + def _check(repo_path, agent_output, log_path): + raise RuntimeError("check exploded") + + task.check = _check + + outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o") + + assert outcome.passed is False + assert "check exploded" in outcome.result.reason + + +@pytest.mark.unit +@patch(f"{MODULE}.MemoryTool") +@patch(f"{MODULE}.WritingBot") +def test_run_still_calls_teardown_when_body_raises(mock_bot_cls, mock_memory_tool): + mock_bot_cls.side_effect = RuntimeError("bot construction failed") + + task = SweBenchVerifiedTask(_instance()) + teardown_calls = [] + task.setup = lambda repo_path: None + task.build_prompt = lambda repo_path: "do the task" + task.teardown = lambda repo_path: teardown_calls.append(repo_path) + + task.run("/repo", "/memory", "azure-openai/gpt-4o") + + assert teardown_calls == ["/repo"] + + +@pytest.mark.unit +@patch(f"{MODULE}.MemoryTool") +@patch(f"{MODULE}.WritingBot") +def test_run_teardown_exception_does_not_clobber_returned_outcome(mock_bot_cls, mock_memory_tool): + from microbots.MicroBot import BotRunResult + + mock_bot = MagicMock() + mock_bot.run.return_value = BotRunResult(status=True, result="output", error=None) + mock_bot_cls.return_value = mock_bot + + task = SweBenchVerifiedTask(_instance()) + task.setup = lambda repo_path: None + task.build_prompt = lambda repo_path: "do the task" + task.check = lambda repo_path, agent_output, log_path: CallbackResult(passed=True, reason="ok") + + def _teardown(repo_path): + raise RuntimeError("teardown boom") + + task.teardown = _teardown + + outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o") + + # teardown() raised, but the already-computed EvalOutcome must still be returned + assert outcome.passed is True + + + +# --------------------------------------------------------------------------- +# SweBenchVerifiedTask.add_cli_args / from_cli_args +# --------------------------------------------------------------------------- + +@pytest.mark.unit +def test_add_cli_args_registers_instance_id_and_repo_flags(): + import argparse + + parser = argparse.ArgumentParser() + SweBenchVerifiedTask.add_cli_args(parser) + + args = parser.parse_args(["--instance-id", "django__django-1", "--swebench-repo", "django/django"]) + assert args.instance_id == "django__django-1" + assert args.swebench_repo == "django/django" + + +@pytest.mark.unit +@patch(f"{MODULE}.load_instance_using_id") +def test_from_cli_args_uses_instance_id_when_given(mock_load_instance_using_id): + mock_load_instance_using_id.return_value = _instance() + args = MagicMock(instance_id="django__django-1", swebench_repo=None) + + tasks = SweBenchVerifiedTask.from_cli_args(args) + + mock_load_instance_using_id.assert_called_once_with("django__django-1") + assert len(tasks) == 1 + assert isinstance(tasks[0], SweBenchVerifiedTask) + assert tasks[0].instance == _instance() + + +@pytest.mark.unit +@patch(f"{MODULE}.load_instances_of_repo") +def test_from_cli_args_falls_back_to_repo_filter_when_no_instance_id(mock_load_instances_of_repo): + mock_load_instances_of_repo.return_value = [_instance(), _instance()] + args = MagicMock(instance_id=None, swebench_repo="django/django") + + tasks = SweBenchVerifiedTask.from_cli_args(args) + + mock_load_instances_of_repo.assert_called_once_with(repo="django/django") + assert len(tasks) == 2 + assert all(isinstance(t, SweBenchVerifiedTask) for t in tasks) + + +@pytest.mark.unit +@patch(f"{MODULE}.load_instances_of_repo") +def test_from_cli_args_handles_missing_attrs_gracefully(mock_load_instances_of_repo): + """Namespace without instance_id/swebench_repo attrs at all (not just None).""" + mock_load_instances_of_repo.return_value = [_instance()] + + class _EmptyArgs: + pass + + tasks = SweBenchVerifiedTask.from_cli_args(_EmptyArgs()) + + mock_load_instances_of_repo.assert_called_once_with(repo=None) + assert len(tasks) == 1 diff --git a/test/auto_memory/eval_swebenchverified/test_eval.py b/test/auto_memory/eval_swebenchverified/test_eval.py deleted file mode 100644 index 4115e45..0000000 --- a/test/auto_memory/eval_swebenchverified/test_eval.py +++ /dev/null @@ -1,243 +0,0 @@ -"""Unit tests for microbots.auto_memory.eval_swebenchverified.eval.""" - -import json -import os -import sys -from pathlib import Path -from unittest.mock import MagicMock, patch - -import pytest - -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../src/"))) - -from microbots.auto_memory.eval_swebenchverified.eval import ( - SweBenchInstance, - SweBenchVerifiedTask, - load_instance_using_id, - load_instances_of_repo, -) - -MODULE = "microbots.auto_memory.eval_swebenchverified.eval" - - -def _fake_rows(): - return [ - { - "instance_id": "django__django-1", - "repo": "django/django", - "base_commit": "abc123", - "problem_statement": "fix bug 1", - }, - { - "instance_id": "astropy__astropy-1", - "repo": "astropy/astropy", - "base_commit": "def456", - "problem_statement": "fix bug 2", - }, - { - "instance_id": "django__django-2", - "repo": "django/django", - "base_commit": "ghi789", - "problem_statement": "fix bug 3", - }, - ] - - -# --------------------------------------------------------------------------- -# load_instances_of_repo / load_instance_using_id -# --------------------------------------------------------------------------- - -@pytest.mark.unit -@patch(f"{MODULE}.load_dataset") -def test_load_instances_of_repo_filters_by_repo(mock_load_dataset): - mock_load_dataset.return_value = _fake_rows() - - instances = load_instances_of_repo(repo="django/django") - - assert [i.instance_id for i in instances] == ["django__django-1", "django__django-2"] - assert all(isinstance(i, SweBenchInstance) for i in instances) - - -@pytest.mark.unit -@patch(f"{MODULE}.load_dataset") -def test_load_instances_of_repo_returns_all_when_repo_none(mock_load_dataset): - mock_load_dataset.return_value = _fake_rows() - - instances = load_instances_of_repo(repo=None) - - assert len(instances) == 3 - - -@pytest.mark.unit -@patch(f"{MODULE}.load_dataset") -def test_load_instance_using_id_returns_matching_instance(mock_load_dataset): - mock_load_dataset.return_value = _fake_rows() - - instance = load_instance_using_id("astropy__astropy-1") - - assert instance.repo == "astropy/astropy" - assert instance.problem_statement == "fix bug 2" - - -@pytest.mark.unit -@patch(f"{MODULE}.load_dataset") -def test_load_instance_using_id_raises_when_not_found(mock_load_dataset): - mock_load_dataset.return_value = _fake_rows() - - with pytest.raises(ValueError, match="not found"): - load_instance_using_id("does-not-exist") - - -# --------------------------------------------------------------------------- -# SweBenchVerifiedTask.setup / build_prompt / teardown -# --------------------------------------------------------------------------- - -def _instance(): - return SweBenchInstance( - instance_id="django__django-1", - repo="django/django", - base_commit="abc123", - problem_statement="fix the bug", - ) - - -@pytest.mark.unit -@patch(f"{MODULE}.subprocess.run") -def test_setup_clones_and_checks_out_base_commit(mock_run): - task = SweBenchVerifiedTask(_instance()) - task.setup("/repo") - - clone_call, checkout_call = mock_run.call_args_list - assert clone_call.args[0] == ["git", "clone", "https://github.com/django/django.git", "/repo"] - assert checkout_call.args[0] == ["git", "checkout", "abc123"] - assert checkout_call.kwargs["cwd"] == "/repo" - - -@pytest.mark.unit -def test_build_prompt_returns_problem_statement(): - task = SweBenchVerifiedTask(_instance()) - assert task.build_prompt("/repo") == "fix the bug" - - -@pytest.mark.unit -@patch(f"{MODULE}.subprocess.run") -def test_teardown_removes_repo_path(mock_run): - task = SweBenchVerifiedTask(_instance()) - task.teardown("/repo") - - mock_run.assert_called_once_with(["rm", "-rf", "/repo"], check=False) - - -# --------------------------------------------------------------------------- -# SweBenchVerifiedTask.check -# --------------------------------------------------------------------------- - -def _make_fake_subprocess_run(resolved: bool, raise_on_harness: bool = False): - """Build a subprocess.run stand-in that fakes git diff + the harness call.""" - - def _fake_run(cmd, **kwargs): - if cmd[:2] == ["git", "diff"]: - return MagicMock(stdout="diff --git a/x.py b/x.py\n+fix", stderr="", returncode=0) - if "swebench.harness.run_evaluation" in cmd: - if raise_on_harness: - raise RuntimeError("harness crashed") - run_id = cmd[cmd.index("--run_id") + 1] - report_dir = kwargs["cwd"] - instance_id = cmd[cmd.index("--instance_ids") + 1] - report = {"resolved_ids": [instance_id] if resolved else []} - (Path(report_dir) / f"microbots-eval-agent.{run_id}.json").write_text(json.dumps(report)) - return MagicMock(stdout="harness ran\n", stderr="", returncode=0) - return MagicMock(stdout="", stderr="", returncode=0) - - return _fake_run - - -@pytest.mark.unit -@patch(f"{MODULE}.subprocess.run") -def test_check_passed_true_when_instance_in_resolved_ids(mock_run, tmp_path): - mock_run.side_effect = _make_fake_subprocess_run(resolved=True) - log_path = tmp_path / "check.log" - log_path.write_text("") - - task = SweBenchVerifiedTask(_instance()) - result = task.check("/repo", "agent output", str(log_path)) - - assert result.passed is True - assert result.reason == "resolved" - - -@pytest.mark.unit -@patch(f"{MODULE}.subprocess.run") -def test_check_passed_false_when_instance_not_in_resolved_ids(mock_run, tmp_path): - mock_run.side_effect = _make_fake_subprocess_run(resolved=False) - log_path = tmp_path / "check.log" - log_path.write_text("") - - task = SweBenchVerifiedTask(_instance()) - result = task.check("/repo", "agent output", str(log_path)) - - assert result.passed is False - assert result.reason == "not resolved" - - -@pytest.mark.unit -@patch(f"{MODULE}.subprocess.run") -def test_check_passed_false_when_report_file_never_written(mock_run, tmp_path): - # harness call succeeds but never writes a report file (e.g. it errored internally) - def _fake_run(cmd, **kwargs): - if cmd[:2] == ["git", "diff"]: - return MagicMock(stdout="diff", stderr="", returncode=0) - return MagicMock(stdout="", stderr="", returncode=1) - - mock_run.side_effect = _fake_run - log_path = tmp_path / "check.log" - log_path.write_text("") - - task = SweBenchVerifiedTask(_instance()) - result = task.check("/repo", "agent output", str(log_path)) - - assert result.passed is False - - -@pytest.mark.unit -@patch(f"{MODULE}.subprocess.run") -def test_check_appends_to_log_file_without_truncating_existing_content(mock_run, tmp_path): - mock_run.side_effect = _make_fake_subprocess_run(resolved=True) - log_path = tmp_path / "check.log" - log_path.write_text("Agent output:\nprevious content\n") - - task = SweBenchVerifiedTask(_instance()) - task.check("/repo", "agent output", str(log_path)) - - content = log_path.read_text() - assert "previous content" in content - assert "harness ran" in content - - -@pytest.mark.unit -@patch(f"{MODULE}.shutil.rmtree") -@patch(f"{MODULE}.subprocess.run") -def test_check_cleans_up_pred_path_and_report_dir_on_success(mock_run, mock_rmtree, tmp_path): - mock_run.side_effect = _make_fake_subprocess_run(resolved=True) - log_path = tmp_path / "check.log" - log_path.write_text("") - - task = SweBenchVerifiedTask(_instance()) - task.check("/repo", "agent output", str(log_path)) - - mock_rmtree.assert_called_once() - - -@pytest.mark.unit -@patch(f"{MODULE}.shutil.rmtree") -@patch(f"{MODULE}.subprocess.run") -def test_check_cleans_up_even_when_harness_raises(mock_run, mock_rmtree, tmp_path): - mock_run.side_effect = _make_fake_subprocess_run(resolved=True, raise_on_harness=True) - log_path = tmp_path / "check.log" - log_path.write_text("") - - task = SweBenchVerifiedTask(_instance()) - with pytest.raises(RuntimeError, match="harness crashed"): - task.check("/repo", "agent output", str(log_path)) - - mock_rmtree.assert_called_once() diff --git a/test/auto_memory/test_cli.py b/test/auto_memory/test_cli.py new file mode 100644 index 0000000..da3ca8b --- /dev/null +++ b/test/auto_memory/test_cli.py @@ -0,0 +1,108 @@ +"""Unit tests for microbots.auto_memory.cli.""" + +import os +import sys +from unittest.mock import MagicMock, patch + +import pytest + +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../src/"))) + +from microbots.auto_memory.cli import main, parse_args + +MODULE = "microbots.auto_memory.cli" + +BASE_ARGS = ["--repo", "/repo", "--memory-dir", "/memory", "--model", "azure-openai/gpt-4o"] + + +@pytest.mark.unit +def test_parse_args_defaults(): + args = parse_args(BASE_ARGS) + + assert args.repo == "/repo" + assert args.memory_dir == "/memory" + assert args.model == "azure-openai/gpt-4o" + assert args.task is None + assert args.max_rounds == 5 + assert args.training_iterations == 1 + + +@pytest.mark.unit +def test_parse_args_with_known_task_adds_its_flags(): + args = parse_args(BASE_ARGS + ["--task", "swebenchverified", "--instance-id", "django__django-1"]) + + assert args.task == "swebenchverified" + assert args.instance_id == "django__django-1" + + +@pytest.mark.unit +def test_parse_args_rejects_unknown_task(): + with pytest.raises(SystemExit): + parse_args(BASE_ARGS + ["--task", "does-not-exist"]) + + +@pytest.mark.unit +@patch(f"{MODULE}.run_training_loop") +def test_main_runs_training_only_when_task_omitted(mock_run_training_loop): + main(BASE_ARGS) + + mock_run_training_loop.assert_called_once_with( + repo_path="/repo", + feedback="", + memory_dir="/memory", + model="azure-openai/gpt-4o", + iterations=1, + ) + + +@pytest.mark.unit +@patch(f"{MODULE}.run_train_eval_loop") +@patch(f"{MODULE}.run_training_loop") +def test_main_does_not_run_eval_loop_when_task_omitted(mock_run_training_loop, mock_run_train_eval_loop): + main(BASE_ARGS) + + mock_run_train_eval_loop.assert_not_called() + + +@pytest.mark.unit +@patch(f"{MODULE}.run_train_eval_loop") +def test_main_runs_eval_loop_for_each_task_when_task_given(mock_run_train_eval_loop): + fake_task = MagicMock() + mock_run_train_eval_loop.return_value = MagicMock(passed=True, rounds_run=1) + + with patch(f"{MODULE}.TASK_REGISTRY", {"swebenchverified": MagicMock(from_cli_args=lambda args: [fake_task])}): + main(BASE_ARGS + ["--task", "swebenchverified"]) + + mock_run_train_eval_loop.assert_called_once_with( + repo_path="/repo", + memory_dir="/memory", + model="azure-openai/gpt-4o", + task=fake_task, + max_rounds=5, + training_iterations=1, + ) + + +@pytest.mark.unit +@patch(f"{MODULE}.run_training_loop") +def test_main_does_not_run_training_only_path_when_task_given(mock_run_training_loop): + fake_task = MagicMock() + + with patch(f"{MODULE}.TASK_REGISTRY", {"swebenchverified": MagicMock(from_cli_args=lambda args: [fake_task])}): + with patch(f"{MODULE}.run_train_eval_loop") as mock_run_train_eval_loop: + mock_run_train_eval_loop.return_value = MagicMock(passed=True, rounds_run=1) + main(BASE_ARGS + ["--task", "swebenchverified"]) + + mock_run_training_loop.assert_not_called() + + +@pytest.mark.unit +@patch(f"{MODULE}.run_train_eval_loop") +def test_main_runs_eval_loop_once_per_returned_task(mock_run_train_eval_loop): + fake_tasks = [MagicMock(), MagicMock()] + mock_run_train_eval_loop.return_value = MagicMock(passed=False, rounds_run=5) + + with patch(f"{MODULE}.TASK_REGISTRY", {"swebenchverified": MagicMock(from_cli_args=lambda args: fake_tasks)}): + main(BASE_ARGS + ["--task", "swebenchverified"]) + + assert mock_run_train_eval_loop.call_count == 2 diff --git a/test/auto_memory/test_orchestrator.py b/test/auto_memory/test_orchestrator.py index ac21773..e869c55 100644 --- a/test/auto_memory/test_orchestrator.py +++ b/test/auto_memory/test_orchestrator.py @@ -9,7 +9,7 @@ sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../src/"))) -from microbots.auto_memory.orchestrator import LoopResult, run_train_eval_loop +from microbots.auto_memory.orchestrator import LoopResult, run_train_eval_loop, run_training_loop from microbots.auto_memory.task import CallbackResult, EvalOutcome @@ -183,3 +183,37 @@ def test_run_training_exception_does_not_crash_loop(mock_build_feedback, mock_ru assert result.passed is True assert result.rounds_run == 2 assert not Path(log1).exists() + + +@pytest.mark.unit +@patch("microbots.auto_memory.orchestrator.run_training") +def test_run_training_loop_calls_run_training_once_by_default(mock_run_training): + run_training_loop(repo_path="/repo", feedback="fb", memory_dir="/memory", model="azure-openai/gpt-4o") + + mock_run_training.assert_called_once_with( + repo_path="/repo", feedback="fb", memory_dir="/memory", model="azure-openai/gpt-4o" + ) + + +@pytest.mark.unit +@patch("microbots.auto_memory.orchestrator.run_training") +def test_run_training_loop_calls_run_training_n_times(mock_run_training): + run_training_loop( + repo_path="/repo", feedback="fb", memory_dir="/memory", model="azure-openai/gpt-4o", iterations=3 + ) + + assert mock_run_training.call_count == 3 + mock_run_training.assert_called_with( + repo_path="/repo", feedback="fb", memory_dir="/memory", model="azure-openai/gpt-4o" + ) + + +@pytest.mark.unit +@patch("microbots.auto_memory.orchestrator.run_training") +def test_run_training_loop_reuses_same_memory_dir_each_pass(mock_run_training): + run_training_loop( + repo_path="/repo", feedback="fb", memory_dir="/memory", model="azure-openai/gpt-4o", iterations=4 + ) + + memory_dirs = {call.kwargs["memory_dir"] for call in mock_run_training.call_args_list} + assert memory_dirs == {"/memory"} diff --git a/test/auto_memory/test_task.py b/test/auto_memory/test_task.py index bfc2cb6..aee0f9d 100644 --- a/test/auto_memory/test_task.py +++ b/test/auto_memory/test_task.py @@ -2,191 +2,61 @@ import os import sys -from unittest.mock import MagicMock, patch import pytest sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../src/"))) from microbots.auto_memory.task import CallbackResult, EvalOutcome, EvalTask -from microbots.MicroBot import BotRunResult -class _StubTask(EvalTask): - """A minimal concrete EvalTask used to exercise the base run() logic.""" +class _RunOnlyTask(EvalTask): + """A task that overrides only run(), never touching the optional hooks.""" - def __init__(self, check_result=None, check_side_effect=None, build_prompt_side_effect=None): - self.setup_calls = [] - self.teardown_calls = [] - self.check_calls = [] - self._check_result = check_result or CallbackResult(passed=True, reason="ok") - self._check_side_effect = check_side_effect - self._build_prompt_side_effect = build_prompt_side_effect - - def setup(self, repo_path): - self.setup_calls.append(repo_path) - - def build_prompt(self, repo_path): - if self._build_prompt_side_effect: - raise self._build_prompt_side_effect - return "do the task" - - def check(self, repo_path, agent_output, log_path): - self.check_calls.append((repo_path, agent_output, log_path)) - if self._check_side_effect: - raise self._check_side_effect - return self._check_result - - def teardown(self, repo_path): - self.teardown_calls.append(repo_path) - - -class _RaisingTeardownTask(_StubTask): - def teardown(self, repo_path): - super().teardown(repo_path) - raise RuntimeError("teardown boom") - - -class _DefaultTeardownTask(EvalTask): - """A task that relies on EvalTask's default no-op teardown.""" - - def setup(self, repo_path): - pass - - def build_prompt(self, repo_path): - return "do the task" - - def check(self, repo_path, agent_output, log_path): - return CallbackResult(passed=True, reason="ok") + def run(self, repo_path, memory_dir, model): + return EvalOutcome( + passed=True, + output="custom output", + result=None, + log_path="/dev/null", + ) @pytest.mark.unit -def test_setup_and_check_are_abstract(): +def test_run_is_abstract(): with pytest.raises(TypeError): EvalTask() @pytest.mark.unit -@patch("microbots.auto_memory.task.MemoryTool") -@patch("microbots.auto_memory.task.WritingBot") -def test_run_calls_setup_build_prompt_check_teardown_in_order(mock_bot_cls, mock_memory_tool): - mock_bot = MagicMock() - mock_bot.run.return_value = BotRunResult(status=True, result="agent did stuff", error=None) - mock_bot_cls.return_value = mock_bot - - task = _StubTask() +def test_subclass_overriding_only_run_is_instantiable(): + task = _RunOnlyTask() outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o") - assert task.setup_calls == ["/repo"] - assert task.check_calls == [("/repo", "agent did stuff", outcome.log_path)] - assert task.teardown_calls == ["/repo"] assert outcome.passed is True - assert outcome.output == "agent did stuff" - - -@pytest.mark.unit -@patch("microbots.auto_memory.task.MemoryTool") -@patch("microbots.auto_memory.task.WritingBot") -def test_run_creates_log_file_before_check_is_called(mock_bot_cls, mock_memory_tool): - mock_bot = MagicMock() - mock_bot.run.return_value = BotRunResult(status=True, result="output", error=None) - mock_bot_cls.return_value = mock_bot - - seen_log_exists = {} - - class _CheckingTask(_StubTask): - def check(self, repo_path, agent_output, log_path): - seen_log_exists["exists"] = os.path.exists(log_path) - return super().check(repo_path, agent_output, log_path) - - task = _CheckingTask() - task.run("/repo", "/memory", "azure-openai/gpt-4o") - - assert seen_log_exists["exists"] is True - - -@pytest.mark.unit -@patch("microbots.auto_memory.task.MemoryTool") -@patch("microbots.auto_memory.task.WritingBot") -def test_run_skips_check_when_bot_status_is_false(mock_bot_cls, mock_memory_tool): - mock_bot = MagicMock() - mock_bot.run.return_value = BotRunResult(status=False, result=None, error="bot crashed") - mock_bot_cls.return_value = mock_bot - - task = _StubTask() - outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o") - - assert task.check_calls == [] - assert outcome.passed is False - assert "bot crashed" in outcome.result.reason + assert outcome.output == "custom output" @pytest.mark.unit -@patch("microbots.auto_memory.task.MemoryTool") -@patch("microbots.auto_memory.task.WritingBot") -def test_run_converts_build_prompt_exception_to_failed_outcome(mock_bot_cls, mock_memory_tool): - task = _StubTask(build_prompt_side_effect=ValueError("bad prompt")) - outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o") - - assert outcome.passed is False - assert "bad prompt" in outcome.result.reason - with open(outcome.log_path) as f: - assert "bad prompt" in f.read() +def test_default_setup_is_a_noop(): + # Should not raise. + _RunOnlyTask().setup("/repo") @pytest.mark.unit -@patch("microbots.auto_memory.task.MemoryTool") -@patch("microbots.auto_memory.task.WritingBot") -def test_run_converts_check_exception_to_failed_outcome(mock_bot_cls, mock_memory_tool): - mock_bot = MagicMock() - mock_bot.run.return_value = BotRunResult(status=True, result="output", error=None) - mock_bot_cls.return_value = mock_bot - - task = _StubTask(check_side_effect=RuntimeError("check exploded")) - outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o") - - assert outcome.passed is False - assert "check exploded" in outcome.result.reason +def test_default_teardown_is_a_noop(): + # Should not raise. + _RunOnlyTask().teardown("/repo") @pytest.mark.unit -@patch("microbots.auto_memory.task.MemoryTool") -@patch("microbots.auto_memory.task.WritingBot") -def test_run_still_calls_teardown_when_body_raises(mock_bot_cls, mock_memory_tool): - mock_bot_cls.side_effect = RuntimeError("bot construction failed") - - task = _StubTask() - task.run("/repo", "/memory", "azure-openai/gpt-4o") - - assert task.teardown_calls == ["/repo"] +def test_default_build_prompt_returns_empty_string(): + assert _RunOnlyTask().build_prompt("/repo") == "" @pytest.mark.unit -@patch("microbots.auto_memory.task.MemoryTool") -@patch("microbots.auto_memory.task.WritingBot") -def test_run_teardown_exception_does_not_clobber_returned_outcome(mock_bot_cls, mock_memory_tool): - mock_bot = MagicMock() - mock_bot.run.return_value = BotRunResult(status=True, result="output", error=None) - mock_bot_cls.return_value = mock_bot - - task = _RaisingTeardownTask() - outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o") - - # teardown() raised, but the already-computed EvalOutcome must still be returned - assert isinstance(outcome, EvalOutcome) - assert outcome.passed is True - - -@pytest.mark.unit -@patch("microbots.auto_memory.task.MemoryTool") -@patch("microbots.auto_memory.task.WritingBot") -def test_run_uses_default_noop_teardown_when_not_overridden(mock_bot_cls, mock_memory_tool): - mock_bot = MagicMock() - mock_bot.run.return_value = BotRunResult(status=True, result="output", error=None) - mock_bot_cls.return_value = mock_bot - - task = _DefaultTeardownTask() - outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o") - - assert outcome.passed is True +def test_default_check_passes_by_default(): + result = _RunOnlyTask().check("/repo", "output", "/log") + assert isinstance(result, CallbackResult) + assert result.passed is True diff --git a/test/auto_memory/test_task_registry.py b/test/auto_memory/test_task_registry.py new file mode 100644 index 0000000..d06247f --- /dev/null +++ b/test/auto_memory/test_task_registry.py @@ -0,0 +1,104 @@ +"""Unit tests for microbots.auto_memory.task_registry.""" + +import os +import sys +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pytest + +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../src/"))) + +from microbots.auto_memory.task import EvalTask +from microbots.auto_memory.task_registry import TASK_REGISTRY, create_task, discover_tasks, register_task + +MODULE_PATH = "microbots.auto_memory.task_registry" + + +class _DummyTask(EvalTask): + def __init__(self, value=None): + self.value = value + + def setup(self, repo_path): + pass + + def build_prompt(self): + return "prompt" + + def check(self, output): + pass + + def teardown(self, repo_path): + pass + + def run(self, repo_path, memory_dir, model): + return super().run(repo_path, memory_dir, model) + + +@pytest.fixture(autouse=True) +def _clean_registry(): + """Snapshot/restore TASK_REGISTRY so tests don't leak state into each other.""" + original = dict(TASK_REGISTRY) + yield + TASK_REGISTRY.clear() + TASK_REGISTRY.update(original) + + +@pytest.mark.unit +def test_register_task_adds_class_to_registry(): + register_task("dummy")(_DummyTask) + + assert TASK_REGISTRY["dummy"] is _DummyTask + + +@pytest.mark.unit +def test_register_task_returns_class_unchanged(): + decorated = register_task("dummy")(_DummyTask) + + assert decorated is _DummyTask + + +@pytest.mark.unit +def test_create_task_constructs_registered_task_with_kwargs(): + register_task("dummy")(_DummyTask) + + task = create_task("dummy", value=42) + + assert isinstance(task, _DummyTask) + assert task.value == 42 + + +@pytest.mark.unit +def test_create_task_raises_for_unknown_name(): + with pytest.raises(ValueError, match="Unknown task 'nonexistent'"): + create_task("nonexistent") + + +@pytest.mark.unit +def test_discover_tasks_registers_swebenchverified(): + """Non-destructive: confirms discover_tasks() works against the real package.""" + discover_tasks() + + assert "swebenchverified" in TASK_REGISTRY + + +@pytest.mark.unit +@patch(f"{MODULE_PATH}.importlib.import_module") +@patch(f"{MODULE_PATH}.pkgutil.iter_modules") +def test_discover_tasks_imports_every_module_found_in_package(mock_iter_modules, mock_import_module): + fake_package = MagicMock() + fake_package.__path__ = ["/fake/path"] + mock_import_module.side_effect = ( + lambda name: fake_package if name == "fake.pkg" else MagicMock() + ) + mock_iter_modules.return_value = [ + SimpleNamespace(name="task_a"), + SimpleNamespace(name="task_b"), + ] + + discover_tasks(package_name="fake.pkg") + + mock_iter_modules.assert_called_once_with(["/fake/path"]) + mock_import_module.assert_any_call("fake.pkg") + mock_import_module.assert_any_call("fake.pkg.task_a") + mock_import_module.assert_any_call("fake.pkg.task_b") From 4a952abbb917f70b422502f31d2520c3cd68dd7e Mon Sep 17 00:00:00 2001 From: Kavya Sree Kaitepalli Date: Wed, 2 Sep 2026 07:18:52 +0000 Subject: [PATCH 07/13] Refactor evalTask module and improve functionality --- pyproject.toml | 1 + requirements.txt | 11 - src/microbots/auto_memory/__init__.py | 2 +- src/microbots/auto_memory/analyzer.py | 2 +- src/microbots/auto_memory/cli.py | 64 +-- .../auto_memory/eval/swebenchverified.py | 128 ++++-- .../auto_memory/{task.py => evalTask.py} | 48 ++- src/microbots/auto_memory/orchestrator.py | 195 +++++++-- src/microbots/auto_memory/task_registry.py | 3 +- src/microbots/auto_memory/workdir.py | 383 ++++++++++++++++++ .../auto_memory/eval/test_swebenchverified.py | 94 +++-- test/auto_memory/test_analyzer.py | 2 +- test/auto_memory/test_cli.py | 127 ++++-- test/auto_memory/test_orchestrator.py | 295 ++++++++++++-- test/auto_memory/test_task.py | 27 +- test/auto_memory/test_task_registry.py | 2 +- test/auto_memory/test_workdir.py | 94 +++++ 17 files changed, 1276 insertions(+), 202 deletions(-) rename src/microbots/auto_memory/{task.py => evalTask.py} (73%) create mode 100644 src/microbots/auto_memory/workdir.py create mode 100644 test/auto_memory/test_workdir.py diff --git a/pyproject.toml b/pyproject.toml index 3917589..a66a138 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,7 @@ requires-python = ">=3.11" ghcp = ["github-copilot-sdk==0.3.0"] azure_ad = ["azure-identity>=1.15.0"] dev = ["pre-commit>=3.7", "numpydoc>=1.8"] +training = ["datasets==4.5.0", "swebench==4.1.0"] [tool.setuptools.dynamic] dependencies = { file = ["requirements.txt"] } diff --git a/requirements.txt b/requirements.txt index 4bf03b5..bfb90ec 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,36 +10,26 @@ certifi==2025.8.3 charset-normalizer==3.4.3 click==8.3.0 coverage==7.11.3 -datasets==4.5.0 -dill==0.4.0 distro==1.9.0 docker==7.1.0 docstring_parser==0.17.0 fastapi==0.116.1 -filelock==3.20.3 frozenlist==1.7.0 -fsspec==2025.10.0 h11==0.16.0 -hf-xet==1.2.0 httpcore==1.0.9 httpx==0.28.1 -huggingface_hub==1.3.2 idna==3.10 iniconfig==2.1.0 jiter==0.11.0 markdown-it-py==4.0.0 mdurl==0.1.2 multidict==6.6.4 -multiprocess==0.70.18 -numpy==1.26.4 openai==1.107.3 packaging==25.0 -pandas==3.0.0 pexpect==4.9.0 pluggy==1.6.0 propcache==0.3.2 ptyprocess==0.7.0 -pyarrow==23.0.0 pydantic==2.11.9 pydantic_core==2.33.2 Pygments==2.19.2 @@ -62,5 +52,4 @@ typing-inspection==0.4.1 typing_extensions==4.15.0 urllib3==2.5.0 uvicorn==0.35.0 -xxhash==3.6.0 yarl==1.20.1 diff --git a/src/microbots/auto_memory/__init__.py b/src/microbots/auto_memory/__init__.py index 27f9a4b..e959bfa 100644 --- a/src/microbots/auto_memory/__init__.py +++ b/src/microbots/auto_memory/__init__.py @@ -4,5 +4,5 @@ an evaluation task and run it in a loop against a training agent. """ -from .task import CallbackResult, EvalOutcome, EvalTask +from .evalTask import CallbackResult, EvalOutcome, EvalTask from .orchestrator import LoopResult, run_train_eval_loop \ No newline at end of file diff --git a/src/microbots/auto_memory/analyzer.py b/src/microbots/auto_memory/analyzer.py index 83e75cf..ca35c16 100644 --- a/src/microbots/auto_memory/analyzer.py +++ b/src/microbots/auto_memory/analyzer.py @@ -7,7 +7,7 @@ from logging import getLogger -from microbots.auto_memory.task import EvalOutcome, EvalTask +from microbots.auto_memory.evalTask import EvalOutcome, EvalTask from microbots.bot.LogAnalysisBot import LogAnalysisBot from microbots.MicroBot import BotRunResult diff --git a/src/microbots/auto_memory/cli.py b/src/microbots/auto_memory/cli.py index 98d3147..66ab0e6 100644 --- a/src/microbots/auto_memory/cli.py +++ b/src/microbots/auto_memory/cli.py @@ -3,16 +3,19 @@ Two modes, selected by ``--task``: - ``--task `` given: run the full train <-> eval loop for that - task (via ``run_train_eval_loop``). -- ``--task`` omitted: train only, no eval task (via ``run_training_loop``, - with empty feedback). + task. +- ``--task`` omitted: train only, no eval task, with empty feedback. + +Both modes are dispatched via ``orchestrator.run``. """ import argparse import logging +from pathlib import Path -from microbots.auto_memory.orchestrator import run_train_eval_loop, run_training_loop +from microbots.auto_memory.orchestrator import run from microbots.auto_memory.task_registry import TASK_REGISTRY, discover_tasks +from microbots.auto_memory.workdir import load_config, require_workdir, resolve_workdir logger = logging.getLogger(__name__) @@ -20,7 +23,10 @@ discover_tasks() def parse_args(argv: list[str] | None = None) -> argparse.Namespace: - """Parse CLI args, including task-specific args when ``--task`` is given. + """Parse the CLI's top-level args. + + Task-specific values (e.g. an eval task's instance ID) are not + parsed here; they come from the workdir's config file instead. Parameters ---------- @@ -33,22 +39,19 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: The parsed args. """ parser = argparse.ArgumentParser(description="Run the auto-memory train/eval loop.") - parser.add_argument("--repo", required=True, help="Absolute path to the repo.") - parser.add_argument("--memory-dir", required=True, help="Directory for memory files.") parser.add_argument("--model", required=True, help='Model, e.g. "azure-openai/gpt-5.5".') + parser.add_argument( + "--workdir", + help="Directory holding this run's files (repo clone, logs, memory, " + "config). Defaults to './workdir' relative to the current directory.", + ) parser.add_argument( "--task", choices=sorted(TASK_REGISTRY), help="Eval task to run. Omit to only run training, with no eval task.", ) parser.add_argument("--max-rounds", type=int, default=5) - parser.add_argument("--training-iterations", type=int, default=1) - - # First pass just to discover --task, so we can register its - # task-specific flags before the real parse. - known_args, _ = parser.parse_known_args(argv) - if known_args.task: - TASK_REGISTRY[known_args.task].add_cli_args(parser) + parser.add_argument("--training-iterations", type=int, default=10) return parser.parse_args(argv) @@ -62,29 +65,28 @@ def main(argv: list[str] | None = None) -> None: """ args = parse_args(argv) - if not args.task: - run_training_loop( - repo_path=args.repo, - feedback="", - memory_dir=args.memory_dir, - model=args.model, - iterations=args.training_iterations, - ) - return + workdir = Path(args.workdir) if args.workdir else resolve_workdir() + require_workdir(workdir) - task_cls = TASK_REGISTRY[args.task] - for task in task_cls.from_cli_args(args): - result = run_train_eval_loop( - repo_path=args.repo, - memory_dir=args.memory_dir, + config = load_config(workdir) + tasks = ( + TASK_REGISTRY[args.task].from_config(config.get("task_args", {})) + if args.task + else [None] + ) + for task in tasks: + result = run( + workdir=workdir, model=args.model, task=task, max_rounds=args.max_rounds, training_iterations=args.training_iterations, + config=config, ) - logger.info( - "task=%s passed=%s rounds_run=%d", args.task, result.passed, result.rounds_run - ) + if result is not None: + logger.info( + "task=%s passed=%s rounds_run=%d", args.task, result.passed, result.rounds_run + ) if __name__ == "__main__": main() diff --git a/src/microbots/auto_memory/eval/swebenchverified.py b/src/microbots/auto_memory/eval/swebenchverified.py index 2cd974b..96017b8 100644 --- a/src/microbots/auto_memory/eval/swebenchverified.py +++ b/src/microbots/auto_memory/eval/swebenchverified.py @@ -13,18 +13,44 @@ import tempfile import uuid from dataclasses import dataclass +from functools import lru_cache from logging import getLogger from pathlib import Path from datasets import load_dataset -from microbots.auto_memory.task import CallbackResult, EvalOutcome, EvalTask +from microbots.auto_memory.evalTask import CallbackResult, EvalOutcome, EvalTask from microbots.auto_memory.task_registry import register_task from microbots.bot.WritingBot import WritingBot from microbots.tools.tool_definitions.memory_tool import MemoryTool logger = getLogger(__name__) -SWE_BENCH_SUITE = "SWE-bench/SWE-bench_Verified" +SWE_BENCH_VERIFIED = "SWE-bench/SWE-bench_Verified" +EVAL_AGENT_MODEL_NAME = "microbots-eval-agent" + + +@lru_cache(maxsize=None) +def _load_dataset_rows(dataset_name: str): + """Load and cache ``dataset_name``'s ``test`` split for the process's lifetime. + + ``load_dataset`` caches the downloaded files on disk, but still + re-reads and rebuilds the in-memory ``Dataset`` object on every + call. Since ``load_instances_of_repo``/``load_instance_using_id`` + may each be called many times (e.g. once per eval task instance), + this wraps ``load_dataset`` with an in-memory cache keyed by + ``dataset_name``, so the dataset is only loaded once per process. + + Parameters + ---------- + dataset_name : str + Hugging Face dataset name to load. + + Returns + ------- + datasets.Dataset + The loaded ``test`` split. + """ + return load_dataset(dataset_name, split="test") @dataclass @@ -50,7 +76,7 @@ class SweBenchInstance: def load_instances_of_repo( - dataset_name: str = SWE_BENCH_SUITE, + dataset_name: str = SWE_BENCH_VERIFIED, repo: str | None = None, ) -> list[SweBenchInstance]: """Load all dataset instances, optionally filtered to a single repo. @@ -59,7 +85,7 @@ def load_instances_of_repo( ---------- dataset_name : str Hugging Face dataset name to load. Defaults to - ``SWE_BENCH_SUITE``. + ``SWE_BENCH_VERIFIED``. repo : str | None If given, only instances whose ``repo`` matches this value are returned, e.g. ``"django/django"``. If ``None``, all instances @@ -70,7 +96,7 @@ def load_instances_of_repo( list[SweBenchInstance] The matching instances. """ - rows = load_dataset(dataset_name, split="test") + rows = _load_dataset_rows(dataset_name) instances = [ SweBenchInstance( instance_id=row["instance_id"], @@ -83,7 +109,7 @@ def load_instances_of_repo( ] return instances -def load_instance_using_id(instance_id: str, dataset_name: str = SWE_BENCH_SUITE) -> SweBenchInstance: +def load_instance_using_id(instance_id: str, dataset_name: str = SWE_BENCH_VERIFIED) -> SweBenchInstance: """Load a single dataset instance by its instance ID. Parameters @@ -92,7 +118,7 @@ def load_instance_using_id(instance_id: str, dataset_name: str = SWE_BENCH_SUITE The instance ID to look up, e.g. ``"django__django-11099"``. dataset_name : str Hugging Face dataset name to load. Defaults to - ``SWE_BENCH_SUITE``. + ``SWE_BENCH_VERIFIED``. Returns ------- @@ -105,7 +131,7 @@ def load_instance_using_id(instance_id: str, dataset_name: str = SWE_BENCH_SUITE If no instance with the given ``instance_id`` exists in the dataset. """ - rows = load_dataset(dataset_name, split="test") + rows = _load_dataset_rows(dataset_name) for row in rows: if row["instance_id"] == instance_id: return SweBenchInstance( @@ -130,13 +156,15 @@ class SweBenchVerifiedTask(EvalTask): The dataset instance this task evaluates against. """ - def __init__(self, instance: SweBenchInstance): - """Initialize the task for a single dataset instance. + def __init__(self, instance: SweBenchInstance | None = None): + """Initialize the task, optionally for a single dataset instance. Parameters ---------- - instance : SweBenchInstance - The dataset instance this task evaluates against. + instance : SweBenchInstance | None + The dataset instance this task evaluates against. May be + omitted and set later via ``self.instance``, but must be + set before any other method on this task is called. """ self.instance = instance @@ -181,6 +209,62 @@ def from_cli_args(cls, args: argparse.Namespace) -> list["SweBenchVerifiedTask"] instances = load_instances_of_repo(repo=getattr(args, "swebench_repo", None)) return [cls(instance) for instance in instances] + @classmethod + def from_config(cls, task_args: dict) -> list["SweBenchVerifiedTask"]: + """Build task(s) from a config's ``task_args`` dict. + + Parameters + ---------- + task_args : dict + Task-specific config values, expected to include + ``instance_id`` and/or ``swebench_repo`` (mirrors + ``add_cli_args``'s flags). + + Returns + ------- + list[SweBenchVerifiedTask] + One task per matching dataset instance. A single-element + list when ``instance_id`` is given. + """ + if task_args.get("instance_id"): + instances = [load_instance_using_id(task_args["instance_id"])] + else: + instances = load_instances_of_repo(repo=task_args.get("swebench_repo")) + return [cls(instance) for instance in instances] + + @property + def task_id(self) -> str: + """Return this instance's SWE-bench-verified ``instance_id``. + + Returns + ------- + str + The dataset instance's ``instance_id``. + """ + return self.instance.instance_id + + def build_result(self, outcome: EvalOutcome) -> dict: + """Summarize a round's outcome, including the instance's dataset fields. + + Parameters + ---------- + outcome : EvalOutcome + The round's outcome to summarize. + + Returns + ------- + dict + ``passed``/``reason`` plus ``instance_id``, ``repo``, and + ``base_commit`` identifying which dataset row this is. + """ + return { + "passed": outcome.result.passed, + "reason": outcome.result.reason, + "instance_id": self.instance.instance_id, + "repo": self.instance.repo, + "base_commit": self.instance.base_commit, + } + def setup(self, repo_path: str) -> None: """Clone the instance's repo and check out its base commit. @@ -197,14 +281,9 @@ def setup(self, repo_path: str) -> None: ["git", "checkout", self.instance.base_commit], cwd=repo_path, check=True ) - def build_prompt(self, repo_path: str) -> str: + def build_prompt(self) -> str: """Return the instance's issue text as the agent's prompt. - Parameters - ---------- - repo_path : str - Absolute path to the repo the agent will operate on. - Returns ------- str @@ -240,7 +319,7 @@ def check(self, repo_path: str, agent_output: str, log_path: str) -> CallbackRes ).stdout run_id = f"microbots-{uuid.uuid4().hex[:8]}" - model_name_or_path = "microbots-eval-agent" + model_name_or_path = EVAL_AGENT_MODEL_NAME pred_path = Path(tempfile.mktemp(suffix=".json")) report_dir = Path(tempfile.mkdtemp()) pred_path.write_text(json.dumps([{ @@ -252,7 +331,7 @@ def check(self, repo_path: str, agent_output: str, log_path: str) -> CallbackRes try: proc = subprocess.run( [sys.executable, "-m", "swebench.harness.run_evaluation", - "--dataset_name", SWE_BENCH_SUITE, + "--dataset_name", SWE_BENCH_VERIFIED, "--max_workers", "1", "--predictions_path", str(pred_path), "--run_id", run_id, @@ -286,7 +365,7 @@ def teardown(self, repo_path: str) -> None: """ subprocess.run(["rm", "-rf", repo_path], check=False) - def run(self, repo_path: str, memory_dir: str, model: str) -> EvalOutcome: + def run(self, repo_path: str, memory_dir: str, model: str, log_path: str) -> EvalOutcome: """Run one eval iteration: setup -> build_prompt -> WritingBot -> check -> teardown. Parameters @@ -298,6 +377,9 @@ def run(self, repo_path: str, memory_dir: str, model: str) -> EvalOutcome: ``MemoryTool``. model : str The model to use, in the format ``/``. + log_path : str + Path to write this round's log to. Caller-provided, so the + log persists under the run's own layout. Returns ------- @@ -306,12 +388,12 @@ def run(self, repo_path: str, memory_dir: str, model: str) -> EvalOutcome: the check verdict, and the round's log file path. """ self.setup(repo_path) - log_path = tempfile.mktemp(suffix=".log") + Path(log_path).parent.mkdir(parents=True, exist_ok=True) Path(log_path).write_text("") try: try: - prompt = self.build_prompt(repo_path) + prompt = self.build_prompt() bot = WritingBot( model=model, folder_to_mount=repo_path, diff --git a/src/microbots/auto_memory/task.py b/src/microbots/auto_memory/evalTask.py similarity index 73% rename from src/microbots/auto_memory/task.py rename to src/microbots/auto_memory/evalTask.py index 9eb00ae..6a8925c 100644 --- a/src/microbots/auto_memory/task.py +++ b/src/microbots/auto_memory/evalTask.py @@ -56,6 +56,41 @@ class EvalTask(ABC): class calls them automatically. """ + @property + def task_id(self) -> str: + """Identifier for this task instance, used to name its output folder. + + Defaults to the class name, which is fine for tasks with only + one instance per run. Override for tasks with several distinct + instances per class (e.g. ``SweBenchVerifiedTask``, where each + dataset row needs its own folder). + + Returns + ------- + str + This task instance's identifier. + """ + return type(self).__name__ + + def build_result(self, outcome: EvalOutcome) -> dict: + """Optional. Build the dict written to this round's ``result.json``. + + Not called automatically; the orchestrator calls this after + each round to decide what to persist. Override to include + task-specific details (e.g. dataset fields, repo info). + + Parameters + ---------- + outcome : EvalOutcome + The round's outcome to summarize. + + Returns + ------- + dict + JSON-serializable summary. Defaults to ``passed``/``reason``. + """ + return {"passed": outcome.result.passed, "reason": outcome.result.reason} + def setup(self, repo_path: str) -> None: """Optional. Prepare repo/environment before the agent runs. @@ -69,17 +104,12 @@ def setup(self, repo_path: str) -> None: """ pass - def build_prompt(self, repo_path: str) -> str: + def build_prompt(self) -> str: """Optional. Return the task prompt/instructions for the agent. Not called automatically; only useful if your ``run`` implementation calls it. - Parameters - ---------- - repo_path : str - Absolute path to the repo the agent will operate on. - Returns ------- str @@ -123,7 +153,7 @@ def teardown(self, repo_path: str) -> None: pass @abstractmethod - def run(self, repo_path: str, memory_dir: str, model: str) -> EvalOutcome: + def run(self, repo_path: str, memory_dir: str, model: str, log_path: str) -> EvalOutcome: """Required. Run one eval iteration and return its outcome. Parameters @@ -135,6 +165,10 @@ def run(self, repo_path: str, memory_dir: str, model: str) -> EvalOutcome: ``MemoryTool``. model : str The model to use, in the format ``/``. + log_path : str + Path to write this round's log to. Caller-provided (e.g. a + workdir-managed path) so logs persist under the run's + layout instead of each task inventing its own temp file. Returns ------- diff --git a/src/microbots/auto_memory/orchestrator.py b/src/microbots/auto_memory/orchestrator.py index f4ca198..0c29538 100644 --- a/src/microbots/auto_memory/orchestrator.py +++ b/src/microbots/auto_memory/orchestrator.py @@ -8,10 +8,20 @@ from dataclasses import dataclass, field from logging import getLogger from pathlib import Path +import json +import subprocess from microbots.auto_memory.analyzer import build_feedback -from microbots.auto_memory.task import EvalOutcome, EvalTask +from microbots.auto_memory.evalTask import EvalOutcome, EvalTask from microbots.auto_memory.training.runner import run_training +from microbots.auto_memory.workdir import ( + eval_log_path, + eval_result_path, + load_config, + load_round_memory, + repo_dir, + save_round_memory, +) logger = getLogger(__name__) @@ -36,12 +46,68 @@ class LoopResult: final_outcome: EvalOutcome outcomes: list[EvalOutcome] = field(default_factory=list) +def clone_repo(url: str, repo_path: Path) -> None: + """Clone ``url`` into ``repo_path`` if it isn't already cloned there. + + Parameters + ---------- + url : str + Git URL (or local path) to clone from. + repo_path : Path + Destination directory for the clone. If it already exists (e.g. + a previous round already cloned here), this is a no-op. + """ + if repo_path.exists(): + return + subprocess.run(["git", "clone", url, str(repo_path)], check=True) + +def reset_repo(repo_path: Path, base_commit: str) -> None: + """Reset ``repo_path`` to ``base_commit``, discarding all local changes. + + Runs ``git reset --hard `` followed by ``git clean -fd``, + so every round/instance starts from the same pristine state instead + of carrying forward whatever a previous round or eval attempt left + behind. + + Parameters + ---------- + repo_path : Path + Path to the repo to reset. + base_commit : str + Commit-ish to reset to. + """ + subprocess.run(["git", "reset", "--hard", base_commit], cwd=repo_path, check=True) + subprocess.run(["git", "clean", "-fd"], cwd=repo_path, check=True) + +def write_eval_result(workdir: Path, round_num: int, task: EvalTask, outcome: EvalOutcome) -> None: + """Write a round's eval result to ``result.json``. + + Delegates the content to ``task.build_result(outcome)`` so each + task decides what's worth persisting (e.g. ``SweBenchVerifiedTask`` + includes its dataset instance's fields). + + Parameters + ---------- + workdir : Path + The run's workdir. + round_num : int + 1-based round number this outcome belongs to. + task : EvalTask + The task that produced ``outcome``, used for both its + ``task_id`` (folder name) and ``build_result`` (file content). + outcome : EvalOutcome + The round's outcome to persist. + """ + path = eval_result_path(workdir, round_num, task.task_id) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(task.build_result(outcome), indent=2)) + def run_training_loop( repo_path: str, feedback: str, memory_dir: str, model: str, - iterations: int = 1, + iterations: int = 10, ) -> None: """Run ``run_training`` ``iterations`` times, reusing the same memory dir. @@ -60,7 +126,7 @@ def run_training_loop( The model to use, in the format ``/``. iterations : int Number of training passes to run, each reusing the same - ``memory_dir``. Defaults to 1. + ``memory_dir``. Defaults to 10. """ for iteration in range(1, iterations + 1): logger.info( @@ -77,27 +143,37 @@ def run_training_loop( def run_train_eval_loop( repo_path: str, - memory_dir: str, + workdir: Path, model: str, task: EvalTask, max_rounds: int = 5, - training_iterations: int = 1, + training_iterations: int = 10, ) -> LoopResult: """Run an eval task in a loop, retraining on failure until it passes. - Each round runs ``task.run(...)``. If the task passes, the loop - returns immediately. If it fails, feedback is built from the round's - log and used to retrain via ``run_training`` (called - ``training_iterations`` times, each pass reusing the same - ``memory_dir``) before the next round. The round's log file is - always deleted before the next round starts. + Each round loads the current top-level memory into its own + ``rounds_/round_N/memory`` (carried forward from the + previous round, or empty on round 1), then runs ``task.run(...)`` + against it, writing its log to a workdir-managed path + (``rounds_/round_N/eval/eval.log``) so it persists. Since + each eval task instance gets its own ``rounds_`` dir, + different instances sharing the same ``workdir`` never collide on + round numbers, and each instance's per-round memory is preserved + individually. If the task passes, the loop returns immediately. If + it fails, feedback is built from the round's log and used to + retrain via ``run_training`` (called ``training_iterations`` times, + each pass reusing the same round memory dir) before the next round. + Either way, the round's result is written to ``result.json`` and + its memory is saved back to the top-level memory dir before the + next round starts. Parameters ---------- repo_path : str Absolute path to the repo to evaluate and train against. - memory_dir : str - Directory where the training agent reads/writes memory files. + workdir : Path + This run's workdir, used to carry memory forward between rounds + (see ``microbots.auto_memory.workdir``). model : str The model to use, in the format ``/``. task : EvalTask @@ -106,7 +182,7 @@ def run_train_eval_loop( Maximum number of train/eval rounds to attempt. Defaults to 5. training_iterations : int Number of training passes to run per retraining round, each - reusing the same ``memory_dir``. Defaults to 1. + reusing the same round memory dir. Defaults to 10. Returns ------- @@ -116,28 +192,31 @@ def run_train_eval_loop( """ outcomes: list[EvalOutcome] = [] - for round_idx in range(max_rounds): + for round_idx in range(1, max_rounds+1): logger.info( - "run_train_eval_loop: round %d/%d starting", round_idx + 1, max_rounds + "run_train_eval_loop: round %d/%d starting", round_idx, max_rounds + ) + memory_dir = str(load_round_memory(workdir, round_idx, instance_id=task.task_id)) + outcome = task.run( + repo_path, memory_dir, model, str(eval_log_path(workdir, round_idx, task.task_id)) ) - outcome = task.run(repo_path, memory_dir, model) outcomes.append(outcome) try: if outcome.passed: logger.info( - "run_train_eval_loop: passed on round %d/%d", round_idx + 1, max_rounds + "run_train_eval_loop: passed on round %d/%d", round_idx, max_rounds ) return LoopResult( passed=True, - rounds_run=round_idx + 1, + rounds_run=round_idx, final_outcome=outcome, outcomes=outcomes, ) logger.info( "run_train_eval_loop: round %d failed (%s), retraining", - round_idx + 1, + round_idx, outcome.result.reason, ) try: @@ -153,10 +232,11 @@ def run_train_eval_loop( logger.exception( "run_train_eval_loop: round %d failed to build feedback/retrain; " "continuing to next round without retraining", - round_idx + 1, + round_idx, ) finally: - Path(outcome.log_path).unlink(missing_ok=True) + write_eval_result(workdir, round_idx, task, outcome) + save_round_memory(workdir, round_idx, instance_id=task.task_id) logger.info( "run_train_eval_loop: exhausted %d rounds without passing", max_rounds @@ -166,4 +246,73 @@ def run_train_eval_loop( rounds_run=max_rounds, final_outcome=outcomes[-1], outcomes=outcomes, - ) \ No newline at end of file + ) + +def run( + workdir: Path, + model: str, + task: EvalTask | None, + max_rounds: int = 5, + training_iterations: int = 10, + config: dict | None = None, +) -> LoopResult | None: + """Run training only, or the full train/eval loop, depending on ``task``. + + Parameters + ---------- + workdir : Path + This run's workdir (see ``microbots.auto_memory.workdir``), + holding ``config.yaml``, the shared repo clone, and all output. + model : str + The model to use, in the format ``/``. + task : EvalTask | None + The eval task to run each round, or ``None`` to only run + training (with empty feedback, once per ``training_iterations``). + max_rounds : int + Maximum number of train/eval rounds to attempt, if ``task`` is + given. Defaults to 5. + training_iterations : int + Number of training passes to run per retraining round, each + reusing the same round memory dir. Defaults to 10. + config : dict | None + This run's already-loaded ``config.yaml`` contents. If ``None`` + (the default), it is loaded from ``workdir`` here. Callers that + invoke ``run`` repeatedly for the same ``workdir`` (e.g. once + per eval task) can load it once and pass it in, to avoid + re-reading/re-parsing the file on every call. + + Returns + ------- + LoopResult | None + The eval loop's result if ``task`` was given, otherwise ``None``. + """ + if config is None: + config = load_config(workdir) + repo_url = config.get("repo") + if repo_url: + clone_repo(repo_url, repo_dir(workdir)) + + repo_path = str(repo_dir(workdir)) + + if task is None: + # Train-only mode has no rounds of its own; round 1 is just a + # scratch dir seeded from (and saved back to) top-level memory. + memory_dir = str(load_round_memory(workdir, 1)) + run_training_loop( + repo_path=repo_path, + feedback="", + memory_dir=memory_dir, + model=model, + iterations=training_iterations, + ) + save_round_memory(workdir, 1) + return None + + return run_train_eval_loop( + repo_path=repo_path, + workdir=workdir, + model=model, + task=task, + max_rounds=max_rounds, + training_iterations=training_iterations, + ) diff --git a/src/microbots/auto_memory/task_registry.py b/src/microbots/auto_memory/task_registry.py index 63367f7..63bfb74 100644 --- a/src/microbots/auto_memory/task_registry.py +++ b/src/microbots/auto_memory/task_registry.py @@ -8,7 +8,7 @@ import importlib import pkgutil -from microbots.auto_memory.task import EvalTask +from microbots.auto_memory.evalTask import EvalTask TASK_REGISTRY: dict[str, type[EvalTask]] = {} @@ -46,6 +46,7 @@ def decorator(task_cls: type[EvalTask]) -> type[EvalTask]: return decorator +# Not being used currently, but kept it for future use if required. def create_task(name: str, **kwargs) -> EvalTask: """Construct a registered ``EvalTask`` by name. diff --git a/src/microbots/auto_memory/workdir.py b/src/microbots/auto_memory/workdir.py new file mode 100644 index 0000000..771066c --- /dev/null +++ b/src/microbots/auto_memory/workdir.py @@ -0,0 +1,383 @@ +"""Path/layout helpers for a training run's workdir. + +Centralizes every path this package reads or writes under a run's +``workdir`` (config, repo clone, logs, memory, and per-round/per-eval +outputs), so callers never hard-code layout details themselves. +""" + +from pathlib import Path +import shutil + +import yaml + +WORKDIR_NAME = "workdir" +CONFIG_FILENAME = "config.yaml" +REPO_DIRNAME = "repo" +RUN_LOG_FILENAME = "run.log" +MEMORY_DIRNAME = "memory" +ROUNDS_DIRNAME = "rounds" +ROUND_LOG_FILENAME = "round.log" +ROUND_PATCH_FILENAME = "repo.patch" +EVAL_DIRNAME = "eval" +RESULT_FILENAME = "result.json" +EVAL_LOG_FILENAME = "eval.log" + + +def resolve_workdir(base: Path | None = None) -> Path: + """Resolve the fixed workdir path relative to ``base``. + + Parameters + ---------- + base : Path | None + Directory to resolve ``workdir/`` relative to. Defaults to the + current working directory. + + Returns + ------- + Path + ``workdir`` resolved relative to ``base`` (or ``Path.cwd()``). + """ + return (base or Path.cwd()) / WORKDIR_NAME + + +def require_workdir(workdir: Path) -> None: + """Validate that ``workdir`` exist. + + Parameters + ---------- + workdir : Path + The workdir to validate. + + Raises + ------ + FileNotFoundError + If ``workdir`` does not exist. + """ + if not workdir.is_dir(): + raise FileNotFoundError(f"workdir not found: {workdir}") + + +def config_path(workdir: Path) -> Path: + """Return the path to ``workdir``'s config file. + + Parameters + ---------- + workdir : Path + The run's workdir. + + Returns + ------- + Path + ``workdir/config.yaml``. + """ + return workdir / CONFIG_FILENAME + + +def load_config(workdir: Path) -> dict: + """Load and parse ``workdir``'s config file. + + Parameters + ---------- + workdir : Path + The workdir whose config file should be loaded. + + Returns + ------- + dict + The parsed config, or ``{}`` if the config file doesn't exist + or is empty. + """ + path = config_path(workdir) + if not path.is_file(): + return {} + return yaml.safe_load(path.read_text()) or {} + + +def repo_dir(workdir: Path) -> Path: + """Return the path to the single cloned repo shared across rounds. + + Parameters + ---------- + workdir : Path + The run's workdir. + + Returns + ------- + Path + ``workdir/repo``. + """ + return workdir / REPO_DIRNAME + + +def run_log_path(workdir: Path) -> Path: + """Return the path to the top-level orchestrator log. + + Parameters + ---------- + workdir : Path + The run's workdir. + + Returns + ------- + Path + ``workdir/run.log``. + """ + return workdir / RUN_LOG_FILENAME + + +def memory_dir(workdir: Path) -> Path: + """Return the path to the current top-level (latest) memory directory. + + Parameters + ---------- + workdir : Path + The run's workdir. + + Returns + ------- + Path + ``workdir/memory``. + """ + return workdir / MEMORY_DIRNAME + + +def round_dir( + workdir: Path, round_num: int, *, instance_id: str | None = None, create: bool = False +) -> Path: + """Return (and optionally create) the directory for a training round. + + Parameters + ---------- + workdir : Path + The run's workdir. + round_num : int + 1-based round number. + instance_id : str | None + If given, rounds are kept under a per-instance rounds dir + (``rounds_{instance_id}``) instead of the shared ``rounds`` dir, + so different eval task instances sharing the same ``workdir`` + don't collide on round numbers. Pass the eval task's + ``task_id`` when running an eval task; omit for training-only + mode. + create : bool + If True, create the directory (and parents) if missing. + + Returns + ------- + Path + ``workdir/rounds/round_{round_num}`` (no ``instance_id``), or + ``workdir/rounds_{instance_id}/round_{round_num}``. + """ + rounds_dirname = f"{ROUNDS_DIRNAME}_{instance_id}" if instance_id else ROUNDS_DIRNAME + path = workdir / rounds_dirname / f"round_{round_num}" + if create: + path.mkdir(parents=True, exist_ok=True) + return path + + +def round_memory_dir(workdir: Path, round_num: int, *, instance_id: str | None = None) -> Path: + """Return the path to a round's own memory snapshot (a directory). + + Parameters + ---------- + workdir : Path + The run's workdir. + round_num : int + 1-based round number. + instance_id : str | None + The eval task's ``task_id``, if running an eval task (see + ``round_dir``). Omit for training-only mode. + + Returns + ------- + Path + This round's own memory directory. + """ + return round_dir(workdir, round_num, instance_id=instance_id) / MEMORY_DIRNAME + + +def load_round_memory(workdir: Path, round_num: int, *, instance_id: str | None = None) -> Path: + """Copy the current top-level memory into this round's own memory dir. + + Called before a round's training pass, so it starts from whatever + memory the previous round left behind (or empty, on round 1). + + Parameters + ---------- + workdir : Path + The run's workdir. + round_num : int + 1-based round number to load memory into. + instance_id : str | None + The eval task's ``task_id``, if running an eval task (see + ``round_dir``). Omit for training-only mode. + + Returns + ------- + Path + This round's own memory dir, ready for the round to use. + """ + src = memory_dir(workdir) + dst = round_memory_dir(workdir, round_num, instance_id=instance_id) + dst.mkdir(parents=True, exist_ok=True) + if src.is_dir(): + shutil.copytree(src, dst, dirs_exist_ok=True) + return dst + + +def save_round_memory(workdir: Path, round_num: int, *, instance_id: str | None = None) -> Path: + """Copy this round's memory back up to the top-level memory dir. + + Called after a round's training pass, so later rounds (and the + final saved memory) see what this round learned. + + Parameters + ---------- + workdir : Path + The run's workdir. + round_num : int + 1-based round number whose memory should be saved. + instance_id : str | None + The eval task's ``task_id``, if running an eval task (see + ``round_dir``). Omit for training-only mode. + + Returns + ------- + Path + The top-level ``memory`` dir, now updated with this round's changes. + """ + src = round_memory_dir(workdir, round_num, instance_id=instance_id) + dst = memory_dir(workdir) + dst.mkdir(parents=True, exist_ok=True) + if src.is_dir(): + shutil.copytree(src, dst, dirs_exist_ok=True) + return dst + + +def round_log_path(workdir: Path, round_num: int, *, instance_id: str | None = None) -> Path: + """Return the path to a round's training log. + + Parameters + ---------- + workdir : Path + The run's workdir. + round_num : int + 1-based round number. + instance_id : str | None + The eval task's ``task_id``, if running an eval task (see + ``round_dir``). Omit for training-only mode. + + Returns + ------- + Path + This round's ``round.log``. + """ + return round_dir(workdir, round_num, instance_id=instance_id) / ROUND_LOG_FILENAME + + +def round_patch_path(workdir: Path, round_num: int, *, instance_id: str | None = None) -> Path: + """Return the path to a round's captured repo diff. + + Parameters + ---------- + workdir : Path + The run's workdir. + round_num : int + 1-based round number. + instance_id : str | None + The eval task's ``task_id``, if running an eval task (see + ``round_dir``). Omit for training-only mode. + + Returns + ------- + Path + This round's ``repo.patch``. + """ + return round_dir(workdir, round_num, instance_id=instance_id) / ROUND_PATCH_FILENAME + + +def eval_dir( + workdir: Path, round_num: int, instance_id: str, *, create: bool = False +) -> Path: + """Return (and optionally create) an eval task instance's eval directory. + + Parameters + ---------- + workdir : Path + The run's workdir. + round_num : int + 1-based round number this eval instance belongs to. + instance_id : str + The eval task instance identifier. + create : bool + If True, create the directory (and parents) if missing. + + Returns + ------- + Path + ``workdir/rounds_{instance_id}/round_{round_num}/eval``. + """ + path = round_dir(workdir, round_num, instance_id=instance_id) / EVAL_DIRNAME + if create: + path.mkdir(parents=True, exist_ok=True) + return path + + +def eval_result_path(workdir: Path, round_num: int, instance_id: str) -> Path: + """Return the path to an eval instance's result file. + + Parameters + ---------- + workdir : Path + The run's workdir. + round_num : int + 1-based round number this eval instance belongs to. + instance_id : str + The eval task instance identifier. + + Returns + ------- + Path + This eval instance's ``result.json``. + """ + return eval_dir(workdir, round_num, instance_id) / RESULT_FILENAME + + +def eval_log_path(workdir: Path, round_num: int, instance_id: str) -> Path: + """Return the path to an eval instance's log file. + + Parameters + ---------- + workdir : Path + The run's workdir. + round_num : int + 1-based round number this eval instance belongs to. + instance_id : str + The eval task instance identifier. + + Returns + ------- + Path + This eval instance's ``eval.log``. + """ + return eval_dir(workdir, round_num, instance_id) / EVAL_LOG_FILENAME + + +def eval_patch_path(workdir: Path, round_num: int, instance_id: str) -> Path: + """Return the path to an eval instance's captured repo diff. + + Parameters + ---------- + workdir : Path + The run's workdir. + round_num : int + 1-based round number this eval instance belongs to. + instance_id : str + The eval task instance identifier. + + Returns + ------- + Path + This eval instance's ``repo.patch``. + """ + return eval_dir(workdir, round_num, instance_id) / ROUND_PATCH_FILENAME \ No newline at end of file diff --git a/test/auto_memory/eval/test_swebenchverified.py b/test/auto_memory/eval/test_swebenchverified.py index 8754211..1835e8d 100644 --- a/test/auto_memory/eval/test_swebenchverified.py +++ b/test/auto_memory/eval/test_swebenchverified.py @@ -13,14 +13,23 @@ from microbots.auto_memory.eval.swebenchverified import ( SweBenchInstance, SweBenchVerifiedTask, + _load_dataset_rows, load_instance_using_id, load_instances_of_repo, ) -from microbots.auto_memory.task import CallbackResult +from microbots.auto_memory.evalTask import CallbackResult, EvalOutcome MODULE = "microbots.auto_memory.eval.swebenchverified" +@pytest.fixture(autouse=True) +def _clear_dataset_cache(): + """Clear ``_load_dataset_rows``'s cache so each test's ``load_dataset`` mock takes effect.""" + _load_dataset_rows.cache_clear() + yield + _load_dataset_rows.cache_clear() + + def _fake_rows(): return [ { @@ -89,6 +98,20 @@ def test_load_instance_using_id_raises_when_not_found(mock_load_dataset): load_instance_using_id("does-not-exist") +@pytest.mark.unit +@patch(f"{MODULE}.load_dataset") +def test_dataset_rows_are_cached_across_repeated_calls(mock_load_dataset): + """``load_dataset`` should only be called once per ``dataset_name``, even + across multiple ``load_instances_of_repo``/``load_instance_using_id`` calls.""" + mock_load_dataset.return_value = _fake_rows() + + load_instances_of_repo(repo="django/django") + load_instances_of_repo(repo=None) + load_instance_using_id("astropy__astropy-1") + + mock_load_dataset.assert_called_once() + + # --------------------------------------------------------------------------- # SweBenchVerifiedTask.setup / build_prompt / teardown # --------------------------------------------------------------------------- @@ -117,7 +140,32 @@ def test_setup_clones_and_checks_out_base_commit(mock_run): @pytest.mark.unit def test_build_prompt_returns_problem_statement(): task = SweBenchVerifiedTask(_instance()) - assert task.build_prompt("/repo") == "fix the bug" + assert task.build_prompt() == "fix the bug" + + +@pytest.mark.unit +def test_task_id_is_instance_id(): + task = SweBenchVerifiedTask(_instance()) + assert task.task_id == "django__django-1" + + +@pytest.mark.unit +def test_build_result_includes_dataset_fields(): + task = SweBenchVerifiedTask(_instance()) + outcome = EvalOutcome( + passed=True, + output="agent output", + result=CallbackResult(passed=True, reason="resolved"), + log_path="/dev/null", + ) + + assert task.build_result(outcome) == { + "passed": True, + "reason": "resolved", + "instance_id": "django__django-1", + "repo": "django/django", + "base_commit": "abc123", + } @pytest.mark.unit @@ -251,7 +299,7 @@ def test_check_cleans_up_even_when_harness_raises(mock_run, mock_rmtree, tmp_pat @pytest.mark.unit @patch(f"{MODULE}.MemoryTool") @patch(f"{MODULE}.WritingBot") -def test_run_calls_setup_build_prompt_check_teardown_in_order(mock_bot_cls, mock_memory_tool): +def test_run_calls_setup_build_prompt_check_teardown_in_order(mock_bot_cls, mock_memory_tool, tmp_path): from microbots.MicroBot import BotRunResult mock_bot = MagicMock() @@ -261,14 +309,14 @@ def test_run_calls_setup_build_prompt_check_teardown_in_order(mock_bot_cls, mock task = SweBenchVerifiedTask(_instance()) calls = [] task.setup = lambda repo_path: calls.append(("setup", repo_path)) - task.build_prompt = lambda repo_path: "do the task" + task.build_prompt = lambda: "do the task" task.check = lambda repo_path, agent_output, log_path: ( calls.append(("check", repo_path, agent_output, log_path)) or CallbackResult(passed=True, reason="ok") ) task.teardown = lambda repo_path: calls.append(("teardown", repo_path)) - outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o") + outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o", str(tmp_path / "eval.log")) assert calls[0] == ("setup", "/repo") assert calls[1] == ("check", "/repo", "agent did stuff", outcome.log_path) @@ -280,7 +328,7 @@ def test_run_calls_setup_build_prompt_check_teardown_in_order(mock_bot_cls, mock @pytest.mark.unit @patch(f"{MODULE}.MemoryTool") @patch(f"{MODULE}.WritingBot") -def test_run_creates_log_file_before_check_is_called(mock_bot_cls, mock_memory_tool): +def test_run_creates_log_file_before_check_is_called(mock_bot_cls, mock_memory_tool, tmp_path): from microbots.MicroBot import BotRunResult mock_bot = MagicMock() @@ -289,7 +337,7 @@ def test_run_creates_log_file_before_check_is_called(mock_bot_cls, mock_memory_t task = SweBenchVerifiedTask(_instance()) task.setup = lambda repo_path: None - task.build_prompt = lambda repo_path: "do the task" + task.build_prompt = lambda: "do the task" task.teardown = lambda repo_path: None seen_log_exists = {} @@ -299,7 +347,7 @@ def _check(repo_path, agent_output, log_path): task.check = _check - task.run("/repo", "/memory", "azure-openai/gpt-4o") + task.run("/repo", "/memory", "azure-openai/gpt-4o", str(tmp_path / "eval.log")) assert seen_log_exists["exists"] is True @@ -307,7 +355,7 @@ def _check(repo_path, agent_output, log_path): @pytest.mark.unit @patch(f"{MODULE}.MemoryTool") @patch(f"{MODULE}.WritingBot") -def test_run_skips_check_when_bot_status_is_false(mock_bot_cls, mock_memory_tool): +def test_run_skips_check_when_bot_status_is_false(mock_bot_cls, mock_memory_tool, tmp_path): from microbots.MicroBot import BotRunResult mock_bot = MagicMock() @@ -317,11 +365,11 @@ def test_run_skips_check_when_bot_status_is_false(mock_bot_cls, mock_memory_tool task = SweBenchVerifiedTask(_instance()) check_calls = [] task.setup = lambda repo_path: None - task.build_prompt = lambda repo_path: "do the task" + task.build_prompt = lambda: "do the task" task.check = lambda *a: check_calls.append(a) task.teardown = lambda repo_path: None - outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o") + outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o", str(tmp_path / "eval.log")) assert check_calls == [] assert outcome.passed is False @@ -331,17 +379,17 @@ def test_run_skips_check_when_bot_status_is_false(mock_bot_cls, mock_memory_tool @pytest.mark.unit @patch(f"{MODULE}.MemoryTool") @patch(f"{MODULE}.WritingBot") -def test_run_converts_build_prompt_exception_to_failed_outcome(mock_bot_cls, mock_memory_tool): +def test_run_converts_build_prompt_exception_to_failed_outcome(mock_bot_cls, mock_memory_tool, tmp_path): task = SweBenchVerifiedTask(_instance()) task.setup = lambda repo_path: None task.teardown = lambda repo_path: None - def _build_prompt(repo_path): + def _build_prompt(): raise ValueError("bad prompt") task.build_prompt = _build_prompt - outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o") + outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o", str(tmp_path / "eval.log")) assert outcome.passed is False assert "bad prompt" in outcome.result.reason @@ -352,7 +400,7 @@ def _build_prompt(repo_path): @pytest.mark.unit @patch(f"{MODULE}.MemoryTool") @patch(f"{MODULE}.WritingBot") -def test_run_converts_check_exception_to_failed_outcome(mock_bot_cls, mock_memory_tool): +def test_run_converts_check_exception_to_failed_outcome(mock_bot_cls, mock_memory_tool, tmp_path): from microbots.MicroBot import BotRunResult mock_bot = MagicMock() @@ -361,7 +409,7 @@ def test_run_converts_check_exception_to_failed_outcome(mock_bot_cls, mock_memor task = SweBenchVerifiedTask(_instance()) task.setup = lambda repo_path: None - task.build_prompt = lambda repo_path: "do the task" + task.build_prompt = lambda: "do the task" task.teardown = lambda repo_path: None def _check(repo_path, agent_output, log_path): @@ -369,7 +417,7 @@ def _check(repo_path, agent_output, log_path): task.check = _check - outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o") + outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o", str(tmp_path / "eval.log")) assert outcome.passed is False assert "check exploded" in outcome.result.reason @@ -378,16 +426,16 @@ def _check(repo_path, agent_output, log_path): @pytest.mark.unit @patch(f"{MODULE}.MemoryTool") @patch(f"{MODULE}.WritingBot") -def test_run_still_calls_teardown_when_body_raises(mock_bot_cls, mock_memory_tool): +def test_run_still_calls_teardown_when_body_raises(mock_bot_cls, mock_memory_tool, tmp_path): mock_bot_cls.side_effect = RuntimeError("bot construction failed") task = SweBenchVerifiedTask(_instance()) teardown_calls = [] task.setup = lambda repo_path: None - task.build_prompt = lambda repo_path: "do the task" + task.build_prompt = lambda: "do the task" task.teardown = lambda repo_path: teardown_calls.append(repo_path) - task.run("/repo", "/memory", "azure-openai/gpt-4o") + task.run("/repo", "/memory", "azure-openai/gpt-4o", str(tmp_path / "eval.log")) assert teardown_calls == ["/repo"] @@ -395,7 +443,7 @@ def test_run_still_calls_teardown_when_body_raises(mock_bot_cls, mock_memory_too @pytest.mark.unit @patch(f"{MODULE}.MemoryTool") @patch(f"{MODULE}.WritingBot") -def test_run_teardown_exception_does_not_clobber_returned_outcome(mock_bot_cls, mock_memory_tool): +def test_run_teardown_exception_does_not_clobber_returned_outcome(mock_bot_cls, mock_memory_tool, tmp_path): from microbots.MicroBot import BotRunResult mock_bot = MagicMock() @@ -404,7 +452,7 @@ def test_run_teardown_exception_does_not_clobber_returned_outcome(mock_bot_cls, task = SweBenchVerifiedTask(_instance()) task.setup = lambda repo_path: None - task.build_prompt = lambda repo_path: "do the task" + task.build_prompt = lambda: "do the task" task.check = lambda repo_path, agent_output, log_path: CallbackResult(passed=True, reason="ok") def _teardown(repo_path): @@ -412,7 +460,7 @@ def _teardown(repo_path): task.teardown = _teardown - outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o") + outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o", str(tmp_path / "eval.log")) # teardown() raised, but the already-computed EvalOutcome must still be returned assert outcome.passed is True diff --git a/test/auto_memory/test_analyzer.py b/test/auto_memory/test_analyzer.py index 601b044..3566b65 100644 --- a/test/auto_memory/test_analyzer.py +++ b/test/auto_memory/test_analyzer.py @@ -9,7 +9,7 @@ sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../src/"))) from microbots.auto_memory.analyzer import build_feedback -from microbots.auto_memory.task import CallbackResult, EvalOutcome +from microbots.auto_memory.evalTask import CallbackResult, EvalOutcome from microbots.MicroBot import BotRunResult diff --git a/test/auto_memory/test_cli.py b/test/auto_memory/test_cli.py index da3ca8b..6fcf716 100644 --- a/test/auto_memory/test_cli.py +++ b/test/auto_memory/test_cli.py @@ -2,6 +2,7 @@ import os import sys +from pathlib import Path from unittest.mock import MagicMock, patch import pytest @@ -12,27 +13,25 @@ MODULE = "microbots.auto_memory.cli" -BASE_ARGS = ["--repo", "/repo", "--memory-dir", "/memory", "--model", "azure-openai/gpt-4o"] +BASE_ARGS = ["--model", "azure-openai/gpt-4o"] +FAKE_WORKDIR = Path("/workdir") @pytest.mark.unit def test_parse_args_defaults(): args = parse_args(BASE_ARGS) - assert args.repo == "/repo" - assert args.memory_dir == "/memory" assert args.model == "azure-openai/gpt-4o" assert args.task is None assert args.max_rounds == 5 - assert args.training_iterations == 1 + assert args.training_iterations == 10 @pytest.mark.unit -def test_parse_args_with_known_task_adds_its_flags(): - args = parse_args(BASE_ARGS + ["--task", "swebenchverified", "--instance-id", "django__django-1"]) +def test_parse_args_accepts_known_task(): + args = parse_args(BASE_ARGS + ["--task", "swebenchverified"]) assert args.task == "swebenchverified" - assert args.instance_id == "django__django-1" @pytest.mark.unit @@ -42,67 +41,107 @@ def test_parse_args_rejects_unknown_task(): @pytest.mark.unit -@patch(f"{MODULE}.run_training_loop") -def test_main_runs_training_only_when_task_omitted(mock_run_training_loop): +def test_parse_args_workdir_defaults_to_none(): + args = parse_args(BASE_ARGS) + + assert args.workdir is None + + +@pytest.mark.unit +def test_parse_args_picks_up_explicit_workdir(): + args = parse_args(BASE_ARGS + ["--workdir", "/custom/workdir"]) + + assert args.workdir == "/custom/workdir" + + +@pytest.mark.unit +@patch(f"{MODULE}.require_workdir") +@patch(f"{MODULE}.resolve_workdir") +@patch(f"{MODULE}.load_config", return_value={}) +@patch(f"{MODULE}.run") +def test_main_uses_explicit_workdir_over_resolve_workdir( + mock_run, mock_load_config, mock_resolve_workdir, mock_require_workdir +): + main(BASE_ARGS + ["--workdir", "/custom/workdir"]) + + mock_resolve_workdir.assert_not_called() + mock_require_workdir.assert_called_once_with(Path("/custom/workdir")) + mock_run.assert_called_once_with( + workdir=Path("/custom/workdir"), + model="azure-openai/gpt-4o", + task=None, + max_rounds=5, + training_iterations=10, + config={}, + ) + + +@pytest.mark.unit +@patch(f"{MODULE}.require_workdir") +@patch(f"{MODULE}.resolve_workdir", return_value=FAKE_WORKDIR) +@patch(f"{MODULE}.load_config", return_value={}) +@patch(f"{MODULE}.run") +def test_main_falls_back_to_resolve_workdir_when_not_given( + mock_run, mock_load_config, mock_resolve_workdir, mock_require_workdir +): main(BASE_ARGS) - mock_run_training_loop.assert_called_once_with( - repo_path="/repo", - feedback="", - memory_dir="/memory", + mock_resolve_workdir.assert_called_once_with() + mock_require_workdir.assert_called_once_with(FAKE_WORKDIR) + mock_run.assert_called_once_with( + workdir=FAKE_WORKDIR, model="azure-openai/gpt-4o", - iterations=1, + task=None, + max_rounds=5, + training_iterations=10, + config={}, ) @pytest.mark.unit -@patch(f"{MODULE}.run_train_eval_loop") -@patch(f"{MODULE}.run_training_loop") -def test_main_does_not_run_eval_loop_when_task_omitted(mock_run_training_loop, mock_run_train_eval_loop): +@patch(f"{MODULE}.require_workdir") +@patch(f"{MODULE}.resolve_workdir", return_value=FAKE_WORKDIR) +@patch(f"{MODULE}.run") +def test_main_calls_run_with_task_none_when_task_omitted(mock_run, mock_resolve_workdir, mock_require_workdir): main(BASE_ARGS) - mock_run_train_eval_loop.assert_not_called() + assert mock_run.call_args.kwargs["task"] is None @pytest.mark.unit -@patch(f"{MODULE}.run_train_eval_loop") -def test_main_runs_eval_loop_for_each_task_when_task_given(mock_run_train_eval_loop): +@patch(f"{MODULE}.require_workdir") +@patch(f"{MODULE}.resolve_workdir", return_value=FAKE_WORKDIR) +@patch(f"{MODULE}.load_config", return_value={}) +@patch(f"{MODULE}.run") +def test_main_calls_run_for_each_task_when_task_given( + mock_run, mock_load_config, mock_resolve_workdir, mock_require_workdir +): fake_task = MagicMock() - mock_run_train_eval_loop.return_value = MagicMock(passed=True, rounds_run=1) + mock_run.return_value = MagicMock(passed=True, rounds_run=1) - with patch(f"{MODULE}.TASK_REGISTRY", {"swebenchverified": MagicMock(from_cli_args=lambda args: [fake_task])}): + with patch(f"{MODULE}.TASK_REGISTRY", {"swebenchverified": MagicMock(from_config=lambda task_args: [fake_task])}): main(BASE_ARGS + ["--task", "swebenchverified"]) - mock_run_train_eval_loop.assert_called_once_with( - repo_path="/repo", - memory_dir="/memory", + mock_run.assert_called_once_with( + workdir=FAKE_WORKDIR, model="azure-openai/gpt-4o", task=fake_task, max_rounds=5, - training_iterations=1, + training_iterations=10, + config={}, ) @pytest.mark.unit -@patch(f"{MODULE}.run_training_loop") -def test_main_does_not_run_training_only_path_when_task_given(mock_run_training_loop): - fake_task = MagicMock() - - with patch(f"{MODULE}.TASK_REGISTRY", {"swebenchverified": MagicMock(from_cli_args=lambda args: [fake_task])}): - with patch(f"{MODULE}.run_train_eval_loop") as mock_run_train_eval_loop: - mock_run_train_eval_loop.return_value = MagicMock(passed=True, rounds_run=1) - main(BASE_ARGS + ["--task", "swebenchverified"]) - - mock_run_training_loop.assert_not_called() - - -@pytest.mark.unit -@patch(f"{MODULE}.run_train_eval_loop") -def test_main_runs_eval_loop_once_per_returned_task(mock_run_train_eval_loop): +@patch(f"{MODULE}.require_workdir") +@patch(f"{MODULE}.resolve_workdir", return_value=FAKE_WORKDIR) +@patch(f"{MODULE}.load_config", return_value={}) +@patch(f"{MODULE}.run") +def test_main_runs_once_per_returned_task(mock_run, mock_load_config, mock_resolve_workdir, mock_require_workdir): fake_tasks = [MagicMock(), MagicMock()] - mock_run_train_eval_loop.return_value = MagicMock(passed=False, rounds_run=5) + mock_run.return_value = MagicMock(passed=False, rounds_run=5) - with patch(f"{MODULE}.TASK_REGISTRY", {"swebenchverified": MagicMock(from_cli_args=lambda args: fake_tasks)}): + with patch(f"{MODULE}.TASK_REGISTRY", {"swebenchverified": MagicMock(from_config=lambda task_args: fake_tasks)}): main(BASE_ARGS + ["--task", "swebenchverified"]) - assert mock_run_train_eval_loop.call_count == 2 + assert mock_run.call_count == 2 diff --git a/test/auto_memory/test_orchestrator.py b/test/auto_memory/test_orchestrator.py index e869c55..10867f2 100644 --- a/test/auto_memory/test_orchestrator.py +++ b/test/auto_memory/test_orchestrator.py @@ -1,16 +1,28 @@ """Unit tests for microbots.auto_memory.orchestrator.""" +import json import os import sys from pathlib import Path -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock, call, patch import pytest sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../src/"))) -from microbots.auto_memory.orchestrator import LoopResult, run_train_eval_loop, run_training_loop -from microbots.auto_memory.task import CallbackResult, EvalOutcome +from microbots.auto_memory.orchestrator import ( + LoopResult, + clone_repo, + reset_repo, + run, + run_train_eval_loop, + run_training_loop, + write_eval_result, +) +from microbots.auto_memory.evalTask import CallbackResult, EvalOutcome +from microbots.auto_memory.workdir import eval_result_path, memory_dir, round_memory_dir + +MODULE = "microbots.auto_memory.orchestrator" def _make_outcome(passed: bool, log_path: str, reason: str = "reason") -> EvalOutcome: @@ -27,15 +39,26 @@ def _touch(path: str) -> str: return path +def _make_task() -> MagicMock: + """A MagicMock task with a real-ish task_id/build_result, for round tests.""" + task = MagicMock() + task.task_id = "task-1" + task.build_result.side_effect = lambda outcome: { + "passed": outcome.result.passed, + "reason": outcome.result.reason, + } + return task + + @pytest.mark.unit @patch("microbots.auto_memory.orchestrator.run_training_loop") @patch("microbots.auto_memory.orchestrator.build_feedback") def test_loop_returns_immediately_when_first_round_passes(mock_build_feedback, mock_run_training_loop, tmp_path): log_path = _touch(str(tmp_path / "round1.log")) - task = MagicMock() + task = _make_task() task.run.return_value = _make_outcome(passed=True, log_path=log_path) - result = run_train_eval_loop("/repo", "/memory", "azure-openai/gpt-4o", task, max_rounds=5) + result = run_train_eval_loop("/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) assert isinstance(result, LoopResult) assert result.passed is True @@ -51,20 +74,24 @@ def test_loop_returns_immediately_when_first_round_passes(mock_build_feedback, m def test_loop_retrains_and_continues_on_failure_then_passes(mock_build_feedback, mock_run_training_loop, tmp_path): log1 = _touch(str(tmp_path / "round1.log")) log2 = _touch(str(tmp_path / "round2.log")) - task = MagicMock() + task = _make_task() task.run.side_effect = [ _make_outcome(passed=False, log_path=log1), _make_outcome(passed=True, log_path=log2), ] mock_build_feedback.return_value = "feedback text" - result = run_train_eval_loop("/repo", "/memory", "azure-openai/gpt-4o", task, max_rounds=5) + result = run_train_eval_loop("/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) assert result.passed is True assert result.rounds_run == 2 mock_build_feedback.assert_called_once() mock_run_training_loop.assert_called_once_with( - repo_path="/repo", feedback="feedback text", memory_dir="/memory", model="azure-openai/gpt-4o", iterations=1 + repo_path="/repo", + feedback="feedback text", + memory_dir=str(round_memory_dir(tmp_path, 1, instance_id="task-1")), + model="azure-openai/gpt-4o", + iterations=10, ) @@ -72,14 +99,14 @@ def test_loop_retrains_and_continues_on_failure_then_passes(mock_build_feedback, @patch("microbots.auto_memory.orchestrator.run_training_loop") @patch("microbots.auto_memory.orchestrator.build_feedback") def test_loop_exhausts_max_rounds_without_passing(mock_build_feedback, mock_run_training_loop, tmp_path): - task = MagicMock() + task = _make_task() task.run.side_effect = [ _make_outcome(passed=False, log_path=_touch(str(tmp_path / f"round{i}.log"))) for i in range(3) ] mock_build_feedback.return_value = "feedback text" - result = run_train_eval_loop("/repo", "/memory", "azure-openai/gpt-4o", task, max_rounds=3) + result = run_train_eval_loop("/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=3) assert result.passed is False assert result.rounds_run == 3 @@ -92,33 +119,33 @@ def test_loop_exhausts_max_rounds_without_passing(mock_build_feedback, mock_run_ @pytest.mark.unit @patch("microbots.auto_memory.orchestrator.run_training_loop") @patch("microbots.auto_memory.orchestrator.build_feedback") -def test_log_path_deleted_after_passing_round(mock_build_feedback, mock_run_training_loop, tmp_path): +def test_log_path_persists_after_passing_round(mock_build_feedback, mock_run_training_loop, tmp_path): log_path = _touch(str(tmp_path / "round1.log")) - task = MagicMock() + task = _make_task() task.run.return_value = _make_outcome(passed=True, log_path=log_path) - run_train_eval_loop("/repo", "/memory", "azure-openai/gpt-4o", task, max_rounds=5) + run_train_eval_loop("/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) - assert not Path(log_path).exists() + assert Path(log_path).exists() @pytest.mark.unit @patch("microbots.auto_memory.orchestrator.run_training_loop") @patch("microbots.auto_memory.orchestrator.build_feedback") -def test_log_path_deleted_after_failing_round(mock_build_feedback, mock_run_training_loop, tmp_path): +def test_log_path_persists_after_failing_round(mock_build_feedback, mock_run_training_loop, tmp_path): log1 = _touch(str(tmp_path / "round1.log")) log2 = _touch(str(tmp_path / "round2.log")) - task = MagicMock() + task = _make_task() task.run.side_effect = [ _make_outcome(passed=False, log_path=log1), _make_outcome(passed=True, log_path=log2), ] mock_build_feedback.return_value = "feedback text" - run_train_eval_loop("/repo", "/memory", "azure-openai/gpt-4o", task, max_rounds=5) + run_train_eval_loop("/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) - assert not Path(log1).exists() - assert not Path(log2).exists() + assert Path(log1).exists() + assert Path(log2).exists() @pytest.mark.unit @@ -127,19 +154,19 @@ def test_log_path_deleted_after_failing_round(mock_build_feedback, mock_run_trai def test_build_feedback_exception_does_not_crash_loop(mock_build_feedback, mock_run_training_loop, tmp_path): log1 = _touch(str(tmp_path / "round1.log")) log2 = _touch(str(tmp_path / "round2.log")) - task = MagicMock() + task = _make_task() task.run.side_effect = [ _make_outcome(passed=False, log_path=log1), _make_outcome(passed=True, log_path=log2), ] mock_build_feedback.side_effect = RuntimeError("analysis bot crashed") - result = run_train_eval_loop("/repo", "/memory", "azure-openai/gpt-4o", task, max_rounds=5) + result = run_train_eval_loop("/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) assert result.passed is True assert result.rounds_run == 2 mock_run_training_loop.assert_not_called() - assert not Path(log1).exists() + assert Path(log1).exists() @pytest.mark.unit @@ -148,7 +175,7 @@ def test_build_feedback_exception_does_not_crash_loop(mock_build_feedback, mock_ def test_loop_forwards_training_iterations_to_run_training_loop(mock_build_feedback, mock_run_training_loop, tmp_path): log1 = _touch(str(tmp_path / "round1.log")) log2 = _touch(str(tmp_path / "round2.log")) - task = MagicMock() + task = _make_task() task.run.side_effect = [ _make_outcome(passed=False, log_path=log1), _make_outcome(passed=True, log_path=log2), @@ -156,11 +183,15 @@ def test_loop_forwards_training_iterations_to_run_training_loop(mock_build_feedb mock_build_feedback.return_value = "feedback text" run_train_eval_loop( - "/repo", "/memory", "azure-openai/gpt-4o", task, max_rounds=5, training_iterations=4 + "/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5, training_iterations=4 ) mock_run_training_loop.assert_called_once_with( - repo_path="/repo", feedback="feedback text", memory_dir="/memory", model="azure-openai/gpt-4o", iterations=4 + repo_path="/repo", + feedback="feedback text", + memory_dir=str(round_memory_dir(tmp_path, 1, instance_id="task-1")), + model="azure-openai/gpt-4o", + iterations=4, ) @@ -170,7 +201,7 @@ def test_loop_forwards_training_iterations_to_run_training_loop(mock_build_feedb def test_run_training_exception_does_not_crash_loop(mock_build_feedback, mock_run_training_loop, tmp_path): log1 = _touch(str(tmp_path / "round1.log")) log2 = _touch(str(tmp_path / "round2.log")) - task = MagicMock() + task = _make_task() task.run.side_effect = [ _make_outcome(passed=False, log_path=log1), _make_outcome(passed=True, log_path=log2), @@ -178,19 +209,20 @@ def test_run_training_exception_does_not_crash_loop(mock_build_feedback, mock_ru mock_build_feedback.return_value = "feedback text" mock_run_training_loop.side_effect = RuntimeError("training crashed") - result = run_train_eval_loop("/repo", "/memory", "azure-openai/gpt-4o", task, max_rounds=5) + result = run_train_eval_loop("/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) assert result.passed is True assert result.rounds_run == 2 - assert not Path(log1).exists() + assert Path(log1).exists() @pytest.mark.unit @patch("microbots.auto_memory.orchestrator.run_training") -def test_run_training_loop_calls_run_training_once_by_default(mock_run_training): +def test_run_training_loop_calls_run_training_ten_times_by_default(mock_run_training): run_training_loop(repo_path="/repo", feedback="fb", memory_dir="/memory", model="azure-openai/gpt-4o") - mock_run_training.assert_called_once_with( + assert mock_run_training.call_count == 10 + mock_run_training.assert_called_with( repo_path="/repo", feedback="fb", memory_dir="/memory", model="azure-openai/gpt-4o" ) @@ -217,3 +249,206 @@ def test_run_training_loop_reuses_same_memory_dir_each_pass(mock_run_training): memory_dirs = {call.kwargs["memory_dir"] for call in mock_run_training.call_args_list} assert memory_dirs == {"/memory"} + + +@pytest.mark.unit +@patch(f"{MODULE}.subprocess.run") +def test_clone_repo_clones_when_missing(mock_run, tmp_path): + repo_path = tmp_path / "repo" + + clone_repo("https://example.com/repo.git", repo_path) + + mock_run.assert_called_once_with( + ["git", "clone", "https://example.com/repo.git", str(repo_path)], check=True + ) + + +@pytest.mark.unit +@patch(f"{MODULE}.subprocess.run") +def test_clone_repo_is_noop_when_already_present(mock_run, tmp_path): + repo_path = tmp_path / "repo" + repo_path.mkdir() + + clone_repo("https://example.com/repo.git", repo_path) + + mock_run.assert_not_called() + + +@pytest.mark.unit +@patch(f"{MODULE}.subprocess.run") +def test_reset_repo_runs_hard_reset_then_clean(mock_run, tmp_path): + repo_path = tmp_path / "repo" + + reset_repo(repo_path, "abc123") + + assert mock_run.call_args_list == [ + call(["git", "reset", "--hard", "abc123"], cwd=repo_path, check=True), + call(["git", "clean", "-fd"], cwd=repo_path, check=True), + ] + + +@pytest.mark.unit +def test_write_eval_result_writes_task_build_result_as_json(tmp_path): + task = MagicMock() + task.task_id = "django__django-1" + task.build_result.return_value = {"passed": True, "reason": "resolved"} + outcome = _make_outcome(passed=True, log_path="/dev/null") + + write_eval_result(tmp_path, 2, task, outcome) + + result_path = eval_result_path(tmp_path, 2, "django__django-1") + assert json.loads(result_path.read_text()) == {"passed": True, "reason": "resolved"} + task.build_result.assert_called_once_with(outcome) + + +@pytest.mark.unit +def test_write_eval_result_creates_missing_parent_dirs(tmp_path): + task = MagicMock() + task.task_id = "some-task" + task.build_result.return_value = {"passed": False, "reason": "nope"} + outcome = _make_outcome(passed=False, log_path="/dev/null") + + write_eval_result(tmp_path, 1, task, outcome) + + assert eval_result_path(tmp_path, 1, "some-task").exists() + + +@pytest.mark.unit +@patch(f"{MODULE}.build_feedback") +def test_loop_writes_eval_result_for_every_round(mock_build_feedback, tmp_path): + log1 = _touch(str(tmp_path / "round1.log")) + log2 = _touch(str(tmp_path / "round2.log")) + task = _make_task() + task.run.side_effect = [ + _make_outcome(passed=False, log_path=log1), + _make_outcome(passed=True, log_path=log2), + ] + mock_build_feedback.return_value = "feedback text" + + with patch(f"{MODULE}.run_training_loop"): + run_train_eval_loop("/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) + + assert eval_result_path(tmp_path, 1, "task-1").exists() + assert eval_result_path(tmp_path, 2, "task-1").exists() + assert json.loads(eval_result_path(tmp_path, 2, "task-1").read_text()) == { + "passed": True, + "reason": "reason", + } + + +@pytest.mark.unit +@patch(f"{MODULE}.run_training_loop") +def test_run_calls_run_training_loop_when_task_is_none(mock_run_training_loop, tmp_path): + result = run(workdir=tmp_path, model="azure-openai/gpt-4o", task=None, training_iterations=2) + + mock_run_training_loop.assert_called_once_with( + repo_path=str(tmp_path / "repo"), + feedback="", + memory_dir=str(round_memory_dir(tmp_path, 1)), + model="azure-openai/gpt-4o", + iterations=2, + ) + assert result is None + + +@pytest.mark.unit +@patch(f"{MODULE}.run_train_eval_loop") +def test_run_calls_run_train_eval_loop_when_task_given(mock_run_train_eval_loop, tmp_path): + fake_task = MagicMock() + mock_run_train_eval_loop.return_value = "loop-result" + + result = run( + workdir=tmp_path, + model="azure-openai/gpt-4o", + task=fake_task, + max_rounds=3, + training_iterations=2, + ) + + mock_run_train_eval_loop.assert_called_once_with( + repo_path=str(tmp_path / "repo"), + workdir=tmp_path, + model="azure-openai/gpt-4o", + task=fake_task, + max_rounds=3, + training_iterations=2, + ) + assert result == "loop-result" + + +@pytest.mark.unit +@patch(f"{MODULE}.run_train_eval_loop") +@patch(f"{MODULE}.run_training_loop") +def test_run_does_not_call_eval_loop_when_task_is_none(mock_run_training_loop, mock_run_train_eval_loop, tmp_path): + run(workdir=tmp_path, model="azure-openai/gpt-4o", task=None) + + mock_run_train_eval_loop.assert_not_called() + + +@pytest.mark.unit +@patch(f"{MODULE}.run_train_eval_loop") +@patch(f"{MODULE}.run_training_loop") +def test_run_does_not_call_training_loop_when_task_given(mock_run_training_loop, mock_run_train_eval_loop, tmp_path): + run(workdir=tmp_path, model="azure-openai/gpt-4o", task=MagicMock()) + + mock_run_training_loop.assert_not_called() + + +@pytest.mark.unit +@patch(f"{MODULE}.clone_repo") +@patch(f"{MODULE}.run_training_loop") +def test_run_clones_repo_from_config_when_repo_url_given(mock_run_training_loop, mock_clone_repo, tmp_path): + (tmp_path / "config.yaml").write_text("repo: https://example.com/repo.git\n") + + run(workdir=tmp_path, model="azure-openai/gpt-4o", task=None) + + mock_clone_repo.assert_called_once_with("https://example.com/repo.git", tmp_path / "repo") + + +@pytest.mark.unit +@patch(f"{MODULE}.clone_repo") +@patch(f"{MODULE}.run_training_loop") +def test_run_does_not_clone_when_config_has_no_repo(mock_run_training_loop, mock_clone_repo, tmp_path): + run(workdir=tmp_path, model="azure-openai/gpt-4o", task=None) + + mock_clone_repo.assert_not_called() + + +@pytest.mark.unit +@patch(f"{MODULE}.run_training_loop") +def test_run_promotes_round1_memory_to_top_level_for_train_only_mode(mock_run_training_loop, tmp_path): + def fake_train(repo_path, feedback, memory_dir, model, iterations=1): + Path(memory_dir, "notes.md").write_text("learned something") + + mock_run_training_loop.side_effect = fake_train + + run(workdir=tmp_path, model="azure-openai/gpt-4o", task=None) + + assert (memory_dir(tmp_path) / "notes.md").read_text() == "learned something" + + +@pytest.mark.unit +@patch(f"{MODULE}.build_feedback") +@patch(f"{MODULE}.run_training_loop") +def test_loop_carries_memory_forward_between_rounds(mock_run_training_loop, mock_build_feedback, tmp_path): + log1 = _touch(str(tmp_path / "round1.log")) + log2 = _touch(str(tmp_path / "round2.log")) + mock_build_feedback.return_value = "feedback text" + seen_memory_dirs = [] + + def fake_run(repo_path, memory_dir, model, log_path): + round_num = len(seen_memory_dirs) + 1 + if round_num == 2: + # Round 2 should start with whatever round 1 saved. + assert (Path(memory_dir) / "notes.md").read_text() == "round 1 progress" + seen_memory_dirs.append(memory_dir) + Path(memory_dir, "notes.md").write_text(f"round {round_num} progress") + log_path = log1 if round_num == 1 else log2 + return _make_outcome(passed=round_num == 2, log_path=log_path) + + task = _make_task() + task.run.side_effect = fake_run + + run_train_eval_loop("/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) + + assert (memory_dir(tmp_path) / "notes.md").read_text() == "round 2 progress" diff --git a/test/auto_memory/test_task.py b/test/auto_memory/test_task.py index aee0f9d..ed11bb2 100644 --- a/test/auto_memory/test_task.py +++ b/test/auto_memory/test_task.py @@ -1,4 +1,4 @@ -"""Unit tests for microbots.auto_memory.task.""" +"""Unit tests for microbots.auto_memory.evalTask.""" import os import sys @@ -7,13 +7,13 @@ sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../src/"))) -from microbots.auto_memory.task import CallbackResult, EvalOutcome, EvalTask +from microbots.auto_memory.evalTask import CallbackResult, EvalOutcome, EvalTask class _RunOnlyTask(EvalTask): """A task that overrides only run(), never touching the optional hooks.""" - def run(self, repo_path, memory_dir, model): + def run(self, repo_path, memory_dir, model, log_path): return EvalOutcome( passed=True, output="custom output", @@ -31,7 +31,7 @@ def test_run_is_abstract(): @pytest.mark.unit def test_subclass_overriding_only_run_is_instantiable(): task = _RunOnlyTask() - outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o") + outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o", "/log") assert outcome.passed is True assert outcome.output == "custom output" @@ -51,7 +51,7 @@ def test_default_teardown_is_a_noop(): @pytest.mark.unit def test_default_build_prompt_returns_empty_string(): - assert _RunOnlyTask().build_prompt("/repo") == "" + assert _RunOnlyTask().build_prompt() == "" @pytest.mark.unit @@ -60,3 +60,20 @@ def test_default_check_passes_by_default(): assert isinstance(result, CallbackResult) assert result.passed is True + + +@pytest.mark.unit +def test_default_task_id_is_class_name(): + assert _RunOnlyTask().task_id == "_RunOnlyTask" + + +@pytest.mark.unit +def test_default_build_result_returns_passed_and_reason(): + outcome = EvalOutcome( + passed=False, + output="agent output", + result=CallbackResult(passed=False, reason="check failed"), + log_path="/dev/null", + ) + + assert _RunOnlyTask().build_result(outcome) == {"passed": False, "reason": "check failed"} diff --git a/test/auto_memory/test_task_registry.py b/test/auto_memory/test_task_registry.py index d06247f..4caa88f 100644 --- a/test/auto_memory/test_task_registry.py +++ b/test/auto_memory/test_task_registry.py @@ -9,7 +9,7 @@ sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../src/"))) -from microbots.auto_memory.task import EvalTask +from microbots.auto_memory.evalTask import EvalTask from microbots.auto_memory.task_registry import TASK_REGISTRY, create_task, discover_tasks, register_task MODULE_PATH = "microbots.auto_memory.task_registry" diff --git a/test/auto_memory/test_workdir.py b/test/auto_memory/test_workdir.py new file mode 100644 index 0000000..f899588 --- /dev/null +++ b/test/auto_memory/test_workdir.py @@ -0,0 +1,94 @@ +"""Unit tests for microbots.auto_memory.workdir.""" + +import os +import sys + +import pytest + +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../src/"))) + +from microbots.auto_memory.workdir import ( + CONFIG_FILENAME, + load_config, + load_round_memory, + memory_dir, + round_memory_dir, + save_round_memory, +) + + +@pytest.mark.unit +def test_load_config_returns_empty_dict_when_file_missing(tmp_path): + assert load_config(tmp_path) == {} + + +@pytest.mark.unit +def test_load_config_returns_empty_dict_when_file_empty(tmp_path): + (tmp_path / CONFIG_FILENAME).write_text("") + + assert load_config(tmp_path) == {} + + +@pytest.mark.unit +def test_load_config_parses_yaml_contents(tmp_path): + (tmp_path / CONFIG_FILENAME).write_text("repo: https://example.com/repo.git\ntask: swebenchverified\n") + + assert load_config(tmp_path) == { + "repo": "https://example.com/repo.git", + "task": "swebenchverified", + } + + +@pytest.mark.unit +def test_load_round_memory_creates_empty_dir_when_no_top_level_memory(tmp_path): + result = load_round_memory(tmp_path, 1) + + assert result == round_memory_dir(tmp_path, 1) + assert result.is_dir() + assert list(result.iterdir()) == [] + + +@pytest.mark.unit +def test_load_round_memory_copies_top_level_memory_into_round(tmp_path): + top_memory = memory_dir(tmp_path) + top_memory.mkdir(parents=True) + (top_memory / "notes.md").write_text("prior findings") + + result = load_round_memory(tmp_path, 2) + + assert (result / "notes.md").read_text() == "prior findings" + + +@pytest.mark.unit +def test_save_round_memory_creates_empty_dir_when_no_round_memory(tmp_path): + result = save_round_memory(tmp_path, 1) + + assert result == memory_dir(tmp_path) + assert result.is_dir() + assert list(result.iterdir()) == [] + + +@pytest.mark.unit +def test_save_round_memory_copies_round_memory_to_top_level(tmp_path): + round_memory = round_memory_dir(tmp_path, 3) + round_memory.mkdir(parents=True) + (round_memory / "learned.md").write_text("new insight") + + result = save_round_memory(tmp_path, 3) + + assert (result / "learned.md").read_text() == "new insight" + + +@pytest.mark.unit +def test_save_round_memory_overwrites_stale_top_level_files(tmp_path): + top_memory = memory_dir(tmp_path) + top_memory.mkdir(parents=True) + (top_memory / "notes.md").write_text("old") + + round_memory = round_memory_dir(tmp_path, 1) + round_memory.mkdir(parents=True) + (round_memory / "notes.md").write_text("new") + + save_round_memory(tmp_path, 1) + + assert (top_memory / "notes.md").read_text() == "new" From 085f462492e15397f14ea1d41c2138e029599b74 Mon Sep 17 00:00:00 2001 From: Kavya Sree Kaitepalli Date: Wed, 2 Sep 2026 07:27:22 +0000 Subject: [PATCH 08/13] Update development mode package installation to include training dependencies --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e898772..a87c7a7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -165,7 +165,7 @@ jobs: - name: Install package in development mode run: | - pip install -e . + pip install -e ".[training]" - name: Install GitHub Copilot SDK dependencies for GHCP tests if: matrix.test-type == 'ghcp' From 25e0b9186d1cf870b65af2fce58607a134f27f67 Mon Sep 17 00:00:00 2001 From: Kavya Sree Kaitepalli Date: Wed, 2 Sep 2026 07:43:17 +0000 Subject: [PATCH 09/13] Add unit tests --- .../auto_memory/eval/test_swebenchverified.py | 42 ++++++ test/auto_memory/test_workdir.py | 124 ++++++++++++++++++ 2 files changed, 166 insertions(+) diff --git a/test/auto_memory/eval/test_swebenchverified.py b/test/auto_memory/eval/test_swebenchverified.py index 1835e8d..98a89cc 100644 --- a/test/auto_memory/eval/test_swebenchverified.py +++ b/test/auto_memory/eval/test_swebenchverified.py @@ -523,3 +523,45 @@ class _EmptyArgs: mock_load_instances_of_repo.assert_called_once_with(repo=None) assert len(tasks) == 1 + + +# --------------------------------------------------------------------------- +# SweBenchVerifiedTask.from_config +# --------------------------------------------------------------------------- + +@pytest.mark.unit +@patch(f"{MODULE}.load_instance_using_id") +def test_from_config_uses_instance_id_when_given(mock_load_instance_using_id): + mock_load_instance_using_id.return_value = _instance() + + tasks = SweBenchVerifiedTask.from_config( + {"instance_id": "django__django-1", "swebench_repo": None} + ) + + mock_load_instance_using_id.assert_called_once_with("django__django-1") + assert len(tasks) == 1 + assert isinstance(tasks[0], SweBenchVerifiedTask) + assert tasks[0].instance == _instance() + + +@pytest.mark.unit +@patch(f"{MODULE}.load_instances_of_repo") +def test_from_config_falls_back_to_repo_filter_when_no_instance_id(mock_load_instances_of_repo): + mock_load_instances_of_repo.return_value = [_instance(), _instance()] + + tasks = SweBenchVerifiedTask.from_config({"swebench_repo": "django/django"}) + + mock_load_instances_of_repo.assert_called_once_with(repo="django/django") + assert len(tasks) == 2 + assert all(isinstance(t, SweBenchVerifiedTask) for t in tasks) + + +@pytest.mark.unit +@patch(f"{MODULE}.load_instances_of_repo") +def test_from_config_handles_empty_dict_gracefully(mock_load_instances_of_repo): + mock_load_instances_of_repo.return_value = [_instance()] + + tasks = SweBenchVerifiedTask.from_config({}) + + mock_load_instances_of_repo.assert_called_once_with(repo=None) + assert len(tasks) == 1 diff --git a/test/auto_memory/test_workdir.py b/test/auto_memory/test_workdir.py index f899588..149b223 100644 --- a/test/auto_memory/test_workdir.py +++ b/test/auto_memory/test_workdir.py @@ -9,10 +9,21 @@ from microbots.auto_memory.workdir import ( CONFIG_FILENAME, + eval_dir, + eval_log_path, + eval_patch_path, + eval_result_path, load_config, load_round_memory, memory_dir, + repo_dir, + require_workdir, + resolve_workdir, + round_dir, + round_log_path, round_memory_dir, + round_patch_path, + run_log_path, save_round_memory, ) @@ -92,3 +103,116 @@ def test_save_round_memory_overwrites_stale_top_level_files(tmp_path): save_round_memory(tmp_path, 1) assert (top_memory / "notes.md").read_text() == "new" + + +@pytest.mark.unit +def test_resolve_workdir_defaults_to_cwd(monkeypatch, tmp_path): + monkeypatch.chdir(tmp_path) + + assert resolve_workdir() == tmp_path / "workdir" + + +@pytest.mark.unit +def test_resolve_workdir_uses_given_base(tmp_path): + assert resolve_workdir(tmp_path) == tmp_path / "workdir" + + +@pytest.mark.unit +def test_require_workdir_raises_when_missing(tmp_path): + missing = tmp_path / "nope" + + with pytest.raises(FileNotFoundError): + require_workdir(missing) + + +@pytest.mark.unit +def test_require_workdir_passes_when_present(tmp_path): + require_workdir(tmp_path) + + +@pytest.mark.unit +def test_repo_dir_returns_workdir_repo(tmp_path): + assert repo_dir(tmp_path) == tmp_path / "repo" + + +@pytest.mark.unit +def test_run_log_path_returns_workdir_run_log(tmp_path): + assert run_log_path(tmp_path) == tmp_path / "run.log" + + +@pytest.mark.unit +def test_round_dir_creates_directory_when_requested(tmp_path): + path = round_dir(tmp_path, 1, create=True) + + assert path == tmp_path / "rounds" / "round_1" + assert path.is_dir() + + +@pytest.mark.unit +def test_round_dir_does_not_create_directory_by_default(tmp_path): + path = round_dir(tmp_path, 1) + + assert path == tmp_path / "rounds" / "round_1" + assert not path.exists() + + +@pytest.mark.unit +def test_round_dir_uses_per_instance_dir_when_instance_id_given(tmp_path): + path = round_dir(tmp_path, 1, instance_id="task-1") + + assert path == tmp_path / "rounds_task-1" / "round_1" + + +@pytest.mark.unit +def test_round_log_path_returns_round_log(tmp_path): + assert round_log_path(tmp_path, 2) == round_dir(tmp_path, 2) / "round.log" + + +@pytest.mark.unit +def test_round_log_path_with_instance_id(tmp_path): + assert round_log_path(tmp_path, 2, instance_id="task-1") == round_dir( + tmp_path, 2, instance_id="task-1" + ) / "round.log" + + +@pytest.mark.unit +def test_round_patch_path_returns_repo_patch(tmp_path): + assert round_patch_path(tmp_path, 2) == round_dir(tmp_path, 2) / "repo.patch" + + +@pytest.mark.unit +def test_round_patch_path_with_instance_id(tmp_path): + assert round_patch_path(tmp_path, 2, instance_id="task-1") == round_dir( + tmp_path, 2, instance_id="task-1" + ) / "repo.patch" + + +@pytest.mark.unit +def test_eval_dir_creates_directory_when_requested(tmp_path): + path = eval_dir(tmp_path, 1, "task-1", create=True) + + assert path == tmp_path / "rounds_task-1" / "round_1" / "eval" + assert path.is_dir() + + +@pytest.mark.unit +def test_eval_dir_does_not_create_directory_by_default(tmp_path): + path = eval_dir(tmp_path, 1, "task-1") + + assert path == tmp_path / "rounds_task-1" / "round_1" / "eval" + assert not path.exists() + + +@pytest.mark.unit +def test_eval_result_path_returns_result_json(tmp_path): + assert eval_result_path(tmp_path, 1, "task-1") == eval_dir(tmp_path, 1, "task-1") / "result.json" + + +@pytest.mark.unit +def test_eval_log_path_returns_eval_log(tmp_path): + assert eval_log_path(tmp_path, 1, "task-1") == eval_dir(tmp_path, 1, "task-1") / "eval.log" + + +@pytest.mark.unit +def test_eval_patch_path_returns_repo_patch(tmp_path): + assert eval_patch_path(tmp_path, 1, "task-1") == eval_dir(tmp_path, 1, "task-1") / "repo.patch" From 00d095bef39a81b00e6fe148aec0015ed80c9646 Mon Sep 17 00:00:00 2001 From: Kavya Sree Kaitepalli Date: Wed, 2 Sep 2026 12:39:04 +0000 Subject: [PATCH 10/13] Refactor evalTask and related modules: implement build_feedback method, remove analyzer, and update tests --- src/microbots/auto_memory/analyzer.py | 65 -------- .../auto_memory/eval/swebenchverified.py | 96 ++++++------ src/microbots/auto_memory/evalTask.py | 35 ++++- src/microbots/auto_memory/orchestrator.py | 8 +- src/microbots/auto_memory/workdir.py | 21 --- .../auto_memory/eval/test_swebenchverified.py | 142 ++++++++++-------- test/auto_memory/test_analyzer.py | 81 ---------- test/auto_memory/test_orchestrator.py | 103 +++++-------- test/auto_memory/test_task.py | 17 ++- test/auto_memory/test_task_registry.py | 3 + test/auto_memory/test_workdir.py | 13 -- 11 files changed, 221 insertions(+), 363 deletions(-) delete mode 100644 src/microbots/auto_memory/analyzer.py delete mode 100644 test/auto_memory/test_analyzer.py diff --git a/src/microbots/auto_memory/analyzer.py b/src/microbots/auto_memory/analyzer.py deleted file mode 100644 index ca35c16..0000000 --- a/src/microbots/auto_memory/analyzer.py +++ /dev/null @@ -1,65 +0,0 @@ -"""Build feedback text for a failed evaluation round. - -Uses ``LogAnalysisBot`` to analyze the eval callback's raw log and produce -concrete feedback describing what went wrong, to be passed into the -training agent as ``feedback`` for the next round. -""" - -from logging import getLogger - -from microbots.auto_memory.evalTask import EvalOutcome, EvalTask -from microbots.bot.LogAnalysisBot import LogAnalysisBot -from microbots.MicroBot import BotRunResult - -logger = getLogger(__name__) -#make this abstract -def build_feedback( - task: EvalTask, - outcome: EvalOutcome, - repo_path: str, - model: str, -) -> str: - """Analyze a failed eval outcome's log and produce training feedback. - - Parameters - ---------- - task : EvalTask - The eval task that produced ``outcome``. - outcome : EvalOutcome - The failed outcome to analyze, including its ``log_path``. - repo_path : str - Absolute path to the repo the task was evaluated against. - model : str - The model to use, in the format ``/``. - - Returns - ------- - str - Feedback text describing the root cause of the failure and what - the agent's memory notes should cover next time, suitable for - passing as ``feedback`` to ``run_training``. - """ - bot = LogAnalysisBot(model=model, folder_to_mount=repo_path) - result: BotRunResult = bot.run( - file_name=outcome.log_path, - user_prompt=( - "This log was produced while verifying whether an " - "agent completed its task correctly. Identify " - "the root cause of the failure and describe concretely " - "what the agent's memory notes should cover next time to " - "avoid this failure." - ), - ) - - if result.status and result.result: - return result.result - - logger.warning( - "LogAnalysisBot failed to analyze failure (%s); falling back to plain feedback", - result.error, - ) - return ( - f"Evaluation failed. Agent output: {outcome.output}\n" - f"Callback reason: {outcome.result.reason}" - ) - \ No newline at end of file diff --git a/src/microbots/auto_memory/eval/swebenchverified.py b/src/microbots/auto_memory/eval/swebenchverified.py index 96017b8..2fd64b1 100644 --- a/src/microbots/auto_memory/eval/swebenchverified.py +++ b/src/microbots/auto_memory/eval/swebenchverified.py @@ -5,7 +5,6 @@ verifies the result via ``swebench.harness.run_evaluation``. """ -import argparse import json import shutil import subprocess @@ -20,7 +19,9 @@ from datasets import load_dataset from microbots.auto_memory.evalTask import CallbackResult, EvalOutcome, EvalTask from microbots.auto_memory.task_registry import register_task +from microbots.bot.LogAnalysisBot import LogAnalysisBot from microbots.bot.WritingBot import WritingBot +from microbots.MicroBot import BotRunResult from microbots.tools.tool_definitions.memory_tool import MemoryTool logger = getLogger(__name__) @@ -168,47 +169,6 @@ def __init__(self, instance: SweBenchInstance | None = None): """ self.instance = instance - @staticmethod - def add_cli_args(parser: argparse.ArgumentParser) -> None: - """Register this task's CLI flags on ``parser``. - - Parameters - ---------- - parser : argparse.ArgumentParser - The CLI's argument parser to add task-specific flags to. - """ - parser.add_argument( - "--instance-id", - help='SWE-bench-verified instance ID, e.g. "django__django-11099".', - ) - parser.add_argument( - "--swebench-repo", - help='Restrict to instances for this repo, e.g. "django/django". ' - "Ignored if --instance-id is given.", - ) - - @classmethod - def from_cli_args(cls, args: argparse.Namespace) -> list["SweBenchVerifiedTask"]: - """Build task(s) from parsed CLI args. - - Parameters - ---------- - args : argparse.Namespace - Parsed CLI args, expected to include ``instance_id`` and/or - ``swebench_repo`` (see ``add_cli_args``). - - Returns - ------- - list[SweBenchVerifiedTask] - One task per matching dataset instance. A single-element - list when ``--instance-id`` is given. - """ - if getattr(args, "instance_id", None): - instances = [load_instance_using_id(args.instance_id)] - else: - instances = load_instances_of_repo(repo=getattr(args, "swebench_repo", None)) - return [cls(instance) for instance in instances] - @classmethod def from_config(cls, task_args: dict) -> list["SweBenchVerifiedTask"]: """Build task(s) from a config's ``task_args`` dict. @@ -217,8 +177,7 @@ def from_config(cls, task_args: dict) -> list["SweBenchVerifiedTask"]: ---------- task_args : dict Task-specific config values, expected to include - ``instance_id`` and/or ``swebench_repo`` (mirrors - ``add_cli_args``'s flags). + ``instance_id`` and/or ``swebench_repo``. Returns ------- @@ -365,6 +324,51 @@ def teardown(self, repo_path: str) -> None: """ subprocess.run(["rm", "-rf", repo_path], check=False) + def build_feedback(self, outcome: EvalOutcome, repo_path: str, model: str, log_path: str) -> str: + """Analyze a failed round's log via ``LogAnalysisBot`` for training feedback. + + Parameters + ---------- + outcome : EvalOutcome + The failed outcome to analyze. + repo_path : str + Absolute path to the repo the task was evaluated against. + model : str + The model to use, in the format ``/``. + log_path : str + Path to the round's log file (the same path passed to + ``run``), analyzed by ``LogAnalysisBot``. + + Returns + ------- + str + Feedback text describing the root cause of the failure and + what the agent's memory notes should cover next time. + """ + bot = LogAnalysisBot(model=model, folder_to_mount=repo_path) + result: BotRunResult = bot.run( + file_name=log_path, + user_prompt=( + "This log was produced while verifying whether an " + "agent completed its task correctly. Identify " + "the root cause of the failure and describe concretely " + "what the agent's memory notes should cover next time to " + "avoid this failure." + ), + ) + + if result.status and result.result: + return result.result + + logger.warning( + "LogAnalysisBot failed to analyze failure (%s); falling back to plain feedback", + result.error, + ) + return ( + f"Evaluation failed. Agent output: {outcome.output}\n" + f"Callback reason: {outcome.result.reason}" + ) + def run(self, repo_path: str, memory_dir: str, model: str, log_path: str) -> EvalOutcome: """Run one eval iteration: setup -> build_prompt -> WritingBot -> check -> teardown. @@ -385,7 +389,7 @@ def run(self, repo_path: str, memory_dir: str, model: str, log_path: str) -> Eva ------- EvalOutcome The result of this eval round, including the agent's output, - the check verdict, and the round's log file path. + the check verdict. """ self.setup(repo_path) Path(log_path).parent.mkdir(parents=True, exist_ok=True) @@ -416,7 +420,6 @@ def run(self, repo_path: str, memory_dir: str, model: str, log_path: str) -> Eva passed=result.passed, output=bot_result.result, result=result, - log_path=log_path, ) except Exception as exc: logger.exception( @@ -430,7 +433,6 @@ def run(self, repo_path: str, memory_dir: str, model: str, log_path: str) -> Eva result=CallbackResult( passed=False, reason=f"{type(exc).__name__}: {exc}" ), - log_path=log_path, ) finally: try: diff --git a/src/microbots/auto_memory/evalTask.py b/src/microbots/auto_memory/evalTask.py index 6a8925c..d58476a 100644 --- a/src/microbots/auto_memory/evalTask.py +++ b/src/microbots/auto_memory/evalTask.py @@ -35,15 +35,11 @@ class EvalOutcome: The agent's raw output for the round, if any. result : CallbackResult The verdict produced by ``EvalTask.check``. - log_path : str - Path to the round's log file, containing the agent output and - any failure/exception details recorded during the round. """ passed: bool output: str | None result: CallbackResult - log_path: str class EvalTask(ABC): @@ -152,6 +148,35 @@ def teardown(self, repo_path: str) -> None: """ pass + @abstractmethod + def build_feedback(self, outcome: EvalOutcome, repo_path: str, model: str, log_path: str) -> str: + """Required. Analyze a failed eval outcome and produce training feedback. + + Called by the orchestrator after a failed round, before + retraining, to turn the round's outcome/log into concrete + feedback text describing what went wrong and what the agent's + memory notes should cover next time. + + Parameters + ---------- + outcome : EvalOutcome + The failed outcome to analyze. + repo_path : str + Absolute path to the repo the task was evaluated against. + model : str + The model to use, in the format ``/``. + log_path : str + Path to the round's log file, containing the agent output + and any failure/exception details recorded during the + round (the same path passed to ``run``). + + Returns + ------- + str + Feedback text to pass as ``feedback`` to the next round's + training. + """ + @abstractmethod def run(self, repo_path: str, memory_dir: str, model: str, log_path: str) -> EvalOutcome: """Required. Run one eval iteration and return its outcome. @@ -174,5 +199,5 @@ def run(self, repo_path: str, memory_dir: str, model: str, log_path: str) -> Eva ------- EvalOutcome The result of this eval round, including the agent's output, - the check verdict, and the round's log file path. + the check verdict. """ diff --git a/src/microbots/auto_memory/orchestrator.py b/src/microbots/auto_memory/orchestrator.py index 0c29538..69b36dc 100644 --- a/src/microbots/auto_memory/orchestrator.py +++ b/src/microbots/auto_memory/orchestrator.py @@ -11,7 +11,6 @@ import json import subprocess -from microbots.auto_memory.analyzer import build_feedback from microbots.auto_memory.evalTask import EvalOutcome, EvalTask from microbots.auto_memory.training.runner import run_training from microbots.auto_memory.workdir import ( @@ -197,9 +196,8 @@ def run_train_eval_loop( "run_train_eval_loop: round %d/%d starting", round_idx, max_rounds ) memory_dir = str(load_round_memory(workdir, round_idx, instance_id=task.task_id)) - outcome = task.run( - repo_path, memory_dir, model, str(eval_log_path(workdir, round_idx, task.task_id)) - ) + log_path = str(eval_log_path(workdir, round_idx, task.task_id)) + outcome = task.run(repo_path, memory_dir, model, log_path) outcomes.append(outcome) try: @@ -220,7 +218,7 @@ def run_train_eval_loop( outcome.result.reason, ) try: - feedback = build_feedback(task, outcome, repo_path, model) + feedback = task.build_feedback(outcome, repo_path, model, log_path) run_training_loop( repo_path=repo_path, feedback=feedback, diff --git a/src/microbots/auto_memory/workdir.py b/src/microbots/auto_memory/workdir.py index 771066c..bcaf616 100644 --- a/src/microbots/auto_memory/workdir.py +++ b/src/microbots/auto_memory/workdir.py @@ -275,27 +275,6 @@ def round_log_path(workdir: Path, round_num: int, *, instance_id: str | None = N return round_dir(workdir, round_num, instance_id=instance_id) / ROUND_LOG_FILENAME -def round_patch_path(workdir: Path, round_num: int, *, instance_id: str | None = None) -> Path: - """Return the path to a round's captured repo diff. - - Parameters - ---------- - workdir : Path - The run's workdir. - round_num : int - 1-based round number. - instance_id : str | None - The eval task's ``task_id``, if running an eval task (see - ``round_dir``). Omit for training-only mode. - - Returns - ------- - Path - This round's ``repo.patch``. - """ - return round_dir(workdir, round_num, instance_id=instance_id) / ROUND_PATCH_FILENAME - - def eval_dir( workdir: Path, round_num: int, instance_id: str, *, create: bool = False ) -> Path: diff --git a/test/auto_memory/eval/test_swebenchverified.py b/test/auto_memory/eval/test_swebenchverified.py index 98a89cc..56e1906 100644 --- a/test/auto_memory/eval/test_swebenchverified.py +++ b/test/auto_memory/eval/test_swebenchverified.py @@ -18,6 +18,7 @@ load_instances_of_repo, ) from microbots.auto_memory.evalTask import CallbackResult, EvalOutcome +from microbots.MicroBot import BotRunResult MODULE = "microbots.auto_memory.eval.swebenchverified" @@ -156,7 +157,6 @@ def test_build_result_includes_dataset_fields(): passed=True, output="agent output", result=CallbackResult(passed=True, reason="resolved"), - log_path="/dev/null", ) assert task.build_result(outcome) == { @@ -177,6 +177,85 @@ def test_teardown_removes_repo_path(mock_run): mock_run.assert_called_once_with(["rm", "-rf", "/repo"], check=False) +# --------------------------------------------------------------------------- +# SweBenchVerifiedTask.build_feedback +# --------------------------------------------------------------------------- + +def _failed_outcome(reason: str = "tests failed", output: str = "agent output") -> EvalOutcome: + return EvalOutcome( + passed=False, + output=output, + result=CallbackResult(passed=False, reason=reason), + ) + + +@pytest.mark.unit +@patch(f"{MODULE}.LogAnalysisBot") +def test_build_feedback_returns_bot_result_on_success(mock_bot_cls): + mock_bot = MagicMock() + mock_bot.run.return_value = BotRunResult( + status=True, result="root cause: missing edge case handling", error=None + ) + mock_bot_cls.return_value = mock_bot + + task = SweBenchVerifiedTask(_instance()) + outcome = _failed_outcome() + + feedback = task.build_feedback(outcome, "/repo", "azure-openai/gpt-4o", "/tmp/some.log") + + assert feedback == "root cause: missing edge case handling" + mock_bot_cls.assert_called_once_with(model="azure-openai/gpt-4o", folder_to_mount="/repo") + mock_bot.run.assert_called_once() + assert mock_bot.run.call_args.kwargs["file_name"] == "/tmp/some.log" + + +@pytest.mark.unit +@patch(f"{MODULE}.LogAnalysisBot") +def test_build_feedback_falls_back_when_bot_status_false(mock_bot_cls): + mock_bot = MagicMock() + mock_bot.run.return_value = BotRunResult(status=False, result=None, error="bot crashed") + mock_bot_cls.return_value = mock_bot + + task = SweBenchVerifiedTask(_instance()) + outcome = _failed_outcome(reason="tests failed", output="some output") + + feedback = task.build_feedback(outcome, "/repo", "azure-openai/gpt-4o", "/tmp/some.log") + + assert "some output" in feedback + assert "tests failed" in feedback + + +@pytest.mark.unit +@patch(f"{MODULE}.LogAnalysisBot") +def test_build_feedback_falls_back_when_result_is_empty(mock_bot_cls): + mock_bot = MagicMock() + mock_bot.run.return_value = BotRunResult(status=True, result="", error=None) + mock_bot_cls.return_value = mock_bot + + task = SweBenchVerifiedTask(_instance()) + outcome = _failed_outcome(reason="assertion error", output="agent tried X") + + feedback = task.build_feedback(outcome, "/repo", "azure-openai/gpt-4o", "/tmp/some.log") + + assert "agent tried X" in feedback + assert "assertion error" in feedback + + +@pytest.mark.unit +@patch(f"{MODULE}.LogAnalysisBot") +def test_build_feedback_falls_back_when_result_is_none(mock_bot_cls): + mock_bot = MagicMock() + mock_bot.run.return_value = BotRunResult(status=True, result=None, error=None) + mock_bot_cls.return_value = mock_bot + + task = SweBenchVerifiedTask(_instance()) + outcome = _failed_outcome() + + feedback = task.build_feedback(outcome, "/repo", "azure-openai/gpt-4o", "/tmp/some.log") + + assert "Evaluation failed" in feedback + + # --------------------------------------------------------------------------- # SweBenchVerifiedTask.check # --------------------------------------------------------------------------- @@ -319,7 +398,7 @@ def test_run_calls_setup_build_prompt_check_teardown_in_order(mock_bot_cls, mock outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o", str(tmp_path / "eval.log")) assert calls[0] == ("setup", "/repo") - assert calls[1] == ("check", "/repo", "agent did stuff", outcome.log_path) + assert calls[1] == ("check", "/repo", "agent did stuff", str(tmp_path / "eval.log")) assert calls[2] == ("teardown", "/repo") assert outcome.passed is True assert outcome.output == "agent did stuff" @@ -393,7 +472,7 @@ def _build_prompt(): assert outcome.passed is False assert "bad prompt" in outcome.result.reason - with open(outcome.log_path) as f: + with open(str(tmp_path / "eval.log")) as f: assert "bad prompt" in f.read() @@ -467,63 +546,6 @@ def _teardown(repo_path): -# --------------------------------------------------------------------------- -# SweBenchVerifiedTask.add_cli_args / from_cli_args -# --------------------------------------------------------------------------- - -@pytest.mark.unit -def test_add_cli_args_registers_instance_id_and_repo_flags(): - import argparse - - parser = argparse.ArgumentParser() - SweBenchVerifiedTask.add_cli_args(parser) - - args = parser.parse_args(["--instance-id", "django__django-1", "--swebench-repo", "django/django"]) - assert args.instance_id == "django__django-1" - assert args.swebench_repo == "django/django" - - -@pytest.mark.unit -@patch(f"{MODULE}.load_instance_using_id") -def test_from_cli_args_uses_instance_id_when_given(mock_load_instance_using_id): - mock_load_instance_using_id.return_value = _instance() - args = MagicMock(instance_id="django__django-1", swebench_repo=None) - - tasks = SweBenchVerifiedTask.from_cli_args(args) - - mock_load_instance_using_id.assert_called_once_with("django__django-1") - assert len(tasks) == 1 - assert isinstance(tasks[0], SweBenchVerifiedTask) - assert tasks[0].instance == _instance() - - -@pytest.mark.unit -@patch(f"{MODULE}.load_instances_of_repo") -def test_from_cli_args_falls_back_to_repo_filter_when_no_instance_id(mock_load_instances_of_repo): - mock_load_instances_of_repo.return_value = [_instance(), _instance()] - args = MagicMock(instance_id=None, swebench_repo="django/django") - - tasks = SweBenchVerifiedTask.from_cli_args(args) - - mock_load_instances_of_repo.assert_called_once_with(repo="django/django") - assert len(tasks) == 2 - assert all(isinstance(t, SweBenchVerifiedTask) for t in tasks) - - -@pytest.mark.unit -@patch(f"{MODULE}.load_instances_of_repo") -def test_from_cli_args_handles_missing_attrs_gracefully(mock_load_instances_of_repo): - """Namespace without instance_id/swebench_repo attrs at all (not just None).""" - mock_load_instances_of_repo.return_value = [_instance()] - - class _EmptyArgs: - pass - - tasks = SweBenchVerifiedTask.from_cli_args(_EmptyArgs()) - - mock_load_instances_of_repo.assert_called_once_with(repo=None) - assert len(tasks) == 1 - # --------------------------------------------------------------------------- # SweBenchVerifiedTask.from_config diff --git a/test/auto_memory/test_analyzer.py b/test/auto_memory/test_analyzer.py deleted file mode 100644 index 3566b65..0000000 --- a/test/auto_memory/test_analyzer.py +++ /dev/null @@ -1,81 +0,0 @@ -"""Unit tests for microbots.auto_memory.analyzer.""" - -import os -import sys -from unittest.mock import MagicMock, patch - -import pytest - -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../src/"))) - -from microbots.auto_memory.analyzer import build_feedback -from microbots.auto_memory.evalTask import CallbackResult, EvalOutcome -from microbots.MicroBot import BotRunResult - - -def _make_outcome(reason: str = "tests failed", output: str = "agent output") -> EvalOutcome: - return EvalOutcome( - passed=False, - output=output, - result=CallbackResult(passed=False, reason=reason), - log_path="/tmp/some.log", - ) - - -@pytest.mark.unit -@patch("microbots.auto_memory.analyzer.LogAnalysisBot") -def test_build_feedback_returns_bot_result_on_success(mock_bot_cls): - mock_bot = MagicMock() - mock_bot.run.return_value = BotRunResult( - status=True, result="root cause: missing edge case handling", error=None - ) - mock_bot_cls.return_value = mock_bot - - outcome = _make_outcome() - feedback = build_feedback(task=MagicMock(), outcome=outcome, repo_path="/repo", model="azure-openai/gpt-4o") - - assert feedback == "root cause: missing edge case handling" - mock_bot_cls.assert_called_once_with(model="azure-openai/gpt-4o", folder_to_mount="/repo") - mock_bot.run.assert_called_once() - assert mock_bot.run.call_args.kwargs["file_name"] == outcome.log_path - - -@pytest.mark.unit -@patch("microbots.auto_memory.analyzer.LogAnalysisBot") -def test_build_feedback_falls_back_when_bot_status_false(mock_bot_cls): - mock_bot = MagicMock() - mock_bot.run.return_value = BotRunResult(status=False, result=None, error="bot crashed") - mock_bot_cls.return_value = mock_bot - - outcome = _make_outcome(reason="tests failed", output="some output") - feedback = build_feedback(task=MagicMock(), outcome=outcome, repo_path="/repo", model="azure-openai/gpt-4o") - - assert "some output" in feedback - assert "tests failed" in feedback - - -@pytest.mark.unit -@patch("microbots.auto_memory.analyzer.LogAnalysisBot") -def test_build_feedback_falls_back_when_result_is_empty(mock_bot_cls): - mock_bot = MagicMock() - mock_bot.run.return_value = BotRunResult(status=True, result="", error=None) - mock_bot_cls.return_value = mock_bot - - outcome = _make_outcome(reason="assertion error", output="agent tried X") - feedback = build_feedback(task=MagicMock(), outcome=outcome, repo_path="/repo", model="azure-openai/gpt-4o") - - assert "agent tried X" in feedback - assert "assertion error" in feedback - - -@pytest.mark.unit -@patch("microbots.auto_memory.analyzer.LogAnalysisBot") -def test_build_feedback_falls_back_when_result_is_none(mock_bot_cls): - mock_bot = MagicMock() - mock_bot.run.return_value = BotRunResult(status=True, result=None, error=None) - mock_bot_cls.return_value = mock_bot - - outcome = _make_outcome() - feedback = build_feedback(task=MagicMock(), outcome=outcome, repo_path="/repo", model="azure-openai/gpt-4o") - - assert "Evaluation failed" in feedback diff --git a/test/auto_memory/test_orchestrator.py b/test/auto_memory/test_orchestrator.py index 10867f2..fba3829 100644 --- a/test/auto_memory/test_orchestrator.py +++ b/test/auto_memory/test_orchestrator.py @@ -25,12 +25,11 @@ MODULE = "microbots.auto_memory.orchestrator" -def _make_outcome(passed: bool, log_path: str, reason: str = "reason") -> EvalOutcome: +def _make_outcome(passed: bool, reason: str = "reason") -> EvalOutcome: return EvalOutcome( passed=passed, output="agent output", result=CallbackResult(passed=passed, reason=reason), - log_path=log_path, ) @@ -52,11 +51,9 @@ def _make_task() -> MagicMock: @pytest.mark.unit @patch("microbots.auto_memory.orchestrator.run_training_loop") -@patch("microbots.auto_memory.orchestrator.build_feedback") -def test_loop_returns_immediately_when_first_round_passes(mock_build_feedback, mock_run_training_loop, tmp_path): - log_path = _touch(str(tmp_path / "round1.log")) +def test_loop_returns_immediately_when_first_round_passes(mock_run_training_loop, tmp_path): task = _make_task() - task.run.return_value = _make_outcome(passed=True, log_path=log_path) + task.run.return_value = _make_outcome(passed=True) result = run_train_eval_loop("/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) @@ -64,28 +61,25 @@ def test_loop_returns_immediately_when_first_round_passes(mock_build_feedback, m assert result.passed is True assert result.rounds_run == 1 assert task.run.call_count == 1 - mock_build_feedback.assert_not_called() + task.build_feedback.assert_not_called() mock_run_training_loop.assert_not_called() @pytest.mark.unit @patch("microbots.auto_memory.orchestrator.run_training_loop") -@patch("microbots.auto_memory.orchestrator.build_feedback") -def test_loop_retrains_and_continues_on_failure_then_passes(mock_build_feedback, mock_run_training_loop, tmp_path): - log1 = _touch(str(tmp_path / "round1.log")) - log2 = _touch(str(tmp_path / "round2.log")) +def test_loop_retrains_and_continues_on_failure_then_passes(mock_run_training_loop, tmp_path): task = _make_task() task.run.side_effect = [ - _make_outcome(passed=False, log_path=log1), - _make_outcome(passed=True, log_path=log2), + _make_outcome(passed=False), + _make_outcome(passed=True), ] - mock_build_feedback.return_value = "feedback text" + task.build_feedback.return_value = "feedback text" result = run_train_eval_loop("/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) assert result.passed is True assert result.rounds_run == 2 - mock_build_feedback.assert_called_once() + task.build_feedback.assert_called_once() mock_run_training_loop.assert_called_once_with( repo_path="/repo", feedback="feedback text", @@ -97,14 +91,13 @@ def test_loop_retrains_and_continues_on_failure_then_passes(mock_build_feedback, @pytest.mark.unit @patch("microbots.auto_memory.orchestrator.run_training_loop") -@patch("microbots.auto_memory.orchestrator.build_feedback") -def test_loop_exhausts_max_rounds_without_passing(mock_build_feedback, mock_run_training_loop, tmp_path): +def test_loop_exhausts_max_rounds_without_passing(mock_run_training_loop, tmp_path): task = _make_task() task.run.side_effect = [ - _make_outcome(passed=False, log_path=_touch(str(tmp_path / f"round{i}.log"))) + _make_outcome(passed=False) for i in range(3) ] - mock_build_feedback.return_value = "feedback text" + task.build_feedback.return_value = "feedback text" result = run_train_eval_loop("/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=3) @@ -112,17 +105,16 @@ def test_loop_exhausts_max_rounds_without_passing(mock_build_feedback, mock_run_ assert result.rounds_run == 3 assert len(result.outcomes) == 3 assert result.final_outcome is result.outcomes[-1] - assert mock_build_feedback.call_count == 3 + assert task.build_feedback.call_count == 3 assert mock_run_training_loop.call_count == 3 @pytest.mark.unit @patch("microbots.auto_memory.orchestrator.run_training_loop") -@patch("microbots.auto_memory.orchestrator.build_feedback") -def test_log_path_persists_after_passing_round(mock_build_feedback, mock_run_training_loop, tmp_path): +def test_log_path_persists_after_passing_round(mock_run_training_loop, tmp_path): log_path = _touch(str(tmp_path / "round1.log")) task = _make_task() - task.run.return_value = _make_outcome(passed=True, log_path=log_path) + task.run.return_value = _make_outcome(passed=True) run_train_eval_loop("/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) @@ -131,16 +123,15 @@ def test_log_path_persists_after_passing_round(mock_build_feedback, mock_run_tra @pytest.mark.unit @patch("microbots.auto_memory.orchestrator.run_training_loop") -@patch("microbots.auto_memory.orchestrator.build_feedback") -def test_log_path_persists_after_failing_round(mock_build_feedback, mock_run_training_loop, tmp_path): +def test_log_path_persists_after_failing_round(mock_run_training_loop, tmp_path): log1 = _touch(str(tmp_path / "round1.log")) log2 = _touch(str(tmp_path / "round2.log")) task = _make_task() task.run.side_effect = [ - _make_outcome(passed=False, log_path=log1), - _make_outcome(passed=True, log_path=log2), + _make_outcome(passed=False), + _make_outcome(passed=True), ] - mock_build_feedback.return_value = "feedback text" + task.build_feedback.return_value = "feedback text" run_train_eval_loop("/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) @@ -150,16 +141,14 @@ def test_log_path_persists_after_failing_round(mock_build_feedback, mock_run_tra @pytest.mark.unit @patch("microbots.auto_memory.orchestrator.run_training_loop") -@patch("microbots.auto_memory.orchestrator.build_feedback") -def test_build_feedback_exception_does_not_crash_loop(mock_build_feedback, mock_run_training_loop, tmp_path): +def test_build_feedback_exception_does_not_crash_loop(mock_run_training_loop, tmp_path): log1 = _touch(str(tmp_path / "round1.log")) - log2 = _touch(str(tmp_path / "round2.log")) task = _make_task() task.run.side_effect = [ - _make_outcome(passed=False, log_path=log1), - _make_outcome(passed=True, log_path=log2), + _make_outcome(passed=False), + _make_outcome(passed=True), ] - mock_build_feedback.side_effect = RuntimeError("analysis bot crashed") + task.build_feedback.side_effect = RuntimeError("analysis bot crashed") result = run_train_eval_loop("/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) @@ -171,16 +160,13 @@ def test_build_feedback_exception_does_not_crash_loop(mock_build_feedback, mock_ @pytest.mark.unit @patch("microbots.auto_memory.orchestrator.run_training_loop") -@patch("microbots.auto_memory.orchestrator.build_feedback") -def test_loop_forwards_training_iterations_to_run_training_loop(mock_build_feedback, mock_run_training_loop, tmp_path): - log1 = _touch(str(tmp_path / "round1.log")) - log2 = _touch(str(tmp_path / "round2.log")) +def test_loop_forwards_training_iterations_to_run_training_loop(mock_run_training_loop, tmp_path): task = _make_task() task.run.side_effect = [ - _make_outcome(passed=False, log_path=log1), - _make_outcome(passed=True, log_path=log2), + _make_outcome(passed=False), + _make_outcome(passed=True), ] - mock_build_feedback.return_value = "feedback text" + task.build_feedback.return_value = "feedback text" run_train_eval_loop( "/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5, training_iterations=4 @@ -197,16 +183,14 @@ def test_loop_forwards_training_iterations_to_run_training_loop(mock_build_feedb @pytest.mark.unit @patch("microbots.auto_memory.orchestrator.run_training_loop") -@patch("microbots.auto_memory.orchestrator.build_feedback") -def test_run_training_exception_does_not_crash_loop(mock_build_feedback, mock_run_training_loop, tmp_path): +def test_run_training_exception_does_not_crash_loop(mock_run_training_loop, tmp_path): log1 = _touch(str(tmp_path / "round1.log")) - log2 = _touch(str(tmp_path / "round2.log")) task = _make_task() task.run.side_effect = [ - _make_outcome(passed=False, log_path=log1), - _make_outcome(passed=True, log_path=log2), + _make_outcome(passed=False), + _make_outcome(passed=True), ] - mock_build_feedback.return_value = "feedback text" + task.build_feedback.return_value = "feedback text" mock_run_training_loop.side_effect = RuntimeError("training crashed") result = run_train_eval_loop("/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) @@ -292,7 +276,7 @@ def test_write_eval_result_writes_task_build_result_as_json(tmp_path): task = MagicMock() task.task_id = "django__django-1" task.build_result.return_value = {"passed": True, "reason": "resolved"} - outcome = _make_outcome(passed=True, log_path="/dev/null") + outcome = _make_outcome(passed=True) write_eval_result(tmp_path, 2, task, outcome) @@ -306,7 +290,7 @@ def test_write_eval_result_creates_missing_parent_dirs(tmp_path): task = MagicMock() task.task_id = "some-task" task.build_result.return_value = {"passed": False, "reason": "nope"} - outcome = _make_outcome(passed=False, log_path="/dev/null") + outcome = _make_outcome(passed=False) write_eval_result(tmp_path, 1, task, outcome) @@ -314,16 +298,13 @@ def test_write_eval_result_creates_missing_parent_dirs(tmp_path): @pytest.mark.unit -@patch(f"{MODULE}.build_feedback") -def test_loop_writes_eval_result_for_every_round(mock_build_feedback, tmp_path): - log1 = _touch(str(tmp_path / "round1.log")) - log2 = _touch(str(tmp_path / "round2.log")) +def test_loop_writes_eval_result_for_every_round(tmp_path): task = _make_task() task.run.side_effect = [ - _make_outcome(passed=False, log_path=log1), - _make_outcome(passed=True, log_path=log2), + _make_outcome(passed=False), + _make_outcome(passed=True), ] - mock_build_feedback.return_value = "feedback text" + task.build_feedback.return_value = "feedback text" with patch(f"{MODULE}.run_training_loop"): run_train_eval_loop("/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) @@ -428,12 +409,8 @@ def fake_train(repo_path, feedback, memory_dir, model, iterations=1): @pytest.mark.unit -@patch(f"{MODULE}.build_feedback") @patch(f"{MODULE}.run_training_loop") -def test_loop_carries_memory_forward_between_rounds(mock_run_training_loop, mock_build_feedback, tmp_path): - log1 = _touch(str(tmp_path / "round1.log")) - log2 = _touch(str(tmp_path / "round2.log")) - mock_build_feedback.return_value = "feedback text" +def test_loop_carries_memory_forward_between_rounds(mock_run_training_loop, tmp_path): seen_memory_dirs = [] def fake_run(repo_path, memory_dir, model, log_path): @@ -443,11 +420,11 @@ def fake_run(repo_path, memory_dir, model, log_path): assert (Path(memory_dir) / "notes.md").read_text() == "round 1 progress" seen_memory_dirs.append(memory_dir) Path(memory_dir, "notes.md").write_text(f"round {round_num} progress") - log_path = log1 if round_num == 1 else log2 - return _make_outcome(passed=round_num == 2, log_path=log_path) + return _make_outcome(passed=round_num == 2) task = _make_task() task.run.side_effect = fake_run + task.build_feedback.return_value = "feedback text" run_train_eval_loop("/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) diff --git a/test/auto_memory/test_task.py b/test/auto_memory/test_task.py index ed11bb2..b66deb1 100644 --- a/test/auto_memory/test_task.py +++ b/test/auto_memory/test_task.py @@ -11,16 +11,18 @@ class _RunOnlyTask(EvalTask): - """A task that overrides only run(), never touching the optional hooks.""" + """A task that overrides only run()/build_feedback(), never touching the optional hooks.""" def run(self, repo_path, memory_dir, model, log_path): return EvalOutcome( passed=True, output="custom output", result=None, - log_path="/dev/null", ) + def build_feedback(self, outcome, repo_path, model, log_path): + return "feedback text" + @pytest.mark.unit def test_run_is_abstract(): @@ -28,6 +30,16 @@ def test_run_is_abstract(): EvalTask() +@pytest.mark.unit +def test_build_feedback_is_abstract(): + class _MissingBuildFeedback(EvalTask): + def run(self, repo_path, memory_dir, model, log_path): + raise NotImplementedError + + with pytest.raises(TypeError): + _MissingBuildFeedback() + + @pytest.mark.unit def test_subclass_overriding_only_run_is_instantiable(): task = _RunOnlyTask() @@ -73,7 +85,6 @@ def test_default_build_result_returns_passed_and_reason(): passed=False, output="agent output", result=CallbackResult(passed=False, reason="check failed"), - log_path="/dev/null", ) assert _RunOnlyTask().build_result(outcome) == {"passed": False, "reason": "check failed"} diff --git a/test/auto_memory/test_task_registry.py b/test/auto_memory/test_task_registry.py index 4caa88f..ea34cf8 100644 --- a/test/auto_memory/test_task_registry.py +++ b/test/auto_memory/test_task_registry.py @@ -31,6 +31,9 @@ def check(self, output): def teardown(self, repo_path): pass + def build_feedback(self, outcome, repo_path, model, log_path): + return "feedback" + def run(self, repo_path, memory_dir, model): return super().run(repo_path, memory_dir, model) diff --git a/test/auto_memory/test_workdir.py b/test/auto_memory/test_workdir.py index 149b223..1eb2fae 100644 --- a/test/auto_memory/test_workdir.py +++ b/test/auto_memory/test_workdir.py @@ -22,7 +22,6 @@ round_dir, round_log_path, round_memory_dir, - round_patch_path, run_log_path, save_round_memory, ) @@ -175,18 +174,6 @@ def test_round_log_path_with_instance_id(tmp_path): ) / "round.log" -@pytest.mark.unit -def test_round_patch_path_returns_repo_patch(tmp_path): - assert round_patch_path(tmp_path, 2) == round_dir(tmp_path, 2) / "repo.patch" - - -@pytest.mark.unit -def test_round_patch_path_with_instance_id(tmp_path): - assert round_patch_path(tmp_path, 2, instance_id="task-1") == round_dir( - tmp_path, 2, instance_id="task-1" - ) / "repo.patch" - - @pytest.mark.unit def test_eval_dir_creates_directory_when_requested(tmp_path): path = eval_dir(tmp_path, 1, "task-1", create=True) From 021c7efe80cbf99c55431916d8ebf7d4d286224f Mon Sep 17 00:00:00 2001 From: Kavya Sree Kaitepalli Date: Thu, 3 Sep 2026 07:31:11 +0000 Subject: [PATCH 11/13] Refactor --- .../auto_memory/eval/swebenchverified.py | 198 ++++++++++-------- src/microbots/auto_memory/evalTask.py | 40 +++- src/microbots/auto_memory/orchestrator.py | 49 ++--- src/microbots/auto_memory/workdir.py | 60 +++++- .../auto_memory/eval/test_swebenchverified.py | 186 +++++++++------- test/auto_memory/test_orchestrator.py | 55 ++--- test/auto_memory/test_task.py | 27 +++ test/auto_memory/test_task_registry.py | 4 + test/auto_memory/test_workdir.py | 42 +++- 9 files changed, 440 insertions(+), 221 deletions(-) diff --git a/src/microbots/auto_memory/eval/swebenchverified.py b/src/microbots/auto_memory/eval/swebenchverified.py index 2fd64b1..889693b 100644 --- a/src/microbots/auto_memory/eval/swebenchverified.py +++ b/src/microbots/auto_memory/eval/swebenchverified.py @@ -54,32 +54,10 @@ def _load_dataset_rows(dataset_name: str): return load_dataset(dataset_name, split="test") -@dataclass -class SweBenchInstance: - """A single SWE-bench-verified dataset row. - - Attributes - ---------- - instance_id : str - Unique identifier for the instance, e.g. ``"django__django-11099"``. - repo : str - The GitHub repo this instance belongs to, e.g. ``"django/django"``. - base_commit : str - Commit hash representing the repo state before the issue's fix. - problem_statement : str - The GitHub issue title and body describing the bug to fix. - """ - - instance_id: str - repo: str - base_commit: str - problem_statement: str - - def load_instances_of_repo( dataset_name: str = SWE_BENCH_VERIFIED, repo: str | None = None, -) -> list[SweBenchInstance]: +) -> list["SweBenchInstance"]: """Load all dataset instances, optionally filtered to a single repo. Parameters @@ -110,7 +88,7 @@ def load_instances_of_repo( ] return instances -def load_instance_using_id(instance_id: str, dataset_name: str = SWE_BENCH_VERIFIED) -> SweBenchInstance: +def load_instance_using_id(instance_id: str, dataset_name: str = SWE_BENCH_VERIFIED) -> "SweBenchInstance": """Load a single dataset instance by its instance ID. Parameters @@ -143,6 +121,27 @@ def load_instance_using_id(instance_id: str, dataset_name: str = SWE_BENCH_VERIF ) raise ValueError(f"instance_id not found: {instance_id}") +@dataclass +class SweBenchInstance: + """A single SWE-bench-verified dataset row. + + Attributes + ---------- + instance_id : str + Unique identifier for the instance, e.g. ``"django__django-11099"``. + repo : str + The GitHub repo this instance belongs to, e.g. ``"django/django"``. + base_commit : str + Commit hash representing the repo state before the issue's fix. + problem_statement : str + The GitHub issue title and body describing the bug to fix. + """ + + instance_id: str + repo: str + base_commit: str + problem_statement: str + @register_task("swebenchverified") class SweBenchVerifiedTask(EvalTask): """Eval task that verifies a fix against one SWE-bench-verified instance. @@ -225,17 +224,48 @@ def build_result(self, outcome: EvalOutcome) -> dict: } def setup(self, repo_path: str) -> None: - """Clone the instance's repo and check out its base commit. + """Clone the instance's repo, or reset it, to its base commit. + + Clones fresh on first use. If ``repo_path`` already exists + (e.g. left behind by a previous round) *and* its ``origin`` + remote matches this instance's repo, it's reset instead of + re-cloned: ``git reset --hard `` followed by + ``git clean -fd`` discards whatever the agent changed, without + the cost of a full re-clone and without deleting the directory + ``build_feedback`` may still need to inspect afterward. If + ``repo_path`` exists but isn't a checkout of this repo (e.g. a + stale directory left over from a different run/task), it's + removed and cloned fresh instead, to avoid silently operating + on the wrong codebase. Parameters ---------- repo_path : str - Absolute path to clone the repo into. + Absolute path to clone (or reset) the repo into. """ - subprocess.run( - ["git", "clone", f"https://github.com/{self.instance.repo}.git", repo_path], - check=True, - ) + expected_url = f"https://github.com/{self.instance.repo}.git" + + if Path(repo_path).exists(): + origin = subprocess.run( + ["git", "remote", "get-url", "origin"], + cwd=repo_path, capture_output=True, text=True, + ) + if origin.returncode == 0 and origin.stdout.strip() == expected_url: + subprocess.run( + ["git", "reset", "--hard", self.instance.base_commit], + cwd=repo_path, check=True, + ) + subprocess.run(["git", "clean", "-fd"], cwd=repo_path, check=True) + return + + logger.warning( + "SweBenchVerifiedTask.setup: %s exists but isn't a checkout of %s " + "(origin=%r); removing and re-cloning", + repo_path, expected_url, origin.stdout.strip(), + ) + shutil.rmtree(repo_path) + + subprocess.run(["git", "clone", expected_url, repo_path], check=True) subprocess.run( ["git", "checkout", self.instance.base_commit], cwd=repo_path, check=True ) @@ -266,7 +296,10 @@ def check(self, repo_path: str, agent_output: str, log_path: str) -> CallbackRes verification is based on the repo's git diff, not the agent's textual output. log_path : str - Path to a log file to append the harness's output to. + Path to a log file to append the harness's output to, + including the per-instance ``run_instance.log`` and + ``test_output.txt`` artifacts if the harness produced them + (read before the harness's ``report_dir`` is cleaned up). Returns ------- @@ -280,6 +313,8 @@ def check(self, repo_path: str, agent_output: str, log_path: str) -> CallbackRes run_id = f"microbots-{uuid.uuid4().hex[:8]}" model_name_or_path = EVAL_AGENT_MODEL_NAME pred_path = Path(tempfile.mktemp(suffix=".json")) + #will need to update when upgraded to ~5.0.2 , removed this flag in the new version + #https://github.com/SWE-bench/SWE-bench/commit/e2c13307b6cf7764a50958b9c8bfbfb3f72cb70a report_dir = Path(tempfile.mkdtemp()) pred_path.write_text(json.dumps([{ "instance_id": self.instance.instance_id, @@ -300,30 +335,29 @@ def check(self, repo_path: str, agent_output: str, log_path: str) -> CallbackRes capture_output=True, text=True, cwd=report_dir, ) + #need to update this instance_log_dir path when swebench is upgraded + instance_log_dir = ( + report_dir / "logs" / "run_evaluation" / run_id + / model_name_or_path / self.instance.instance_id + ) with open(log_path, "a") as f: f.write(proc.stdout + proc.stderr) + for log_filename in ("run_instance.log", "test_output.txt"): + log_file = instance_log_dir / log_filename + if log_file.exists(): + f.write(f"\n--- {log_filename} ---\n{log_file.read_text()}\n") - report_file = report_dir / f"{model_name_or_path}.{run_id}.json" + report_file = instance_log_dir / "report.json" passed = False if report_file.exists(): report = json.loads(report_file.read_text()) - passed = self.instance.instance_id in report.get("resolved_ids", []) + passed = report.get(self.instance.instance_id, {}).get("resolved", False) finally: pred_path.unlink(missing_ok=True) shutil.rmtree(report_dir, ignore_errors=True) return CallbackResult(passed=passed, reason="resolved" if passed else "not resolved") - def teardown(self, repo_path: str) -> None: - """Remove the cloned repo working directory. - - Parameters - ---------- - repo_path : str - Absolute path to the repo cloned by ``setup``. - """ - subprocess.run(["rm", "-rf", repo_path], check=False) - def build_feedback(self, outcome: EvalOutcome, repo_path: str, model: str, log_path: str) -> str: """Analyze a failed round's log via ``LogAnalysisBot`` for training feedback. @@ -370,7 +404,7 @@ def build_feedback(self, outcome: EvalOutcome, repo_path: str, model: str, log_p ) def run(self, repo_path: str, memory_dir: str, model: str, log_path: str) -> EvalOutcome: - """Run one eval iteration: setup -> build_prompt -> WritingBot -> check -> teardown. + """Run one eval iteration: setup -> build_prompt -> WritingBot -> check. Parameters ---------- @@ -396,48 +430,42 @@ def run(self, repo_path: str, memory_dir: str, model: str, log_path: str) -> Eva Path(log_path).write_text("") try: - try: - prompt = self.build_prompt() - bot = WritingBot( - model=model, - folder_to_mount=repo_path, - additional_tools=[MemoryTool(memory_dir=memory_dir)], - ) - bot_result = bot.run(prompt) + prompt = self.build_prompt() + bot = WritingBot( + model=model, + folder_to_mount=repo_path, + additional_tools=[MemoryTool(memory_dir=memory_dir)], + ) + bot_result = bot.run(prompt) + with open(log_path, "a") as f: + f.write(f"Agent output:\n{bot_result.result}\n") + + if not bot_result.status: + reason = f"Bot run failed: {bot_result.error}" with open(log_path, "a") as f: - f.write(f"Agent output:\n{bot_result.result}\n") - - if not bot_result.status: - reason = f"Bot run failed: {bot_result.error}" - with open(log_path, "a") as f: - f.write(f"\n{reason}\n") - result = CallbackResult(passed=False, reason=reason) - else: - result = self.check(repo_path, bot_result.result or "", log_path) - - return EvalOutcome( - passed=result.passed, - output=bot_result.result, - result=result, - ) - except Exception as exc: - logger.exception( - "SweBenchVerifiedTask.run: iteration raised %s", type(exc).__name__ - ) - with open(log_path, "a") as f: - f.write(f"\nException during eval iteration: {type(exc).__name__}: {exc}\n") - return EvalOutcome( - passed=False, - output=None, - result=CallbackResult( - passed=False, reason=f"{type(exc).__name__}: {exc}" - ), - ) - finally: - try: - self.teardown(repo_path) - except Exception: - logger.exception("SweBenchVerifiedTask.run: teardown() raised exception; ignoring") + f.write(f"\n{reason}\n") + result = CallbackResult(passed=False, reason=reason) + else: + result = self.check(repo_path, bot_result.result or "", log_path) + + return EvalOutcome( + passed=result.passed, + output=bot_result.result, + result=result, + ) + except Exception as exc: + logger.exception( + "SweBenchVerifiedTask.run: iteration raised %s", type(exc).__name__ + ) + with open(log_path, "a") as f: + f.write(f"\nException during eval iteration: {type(exc).__name__}: {exc}\n") + return EvalOutcome( + passed=False, + output=None, + result=CallbackResult( + passed=False, reason=f"{type(exc).__name__}: {exc}" + ), + ) diff --git a/src/microbots/auto_memory/evalTask.py b/src/microbots/auto_memory/evalTask.py index d58476a..aacee20 100644 --- a/src/microbots/auto_memory/evalTask.py +++ b/src/microbots/auto_memory/evalTask.py @@ -7,6 +7,7 @@ from abc import ABC, abstractmethod from dataclasses import dataclass +from typing import Any @dataclass class CallbackResult: @@ -45,11 +46,11 @@ class EvalOutcome: class EvalTask(ABC): """Base class for a single evaluation task in the train <-> eval loop. - Subclasses must implement ``run``. ``setup``, ``build_prompt``, - ``check``, and ``teardown`` are optional hooks subclasses may use - to structure their own ``run`` implementation (see - ``SweBenchVerifiedTask`` for an example), but nothing in this base - class calls them automatically. + Subclasses must implement ``run`` and ``from_config``. ``setup``, + ``build_prompt``, ``check``, and ``teardown`` are optional hooks + subclasses may use to structure their own ``run`` implementation + (see ``SweBenchVerifiedTask`` for an example), but nothing in this + base class calls them automatically. """ @property @@ -148,6 +149,35 @@ def teardown(self, repo_path: str) -> None: """ pass + @classmethod + @abstractmethod + def from_config(cls, task_args: dict[str, Any]) -> list["EvalTask"]: + """Required. Build task instance(s) from a config's ``task_args`` dict. + + Called by the CLI at runtime (driven by ``--task``) to + construct the actual task object(s) to run, using whatever + config values the task needs (e.g. a dataset instance ID, a + repo filter). Object creation must go through this method + rather than being constructed elsewhere, so behavior stays + driven by the CLI/config at runtime. + + Parameters + ---------- + task_args : dict[str, Any] + Task-specific config values (the config file's + ``task_args`` section). + + Returns + ------- + list[EvalTask] + One task instance per unit of work this config describes + (often just one, but e.g. ``SweBenchVerifiedTask`` returns + one per matching dataset instance). + """ + raise NotImplementedError( + f"{cls.__name__} must implement from_config() to be usable via --task" + ) + @abstractmethod def build_feedback(self, outcome: EvalOutcome, repo_path: str, model: str, log_path: str) -> str: """Required. Analyze a failed eval outcome and produce training feedback. diff --git a/src/microbots/auto_memory/orchestrator.py b/src/microbots/auto_memory/orchestrator.py index 69b36dc..7dc5662 100644 --- a/src/microbots/auto_memory/orchestrator.py +++ b/src/microbots/auto_memory/orchestrator.py @@ -15,11 +15,13 @@ from microbots.auto_memory.training.runner import run_training from microbots.auto_memory.workdir import ( eval_log_path, + eval_repo_dir, eval_result_path, load_config, load_round_memory, repo_dir, save_round_memory, + snapshot_seed_memory, ) logger = getLogger(__name__) @@ -60,24 +62,6 @@ def clone_repo(url: str, repo_path: Path) -> None: return subprocess.run(["git", "clone", url, str(repo_path)], check=True) -def reset_repo(repo_path: Path, base_commit: str) -> None: - """Reset ``repo_path`` to ``base_commit``, discarding all local changes. - - Runs ``git reset --hard `` followed by ``git clean -fd``, - so every round/instance starts from the same pristine state instead - of carrying forward whatever a previous round or eval attempt left - behind. - - Parameters - ---------- - repo_path : Path - Path to the repo to reset. - base_commit : str - Commit-ish to reset to. - """ - subprocess.run(["git", "reset", "--hard", base_commit], cwd=repo_path, check=True) - subprocess.run(["git", "clean", "-fd"], cwd=repo_path, check=True) - def write_eval_result(workdir: Path, round_num: int, task: EvalTask, outcome: EvalOutcome) -> None: """Write a round's eval result to ``result.json``. @@ -141,7 +125,8 @@ def run_training_loop( ) def run_train_eval_loop( - repo_path: str, + training_repo_path: str, + eval_repo_path: str, workdir: Path, model: str, task: EvalTask, @@ -168,8 +153,15 @@ def run_train_eval_loop( Parameters ---------- - repo_path : str - Absolute path to the repo to evaluate and train against. + training_repo_path : str + Absolute path to the persistent repo checkout used only for + retraining (``run_training_loop``). Kept separate from + ``eval_repo_path`` since the task manages the latter's + lifecycle itself (clone/teardown each round). + eval_repo_path : str + Absolute path to the repo the task clones/manages itself (via + its own ``setup``) and runs/checks the agent against each + round. workdir : Path This run's workdir, used to carry memory forward between rounds (see ``microbots.auto_memory.workdir``). @@ -197,7 +189,7 @@ def run_train_eval_loop( ) memory_dir = str(load_round_memory(workdir, round_idx, instance_id=task.task_id)) log_path = str(eval_log_path(workdir, round_idx, task.task_id)) - outcome = task.run(repo_path, memory_dir, model, log_path) + outcome = task.run(eval_repo_path, memory_dir, model, log_path) outcomes.append(outcome) try: @@ -218,9 +210,9 @@ def run_train_eval_loop( outcome.result.reason, ) try: - feedback = task.build_feedback(outcome, repo_path, model, log_path) + feedback = task.build_feedback(outcome, eval_repo_path, model, log_path) run_training_loop( - repo_path=repo_path, + repo_path=training_repo_path, feedback=feedback, memory_dir=memory_dir, model=model, @@ -290,14 +282,16 @@ def run( if repo_url: clone_repo(repo_url, repo_dir(workdir)) - repo_path = str(repo_dir(workdir)) + snapshot_seed_memory(workdir) + + training_repo_path = str(repo_dir(workdir)) if task is None: # Train-only mode has no rounds of its own; round 1 is just a # scratch dir seeded from (and saved back to) top-level memory. memory_dir = str(load_round_memory(workdir, 1)) run_training_loop( - repo_path=repo_path, + repo_path=training_repo_path, feedback="", memory_dir=memory_dir, model=model, @@ -307,7 +301,8 @@ def run( return None return run_train_eval_loop( - repo_path=repo_path, + training_repo_path=training_repo_path, + eval_repo_path=str(eval_repo_dir(workdir)), workdir=workdir, model=model, task=task, diff --git a/src/microbots/auto_memory/workdir.py b/src/microbots/auto_memory/workdir.py index bcaf616..65cb3ff 100644 --- a/src/microbots/auto_memory/workdir.py +++ b/src/microbots/auto_memory/workdir.py @@ -13,8 +13,9 @@ WORKDIR_NAME = "workdir" CONFIG_FILENAME = "config.yaml" REPO_DIRNAME = "repo" -RUN_LOG_FILENAME = "run.log" +EVAL_REPO_DIRNAME = "eval_repo" MEMORY_DIRNAME = "memory" +MEMORY_SEED_DIRNAME = "memory_seed" ROUNDS_DIRNAME = "rounds" ROUND_LOG_FILENAME = "round.log" ROUND_PATCH_FILENAME = "repo.patch" @@ -96,6 +97,13 @@ def load_config(workdir: Path) -> dict: def repo_dir(workdir: Path) -> Path: """Return the path to the single cloned repo shared across rounds. + Used only for training (both train-only mode and the eval loop's + retrain step): a persistent checkout that stays in place across + rounds. Eval tasks that manage their own repo checkout (e.g. + ``SweBenchVerifiedTask``, which clones a different repo/commit per + dataset instance) use ``eval_repo_dir`` instead, so the two never + collide. + Parameters ---------- workdir : Path @@ -109,8 +117,14 @@ def repo_dir(workdir: Path) -> Path: return workdir / REPO_DIRNAME -def run_log_path(workdir: Path) -> Path: - """Return the path to the top-level orchestrator log. +def eval_repo_dir(workdir: Path) -> Path: + """Return the path to the repo an eval task clones/manages itself. + + Kept separate from ``repo_dir`` (the training repo) because a + task's ``setup`` may clone or reset this directory every round + (e.g. ``SweBenchVerifiedTask`` checks out a different repo/commit + per dataset instance), which would otherwise conflict with the + persistent training checkout at ``repo_dir``. Parameters ---------- @@ -120,9 +134,9 @@ def run_log_path(workdir: Path) -> Path: Returns ------- Path - ``workdir/run.log``. + ``workdir/eval_repo``. """ - return workdir / RUN_LOG_FILENAME + return workdir / EVAL_REPO_DIRNAME def memory_dir(workdir: Path) -> Path: @@ -141,6 +155,42 @@ def memory_dir(workdir: Path) -> Path: return workdir / MEMORY_DIRNAME +def snapshot_seed_memory(workdir: Path) -> Path: + """Snapshot the current top-level memory dir as the run's restorable baseline. + + ``memory_dir`` is shared and mutated in place across every + training/eval round and every eval task instance (so later + instances benefit from what earlier ones learned), which means the + original, pre-run memory is otherwise overwritten and lost with no + way to get back to it. Call this once, before anything trains, + to preserve that original state at ``workdir/memory_seed``. A + no-op if a snapshot already exists, so later calls (e.g. once per + eval task instance in the same run) never clobber the very first + snapshot with already-mutated memory. + + Parameters + ---------- + workdir : Path + The run's workdir. + + Returns + ------- + Path + ``workdir/memory_seed``, containing a copy of whatever + ``memory_dir`` held the first time this was called (or empty, + if there was no pre-existing memory). + """ + dst = workdir / MEMORY_SEED_DIRNAME + if dst.exists(): + return dst + src = memory_dir(workdir) + if src.is_dir(): + shutil.copytree(src, dst) + else: + dst.mkdir(parents=True, exist_ok=True) + return dst + + def round_dir( workdir: Path, round_num: int, *, instance_id: str | None = None, create: bool = False ) -> Path: diff --git a/test/auto_memory/eval/test_swebenchverified.py b/test/auto_memory/eval/test_swebenchverified.py index 56e1906..79477ec 100644 --- a/test/auto_memory/eval/test_swebenchverified.py +++ b/test/auto_memory/eval/test_swebenchverified.py @@ -114,7 +114,7 @@ def test_dataset_rows_are_cached_across_repeated_calls(mock_load_dataset): # --------------------------------------------------------------------------- -# SweBenchVerifiedTask.setup / build_prompt / teardown +# SweBenchVerifiedTask.setup / build_prompt # --------------------------------------------------------------------------- def _instance(): @@ -128,14 +128,75 @@ def _instance(): @pytest.mark.unit @patch(f"{MODULE}.subprocess.run") -def test_setup_clones_and_checks_out_base_commit(mock_run): +def test_setup_clones_and_checks_out_base_commit_when_repo_missing(mock_run, tmp_path): + repo_path = tmp_path / "repo" task = SweBenchVerifiedTask(_instance()) - task.setup("/repo") + task.setup(str(repo_path)) clone_call, checkout_call = mock_run.call_args_list - assert clone_call.args[0] == ["git", "clone", "https://github.com/django/django.git", "/repo"] + assert clone_call.args[0] == [ + "git", "clone", "https://github.com/django/django.git", str(repo_path) + ] + assert checkout_call.args[0] == ["git", "checkout", "abc123"] + assert checkout_call.kwargs["cwd"] == str(repo_path) + + +@pytest.mark.unit +@patch(f"{MODULE}.subprocess.run") +def test_setup_resets_instead_of_recloning_when_origin_matches(mock_run, tmp_path): + repo_path = tmp_path / "repo" + repo_path.mkdir() + mock_run.return_value = MagicMock( + returncode=0, stdout="https://github.com/django/django.git\n" + ) + task = SweBenchVerifiedTask(_instance()) + task.setup(str(repo_path)) + + origin_call, reset_call, clean_call = mock_run.call_args_list + assert origin_call.args[0] == ["git", "remote", "get-url", "origin"] + assert origin_call.kwargs["cwd"] == str(repo_path) + assert reset_call.args[0] == ["git", "reset", "--hard", "abc123"] + assert reset_call.kwargs["cwd"] == str(repo_path) + assert clean_call.args[0] == ["git", "clean", "-fd"] + assert clean_call.kwargs["cwd"] == str(repo_path) + + +@pytest.mark.unit +@patch(f"{MODULE}.subprocess.run") +def test_setup_removes_and_reclones_when_origin_mismatched(mock_run, tmp_path): + repo_path = tmp_path / "repo" + repo_path.mkdir() + (repo_path / "stale_file.txt").write_text("leftover from a different repo") + mock_run.return_value = MagicMock( + returncode=0, stdout="https://github.com/other/repo.git\n" + ) + task = SweBenchVerifiedTask(_instance()) + task.setup(str(repo_path)) + + assert not (repo_path / "stale_file.txt").exists() + origin_call, clone_call, checkout_call = mock_run.call_args_list + assert origin_call.args[0] == ["git", "remote", "get-url", "origin"] + assert clone_call.args[0] == [ + "git", "clone", "https://github.com/django/django.git", str(repo_path) + ] assert checkout_call.args[0] == ["git", "checkout", "abc123"] - assert checkout_call.kwargs["cwd"] == "/repo" + + +@pytest.mark.unit +@patch(f"{MODULE}.subprocess.run") +def test_setup_removes_and_reclones_when_repo_path_not_a_git_repo(mock_run, tmp_path): + repo_path = tmp_path / "repo" + repo_path.mkdir() + mock_run.return_value = MagicMock(returncode=128, stdout="") + task = SweBenchVerifiedTask(_instance()) + task.setup(str(repo_path)) + + assert not repo_path.exists() + origin_call, clone_call, checkout_call = mock_run.call_args_list + assert origin_call.args[0] == ["git", "remote", "get-url", "origin"] + assert clone_call.args[0] == [ + "git", "clone", "https://github.com/django/django.git", str(repo_path) + ] @pytest.mark.unit @@ -168,15 +229,6 @@ def test_build_result_includes_dataset_fields(): } -@pytest.mark.unit -@patch(f"{MODULE}.subprocess.run") -def test_teardown_removes_repo_path(mock_run): - task = SweBenchVerifiedTask(_instance()) - task.teardown("/repo") - - mock_run.assert_called_once_with(["rm", "-rf", "/repo"], check=False) - - # --------------------------------------------------------------------------- # SweBenchVerifiedTask.build_feedback # --------------------------------------------------------------------------- @@ -270,10 +322,15 @@ def _fake_run(cmd, **kwargs): if raise_on_harness: raise RuntimeError("harness crashed") run_id = cmd[cmd.index("--run_id") + 1] - report_dir = kwargs["cwd"] + report_dir = Path(kwargs["cwd"]) instance_id = cmd[cmd.index("--instance_ids") + 1] - report = {"resolved_ids": [instance_id] if resolved else []} - (Path(report_dir) / f"microbots-eval-agent.{run_id}.json").write_text(json.dumps(report)) + instance_log_dir = ( + report_dir / "logs" / "run_evaluation" / run_id + / "microbots-eval-agent" / instance_id + ) + instance_log_dir.mkdir(parents=True) + report = {instance_id: {"resolved": resolved}} + (instance_log_dir / "report.json").write_text(json.dumps(report)) return MagicMock(stdout="harness ran\n", stderr="", returncode=0) return MagicMock(stdout="", stderr="", returncode=0) @@ -282,7 +339,7 @@ def _fake_run(cmd, **kwargs): @pytest.mark.unit @patch(f"{MODULE}.subprocess.run") -def test_check_passed_true_when_instance_in_resolved_ids(mock_run, tmp_path): +def test_check_passed_true_when_report_marks_resolved(mock_run, tmp_path): mock_run.side_effect = _make_fake_subprocess_run(resolved=True) log_path = tmp_path / "check.log" log_path.write_text("") @@ -296,7 +353,7 @@ def test_check_passed_true_when_instance_in_resolved_ids(mock_run, tmp_path): @pytest.mark.unit @patch(f"{MODULE}.subprocess.run") -def test_check_passed_false_when_instance_not_in_resolved_ids(mock_run, tmp_path): +def test_check_passed_false_when_report_marks_not_resolved(mock_run, tmp_path): mock_run.side_effect = _make_fake_subprocess_run(resolved=False) log_path = tmp_path / "check.log" log_path.write_text("") @@ -342,6 +399,44 @@ def test_check_appends_to_log_file_without_truncating_existing_content(mock_run, assert "harness ran" in content +@pytest.mark.unit +@patch(f"{MODULE}.subprocess.run") +def test_check_appends_instance_log_and_test_output_when_present(mock_run, tmp_path): + def _fake_run(cmd, **kwargs): + if cmd[:2] == ["git", "diff"]: + return MagicMock(stdout="diff", stderr="", returncode=0) + if "swebench.harness.run_evaluation" in cmd: + run_id = cmd[cmd.index("--run_id") + 1] + report_dir = Path(kwargs["cwd"]) + instance_id = cmd[cmd.index("--instance_ids") + 1] + + instance_log_dir = ( + report_dir / "logs" / "run_evaluation" / run_id + / "microbots-eval-agent" / instance_id + ) + instance_log_dir.mkdir(parents=True) + report = {instance_id: {"resolved": True}} + (instance_log_dir / "report.json").write_text(json.dumps(report)) + (instance_log_dir / "run_instance.log").write_text("build+test steps") + (instance_log_dir / "test_output.txt").write_text("FAILED test_foo") + + return MagicMock(stdout="harness ran\n", stderr="", returncode=0) + return MagicMock(stdout="", stderr="", returncode=0) + + mock_run.side_effect = _fake_run + log_path = tmp_path / "check.log" + log_path.write_text("") + + task = SweBenchVerifiedTask(_instance()) + task.check("/repo", "agent output", str(log_path)) + + content = log_path.read_text() + assert "run_instance.log" in content + assert "build+test steps" in content + assert "test_output.txt" in content + assert "FAILED test_foo" in content + + @pytest.mark.unit @patch(f"{MODULE}.shutil.rmtree") @patch(f"{MODULE}.subprocess.run") @@ -378,7 +473,7 @@ def test_check_cleans_up_even_when_harness_raises(mock_run, mock_rmtree, tmp_pat @pytest.mark.unit @patch(f"{MODULE}.MemoryTool") @patch(f"{MODULE}.WritingBot") -def test_run_calls_setup_build_prompt_check_teardown_in_order(mock_bot_cls, mock_memory_tool, tmp_path): +def test_run_calls_setup_build_prompt_check_in_order(mock_bot_cls, mock_memory_tool, tmp_path): from microbots.MicroBot import BotRunResult mock_bot = MagicMock() @@ -393,13 +488,11 @@ def test_run_calls_setup_build_prompt_check_teardown_in_order(mock_bot_cls, mock calls.append(("check", repo_path, agent_output, log_path)) or CallbackResult(passed=True, reason="ok") ) - task.teardown = lambda repo_path: calls.append(("teardown", repo_path)) outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o", str(tmp_path / "eval.log")) assert calls[0] == ("setup", "/repo") assert calls[1] == ("check", "/repo", "agent did stuff", str(tmp_path / "eval.log")) - assert calls[2] == ("teardown", "/repo") assert outcome.passed is True assert outcome.output == "agent did stuff" @@ -417,7 +510,6 @@ def test_run_creates_log_file_before_check_is_called(mock_bot_cls, mock_memory_t task = SweBenchVerifiedTask(_instance()) task.setup = lambda repo_path: None task.build_prompt = lambda: "do the task" - task.teardown = lambda repo_path: None seen_log_exists = {} def _check(repo_path, agent_output, log_path): @@ -446,7 +538,6 @@ def test_run_skips_check_when_bot_status_is_false(mock_bot_cls, mock_memory_tool task.setup = lambda repo_path: None task.build_prompt = lambda: "do the task" task.check = lambda *a: check_calls.append(a) - task.teardown = lambda repo_path: None outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o", str(tmp_path / "eval.log")) @@ -461,7 +552,6 @@ def test_run_skips_check_when_bot_status_is_false(mock_bot_cls, mock_memory_tool def test_run_converts_build_prompt_exception_to_failed_outcome(mock_bot_cls, mock_memory_tool, tmp_path): task = SweBenchVerifiedTask(_instance()) task.setup = lambda repo_path: None - task.teardown = lambda repo_path: None def _build_prompt(): raise ValueError("bad prompt") @@ -489,7 +579,6 @@ def test_run_converts_check_exception_to_failed_outcome(mock_bot_cls, mock_memor task = SweBenchVerifiedTask(_instance()) task.setup = lambda repo_path: None task.build_prompt = lambda: "do the task" - task.teardown = lambda repo_path: None def _check(repo_path, agent_output, log_path): raise RuntimeError("check exploded") @@ -502,51 +591,6 @@ def _check(repo_path, agent_output, log_path): assert "check exploded" in outcome.result.reason -@pytest.mark.unit -@patch(f"{MODULE}.MemoryTool") -@patch(f"{MODULE}.WritingBot") -def test_run_still_calls_teardown_when_body_raises(mock_bot_cls, mock_memory_tool, tmp_path): - mock_bot_cls.side_effect = RuntimeError("bot construction failed") - - task = SweBenchVerifiedTask(_instance()) - teardown_calls = [] - task.setup = lambda repo_path: None - task.build_prompt = lambda: "do the task" - task.teardown = lambda repo_path: teardown_calls.append(repo_path) - - task.run("/repo", "/memory", "azure-openai/gpt-4o", str(tmp_path / "eval.log")) - - assert teardown_calls == ["/repo"] - - -@pytest.mark.unit -@patch(f"{MODULE}.MemoryTool") -@patch(f"{MODULE}.WritingBot") -def test_run_teardown_exception_does_not_clobber_returned_outcome(mock_bot_cls, mock_memory_tool, tmp_path): - from microbots.MicroBot import BotRunResult - - mock_bot = MagicMock() - mock_bot.run.return_value = BotRunResult(status=True, result="output", error=None) - mock_bot_cls.return_value = mock_bot - - task = SweBenchVerifiedTask(_instance()) - task.setup = lambda repo_path: None - task.build_prompt = lambda: "do the task" - task.check = lambda repo_path, agent_output, log_path: CallbackResult(passed=True, reason="ok") - - def _teardown(repo_path): - raise RuntimeError("teardown boom") - - task.teardown = _teardown - - outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o", str(tmp_path / "eval.log")) - - # teardown() raised, but the already-computed EvalOutcome must still be returned - assert outcome.passed is True - - - - # --------------------------------------------------------------------------- # SweBenchVerifiedTask.from_config # --------------------------------------------------------------------------- diff --git a/test/auto_memory/test_orchestrator.py b/test/auto_memory/test_orchestrator.py index fba3829..2cabafb 100644 --- a/test/auto_memory/test_orchestrator.py +++ b/test/auto_memory/test_orchestrator.py @@ -13,7 +13,6 @@ from microbots.auto_memory.orchestrator import ( LoopResult, clone_repo, - reset_repo, run, run_train_eval_loop, run_training_loop, @@ -55,7 +54,7 @@ def test_loop_returns_immediately_when_first_round_passes(mock_run_training_loop task = _make_task() task.run.return_value = _make_outcome(passed=True) - result = run_train_eval_loop("/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) + result = run_train_eval_loop("/repo", "/eval_repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) assert isinstance(result, LoopResult) assert result.passed is True @@ -75,7 +74,7 @@ def test_loop_retrains_and_continues_on_failure_then_passes(mock_run_training_lo ] task.build_feedback.return_value = "feedback text" - result = run_train_eval_loop("/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) + result = run_train_eval_loop("/repo", "/eval_repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) assert result.passed is True assert result.rounds_run == 2 @@ -99,7 +98,7 @@ def test_loop_exhausts_max_rounds_without_passing(mock_run_training_loop, tmp_pa ] task.build_feedback.return_value = "feedback text" - result = run_train_eval_loop("/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=3) + result = run_train_eval_loop("/repo", "/eval_repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=3) assert result.passed is False assert result.rounds_run == 3 @@ -116,7 +115,7 @@ def test_log_path_persists_after_passing_round(mock_run_training_loop, tmp_path) task = _make_task() task.run.return_value = _make_outcome(passed=True) - run_train_eval_loop("/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) + run_train_eval_loop("/repo", "/eval_repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) assert Path(log_path).exists() @@ -133,7 +132,7 @@ def test_log_path_persists_after_failing_round(mock_run_training_loop, tmp_path) ] task.build_feedback.return_value = "feedback text" - run_train_eval_loop("/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) + run_train_eval_loop("/repo", "/eval_repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) assert Path(log1).exists() assert Path(log2).exists() @@ -150,7 +149,7 @@ def test_build_feedback_exception_does_not_crash_loop(mock_run_training_loop, tm ] task.build_feedback.side_effect = RuntimeError("analysis bot crashed") - result = run_train_eval_loop("/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) + result = run_train_eval_loop("/repo", "/eval_repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) assert result.passed is True assert result.rounds_run == 2 @@ -169,7 +168,7 @@ def test_loop_forwards_training_iterations_to_run_training_loop(mock_run_trainin task.build_feedback.return_value = "feedback text" run_train_eval_loop( - "/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5, training_iterations=4 + "/repo", "/eval_repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5, training_iterations=4 ) mock_run_training_loop.assert_called_once_with( @@ -193,7 +192,7 @@ def test_run_training_exception_does_not_crash_loop(mock_run_training_loop, tmp_ task.build_feedback.return_value = "feedback text" mock_run_training_loop.side_effect = RuntimeError("training crashed") - result = run_train_eval_loop("/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) + result = run_train_eval_loop("/repo", "/eval_repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) assert result.passed is True assert result.rounds_run == 2 @@ -258,19 +257,6 @@ def test_clone_repo_is_noop_when_already_present(mock_run, tmp_path): mock_run.assert_not_called() -@pytest.mark.unit -@patch(f"{MODULE}.subprocess.run") -def test_reset_repo_runs_hard_reset_then_clean(mock_run, tmp_path): - repo_path = tmp_path / "repo" - - reset_repo(repo_path, "abc123") - - assert mock_run.call_args_list == [ - call(["git", "reset", "--hard", "abc123"], cwd=repo_path, check=True), - call(["git", "clean", "-fd"], cwd=repo_path, check=True), - ] - - @pytest.mark.unit def test_write_eval_result_writes_task_build_result_as_json(tmp_path): task = MagicMock() @@ -307,7 +293,7 @@ def test_loop_writes_eval_result_for_every_round(tmp_path): task.build_feedback.return_value = "feedback text" with patch(f"{MODULE}.run_training_loop"): - run_train_eval_loop("/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) + run_train_eval_loop("/repo", "/eval_repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) assert eval_result_path(tmp_path, 1, "task-1").exists() assert eval_result_path(tmp_path, 2, "task-1").exists() @@ -347,7 +333,8 @@ def test_run_calls_run_train_eval_loop_when_task_given(mock_run_train_eval_loop, ) mock_run_train_eval_loop.assert_called_once_with( - repo_path=str(tmp_path / "repo"), + training_repo_path=str(tmp_path / "repo"), + eval_repo_path=str(tmp_path / "eval_repo"), workdir=tmp_path, model="azure-openai/gpt-4o", task=fake_task, @@ -408,6 +395,24 @@ def fake_train(repo_path, feedback, memory_dir, model, iterations=1): assert (memory_dir(tmp_path) / "notes.md").read_text() == "learned something" +@pytest.mark.unit +@patch(f"{MODULE}.run_training_loop") +def test_run_preserves_original_memory_as_a_seed_snapshot(mock_run_training_loop, tmp_path): + memory_dir(tmp_path).mkdir(parents=True) + (memory_dir(tmp_path) / "notes.md").write_text("original seed") + + def fake_train(repo_path, feedback, memory_dir, model, iterations=1): + Path(memory_dir, "notes.md").write_text("overwritten by training") + + mock_run_training_loop.side_effect = fake_train + + run(workdir=tmp_path, model="azure-openai/gpt-4o", task=None) + + assert (memory_dir(tmp_path) / "notes.md").read_text() == "overwritten by training" + assert (tmp_path / "memory_seed" / "notes.md").read_text() == "original seed" + + + @pytest.mark.unit @patch(f"{MODULE}.run_training_loop") def test_loop_carries_memory_forward_between_rounds(mock_run_training_loop, tmp_path): @@ -426,6 +431,6 @@ def fake_run(repo_path, memory_dir, model, log_path): task.run.side_effect = fake_run task.build_feedback.return_value = "feedback text" - run_train_eval_loop("/repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) + run_train_eval_loop("/repo", "/eval_repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5) assert (memory_dir(tmp_path) / "notes.md").read_text() == "round 2 progress" diff --git a/test/auto_memory/test_task.py b/test/auto_memory/test_task.py index b66deb1..e04d0e9 100644 --- a/test/auto_memory/test_task.py +++ b/test/auto_memory/test_task.py @@ -13,6 +13,10 @@ class _RunOnlyTask(EvalTask): """A task that overrides only run()/build_feedback(), never touching the optional hooks.""" + @classmethod + def from_config(cls, task_args): + return [cls()] + def run(self, repo_path, memory_dir, model, log_path): return EvalOutcome( passed=True, @@ -33,6 +37,10 @@ def test_run_is_abstract(): @pytest.mark.unit def test_build_feedback_is_abstract(): class _MissingBuildFeedback(EvalTask): + @classmethod + def from_config(cls, task_args): + return [cls()] + def run(self, repo_path, memory_dir, model, log_path): raise NotImplementedError @@ -40,6 +48,25 @@ def run(self, repo_path, memory_dir, model, log_path): _MissingBuildFeedback() +@pytest.mark.unit +def test_from_config_is_abstract(): + class _MissingFromConfig(EvalTask): + def run(self, repo_path, memory_dir, model, log_path): + raise NotImplementedError + + def build_feedback(self, outcome, repo_path, model, log_path): + raise NotImplementedError + + with pytest.raises(TypeError): + _MissingFromConfig() + + +@pytest.mark.unit +def test_from_config_default_body_raises_not_implemented_error(): + with pytest.raises(NotImplementedError): + EvalTask.from_config({}) + + @pytest.mark.unit def test_subclass_overriding_only_run_is_instantiable(): task = _RunOnlyTask() diff --git a/test/auto_memory/test_task_registry.py b/test/auto_memory/test_task_registry.py index ea34cf8..8de0d48 100644 --- a/test/auto_memory/test_task_registry.py +++ b/test/auto_memory/test_task_registry.py @@ -19,6 +19,10 @@ class _DummyTask(EvalTask): def __init__(self, value=None): self.value = value + @classmethod + def from_config(cls, task_args): + return [cls(**task_args)] + def setup(self, repo_path): pass diff --git a/test/auto_memory/test_workdir.py b/test/auto_memory/test_workdir.py index 1eb2fae..96a04af 100644 --- a/test/auto_memory/test_workdir.py +++ b/test/auto_memory/test_workdir.py @@ -12,6 +12,7 @@ eval_dir, eval_log_path, eval_patch_path, + eval_repo_dir, eval_result_path, load_config, load_round_memory, @@ -22,8 +23,8 @@ round_dir, round_log_path, round_memory_dir, - run_log_path, save_round_memory, + snapshot_seed_memory, ) @@ -104,6 +105,41 @@ def test_save_round_memory_overwrites_stale_top_level_files(tmp_path): assert (top_memory / "notes.md").read_text() == "new" +@pytest.mark.unit +def test_snapshot_seed_memory_creates_empty_dir_when_no_top_level_memory(tmp_path): + result = snapshot_seed_memory(tmp_path) + + assert result == tmp_path / "memory_seed" + assert result.is_dir() + assert list(result.iterdir()) == [] + + +@pytest.mark.unit +def test_snapshot_seed_memory_copies_current_top_level_memory(tmp_path): + top_memory = memory_dir(tmp_path) + top_memory.mkdir(parents=True) + (top_memory / "notes.md").write_text("original seed") + + result = snapshot_seed_memory(tmp_path) + + assert (result / "notes.md").read_text() == "original seed" + + +@pytest.mark.unit +def test_snapshot_seed_memory_is_a_noop_once_a_snapshot_exists(tmp_path): + top_memory = memory_dir(tmp_path) + top_memory.mkdir(parents=True) + (top_memory / "notes.md").write_text("original seed") + snapshot_seed_memory(tmp_path) + + # Mutate top-level memory as later rounds/instances would. + (top_memory / "notes.md").write_text("overwritten by later training") + + result = snapshot_seed_memory(tmp_path) + + assert (result / "notes.md").read_text() == "original seed" + + @pytest.mark.unit def test_resolve_workdir_defaults_to_cwd(monkeypatch, tmp_path): monkeypatch.chdir(tmp_path) @@ -135,8 +171,8 @@ def test_repo_dir_returns_workdir_repo(tmp_path): @pytest.mark.unit -def test_run_log_path_returns_workdir_run_log(tmp_path): - assert run_log_path(tmp_path) == tmp_path / "run.log" +def test_eval_repo_dir_returns_workdir_eval_repo(tmp_path): + assert eval_repo_dir(tmp_path) == tmp_path / "eval_repo" @pytest.mark.unit From 998040a2f82c95af8c63633227c8d3e212869d92 Mon Sep 17 00:00:00 2001 From: Kavya Sree Kaitepalli Date: Thu, 3 Sep 2026 09:35:34 +0000 Subject: [PATCH 12/13] Resolve copilot comments --- .../auto_memory/eval/swebenchverified.py | 61 ++++++---- src/microbots/auto_memory/evalTask.py | 7 -- src/microbots/auto_memory/orchestrator.py | 56 ++++++++- src/microbots/auto_memory/workdir.py | 23 +++- .../auto_memory/eval/test_swebenchverified.py | 52 +++++++- test/auto_memory/test_orchestrator.py | 115 +++++++++++++++--- test/auto_memory/test_workdir.py | 36 ++++++ 7 files changed, 287 insertions(+), 63 deletions(-) diff --git a/src/microbots/auto_memory/eval/swebenchverified.py b/src/microbots/auto_memory/eval/swebenchverified.py index 889693b..613396d 100644 --- a/src/microbots/auto_memory/eval/swebenchverified.py +++ b/src/microbots/auto_memory/eval/swebenchverified.py @@ -16,7 +16,6 @@ from logging import getLogger from pathlib import Path -from datasets import load_dataset from microbots.auto_memory.evalTask import CallbackResult, EvalOutcome, EvalTask from microbots.auto_memory.task_registry import register_task from microbots.bot.LogAnalysisBot import LogAnalysisBot @@ -50,7 +49,20 @@ def _load_dataset_rows(dataset_name: str): ------- datasets.Dataset The loaded ``test`` split. + + Raises + ------ + ImportError + If the optional ``datasets`` package (the ``training`` extra) + isn't installed. """ + try: + from datasets import load_dataset + except ImportError as exc: + raise ImportError( + "SWE-bench-verified evaluation requires the 'training' extra: " + "pip install 'microbots[training]'" + ) from exc return load_dataset(dataset_name, split="test") @@ -226,18 +238,6 @@ def build_result(self, outcome: EvalOutcome) -> dict: def setup(self, repo_path: str) -> None: """Clone the instance's repo, or reset it, to its base commit. - Clones fresh on first use. If ``repo_path`` already exists - (e.g. left behind by a previous round) *and* its ``origin`` - remote matches this instance's repo, it's reset instead of - re-cloned: ``git reset --hard `` followed by - ``git clean -fd`` discards whatever the agent changed, without - the cost of a full re-clone and without deleting the directory - ``build_feedback`` may still need to inspect afterward. If - ``repo_path`` exists but isn't a checkout of this repo (e.g. a - stale directory left over from a different run/task), it's - removed and cloned fresh instead, to avoid silently operating - on the wrong codebase. - Parameters ---------- repo_path : str @@ -283,9 +283,12 @@ def build_prompt(self) -> str: def check(self, repo_path: str, agent_output: str, log_path: str) -> CallbackResult: """Verify the agent's patch using the SWE-bench evaluation harness. - Captures the agent's changes as a git diff, submits it as a - prediction to ``swebench.harness.run_evaluation``, and checks - whether the harness marked this instance as resolved. + Captures the agent's changes as a git diff (after marking any + untracked new files intent-to-add, so files the agent newly + created are included in the diff rather than silently + dropped), submits it as a prediction to + ``swebench.harness.run_evaluation``, and checks whether the + harness marked this instance as resolved. Parameters ---------- @@ -306,21 +309,31 @@ def check(self, repo_path: str, agent_output: str, log_path: str) -> CallbackRes CallbackResult Whether the harness marked this instance as resolved. """ + subprocess.run( + ["git", "add", "--intent-to-add", "."], cwd=repo_path, check=True + ) diff = subprocess.run( - ["git", "diff"], cwd=repo_path, capture_output=True, text=True + ["git", "diff", "--binary"], + cwd=repo_path, + capture_output=True, + text=True, + check=True, ).stdout run_id = f"microbots-{uuid.uuid4().hex[:8]}" model_name_or_path = EVAL_AGENT_MODEL_NAME - pred_path = Path(tempfile.mktemp(suffix=".json")) #will need to update when upgraded to ~5.0.2 , removed this flag in the new version #https://github.com/SWE-bench/SWE-bench/commit/e2c13307b6cf7764a50958b9c8bfbfb3f72cb70a report_dir = Path(tempfile.mkdtemp()) - pred_path.write_text(json.dumps([{ - "instance_id": self.instance.instance_id, - "model_patch": diff, - "model_name_or_path": model_name_or_path, - }])) + with tempfile.NamedTemporaryFile( + mode="w", suffix=".json", delete=False + ) as pred_file: + pred_path = Path(pred_file.name) + pred_file.write(json.dumps([{ + "instance_id": self.instance.instance_id, + "model_patch": diff, + "model_name_or_path": model_name_or_path, + }])) try: proc = subprocess.run( @@ -425,11 +438,11 @@ def run(self, repo_path: str, memory_dir: str, model: str, log_path: str) -> Eva The result of this eval round, including the agent's output, the check verdict. """ - self.setup(repo_path) Path(log_path).parent.mkdir(parents=True, exist_ok=True) Path(log_path).write_text("") try: + self.setup(repo_path) prompt = self.build_prompt() bot = WritingBot( model=model, diff --git a/src/microbots/auto_memory/evalTask.py b/src/microbots/auto_memory/evalTask.py index aacee20..be3af99 100644 --- a/src/microbots/auto_memory/evalTask.py +++ b/src/microbots/auto_memory/evalTask.py @@ -154,13 +154,6 @@ def teardown(self, repo_path: str) -> None: def from_config(cls, task_args: dict[str, Any]) -> list["EvalTask"]: """Required. Build task instance(s) from a config's ``task_args`` dict. - Called by the CLI at runtime (driven by ``--task``) to - construct the actual task object(s) to run, using whatever - config values the task needs (e.g. a dataset instance ID, a - repo filter). Object creation must go through this method - rather than being constructed elsewhere, so behavior stays - driven by the CLI/config at runtime. - Parameters ---------- task_args : dict[str, Any] diff --git a/src/microbots/auto_memory/orchestrator.py b/src/microbots/auto_memory/orchestrator.py index 7dc5662..e13ad3e 100644 --- a/src/microbots/auto_memory/orchestrator.py +++ b/src/microbots/auto_memory/orchestrator.py @@ -9,6 +9,7 @@ from logging import getLogger from pathlib import Path import json +import shutil import subprocess from microbots.auto_memory.evalTask import EvalOutcome, EvalTask @@ -48,18 +49,39 @@ class LoopResult: outcomes: list[EvalOutcome] = field(default_factory=list) def clone_repo(url: str, repo_path: Path) -> None: - """Clone ``url`` into ``repo_path`` if it isn't already cloned there. + """Clone ``url`` into ``repo_path``, or reuse it if already cloned from ``url``. + + Existence alone isn't enough to trust ``repo_path``: it could be an + empty/partial directory left by a previous failed clone, or a + reused workdir whose config now points at a different ``url``. So + if ``repo_path`` exists, its ``origin`` remote is checked against + ``url`` first. Only an exact match is reused as-is; anything else + (mismatched origin, or not a git checkout at all) is removed and + re-cloned, so training never silently runs against missing or + wrong code. Parameters ---------- url : str Git URL (or local path) to clone from. repo_path : Path - Destination directory for the clone. If it already exists (e.g. - a previous round already cloned here), this is a no-op. + Destination directory for the clone. """ if repo_path.exists(): - return + origin = subprocess.run( + ["git", "remote", "get-url", "origin"], + cwd=repo_path, capture_output=True, text=True, + ) + if origin.returncode == 0 and origin.stdout.strip() == url: + return + + logger.warning( + "clone_repo: %s exists but isn't a checkout of %s (origin=%r); " + "removing and re-cloning", + repo_path, url, origin.stdout.strip(), + ) + shutil.rmtree(repo_path) + subprocess.run(["git", "clone", url, str(repo_path)], check=True) def write_eval_result(workdir: Path, round_num: int, task: EvalTask, outcome: EvalOutcome) -> None: @@ -180,7 +202,15 @@ def run_train_eval_loop( LoopResult Whether the task passed, how many rounds ran, and every round's outcome. + + Raises + ------ + ValueError + If ``max_rounds`` is less than 1. """ + if max_rounds < 1: + raise ValueError(f"max_rounds must be >= 1, got {max_rounds}") + outcomes: list[EvalOutcome] = [] for round_idx in range(1, max_rounds+1): @@ -275,12 +305,26 @@ def run( ------- LoopResult | None The eval loop's result if ``task`` was given, otherwise ``None``. + + Raises + ------ + ValueError + If ``config`` has no ``repo`` entry. Every run needs a training + checkout (``run_training_loop`` always mounts + ``training_repo_path``, regardless of ``task``), so ``repo`` + must be configured even for tasks like ``SweBenchVerifiedTask`` + that manage their own separate eval checkout. """ if config is None: config = load_config(workdir) repo_url = config.get("repo") - if repo_url: - clone_repo(repo_url, repo_dir(workdir)) + if not repo_url: + raise ValueError( + "config.yaml must specify 'repo' (the training checkout's clone " + "URL); it is required even when the eval task manages its own " + "separate eval repo checkout." + ) + clone_repo(repo_url, repo_dir(workdir)) snapshot_seed_memory(workdir) diff --git a/src/microbots/auto_memory/workdir.py b/src/microbots/auto_memory/workdir.py index 65cb3ff..154699b 100644 --- a/src/microbots/auto_memory/workdir.py +++ b/src/microbots/auto_memory/workdir.py @@ -250,7 +250,11 @@ def load_round_memory(workdir: Path, round_num: int, *, instance_id: str | None """Copy the current top-level memory into this round's own memory dir. Called before a round's training pass, so it starts from whatever - memory the previous round left behind (or empty, on round 1). + memory the previous round left behind (or empty, on round 1). This + round's memory dir is replaced, not merged into: any stale files + left behind by a previous attempt at this same round (e.g. a + crashed/re-run process) are discarded first, so the round always + starts from an exact snapshot of the current top-level memory. Parameters ---------- @@ -269,9 +273,11 @@ def load_round_memory(workdir: Path, round_num: int, *, instance_id: str | None """ src = memory_dir(workdir) dst = round_memory_dir(workdir, round_num, instance_id=instance_id) - dst.mkdir(parents=True, exist_ok=True) + shutil.rmtree(dst, ignore_errors=True) if src.is_dir(): - shutil.copytree(src, dst, dirs_exist_ok=True) + shutil.copytree(src, dst) + else: + dst.mkdir(parents=True, exist_ok=True) return dst @@ -279,7 +285,10 @@ def save_round_memory(workdir: Path, round_num: int, *, instance_id: str | None """Copy this round's memory back up to the top-level memory dir. Called after a round's training pass, so later rounds (and the - final saved memory) see what this round learned. + final saved memory) see what this round learned. The top-level + memory dir is replaced, not merged into: files the round deleted + (e.g. via the agent's ``memory delete`` command) are gone from + the top level too, instead of surviving from a previous save. Parameters ---------- @@ -298,9 +307,11 @@ def save_round_memory(workdir: Path, round_num: int, *, instance_id: str | None """ src = round_memory_dir(workdir, round_num, instance_id=instance_id) dst = memory_dir(workdir) - dst.mkdir(parents=True, exist_ok=True) + shutil.rmtree(dst, ignore_errors=True) if src.is_dir(): - shutil.copytree(src, dst, dirs_exist_ok=True) + shutil.copytree(src, dst) + else: + dst.mkdir(parents=True, exist_ok=True) return dst diff --git a/test/auto_memory/eval/test_swebenchverified.py b/test/auto_memory/eval/test_swebenchverified.py index 79477ec..7caffea 100644 --- a/test/auto_memory/eval/test_swebenchverified.py +++ b/test/auto_memory/eval/test_swebenchverified.py @@ -11,6 +11,7 @@ sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../src/"))) from microbots.auto_memory.eval.swebenchverified import ( + SWE_BENCH_VERIFIED, SweBenchInstance, SweBenchVerifiedTask, _load_dataset_rows, @@ -59,7 +60,7 @@ def _fake_rows(): # --------------------------------------------------------------------------- @pytest.mark.unit -@patch(f"{MODULE}.load_dataset") +@patch("datasets.load_dataset") def test_load_instances_of_repo_filters_by_repo(mock_load_dataset): mock_load_dataset.return_value = _fake_rows() @@ -70,7 +71,7 @@ def test_load_instances_of_repo_filters_by_repo(mock_load_dataset): @pytest.mark.unit -@patch(f"{MODULE}.load_dataset") +@patch("datasets.load_dataset") def test_load_instances_of_repo_returns_all_when_repo_none(mock_load_dataset): mock_load_dataset.return_value = _fake_rows() @@ -80,7 +81,7 @@ def test_load_instances_of_repo_returns_all_when_repo_none(mock_load_dataset): @pytest.mark.unit -@patch(f"{MODULE}.load_dataset") +@patch("datasets.load_dataset") def test_load_instance_using_id_returns_matching_instance(mock_load_dataset): mock_load_dataset.return_value = _fake_rows() @@ -91,7 +92,7 @@ def test_load_instance_using_id_returns_matching_instance(mock_load_dataset): @pytest.mark.unit -@patch(f"{MODULE}.load_dataset") +@patch("datasets.load_dataset") def test_load_instance_using_id_raises_when_not_found(mock_load_dataset): mock_load_dataset.return_value = _fake_rows() @@ -100,7 +101,7 @@ def test_load_instance_using_id_raises_when_not_found(mock_load_dataset): @pytest.mark.unit -@patch(f"{MODULE}.load_dataset") +@patch("datasets.load_dataset") def test_dataset_rows_are_cached_across_repeated_calls(mock_load_dataset): """``load_dataset`` should only be called once per ``dataset_name``, even across multiple ``load_instances_of_repo``/``load_instance_using_id`` calls.""" @@ -113,6 +114,13 @@ def test_dataset_rows_are_cached_across_repeated_calls(mock_load_dataset): mock_load_dataset.assert_called_once() +@pytest.mark.unit +def test_load_dataset_rows_raises_helpful_error_when_datasets_not_installed(): + with patch.dict(sys.modules, {"datasets": None}): + with pytest.raises(ImportError, match=r"pip install 'microbots\[training\]'"): + _load_dataset_rows(SWE_BENCH_VERIFIED) + + # --------------------------------------------------------------------------- # SweBenchVerifiedTask.setup / build_prompt # --------------------------------------------------------------------------- @@ -337,6 +345,23 @@ def _fake_run(cmd, **kwargs): return _fake_run +@pytest.mark.unit +@patch(f"{MODULE}.subprocess.run") +def test_check_marks_untracked_files_intent_to_add_before_diffing(mock_run, tmp_path): + mock_run.side_effect = _make_fake_subprocess_run(resolved=True) + log_path = tmp_path / "check.log" + log_path.write_text("") + + task = SweBenchVerifiedTask(_instance()) + task.check("/repo", "agent output", str(log_path)) + + calls = [c.args[0] for c in mock_run.call_args_list] + add_idx = calls.index(["git", "add", "--intent-to-add", "."]) + diff_idx = calls.index(["git", "diff", "--binary"]) + assert add_idx < diff_idx + assert mock_run.call_args_list[add_idx].kwargs["cwd"] == "/repo" + + @pytest.mark.unit @patch(f"{MODULE}.subprocess.run") def test_check_passed_true_when_report_marks_resolved(mock_run, tmp_path): @@ -566,6 +591,23 @@ def _build_prompt(): assert "bad prompt" in f.read() +@pytest.mark.unit +def test_run_converts_setup_exception_to_failed_outcome(tmp_path): + task = SweBenchVerifiedTask(_instance()) + + def _setup(repo_path): + raise RuntimeError("clone failed") + + task.setup = _setup + + outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o", str(tmp_path / "eval.log")) + + assert outcome.passed is False + assert "clone failed" in outcome.result.reason + with open(str(tmp_path / "eval.log")) as f: + assert "clone failed" in f.read() + + @pytest.mark.unit @patch(f"{MODULE}.MemoryTool") @patch(f"{MODULE}.WritingBot") diff --git a/test/auto_memory/test_orchestrator.py b/test/auto_memory/test_orchestrator.py index 2cabafb..ba193be 100644 --- a/test/auto_memory/test_orchestrator.py +++ b/test/auto_memory/test_orchestrator.py @@ -48,6 +48,17 @@ def _make_task() -> MagicMock: return task +@pytest.mark.unit +@pytest.mark.parametrize("max_rounds", [0, -1]) +def test_loop_raises_for_non_positive_max_rounds(max_rounds): + task = _make_task() + + with pytest.raises(ValueError, match="max_rounds must be >= 1"): + run_train_eval_loop("/repo", "/eval_repo", Path("/workdir"), "azure-openai/gpt-4o", task, max_rounds=max_rounds) + + task.run.assert_not_called() + + @pytest.mark.unit @patch("microbots.auto_memory.orchestrator.run_training_loop") def test_loop_returns_immediately_when_first_round_passes(mock_run_training_loop, tmp_path): @@ -248,13 +259,48 @@ def test_clone_repo_clones_when_missing(mock_run, tmp_path): @pytest.mark.unit @patch(f"{MODULE}.subprocess.run") -def test_clone_repo_is_noop_when_already_present(mock_run, tmp_path): +def test_clone_repo_is_noop_when_origin_matches(mock_run, tmp_path): repo_path = tmp_path / "repo" repo_path.mkdir() + mock_run.return_value = MagicMock(returncode=0, stdout="https://example.com/repo.git\n") clone_repo("https://example.com/repo.git", repo_path) - mock_run.assert_not_called() + mock_run.assert_called_once_with( + ["git", "remote", "get-url", "origin"], + cwd=repo_path, capture_output=True, text=True, + ) + + +@pytest.mark.unit +@patch(f"{MODULE}.subprocess.run") +def test_clone_repo_removes_and_reclones_when_origin_mismatched(mock_run, tmp_path): + repo_path = tmp_path / "repo" + repo_path.mkdir() + (repo_path / "stale_marker.txt").write_text("leftover from a different repo") + mock_run.return_value = MagicMock(returncode=0, stdout="https://example.com/other-repo.git\n") + + clone_repo("https://example.com/repo.git", repo_path) + + assert not (repo_path / "stale_marker.txt").exists() + mock_run.assert_called_with( + ["git", "clone", "https://example.com/repo.git", str(repo_path)], check=True + ) + + +@pytest.mark.unit +@patch(f"{MODULE}.subprocess.run") +def test_clone_repo_removes_and_reclones_when_repo_path_not_a_git_repo(mock_run, tmp_path): + repo_path = tmp_path / "repo" + repo_path.mkdir() + mock_run.return_value = MagicMock(returncode=128, stdout="") + + clone_repo("https://example.com/repo.git", repo_path) + + assert not repo_path.exists() + mock_run.assert_called_with( + ["git", "clone", "https://example.com/repo.git", str(repo_path)], check=True + ) @pytest.mark.unit @@ -304,9 +350,16 @@ def test_loop_writes_eval_result_for_every_round(tmp_path): @pytest.mark.unit +@patch(f"{MODULE}.clone_repo") @patch(f"{MODULE}.run_training_loop") -def test_run_calls_run_training_loop_when_task_is_none(mock_run_training_loop, tmp_path): - result = run(workdir=tmp_path, model="azure-openai/gpt-4o", task=None, training_iterations=2) +def test_run_calls_run_training_loop_when_task_is_none(mock_run_training_loop, mock_clone_repo, tmp_path): + result = run( + workdir=tmp_path, + model="azure-openai/gpt-4o", + task=None, + training_iterations=2, + config={"repo": "https://example.com/repo.git"}, + ) mock_run_training_loop.assert_called_once_with( repo_path=str(tmp_path / "repo"), @@ -319,8 +372,9 @@ def test_run_calls_run_training_loop_when_task_is_none(mock_run_training_loop, t @pytest.mark.unit +@patch(f"{MODULE}.clone_repo") @patch(f"{MODULE}.run_train_eval_loop") -def test_run_calls_run_train_eval_loop_when_task_given(mock_run_train_eval_loop, tmp_path): +def test_run_calls_run_train_eval_loop_when_task_given(mock_run_train_eval_loop, mock_clone_repo, tmp_path): fake_task = MagicMock() mock_run_train_eval_loop.return_value = "loop-result" @@ -330,6 +384,7 @@ def test_run_calls_run_train_eval_loop_when_task_given(mock_run_train_eval_loop, task=fake_task, max_rounds=3, training_iterations=2, + config={"repo": "https://example.com/repo.git"}, ) mock_run_train_eval_loop.assert_called_once_with( @@ -345,19 +400,35 @@ def test_run_calls_run_train_eval_loop_when_task_given(mock_run_train_eval_loop, @pytest.mark.unit +@patch(f"{MODULE}.clone_repo") @patch(f"{MODULE}.run_train_eval_loop") @patch(f"{MODULE}.run_training_loop") -def test_run_does_not_call_eval_loop_when_task_is_none(mock_run_training_loop, mock_run_train_eval_loop, tmp_path): - run(workdir=tmp_path, model="azure-openai/gpt-4o", task=None) +def test_run_does_not_call_eval_loop_when_task_is_none( + mock_run_training_loop, mock_run_train_eval_loop, mock_clone_repo, tmp_path +): + run( + workdir=tmp_path, + model="azure-openai/gpt-4o", + task=None, + config={"repo": "https://example.com/repo.git"}, + ) mock_run_train_eval_loop.assert_not_called() @pytest.mark.unit +@patch(f"{MODULE}.clone_repo") @patch(f"{MODULE}.run_train_eval_loop") @patch(f"{MODULE}.run_training_loop") -def test_run_does_not_call_training_loop_when_task_given(mock_run_training_loop, mock_run_train_eval_loop, tmp_path): - run(workdir=tmp_path, model="azure-openai/gpt-4o", task=MagicMock()) +def test_run_does_not_call_training_loop_when_task_given( + mock_run_training_loop, mock_run_train_eval_loop, mock_clone_repo, tmp_path +): + run( + workdir=tmp_path, + model="azure-openai/gpt-4o", + task=MagicMock(), + config={"repo": "https://example.com/repo.git"}, + ) mock_run_training_loop.assert_not_called() @@ -376,28 +447,37 @@ def test_run_clones_repo_from_config_when_repo_url_given(mock_run_training_loop, @pytest.mark.unit @patch(f"{MODULE}.clone_repo") @patch(f"{MODULE}.run_training_loop") -def test_run_does_not_clone_when_config_has_no_repo(mock_run_training_loop, mock_clone_repo, tmp_path): - run(workdir=tmp_path, model="azure-openai/gpt-4o", task=None) +def test_run_raises_when_config_has_no_repo(mock_run_training_loop, mock_clone_repo, tmp_path): + with pytest.raises(ValueError, match="repo"): + run(workdir=tmp_path, model="azure-openai/gpt-4o", task=None, config={}) mock_clone_repo.assert_not_called() + mock_run_training_loop.assert_not_called() @pytest.mark.unit +@patch(f"{MODULE}.clone_repo") @patch(f"{MODULE}.run_training_loop") -def test_run_promotes_round1_memory_to_top_level_for_train_only_mode(mock_run_training_loop, tmp_path): +def test_run_promotes_round1_memory_to_top_level_for_train_only_mode(mock_run_training_loop, mock_clone_repo, tmp_path): def fake_train(repo_path, feedback, memory_dir, model, iterations=1): Path(memory_dir, "notes.md").write_text("learned something") mock_run_training_loop.side_effect = fake_train - run(workdir=tmp_path, model="azure-openai/gpt-4o", task=None) + run( + workdir=tmp_path, + model="azure-openai/gpt-4o", + task=None, + config={"repo": "https://example.com/repo.git"}, + ) assert (memory_dir(tmp_path) / "notes.md").read_text() == "learned something" @pytest.mark.unit +@patch(f"{MODULE}.clone_repo") @patch(f"{MODULE}.run_training_loop") -def test_run_preserves_original_memory_as_a_seed_snapshot(mock_run_training_loop, tmp_path): +def test_run_preserves_original_memory_as_a_seed_snapshot(mock_run_training_loop, mock_clone_repo, tmp_path): memory_dir(tmp_path).mkdir(parents=True) (memory_dir(tmp_path) / "notes.md").write_text("original seed") @@ -406,7 +486,12 @@ def fake_train(repo_path, feedback, memory_dir, model, iterations=1): mock_run_training_loop.side_effect = fake_train - run(workdir=tmp_path, model="azure-openai/gpt-4o", task=None) + run( + workdir=tmp_path, + model="azure-openai/gpt-4o", + task=None, + config={"repo": "https://example.com/repo.git"}, + ) assert (memory_dir(tmp_path) / "notes.md").read_text() == "overwritten by training" assert (tmp_path / "memory_seed" / "notes.md").read_text() == "original seed" diff --git a/test/auto_memory/test_workdir.py b/test/auto_memory/test_workdir.py index 96a04af..4c87baa 100644 --- a/test/auto_memory/test_workdir.py +++ b/test/auto_memory/test_workdir.py @@ -70,6 +70,24 @@ def test_load_round_memory_copies_top_level_memory_into_round(tmp_path): assert (result / "notes.md").read_text() == "prior findings" +@pytest.mark.unit +def test_load_round_memory_discards_stale_files_left_in_round_dir(tmp_path): + # Simulate a previous crashed/re-run attempt at this same round that + # left behind a file no longer present in top-level memory. + round_memory = round_memory_dir(tmp_path, 1) + round_memory.mkdir(parents=True) + (round_memory / "stale.md").write_text("leftover from a crashed attempt") + + top_memory = memory_dir(tmp_path) + top_memory.mkdir(parents=True) + (top_memory / "notes.md").write_text("current memory") + + result = load_round_memory(tmp_path, 1) + + assert (result / "notes.md").read_text() == "current memory" + assert not (result / "stale.md").exists() + + @pytest.mark.unit def test_save_round_memory_creates_empty_dir_when_no_round_memory(tmp_path): result = save_round_memory(tmp_path, 1) @@ -105,6 +123,24 @@ def test_save_round_memory_overwrites_stale_top_level_files(tmp_path): assert (top_memory / "notes.md").read_text() == "new" +@pytest.mark.unit +def test_save_round_memory_propagates_deletions_to_top_level(tmp_path): + # The agent deleted a file during this round (e.g. via `memory + # delete`); the top level shouldn't resurrect it from a prior save. + top_memory = memory_dir(tmp_path) + top_memory.mkdir(parents=True) + (top_memory / "stale.md").write_text("no longer relevant") + + round_memory = round_memory_dir(tmp_path, 1) + round_memory.mkdir(parents=True) + (round_memory / "notes.md").write_text("kept") + + save_round_memory(tmp_path, 1) + + assert not (top_memory / "stale.md").exists() + assert (top_memory / "notes.md").read_text() == "kept" + + @pytest.mark.unit def test_snapshot_seed_memory_creates_empty_dir_when_no_top_level_memory(tmp_path): result = snapshot_seed_memory(tmp_path) From db258daa0d9e659a769b257f97d9b671c688cfb7 Mon Sep 17 00:00:00 2001 From: bala Date: Mon, 7 Sep 2026 15:18:30 +0000 Subject: [PATCH 13/13] Single training should be validated by all the instances in the task --- src/microbots/auto_memory/__init__.py | 2 +- src/microbots/auto_memory/architecture.md | 196 ++++++++++ src/microbots/auto_memory/cli.py | 42 ++- .../auto_memory/eval/swebenchverified.py | 357 +++++++++++------- src/microbots/auto_memory/evalTask.py | 159 ++------ src/microbots/auto_memory/orchestrator.py | 84 ++--- src/microbots/auto_memory/task_registry.py | 32 +- src/microbots/auto_memory/workdir.py | 38 +- src/microbots/auto_memory/workdir/config.yaml | 3 + 9 files changed, 523 insertions(+), 390 deletions(-) create mode 100644 src/microbots/auto_memory/architecture.md create mode 100644 src/microbots/auto_memory/workdir/config.yaml diff --git a/src/microbots/auto_memory/__init__.py b/src/microbots/auto_memory/__init__.py index e959bfa..e967e45 100644 --- a/src/microbots/auto_memory/__init__.py +++ b/src/microbots/auto_memory/__init__.py @@ -4,5 +4,5 @@ an evaluation task and run it in a loop against a training agent. """ -from .evalTask import CallbackResult, EvalOutcome, EvalTask +from .evalTask import EvalOutcome, EvalTask from .orchestrator import LoopResult, run_train_eval_loop \ No newline at end of file diff --git a/src/microbots/auto_memory/architecture.md b/src/microbots/auto_memory/architecture.md new file mode 100644 index 0000000..01c9135 --- /dev/null +++ b/src/microbots/auto_memory/architecture.md @@ -0,0 +1,196 @@ +# auto_memory — Architecture + +An agent that **learns a repository into memory notes**, then **proves those notes work** by +solving a real task with them. If it fails, it learns again from the failure and retries. + +> Train → Eval → Feedback → Train → … until pass (or rounds run out). + +--- + +## 1. The Big Picture + +```mermaid +flowchart LR + subgraph LOOP["Train / Eval Loop"] + direction TB + T["🧠 TRAIN
ReadingBot reads the repo
writes notes to memory/"] + E["🎯 EVAL
WritingBot solves a task
using only those notes"] + C{"Passed?"} + F["🔍 FEEDBACK
LogAnalysisBot reads the failure log
says what the notes were missing"] + + E --> C + C -- "yes" --> DONE(["✅ Done"]) + C -- "no" --> F --> T --> E + end + + CLI["cli.py
--model --task --max-rounds"] --> LOOP +``` + +**Key idea:** the eval agent gets *no* extra hints — only the memory notes. +So a failing eval is direct evidence the notes are wrong or incomplete. + +--- + +## 2. The Cast + +| File | Role | One-liner | +|---|---|---| +| `cli.py` | Entry point | Parses args, builds tasks, calls the orchestrator | +| `orchestrator.py` | Conductor | Owns the round loop, clones repo, wires train ↔ eval | +| `evalTask.py` | Contract | Abstract `EvalTask`: `run`, `check`, `build_feedback`, … | +| `task_registry.py` | Plugin table | `@register_task("name")` + auto-import of `eval/*` | +| `eval/swebenchverified.py` | A real task | One SWE-bench-Verified issue, graded by the official harness | +| `training/runner.py` | Trainer | One `ReadingBot` pass + `MemoryTool` | +| `training/training_instructions.md` | Trainer's brief | "Learn the repo, write notes, never edit code" | +| `workdir.py` | Filing clerk | Every path under `workdir/` lives here — nothing is hard-coded elsewhere | + +--- + +## 3. One Round, Step by Step + +```mermaid +sequenceDiagram + autonumber + participant O as orchestrator + participant W as workdir + participant Task as EvalTask + participant Bot as WritingBot + participant Train as run_training_loop + + O->>W: load_round_memory(round N) + Note over W: copy memory/ ➜ rounds_/round_N/memory + O->>Task: run(eval_repo, memory_dir, model, log) + Task->>Task: setup() – clone/reset repo @ base commit + Task->>Bot: build_prompt() + MemoryTool(memory_dir) + Bot-->>Task: patch in repo + output + Task->>Task: check() – grade it (SWE-bench harness) + Task-->>O: EvalOutcome(passed, output, result) + + alt passed + O-->>O: return LoopResult(passed=True) + else failed + O->>Task: build_feedback(outcome, log) + Task-->>O: "your notes were missing X" + O->>Train: run_training_loop(feedback, memory_dir) × iterations + Train-->>W: notes updated in place + end + + O->>W: write result.json + save_round_memory(round N) + Note over W: copy round memory ➜ back up to memory/ +``` + +--- + +## 4. Memory Lifecycle (the heart of it) + +Memory is a **directory of markdown notes** that is copied down into each round, +mutated by the bots, then copied back up. + +```mermaid +flowchart TD + SEED["workdir/memory_seed/
immutable baseline snapshot"] + TOP["workdir/memory/
current best notes"] + R1["round_1/memory"] + R2["round_2/memory"] + R3["round_N/memory"] + + TOP -. "snapshot once, at run start" .-> SEED + TOP -->|load_round_memory| R1 + R1 -->|save_round_memory| TOP + TOP -->|load_round_memory| R2 + R2 -->|save_round_memory| TOP + TOP -->|load_round_memory| R3 + R3 -->|save_round_memory| TOP +``` + +Rules that matter: + +- **Replace, never merge.** `load`/`save` do `rmtree` + `copytree`, so deleted notes stay deleted + and stale files from a crashed round can't leak in. +- **`memory_seed` is written once.** It preserves the pre-run state, because `memory/` is + mutated in place all run long. +- **Memory carries across tasks.** Multiple task instances in one workdir share `memory/`, + so later instances inherit what earlier ones learned. + +--- + +## 5. Workdir Layout + +```text +workdir/ +├── config.yaml # repo URL + task_args +├── repo/ # persistent clone — TRAINING only +├── eval_repo/ # task-managed clone — EVAL only (reset each round) +├── memory_seed/ # baseline snapshot (write-once) +├── memory/ # current best notes ← the thing being optimized +└── rounds_/ # per-task-instance, so instances never collide + └── round_N/ + ├── memory/ # this round's working copy of the notes + └── eval/ + ├── eval.log # agent output + harness logs (feeds LogAnalysisBot) + └── result.json +``` + +Two repos on purpose: the eval task wipes/resets its checkout every round, which +would otherwise destroy the training checkout. + +--- + +## 6. Two Modes + +```mermaid +flowchart LR + A["orchestrator.run(task=?)"] + A -->|"task is None"| B["Train-only
N training passes, empty feedback
round 1 is just a scratch dir"] + A -->|"task given"| C["run_train_eval_loop
up to max_rounds"] +``` + +```bash +# train only +python -m microbots.auto_memory.cli --model azure-openai/gpt-5.5 + +# train + eval against a SWE-bench instance +python -m microbots.auto_memory.cli \ + --model azure-openai/gpt-5.5 \ + --task swebenchverified \ + --max-rounds 5 --training-iterations 10 +``` + +--- + +## 7. Adding a New Eval Task + +Drop a module in `eval/` — `discover_tasks()` imports everything in that package, +so the `@register_task` decorator fires and the name shows up in `--task`. No central +factory to edit. + +```python +@register_task("mytask") +class MyTask(EvalTask): + @classmethod + def from_config(cls, task_args: dict) -> list["EvalTask"]: + ... # one instance per unit of work + + def run(self, repo_path, memory_dir, model, log_path) -> EvalOutcome: + ... # you drive setup/build_prompt/check yourself + + def build_feedback(self, outcome, repo_path, model, log_path) -> str: + ... # turn the failure log into "what the notes should say" +``` + +Required: `from_config`, `run`, `build_feedback`. +Optional hooks (`setup`, `build_prompt`, `check`, `teardown`, `build_result`, `task_id`) +are **not** called automatically — your `run` decides. + +--- + +## 8. Failure Handling at a Glance + +| Where it breaks | What happens | +|---|---| +| Agent run raises | Caught in `run`; logged; round fails with the exception as the reason | +| `build_feedback` / retraining raises | Logged; loop **continues to the next round** without retraining | +| Repo dir exists with wrong `origin` | Removed and re-cloned (never silently trains on wrong code) | +| `max_rounds` exhausted | `LoopResult(passed=False)` with every round's outcome | + +Whatever happens, the `finally` block still writes `result.json` and saves the round's memory. diff --git a/src/microbots/auto_memory/cli.py b/src/microbots/auto_memory/cli.py index 66ab0e6..383f313 100644 --- a/src/microbots/auto_memory/cli.py +++ b/src/microbots/auto_memory/cli.py @@ -4,7 +4,6 @@ - ``--task `` given: run the full train <-> eval loop for that task. -- ``--task`` omitted: train only, no eval task, with empty feedback. Both modes are dispatched via ``orchestrator.run``. """ @@ -15,7 +14,7 @@ from microbots.auto_memory.orchestrator import run from microbots.auto_memory.task_registry import TASK_REGISTRY, discover_tasks -from microbots.auto_memory.workdir import load_config, require_workdir, resolve_workdir +from microbots.auto_memory.workdir import require_workdir, resolve_workdir logger = logging.getLogger(__name__) @@ -47,8 +46,12 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: ) parser.add_argument( "--task", + required=True, choices=sorted(TASK_REGISTRY), - help="Eval task to run. Omit to only run training, with no eval task.", + help="Eval task to run.", + ) + parser.add_argument( + "--config-file", type=Path, help="Path to the task configuration file.", ) parser.add_argument("--max-rounds", type=int, default=5) parser.add_argument("--training-iterations", type=int, default=10) @@ -68,25 +71,24 @@ def main(argv: list[str] | None = None) -> None: workdir = Path(args.workdir) if args.workdir else resolve_workdir() require_workdir(workdir) - config = load_config(workdir) - tasks = ( - TASK_REGISTRY[args.task].from_config(config.get("task_args", {})) - if args.task - else [None] + if not args.config_file: + config_file = workdir / "task_config.yaml" + else: + config_file = args.config_file + if not config_file.is_file(): + raise FileNotFoundError(f"Config file not found: {config_file}") + + result = run( + workdir=workdir, + model=args.model, + task=TASK_REGISTRY[args.task](config_file=config_file), + max_rounds=args.max_rounds, + training_iterations=args.training_iterations, ) - for task in tasks: - result = run( - workdir=workdir, - model=args.model, - task=task, - max_rounds=args.max_rounds, - training_iterations=args.training_iterations, - config=config, + if result is not None: + logger.info( + "task=%s passed=%s rounds_run=%d", args.task, result.passed, result.rounds_run ) - if result is not None: - logger.info( - "task=%s passed=%s rounds_run=%d", args.task, result.passed, result.rounds_run - ) if __name__ == "__main__": main() diff --git a/src/microbots/auto_memory/eval/swebenchverified.py b/src/microbots/auto_memory/eval/swebenchverified.py index 613396d..31d2866 100644 --- a/src/microbots/auto_memory/eval/swebenchverified.py +++ b/src/microbots/auto_memory/eval/swebenchverified.py @@ -12,13 +12,15 @@ import tempfile import uuid from dataclasses import dataclass -from functools import lru_cache +from functools import cache from logging import getLogger from pathlib import Path -from microbots.auto_memory.evalTask import CallbackResult, EvalOutcome, EvalTask +import yaml + +from microbots.auto_memory.evalTask import EvalOutcome, EvalTask from microbots.auto_memory.task_registry import register_task -from microbots.bot.LogAnalysisBot import LogAnalysisBot +from microbots.bot.ReadingBot import ReadingBot from microbots.bot.WritingBot import WritingBot from microbots.MicroBot import BotRunResult from microbots.tools.tool_definitions.memory_tool import MemoryTool @@ -29,14 +31,36 @@ EVAL_AGENT_MODEL_NAME = "microbots-eval-agent" -@lru_cache(maxsize=None) +@dataclass +class SweBenchInstance: + """A single SWE-bench-verified dataset row. + + Attributes + ---------- + instance_id : str + Unique identifier for the instance, e.g. ``"django__django-11099"``. + repo : str + The GitHub repo this instance belongs to, e.g. ``"django/django"``. + base_commit : str + Commit hash representing the repo state before the issue's fix. + problem_statement : str + The GitHub issue title and body describing the bug to fix. + """ + + instance_id: str + repo: str + base_commit: str + problem_statement: str + + +@cache def _load_dataset_rows(dataset_name: str): """Load and cache ``dataset_name``'s ``test`` split for the process's lifetime. ``load_dataset`` caches the downloaded files on disk, but still re-reads and rebuilds the in-memory ``Dataset`` object on every call. Since ``load_instances_of_repo``/``load_instance_using_id`` - may each be called many times (e.g. once per eval task instance), + may each be called many times, this wraps ``load_dataset`` with an in-memory cache keyed by ``dataset_name``, so the dataset is only loaded once per process. @@ -69,7 +93,7 @@ def _load_dataset_rows(dataset_name: str): def load_instances_of_repo( dataset_name: str = SWE_BENCH_VERIFIED, repo: str | None = None, -) -> list["SweBenchInstance"]: +) -> list[SweBenchInstance]: """Load all dataset instances, optionally filtered to a single repo. Parameters @@ -100,7 +124,7 @@ def load_instances_of_repo( ] return instances -def load_instance_using_id(instance_id: str, dataset_name: str = SWE_BENCH_VERIFIED) -> "SweBenchInstance": +def load_instance_using_id(instance_id: str, dataset_name: str = SWE_BENCH_VERIFIED) -> SweBenchInstance: """Load a single dataset instance by its instance ID. Parameters @@ -133,30 +157,8 @@ def load_instance_using_id(instance_id: str, dataset_name: str = SWE_BENCH_VERIF ) raise ValueError(f"instance_id not found: {instance_id}") -@dataclass -class SweBenchInstance: - """A single SWE-bench-verified dataset row. - - Attributes - ---------- - instance_id : str - Unique identifier for the instance, e.g. ``"django__django-11099"``. - repo : str - The GitHub repo this instance belongs to, e.g. ``"django/django"``. - base_commit : str - Commit hash representing the repo state before the issue's fix. - problem_statement : str - The GitHub issue title and body describing the bug to fix. - """ - - instance_id: str - repo: str - base_commit: str - problem_statement: str - -@register_task("swebenchverified") -class SweBenchVerifiedTask(EvalTask): - """Eval task that verifies a fix against one SWE-bench-verified instance. +class SweBenchVerifiedTask_one(): + """SWE-bench-verified based evaluation task. Checks out the instance's repo at its base commit, gives the agent the issue's problem statement, and verifies the agent's patch using @@ -180,28 +182,6 @@ def __init__(self, instance: SweBenchInstance | None = None): """ self.instance = instance - @classmethod - def from_config(cls, task_args: dict) -> list["SweBenchVerifiedTask"]: - """Build task(s) from a config's ``task_args`` dict. - - Parameters - ---------- - task_args : dict - Task-specific config values, expected to include - ``instance_id`` and/or ``swebench_repo``. - - Returns - ------- - list[SweBenchVerifiedTask] - One task per matching dataset instance. A single-element - list when ``instance_id`` is given. - """ - if task_args.get("instance_id"): - instances = [load_instance_using_id(task_args["instance_id"])] - else: - instances = load_instances_of_repo(repo=task_args.get("swebench_repo")) - return [cls(instance) for instance in instances] - @property def task_id(self) -> str: """Return this instance's SWE-bench-verified ``instance_id``. @@ -213,28 +193,6 @@ def task_id(self) -> str: """ return self.instance.instance_id - def build_result(self, outcome: EvalOutcome) -> dict: - """Summarize a round's outcome, including the instance's dataset fields. - - Parameters - ---------- - outcome : EvalOutcome - The round's outcome to summarize. - - Returns - ------- - dict - ``passed``/``reason`` plus ``instance_id``, ``repo``, and - ``base_commit`` identifying which dataset row this is. - """ - return { - "passed": outcome.result.passed, - "reason": outcome.result.reason, - "instance_id": self.instance.instance_id, - "repo": self.instance.repo, - "base_commit": self.instance.base_commit, - } - def setup(self, repo_path: str) -> None: """Clone the instance's repo, or reset it, to its base commit. @@ -280,7 +238,7 @@ def build_prompt(self) -> str: """ return self.instance.problem_statement - def check(self, repo_path: str, agent_output: str, log_path: str) -> CallbackResult: + def check(self, repo_path: str, agent_output: str, log_path: str) -> BotRunResult: """Verify the agent's patch using the SWE-bench evaluation harness. Captures the agent's changes as a git diff (after marking any @@ -306,8 +264,12 @@ def check(self, repo_path: str, agent_output: str, log_path: str) -> CallbackRes Returns ------- - CallbackResult - Whether the harness marked this instance as resolved. + BotRunResult + ``status`` is whether the harness marked this instance as + resolved. On failure, ``error`` carries the harness's + ``test_output.txt`` (or its console output, if the harness + died before producing one) so the feedback bot can see why + the tests failed. """ subprocess.run( ["git", "add", "--intent-to-add", "."], cwd=repo_path, check=True @@ -353,12 +315,17 @@ def check(self, repo_path: str, agent_output: str, log_path: str) -> CallbackRes report_dir / "logs" / "run_evaluation" / run_id / model_name_or_path / self.instance.instance_id ) + # Read while report_dir still exists; the finally block deletes it. + test_output = "" with open(log_path, "a") as f: f.write(proc.stdout + proc.stderr) for log_filename in ("run_instance.log", "test_output.txt"): log_file = instance_log_dir / log_filename if log_file.exists(): - f.write(f"\n--- {log_filename} ---\n{log_file.read_text()}\n") + content = log_file.read_text() + if log_filename == "test_output.txt": + test_output = content + f.write(f"\n--- {log_filename} ---\n{content}\n") report_file = instance_log_dir / "report.json" passed = False @@ -369,54 +336,14 @@ def check(self, repo_path: str, agent_output: str, log_path: str) -> CallbackRes pred_path.unlink(missing_ok=True) shutil.rmtree(report_dir, ignore_errors=True) - return CallbackResult(passed=passed, reason="resolved" if passed else "not resolved") - - def build_feedback(self, outcome: EvalOutcome, repo_path: str, model: str, log_path: str) -> str: - """Analyze a failed round's log via ``LogAnalysisBot`` for training feedback. - - Parameters - ---------- - outcome : EvalOutcome - The failed outcome to analyze. - repo_path : str - Absolute path to the repo the task was evaluated against. - model : str - The model to use, in the format ``/``. - log_path : str - Path to the round's log file (the same path passed to - ``run``), analyzed by ``LogAnalysisBot``. - - Returns - ------- - str - Feedback text describing the root cause of the failure and - what the agent's memory notes should cover next time. - """ - bot = LogAnalysisBot(model=model, folder_to_mount=repo_path) - result: BotRunResult = bot.run( - file_name=log_path, - user_prompt=( - "This log was produced while verifying whether an " - "agent completed its task correctly. Identify " - "the root cause of the failure and describe concretely " - "what the agent's memory notes should cover next time to " - "avoid this failure." - ), + return BotRunResult( + status = passed, + result = "resolved" if passed else "not resolved", + # Harness can fail before producing test_output.txt; fall back to its console output. + error = None if passed else (test_output or proc.stdout + proc.stderr) ) - if result.status and result.result: - return result.result - - logger.warning( - "LogAnalysisBot failed to analyze failure (%s); falling back to plain feedback", - result.error, - ) - return ( - f"Evaluation failed. Agent output: {outcome.output}\n" - f"Callback reason: {outcome.result.reason}" - ) - - def run(self, repo_path: str, memory_dir: str, model: str, log_path: str) -> EvalOutcome: + def eval(self, repo_path: str, memory_dir: str, model: str, log_path: str) -> BotRunResult: """Run one eval iteration: setup -> build_prompt -> WritingBot -> check. Parameters @@ -434,7 +361,7 @@ def run(self, repo_path: str, memory_dir: str, model: str, log_path: str) -> Eva Returns ------- - EvalOutcome + BotRunResult The result of this eval round, including the agent's output, the check verdict. """ @@ -454,31 +381,171 @@ def run(self, repo_path: str, memory_dir: str, model: str, log_path: str) -> Eva with open(log_path, "a") as f: f.write(f"Agent output:\n{bot_result.result}\n") - if not bot_result.status: - reason = f"Bot run failed: {bot_result.error}" - with open(log_path, "a") as f: - f.write(f"\n{reason}\n") - result = CallbackResult(passed=False, reason=reason) - else: - result = self.check(repo_path, bot_result.result or "", log_path) + return bot_result - return EvalOutcome( - passed=result.passed, - output=bot_result.result, - result=result, - ) except Exception as exc: logger.exception( - "SweBenchVerifiedTask.run: iteration raised %s", type(exc).__name__ + "SweBenchVerifiedTask.eval: iteration raised %s", type(exc).__name__ ) with open(log_path, "a") as f: f.write(f"\nException during eval iteration: {type(exc).__name__}: {exc}\n") - return EvalOutcome( - passed=False, - output=None, - result=CallbackResult( - passed=False, reason=f"{type(exc).__name__}: {exc}" - ), + return BotRunResult( + status=False, + result=None, + error=f"{type(exc).__name__}: {exc}" ) +@register_task("swebenchverified") +class SweBenchVerified(EvalTask): + """SWE-bench-verified based evaluation task. + + It takes the memory provided by the training agent and runs + all the selected SWE-bench-verified instances. Then provides + a combined score and feedback. + """ + + def __init__(self, config_file: Path) -> None: + super().__init__(config_file) + self.dataset: list[SweBenchInstance] = [] + self.parse_config(config_file=config_file) + + def repo_url(self) -> str: + """Return the URL of the repo for the training agent. + + Returns: + str: The URL of the repo for the training agent. + """ + return f"https://github.com/{self.dataset[0].repo}.git" + + def teardown(self, eval_repo_path: Path) -> None: + """Tear down the task, cleaning up any resources if necessary.""" + if eval_repo_path and eval_repo_path.exists(): + shutil.rmtree(eval_repo_path) + + def parse_config(self, config_file: Path) -> None: + """Parse the configuration file for the task. + The config file is a yaml file. It will have array of "instance_id" + or "repo" as the root object. Gather it and load the dataset to + the object variable dataset. + + Args: + config_file (Path): Path to the configuration file. + """ + + with open(config_file, "r") as f: + config = yaml.safe_load(f) + + instance_ids = config.get("instance_id_list", []) + repo = config.get("repo", None) + + if instance_ids: + repo = None + for instance_id in instance_ids: + dataset = load_instance_using_id(instance_id) + if not repo: + repo = dataset.repo + elif repo != dataset.repo: + raise ValueError( + f"Conflicting repos for instance_id {instance_id}: {repo} vs {dataset.repo}" + ) + + self.dataset.append(dataset) + + elif repo: + self.dataset = load_instances_of_repo(repo=repo) + + if len(self.dataset) == 0: + raise ValueError("No instances loaded for evaluation.") + + def eval(self, memory_dir: str, model: str, log_path: str) -> EvalOutcome: + """Runs the evaluation agent with the memory on all the eval instances + and produces a cumulative feedback. + + Args: + memory_dir (str): Path to the directory containing the agent's memory. + model (str): The model identifier used for evaluation. + log_path (str): Path to the log file for recording evaluation details. + + Returns: + EvalOutcome: The outcome of the evaluation, including whether it passed, the output, and the result. + """ + + eval_repo_path = Path(log_path).parent / "eval_repo" + results = [] + + for instance in self.dataset: + inst_log_path = Path(log_path).parent / f"{instance.instance_id}_log.txt" + task = SweBenchVerifiedTask_one(instance) + + res = task.eval(str(eval_repo_path), memory_dir, model, str(inst_log_path)) + + if not res.status: + logger.info(f"Evaluation failed for instance {instance.instance_id}: {res.error if res.error else 'Unknown error'}") + results.append(res) + else: + res = task.check(str(eval_repo_path), "", str(inst_log_path)) + results.append(res) + + score = 0 + for result in results: + if result.status: + score += 1 + + score = score / len(self.dataset) + + if score == 1: + feedback = "All evaluations passed." + else: + feedback = self._combine_result_feedback(results, model, str(eval_repo_path)) + + self.teardown(eval_repo_path) + + return EvalOutcome( + passed = score == 1, + score = score, + feedback = feedback + ) + + + def _combine_result_feedback(self, results: list[BotRunResult], model: str, eval_repo: str) -> str: + """ + Combines the feedback from multiple BotRunResult instances into a single feedback string. + Args: + results (list[BotRunResult]): List of individual bot run results. + model (str): The model identifier used for evaluation. + eval_repo (str): Path to the evaluation repository. + + Returns: + str: Combined feedback from all results. + """ + + serialized_str = f"Total {len(results)} tests ran and their result and feedback:\n" + + for res in results: + serialized_str += f"\nResult: {'Passed' if res.status else 'Failed'}\n" + serialized_str += f"Optional Feedback: {res.result if res.result else 'None'}\n" + serialized_str += f"Error if there are any: {res.error if res.error else 'None'}\n" + + try: + bot = ReadingBot( + model = model, + folder_to_mount=eval_repo + ) + task = f""" + Combine the results of the eval runs into single feedback. + This feedback will be given to the next iteration. + You just combine the results with minimal efforts. + Avoid referring to code whenever possible. + + {serialized_str} + """ + bot_result = bot.run(task=task) + except Exception as e: + logger.warning(f"Combining results failed with exception: {e}") + return f"Combining results failed. raw combined output:\n\n{serialized_str}" + + if bot_result.status: + return bot_result.result if bot_result.result else 'No feedback provided' + else: + return f"Combining results failed. raw combined output:\n\n{serialized_str}" diff --git a/src/microbots/auto_memory/evalTask.py b/src/microbots/auto_memory/evalTask.py index be3af99..fbe1290 100644 --- a/src/microbots/auto_memory/evalTask.py +++ b/src/microbots/auto_memory/evalTask.py @@ -7,52 +7,45 @@ from abc import ABC, abstractmethod from dataclasses import dataclass +from pathlib import Path from typing import Any + @dataclass -class CallbackResult: +class EvalOutcome: """Result of verifying whether an eval task was completed correctly. Attributes ---------- passed : bool Whether the agent's output satisfies the task's check. - reason : str + score : float + A numeric score representing the quality of the agent's output. + feedback : str A short human-readable explanation of the pass/fail verdict. """ passed: bool - reason: str - -@dataclass -class EvalOutcome: - """Full record of one eval round. - - Attributes - ---------- - passed : bool - Whether the round passed, mirrors ``result.passed``. - output : str | None - The agent's raw output for the round, if any. - result : CallbackResult - The verdict produced by ``EvalTask.check``. - """ - - passed: bool - output: str | None - result: CallbackResult - + score: float + feedback: str class EvalTask(ABC): """Base class for a single evaluation task in the train <-> eval loop. - Subclasses must implement ``run`` and ``from_config``. ``setup``, - ``build_prompt``, ``check``, and ``teardown`` are optional hooks + Subclasses must implement ``run``. ``parse_config``, ``setup``, + ``check``, and ``teardown`` are optional hooks subclasses may use to structure their own ``run`` implementation (see ``SweBenchVerifiedTask`` for an example), but nothing in this base class calls them automatically. """ + def __init__(self, config_file: Path) -> None: + super().__init__() + + @abstractmethod + def repo_url(self) -> str: + """Return the URL of the repo for the training agent.""" + @property def task_id(self) -> str: """Identifier for this task instance, used to name its output folder. @@ -69,26 +62,7 @@ def task_id(self) -> str: """ return type(self).__name__ - def build_result(self, outcome: EvalOutcome) -> dict: - """Optional. Build the dict written to this round's ``result.json``. - - Not called automatically; the orchestrator calls this after - each round to decide what to persist. Override to include - task-specific details (e.g. dataset fields, repo info). - - Parameters - ---------- - outcome : EvalOutcome - The round's outcome to summarize. - - Returns - ------- - dict - JSON-serializable summary. Defaults to ``passed``/``reason``. - """ - return {"passed": outcome.result.passed, "reason": outcome.result.reason} - - def setup(self, repo_path: str) -> None: + def setup(self) -> None: """Optional. Prepare repo/environment before the agent runs. Not called automatically; only useful if your ``run`` @@ -101,113 +75,34 @@ def setup(self, repo_path: str) -> None: """ pass - def build_prompt(self) -> str: - """Optional. Return the task prompt/instructions for the agent. - - Not called automatically; only useful if your ``run`` - implementation calls it. - - Returns - ------- - str - The prompt/instructions to give the agent. Empty string by - default. - """ - return "" - - def check(self, repo_path: str, agent_output: str, log_path: str) -> CallbackResult: - """Optional. Verify whether the task was actually completed correctly. - - Not called automatically; only useful if your ``run`` - implementation calls it. - - Parameters - ---------- - repo_path : str - Absolute path to the repo the agent operated on. - agent_output : str - The agent's raw output/result text. - log_path : str - Path to a log file, already created by ``run``, that this - check may append verification details to. - - Returns - ------- - CallbackResult - The pass/fail verdict and its reason. Passes by default. - """ - return CallbackResult(passed=True, reason="not checked") - - - def teardown(self, repo_path: str) -> None: + def teardown(self, eval_repo_path: Path) -> None: """Optional. Clean up anything setup() created. Parameters ---------- - repo_path : str + eval_repo_path : Path Absolute path to the repo that was prepared by ``setup``. """ pass - @classmethod - @abstractmethod - def from_config(cls, task_args: dict[str, Any]) -> list["EvalTask"]: - """Required. Build task instance(s) from a config's ``task_args`` dict. - - Parameters - ---------- - task_args : dict[str, Any] - Task-specific config values (the config file's - ``task_args`` section). - - Returns - ------- - list[EvalTask] - One task instance per unit of work this config describes - (often just one, but e.g. ``SweBenchVerifiedTask`` returns - one per matching dataset instance). - """ - raise NotImplementedError( - f"{cls.__name__} must implement from_config() to be usable via --task" - ) - @abstractmethod - def build_feedback(self, outcome: EvalOutcome, repo_path: str, model: str, log_path: str) -> str: - """Required. Analyze a failed eval outcome and produce training feedback. - - Called by the orchestrator after a failed round, before - retraining, to turn the round's outcome/log into concrete - feedback text describing what went wrong and what the agent's - memory notes should cover next time. + def parse_config(self, config_file: Path) -> None: + """Parse the task-specific config file. Importantly it + parses the config file and get the repo for the training + agent. Parameters ---------- - outcome : EvalOutcome - The failed outcome to analyze. - repo_path : str - Absolute path to the repo the task was evaluated against. - model : str - The model to use, in the format ``/``. - log_path : str - Path to the round's log file, containing the agent output - and any failure/exception details recorded during the - round (the same path passed to ``run``). - - Returns - ------- - str - Feedback text to pass as ``feedback`` to the next round's - training. + config_file : Path + Path to the config file to parse. """ @abstractmethod - def run(self, repo_path: str, memory_dir: str, model: str, log_path: str) -> EvalOutcome: + def eval(self, memory_dir: str, model: str, log_path: str) -> EvalOutcome: """Required. Run one eval iteration and return its outcome. Parameters ---------- - repo_path : str - Absolute path to the repo to run the eval round against. memory_dir : str Directory containing memory files to give the agent via ``MemoryTool``. diff --git a/src/microbots/auto_memory/orchestrator.py b/src/microbots/auto_memory/orchestrator.py index e13ad3e..4691837 100644 --- a/src/microbots/auto_memory/orchestrator.py +++ b/src/microbots/auto_memory/orchestrator.py @@ -6,6 +6,7 @@ """ from dataclasses import dataclass, field +import dataclasses from logging import getLogger from pathlib import Path import json @@ -16,9 +17,7 @@ from microbots.auto_memory.training.runner import run_training from microbots.auto_memory.workdir import ( eval_log_path, - eval_repo_dir, eval_result_path, - load_config, load_round_memory, repo_dir, save_round_memory, @@ -87,10 +86,6 @@ def clone_repo(url: str, repo_path: Path) -> None: def write_eval_result(workdir: Path, round_num: int, task: EvalTask, outcome: EvalOutcome) -> None: """Write a round's eval result to ``result.json``. - Delegates the content to ``task.build_result(outcome)`` so each - task decides what's worth persisting (e.g. ``SweBenchVerifiedTask`` - includes its dataset instance's fields). - Parameters ---------- workdir : Path @@ -105,7 +100,7 @@ def write_eval_result(workdir: Path, round_num: int, task: EvalTask, outcome: Ev """ path = eval_result_path(workdir, round_num, task.task_id) path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(task.build_result(outcome), indent=2)) + path.write_text(json.dumps(dataclasses.asdict(outcome), indent=2)) def run_training_loop( repo_path: str, @@ -148,7 +143,6 @@ def run_training_loop( def run_train_eval_loop( training_repo_path: str, - eval_repo_path: str, workdir: Path, model: str, task: EvalTask, @@ -217,9 +211,10 @@ def run_train_eval_loop( logger.info( "run_train_eval_loop: round %d/%d starting", round_idx, max_rounds ) + # TODO: Instead of loading new memory dir on every iteration, snapshot the memory. memory_dir = str(load_round_memory(workdir, round_idx, instance_id=task.task_id)) log_path = str(eval_log_path(workdir, round_idx, task.task_id)) - outcome = task.run(eval_repo_path, memory_dir, model, log_path) + outcome = task.eval(memory_dir, model, log_path) outcomes.append(outcome) try: @@ -237,13 +232,12 @@ def run_train_eval_loop( logger.info( "run_train_eval_loop: round %d failed (%s), retraining", round_idx, - outcome.result.reason, + outcome.feedback, ) try: - feedback = task.build_feedback(outcome, eval_repo_path, model, log_path) run_training_loop( repo_path=training_repo_path, - feedback=feedback, + feedback=outcome.feedback, memory_dir=memory_dir, model=model, iterations=training_iterations, @@ -271,40 +265,31 @@ def run_train_eval_loop( def run( workdir: Path, model: str, - task: EvalTask | None, + task: EvalTask, max_rounds: int = 5, training_iterations: int = 10, - config: dict | None = None, -) -> LoopResult | None: - """Run training only, or the full train/eval loop, depending on ``task``. +) -> LoopResult: + """Run full train/eval loop, depending on ``task``. Parameters ---------- workdir : Path This run's workdir (see ``microbots.auto_memory.workdir``), - holding ``config.yaml``, the shared repo clone, and all output. + holding ``task_config.yaml``, the shared repo clone, and all output. model : str The model to use, in the format ``/``. - task : EvalTask | None - The eval task to run each round, or ``None`` to only run - training (with empty feedback, once per ``training_iterations``). + task : EvalTask + The eval task to run each round. max_rounds : int - Maximum number of train/eval rounds to attempt, if ``task`` is - given. Defaults to 5. + Maximum number of train/eval rounds to attempt. Defaults to 5. training_iterations : int Number of training passes to run per retraining round, each reusing the same round memory dir. Defaults to 10. - config : dict | None - This run's already-loaded ``config.yaml`` contents. If ``None`` - (the default), it is loaded from ``workdir`` here. Callers that - invoke ``run`` repeatedly for the same ``workdir`` (e.g. once - per eval task) can load it once and pass it in, to avoid - re-reading/re-parsing the file on every call. Returns ------- - LoopResult | None - The eval loop's result if ``task`` was given, otherwise ``None``. + LoopResult + The eval loop's result. Raises ------ @@ -315,38 +300,29 @@ def run( must be configured even for tasks like ``SweBenchVerifiedTask`` that manage their own separate eval checkout. """ - if config is None: - config = load_config(workdir) - repo_url = config.get("repo") - if not repo_url: - raise ValueError( - "config.yaml must specify 'repo' (the training checkout's clone " - "URL); it is required even when the eval task manages its own " - "separate eval repo checkout." - ) - clone_repo(repo_url, repo_dir(workdir)) + clone_repo(task.repo_url(), repo_dir(workdir)) snapshot_seed_memory(workdir) training_repo_path = str(repo_dir(workdir)) - if task is None: - # Train-only mode has no rounds of its own; round 1 is just a - # scratch dir seeded from (and saved back to) top-level memory. - memory_dir = str(load_round_memory(workdir, 1)) - run_training_loop( - repo_path=training_repo_path, - feedback="", - memory_dir=memory_dir, - model=model, - iterations=training_iterations, - ) - save_round_memory(workdir, 1) - return None + # TODO: train-only mode will be implemented if required after proper design + # if task is None: + # # Train-only mode has no rounds of its own; round 1 is just a + # # scratch dir seeded from (and saved back to) top-level memory. + # memory_dir = str(load_round_memory(workdir, 1)) + # run_training_loop( + # repo_path=training_repo_path, + # feedback="", + # memory_dir=memory_dir, + # model=model, + # iterations=training_iterations, + # ) + # save_round_memory(workdir, 1) + # return None return run_train_eval_loop( training_repo_path=training_repo_path, - eval_repo_path=str(eval_repo_dir(workdir)), workdir=workdir, model=model, task=task, diff --git a/src/microbots/auto_memory/task_registry.py b/src/microbots/auto_memory/task_registry.py index 63bfb74..1846d2d 100644 --- a/src/microbots/auto_memory/task_registry.py +++ b/src/microbots/auto_memory/task_registry.py @@ -7,14 +7,18 @@ import importlib import pkgutil +from collections.abc import Callable from microbots.auto_memory.evalTask import EvalTask TASK_REGISTRY: dict[str, type[EvalTask]] = {} -def register_task(name: str): +def register_task(name: str) -> Callable[[type[EvalTask]], type[EvalTask]]: """Register an ``EvalTask`` subclass under ``name`` as a class decorator. + Each name maps to exactly one class; registering a name twice is a + programming error rather than a silent overwrite. + Parameters ---------- name : str @@ -40,27 +44,39 @@ def decorator(task_cls: type[EvalTask]) -> type[EvalTask]: ------- type[EvalTask] ``task_cls``, unchanged. + + Raises + ------ + ValueError + If ``name`` is already registered to a different class. """ + registered = TASK_REGISTRY.get(name) + if registered is not None and registered is not task_cls: + raise ValueError( + f"Task name {name!r} is already registered to " + f"{registered.__module__}.{registered.__qualname__}; " + f"cannot also register {task_cls.__module__}.{task_cls.__qualname__}." + ) TASK_REGISTRY[name] = task_cls return task_cls return decorator -# Not being used currently, but kept it for future use if required. -def create_task(name: str, **kwargs) -> EvalTask: - """Construct a registered ``EvalTask`` by name. +def create_task(name: str) -> EvalTask: + """Construct the registered ``EvalTask`` for ``name``. + + Tasks take no constructor arguments; per-run configuration is + applied afterwards via ``EvalTask.parse_config``. Parameters ---------- name : str The registered task name, e.g. ``"swebenchverified"``. - **kwargs - Keyword arguments forwarded to the task's constructor. Returns ------- EvalTask - The constructed task instance. + A new instance of the class registered under ``name``. Raises ------ @@ -73,7 +89,7 @@ def create_task(name: str, **kwargs) -> EvalTask: raise ValueError( f"Unknown task {name!r}. Registered tasks: {sorted(TASK_REGISTRY)}" ) from None - return task_cls(**kwargs) + return task_cls() def discover_tasks(package_name: str = "microbots.auto_memory.eval") -> None: """Import every module in ``package_name`` so ``@register_task`` fires. diff --git a/src/microbots/auto_memory/workdir.py b/src/microbots/auto_memory/workdir.py index 154699b..1276ff2 100644 --- a/src/microbots/auto_memory/workdir.py +++ b/src/microbots/auto_memory/workdir.py @@ -38,24 +38,24 @@ def resolve_workdir(base: Path | None = None) -> Path: Path ``workdir`` resolved relative to ``base`` (or ``Path.cwd()``). """ - return (base or Path.cwd()) / WORKDIR_NAME + if base is not None and not base.is_absolute(): + raise ValueError(f"base must be an absolute path: {base}") + workdir = (base or Path.cwd()) / WORKDIR_NAME + return workdir def require_workdir(workdir: Path) -> None: - """Validate that ``workdir`` exist. + """Validate that ``workdir`` exist. Create if not exist Parameters ---------- workdir : Path The workdir to validate. - Raises - ------ - FileNotFoundError - If ``workdir`` does not exist. """ - if not workdir.is_dir(): - raise FileNotFoundError(f"workdir not found: {workdir}") + # create workdir if not existing + if not workdir.exists(): + workdir.mkdir(parents=True) def config_path(workdir: Path) -> Path: @@ -117,28 +117,6 @@ def repo_dir(workdir: Path) -> Path: return workdir / REPO_DIRNAME -def eval_repo_dir(workdir: Path) -> Path: - """Return the path to the repo an eval task clones/manages itself. - - Kept separate from ``repo_dir`` (the training repo) because a - task's ``setup`` may clone or reset this directory every round - (e.g. ``SweBenchVerifiedTask`` checks out a different repo/commit - per dataset instance), which would otherwise conflict with the - persistent training checkout at ``repo_dir``. - - Parameters - ---------- - workdir : Path - The run's workdir. - - Returns - ------- - Path - ``workdir/eval_repo``. - """ - return workdir / EVAL_REPO_DIRNAME - - def memory_dir(workdir: Path) -> Path: """Return the path to the current top-level (latest) memory directory. diff --git a/src/microbots/auto_memory/workdir/config.yaml b/src/microbots/auto_memory/workdir/config.yaml new file mode 100644 index 0000000..03de1e5 --- /dev/null +++ b/src/microbots/auto_memory/workdir/config.yaml @@ -0,0 +1,3 @@ +instance_id_list: + - astropy__astropy-12907 + - astropy__astropy-13033 \ No newline at end of file