diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index e898772..a87c7a7 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -165,7 +165,7 @@ jobs:
- name: Install package in development mode
run: |
- pip install -e .
+ pip install -e ".[training]"
- name: Install GitHub Copilot SDK dependencies for GHCP tests
if: matrix.test-type == 'ghcp'
diff --git a/pyproject.toml b/pyproject.toml
index 3917589..a66a138 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -25,6 +25,7 @@ requires-python = ">=3.11"
ghcp = ["github-copilot-sdk==0.3.0"]
azure_ad = ["azure-identity>=1.15.0"]
dev = ["pre-commit>=3.7", "numpydoc>=1.8"]
+training = ["datasets==4.5.0", "swebench==4.1.0"]
[tool.setuptools.dynamic]
dependencies = { file = ["requirements.txt"] }
diff --git a/requirements.txt b/requirements.txt
index 4bf03b5..bfb90ec 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -10,36 +10,26 @@ certifi==2025.8.3
charset-normalizer==3.4.3
click==8.3.0
coverage==7.11.3
-datasets==4.5.0
-dill==0.4.0
distro==1.9.0
docker==7.1.0
docstring_parser==0.17.0
fastapi==0.116.1
-filelock==3.20.3
frozenlist==1.7.0
-fsspec==2025.10.0
h11==0.16.0
-hf-xet==1.2.0
httpcore==1.0.9
httpx==0.28.1
-huggingface_hub==1.3.2
idna==3.10
iniconfig==2.1.0
jiter==0.11.0
markdown-it-py==4.0.0
mdurl==0.1.2
multidict==6.6.4
-multiprocess==0.70.18
-numpy==1.26.4
openai==1.107.3
packaging==25.0
-pandas==3.0.0
pexpect==4.9.0
pluggy==1.6.0
propcache==0.3.2
ptyprocess==0.7.0
-pyarrow==23.0.0
pydantic==2.11.9
pydantic_core==2.33.2
Pygments==2.19.2
@@ -62,5 +52,4 @@ typing-inspection==0.4.1
typing_extensions==4.15.0
urllib3==2.5.0
uvicorn==0.35.0
-xxhash==3.6.0
yarl==1.20.1
diff --git a/src/microbots/auto_memory/__init__.py b/src/microbots/auto_memory/__init__.py
new file mode 100644
index 0000000..e967e45
--- /dev/null
+++ b/src/microbots/auto_memory/__init__.py
@@ -0,0 +1,8 @@
+"""Train <-> eval loop for repo-learning agents.
+
+Re-exports the public task, outcome, and orchestrator types used to define
+an evaluation task and run it in a loop against a training agent.
+"""
+
+from .evalTask import EvalOutcome, EvalTask
+from .orchestrator import LoopResult, run_train_eval_loop
\ No newline at end of file
diff --git a/src/microbots/auto_memory/architecture.md b/src/microbots/auto_memory/architecture.md
new file mode 100644
index 0000000..01c9135
--- /dev/null
+++ b/src/microbots/auto_memory/architecture.md
@@ -0,0 +1,196 @@
+# auto_memory — Architecture
+
+An agent that **learns a repository into memory notes**, then **proves those notes work** by
+solving a real task with them. If it fails, it learns again from the failure and retries.
+
+> Train → Eval → Feedback → Train → … until pass (or rounds run out).
+
+---
+
+## 1. The Big Picture
+
+```mermaid
+flowchart LR
+ subgraph LOOP["Train / Eval Loop"]
+ direction TB
+ T["🧠 TRAIN
ReadingBot reads the repo
writes notes to memory/"]
+ E["🎯 EVAL
WritingBot solves a task
using only those notes"]
+ C{"Passed?"}
+ F["🔍 FEEDBACK
LogAnalysisBot reads the failure log
says what the notes were missing"]
+
+ E --> C
+ C -- "yes" --> DONE(["✅ Done"])
+ C -- "no" --> F --> T --> E
+ end
+
+ CLI["cli.py
--model --task --max-rounds"] --> LOOP
+```
+
+**Key idea:** the eval agent gets *no* extra hints — only the memory notes.
+So a failing eval is direct evidence the notes are wrong or incomplete.
+
+---
+
+## 2. The Cast
+
+| File | Role | One-liner |
+|---|---|---|
+| `cli.py` | Entry point | Parses args, builds tasks, calls the orchestrator |
+| `orchestrator.py` | Conductor | Owns the round loop, clones repo, wires train ↔ eval |
+| `evalTask.py` | Contract | Abstract `EvalTask`: `run`, `check`, `build_feedback`, … |
+| `task_registry.py` | Plugin table | `@register_task("name")` + auto-import of `eval/*` |
+| `eval/swebenchverified.py` | A real task | One SWE-bench-Verified issue, graded by the official harness |
+| `training/runner.py` | Trainer | One `ReadingBot` pass + `MemoryTool` |
+| `training/training_instructions.md` | Trainer's brief | "Learn the repo, write notes, never edit code" |
+| `workdir.py` | Filing clerk | Every path under `workdir/` lives here — nothing is hard-coded elsewhere |
+
+---
+
+## 3. One Round, Step by Step
+
+```mermaid
+sequenceDiagram
+ autonumber
+ participant O as orchestrator
+ participant W as workdir
+ participant Task as EvalTask
+ participant Bot as WritingBot
+ participant Train as run_training_loop
+
+ O->>W: load_round_memory(round N)
+ Note over W: copy memory/ ➜ rounds_/round_N/memory
+ O->>Task: run(eval_repo, memory_dir, model, log)
+ Task->>Task: setup() – clone/reset repo @ base commit
+ Task->>Bot: build_prompt() + MemoryTool(memory_dir)
+ Bot-->>Task: patch in repo + output
+ Task->>Task: check() – grade it (SWE-bench harness)
+ Task-->>O: EvalOutcome(passed, output, result)
+
+ alt passed
+ O-->>O: return LoopResult(passed=True)
+ else failed
+ O->>Task: build_feedback(outcome, log)
+ Task-->>O: "your notes were missing X"
+ O->>Train: run_training_loop(feedback, memory_dir) × iterations
+ Train-->>W: notes updated in place
+ end
+
+ O->>W: write result.json + save_round_memory(round N)
+ Note over W: copy round memory ➜ back up to memory/
+```
+
+---
+
+## 4. Memory Lifecycle (the heart of it)
+
+Memory is a **directory of markdown notes** that is copied down into each round,
+mutated by the bots, then copied back up.
+
+```mermaid
+flowchart TD
+ SEED["workdir/memory_seed/
immutable baseline snapshot"]
+ TOP["workdir/memory/
current best notes"]
+ R1["round_1/memory"]
+ R2["round_2/memory"]
+ R3["round_N/memory"]
+
+ TOP -. "snapshot once, at run start" .-> SEED
+ TOP -->|load_round_memory| R1
+ R1 -->|save_round_memory| TOP
+ TOP -->|load_round_memory| R2
+ R2 -->|save_round_memory| TOP
+ TOP -->|load_round_memory| R3
+ R3 -->|save_round_memory| TOP
+```
+
+Rules that matter:
+
+- **Replace, never merge.** `load`/`save` do `rmtree` + `copytree`, so deleted notes stay deleted
+ and stale files from a crashed round can't leak in.
+- **`memory_seed` is written once.** It preserves the pre-run state, because `memory/` is
+ mutated in place all run long.
+- **Memory carries across tasks.** Multiple task instances in one workdir share `memory/`,
+ so later instances inherit what earlier ones learned.
+
+---
+
+## 5. Workdir Layout
+
+```text
+workdir/
+├── config.yaml # repo URL + task_args
+├── repo/ # persistent clone — TRAINING only
+├── eval_repo/ # task-managed clone — EVAL only (reset each round)
+├── memory_seed/ # baseline snapshot (write-once)
+├── memory/ # current best notes ← the thing being optimized
+└── rounds_/ # per-task-instance, so instances never collide
+ └── round_N/
+ ├── memory/ # this round's working copy of the notes
+ └── eval/
+ ├── eval.log # agent output + harness logs (feeds LogAnalysisBot)
+ └── result.json
+```
+
+Two repos on purpose: the eval task wipes/resets its checkout every round, which
+would otherwise destroy the training checkout.
+
+---
+
+## 6. Two Modes
+
+```mermaid
+flowchart LR
+ A["orchestrator.run(task=?)"]
+ A -->|"task is None"| B["Train-only
N training passes, empty feedback
round 1 is just a scratch dir"]
+ A -->|"task given"| C["run_train_eval_loop
up to max_rounds"]
+```
+
+```bash
+# train only
+python -m microbots.auto_memory.cli --model azure-openai/gpt-5.5
+
+# train + eval against a SWE-bench instance
+python -m microbots.auto_memory.cli \
+ --model azure-openai/gpt-5.5 \
+ --task swebenchverified \
+ --max-rounds 5 --training-iterations 10
+```
+
+---
+
+## 7. Adding a New Eval Task
+
+Drop a module in `eval/` — `discover_tasks()` imports everything in that package,
+so the `@register_task` decorator fires and the name shows up in `--task`. No central
+factory to edit.
+
+```python
+@register_task("mytask")
+class MyTask(EvalTask):
+ @classmethod
+ def from_config(cls, task_args: dict) -> list["EvalTask"]:
+ ... # one instance per unit of work
+
+ def run(self, repo_path, memory_dir, model, log_path) -> EvalOutcome:
+ ... # you drive setup/build_prompt/check yourself
+
+ def build_feedback(self, outcome, repo_path, model, log_path) -> str:
+ ... # turn the failure log into "what the notes should say"
+```
+
+Required: `from_config`, `run`, `build_feedback`.
+Optional hooks (`setup`, `build_prompt`, `check`, `teardown`, `build_result`, `task_id`)
+are **not** called automatically — your `run` decides.
+
+---
+
+## 8. Failure Handling at a Glance
+
+| Where it breaks | What happens |
+|---|---|
+| Agent run raises | Caught in `run`; logged; round fails with the exception as the reason |
+| `build_feedback` / retraining raises | Logged; loop **continues to the next round** without retraining |
+| Repo dir exists with wrong `origin` | Removed and re-cloned (never silently trains on wrong code) |
+| `max_rounds` exhausted | `LoopResult(passed=False)` with every round's outcome |
+
+Whatever happens, the `finally` block still writes `result.json` and saves the round's memory.
diff --git a/src/microbots/auto_memory/cli.py b/src/microbots/auto_memory/cli.py
new file mode 100644
index 0000000..383f313
--- /dev/null
+++ b/src/microbots/auto_memory/cli.py
@@ -0,0 +1,94 @@
+"""Command-line entry point for the auto-memory train/eval loop.
+
+Two modes, selected by ``--task``:
+
+- ``--task `` given: run the full train <-> eval loop for that
+ task.
+
+Both modes are dispatched via ``orchestrator.run``.
+"""
+
+import argparse
+import logging
+from pathlib import Path
+
+from microbots.auto_memory.orchestrator import run
+from microbots.auto_memory.task_registry import TASK_REGISTRY, discover_tasks
+from microbots.auto_memory.workdir import require_workdir, resolve_workdir
+
+logger = logging.getLogger(__name__)
+
+# Import every task module so their @register_task decorators fire.
+discover_tasks()
+
+def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
+ """Parse the CLI's top-level args.
+
+ Task-specific values (e.g. an eval task's instance ID) are not
+ parsed here; they come from the workdir's config file instead.
+
+ Parameters
+ ----------
+ argv : list[str] | None
+ Args to parse. Defaults to ``sys.argv[1:]`` when ``None``.
+
+ Returns
+ -------
+ argparse.Namespace
+ The parsed args.
+ """
+ parser = argparse.ArgumentParser(description="Run the auto-memory train/eval loop.")
+ parser.add_argument("--model", required=True, help='Model, e.g. "azure-openai/gpt-5.5".')
+ parser.add_argument(
+ "--workdir",
+ help="Directory holding this run's files (repo clone, logs, memory, "
+ "config). Defaults to './workdir' relative to the current directory.",
+ )
+ parser.add_argument(
+ "--task",
+ required=True,
+ choices=sorted(TASK_REGISTRY),
+ help="Eval task to run.",
+ )
+ parser.add_argument(
+ "--config-file", type=Path, help="Path to the task configuration file.",
+ )
+ parser.add_argument("--max-rounds", type=int, default=5)
+ parser.add_argument("--training-iterations", type=int, default=10)
+
+ return parser.parse_args(argv)
+
+def main(argv: list[str] | None = None) -> None:
+ """CLI entry point: run training only, or the full train/eval loop.
+
+ Parameters
+ ----------
+ argv : list[str] | None
+ Args to parse. Defaults to ``sys.argv[1:]`` when ``None``.
+ """
+ args = parse_args(argv)
+
+ workdir = Path(args.workdir) if args.workdir else resolve_workdir()
+ require_workdir(workdir)
+
+ if not args.config_file:
+ config_file = workdir / "task_config.yaml"
+ else:
+ config_file = args.config_file
+ if not config_file.is_file():
+ raise FileNotFoundError(f"Config file not found: {config_file}")
+
+ result = run(
+ workdir=workdir,
+ model=args.model,
+ task=TASK_REGISTRY[args.task](config_file=config_file),
+ max_rounds=args.max_rounds,
+ training_iterations=args.training_iterations,
+ )
+ if result is not None:
+ logger.info(
+ "task=%s passed=%s rounds_run=%d", args.task, result.passed, result.rounds_run
+ )
+
+if __name__ == "__main__":
+ main()
diff --git a/src/microbots/auto_memory/eval/swebenchverified.py b/src/microbots/auto_memory/eval/swebenchverified.py
new file mode 100644
index 0000000..31d2866
--- /dev/null
+++ b/src/microbots/auto_memory/eval/swebenchverified.py
@@ -0,0 +1,551 @@
+"""SWE-bench-verified eval task.
+
+Loads instances from the SWE-bench-verified dataset, checks out each
+instance's repo at its base commit, has the agent attempt a fix, and
+verifies the result via ``swebench.harness.run_evaluation``.
+"""
+
+import json
+import shutil
+import subprocess
+import sys
+import tempfile
+import uuid
+from dataclasses import dataclass
+from functools import cache
+from logging import getLogger
+from pathlib import Path
+
+import yaml
+
+from microbots.auto_memory.evalTask import EvalOutcome, EvalTask
+from microbots.auto_memory.task_registry import register_task
+from microbots.bot.ReadingBot import ReadingBot
+from microbots.bot.WritingBot import WritingBot
+from microbots.MicroBot import BotRunResult
+from microbots.tools.tool_definitions.memory_tool import MemoryTool
+
+logger = getLogger(__name__)
+
+SWE_BENCH_VERIFIED = "SWE-bench/SWE-bench_Verified"
+EVAL_AGENT_MODEL_NAME = "microbots-eval-agent"
+
+
+@dataclass
+class SweBenchInstance:
+ """A single SWE-bench-verified dataset row.
+
+ Attributes
+ ----------
+ instance_id : str
+ Unique identifier for the instance, e.g. ``"django__django-11099"``.
+ repo : str
+ The GitHub repo this instance belongs to, e.g. ``"django/django"``.
+ base_commit : str
+ Commit hash representing the repo state before the issue's fix.
+ problem_statement : str
+ The GitHub issue title and body describing the bug to fix.
+ """
+
+ instance_id: str
+ repo: str
+ base_commit: str
+ problem_statement: str
+
+
+@cache
+def _load_dataset_rows(dataset_name: str):
+ """Load and cache ``dataset_name``'s ``test`` split for the process's lifetime.
+
+ ``load_dataset`` caches the downloaded files on disk, but still
+ re-reads and rebuilds the in-memory ``Dataset`` object on every
+ call. Since ``load_instances_of_repo``/``load_instance_using_id``
+ may each be called many times,
+ this wraps ``load_dataset`` with an in-memory cache keyed by
+ ``dataset_name``, so the dataset is only loaded once per process.
+
+ Parameters
+ ----------
+ dataset_name : str
+ Hugging Face dataset name to load.
+
+ Returns
+ -------
+ datasets.Dataset
+ The loaded ``test`` split.
+
+ Raises
+ ------
+ ImportError
+ If the optional ``datasets`` package (the ``training`` extra)
+ isn't installed.
+ """
+ try:
+ from datasets import load_dataset
+ except ImportError as exc:
+ raise ImportError(
+ "SWE-bench-verified evaluation requires the 'training' extra: "
+ "pip install 'microbots[training]'"
+ ) from exc
+ return load_dataset(dataset_name, split="test")
+
+
+def load_instances_of_repo(
+ dataset_name: str = SWE_BENCH_VERIFIED,
+ repo: str | None = None,
+) -> list[SweBenchInstance]:
+ """Load all dataset instances, optionally filtered to a single repo.
+
+ Parameters
+ ----------
+ dataset_name : str
+ Hugging Face dataset name to load. Defaults to
+ ``SWE_BENCH_VERIFIED``.
+ repo : str | None
+ If given, only instances whose ``repo`` matches this value are
+ returned, e.g. ``"django/django"``. If ``None``, all instances
+ are returned.
+
+ Returns
+ -------
+ list[SweBenchInstance]
+ The matching instances.
+ """
+ rows = _load_dataset_rows(dataset_name)
+ instances = [
+ SweBenchInstance(
+ instance_id=row["instance_id"],
+ repo=row["repo"],
+ base_commit=row["base_commit"],
+ problem_statement=row["problem_statement"],
+ )
+ for row in rows
+ if repo is None or row["repo"] == repo
+ ]
+ return instances
+
+def load_instance_using_id(instance_id: str, dataset_name: str = SWE_BENCH_VERIFIED) -> SweBenchInstance:
+ """Load a single dataset instance by its instance ID.
+
+ Parameters
+ ----------
+ instance_id : str
+ The instance ID to look up, e.g. ``"django__django-11099"``.
+ dataset_name : str
+ Hugging Face dataset name to load. Defaults to
+ ``SWE_BENCH_VERIFIED``.
+
+ Returns
+ -------
+ SweBenchInstance
+ The matching instance.
+
+ Raises
+ ------
+ ValueError
+ If no instance with the given ``instance_id`` exists in the
+ dataset.
+ """
+ rows = _load_dataset_rows(dataset_name)
+ for row in rows:
+ if row["instance_id"] == instance_id:
+ return SweBenchInstance(
+ instance_id=row["instance_id"],
+ repo=row["repo"],
+ base_commit=row["base_commit"],
+ problem_statement=row["problem_statement"],
+ )
+ raise ValueError(f"instance_id not found: {instance_id}")
+
+class SweBenchVerifiedTask_one():
+ """SWE-bench-verified based evaluation task.
+
+ Checks out the instance's repo at its base commit, gives the agent
+ the issue's problem statement, and verifies the agent's patch using
+ the official SWE-bench evaluation harness.
+
+ Parameters
+ ----------
+ instance : SweBenchInstance
+ The dataset instance this task evaluates against.
+ """
+
+ def __init__(self, instance: SweBenchInstance | None = None):
+ """Initialize the task, optionally for a single dataset instance.
+
+ Parameters
+ ----------
+ instance : SweBenchInstance | None
+ The dataset instance this task evaluates against. May be
+ omitted and set later via ``self.instance``, but must be
+ set before any other method on this task is called.
+ """
+ self.instance = instance
+
+ @property
+ def task_id(self) -> str:
+ """Return this instance's SWE-bench-verified ``instance_id``.
+
+ Returns
+ -------
+ str
+ The dataset instance's ``instance_id``.
+ """
+ return self.instance.instance_id
+
+ def setup(self, repo_path: str) -> None:
+ """Clone the instance's repo, or reset it, to its base commit.
+
+ Parameters
+ ----------
+ repo_path : str
+ Absolute path to clone (or reset) the repo into.
+ """
+ expected_url = f"https://github.com/{self.instance.repo}.git"
+
+ if Path(repo_path).exists():
+ origin = subprocess.run(
+ ["git", "remote", "get-url", "origin"],
+ cwd=repo_path, capture_output=True, text=True,
+ )
+ if origin.returncode == 0 and origin.stdout.strip() == expected_url:
+ subprocess.run(
+ ["git", "reset", "--hard", self.instance.base_commit],
+ cwd=repo_path, check=True,
+ )
+ subprocess.run(["git", "clean", "-fd"], cwd=repo_path, check=True)
+ return
+
+ logger.warning(
+ "SweBenchVerifiedTask.setup: %s exists but isn't a checkout of %s "
+ "(origin=%r); removing and re-cloning",
+ repo_path, expected_url, origin.stdout.strip(),
+ )
+ shutil.rmtree(repo_path)
+
+ subprocess.run(["git", "clone", expected_url, repo_path], check=True)
+ subprocess.run(
+ ["git", "checkout", self.instance.base_commit], cwd=repo_path, check=True
+ )
+
+ def build_prompt(self) -> str:
+ """Return the instance's issue text as the agent's prompt.
+
+ Returns
+ -------
+ str
+ The instance's ``problem_statement``.
+ """
+ return self.instance.problem_statement
+
+ def check(self, repo_path: str, agent_output: str, log_path: str) -> BotRunResult:
+ """Verify the agent's patch using the SWE-bench evaluation harness.
+
+ Captures the agent's changes as a git diff (after marking any
+ untracked new files intent-to-add, so files the agent newly
+ created are included in the diff rather than silently
+ dropped), submits it as a prediction to
+ ``swebench.harness.run_evaluation``, and checks whether the
+ harness marked this instance as resolved.
+
+ Parameters
+ ----------
+ repo_path : str
+ Absolute path to the repo the agent operated on.
+ agent_output : str
+ The agent's raw output/result text. Unused here, since
+ verification is based on the repo's git diff, not the
+ agent's textual output.
+ log_path : str
+ Path to a log file to append the harness's output to,
+ including the per-instance ``run_instance.log`` and
+ ``test_output.txt`` artifacts if the harness produced them
+ (read before the harness's ``report_dir`` is cleaned up).
+
+ Returns
+ -------
+ BotRunResult
+ ``status`` is whether the harness marked this instance as
+ resolved. On failure, ``error`` carries the harness's
+ ``test_output.txt`` (or its console output, if the harness
+ died before producing one) so the feedback bot can see why
+ the tests failed.
+ """
+ subprocess.run(
+ ["git", "add", "--intent-to-add", "."], cwd=repo_path, check=True
+ )
+ diff = subprocess.run(
+ ["git", "diff", "--binary"],
+ cwd=repo_path,
+ capture_output=True,
+ text=True,
+ check=True,
+ ).stdout
+
+ run_id = f"microbots-{uuid.uuid4().hex[:8]}"
+ model_name_or_path = EVAL_AGENT_MODEL_NAME
+ #will need to update when upgraded to ~5.0.2 , removed this flag in the new version
+ #https://github.com/SWE-bench/SWE-bench/commit/e2c13307b6cf7764a50958b9c8bfbfb3f72cb70a
+ report_dir = Path(tempfile.mkdtemp())
+ with tempfile.NamedTemporaryFile(
+ mode="w", suffix=".json", delete=False
+ ) as pred_file:
+ pred_path = Path(pred_file.name)
+ pred_file.write(json.dumps([{
+ "instance_id": self.instance.instance_id,
+ "model_patch": diff,
+ "model_name_or_path": model_name_or_path,
+ }]))
+
+ try:
+ proc = subprocess.run(
+ [sys.executable, "-m", "swebench.harness.run_evaluation",
+ "--dataset_name", SWE_BENCH_VERIFIED,
+ "--max_workers", "1",
+ "--predictions_path", str(pred_path),
+ "--run_id", run_id,
+ "--report_dir", str(report_dir),
+ "--instance_ids", self.instance.instance_id],
+ #can add timeout if needed
+ capture_output=True, text=True,
+ cwd=report_dir,
+ )
+ #need to update this instance_log_dir path when swebench is upgraded
+ instance_log_dir = (
+ report_dir / "logs" / "run_evaluation" / run_id
+ / model_name_or_path / self.instance.instance_id
+ )
+ # Read while report_dir still exists; the finally block deletes it.
+ test_output = ""
+ with open(log_path, "a") as f:
+ f.write(proc.stdout + proc.stderr)
+ for log_filename in ("run_instance.log", "test_output.txt"):
+ log_file = instance_log_dir / log_filename
+ if log_file.exists():
+ content = log_file.read_text()
+ if log_filename == "test_output.txt":
+ test_output = content
+ f.write(f"\n--- {log_filename} ---\n{content}\n")
+
+ report_file = instance_log_dir / "report.json"
+ passed = False
+ if report_file.exists():
+ report = json.loads(report_file.read_text())
+ passed = report.get(self.instance.instance_id, {}).get("resolved", False)
+ finally:
+ pred_path.unlink(missing_ok=True)
+ shutil.rmtree(report_dir, ignore_errors=True)
+
+ return BotRunResult(
+ status = passed,
+ result = "resolved" if passed else "not resolved",
+ # Harness can fail before producing test_output.txt; fall back to its console output.
+ error = None if passed else (test_output or proc.stdout + proc.stderr)
+ )
+
+ def eval(self, repo_path: str, memory_dir: str, model: str, log_path: str) -> BotRunResult:
+ """Run one eval iteration: setup -> build_prompt -> WritingBot -> check.
+
+ Parameters
+ ----------
+ repo_path : str
+ Absolute path to the repo to run the eval round against.
+ memory_dir : str
+ Directory containing memory files to give the agent via
+ ``MemoryTool``.
+ model : str
+ The model to use, in the format ``/``.
+ log_path : str
+ Path to write this round's log to. Caller-provided, so the
+ log persists under the run's own layout.
+
+ Returns
+ -------
+ BotRunResult
+ The result of this eval round, including the agent's output,
+ the check verdict.
+ """
+ Path(log_path).parent.mkdir(parents=True, exist_ok=True)
+ Path(log_path).write_text("")
+
+ try:
+ self.setup(repo_path)
+ prompt = self.build_prompt()
+ bot = WritingBot(
+ model=model,
+ folder_to_mount=repo_path,
+ additional_tools=[MemoryTool(memory_dir=memory_dir)],
+ )
+ bot_result = bot.run(prompt)
+
+ with open(log_path, "a") as f:
+ f.write(f"Agent output:\n{bot_result.result}\n")
+
+ return bot_result
+
+ except Exception as exc:
+ logger.exception(
+ "SweBenchVerifiedTask.eval: iteration raised %s", type(exc).__name__
+ )
+ with open(log_path, "a") as f:
+ f.write(f"\nException during eval iteration: {type(exc).__name__}: {exc}\n")
+ return BotRunResult(
+ status=False,
+ result=None,
+ error=f"{type(exc).__name__}: {exc}"
+ )
+
+
+@register_task("swebenchverified")
+class SweBenchVerified(EvalTask):
+ """SWE-bench-verified based evaluation task.
+
+ It takes the memory provided by the training agent and runs
+ all the selected SWE-bench-verified instances. Then provides
+ a combined score and feedback.
+ """
+
+ def __init__(self, config_file: Path) -> None:
+ super().__init__(config_file)
+ self.dataset: list[SweBenchInstance] = []
+ self.parse_config(config_file=config_file)
+
+ def repo_url(self) -> str:
+ """Return the URL of the repo for the training agent.
+
+ Returns:
+ str: The URL of the repo for the training agent.
+ """
+ return f"https://github.com/{self.dataset[0].repo}.git"
+
+ def teardown(self, eval_repo_path: Path) -> None:
+ """Tear down the task, cleaning up any resources if necessary."""
+ if eval_repo_path and eval_repo_path.exists():
+ shutil.rmtree(eval_repo_path)
+
+ def parse_config(self, config_file: Path) -> None:
+ """Parse the configuration file for the task.
+ The config file is a yaml file. It will have array of "instance_id"
+ or "repo" as the root object. Gather it and load the dataset to
+ the object variable dataset.
+
+ Args:
+ config_file (Path): Path to the configuration file.
+ """
+
+ with open(config_file, "r") as f:
+ config = yaml.safe_load(f)
+
+ instance_ids = config.get("instance_id_list", [])
+ repo = config.get("repo", None)
+
+ if instance_ids:
+ repo = None
+ for instance_id in instance_ids:
+ dataset = load_instance_using_id(instance_id)
+ if not repo:
+ repo = dataset.repo
+ elif repo != dataset.repo:
+ raise ValueError(
+ f"Conflicting repos for instance_id {instance_id}: {repo} vs {dataset.repo}"
+ )
+
+ self.dataset.append(dataset)
+
+ elif repo:
+ self.dataset = load_instances_of_repo(repo=repo)
+
+ if len(self.dataset) == 0:
+ raise ValueError("No instances loaded for evaluation.")
+
+ def eval(self, memory_dir: str, model: str, log_path: str) -> EvalOutcome:
+ """Runs the evaluation agent with the memory on all the eval instances
+ and produces a cumulative feedback.
+
+ Args:
+ memory_dir (str): Path to the directory containing the agent's memory.
+ model (str): The model identifier used for evaluation.
+ log_path (str): Path to the log file for recording evaluation details.
+
+ Returns:
+ EvalOutcome: The outcome of the evaluation, including whether it passed, the output, and the result.
+ """
+
+ eval_repo_path = Path(log_path).parent / "eval_repo"
+ results = []
+
+ for instance in self.dataset:
+ inst_log_path = Path(log_path).parent / f"{instance.instance_id}_log.txt"
+ task = SweBenchVerifiedTask_one(instance)
+
+ res = task.eval(str(eval_repo_path), memory_dir, model, str(inst_log_path))
+
+ if not res.status:
+ logger.info(f"Evaluation failed for instance {instance.instance_id}: {res.error if res.error else 'Unknown error'}")
+ results.append(res)
+ else:
+ res = task.check(str(eval_repo_path), "", str(inst_log_path))
+ results.append(res)
+
+ score = 0
+ for result in results:
+ if result.status:
+ score += 1
+
+ score = score / len(self.dataset)
+
+ if score == 1:
+ feedback = "All evaluations passed."
+ else:
+ feedback = self._combine_result_feedback(results, model, str(eval_repo_path))
+
+ self.teardown(eval_repo_path)
+
+ return EvalOutcome(
+ passed = score == 1,
+ score = score,
+ feedback = feedback
+ )
+
+
+ def _combine_result_feedback(self, results: list[BotRunResult], model: str, eval_repo: str) -> str:
+ """
+ Combines the feedback from multiple BotRunResult instances into a single feedback string.
+ Args:
+ results (list[BotRunResult]): List of individual bot run results.
+ model (str): The model identifier used for evaluation.
+ eval_repo (str): Path to the evaluation repository.
+
+ Returns:
+ str: Combined feedback from all results.
+ """
+
+ serialized_str = f"Total {len(results)} tests ran and their result and feedback:\n"
+
+ for res in results:
+ serialized_str += f"\nResult: {'Passed' if res.status else 'Failed'}\n"
+ serialized_str += f"Optional Feedback: {res.result if res.result else 'None'}\n"
+ serialized_str += f"Error if there are any: {res.error if res.error else 'None'}\n"
+
+ try:
+ bot = ReadingBot(
+ model = model,
+ folder_to_mount=eval_repo
+ )
+ task = f"""
+ Combine the results of the eval runs into single feedback.
+ This feedback will be given to the next iteration.
+ You just combine the results with minimal efforts.
+ Avoid referring to code whenever possible.
+
+ {serialized_str}
+ """
+ bot_result = bot.run(task=task)
+ except Exception as e:
+ logger.warning(f"Combining results failed with exception: {e}")
+ return f"Combining results failed. raw combined output:\n\n{serialized_str}"
+
+ if bot_result.status:
+ return bot_result.result if bot_result.result else 'No feedback provided'
+ else:
+ return f"Combining results failed. raw combined output:\n\n{serialized_str}"
diff --git a/src/microbots/auto_memory/evalTask.py b/src/microbots/auto_memory/evalTask.py
new file mode 100644
index 0000000..fbe1290
--- /dev/null
+++ b/src/microbots/auto_memory/evalTask.py
@@ -0,0 +1,121 @@
+"""Defines the abstract eval task interface for the train <-> eval loop.
+
+An ``EvalTask`` describes one unit of work: how to prepare a repo, what
+prompt to give the agent, how to verify the agent's output, and how to
+clean up afterward.
+"""
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+
+@dataclass
+class EvalOutcome:
+ """Result of verifying whether an eval task was completed correctly.
+
+ Attributes
+ ----------
+ passed : bool
+ Whether the agent's output satisfies the task's check.
+ score : float
+ A numeric score representing the quality of the agent's output.
+ feedback : str
+ A short human-readable explanation of the pass/fail verdict.
+ """
+
+ passed: bool
+ score: float
+ feedback: str
+
+class EvalTask(ABC):
+ """Base class for a single evaluation task in the train <-> eval loop.
+
+ Subclasses must implement ``run``. ``parse_config``, ``setup``,
+ ``check``, and ``teardown`` are optional hooks
+ subclasses may use to structure their own ``run`` implementation
+ (see ``SweBenchVerifiedTask`` for an example), but nothing in this
+ base class calls them automatically.
+ """
+
+ def __init__(self, config_file: Path) -> None:
+ super().__init__()
+
+ @abstractmethod
+ def repo_url(self) -> str:
+ """Return the URL of the repo for the training agent."""
+
+ @property
+ def task_id(self) -> str:
+ """Identifier for this task instance, used to name its output folder.
+
+ Defaults to the class name, which is fine for tasks with only
+ one instance per run. Override for tasks with several distinct
+ instances per class (e.g. ``SweBenchVerifiedTask``, where each
+ dataset row needs its own folder).
+
+ Returns
+ -------
+ str
+ This task instance's identifier.
+ """
+ return type(self).__name__
+
+ def setup(self) -> None:
+ """Optional. Prepare repo/environment before the agent runs.
+
+ Not called automatically; only useful if your ``run``
+ implementation calls it.
+
+ Parameters
+ ----------
+ repo_path : str
+ Absolute path to the repo to prepare.
+ """
+ pass
+
+ def teardown(self, eval_repo_path: Path) -> None:
+ """Optional. Clean up anything setup() created.
+
+ Parameters
+ ----------
+ eval_repo_path : Path
+ Absolute path to the repo that was prepared by ``setup``.
+ """
+ pass
+
+ @abstractmethod
+ def parse_config(self, config_file: Path) -> None:
+ """Parse the task-specific config file. Importantly it
+ parses the config file and get the repo for the training
+ agent.
+
+ Parameters
+ ----------
+ config_file : Path
+ Path to the config file to parse.
+ """
+
+ @abstractmethod
+ def eval(self, memory_dir: str, model: str, log_path: str) -> EvalOutcome:
+ """Required. Run one eval iteration and return its outcome.
+
+ Parameters
+ ----------
+ memory_dir : str
+ Directory containing memory files to give the agent via
+ ``MemoryTool``.
+ model : str
+ The model to use, in the format ``/``.
+ log_path : str
+ Path to write this round's log to. Caller-provided (e.g. a
+ workdir-managed path) so logs persist under the run's
+ layout instead of each task inventing its own temp file.
+
+ Returns
+ -------
+ EvalOutcome
+ The result of this eval round, including the agent's output,
+ the check verdict.
+ """
diff --git a/src/microbots/auto_memory/orchestrator.py b/src/microbots/auto_memory/orchestrator.py
new file mode 100644
index 0000000..4691837
--- /dev/null
+++ b/src/microbots/auto_memory/orchestrator.py
@@ -0,0 +1,331 @@
+"""Orchestrates the train <-> eval loop for repo-learning agents.
+
+Repeatedly runs an ``EvalTask`` against a repo, and on failure builds
+feedback and retrains via the training agent, looping until the task
+passes or ``max_rounds`` is exhausted.
+"""
+
+from dataclasses import dataclass, field
+import dataclasses
+from logging import getLogger
+from pathlib import Path
+import json
+import shutil
+import subprocess
+
+from microbots.auto_memory.evalTask import EvalOutcome, EvalTask
+from microbots.auto_memory.training.runner import run_training
+from microbots.auto_memory.workdir import (
+ eval_log_path,
+ eval_result_path,
+ load_round_memory,
+ repo_dir,
+ save_round_memory,
+ snapshot_seed_memory,
+)
+
+logger = getLogger(__name__)
+
+@dataclass
+class LoopResult:
+ """Result of running ``run_train_eval_loop``.
+
+ Attributes
+ ----------
+ passed : bool
+ Whether the task passed within ``max_rounds``.
+ rounds_run : int
+ Number of eval rounds actually run.
+ final_outcome : EvalOutcome
+ The outcome of the last round run.
+ outcomes : list[EvalOutcome]
+ The outcome of every round run, in order.
+ """
+
+ passed: bool
+ rounds_run: int
+ final_outcome: EvalOutcome
+ outcomes: list[EvalOutcome] = field(default_factory=list)
+
+def clone_repo(url: str, repo_path: Path) -> None:
+ """Clone ``url`` into ``repo_path``, or reuse it if already cloned from ``url``.
+
+ Existence alone isn't enough to trust ``repo_path``: it could be an
+ empty/partial directory left by a previous failed clone, or a
+ reused workdir whose config now points at a different ``url``. So
+ if ``repo_path`` exists, its ``origin`` remote is checked against
+ ``url`` first. Only an exact match is reused as-is; anything else
+ (mismatched origin, or not a git checkout at all) is removed and
+ re-cloned, so training never silently runs against missing or
+ wrong code.
+
+ Parameters
+ ----------
+ url : str
+ Git URL (or local path) to clone from.
+ repo_path : Path
+ Destination directory for the clone.
+ """
+ if repo_path.exists():
+ origin = subprocess.run(
+ ["git", "remote", "get-url", "origin"],
+ cwd=repo_path, capture_output=True, text=True,
+ )
+ if origin.returncode == 0 and origin.stdout.strip() == url:
+ return
+
+ logger.warning(
+ "clone_repo: %s exists but isn't a checkout of %s (origin=%r); "
+ "removing and re-cloning",
+ repo_path, url, origin.stdout.strip(),
+ )
+ shutil.rmtree(repo_path)
+
+ subprocess.run(["git", "clone", url, str(repo_path)], check=True)
+
+def write_eval_result(workdir: Path, round_num: int, task: EvalTask, outcome: EvalOutcome) -> None:
+ """Write a round's eval result to ``result.json``.
+
+ Parameters
+ ----------
+ workdir : Path
+ The run's workdir.
+ round_num : int
+ 1-based round number this outcome belongs to.
+ task : EvalTask
+ The task that produced ``outcome``, used for both its
+ ``task_id`` (folder name) and ``build_result`` (file content).
+ outcome : EvalOutcome
+ The round's outcome to persist.
+ """
+ path = eval_result_path(workdir, round_num, task.task_id)
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(json.dumps(dataclasses.asdict(outcome), indent=2))
+
+def run_training_loop(
+ repo_path: str,
+ feedback: str,
+ memory_dir: str,
+ model: str,
+ iterations: int = 10,
+) -> None:
+ """Run ``run_training`` ``iterations`` times, reusing the same memory dir.
+
+ Shared by the eval-loop's retrain step and any training-only entry
+ point (e.g. a CLI) that needs to run training without an eval task.
+
+ Parameters
+ ----------
+ repo_path : str
+ Absolute path to the repo to train against.
+ feedback : str
+ Feedback from a prior failed eval attempt, or ``""`` if none.
+ memory_dir : str
+ Directory where the training agent reads/writes memory files.
+ model : str
+ The model to use, in the format ``/``.
+ iterations : int
+ Number of training passes to run, each reusing the same
+ ``memory_dir``. Defaults to 10.
+ """
+ for iteration in range(1, iterations + 1):
+ logger.info(
+ "run_training_loop: training iteration %d/%d",
+ iteration,
+ iterations,
+ )
+ run_training(
+ repo_path=repo_path,
+ feedback=feedback,
+ memory_dir=memory_dir,
+ model=model,
+ )
+
+def run_train_eval_loop(
+ training_repo_path: str,
+ workdir: Path,
+ model: str,
+ task: EvalTask,
+ max_rounds: int = 5,
+ training_iterations: int = 10,
+) -> LoopResult:
+ """Run an eval task in a loop, retraining on failure until it passes.
+
+ Each round loads the current top-level memory into its own
+ ``rounds_/round_N/memory`` (carried forward from the
+ previous round, or empty on round 1), then runs ``task.run(...)``
+ against it, writing its log to a workdir-managed path
+ (``rounds_/round_N/eval/eval.log``) so it persists. Since
+ each eval task instance gets its own ``rounds_`` dir,
+ different instances sharing the same ``workdir`` never collide on
+ round numbers, and each instance's per-round memory is preserved
+ individually. If the task passes, the loop returns immediately. If
+ it fails, feedback is built from the round's log and used to
+ retrain via ``run_training`` (called ``training_iterations`` times,
+ each pass reusing the same round memory dir) before the next round.
+ Either way, the round's result is written to ``result.json`` and
+ its memory is saved back to the top-level memory dir before the
+ next round starts.
+
+ Parameters
+ ----------
+ training_repo_path : str
+ Absolute path to the persistent repo checkout used only for
+ retraining (``run_training_loop``). Kept separate from
+ ``eval_repo_path`` since the task manages the latter's
+ lifecycle itself (clone/teardown each round).
+ eval_repo_path : str
+ Absolute path to the repo the task clones/manages itself (via
+ its own ``setup``) and runs/checks the agent against each
+ round.
+ workdir : Path
+ This run's workdir, used to carry memory forward between rounds
+ (see ``microbots.auto_memory.workdir``).
+ model : str
+ The model to use, in the format ``/``.
+ task : EvalTask
+ The eval task to run each round.
+ max_rounds : int
+ Maximum number of train/eval rounds to attempt. Defaults to 5.
+ training_iterations : int
+ Number of training passes to run per retraining round, each
+ reusing the same round memory dir. Defaults to 10.
+
+ Returns
+ -------
+ LoopResult
+ Whether the task passed, how many rounds ran, and every round's
+ outcome.
+
+ Raises
+ ------
+ ValueError
+ If ``max_rounds`` is less than 1.
+ """
+ if max_rounds < 1:
+ raise ValueError(f"max_rounds must be >= 1, got {max_rounds}")
+
+ outcomes: list[EvalOutcome] = []
+
+ for round_idx in range(1, max_rounds+1):
+ logger.info(
+ "run_train_eval_loop: round %d/%d starting", round_idx, max_rounds
+ )
+ # TODO: Instead of loading new memory dir on every iteration, snapshot the memory.
+ memory_dir = str(load_round_memory(workdir, round_idx, instance_id=task.task_id))
+ log_path = str(eval_log_path(workdir, round_idx, task.task_id))
+ outcome = task.eval(memory_dir, model, log_path)
+ outcomes.append(outcome)
+
+ try:
+ if outcome.passed:
+ logger.info(
+ "run_train_eval_loop: passed on round %d/%d", round_idx, max_rounds
+ )
+ return LoopResult(
+ passed=True,
+ rounds_run=round_idx,
+ final_outcome=outcome,
+ outcomes=outcomes,
+ )
+
+ logger.info(
+ "run_train_eval_loop: round %d failed (%s), retraining",
+ round_idx,
+ outcome.feedback,
+ )
+ try:
+ run_training_loop(
+ repo_path=training_repo_path,
+ feedback=outcome.feedback,
+ memory_dir=memory_dir,
+ model=model,
+ iterations=training_iterations,
+ )
+ except Exception:
+ logger.exception(
+ "run_train_eval_loop: round %d failed to build feedback/retrain; "
+ "continuing to next round without retraining",
+ round_idx,
+ )
+ finally:
+ write_eval_result(workdir, round_idx, task, outcome)
+ save_round_memory(workdir, round_idx, instance_id=task.task_id)
+
+ logger.info(
+ "run_train_eval_loop: exhausted %d rounds without passing", max_rounds
+ )
+ return LoopResult(
+ passed=False,
+ rounds_run=max_rounds,
+ final_outcome=outcomes[-1],
+ outcomes=outcomes,
+ )
+
+def run(
+ workdir: Path,
+ model: str,
+ task: EvalTask,
+ max_rounds: int = 5,
+ training_iterations: int = 10,
+) -> LoopResult:
+ """Run full train/eval loop, depending on ``task``.
+
+ Parameters
+ ----------
+ workdir : Path
+ This run's workdir (see ``microbots.auto_memory.workdir``),
+ holding ``task_config.yaml``, the shared repo clone, and all output.
+ model : str
+ The model to use, in the format ``/``.
+ task : EvalTask
+ The eval task to run each round.
+ max_rounds : int
+ Maximum number of train/eval rounds to attempt. Defaults to 5.
+ training_iterations : int
+ Number of training passes to run per retraining round, each
+ reusing the same round memory dir. Defaults to 10.
+
+ Returns
+ -------
+ LoopResult
+ The eval loop's result.
+
+ Raises
+ ------
+ ValueError
+ If ``config`` has no ``repo`` entry. Every run needs a training
+ checkout (``run_training_loop`` always mounts
+ ``training_repo_path``, regardless of ``task``), so ``repo``
+ must be configured even for tasks like ``SweBenchVerifiedTask``
+ that manage their own separate eval checkout.
+ """
+ clone_repo(task.repo_url(), repo_dir(workdir))
+
+ snapshot_seed_memory(workdir)
+
+ training_repo_path = str(repo_dir(workdir))
+
+ # TODO: train-only mode will be implemented if required after proper design
+ # if task is None:
+ # # Train-only mode has no rounds of its own; round 1 is just a
+ # # scratch dir seeded from (and saved back to) top-level memory.
+ # memory_dir = str(load_round_memory(workdir, 1))
+ # run_training_loop(
+ # repo_path=training_repo_path,
+ # feedback="",
+ # memory_dir=memory_dir,
+ # model=model,
+ # iterations=training_iterations,
+ # )
+ # save_round_memory(workdir, 1)
+ # return None
+
+ return run_train_eval_loop(
+ training_repo_path=training_repo_path,
+ workdir=workdir,
+ model=model,
+ task=task,
+ max_rounds=max_rounds,
+ training_iterations=training_iterations,
+ )
diff --git a/src/microbots/auto_memory/task_registry.py b/src/microbots/auto_memory/task_registry.py
new file mode 100644
index 0000000..1846d2d
--- /dev/null
+++ b/src/microbots/auto_memory/task_registry.py
@@ -0,0 +1,109 @@
+"""Registry for constructing ``EvalTask`` instances by name.
+
+Tasks self-register via the ``@register_task`` decorator, so new task
+types can be added without editing a central if/elif factory function.
+Callers (e.g. a CLI) look tasks up by name via ``create_task``.
+"""
+
+import importlib
+import pkgutil
+from collections.abc import Callable
+
+from microbots.auto_memory.evalTask import EvalTask
+
+TASK_REGISTRY: dict[str, type[EvalTask]] = {}
+
+def register_task(name: str) -> Callable[[type[EvalTask]], type[EvalTask]]:
+ """Register an ``EvalTask`` subclass under ``name`` as a class decorator.
+
+ Each name maps to exactly one class; registering a name twice is a
+ programming error rather than a silent overwrite.
+
+ Parameters
+ ----------
+ name : str
+ The key other code will use to look up this task via
+ ``create_task``, e.g. ``"swebenchverified"``.
+
+ Returns
+ -------
+ Callable[[type[EvalTask]], type[EvalTask]]
+ A decorator that registers the class in ``TASK_REGISTRY`` and
+ returns it unchanged.
+ """
+
+ def decorator(task_cls: type[EvalTask]) -> type[EvalTask]:
+ """Register ``task_cls`` in ``TASK_REGISTRY`` under the enclosing ``name``.
+
+ Parameters
+ ----------
+ task_cls : type[EvalTask]
+ The ``EvalTask`` subclass to register.
+
+ Returns
+ -------
+ type[EvalTask]
+ ``task_cls``, unchanged.
+
+ Raises
+ ------
+ ValueError
+ If ``name`` is already registered to a different class.
+ """
+ registered = TASK_REGISTRY.get(name)
+ if registered is not None and registered is not task_cls:
+ raise ValueError(
+ f"Task name {name!r} is already registered to "
+ f"{registered.__module__}.{registered.__qualname__}; "
+ f"cannot also register {task_cls.__module__}.{task_cls.__qualname__}."
+ )
+ TASK_REGISTRY[name] = task_cls
+ return task_cls
+
+ return decorator
+
+def create_task(name: str) -> EvalTask:
+ """Construct the registered ``EvalTask`` for ``name``.
+
+ Tasks take no constructor arguments; per-run configuration is
+ applied afterwards via ``EvalTask.parse_config``.
+
+ Parameters
+ ----------
+ name : str
+ The registered task name, e.g. ``"swebenchverified"``.
+
+ Returns
+ -------
+ EvalTask
+ A new instance of the class registered under ``name``.
+
+ Raises
+ ------
+ ValueError
+ If ``name`` has not been registered via ``register_task``.
+ """
+ try:
+ task_cls = TASK_REGISTRY[name]
+ except KeyError:
+ raise ValueError(
+ f"Unknown task {name!r}. Registered tasks: {sorted(TASK_REGISTRY)}"
+ ) from None
+ return task_cls()
+
+def discover_tasks(package_name: str = "microbots.auto_memory.eval") -> None:
+ """Import every module in ``package_name`` so ``@register_task`` fires.
+
+ Adding a new task only requires dropping a new module into this
+ package (with its own ``@register_task`` decorator) — no other code
+ needs to change to make it discoverable.
+
+ Parameters
+ ----------
+ package_name : str
+ Dotted path of the package to scan for task modules. Defaults
+ to ``"microbots.auto_memory.eval"``.
+ """
+ package = importlib.import_module(package_name)
+ for module_info in pkgutil.iter_modules(package.__path__):
+ importlib.import_module(f"{package_name}.{module_info.name}")
diff --git a/src/microbots/auto_memory/workdir.py b/src/microbots/auto_memory/workdir.py
new file mode 100644
index 0000000..1276ff2
--- /dev/null
+++ b/src/microbots/auto_memory/workdir.py
@@ -0,0 +1,401 @@
+"""Path/layout helpers for a training run's workdir.
+
+Centralizes every path this package reads or writes under a run's
+``workdir`` (config, repo clone, logs, memory, and per-round/per-eval
+outputs), so callers never hard-code layout details themselves.
+"""
+
+from pathlib import Path
+import shutil
+
+import yaml
+
+WORKDIR_NAME = "workdir"
+CONFIG_FILENAME = "config.yaml"
+REPO_DIRNAME = "repo"
+EVAL_REPO_DIRNAME = "eval_repo"
+MEMORY_DIRNAME = "memory"
+MEMORY_SEED_DIRNAME = "memory_seed"
+ROUNDS_DIRNAME = "rounds"
+ROUND_LOG_FILENAME = "round.log"
+ROUND_PATCH_FILENAME = "repo.patch"
+EVAL_DIRNAME = "eval"
+RESULT_FILENAME = "result.json"
+EVAL_LOG_FILENAME = "eval.log"
+
+
+def resolve_workdir(base: Path | None = None) -> Path:
+ """Resolve the fixed workdir path relative to ``base``.
+
+ Parameters
+ ----------
+ base : Path | None
+ Directory to resolve ``workdir/`` relative to. Defaults to the
+ current working directory.
+
+ Returns
+ -------
+ Path
+ ``workdir`` resolved relative to ``base`` (or ``Path.cwd()``).
+ """
+ if base is not None and not base.is_absolute():
+ raise ValueError(f"base must be an absolute path: {base}")
+ workdir = (base or Path.cwd()) / WORKDIR_NAME
+ return workdir
+
+
+def require_workdir(workdir: Path) -> None:
+ """Validate that ``workdir`` exist. Create if not exist
+
+ Parameters
+ ----------
+ workdir : Path
+ The workdir to validate.
+
+ """
+ # create workdir if not existing
+ if not workdir.exists():
+ workdir.mkdir(parents=True)
+
+
+def config_path(workdir: Path) -> Path:
+ """Return the path to ``workdir``'s config file.
+
+ Parameters
+ ----------
+ workdir : Path
+ The run's workdir.
+
+ Returns
+ -------
+ Path
+ ``workdir/config.yaml``.
+ """
+ return workdir / CONFIG_FILENAME
+
+
+def load_config(workdir: Path) -> dict:
+ """Load and parse ``workdir``'s config file.
+
+ Parameters
+ ----------
+ workdir : Path
+ The workdir whose config file should be loaded.
+
+ Returns
+ -------
+ dict
+ The parsed config, or ``{}`` if the config file doesn't exist
+ or is empty.
+ """
+ path = config_path(workdir)
+ if not path.is_file():
+ return {}
+ return yaml.safe_load(path.read_text()) or {}
+
+
+def repo_dir(workdir: Path) -> Path:
+ """Return the path to the single cloned repo shared across rounds.
+
+ Used only for training (both train-only mode and the eval loop's
+ retrain step): a persistent checkout that stays in place across
+ rounds. Eval tasks that manage their own repo checkout (e.g.
+ ``SweBenchVerifiedTask``, which clones a different repo/commit per
+ dataset instance) use ``eval_repo_dir`` instead, so the two never
+ collide.
+
+ Parameters
+ ----------
+ workdir : Path
+ The run's workdir.
+
+ Returns
+ -------
+ Path
+ ``workdir/repo``.
+ """
+ return workdir / REPO_DIRNAME
+
+
+def memory_dir(workdir: Path) -> Path:
+ """Return the path to the current top-level (latest) memory directory.
+
+ Parameters
+ ----------
+ workdir : Path
+ The run's workdir.
+
+ Returns
+ -------
+ Path
+ ``workdir/memory``.
+ """
+ return workdir / MEMORY_DIRNAME
+
+
+def snapshot_seed_memory(workdir: Path) -> Path:
+ """Snapshot the current top-level memory dir as the run's restorable baseline.
+
+ ``memory_dir`` is shared and mutated in place across every
+ training/eval round and every eval task instance (so later
+ instances benefit from what earlier ones learned), which means the
+ original, pre-run memory is otherwise overwritten and lost with no
+ way to get back to it. Call this once, before anything trains,
+ to preserve that original state at ``workdir/memory_seed``. A
+ no-op if a snapshot already exists, so later calls (e.g. once per
+ eval task instance in the same run) never clobber the very first
+ snapshot with already-mutated memory.
+
+ Parameters
+ ----------
+ workdir : Path
+ The run's workdir.
+
+ Returns
+ -------
+ Path
+ ``workdir/memory_seed``, containing a copy of whatever
+ ``memory_dir`` held the first time this was called (or empty,
+ if there was no pre-existing memory).
+ """
+ dst = workdir / MEMORY_SEED_DIRNAME
+ if dst.exists():
+ return dst
+ src = memory_dir(workdir)
+ if src.is_dir():
+ shutil.copytree(src, dst)
+ else:
+ dst.mkdir(parents=True, exist_ok=True)
+ return dst
+
+
+def round_dir(
+ workdir: Path, round_num: int, *, instance_id: str | None = None, create: bool = False
+) -> Path:
+ """Return (and optionally create) the directory for a training round.
+
+ Parameters
+ ----------
+ workdir : Path
+ The run's workdir.
+ round_num : int
+ 1-based round number.
+ instance_id : str | None
+ If given, rounds are kept under a per-instance rounds dir
+ (``rounds_{instance_id}``) instead of the shared ``rounds`` dir,
+ so different eval task instances sharing the same ``workdir``
+ don't collide on round numbers. Pass the eval task's
+ ``task_id`` when running an eval task; omit for training-only
+ mode.
+ create : bool
+ If True, create the directory (and parents) if missing.
+
+ Returns
+ -------
+ Path
+ ``workdir/rounds/round_{round_num}`` (no ``instance_id``), or
+ ``workdir/rounds_{instance_id}/round_{round_num}``.
+ """
+ rounds_dirname = f"{ROUNDS_DIRNAME}_{instance_id}" if instance_id else ROUNDS_DIRNAME
+ path = workdir / rounds_dirname / f"round_{round_num}"
+ if create:
+ path.mkdir(parents=True, exist_ok=True)
+ return path
+
+
+def round_memory_dir(workdir: Path, round_num: int, *, instance_id: str | None = None) -> Path:
+ """Return the path to a round's own memory snapshot (a directory).
+
+ Parameters
+ ----------
+ workdir : Path
+ The run's workdir.
+ round_num : int
+ 1-based round number.
+ instance_id : str | None
+ The eval task's ``task_id``, if running an eval task (see
+ ``round_dir``). Omit for training-only mode.
+
+ Returns
+ -------
+ Path
+ This round's own memory directory.
+ """
+ return round_dir(workdir, round_num, instance_id=instance_id) / MEMORY_DIRNAME
+
+
+def load_round_memory(workdir: Path, round_num: int, *, instance_id: str | None = None) -> Path:
+ """Copy the current top-level memory into this round's own memory dir.
+
+ Called before a round's training pass, so it starts from whatever
+ memory the previous round left behind (or empty, on round 1). This
+ round's memory dir is replaced, not merged into: any stale files
+ left behind by a previous attempt at this same round (e.g. a
+ crashed/re-run process) are discarded first, so the round always
+ starts from an exact snapshot of the current top-level memory.
+
+ Parameters
+ ----------
+ workdir : Path
+ The run's workdir.
+ round_num : int
+ 1-based round number to load memory into.
+ instance_id : str | None
+ The eval task's ``task_id``, if running an eval task (see
+ ``round_dir``). Omit for training-only mode.
+
+ Returns
+ -------
+ Path
+ This round's own memory dir, ready for the round to use.
+ """
+ src = memory_dir(workdir)
+ dst = round_memory_dir(workdir, round_num, instance_id=instance_id)
+ shutil.rmtree(dst, ignore_errors=True)
+ if src.is_dir():
+ shutil.copytree(src, dst)
+ else:
+ dst.mkdir(parents=True, exist_ok=True)
+ return dst
+
+
+def save_round_memory(workdir: Path, round_num: int, *, instance_id: str | None = None) -> Path:
+ """Copy this round's memory back up to the top-level memory dir.
+
+ Called after a round's training pass, so later rounds (and the
+ final saved memory) see what this round learned. The top-level
+ memory dir is replaced, not merged into: files the round deleted
+ (e.g. via the agent's ``memory delete`` command) are gone from
+ the top level too, instead of surviving from a previous save.
+
+ Parameters
+ ----------
+ workdir : Path
+ The run's workdir.
+ round_num : int
+ 1-based round number whose memory should be saved.
+ instance_id : str | None
+ The eval task's ``task_id``, if running an eval task (see
+ ``round_dir``). Omit for training-only mode.
+
+ Returns
+ -------
+ Path
+ The top-level ``memory`` dir, now updated with this round's changes.
+ """
+ src = round_memory_dir(workdir, round_num, instance_id=instance_id)
+ dst = memory_dir(workdir)
+ shutil.rmtree(dst, ignore_errors=True)
+ if src.is_dir():
+ shutil.copytree(src, dst)
+ else:
+ dst.mkdir(parents=True, exist_ok=True)
+ return dst
+
+
+def round_log_path(workdir: Path, round_num: int, *, instance_id: str | None = None) -> Path:
+ """Return the path to a round's training log.
+
+ Parameters
+ ----------
+ workdir : Path
+ The run's workdir.
+ round_num : int
+ 1-based round number.
+ instance_id : str | None
+ The eval task's ``task_id``, if running an eval task (see
+ ``round_dir``). Omit for training-only mode.
+
+ Returns
+ -------
+ Path
+ This round's ``round.log``.
+ """
+ return round_dir(workdir, round_num, instance_id=instance_id) / ROUND_LOG_FILENAME
+
+
+def eval_dir(
+ workdir: Path, round_num: int, instance_id: str, *, create: bool = False
+) -> Path:
+ """Return (and optionally create) an eval task instance's eval directory.
+
+ Parameters
+ ----------
+ workdir : Path
+ The run's workdir.
+ round_num : int
+ 1-based round number this eval instance belongs to.
+ instance_id : str
+ The eval task instance identifier.
+ create : bool
+ If True, create the directory (and parents) if missing.
+
+ Returns
+ -------
+ Path
+ ``workdir/rounds_{instance_id}/round_{round_num}/eval``.
+ """
+ path = round_dir(workdir, round_num, instance_id=instance_id) / EVAL_DIRNAME
+ if create:
+ path.mkdir(parents=True, exist_ok=True)
+ return path
+
+
+def eval_result_path(workdir: Path, round_num: int, instance_id: str) -> Path:
+ """Return the path to an eval instance's result file.
+
+ Parameters
+ ----------
+ workdir : Path
+ The run's workdir.
+ round_num : int
+ 1-based round number this eval instance belongs to.
+ instance_id : str
+ The eval task instance identifier.
+
+ Returns
+ -------
+ Path
+ This eval instance's ``result.json``.
+ """
+ return eval_dir(workdir, round_num, instance_id) / RESULT_FILENAME
+
+
+def eval_log_path(workdir: Path, round_num: int, instance_id: str) -> Path:
+ """Return the path to an eval instance's log file.
+
+ Parameters
+ ----------
+ workdir : Path
+ The run's workdir.
+ round_num : int
+ 1-based round number this eval instance belongs to.
+ instance_id : str
+ The eval task instance identifier.
+
+ Returns
+ -------
+ Path
+ This eval instance's ``eval.log``.
+ """
+ return eval_dir(workdir, round_num, instance_id) / EVAL_LOG_FILENAME
+
+
+def eval_patch_path(workdir: Path, round_num: int, instance_id: str) -> Path:
+ """Return the path to an eval instance's captured repo diff.
+
+ Parameters
+ ----------
+ workdir : Path
+ The run's workdir.
+ round_num : int
+ 1-based round number this eval instance belongs to.
+ instance_id : str
+ The eval task instance identifier.
+
+ Returns
+ -------
+ Path
+ This eval instance's ``repo.patch``.
+ """
+ return eval_dir(workdir, round_num, instance_id) / ROUND_PATCH_FILENAME
\ No newline at end of file
diff --git a/src/microbots/auto_memory/workdir/config.yaml b/src/microbots/auto_memory/workdir/config.yaml
new file mode 100644
index 0000000..03de1e5
--- /dev/null
+++ b/src/microbots/auto_memory/workdir/config.yaml
@@ -0,0 +1,3 @@
+instance_id_list:
+ - astropy__astropy-12907
+ - astropy__astropy-13033
\ No newline at end of file
diff --git a/test/auto_memory/eval/test_swebenchverified.py b/test/auto_memory/eval/test_swebenchverified.py
new file mode 100644
index 0000000..7caffea
--- /dev/null
+++ b/test/auto_memory/eval/test_swebenchverified.py
@@ -0,0 +1,675 @@
+"""Unit tests for microbots.auto_memory.eval.swebenchverified."""
+
+import json
+import os
+import sys
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../src/")))
+
+from microbots.auto_memory.eval.swebenchverified import (
+ SWE_BENCH_VERIFIED,
+ SweBenchInstance,
+ SweBenchVerifiedTask,
+ _load_dataset_rows,
+ load_instance_using_id,
+ load_instances_of_repo,
+)
+from microbots.auto_memory.evalTask import CallbackResult, EvalOutcome
+from microbots.MicroBot import BotRunResult
+
+MODULE = "microbots.auto_memory.eval.swebenchverified"
+
+
+@pytest.fixture(autouse=True)
+def _clear_dataset_cache():
+ """Clear ``_load_dataset_rows``'s cache so each test's ``load_dataset`` mock takes effect."""
+ _load_dataset_rows.cache_clear()
+ yield
+ _load_dataset_rows.cache_clear()
+
+
+def _fake_rows():
+ return [
+ {
+ "instance_id": "django__django-1",
+ "repo": "django/django",
+ "base_commit": "abc123",
+ "problem_statement": "fix bug 1",
+ },
+ {
+ "instance_id": "astropy__astropy-1",
+ "repo": "astropy/astropy",
+ "base_commit": "def456",
+ "problem_statement": "fix bug 2",
+ },
+ {
+ "instance_id": "django__django-2",
+ "repo": "django/django",
+ "base_commit": "ghi789",
+ "problem_statement": "fix bug 3",
+ },
+ ]
+
+
+# ---------------------------------------------------------------------------
+# load_instances_of_repo / load_instance_using_id
+# ---------------------------------------------------------------------------
+
+@pytest.mark.unit
+@patch("datasets.load_dataset")
+def test_load_instances_of_repo_filters_by_repo(mock_load_dataset):
+ mock_load_dataset.return_value = _fake_rows()
+
+ instances = load_instances_of_repo(repo="django/django")
+
+ assert [i.instance_id for i in instances] == ["django__django-1", "django__django-2"]
+ assert all(isinstance(i, SweBenchInstance) for i in instances)
+
+
+@pytest.mark.unit
+@patch("datasets.load_dataset")
+def test_load_instances_of_repo_returns_all_when_repo_none(mock_load_dataset):
+ mock_load_dataset.return_value = _fake_rows()
+
+ instances = load_instances_of_repo(repo=None)
+
+ assert len(instances) == 3
+
+
+@pytest.mark.unit
+@patch("datasets.load_dataset")
+def test_load_instance_using_id_returns_matching_instance(mock_load_dataset):
+ mock_load_dataset.return_value = _fake_rows()
+
+ instance = load_instance_using_id("astropy__astropy-1")
+
+ assert instance.repo == "astropy/astropy"
+ assert instance.problem_statement == "fix bug 2"
+
+
+@pytest.mark.unit
+@patch("datasets.load_dataset")
+def test_load_instance_using_id_raises_when_not_found(mock_load_dataset):
+ mock_load_dataset.return_value = _fake_rows()
+
+ with pytest.raises(ValueError, match="not found"):
+ load_instance_using_id("does-not-exist")
+
+
+@pytest.mark.unit
+@patch("datasets.load_dataset")
+def test_dataset_rows_are_cached_across_repeated_calls(mock_load_dataset):
+ """``load_dataset`` should only be called once per ``dataset_name``, even
+ across multiple ``load_instances_of_repo``/``load_instance_using_id`` calls."""
+ mock_load_dataset.return_value = _fake_rows()
+
+ load_instances_of_repo(repo="django/django")
+ load_instances_of_repo(repo=None)
+ load_instance_using_id("astropy__astropy-1")
+
+ mock_load_dataset.assert_called_once()
+
+
+@pytest.mark.unit
+def test_load_dataset_rows_raises_helpful_error_when_datasets_not_installed():
+ with patch.dict(sys.modules, {"datasets": None}):
+ with pytest.raises(ImportError, match=r"pip install 'microbots\[training\]'"):
+ _load_dataset_rows(SWE_BENCH_VERIFIED)
+
+
+# ---------------------------------------------------------------------------
+# SweBenchVerifiedTask.setup / build_prompt
+# ---------------------------------------------------------------------------
+
+def _instance():
+ return SweBenchInstance(
+ instance_id="django__django-1",
+ repo="django/django",
+ base_commit="abc123",
+ problem_statement="fix the bug",
+ )
+
+
+@pytest.mark.unit
+@patch(f"{MODULE}.subprocess.run")
+def test_setup_clones_and_checks_out_base_commit_when_repo_missing(mock_run, tmp_path):
+ repo_path = tmp_path / "repo"
+ task = SweBenchVerifiedTask(_instance())
+ task.setup(str(repo_path))
+
+ clone_call, checkout_call = mock_run.call_args_list
+ assert clone_call.args[0] == [
+ "git", "clone", "https://github.com/django/django.git", str(repo_path)
+ ]
+ assert checkout_call.args[0] == ["git", "checkout", "abc123"]
+ assert checkout_call.kwargs["cwd"] == str(repo_path)
+
+
+@pytest.mark.unit
+@patch(f"{MODULE}.subprocess.run")
+def test_setup_resets_instead_of_recloning_when_origin_matches(mock_run, tmp_path):
+ repo_path = tmp_path / "repo"
+ repo_path.mkdir()
+ mock_run.return_value = MagicMock(
+ returncode=0, stdout="https://github.com/django/django.git\n"
+ )
+ task = SweBenchVerifiedTask(_instance())
+ task.setup(str(repo_path))
+
+ origin_call, reset_call, clean_call = mock_run.call_args_list
+ assert origin_call.args[0] == ["git", "remote", "get-url", "origin"]
+ assert origin_call.kwargs["cwd"] == str(repo_path)
+ assert reset_call.args[0] == ["git", "reset", "--hard", "abc123"]
+ assert reset_call.kwargs["cwd"] == str(repo_path)
+ assert clean_call.args[0] == ["git", "clean", "-fd"]
+ assert clean_call.kwargs["cwd"] == str(repo_path)
+
+
+@pytest.mark.unit
+@patch(f"{MODULE}.subprocess.run")
+def test_setup_removes_and_reclones_when_origin_mismatched(mock_run, tmp_path):
+ repo_path = tmp_path / "repo"
+ repo_path.mkdir()
+ (repo_path / "stale_file.txt").write_text("leftover from a different repo")
+ mock_run.return_value = MagicMock(
+ returncode=0, stdout="https://github.com/other/repo.git\n"
+ )
+ task = SweBenchVerifiedTask(_instance())
+ task.setup(str(repo_path))
+
+ assert not (repo_path / "stale_file.txt").exists()
+ origin_call, clone_call, checkout_call = mock_run.call_args_list
+ assert origin_call.args[0] == ["git", "remote", "get-url", "origin"]
+ assert clone_call.args[0] == [
+ "git", "clone", "https://github.com/django/django.git", str(repo_path)
+ ]
+ assert checkout_call.args[0] == ["git", "checkout", "abc123"]
+
+
+@pytest.mark.unit
+@patch(f"{MODULE}.subprocess.run")
+def test_setup_removes_and_reclones_when_repo_path_not_a_git_repo(mock_run, tmp_path):
+ repo_path = tmp_path / "repo"
+ repo_path.mkdir()
+ mock_run.return_value = MagicMock(returncode=128, stdout="")
+ task = SweBenchVerifiedTask(_instance())
+ task.setup(str(repo_path))
+
+ assert not repo_path.exists()
+ origin_call, clone_call, checkout_call = mock_run.call_args_list
+ assert origin_call.args[0] == ["git", "remote", "get-url", "origin"]
+ assert clone_call.args[0] == [
+ "git", "clone", "https://github.com/django/django.git", str(repo_path)
+ ]
+
+
+@pytest.mark.unit
+def test_build_prompt_returns_problem_statement():
+ task = SweBenchVerifiedTask(_instance())
+ assert task.build_prompt() == "fix the bug"
+
+
+@pytest.mark.unit
+def test_task_id_is_instance_id():
+ task = SweBenchVerifiedTask(_instance())
+ assert task.task_id == "django__django-1"
+
+
+@pytest.mark.unit
+def test_build_result_includes_dataset_fields():
+ task = SweBenchVerifiedTask(_instance())
+ outcome = EvalOutcome(
+ passed=True,
+ output="agent output",
+ result=CallbackResult(passed=True, reason="resolved"),
+ )
+
+ assert task.build_result(outcome) == {
+ "passed": True,
+ "reason": "resolved",
+ "instance_id": "django__django-1",
+ "repo": "django/django",
+ "base_commit": "abc123",
+ }
+
+
+# ---------------------------------------------------------------------------
+# SweBenchVerifiedTask.build_feedback
+# ---------------------------------------------------------------------------
+
+def _failed_outcome(reason: str = "tests failed", output: str = "agent output") -> EvalOutcome:
+ return EvalOutcome(
+ passed=False,
+ output=output,
+ result=CallbackResult(passed=False, reason=reason),
+ )
+
+
+@pytest.mark.unit
+@patch(f"{MODULE}.LogAnalysisBot")
+def test_build_feedback_returns_bot_result_on_success(mock_bot_cls):
+ mock_bot = MagicMock()
+ mock_bot.run.return_value = BotRunResult(
+ status=True, result="root cause: missing edge case handling", error=None
+ )
+ mock_bot_cls.return_value = mock_bot
+
+ task = SweBenchVerifiedTask(_instance())
+ outcome = _failed_outcome()
+
+ feedback = task.build_feedback(outcome, "/repo", "azure-openai/gpt-4o", "/tmp/some.log")
+
+ assert feedback == "root cause: missing edge case handling"
+ mock_bot_cls.assert_called_once_with(model="azure-openai/gpt-4o", folder_to_mount="/repo")
+ mock_bot.run.assert_called_once()
+ assert mock_bot.run.call_args.kwargs["file_name"] == "/tmp/some.log"
+
+
+@pytest.mark.unit
+@patch(f"{MODULE}.LogAnalysisBot")
+def test_build_feedback_falls_back_when_bot_status_false(mock_bot_cls):
+ mock_bot = MagicMock()
+ mock_bot.run.return_value = BotRunResult(status=False, result=None, error="bot crashed")
+ mock_bot_cls.return_value = mock_bot
+
+ task = SweBenchVerifiedTask(_instance())
+ outcome = _failed_outcome(reason="tests failed", output="some output")
+
+ feedback = task.build_feedback(outcome, "/repo", "azure-openai/gpt-4o", "/tmp/some.log")
+
+ assert "some output" in feedback
+ assert "tests failed" in feedback
+
+
+@pytest.mark.unit
+@patch(f"{MODULE}.LogAnalysisBot")
+def test_build_feedback_falls_back_when_result_is_empty(mock_bot_cls):
+ mock_bot = MagicMock()
+ mock_bot.run.return_value = BotRunResult(status=True, result="", error=None)
+ mock_bot_cls.return_value = mock_bot
+
+ task = SweBenchVerifiedTask(_instance())
+ outcome = _failed_outcome(reason="assertion error", output="agent tried X")
+
+ feedback = task.build_feedback(outcome, "/repo", "azure-openai/gpt-4o", "/tmp/some.log")
+
+ assert "agent tried X" in feedback
+ assert "assertion error" in feedback
+
+
+@pytest.mark.unit
+@patch(f"{MODULE}.LogAnalysisBot")
+def test_build_feedback_falls_back_when_result_is_none(mock_bot_cls):
+ mock_bot = MagicMock()
+ mock_bot.run.return_value = BotRunResult(status=True, result=None, error=None)
+ mock_bot_cls.return_value = mock_bot
+
+ task = SweBenchVerifiedTask(_instance())
+ outcome = _failed_outcome()
+
+ feedback = task.build_feedback(outcome, "/repo", "azure-openai/gpt-4o", "/tmp/some.log")
+
+ assert "Evaluation failed" in feedback
+
+
+# ---------------------------------------------------------------------------
+# SweBenchVerifiedTask.check
+# ---------------------------------------------------------------------------
+
+def _make_fake_subprocess_run(resolved: bool, raise_on_harness: bool = False):
+ """Build a subprocess.run stand-in that fakes git diff + the harness call."""
+
+ def _fake_run(cmd, **kwargs):
+ if cmd[:2] == ["git", "diff"]:
+ return MagicMock(stdout="diff --git a/x.py b/x.py\n+fix", stderr="", returncode=0)
+ if "swebench.harness.run_evaluation" in cmd:
+ if raise_on_harness:
+ raise RuntimeError("harness crashed")
+ run_id = cmd[cmd.index("--run_id") + 1]
+ report_dir = Path(kwargs["cwd"])
+ instance_id = cmd[cmd.index("--instance_ids") + 1]
+ instance_log_dir = (
+ report_dir / "logs" / "run_evaluation" / run_id
+ / "microbots-eval-agent" / instance_id
+ )
+ instance_log_dir.mkdir(parents=True)
+ report = {instance_id: {"resolved": resolved}}
+ (instance_log_dir / "report.json").write_text(json.dumps(report))
+ return MagicMock(stdout="harness ran\n", stderr="", returncode=0)
+ return MagicMock(stdout="", stderr="", returncode=0)
+
+ return _fake_run
+
+
+@pytest.mark.unit
+@patch(f"{MODULE}.subprocess.run")
+def test_check_marks_untracked_files_intent_to_add_before_diffing(mock_run, tmp_path):
+ mock_run.side_effect = _make_fake_subprocess_run(resolved=True)
+ log_path = tmp_path / "check.log"
+ log_path.write_text("")
+
+ task = SweBenchVerifiedTask(_instance())
+ task.check("/repo", "agent output", str(log_path))
+
+ calls = [c.args[0] for c in mock_run.call_args_list]
+ add_idx = calls.index(["git", "add", "--intent-to-add", "."])
+ diff_idx = calls.index(["git", "diff", "--binary"])
+ assert add_idx < diff_idx
+ assert mock_run.call_args_list[add_idx].kwargs["cwd"] == "/repo"
+
+
+@pytest.mark.unit
+@patch(f"{MODULE}.subprocess.run")
+def test_check_passed_true_when_report_marks_resolved(mock_run, tmp_path):
+ mock_run.side_effect = _make_fake_subprocess_run(resolved=True)
+ log_path = tmp_path / "check.log"
+ log_path.write_text("")
+
+ task = SweBenchVerifiedTask(_instance())
+ result = task.check("/repo", "agent output", str(log_path))
+
+ assert result.passed is True
+ assert result.reason == "resolved"
+
+
+@pytest.mark.unit
+@patch(f"{MODULE}.subprocess.run")
+def test_check_passed_false_when_report_marks_not_resolved(mock_run, tmp_path):
+ mock_run.side_effect = _make_fake_subprocess_run(resolved=False)
+ log_path = tmp_path / "check.log"
+ log_path.write_text("")
+
+ task = SweBenchVerifiedTask(_instance())
+ result = task.check("/repo", "agent output", str(log_path))
+
+ assert result.passed is False
+ assert result.reason == "not resolved"
+
+
+@pytest.mark.unit
+@patch(f"{MODULE}.subprocess.run")
+def test_check_passed_false_when_report_file_never_written(mock_run, tmp_path):
+ # harness call succeeds but never writes a report file (e.g. it errored internally)
+ def _fake_run(cmd, **kwargs):
+ if cmd[:2] == ["git", "diff"]:
+ return MagicMock(stdout="diff", stderr="", returncode=0)
+ return MagicMock(stdout="", stderr="", returncode=1)
+
+ mock_run.side_effect = _fake_run
+ log_path = tmp_path / "check.log"
+ log_path.write_text("")
+
+ task = SweBenchVerifiedTask(_instance())
+ result = task.check("/repo", "agent output", str(log_path))
+
+ assert result.passed is False
+
+
+@pytest.mark.unit
+@patch(f"{MODULE}.subprocess.run")
+def test_check_appends_to_log_file_without_truncating_existing_content(mock_run, tmp_path):
+ mock_run.side_effect = _make_fake_subprocess_run(resolved=True)
+ log_path = tmp_path / "check.log"
+ log_path.write_text("Agent output:\nprevious content\n")
+
+ task = SweBenchVerifiedTask(_instance())
+ task.check("/repo", "agent output", str(log_path))
+
+ content = log_path.read_text()
+ assert "previous content" in content
+ assert "harness ran" in content
+
+
+@pytest.mark.unit
+@patch(f"{MODULE}.subprocess.run")
+def test_check_appends_instance_log_and_test_output_when_present(mock_run, tmp_path):
+ def _fake_run(cmd, **kwargs):
+ if cmd[:2] == ["git", "diff"]:
+ return MagicMock(stdout="diff", stderr="", returncode=0)
+ if "swebench.harness.run_evaluation" in cmd:
+ run_id = cmd[cmd.index("--run_id") + 1]
+ report_dir = Path(kwargs["cwd"])
+ instance_id = cmd[cmd.index("--instance_ids") + 1]
+
+ instance_log_dir = (
+ report_dir / "logs" / "run_evaluation" / run_id
+ / "microbots-eval-agent" / instance_id
+ )
+ instance_log_dir.mkdir(parents=True)
+ report = {instance_id: {"resolved": True}}
+ (instance_log_dir / "report.json").write_text(json.dumps(report))
+ (instance_log_dir / "run_instance.log").write_text("build+test steps")
+ (instance_log_dir / "test_output.txt").write_text("FAILED test_foo")
+
+ return MagicMock(stdout="harness ran\n", stderr="", returncode=0)
+ return MagicMock(stdout="", stderr="", returncode=0)
+
+ mock_run.side_effect = _fake_run
+ log_path = tmp_path / "check.log"
+ log_path.write_text("")
+
+ task = SweBenchVerifiedTask(_instance())
+ task.check("/repo", "agent output", str(log_path))
+
+ content = log_path.read_text()
+ assert "run_instance.log" in content
+ assert "build+test steps" in content
+ assert "test_output.txt" in content
+ assert "FAILED test_foo" in content
+
+
+@pytest.mark.unit
+@patch(f"{MODULE}.shutil.rmtree")
+@patch(f"{MODULE}.subprocess.run")
+def test_check_cleans_up_pred_path_and_report_dir_on_success(mock_run, mock_rmtree, tmp_path):
+ mock_run.side_effect = _make_fake_subprocess_run(resolved=True)
+ log_path = tmp_path / "check.log"
+ log_path.write_text("")
+
+ task = SweBenchVerifiedTask(_instance())
+ task.check("/repo", "agent output", str(log_path))
+
+ mock_rmtree.assert_called_once()
+
+
+@pytest.mark.unit
+@patch(f"{MODULE}.shutil.rmtree")
+@patch(f"{MODULE}.subprocess.run")
+def test_check_cleans_up_even_when_harness_raises(mock_run, mock_rmtree, tmp_path):
+ mock_run.side_effect = _make_fake_subprocess_run(resolved=True, raise_on_harness=True)
+ log_path = tmp_path / "check.log"
+ log_path.write_text("")
+
+ task = SweBenchVerifiedTask(_instance())
+ with pytest.raises(RuntimeError, match="harness crashed"):
+ task.check("/repo", "agent output", str(log_path))
+
+ mock_rmtree.assert_called_once()
+
+
+# ---------------------------------------------------------------------------
+# SweBenchVerifiedTask.run
+# ---------------------------------------------------------------------------
+
+@pytest.mark.unit
+@patch(f"{MODULE}.MemoryTool")
+@patch(f"{MODULE}.WritingBot")
+def test_run_calls_setup_build_prompt_check_in_order(mock_bot_cls, mock_memory_tool, tmp_path):
+ from microbots.MicroBot import BotRunResult
+
+ mock_bot = MagicMock()
+ mock_bot.run.return_value = BotRunResult(status=True, result="agent did stuff", error=None)
+ mock_bot_cls.return_value = mock_bot
+
+ task = SweBenchVerifiedTask(_instance())
+ calls = []
+ task.setup = lambda repo_path: calls.append(("setup", repo_path))
+ task.build_prompt = lambda: "do the task"
+ task.check = lambda repo_path, agent_output, log_path: (
+ calls.append(("check", repo_path, agent_output, log_path))
+ or CallbackResult(passed=True, reason="ok")
+ )
+
+ outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o", str(tmp_path / "eval.log"))
+
+ assert calls[0] == ("setup", "/repo")
+ assert calls[1] == ("check", "/repo", "agent did stuff", str(tmp_path / "eval.log"))
+ assert outcome.passed is True
+ assert outcome.output == "agent did stuff"
+
+
+@pytest.mark.unit
+@patch(f"{MODULE}.MemoryTool")
+@patch(f"{MODULE}.WritingBot")
+def test_run_creates_log_file_before_check_is_called(mock_bot_cls, mock_memory_tool, tmp_path):
+ from microbots.MicroBot import BotRunResult
+
+ mock_bot = MagicMock()
+ mock_bot.run.return_value = BotRunResult(status=True, result="output", error=None)
+ mock_bot_cls.return_value = mock_bot
+
+ task = SweBenchVerifiedTask(_instance())
+ task.setup = lambda repo_path: None
+ task.build_prompt = lambda: "do the task"
+ seen_log_exists = {}
+
+ def _check(repo_path, agent_output, log_path):
+ seen_log_exists["exists"] = os.path.exists(log_path)
+ return CallbackResult(passed=True, reason="ok")
+
+ task.check = _check
+
+ task.run("/repo", "/memory", "azure-openai/gpt-4o", str(tmp_path / "eval.log"))
+
+ assert seen_log_exists["exists"] is True
+
+
+@pytest.mark.unit
+@patch(f"{MODULE}.MemoryTool")
+@patch(f"{MODULE}.WritingBot")
+def test_run_skips_check_when_bot_status_is_false(mock_bot_cls, mock_memory_tool, tmp_path):
+ from microbots.MicroBot import BotRunResult
+
+ mock_bot = MagicMock()
+ mock_bot.run.return_value = BotRunResult(status=False, result=None, error="bot crashed")
+ mock_bot_cls.return_value = mock_bot
+
+ task = SweBenchVerifiedTask(_instance())
+ check_calls = []
+ task.setup = lambda repo_path: None
+ task.build_prompt = lambda: "do the task"
+ task.check = lambda *a: check_calls.append(a)
+
+ outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o", str(tmp_path / "eval.log"))
+
+ assert check_calls == []
+ assert outcome.passed is False
+ assert "bot crashed" in outcome.result.reason
+
+
+@pytest.mark.unit
+@patch(f"{MODULE}.MemoryTool")
+@patch(f"{MODULE}.WritingBot")
+def test_run_converts_build_prompt_exception_to_failed_outcome(mock_bot_cls, mock_memory_tool, tmp_path):
+ task = SweBenchVerifiedTask(_instance())
+ task.setup = lambda repo_path: None
+
+ def _build_prompt():
+ raise ValueError("bad prompt")
+
+ task.build_prompt = _build_prompt
+
+ outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o", str(tmp_path / "eval.log"))
+
+ assert outcome.passed is False
+ assert "bad prompt" in outcome.result.reason
+ with open(str(tmp_path / "eval.log")) as f:
+ assert "bad prompt" in f.read()
+
+
+@pytest.mark.unit
+def test_run_converts_setup_exception_to_failed_outcome(tmp_path):
+ task = SweBenchVerifiedTask(_instance())
+
+ def _setup(repo_path):
+ raise RuntimeError("clone failed")
+
+ task.setup = _setup
+
+ outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o", str(tmp_path / "eval.log"))
+
+ assert outcome.passed is False
+ assert "clone failed" in outcome.result.reason
+ with open(str(tmp_path / "eval.log")) as f:
+ assert "clone failed" in f.read()
+
+
+@pytest.mark.unit
+@patch(f"{MODULE}.MemoryTool")
+@patch(f"{MODULE}.WritingBot")
+def test_run_converts_check_exception_to_failed_outcome(mock_bot_cls, mock_memory_tool, tmp_path):
+ from microbots.MicroBot import BotRunResult
+
+ mock_bot = MagicMock()
+ mock_bot.run.return_value = BotRunResult(status=True, result="output", error=None)
+ mock_bot_cls.return_value = mock_bot
+
+ task = SweBenchVerifiedTask(_instance())
+ task.setup = lambda repo_path: None
+ task.build_prompt = lambda: "do the task"
+
+ def _check(repo_path, agent_output, log_path):
+ raise RuntimeError("check exploded")
+
+ task.check = _check
+
+ outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o", str(tmp_path / "eval.log"))
+
+ assert outcome.passed is False
+ assert "check exploded" in outcome.result.reason
+
+
+# ---------------------------------------------------------------------------
+# SweBenchVerifiedTask.from_config
+# ---------------------------------------------------------------------------
+
+@pytest.mark.unit
+@patch(f"{MODULE}.load_instance_using_id")
+def test_from_config_uses_instance_id_when_given(mock_load_instance_using_id):
+ mock_load_instance_using_id.return_value = _instance()
+
+ tasks = SweBenchVerifiedTask.from_config(
+ {"instance_id": "django__django-1", "swebench_repo": None}
+ )
+
+ mock_load_instance_using_id.assert_called_once_with("django__django-1")
+ assert len(tasks) == 1
+ assert isinstance(tasks[0], SweBenchVerifiedTask)
+ assert tasks[0].instance == _instance()
+
+
+@pytest.mark.unit
+@patch(f"{MODULE}.load_instances_of_repo")
+def test_from_config_falls_back_to_repo_filter_when_no_instance_id(mock_load_instances_of_repo):
+ mock_load_instances_of_repo.return_value = [_instance(), _instance()]
+
+ tasks = SweBenchVerifiedTask.from_config({"swebench_repo": "django/django"})
+
+ mock_load_instances_of_repo.assert_called_once_with(repo="django/django")
+ assert len(tasks) == 2
+ assert all(isinstance(t, SweBenchVerifiedTask) for t in tasks)
+
+
+@pytest.mark.unit
+@patch(f"{MODULE}.load_instances_of_repo")
+def test_from_config_handles_empty_dict_gracefully(mock_load_instances_of_repo):
+ mock_load_instances_of_repo.return_value = [_instance()]
+
+ tasks = SweBenchVerifiedTask.from_config({})
+
+ mock_load_instances_of_repo.assert_called_once_with(repo=None)
+ assert len(tasks) == 1
diff --git a/test/auto_memory/test_cli.py b/test/auto_memory/test_cli.py
new file mode 100644
index 0000000..6fcf716
--- /dev/null
+++ b/test/auto_memory/test_cli.py
@@ -0,0 +1,147 @@
+"""Unit tests for microbots.auto_memory.cli."""
+
+import os
+import sys
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../src/")))
+
+from microbots.auto_memory.cli import main, parse_args
+
+MODULE = "microbots.auto_memory.cli"
+
+BASE_ARGS = ["--model", "azure-openai/gpt-4o"]
+FAKE_WORKDIR = Path("/workdir")
+
+
+@pytest.mark.unit
+def test_parse_args_defaults():
+ args = parse_args(BASE_ARGS)
+
+ assert args.model == "azure-openai/gpt-4o"
+ assert args.task is None
+ assert args.max_rounds == 5
+ assert args.training_iterations == 10
+
+
+@pytest.mark.unit
+def test_parse_args_accepts_known_task():
+ args = parse_args(BASE_ARGS + ["--task", "swebenchverified"])
+
+ assert args.task == "swebenchverified"
+
+
+@pytest.mark.unit
+def test_parse_args_rejects_unknown_task():
+ with pytest.raises(SystemExit):
+ parse_args(BASE_ARGS + ["--task", "does-not-exist"])
+
+
+@pytest.mark.unit
+def test_parse_args_workdir_defaults_to_none():
+ args = parse_args(BASE_ARGS)
+
+ assert args.workdir is None
+
+
+@pytest.mark.unit
+def test_parse_args_picks_up_explicit_workdir():
+ args = parse_args(BASE_ARGS + ["--workdir", "/custom/workdir"])
+
+ assert args.workdir == "/custom/workdir"
+
+
+@pytest.mark.unit
+@patch(f"{MODULE}.require_workdir")
+@patch(f"{MODULE}.resolve_workdir")
+@patch(f"{MODULE}.load_config", return_value={})
+@patch(f"{MODULE}.run")
+def test_main_uses_explicit_workdir_over_resolve_workdir(
+ mock_run, mock_load_config, mock_resolve_workdir, mock_require_workdir
+):
+ main(BASE_ARGS + ["--workdir", "/custom/workdir"])
+
+ mock_resolve_workdir.assert_not_called()
+ mock_require_workdir.assert_called_once_with(Path("/custom/workdir"))
+ mock_run.assert_called_once_with(
+ workdir=Path("/custom/workdir"),
+ model="azure-openai/gpt-4o",
+ task=None,
+ max_rounds=5,
+ training_iterations=10,
+ config={},
+ )
+
+
+@pytest.mark.unit
+@patch(f"{MODULE}.require_workdir")
+@patch(f"{MODULE}.resolve_workdir", return_value=FAKE_WORKDIR)
+@patch(f"{MODULE}.load_config", return_value={})
+@patch(f"{MODULE}.run")
+def test_main_falls_back_to_resolve_workdir_when_not_given(
+ mock_run, mock_load_config, mock_resolve_workdir, mock_require_workdir
+):
+ main(BASE_ARGS)
+
+ mock_resolve_workdir.assert_called_once_with()
+ mock_require_workdir.assert_called_once_with(FAKE_WORKDIR)
+ mock_run.assert_called_once_with(
+ workdir=FAKE_WORKDIR,
+ model="azure-openai/gpt-4o",
+ task=None,
+ max_rounds=5,
+ training_iterations=10,
+ config={},
+ )
+
+
+@pytest.mark.unit
+@patch(f"{MODULE}.require_workdir")
+@patch(f"{MODULE}.resolve_workdir", return_value=FAKE_WORKDIR)
+@patch(f"{MODULE}.run")
+def test_main_calls_run_with_task_none_when_task_omitted(mock_run, mock_resolve_workdir, mock_require_workdir):
+ main(BASE_ARGS)
+
+ assert mock_run.call_args.kwargs["task"] is None
+
+
+@pytest.mark.unit
+@patch(f"{MODULE}.require_workdir")
+@patch(f"{MODULE}.resolve_workdir", return_value=FAKE_WORKDIR)
+@patch(f"{MODULE}.load_config", return_value={})
+@patch(f"{MODULE}.run")
+def test_main_calls_run_for_each_task_when_task_given(
+ mock_run, mock_load_config, mock_resolve_workdir, mock_require_workdir
+):
+ fake_task = MagicMock()
+ mock_run.return_value = MagicMock(passed=True, rounds_run=1)
+
+ with patch(f"{MODULE}.TASK_REGISTRY", {"swebenchverified": MagicMock(from_config=lambda task_args: [fake_task])}):
+ main(BASE_ARGS + ["--task", "swebenchverified"])
+
+ mock_run.assert_called_once_with(
+ workdir=FAKE_WORKDIR,
+ model="azure-openai/gpt-4o",
+ task=fake_task,
+ max_rounds=5,
+ training_iterations=10,
+ config={},
+ )
+
+
+@pytest.mark.unit
+@patch(f"{MODULE}.require_workdir")
+@patch(f"{MODULE}.resolve_workdir", return_value=FAKE_WORKDIR)
+@patch(f"{MODULE}.load_config", return_value={})
+@patch(f"{MODULE}.run")
+def test_main_runs_once_per_returned_task(mock_run, mock_load_config, mock_resolve_workdir, mock_require_workdir):
+ fake_tasks = [MagicMock(), MagicMock()]
+ mock_run.return_value = MagicMock(passed=False, rounds_run=5)
+
+ with patch(f"{MODULE}.TASK_REGISTRY", {"swebenchverified": MagicMock(from_config=lambda task_args: fake_tasks)}):
+ main(BASE_ARGS + ["--task", "swebenchverified"])
+
+ assert mock_run.call_count == 2
diff --git a/test/auto_memory/test_orchestrator.py b/test/auto_memory/test_orchestrator.py
new file mode 100644
index 0000000..ba193be
--- /dev/null
+++ b/test/auto_memory/test_orchestrator.py
@@ -0,0 +1,521 @@
+"""Unit tests for microbots.auto_memory.orchestrator."""
+
+import json
+import os
+import sys
+from pathlib import Path
+from unittest.mock import MagicMock, call, patch
+
+import pytest
+
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../src/")))
+
+from microbots.auto_memory.orchestrator import (
+ LoopResult,
+ clone_repo,
+ run,
+ run_train_eval_loop,
+ run_training_loop,
+ write_eval_result,
+)
+from microbots.auto_memory.evalTask import CallbackResult, EvalOutcome
+from microbots.auto_memory.workdir import eval_result_path, memory_dir, round_memory_dir
+
+MODULE = "microbots.auto_memory.orchestrator"
+
+
+def _make_outcome(passed: bool, reason: str = "reason") -> EvalOutcome:
+ return EvalOutcome(
+ passed=passed,
+ output="agent output",
+ result=CallbackResult(passed=passed, reason=reason),
+ )
+
+
+def _touch(path: str) -> str:
+ Path(path).write_text("log contents")
+ return path
+
+
+def _make_task() -> MagicMock:
+ """A MagicMock task with a real-ish task_id/build_result, for round tests."""
+ task = MagicMock()
+ task.task_id = "task-1"
+ task.build_result.side_effect = lambda outcome: {
+ "passed": outcome.result.passed,
+ "reason": outcome.result.reason,
+ }
+ return task
+
+
+@pytest.mark.unit
+@pytest.mark.parametrize("max_rounds", [0, -1])
+def test_loop_raises_for_non_positive_max_rounds(max_rounds):
+ task = _make_task()
+
+ with pytest.raises(ValueError, match="max_rounds must be >= 1"):
+ run_train_eval_loop("/repo", "/eval_repo", Path("/workdir"), "azure-openai/gpt-4o", task, max_rounds=max_rounds)
+
+ task.run.assert_not_called()
+
+
+@pytest.mark.unit
+@patch("microbots.auto_memory.orchestrator.run_training_loop")
+def test_loop_returns_immediately_when_first_round_passes(mock_run_training_loop, tmp_path):
+ task = _make_task()
+ task.run.return_value = _make_outcome(passed=True)
+
+ result = run_train_eval_loop("/repo", "/eval_repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5)
+
+ assert isinstance(result, LoopResult)
+ assert result.passed is True
+ assert result.rounds_run == 1
+ assert task.run.call_count == 1
+ task.build_feedback.assert_not_called()
+ mock_run_training_loop.assert_not_called()
+
+
+@pytest.mark.unit
+@patch("microbots.auto_memory.orchestrator.run_training_loop")
+def test_loop_retrains_and_continues_on_failure_then_passes(mock_run_training_loop, tmp_path):
+ task = _make_task()
+ task.run.side_effect = [
+ _make_outcome(passed=False),
+ _make_outcome(passed=True),
+ ]
+ task.build_feedback.return_value = "feedback text"
+
+ result = run_train_eval_loop("/repo", "/eval_repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5)
+
+ assert result.passed is True
+ assert result.rounds_run == 2
+ task.build_feedback.assert_called_once()
+ mock_run_training_loop.assert_called_once_with(
+ repo_path="/repo",
+ feedback="feedback text",
+ memory_dir=str(round_memory_dir(tmp_path, 1, instance_id="task-1")),
+ model="azure-openai/gpt-4o",
+ iterations=10,
+ )
+
+
+@pytest.mark.unit
+@patch("microbots.auto_memory.orchestrator.run_training_loop")
+def test_loop_exhausts_max_rounds_without_passing(mock_run_training_loop, tmp_path):
+ task = _make_task()
+ task.run.side_effect = [
+ _make_outcome(passed=False)
+ for i in range(3)
+ ]
+ task.build_feedback.return_value = "feedback text"
+
+ result = run_train_eval_loop("/repo", "/eval_repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=3)
+
+ assert result.passed is False
+ assert result.rounds_run == 3
+ assert len(result.outcomes) == 3
+ assert result.final_outcome is result.outcomes[-1]
+ assert task.build_feedback.call_count == 3
+ assert mock_run_training_loop.call_count == 3
+
+
+@pytest.mark.unit
+@patch("microbots.auto_memory.orchestrator.run_training_loop")
+def test_log_path_persists_after_passing_round(mock_run_training_loop, tmp_path):
+ log_path = _touch(str(tmp_path / "round1.log"))
+ task = _make_task()
+ task.run.return_value = _make_outcome(passed=True)
+
+ run_train_eval_loop("/repo", "/eval_repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5)
+
+ assert Path(log_path).exists()
+
+
+@pytest.mark.unit
+@patch("microbots.auto_memory.orchestrator.run_training_loop")
+def test_log_path_persists_after_failing_round(mock_run_training_loop, tmp_path):
+ log1 = _touch(str(tmp_path / "round1.log"))
+ log2 = _touch(str(tmp_path / "round2.log"))
+ task = _make_task()
+ task.run.side_effect = [
+ _make_outcome(passed=False),
+ _make_outcome(passed=True),
+ ]
+ task.build_feedback.return_value = "feedback text"
+
+ run_train_eval_loop("/repo", "/eval_repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5)
+
+ assert Path(log1).exists()
+ assert Path(log2).exists()
+
+
+@pytest.mark.unit
+@patch("microbots.auto_memory.orchestrator.run_training_loop")
+def test_build_feedback_exception_does_not_crash_loop(mock_run_training_loop, tmp_path):
+ log1 = _touch(str(tmp_path / "round1.log"))
+ task = _make_task()
+ task.run.side_effect = [
+ _make_outcome(passed=False),
+ _make_outcome(passed=True),
+ ]
+ task.build_feedback.side_effect = RuntimeError("analysis bot crashed")
+
+ result = run_train_eval_loop("/repo", "/eval_repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5)
+
+ assert result.passed is True
+ assert result.rounds_run == 2
+ mock_run_training_loop.assert_not_called()
+ assert Path(log1).exists()
+
+
+@pytest.mark.unit
+@patch("microbots.auto_memory.orchestrator.run_training_loop")
+def test_loop_forwards_training_iterations_to_run_training_loop(mock_run_training_loop, tmp_path):
+ task = _make_task()
+ task.run.side_effect = [
+ _make_outcome(passed=False),
+ _make_outcome(passed=True),
+ ]
+ task.build_feedback.return_value = "feedback text"
+
+ run_train_eval_loop(
+ "/repo", "/eval_repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5, training_iterations=4
+ )
+
+ mock_run_training_loop.assert_called_once_with(
+ repo_path="/repo",
+ feedback="feedback text",
+ memory_dir=str(round_memory_dir(tmp_path, 1, instance_id="task-1")),
+ model="azure-openai/gpt-4o",
+ iterations=4,
+ )
+
+
+@pytest.mark.unit
+@patch("microbots.auto_memory.orchestrator.run_training_loop")
+def test_run_training_exception_does_not_crash_loop(mock_run_training_loop, tmp_path):
+ log1 = _touch(str(tmp_path / "round1.log"))
+ task = _make_task()
+ task.run.side_effect = [
+ _make_outcome(passed=False),
+ _make_outcome(passed=True),
+ ]
+ task.build_feedback.return_value = "feedback text"
+ mock_run_training_loop.side_effect = RuntimeError("training crashed")
+
+ result = run_train_eval_loop("/repo", "/eval_repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5)
+
+ assert result.passed is True
+ assert result.rounds_run == 2
+ assert Path(log1).exists()
+
+
+@pytest.mark.unit
+@patch("microbots.auto_memory.orchestrator.run_training")
+def test_run_training_loop_calls_run_training_ten_times_by_default(mock_run_training):
+ run_training_loop(repo_path="/repo", feedback="fb", memory_dir="/memory", model="azure-openai/gpt-4o")
+
+ assert mock_run_training.call_count == 10
+ mock_run_training.assert_called_with(
+ repo_path="/repo", feedback="fb", memory_dir="/memory", model="azure-openai/gpt-4o"
+ )
+
+
+@pytest.mark.unit
+@patch("microbots.auto_memory.orchestrator.run_training")
+def test_run_training_loop_calls_run_training_n_times(mock_run_training):
+ run_training_loop(
+ repo_path="/repo", feedback="fb", memory_dir="/memory", model="azure-openai/gpt-4o", iterations=3
+ )
+
+ assert mock_run_training.call_count == 3
+ mock_run_training.assert_called_with(
+ repo_path="/repo", feedback="fb", memory_dir="/memory", model="azure-openai/gpt-4o"
+ )
+
+
+@pytest.mark.unit
+@patch("microbots.auto_memory.orchestrator.run_training")
+def test_run_training_loop_reuses_same_memory_dir_each_pass(mock_run_training):
+ run_training_loop(
+ repo_path="/repo", feedback="fb", memory_dir="/memory", model="azure-openai/gpt-4o", iterations=4
+ )
+
+ memory_dirs = {call.kwargs["memory_dir"] for call in mock_run_training.call_args_list}
+ assert memory_dirs == {"/memory"}
+
+
+@pytest.mark.unit
+@patch(f"{MODULE}.subprocess.run")
+def test_clone_repo_clones_when_missing(mock_run, tmp_path):
+ repo_path = tmp_path / "repo"
+
+ clone_repo("https://example.com/repo.git", repo_path)
+
+ mock_run.assert_called_once_with(
+ ["git", "clone", "https://example.com/repo.git", str(repo_path)], check=True
+ )
+
+
+@pytest.mark.unit
+@patch(f"{MODULE}.subprocess.run")
+def test_clone_repo_is_noop_when_origin_matches(mock_run, tmp_path):
+ repo_path = tmp_path / "repo"
+ repo_path.mkdir()
+ mock_run.return_value = MagicMock(returncode=0, stdout="https://example.com/repo.git\n")
+
+ clone_repo("https://example.com/repo.git", repo_path)
+
+ mock_run.assert_called_once_with(
+ ["git", "remote", "get-url", "origin"],
+ cwd=repo_path, capture_output=True, text=True,
+ )
+
+
+@pytest.mark.unit
+@patch(f"{MODULE}.subprocess.run")
+def test_clone_repo_removes_and_reclones_when_origin_mismatched(mock_run, tmp_path):
+ repo_path = tmp_path / "repo"
+ repo_path.mkdir()
+ (repo_path / "stale_marker.txt").write_text("leftover from a different repo")
+ mock_run.return_value = MagicMock(returncode=0, stdout="https://example.com/other-repo.git\n")
+
+ clone_repo("https://example.com/repo.git", repo_path)
+
+ assert not (repo_path / "stale_marker.txt").exists()
+ mock_run.assert_called_with(
+ ["git", "clone", "https://example.com/repo.git", str(repo_path)], check=True
+ )
+
+
+@pytest.mark.unit
+@patch(f"{MODULE}.subprocess.run")
+def test_clone_repo_removes_and_reclones_when_repo_path_not_a_git_repo(mock_run, tmp_path):
+ repo_path = tmp_path / "repo"
+ repo_path.mkdir()
+ mock_run.return_value = MagicMock(returncode=128, stdout="")
+
+ clone_repo("https://example.com/repo.git", repo_path)
+
+ assert not repo_path.exists()
+ mock_run.assert_called_with(
+ ["git", "clone", "https://example.com/repo.git", str(repo_path)], check=True
+ )
+
+
+@pytest.mark.unit
+def test_write_eval_result_writes_task_build_result_as_json(tmp_path):
+ task = MagicMock()
+ task.task_id = "django__django-1"
+ task.build_result.return_value = {"passed": True, "reason": "resolved"}
+ outcome = _make_outcome(passed=True)
+
+ write_eval_result(tmp_path, 2, task, outcome)
+
+ result_path = eval_result_path(tmp_path, 2, "django__django-1")
+ assert json.loads(result_path.read_text()) == {"passed": True, "reason": "resolved"}
+ task.build_result.assert_called_once_with(outcome)
+
+
+@pytest.mark.unit
+def test_write_eval_result_creates_missing_parent_dirs(tmp_path):
+ task = MagicMock()
+ task.task_id = "some-task"
+ task.build_result.return_value = {"passed": False, "reason": "nope"}
+ outcome = _make_outcome(passed=False)
+
+ write_eval_result(tmp_path, 1, task, outcome)
+
+ assert eval_result_path(tmp_path, 1, "some-task").exists()
+
+
+@pytest.mark.unit
+def test_loop_writes_eval_result_for_every_round(tmp_path):
+ task = _make_task()
+ task.run.side_effect = [
+ _make_outcome(passed=False),
+ _make_outcome(passed=True),
+ ]
+ task.build_feedback.return_value = "feedback text"
+
+ with patch(f"{MODULE}.run_training_loop"):
+ run_train_eval_loop("/repo", "/eval_repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5)
+
+ assert eval_result_path(tmp_path, 1, "task-1").exists()
+ assert eval_result_path(tmp_path, 2, "task-1").exists()
+ assert json.loads(eval_result_path(tmp_path, 2, "task-1").read_text()) == {
+ "passed": True,
+ "reason": "reason",
+ }
+
+
+@pytest.mark.unit
+@patch(f"{MODULE}.clone_repo")
+@patch(f"{MODULE}.run_training_loop")
+def test_run_calls_run_training_loop_when_task_is_none(mock_run_training_loop, mock_clone_repo, tmp_path):
+ result = run(
+ workdir=tmp_path,
+ model="azure-openai/gpt-4o",
+ task=None,
+ training_iterations=2,
+ config={"repo": "https://example.com/repo.git"},
+ )
+
+ mock_run_training_loop.assert_called_once_with(
+ repo_path=str(tmp_path / "repo"),
+ feedback="",
+ memory_dir=str(round_memory_dir(tmp_path, 1)),
+ model="azure-openai/gpt-4o",
+ iterations=2,
+ )
+ assert result is None
+
+
+@pytest.mark.unit
+@patch(f"{MODULE}.clone_repo")
+@patch(f"{MODULE}.run_train_eval_loop")
+def test_run_calls_run_train_eval_loop_when_task_given(mock_run_train_eval_loop, mock_clone_repo, tmp_path):
+ fake_task = MagicMock()
+ mock_run_train_eval_loop.return_value = "loop-result"
+
+ result = run(
+ workdir=tmp_path,
+ model="azure-openai/gpt-4o",
+ task=fake_task,
+ max_rounds=3,
+ training_iterations=2,
+ config={"repo": "https://example.com/repo.git"},
+ )
+
+ mock_run_train_eval_loop.assert_called_once_with(
+ training_repo_path=str(tmp_path / "repo"),
+ eval_repo_path=str(tmp_path / "eval_repo"),
+ workdir=tmp_path,
+ model="azure-openai/gpt-4o",
+ task=fake_task,
+ max_rounds=3,
+ training_iterations=2,
+ )
+ assert result == "loop-result"
+
+
+@pytest.mark.unit
+@patch(f"{MODULE}.clone_repo")
+@patch(f"{MODULE}.run_train_eval_loop")
+@patch(f"{MODULE}.run_training_loop")
+def test_run_does_not_call_eval_loop_when_task_is_none(
+ mock_run_training_loop, mock_run_train_eval_loop, mock_clone_repo, tmp_path
+):
+ run(
+ workdir=tmp_path,
+ model="azure-openai/gpt-4o",
+ task=None,
+ config={"repo": "https://example.com/repo.git"},
+ )
+
+ mock_run_train_eval_loop.assert_not_called()
+
+
+@pytest.mark.unit
+@patch(f"{MODULE}.clone_repo")
+@patch(f"{MODULE}.run_train_eval_loop")
+@patch(f"{MODULE}.run_training_loop")
+def test_run_does_not_call_training_loop_when_task_given(
+ mock_run_training_loop, mock_run_train_eval_loop, mock_clone_repo, tmp_path
+):
+ run(
+ workdir=tmp_path,
+ model="azure-openai/gpt-4o",
+ task=MagicMock(),
+ config={"repo": "https://example.com/repo.git"},
+ )
+
+ mock_run_training_loop.assert_not_called()
+
+
+@pytest.mark.unit
+@patch(f"{MODULE}.clone_repo")
+@patch(f"{MODULE}.run_training_loop")
+def test_run_clones_repo_from_config_when_repo_url_given(mock_run_training_loop, mock_clone_repo, tmp_path):
+ (tmp_path / "config.yaml").write_text("repo: https://example.com/repo.git\n")
+
+ run(workdir=tmp_path, model="azure-openai/gpt-4o", task=None)
+
+ mock_clone_repo.assert_called_once_with("https://example.com/repo.git", tmp_path / "repo")
+
+
+@pytest.mark.unit
+@patch(f"{MODULE}.clone_repo")
+@patch(f"{MODULE}.run_training_loop")
+def test_run_raises_when_config_has_no_repo(mock_run_training_loop, mock_clone_repo, tmp_path):
+ with pytest.raises(ValueError, match="repo"):
+ run(workdir=tmp_path, model="azure-openai/gpt-4o", task=None, config={})
+
+ mock_clone_repo.assert_not_called()
+ mock_run_training_loop.assert_not_called()
+
+
+@pytest.mark.unit
+@patch(f"{MODULE}.clone_repo")
+@patch(f"{MODULE}.run_training_loop")
+def test_run_promotes_round1_memory_to_top_level_for_train_only_mode(mock_run_training_loop, mock_clone_repo, tmp_path):
+ def fake_train(repo_path, feedback, memory_dir, model, iterations=1):
+ Path(memory_dir, "notes.md").write_text("learned something")
+
+ mock_run_training_loop.side_effect = fake_train
+
+ run(
+ workdir=tmp_path,
+ model="azure-openai/gpt-4o",
+ task=None,
+ config={"repo": "https://example.com/repo.git"},
+ )
+
+ assert (memory_dir(tmp_path) / "notes.md").read_text() == "learned something"
+
+
+@pytest.mark.unit
+@patch(f"{MODULE}.clone_repo")
+@patch(f"{MODULE}.run_training_loop")
+def test_run_preserves_original_memory_as_a_seed_snapshot(mock_run_training_loop, mock_clone_repo, tmp_path):
+ memory_dir(tmp_path).mkdir(parents=True)
+ (memory_dir(tmp_path) / "notes.md").write_text("original seed")
+
+ def fake_train(repo_path, feedback, memory_dir, model, iterations=1):
+ Path(memory_dir, "notes.md").write_text("overwritten by training")
+
+ mock_run_training_loop.side_effect = fake_train
+
+ run(
+ workdir=tmp_path,
+ model="azure-openai/gpt-4o",
+ task=None,
+ config={"repo": "https://example.com/repo.git"},
+ )
+
+ assert (memory_dir(tmp_path) / "notes.md").read_text() == "overwritten by training"
+ assert (tmp_path / "memory_seed" / "notes.md").read_text() == "original seed"
+
+
+
+@pytest.mark.unit
+@patch(f"{MODULE}.run_training_loop")
+def test_loop_carries_memory_forward_between_rounds(mock_run_training_loop, tmp_path):
+ seen_memory_dirs = []
+
+ def fake_run(repo_path, memory_dir, model, log_path):
+ round_num = len(seen_memory_dirs) + 1
+ if round_num == 2:
+ # Round 2 should start with whatever round 1 saved.
+ assert (Path(memory_dir) / "notes.md").read_text() == "round 1 progress"
+ seen_memory_dirs.append(memory_dir)
+ Path(memory_dir, "notes.md").write_text(f"round {round_num} progress")
+ return _make_outcome(passed=round_num == 2)
+
+ task = _make_task()
+ task.run.side_effect = fake_run
+ task.build_feedback.return_value = "feedback text"
+
+ run_train_eval_loop("/repo", "/eval_repo", tmp_path, "azure-openai/gpt-4o", task, max_rounds=5)
+
+ assert (memory_dir(tmp_path) / "notes.md").read_text() == "round 2 progress"
diff --git a/test/auto_memory/test_task.py b/test/auto_memory/test_task.py
new file mode 100644
index 0000000..e04d0e9
--- /dev/null
+++ b/test/auto_memory/test_task.py
@@ -0,0 +1,117 @@
+"""Unit tests for microbots.auto_memory.evalTask."""
+
+import os
+import sys
+
+import pytest
+
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../src/")))
+
+from microbots.auto_memory.evalTask import CallbackResult, EvalOutcome, EvalTask
+
+
+class _RunOnlyTask(EvalTask):
+ """A task that overrides only run()/build_feedback(), never touching the optional hooks."""
+
+ @classmethod
+ def from_config(cls, task_args):
+ return [cls()]
+
+ def run(self, repo_path, memory_dir, model, log_path):
+ return EvalOutcome(
+ passed=True,
+ output="custom output",
+ result=None,
+ )
+
+ def build_feedback(self, outcome, repo_path, model, log_path):
+ return "feedback text"
+
+
+@pytest.mark.unit
+def test_run_is_abstract():
+ with pytest.raises(TypeError):
+ EvalTask()
+
+
+@pytest.mark.unit
+def test_build_feedback_is_abstract():
+ class _MissingBuildFeedback(EvalTask):
+ @classmethod
+ def from_config(cls, task_args):
+ return [cls()]
+
+ def run(self, repo_path, memory_dir, model, log_path):
+ raise NotImplementedError
+
+ with pytest.raises(TypeError):
+ _MissingBuildFeedback()
+
+
+@pytest.mark.unit
+def test_from_config_is_abstract():
+ class _MissingFromConfig(EvalTask):
+ def run(self, repo_path, memory_dir, model, log_path):
+ raise NotImplementedError
+
+ def build_feedback(self, outcome, repo_path, model, log_path):
+ raise NotImplementedError
+
+ with pytest.raises(TypeError):
+ _MissingFromConfig()
+
+
+@pytest.mark.unit
+def test_from_config_default_body_raises_not_implemented_error():
+ with pytest.raises(NotImplementedError):
+ EvalTask.from_config({})
+
+
+@pytest.mark.unit
+def test_subclass_overriding_only_run_is_instantiable():
+ task = _RunOnlyTask()
+ outcome = task.run("/repo", "/memory", "azure-openai/gpt-4o", "/log")
+
+ assert outcome.passed is True
+ assert outcome.output == "custom output"
+
+
+@pytest.mark.unit
+def test_default_setup_is_a_noop():
+ # Should not raise.
+ _RunOnlyTask().setup("/repo")
+
+
+@pytest.mark.unit
+def test_default_teardown_is_a_noop():
+ # Should not raise.
+ _RunOnlyTask().teardown("/repo")
+
+
+@pytest.mark.unit
+def test_default_build_prompt_returns_empty_string():
+ assert _RunOnlyTask().build_prompt() == ""
+
+
+@pytest.mark.unit
+def test_default_check_passes_by_default():
+ result = _RunOnlyTask().check("/repo", "output", "/log")
+
+ assert isinstance(result, CallbackResult)
+ assert result.passed is True
+
+
+@pytest.mark.unit
+def test_default_task_id_is_class_name():
+ assert _RunOnlyTask().task_id == "_RunOnlyTask"
+
+
+@pytest.mark.unit
+def test_default_build_result_returns_passed_and_reason():
+ outcome = EvalOutcome(
+ passed=False,
+ output="agent output",
+ result=CallbackResult(passed=False, reason="check failed"),
+ )
+
+ assert _RunOnlyTask().build_result(outcome) == {"passed": False, "reason": "check failed"}
diff --git a/test/auto_memory/test_task_registry.py b/test/auto_memory/test_task_registry.py
new file mode 100644
index 0000000..8de0d48
--- /dev/null
+++ b/test/auto_memory/test_task_registry.py
@@ -0,0 +1,111 @@
+"""Unit tests for microbots.auto_memory.task_registry."""
+
+import os
+import sys
+from types import SimpleNamespace
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../src/")))
+
+from microbots.auto_memory.evalTask import EvalTask
+from microbots.auto_memory.task_registry import TASK_REGISTRY, create_task, discover_tasks, register_task
+
+MODULE_PATH = "microbots.auto_memory.task_registry"
+
+
+class _DummyTask(EvalTask):
+ def __init__(self, value=None):
+ self.value = value
+
+ @classmethod
+ def from_config(cls, task_args):
+ return [cls(**task_args)]
+
+ def setup(self, repo_path):
+ pass
+
+ def build_prompt(self):
+ return "prompt"
+
+ def check(self, output):
+ pass
+
+ def teardown(self, repo_path):
+ pass
+
+ def build_feedback(self, outcome, repo_path, model, log_path):
+ return "feedback"
+
+ def run(self, repo_path, memory_dir, model):
+ return super().run(repo_path, memory_dir, model)
+
+
+@pytest.fixture(autouse=True)
+def _clean_registry():
+ """Snapshot/restore TASK_REGISTRY so tests don't leak state into each other."""
+ original = dict(TASK_REGISTRY)
+ yield
+ TASK_REGISTRY.clear()
+ TASK_REGISTRY.update(original)
+
+
+@pytest.mark.unit
+def test_register_task_adds_class_to_registry():
+ register_task("dummy")(_DummyTask)
+
+ assert TASK_REGISTRY["dummy"] is _DummyTask
+
+
+@pytest.mark.unit
+def test_register_task_returns_class_unchanged():
+ decorated = register_task("dummy")(_DummyTask)
+
+ assert decorated is _DummyTask
+
+
+@pytest.mark.unit
+def test_create_task_constructs_registered_task_with_kwargs():
+ register_task("dummy")(_DummyTask)
+
+ task = create_task("dummy", value=42)
+
+ assert isinstance(task, _DummyTask)
+ assert task.value == 42
+
+
+@pytest.mark.unit
+def test_create_task_raises_for_unknown_name():
+ with pytest.raises(ValueError, match="Unknown task 'nonexistent'"):
+ create_task("nonexistent")
+
+
+@pytest.mark.unit
+def test_discover_tasks_registers_swebenchverified():
+ """Non-destructive: confirms discover_tasks() works against the real package."""
+ discover_tasks()
+
+ assert "swebenchverified" in TASK_REGISTRY
+
+
+@pytest.mark.unit
+@patch(f"{MODULE_PATH}.importlib.import_module")
+@patch(f"{MODULE_PATH}.pkgutil.iter_modules")
+def test_discover_tasks_imports_every_module_found_in_package(mock_iter_modules, mock_import_module):
+ fake_package = MagicMock()
+ fake_package.__path__ = ["/fake/path"]
+ mock_import_module.side_effect = (
+ lambda name: fake_package if name == "fake.pkg" else MagicMock()
+ )
+ mock_iter_modules.return_value = [
+ SimpleNamespace(name="task_a"),
+ SimpleNamespace(name="task_b"),
+ ]
+
+ discover_tasks(package_name="fake.pkg")
+
+ mock_iter_modules.assert_called_once_with(["/fake/path"])
+ mock_import_module.assert_any_call("fake.pkg")
+ mock_import_module.assert_any_call("fake.pkg.task_a")
+ mock_import_module.assert_any_call("fake.pkg.task_b")
diff --git a/test/auto_memory/test_workdir.py b/test/auto_memory/test_workdir.py
new file mode 100644
index 0000000..4c87baa
--- /dev/null
+++ b/test/auto_memory/test_workdir.py
@@ -0,0 +1,277 @@
+"""Unit tests for microbots.auto_memory.workdir."""
+
+import os
+import sys
+
+import pytest
+
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../src/")))
+
+from microbots.auto_memory.workdir import (
+ CONFIG_FILENAME,
+ eval_dir,
+ eval_log_path,
+ eval_patch_path,
+ eval_repo_dir,
+ eval_result_path,
+ load_config,
+ load_round_memory,
+ memory_dir,
+ repo_dir,
+ require_workdir,
+ resolve_workdir,
+ round_dir,
+ round_log_path,
+ round_memory_dir,
+ save_round_memory,
+ snapshot_seed_memory,
+)
+
+
+@pytest.mark.unit
+def test_load_config_returns_empty_dict_when_file_missing(tmp_path):
+ assert load_config(tmp_path) == {}
+
+
+@pytest.mark.unit
+def test_load_config_returns_empty_dict_when_file_empty(tmp_path):
+ (tmp_path / CONFIG_FILENAME).write_text("")
+
+ assert load_config(tmp_path) == {}
+
+
+@pytest.mark.unit
+def test_load_config_parses_yaml_contents(tmp_path):
+ (tmp_path / CONFIG_FILENAME).write_text("repo: https://example.com/repo.git\ntask: swebenchverified\n")
+
+ assert load_config(tmp_path) == {
+ "repo": "https://example.com/repo.git",
+ "task": "swebenchverified",
+ }
+
+
+@pytest.mark.unit
+def test_load_round_memory_creates_empty_dir_when_no_top_level_memory(tmp_path):
+ result = load_round_memory(tmp_path, 1)
+
+ assert result == round_memory_dir(tmp_path, 1)
+ assert result.is_dir()
+ assert list(result.iterdir()) == []
+
+
+@pytest.mark.unit
+def test_load_round_memory_copies_top_level_memory_into_round(tmp_path):
+ top_memory = memory_dir(tmp_path)
+ top_memory.mkdir(parents=True)
+ (top_memory / "notes.md").write_text("prior findings")
+
+ result = load_round_memory(tmp_path, 2)
+
+ assert (result / "notes.md").read_text() == "prior findings"
+
+
+@pytest.mark.unit
+def test_load_round_memory_discards_stale_files_left_in_round_dir(tmp_path):
+ # Simulate a previous crashed/re-run attempt at this same round that
+ # left behind a file no longer present in top-level memory.
+ round_memory = round_memory_dir(tmp_path, 1)
+ round_memory.mkdir(parents=True)
+ (round_memory / "stale.md").write_text("leftover from a crashed attempt")
+
+ top_memory = memory_dir(tmp_path)
+ top_memory.mkdir(parents=True)
+ (top_memory / "notes.md").write_text("current memory")
+
+ result = load_round_memory(tmp_path, 1)
+
+ assert (result / "notes.md").read_text() == "current memory"
+ assert not (result / "stale.md").exists()
+
+
+@pytest.mark.unit
+def test_save_round_memory_creates_empty_dir_when_no_round_memory(tmp_path):
+ result = save_round_memory(tmp_path, 1)
+
+ assert result == memory_dir(tmp_path)
+ assert result.is_dir()
+ assert list(result.iterdir()) == []
+
+
+@pytest.mark.unit
+def test_save_round_memory_copies_round_memory_to_top_level(tmp_path):
+ round_memory = round_memory_dir(tmp_path, 3)
+ round_memory.mkdir(parents=True)
+ (round_memory / "learned.md").write_text("new insight")
+
+ result = save_round_memory(tmp_path, 3)
+
+ assert (result / "learned.md").read_text() == "new insight"
+
+
+@pytest.mark.unit
+def test_save_round_memory_overwrites_stale_top_level_files(tmp_path):
+ top_memory = memory_dir(tmp_path)
+ top_memory.mkdir(parents=True)
+ (top_memory / "notes.md").write_text("old")
+
+ round_memory = round_memory_dir(tmp_path, 1)
+ round_memory.mkdir(parents=True)
+ (round_memory / "notes.md").write_text("new")
+
+ save_round_memory(tmp_path, 1)
+
+ assert (top_memory / "notes.md").read_text() == "new"
+
+
+@pytest.mark.unit
+def test_save_round_memory_propagates_deletions_to_top_level(tmp_path):
+ # The agent deleted a file during this round (e.g. via `memory
+ # delete`); the top level shouldn't resurrect it from a prior save.
+ top_memory = memory_dir(tmp_path)
+ top_memory.mkdir(parents=True)
+ (top_memory / "stale.md").write_text("no longer relevant")
+
+ round_memory = round_memory_dir(tmp_path, 1)
+ round_memory.mkdir(parents=True)
+ (round_memory / "notes.md").write_text("kept")
+
+ save_round_memory(tmp_path, 1)
+
+ assert not (top_memory / "stale.md").exists()
+ assert (top_memory / "notes.md").read_text() == "kept"
+
+
+@pytest.mark.unit
+def test_snapshot_seed_memory_creates_empty_dir_when_no_top_level_memory(tmp_path):
+ result = snapshot_seed_memory(tmp_path)
+
+ assert result == tmp_path / "memory_seed"
+ assert result.is_dir()
+ assert list(result.iterdir()) == []
+
+
+@pytest.mark.unit
+def test_snapshot_seed_memory_copies_current_top_level_memory(tmp_path):
+ top_memory = memory_dir(tmp_path)
+ top_memory.mkdir(parents=True)
+ (top_memory / "notes.md").write_text("original seed")
+
+ result = snapshot_seed_memory(tmp_path)
+
+ assert (result / "notes.md").read_text() == "original seed"
+
+
+@pytest.mark.unit
+def test_snapshot_seed_memory_is_a_noop_once_a_snapshot_exists(tmp_path):
+ top_memory = memory_dir(tmp_path)
+ top_memory.mkdir(parents=True)
+ (top_memory / "notes.md").write_text("original seed")
+ snapshot_seed_memory(tmp_path)
+
+ # Mutate top-level memory as later rounds/instances would.
+ (top_memory / "notes.md").write_text("overwritten by later training")
+
+ result = snapshot_seed_memory(tmp_path)
+
+ assert (result / "notes.md").read_text() == "original seed"
+
+
+@pytest.mark.unit
+def test_resolve_workdir_defaults_to_cwd(monkeypatch, tmp_path):
+ monkeypatch.chdir(tmp_path)
+
+ assert resolve_workdir() == tmp_path / "workdir"
+
+
+@pytest.mark.unit
+def test_resolve_workdir_uses_given_base(tmp_path):
+ assert resolve_workdir(tmp_path) == tmp_path / "workdir"
+
+
+@pytest.mark.unit
+def test_require_workdir_raises_when_missing(tmp_path):
+ missing = tmp_path / "nope"
+
+ with pytest.raises(FileNotFoundError):
+ require_workdir(missing)
+
+
+@pytest.mark.unit
+def test_require_workdir_passes_when_present(tmp_path):
+ require_workdir(tmp_path)
+
+
+@pytest.mark.unit
+def test_repo_dir_returns_workdir_repo(tmp_path):
+ assert repo_dir(tmp_path) == tmp_path / "repo"
+
+
+@pytest.mark.unit
+def test_eval_repo_dir_returns_workdir_eval_repo(tmp_path):
+ assert eval_repo_dir(tmp_path) == tmp_path / "eval_repo"
+
+
+@pytest.mark.unit
+def test_round_dir_creates_directory_when_requested(tmp_path):
+ path = round_dir(tmp_path, 1, create=True)
+
+ assert path == tmp_path / "rounds" / "round_1"
+ assert path.is_dir()
+
+
+@pytest.mark.unit
+def test_round_dir_does_not_create_directory_by_default(tmp_path):
+ path = round_dir(tmp_path, 1)
+
+ assert path == tmp_path / "rounds" / "round_1"
+ assert not path.exists()
+
+
+@pytest.mark.unit
+def test_round_dir_uses_per_instance_dir_when_instance_id_given(tmp_path):
+ path = round_dir(tmp_path, 1, instance_id="task-1")
+
+ assert path == tmp_path / "rounds_task-1" / "round_1"
+
+
+@pytest.mark.unit
+def test_round_log_path_returns_round_log(tmp_path):
+ assert round_log_path(tmp_path, 2) == round_dir(tmp_path, 2) / "round.log"
+
+
+@pytest.mark.unit
+def test_round_log_path_with_instance_id(tmp_path):
+ assert round_log_path(tmp_path, 2, instance_id="task-1") == round_dir(
+ tmp_path, 2, instance_id="task-1"
+ ) / "round.log"
+
+
+@pytest.mark.unit
+def test_eval_dir_creates_directory_when_requested(tmp_path):
+ path = eval_dir(tmp_path, 1, "task-1", create=True)
+
+ assert path == tmp_path / "rounds_task-1" / "round_1" / "eval"
+ assert path.is_dir()
+
+
+@pytest.mark.unit
+def test_eval_dir_does_not_create_directory_by_default(tmp_path):
+ path = eval_dir(tmp_path, 1, "task-1")
+
+ assert path == tmp_path / "rounds_task-1" / "round_1" / "eval"
+ assert not path.exists()
+
+
+@pytest.mark.unit
+def test_eval_result_path_returns_result_json(tmp_path):
+ assert eval_result_path(tmp_path, 1, "task-1") == eval_dir(tmp_path, 1, "task-1") / "result.json"
+
+
+@pytest.mark.unit
+def test_eval_log_path_returns_eval_log(tmp_path):
+ assert eval_log_path(tmp_path, 1, "task-1") == eval_dir(tmp_path, 1, "task-1") / "eval.log"
+
+
+@pytest.mark.unit
+def test_eval_patch_path_returns_repo_patch(tmp_path):
+ assert eval_patch_path(tmp_path, 1, "task-1") == eval_dir(tmp_path, 1, "task-1") / "repo.patch"