From 4b255bee6342e402a52d0ca514709b25d12355e2 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 28 Jul 2026 16:36:46 +0000 Subject: [PATCH 1/2] feat(sleep): consolidate skill groups independently with failure isolation Add consolidate_groups: each skill group runs through the existing consolidate() with its own tasks, document, and gate decision. Task-less, name-less, repeated, and raising groups are reported instead of aborting the night, shared memory is read-only, and adoption stays explicit. Refs #120 --- skillopt_sleep/multi_skill.py | 97 ++++++++++++++++++ tests/test_sleep_multi_skill.py | 175 ++++++++++++++++++++++++++++++++ 2 files changed, 272 insertions(+) create mode 100644 skillopt_sleep/multi_skill.py create mode 100644 tests/test_sleep_multi_skill.py diff --git a/skillopt_sleep/multi_skill.py b/skillopt_sleep/multi_skill.py new file mode 100644 index 00000000..409b9090 --- /dev/null +++ b/skillopt_sleep/multi_skill.py @@ -0,0 +1,97 @@ +"""SkillOpt-Sleep — consolidate several skill groups in one night. + +A night normally consolidates one managed skill. When mined tasks have been +grouped per skill, each group is an independent consolidation: its own tasks, its +own document, its own gate decision. This module drives those runs so that a +group with no tasks, an unusable name, or a failing backend is isolated and +reported instead of aborting the groups around it. + +Adoption is unchanged and still explicit: nothing here writes a skill file. +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Callable, Dict, List, Optional, Sequence + +from skillopt_sleep.backend import Backend +from skillopt_sleep.consolidate import ConsolidationResult, consolidate +from skillopt_sleep.types import TaskRecord + +CONSOLIDATED = "consolidated" +SKIPPED = "skipped" +FAILED = "failed" + + +@dataclass +class SkillGroup: + """One skill's slice of the night: its name, its document, its tasks.""" + + skill_name: str + skill: str = "" + tasks: List[TaskRecord] = field(default_factory=list) + + +@dataclass +class GroupConsolidation: + """Per-group outcome. ``result`` is set only when the group actually ran.""" + + skill_name: str + status: str + result: Optional[ConsolidationResult] = None + reason: str = "" + + @property + def accepted(self) -> bool: + return bool(self.result and self.result.accepted) + + +def consolidate_groups( + backend: Backend, + groups: Sequence[SkillGroup], + memory: str = "", + *, + consolidate_fn: Callable[..., ConsolidationResult] = consolidate, + **consolidate_kwargs: object, +) -> Dict[str, GroupConsolidation]: + """Consolidate each group independently, in order, isolating failures. + + Returns one entry per input group, keyed by skill name and ordered + first-seen. A group is ``skipped`` when it is unusable (blank name, no + tasks, repeated name) and ``failed`` when its own consolidation raised; both + leave every other group's decision untouched. + + ``memory`` is the shared agent memory and is passed through read-only: group + runs evolve skills only, so no group can rewrite another group's memory. + """ + out: Dict[str, GroupConsolidation] = {} + for group in groups: + name = (group.skill_name or "").strip() + if not name: + out.setdefault("", GroupConsolidation("", SKIPPED, reason="group has no skill name")) + continue + if name in out: + continue # first group wins; a repeated name is not a second night + if not group.tasks: + out[name] = GroupConsolidation(name, SKIPPED, reason="no mined tasks for this skill") + continue + try: + result = consolidate_fn( + backend, list(group.tasks), group.skill, memory, + evolve_memory=False, **consolidate_kwargs, + ) + except Exception as exc: # one group's failure must not abort the night + out[name] = GroupConsolidation( + name, FAILED, reason=f"{type(exc).__name__}: {exc}"[:300] + ) + continue + out[name] = GroupConsolidation(name, CONSOLIDATED, result=result) + return out + + +def accepted_group_skills(outcomes: Dict[str, GroupConsolidation]) -> Dict[str, str]: + """Skill name -> new document, for groups whose gate accepted an update.""" + return { + name: outcome.result.new_skill + for name, outcome in outcomes.items() + if outcome.accepted and outcome.result is not None + } diff --git a/tests/test_sleep_multi_skill.py b/tests/test_sleep_multi_skill.py new file mode 100644 index 00000000..9fd63888 --- /dev/null +++ b/tests/test_sleep_multi_skill.py @@ -0,0 +1,175 @@ +"""Tests for per-skill-group consolidation (issue #120). + +Pure-stdlib (unittest), deterministic MockBackend, no API key, no network. +Run: python -m pytest tests/test_sleep_multi_skill.py +""" +from __future__ import annotations + +import unittest + +from skillopt_sleep.backend import MockBackend +from skillopt_sleep.consolidate import consolidate +from skillopt_sleep.experiments.personas import programmer_persona, researcher_persona +from skillopt_sleep.memory import set_learned +from skillopt_sleep.mine import assign_splits +from skillopt_sleep.multi_skill import ( + CONSOLIDATED, + FAILED, + SKIPPED, + GroupConsolidation, + SkillGroup, + accepted_group_skills, + consolidate_groups, +) + + +def _tasks(persona, seed=42): + return assign_splits(persona(), holdout_fraction=0.34, seed=seed) + + +class TestConsolidateGroups(unittest.TestCase): + def test_no_groups_is_an_empty_mapping(self): + self.assertEqual(consolidate_groups(MockBackend(), []), {}) + + def test_single_group_matches_the_existing_single_skill_run(self): + skill = set_learned("", []) + direct = consolidate(MockBackend(), _tasks(researcher_persona), skill, "", + edit_budget=4, gate_metric="mixed", night=1, + evolve_memory=False) + grouped = consolidate_groups( + MockBackend(), + [SkillGroup("research-skill", skill, _tasks(researcher_persona))], + edit_budget=4, gate_metric="mixed", night=1, + ) + self.assertEqual(list(grouped), ["research-skill"]) + outcome = grouped["research-skill"] + self.assertEqual(outcome.status, CONSOLIDATED) + self.assertTrue(outcome.accepted) + self.assertEqual(outcome.result.new_skill, direct.new_skill) + self.assertEqual(outcome.result.gate_action, direct.gate_action) + self.assertEqual(outcome.result.candidate_score, direct.candidate_score) + + def test_each_group_gets_its_own_decision_and_document(self): + groups = [ + SkillGroup("research-skill", set_learned("# research\n", []), + _tasks(researcher_persona)), + SkillGroup("programming-skill", set_learned("# programming\n", []), + _tasks(programmer_persona, seed=1)), + ] + outcomes = consolidate_groups(MockBackend(), groups, edit_budget=4, night=1) + self.assertEqual(list(outcomes), ["research-skill", "programming-skill"]) + self.assertIn("# research", outcomes["research-skill"].result.new_skill) + self.assertIn("# programming", outcomes["programming-skill"].result.new_skill) + self.assertNotIn("# programming", outcomes["research-skill"].result.new_skill) + + def test_group_without_tasks_is_skipped_not_fatal(self): + outcomes = consolidate_groups( + MockBackend(), + [ + SkillGroup("empty-skill", set_learned("", []), []), + SkillGroup("research-skill", set_learned("", []), _tasks(researcher_persona)), + ], + edit_budget=4, night=1, + ) + self.assertEqual(outcomes["empty-skill"].status, SKIPPED) + self.assertIsNone(outcomes["empty-skill"].result) + self.assertIn("no mined tasks", outcomes["empty-skill"].reason) + self.assertEqual(outcomes["research-skill"].status, CONSOLIDATED) + + def test_group_without_a_name_is_skipped_and_reported(self): + outcomes = consolidate_groups( + MockBackend(), + [SkillGroup(" ", set_learned("", []), _tasks(researcher_persona))], + edit_budget=4, night=1, + ) + self.assertEqual(outcomes[""].status, SKIPPED) + self.assertIn("no skill name", outcomes[""].reason) + + def test_repeated_group_name_runs_once(self): + calls = [] + + def _fake(backend, tasks, skill, memory, **kwargs): + calls.append(skill) + return consolidate(backend, tasks, skill, memory, **kwargs) + + outcomes = consolidate_groups( + MockBackend(), + [ + SkillGroup("research-skill", set_learned("# first\n", []), + _tasks(researcher_persona)), + SkillGroup("research-skill", set_learned("# second\n", []), + _tasks(researcher_persona)), + ], + consolidate_fn=_fake, edit_budget=4, night=1, + ) + self.assertEqual(len(outcomes), 1) + self.assertEqual(len(calls), 1) + self.assertIn("# first", calls[0]) + + def test_one_failing_group_is_isolated_and_reported(self): + def _fake(backend, tasks, skill, memory, **kwargs): + if "boom" in skill: + raise RuntimeError("backend exploded") + return consolidate(backend, tasks, skill, memory, **kwargs) + + outcomes = consolidate_groups( + MockBackend(), + [ + SkillGroup("broken-skill", "boom", _tasks(researcher_persona)), + SkillGroup("research-skill", set_learned("", []), _tasks(researcher_persona)), + ], + consolidate_fn=_fake, edit_budget=4, night=1, + ) + self.assertEqual(outcomes["broken-skill"].status, FAILED) + self.assertIsNone(outcomes["broken-skill"].result) + self.assertIn("RuntimeError: backend exploded", outcomes["broken-skill"].reason) + self.assertEqual(outcomes["research-skill"].status, CONSOLIDATED) + self.assertTrue(outcomes["research-skill"].accepted) + + def test_group_runs_do_not_evolve_shared_memory(self): + seen = {} + + def _fake(backend, tasks, skill, memory, **kwargs): + seen.update(kwargs) + seen["memory"] = memory + return consolidate(backend, tasks, skill, memory, **kwargs) + + consolidate_groups( + MockBackend(), + [SkillGroup("research-skill", set_learned("", []), _tasks(researcher_persona))], + "# shared memory\n", consolidate_fn=_fake, edit_budget=4, night=1, + ) + self.assertEqual(seen["memory"], "# shared memory\n") + self.assertFalse(seen["evolve_memory"]) + + def test_accepted_group_skills_lists_only_accepted_updates(self): + outcomes = { + "kept": GroupConsolidation( + "kept", CONSOLIDATED, + result=consolidate(MockBackend(), _tasks(researcher_persona), + set_learned("", []), "", edit_budget=4, night=1), + ), + "skipped": GroupConsolidation("skipped", SKIPPED, reason="no mined tasks"), + "failed": GroupConsolidation("failed", FAILED, reason="RuntimeError: x"), + } + self.assertTrue(outcomes["kept"].accepted) + self.assertEqual(list(accepted_group_skills(outcomes)), ["kept"]) + self.assertFalse(outcomes["skipped"].accepted) + self.assertFalse(outcomes["failed"].accepted) + + def test_consolidating_groups_writes_no_files(self): + import os + import tempfile + + with tempfile.TemporaryDirectory() as tmp: + before = sorted(os.listdir(tmp)) + consolidate_groups( + MockBackend(), + [SkillGroup("research-skill", set_learned("", []), _tasks(researcher_persona))], + edit_budget=4, night=1, + ) + self.assertEqual(sorted(os.listdir(tmp)), before) + + +if __name__ == "__main__": + unittest.main() From 184895eeb1c35de67433a7085d918b50eca9d0a6 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 28 Jul 2026 16:39:03 +0000 Subject: [PATCH 2/2] feat(sleep): report per-skill-group gate decisions Add SkillGroupReport plus SleepReport.skill_groups (default empty) and build the rows from each group's own baseline, candidate score, decision, and reason, so a weak group can neither block nor borrow a strong group's evidence and older single-group report consumers keep working. Refs #120 --- skillopt_sleep/multi_skill.py | 43 ++++++- skillopt_sleep/types.py | 28 +++++ tests/test_sleep_skill_group_report.py | 159 +++++++++++++++++++++++++ 3 files changed, 226 insertions(+), 4 deletions(-) create mode 100644 tests/test_sleep_skill_group_report.py diff --git a/skillopt_sleep/multi_skill.py b/skillopt_sleep/multi_skill.py index 409b9090..f9bea8aa 100644 --- a/skillopt_sleep/multi_skill.py +++ b/skillopt_sleep/multi_skill.py @@ -15,7 +15,7 @@ from skillopt_sleep.backend import Backend from skillopt_sleep.consolidate import ConsolidationResult, consolidate -from skillopt_sleep.types import TaskRecord +from skillopt_sleep.types import SkillGroupReport, TaskRecord CONSOLIDATED = "consolidated" SKIPPED = "skipped" @@ -39,6 +39,7 @@ class GroupConsolidation: status: str result: Optional[ConsolidationResult] = None reason: str = "" + n_tasks: int = 0 @property def accepted(self) -> bool: @@ -72,7 +73,9 @@ def consolidate_groups( if name in out: continue # first group wins; a repeated name is not a second night if not group.tasks: - out[name] = GroupConsolidation(name, SKIPPED, reason="no mined tasks for this skill") + out[name] = GroupConsolidation( + name, SKIPPED, reason="no mined tasks for this skill" + ) continue try: result = consolidate_fn( @@ -81,13 +84,45 @@ def consolidate_groups( ) except Exception as exc: # one group's failure must not abort the night out[name] = GroupConsolidation( - name, FAILED, reason=f"{type(exc).__name__}: {exc}"[:300] + name, FAILED, reason=f"{type(exc).__name__}: {exc}"[:300], + n_tasks=len(group.tasks), ) continue - out[name] = GroupConsolidation(name, CONSOLIDATED, result=result) + out[name] = GroupConsolidation( + name, CONSOLIDATED, result=result, n_tasks=len(group.tasks) + ) return out +def skill_group_reports( + outcomes: Dict[str, GroupConsolidation], +) -> List[SkillGroupReport]: + """Build one report row per group, from that group's evidence only. + + A skipped or failed group reports its reason and keeps zeroed scores rather + than inheriting another group's numbers, and an accepted group's row is + unaffected by its neighbours' decisions. + """ + rows: List[SkillGroupReport] = [] + for name, outcome in outcomes.items(): + row = SkillGroupReport( + skill_name=name, + status=outcome.status, + reason=outcome.reason, + n_tasks=outcome.n_tasks, + ) + result = outcome.result + if result is not None: + row.accepted = result.accepted + row.gate_action = result.gate_action + row.baseline_score = result.baseline_score + row.candidate_score = result.candidate_score + row.n_applied_edits = len(result.applied_edits) + row.n_rejected_edits = len(result.rejected_edits) + rows.append(row) + return rows + + def accepted_group_skills(outcomes: Dict[str, GroupConsolidation]) -> Dict[str, str]: """Skill name -> new document, for groups whose gate accepted an update.""" return { diff --git a/skillopt_sleep/types.py b/skillopt_sleep/types.py index c0ed2e5a..385d93e2 100644 --- a/skillopt_sleep/types.py +++ b/skillopt_sleep/types.py @@ -120,6 +120,30 @@ class EditRecord: rationale: str = "" +@dataclass +class SkillGroupReport: + """One skill group's own gate evidence for the night. + + Every field describes that group alone: its baseline, its candidate score, + its decision, and why. Groups never share rows, so a weak group cannot + borrow a strong group's evidence or block its acceptance. + """ + + skill_name: str + status: str = "" # consolidated | skipped | failed + accepted: bool = False + gate_action: str = "" + baseline_score: float = 0.0 + candidate_score: float = 0.0 + n_tasks: int = 0 + n_applied_edits: int = 0 + n_rejected_edits: int = 0 + reason: str = "" + + def to_dict(self) -> Dict[str, Any]: + return asdict(self) + + @dataclass class SleepReport: """Everything one night produced — written to staging for review.""" @@ -143,6 +167,10 @@ class SleepReport: # list above — without them a night can report no edits while the optimizer # actually produced several. unmatched_edits: List[EditRecord] = field(default_factory=list) + # Per-skill gate rows for a multi-skill night, in first-seen order. Empty on + # a single-managed-skill night, so the fields above remain the whole story + # for consumers that predate skill groups. + skill_groups: List[SkillGroupReport] = field(default_factory=list) tokens_used: int = 0 notes: List[str] = field(default_factory=list) diff --git a/tests/test_sleep_skill_group_report.py b/tests/test_sleep_skill_group_report.py new file mode 100644 index 00000000..dfb469e2 --- /dev/null +++ b/tests/test_sleep_skill_group_report.py @@ -0,0 +1,159 @@ +"""Tests for per-skill-group gate reporting (issue #120). + +Pure-stdlib (unittest), deterministic MockBackend, no API key, no network. +Run: python -m pytest tests/test_sleep_skill_group_report.py +""" +from __future__ import annotations + +import unittest + +from skillopt_sleep.backend import MockBackend +from skillopt_sleep.consolidate import ConsolidationResult +from skillopt_sleep.experiments.personas import researcher_persona +from skillopt_sleep.memory import set_learned +from skillopt_sleep.mine import assign_splits +from skillopt_sleep.multi_skill import ( + CONSOLIDATED, + FAILED, + SKIPPED, + GroupConsolidation, + SkillGroup, + consolidate_groups, + skill_group_reports, +) +from skillopt_sleep.types import EditRecord, SkillGroupReport, SleepReport + + +def _tasks(seed=42): + return assign_splits(researcher_persona(), holdout_fraction=0.34, seed=seed) + + +def _result(accepted, action, baseline, candidate, applied=0, rejected=0): + return ConsolidationResult( + accepted=accepted, gate_action=action, baseline_score=baseline, + candidate_score=candidate, new_skill="", new_memory="", + applied_edits=[EditRecord("skill", "add", f"rule {i}") for i in range(applied)], + rejected_edits=[EditRecord("skill", "add", f"bad {i}") for i in range(rejected)], + holdout_baseline=baseline, holdout_candidate=candidate, + ) + + +class TestSkillGroupReport(unittest.TestCase): + def test_row_defaults_are_empty_evidence(self): + row = SkillGroupReport(skill_name="example-skill") + self.assertEqual( + (row.status, row.accepted, row.gate_action, row.baseline_score, + row.candidate_score, row.n_tasks, row.n_applied_edits, + row.n_rejected_edits, row.reason), + ("", False, "", 0.0, 0.0, 0, 0, 0, ""), + ) + self.assertEqual(row.to_dict()["skill_name"], "example-skill") + + +class TestSkillGroupRows(unittest.TestCase): + def test_each_group_keeps_its_own_scores_and_decision(self): + outcomes = { + "strong-skill": GroupConsolidation( + "strong-skill", CONSOLIDATED, n_tasks=6, + result=_result(True, "accept_new_best", 0.25, 0.75, applied=2), + ), + "weak-skill": GroupConsolidation( + "weak-skill", CONSOLIDATED, n_tasks=3, + result=_result(False, "reject", 0.5, 0.1, rejected=1), + ), + } + rows = {r.skill_name: r for r in skill_group_reports(outcomes)} + strong, weak = rows["strong-skill"], rows["weak-skill"] + self.assertTrue(strong.accepted) + self.assertEqual((strong.baseline_score, strong.candidate_score), (0.25, 0.75)) + self.assertEqual((strong.n_tasks, strong.n_applied_edits), (6, 2)) + self.assertFalse(weak.accepted) + self.assertEqual((weak.baseline_score, weak.candidate_score), (0.5, 0.1)) + self.assertEqual((weak.n_applied_edits, weak.n_rejected_edits), (0, 1)) + self.assertEqual(weak.gate_action, "reject") + + def test_rows_follow_first_seen_group_order(self): + outcomes = { + "b-skill": GroupConsolidation("b-skill", SKIPPED, reason="no mined tasks"), + "a-skill": GroupConsolidation("a-skill", SKIPPED, reason="no mined tasks"), + } + self.assertEqual([r.skill_name for r in skill_group_reports(outcomes)], + ["b-skill", "a-skill"]) + + def test_skipped_and_failed_rows_carry_reasons_and_no_borrowed_scores(self): + outcomes = { + "strong-skill": GroupConsolidation( + "strong-skill", CONSOLIDATED, n_tasks=6, + result=_result(True, "accept_new_best", 0.25, 0.75, applied=2), + ), + "empty-skill": GroupConsolidation( + "empty-skill", SKIPPED, reason="no mined tasks for this skill"), + "broken-skill": GroupConsolidation( + "broken-skill", FAILED, reason="RuntimeError: backend exploded", n_tasks=4), + } + rows = {r.skill_name: r for r in skill_group_reports(outcomes)} + for name in ("empty-skill", "broken-skill"): + self.assertFalse(rows[name].accepted) + self.assertEqual(rows[name].gate_action, "") + self.assertEqual((rows[name].baseline_score, rows[name].candidate_score), (0.0, 0.0)) + self.assertTrue(rows[name].reason) + self.assertEqual(rows["broken-skill"].status, FAILED) + self.assertEqual(rows["broken-skill"].n_tasks, 4) + self.assertTrue(rows["strong-skill"].accepted) + + def test_rows_from_a_real_mixed_night(self): + outcomes = consolidate_groups( + MockBackend(), + [ + SkillGroup("research-skill", set_learned("", []), _tasks()), + SkillGroup("empty-skill", set_learned("", []), []), + ], + edit_budget=4, gate_metric="mixed", night=1, + ) + rows = skill_group_reports(outcomes) + self.assertEqual([r.skill_name for r in rows], ["research-skill", "empty-skill"]) + research = rows[0] + self.assertEqual(research.status, CONSOLIDATED) + self.assertTrue(research.accepted) + self.assertGreater(research.candidate_score, research.baseline_score) + self.assertEqual(research.n_tasks, len(_tasks())) + self.assertEqual(rows[1].status, SKIPPED) + self.assertFalse(rows[1].accepted) + + +class TestSleepReportCompatibility(unittest.TestCase): + def test_single_skill_report_has_no_group_rows(self): + report = SleepReport(night=1, project="/repo/example", accepted=True, + gate_action="accept_new_best") + self.assertEqual(report.skill_groups, []) + self.assertEqual(report.to_dict()["skill_groups"], []) + + def test_legacy_keyword_construction_still_works(self): + legacy = { + "night": 2, "project": "/repo/example", "n_sessions": 1, "n_tasks": 3, + "baseline_score": 0.1, "candidate_score": 0.2, "accepted": True, + "gate_action": "accept", "edits": [], "notes": ["ok"], + } + report = SleepReport(**legacy) + self.assertEqual(report.skill_groups, []) + self.assertEqual(report.gate_action, "accept") + + def test_group_rows_serialize_beside_the_single_skill_summary(self): + report = SleepReport(night=3, project="/repo/example", accepted=True, + gate_action="accept_new_best", candidate_score=0.75) + report.skill_groups = skill_group_reports({ + "research-skill": GroupConsolidation( + "research-skill", CONSOLIDATED, n_tasks=6, + result=_result(True, "accept_new_best", 0.25, 0.75, applied=2), + ), + }) + payload = report.to_dict() + # older consumers read the flat summary and ignore the new key + self.assertEqual(payload["gate_action"], "accept_new_best") + self.assertEqual(payload["candidate_score"], 0.75) + self.assertEqual(payload["skill_groups"][0]["skill_name"], "research-skill") + self.assertEqual(payload["skill_groups"][0]["n_applied_edits"], 2) + + +if __name__ == "__main__": + unittest.main()