feat(sleep): report per-skill-group gate decisions

skillopt_sleep/multi_skill.py

@@ -0,0 +1,132 @@
+"""SkillOpt-Sleep — consolidate several skill groups in one night.
+
+A night normally consolidates one managed skill. When mined tasks have been
+grouped per skill, each group is an independent consolidation: its own tasks, its
+own document, its own gate decision. This module drives those runs so that a
+group with no tasks, an unusable name, or a failing backend is isolated and
+reported instead of aborting the groups around it.
+
+Adoption is unchanged and still explicit: nothing here writes a skill file.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Callable, Dict, List, Optional, Sequence
+
+from skillopt_sleep.backend import Backend
+from skillopt_sleep.consolidate import ConsolidationResult, consolidate
+from skillopt_sleep.types import SkillGroupReport, TaskRecord
+
+CONSOLIDATED = "consolidated"
+SKIPPED = "skipped"
+FAILED = "failed"
+
+
+@dataclass
+class SkillGroup:
+    """One skill's slice of the night: its name, its document, its tasks."""
+
+    skill_name: str
+    skill: str = ""
+    tasks: List[TaskRecord] = field(default_factory=list)
+
+
+@dataclass
+class GroupConsolidation:
+    """Per-group outcome. ``result`` is set only when the group actually ran."""
+
+    skill_name: str
+    status: str
+    result: Optional[ConsolidationResult] = None
+    reason: str = ""
+    n_tasks: int = 0
+
+    @property
+    def accepted(self) -> bool:
+        return bool(self.result and self.result.accepted)
+
+
+def consolidate_groups(
+    backend: Backend,
+    groups: Sequence[SkillGroup],
+    memory: str = "",
+    *,
+    consolidate_fn: Callable[..., ConsolidationResult] = consolidate,
+    **consolidate_kwargs: object,
+) -> Dict[str, GroupConsolidation]:
+    """Consolidate each group independently, in order, isolating failures.
+
+    Returns one entry per input group, keyed by skill name and ordered
+    first-seen. A group is ``skipped`` when it is unusable (blank name, no
+    tasks, repeated name) and ``failed`` when its own consolidation raised; both
+    leave every other group's decision untouched.
+
+    ``memory`` is the shared agent memory and is passed through read-only: group
+    runs evolve skills only, so no group can rewrite another group's memory.
+    """
+    out: Dict[str, GroupConsolidation] = {}
+    for group in groups:
+        name = (group.skill_name or "").strip()
+        if not name:
+            out.setdefault("", GroupConsolidation("", SKIPPED, reason="group has no skill name"))
+            continue
+        if name in out:
+            continue  # first group wins; a repeated name is not a second night
+        if not group.tasks:
+            out[name] = GroupConsolidation(
+                name, SKIPPED, reason="no mined tasks for this skill"
+            )
+            continue
+        try:
+            result = consolidate_fn(
+                backend, list(group.tasks), group.skill, memory,
+                evolve_memory=False, **consolidate_kwargs,
+            )
+        except Exception as exc:  # one group's failure must not abort the night
+            out[name] = GroupConsolidation(
+                name, FAILED, reason=f"{type(exc).__name__}: {exc}"[:300],
+                n_tasks=len(group.tasks),
+            )
+            continue
+        out[name] = GroupConsolidation(
+            name, CONSOLIDATED, result=result, n_tasks=len(group.tasks)
+        )
+    return out
+
+
+def skill_group_reports(
+    outcomes: Dict[str, GroupConsolidation],
+) -> List[SkillGroupReport]:
+    """Build one report row per group, from that group's evidence only.
+
+    A skipped or failed group reports its reason and keeps zeroed scores rather
+    than inheriting another group's numbers, and an accepted group's row is
+    unaffected by its neighbours' decisions.
+    """
+    rows: List[SkillGroupReport] = []
+    for name, outcome in outcomes.items():
+        row = SkillGroupReport(
+            skill_name=name,
+            status=outcome.status,
+            reason=outcome.reason,
+            n_tasks=outcome.n_tasks,
+        )
+        result = outcome.result
+        if result is not None:
+            row.accepted = result.accepted
+            row.gate_action = result.gate_action
+            row.baseline_score = result.baseline_score
+            row.candidate_score = result.candidate_score
+            row.n_applied_edits = len(result.applied_edits)
+            row.n_rejected_edits = len(result.rejected_edits)
+        rows.append(row)
+    return rows
+
+
+def accepted_group_skills(outcomes: Dict[str, GroupConsolidation]) -> Dict[str, str]:
+    """Skill name -> new document, for groups whose gate accepted an update."""
+    return {
+        name: outcome.result.new_skill
+        for name, outcome in outcomes.items()
+        if outcome.accepted and outcome.result is not None
+    }

skillopt_sleep/types.py

@@ -120,6 +120,30 @@ class EditRecord:
     rationale: str = ""
 
 
+@dataclass
+class SkillGroupReport:
+    """One skill group's own gate evidence for the night.
+
+    Every field describes that group alone: its baseline, its candidate score,
+    its decision, and why. Groups never share rows, so a weak group cannot
+    borrow a strong group's evidence or block its acceptance.
+    """
+
+    skill_name: str
+    status: str = ""                   # consolidated | skipped | failed
+    accepted: bool = False
+    gate_action: str = ""
+    baseline_score: float = 0.0
+    candidate_score: float = 0.0
+    n_tasks: int = 0
+    n_applied_edits: int = 0
+    n_rejected_edits: int = 0
+    reason: str = ""
+
+    def to_dict(self) -> Dict[str, Any]:
+        return asdict(self)
+
+
 @dataclass
 class SleepReport:
     """Everything one night produced — written to staging for review."""
@@ -143,6 +167,10 @@ class SleepReport:
     # list above — without them a night can report no edits while the optimizer
     # actually produced several.
     unmatched_edits: List[EditRecord] = field(default_factory=list)
+    # Per-skill gate rows for a multi-skill night, in first-seen order. Empty on
+    # a single-managed-skill night, so the fields above remain the whole story
+    # for consumers that predate skill groups.
+    skill_groups: List[SkillGroupReport] = field(default_factory=list)
     tokens_used: int = 0
     notes: List[str] = field(default_factory=list)
 

tests/test_sleep_multi_skill.py

@@ -0,0 +1,175 @@
+"""Tests for per-skill-group consolidation (issue #120).
+
+Pure-stdlib (unittest), deterministic MockBackend, no API key, no network.
+Run:  python -m pytest tests/test_sleep_multi_skill.py
+"""
+from __future__ import annotations
+
+import unittest
+
+from skillopt_sleep.backend import MockBackend
+from skillopt_sleep.consolidate import consolidate
+from skillopt_sleep.experiments.personas import programmer_persona, researcher_persona
+from skillopt_sleep.memory import set_learned
+from skillopt_sleep.mine import assign_splits
+from skillopt_sleep.multi_skill import (
+    CONSOLIDATED,
+    FAILED,
+    SKIPPED,
+    GroupConsolidation,
+    SkillGroup,
+    accepted_group_skills,
+    consolidate_groups,
+)
+
+
+def _tasks(persona, seed=42):
+    return assign_splits(persona(), holdout_fraction=0.34, seed=seed)
+
+
+class TestConsolidateGroups(unittest.TestCase):
+    def test_no_groups_is_an_empty_mapping(self):
+        self.assertEqual(consolidate_groups(MockBackend(), []), {})
+
+    def test_single_group_matches_the_existing_single_skill_run(self):
+        skill = set_learned("", [])
+        direct = consolidate(MockBackend(), _tasks(researcher_persona), skill, "",
+                             edit_budget=4, gate_metric="mixed", night=1,
+                             evolve_memory=False)
+        grouped = consolidate_groups(
+            MockBackend(),
+            [SkillGroup("research-skill", skill, _tasks(researcher_persona))],
+            edit_budget=4, gate_metric="mixed", night=1,
+        )
+        self.assertEqual(list(grouped), ["research-skill"])
+        outcome = grouped["research-skill"]
+        self.assertEqual(outcome.status, CONSOLIDATED)
+        self.assertTrue(outcome.accepted)
+        self.assertEqual(outcome.result.new_skill, direct.new_skill)
+        self.assertEqual(outcome.result.gate_action, direct.gate_action)
+        self.assertEqual(outcome.result.candidate_score, direct.candidate_score)
+
+    def test_each_group_gets_its_own_decision_and_document(self):
+        groups = [
+            SkillGroup("research-skill", set_learned("# research\n", []),
+                       _tasks(researcher_persona)),
+            SkillGroup("programming-skill", set_learned("# programming\n", []),
+                       _tasks(programmer_persona, seed=1)),
+        ]
+        outcomes = consolidate_groups(MockBackend(), groups, edit_budget=4, night=1)
+        self.assertEqual(list(outcomes), ["research-skill", "programming-skill"])
+        self.assertIn("# research", outcomes["research-skill"].result.new_skill)
+        self.assertIn("# programming", outcomes["programming-skill"].result.new_skill)
+        self.assertNotIn("# programming", outcomes["research-skill"].result.new_skill)
+
+    def test_group_without_tasks_is_skipped_not_fatal(self):
+        outcomes = consolidate_groups(
+            MockBackend(),
+            [
+                SkillGroup("empty-skill", set_learned("", []), []),
+                SkillGroup("research-skill", set_learned("", []), _tasks(researcher_persona)),
+            ],
+            edit_budget=4, night=1,
+        )
+        self.assertEqual(outcomes["empty-skill"].status, SKIPPED)
+        self.assertIsNone(outcomes["empty-skill"].result)
+        self.assertIn("no mined tasks", outcomes["empty-skill"].reason)
+        self.assertEqual(outcomes["research-skill"].status, CONSOLIDATED)
+
+    def test_group_without_a_name_is_skipped_and_reported(self):
+        outcomes = consolidate_groups(
+            MockBackend(),
+            [SkillGroup("  ", set_learned("", []), _tasks(researcher_persona))],
+            edit_budget=4, night=1,
+        )
+        self.assertEqual(outcomes[""].status, SKIPPED)
+        self.assertIn("no skill name", outcomes[""].reason)
+
+    def test_repeated_group_name_runs_once(self):
+        calls = []
+
+        def _fake(backend, tasks, skill, memory, **kwargs):
+            calls.append(skill)
+            return consolidate(backend, tasks, skill, memory, **kwargs)
+
+        outcomes = consolidate_groups(
+            MockBackend(),
+            [
+                SkillGroup("research-skill", set_learned("# first\n", []),
+                           _tasks(researcher_persona)),
+                SkillGroup("research-skill", set_learned("# second\n", []),
+                           _tasks(researcher_persona)),
+            ],
+            consolidate_fn=_fake, edit_budget=4, night=1,
+        )
+        self.assertEqual(len(outcomes), 1)
+        self.assertEqual(len(calls), 1)
+        self.assertIn("# first", calls[0])
+
+    def test_one_failing_group_is_isolated_and_reported(self):
+        def _fake(backend, tasks, skill, memory, **kwargs):
+            if "boom" in skill:
+                raise RuntimeError("backend exploded")
+            return consolidate(backend, tasks, skill, memory, **kwargs)
+
+        outcomes = consolidate_groups(
+            MockBackend(),
+            [
+                SkillGroup("broken-skill", "boom", _tasks(researcher_persona)),
+                SkillGroup("research-skill", set_learned("", []), _tasks(researcher_persona)),
+            ],
+            consolidate_fn=_fake, edit_budget=4, night=1,
+        )
+        self.assertEqual(outcomes["broken-skill"].status, FAILED)
+        self.assertIsNone(outcomes["broken-skill"].result)
+        self.assertIn("RuntimeError: backend exploded", outcomes["broken-skill"].reason)
+        self.assertEqual(outcomes["research-skill"].status, CONSOLIDATED)
+        self.assertTrue(outcomes["research-skill"].accepted)
+
+    def test_group_runs_do_not_evolve_shared_memory(self):
+        seen = {}
+
+        def _fake(backend, tasks, skill, memory, **kwargs):
+            seen.update(kwargs)
+            seen["memory"] = memory
+            return consolidate(backend, tasks, skill, memory, **kwargs)
+
+        consolidate_groups(
+            MockBackend(),
+            [SkillGroup("research-skill", set_learned("", []), _tasks(researcher_persona))],
+            "# shared memory\n", consolidate_fn=_fake, edit_budget=4, night=1,
+        )
+        self.assertEqual(seen["memory"], "# shared memory\n")
+        self.assertFalse(seen["evolve_memory"])
+
+    def test_accepted_group_skills_lists_only_accepted_updates(self):
+        outcomes = {
+            "kept": GroupConsolidation(
+                "kept", CONSOLIDATED,
+                result=consolidate(MockBackend(), _tasks(researcher_persona),
+                                   set_learned("", []), "", edit_budget=4, night=1),
+            ),
+            "skipped": GroupConsolidation("skipped", SKIPPED, reason="no mined tasks"),
+            "failed": GroupConsolidation("failed", FAILED, reason="RuntimeError: x"),
+        }
+        self.assertTrue(outcomes["kept"].accepted)
+        self.assertEqual(list(accepted_group_skills(outcomes)), ["kept"])
+        self.assertFalse(outcomes["skipped"].accepted)
+        self.assertFalse(outcomes["failed"].accepted)
+
+    def test_consolidating_groups_writes_no_files(self):
+        import os
+        import tempfile
+
+        with tempfile.TemporaryDirectory() as tmp:
+            before = sorted(os.listdir(tmp))
+            consolidate_groups(
+                MockBackend(),
+                [SkillGroup("research-skill", set_learned("", []), _tasks(researcher_persona))],
+                edit_budget=4, night=1,
+            )
+            self.assertEqual(sorted(os.listdir(tmp)), before)
+
+
+if __name__ == "__main__":
+    unittest.main()