diff --git a/examples/quorum_ensemble.py b/examples/quorum_ensemble.py new file mode 100644 index 0000000..926a4f9 --- /dev/null +++ b/examples/quorum_ensemble.py @@ -0,0 +1,130 @@ +"""A pattern written from scratch: the majority-quorum ensemble. + +``patterns.ensemble`` revises whenever *any* reviewer withholds approval +(unanimity). Say you want a different rule — an author stands pat unless a +strict majority of its reviewers reject, and only the rejecting feedback is +forwarded. That quorum is deliberately not an ``ensemble`` kwarg: a pattern +is ordinary SDK code, so you write the loop yourself. This file is the +anatomy of doing that — every prebuilt pattern has the same skeleton: + +1. attempt — independent first attempts (``expect_independent=True``) +2. freeze — seal the round before any cross-agent influence +3. interact — your control flow: reviews, the custom quorum, merged + feedback via ``patterns.merge_reviews``, revise turns +4. verify + judge — the shared tail, one ``patterns.verify_and_judge`` call + +Everything used here is public API — no privileged hooks. Hire every seat +before the freeze (enrollment is open-round-only), and remember each turn is +journaled: a killed run resumes without re-paying completed turns. + + python examples/quorum_ensemble.py [""] # default: implement quicksort with pytest +""" + +import asyncio +import sys +from dataclasses import dataclass +from typing import Sequence + +from h5i.orchestra import Agent, Artifact, Conductor, Review, Verdict, patterns + +DEMO_TASK = "implement quicksort with pytest" + + +@dataclass +class QuorumOutcome: + #: each agent's latest artifact after the cycles, ordered by agent id + artifacts: list[Artifact] + #: every review posted across all cycles + reviews: list[Review] + verdict: Verdict | None + rounds_run: int + + +async def quorum_ensemble( + c: Conductor, + task: str, + agents: Sequence[Agent], + *, + rounds: int = 2, + verify: Sequence[str] | None = None, +) -> QuorumOutcome: + """Like ``patterns.ensemble``, but an author revises only when a strict + majority of its reviewers reject — and sees only the rejecting feedback.""" + if len(agents) < 3: + raise ValueError("a majority quorum needs at least three agents") + + # 1. Independent first attempts, in parallel. + attempts = await asyncio.gather( + *(a.work(task, expect_independent=True) for a in agents) + ) + latest = {agent.id: artifact for agent, artifact in zip(agents, attempts)} + + # 2. Seal the round: no cross-agent influence before every first attempt + # is frozen. + await c.freeze() + + # 3. Review cycles under the majority quorum — plain Python over + # journaled turns, the part no kwarg could express. + all_reviews: list[Review] = [] + rounds_run = 0 + for _ in range(rounds): + rounds_run += 1 + pairs = [ + (reviewer, target) + for reviewer in agents + for target in agents + if reviewer.id != target.id + ] + cycle = await asyncio.gather( + *(reviewer.review(latest[target.id]) for reviewer, target in pairs) + ) + all_reviews.extend(cycle) + + revising: list[tuple[Agent, Review]] = [] + for agent in agents: + received = [r for r in cycle if r.target == agent.id] + rejections = [r for r in received if not r.approved] + if len(rejections) * 2 <= len(received): # majority approves → stand pat + continue + revising.append( + (agent, patterns.merge_reviews(rejections, latest[agent.id])) + ) + if not revising: + break + revised = await asyncio.gather( + *(agent.revise(latest[agent.id], merged) for agent, merged in revising) + ) + for (agent, _), artifact in zip(revising, revised): + latest[agent.id] = artifact + + # 4. The shared tail: neutral verification, then the CLI finalize rule + # (pass judge=… for a custom policy). + verdict = await patterns.verify_and_judge( + c, list(latest.values()), verify=verify + ) + return QuorumOutcome( + artifacts=[latest[k] for k in sorted(latest)], + reviews=all_reviews, + verdict=verdict, + rounds_run=rounds_run, + ) + + +async def main(task: str) -> None: + async with Conductor(".", "quorum-demo", isolation="supervised") as c: + crew = [ + await c.hire(f"worker{i}", runtime="claude", model="claude-haiku-4-5") + for i in range(3) + ] + outcome = await quorum_ensemble( + c, task, crew, rounds=2, verify=["pytest", "-q"] + ) + print(f"{outcome.rounds_run} cycle(s), {len(outcome.reviews)} reviews") + if outcome.verdict and outcome.verdict.selected_submission: + print("winner:", outcome.verdict.selected_submission) + else: + print("no winner:", *(outcome.verdict.reasons if outcome.verdict else ())) + + +if __name__ == "__main__": + asyncio.run(main(sys.argv[1] if len(sys.argv) > 1 else DEMO_TASK)) diff --git a/src/h5i/orchestra/patterns.py b/src/h5i/orchestra/patterns.py index 2f45e04..c3586d3 100644 --- a/src/h5i/orchestra/patterns.py +++ b/src/h5i/orchestra/patterns.py @@ -1,7 +1,23 @@ """Prebuilt orchestrations, implemented in the public SDK — readable, forkable, no privileged API. Each is a faithful port of the Rust -``h5i_orchestra::patterns`` module: if one doesn't fit, copy its ~40 lines -into your score and edit — that is the intended workflow, not a plugin API. +``h5i_orchestra::patterns`` module, built only from the public primitives +in one shared shape: attempt → ``freeze`` → interact (review / revise / +ask) → ``verify`` → ``judge``. + +Three ways to make one your own, cheapest first: + +- **Parameterize** — the conventions are injectable where that is cheap: + ``ensemble(..., approve=…)`` swaps the approval predicate, + ``judge_panel(..., aggregate=…)`` swaps the ballot aggregation, and every + ``judge=`` kwarg accepts any :data:`~h5i.orchestra.policy.Policy` callable. +- **Compose** — the pieces the patterns are assembled from are public: + `merge_reviews`, `review_cycle`, `verify_and_judge`, + `ask_with_valid_citations`, `render_evidence`, `mean_score_verdict`, + `smaller_diff`. A custom pattern is an ordinary async function over these + (``examples/quorum_ensemble.py`` builds one from scratch). +- **Fork** — if the control flow itself doesn't fit, copy the pattern's ~40 + lines into your score and edit. There is deliberately no plugin API to + learn; patterns are user-space code. Roster note: every agent a pattern uses must be hired before the round is sealed — hire integrators/moderators up front, alongside the workers. @@ -12,7 +28,7 @@ import asyncio import sys from dataclasses import dataclass -from typing import Any, Mapping, Sequence +from typing import Any, Callable, Mapping, Sequence from ._conductor import Agent, Conductor from ._errors import AskParseError, OrchestraError @@ -21,6 +37,16 @@ __all__ = [ "approves", + # composition helpers + "merge_reviews", + "review_cycle", + "ReviewCycleOutcome", + "verify_and_judge", + "ask_with_valid_citations", + "render_evidence", + "mean_score_verdict", + "smaller_diff", + # patterns "ensemble", "EnsembleOutcome", "integrate", @@ -45,6 +71,102 @@ def approves(review: Review) -> bool: return approves_text(review.body) +# ── composition helpers ─────────────────────────────────────────────────────── +# +# The pieces the prebuilt patterns are assembled from — public, so a custom +# pattern composes them instead of forking a whole pattern. + + +def merge_reviews(received: Sequence[Review], target: Artifact) -> Review: + """Fold several reviews of one artifact into a single review, so feedback + from many reviewers costs the author one revise turn. Each reviewer's + feedback is tagged ``[reviewer]`` in the merged body; the merged review + references the reviewed artifact.""" + if not received: + raise ValueError("merge_reviews needs at least one review") + return Review( + reviewer="+".join(r.reviewer for r in received), + target=target.owner_agent, + round=received[0].round, + body="\n\n".join(f"[{r.reviewer}]\n{r.body}" for r in received), + referenced_artifacts=(target.id,), + ) + + +@dataclass +class ReviewCycleOutcome: + #: every review posted this cycle, in (reviewer, target) pair order + reviews: list[Review] + #: latest artifact per agent id after the cycle (revisions applied) + artifacts: dict[str, Artifact] + #: ids of the agents that revised this cycle, in roster order + revised: tuple[str, ...] + + +async def review_cycle( + agents: Sequence[Agent], + latest: Mapping[str, Artifact], + *, + approve: Callable[[Review], bool] = approves, +) -> ReviewCycleOutcome: + """One mutual review → revise cycle over ``latest`` (artifact per agent + id): every ordered (reviewer, target) pair reviews in parallel, then + every author whose received reviews are not unanimously approved — per + ``approve`` — revises once against the merged feedback. Returns updated + artifacts without mutating ``latest``. For a quorum other than unanimity, + write the loop yourself on `merge_reviews` — it is a few lines (see + ``examples/quorum_ensemble.py``).""" + pairs = [ + (reviewer, target) + for reviewer in agents + for target in agents + if reviewer.id != target.id + ] + cycle = await asyncio.gather( + *(reviewer.review(latest[target.id]) for reviewer, target in pairs) + ) + revising: list[tuple[Agent, Review]] = [] + for agent in agents: + received = [r for r in cycle if r.target == agent.id] + if all(approve(r) for r in received): + continue + revising.append((agent, merge_reviews(received, latest[agent.id]))) + revised = await asyncio.gather( + *(agent.revise(latest[agent.id], merged) for agent, merged in revising) + ) + artifacts = dict(latest) + for (agent, _), artifact in zip(revising, revised): + artifacts[agent.id] = artifact + return ReviewCycleOutcome( + reviews=list(cycle), + artifacts=artifacts, + revised=tuple(agent.id for agent, _ in revising), + ) + + +async def verify_and_judge( + c: Conductor, + artifacts: Sequence[Artifact], + *, + verify: Sequence[str] | None = None, + isolation: str | None = None, + judge: Policy | None = None, +) -> Verdict | None: + """The shared pattern tail: neutrally verify each artifact — one at a + time, verify worktrees share on-disk state and parallel creation is racy + — then record a verdict. An explicit ``judge`` always wins; otherwise, + when a verifier ran, the CLI's finalize rule (`tests_then_smallest_diff`) + applies; with neither, nothing is recorded and ``None`` comes back.""" + if verify is not None: + for artifact in artifacts: + await c.verify(artifact, verify, isolation=isolation) + if judge is not None: + return await c.judge(judge) + if verify is not None: + return await c.judge(tests_then_smallest_diff) + return None + + # ── ensemble ────────────────────────────────────────────────────────────────── @@ -69,11 +191,14 @@ async def ensemble( verify: Sequence[str] | None = None, isolation: str | None = None, judge: Policy | None = None, + approve: Callable[[Review], bool] = approves, ) -> EnsembleOutcome: """The classic ensemble: every agent attempts ``task`` independently, the round is sealed, agents mutually review and revise for up to ``rounds`` cycles, then (optionally) a neutral verifier runs and a policy decides. - Apply is never automatic — inspect the outcome and apply yourself.""" + Apply is never automatic — inspect the outcome and apply yourself. + ``approve`` swaps the per-review approval convention (default: + `approves`, the ``APPROVE``/``LGTM`` first-line rule).""" if len(agents) < 2: raise ValueError("ensemble needs at least two agents") @@ -94,53 +219,16 @@ async def ensemble( rounds_run = 0 for _ in range(rounds): rounds_run += 1 - # Every ordered (reviewer, target) pair, in parallel. - pairs = [ - (reviewer, target) - for reviewer in agents - for target in agents - if reviewer.id != target.id - ] - cycle = await asyncio.gather( - *(reviewer.review(latest[target.id]) for reviewer, target in pairs) - ) - - # Revise every artifact that a reviewer did not approve; feedback from - # several reviewers is merged into one revise turn. - revising: list[tuple[Agent, Review]] = [] - for agent in agents: - received = [r for r in cycle if r.target == agent.id] - if all(approves(r) for r in received): - continue - merged = Review( - reviewer="+".join(r.reviewer for r in received), - target=agent.id, - round=received[0].round if received else 1, - body="\n\n".join(f"[{r.reviewer}]\n{r.body}" for r in received), - referenced_artifacts=(latest[agent.id].id,), - ) - revising.append((agent, merged)) - revised = await asyncio.gather( - *(agent.revise(latest[agent.id], merged) for agent, merged in revising) - ) - for (agent, _), artifact in zip(revising, revised): - latest[agent.id] = artifact - all_reviews.extend(cycle) - if not revising: + cycle = await review_cycle(agents, latest, approve=approve) + all_reviews.extend(cycle.reviews) + latest = cycle.artifacts + if not cycle.revised: break - # 4. Neutral verification, one artifact at a time (verify worktrees share - # on-disk state; parallel worktree creation is racy). - for artifact in latest.values(): - if verify is not None: - await c.verify(artifact, verify, isolation=isolation) - - # 5. Verdict. - verdict: Verdict | None = None - if judge is not None: - verdict = await c.judge(judge) - elif verify is not None: - verdict = await c.judge(tests_then_smallest_diff) + # 4-5. Neutral verification + verdict, the shared tail. + verdict = await verify_and_judge( + c, list(latest.values()), verify=verify, isolation=isolation, judge=judge + ) return EnsembleOutcome( artifacts=[latest[k] for k in sorted(latest)], @@ -241,14 +329,9 @@ async def arena( await asyncio.gather(*(a.work(task, expect_independent=True) for a in agents)) ) await c.freeze() - if verify is not None: - for artifact in artifacts: - await c.verify(artifact, verify, isolation=isolation) - verdict: Verdict | None = None - if judge is not None: - verdict = await c.judge(judge) - elif verify is not None: - verdict = await c.judge(tests_then_smallest_diff) + verdict = await verify_and_judge( + c, artifacts, verify=verify, isolation=isolation, judge=judge + ) rows = await c.compare() return ArenaOutcome(artifacts=artifacts, rows=rows, verdict=verdict) @@ -325,7 +408,8 @@ def from_value(cls, value: Mapping[str, Any]) -> "Ballot": class JudgePanelOutcome: #: every validated ballot, by judge id ballots: list[tuple[str, list[Ballot]]] - #: the recorded verdict (highest mean score; ties broken by smallest diff) + #: the recorded verdict (default: highest mean score, ties broken by + #: smallest diff — see ``aggregate``) verdict: Verdict @@ -333,12 +417,16 @@ async def judge_panel( c: Conductor, rubric: str, judges: Sequence[Agent], + *, + aggregate: Callable[[Sequence[Ballot], int, Run], Verdict] | None = None, ) -> JudgePanelOutcome: """A panel of judge agents scores the sealed candidates over the run's recorded evidence; citations are validated against real ids (bounded - re-ask on a hallucinated citation) and the mean-score winner is recorded + re-ask on a hallucinated citation) and the aggregated winner is recorded as the verdict. Judges are read-only seats — they never submit, and the - verdict is advisory (never auto-applicable).""" + verdict is advisory (never auto-applicable). ``aggregate`` swaps the + aggregation rule ``(ballots, n_judges, run) -> Verdict`` (median, quorum, + veto, …); the default is `mean_score_verdict`.""" if not judges: raise ValueError("judge_panel needs at least one judge") status = await c.status() @@ -348,7 +436,7 @@ async def judge_panel( valid_ids = {s.id for s in status.submissions} | {v.id for v in status.verifications} candidate_ids = [s.id for s in candidates] - evidence = _render_evidence(status) + evidence = render_evidence(status) prompt = ( f"You are a neutral judge on a review panel. Rubric: {rubric}\n\n" @@ -362,25 +450,25 @@ async def judge_panel( ballots: list[tuple[str, list[Ballot]]] = [] for judge in judges: - card = await _ask_with_valid_citations( + card = await ask_with_valid_citations( judge, prompt, valid_ids, candidate_ids ) ballots.append((judge.id, card)) # Aggregation is a policy — the panel's contribution is eliciting - # evidence-cited ballots. Swap in your own aggregation (median, quorum, - # veto) by judging the same ballots with a different callable. + # evidence-cited ballots; ``aggregate`` decides over them. flat = [b for _, judge_ballots in ballots for b in judge_ballots] n_judges = len(judges) + rule = aggregate if aggregate is not None else mean_score_verdict - def mean_score_policy(run: Run) -> Verdict: - return _mean_score_verdict(flat, n_judges, run) + def aggregate_policy(run: Run) -> Verdict: + return rule(flat, n_judges, run) - verdict = await c.judge(mean_score_policy) + verdict = await c.judge(aggregate_policy) return JudgePanelOutcome(ballots=ballots, verdict=verdict) -async def _ask_with_valid_citations( +async def ask_with_valid_citations( judge: Agent, base_prompt: str, valid_ids: set[str], @@ -423,10 +511,11 @@ async def _ask_with_valid_citations( raise AssertionError("unreachable") -def _mean_score_verdict(ballots: Sequence[Ballot], n_judges: int, run: Run) -> Verdict: +def mean_score_verdict(ballots: Sequence[Ballot], n_judges: int, run: Run) -> Verdict: """Mean-score aggregation: highest mean wins, ties broken by smallest diff. A panel is advisory over evidence (not a neutral re-execution), so - the verdict is never auto-applicable.""" + the verdict is never auto-applicable. This is `judge_panel`'s default + ``aggregate`` rule — a template for writing your own.""" method = f"panel:mean-score({n_judges} judges)" epsilon = sys.float_info.epsilon best: tuple[str, float] | None = None @@ -441,7 +530,7 @@ def _mean_score_verdict(ballots: Sequence[Ballot], n_judges: int, run: Run) -> V _, current_mean = best better = mean > current_mean + epsilon or ( abs(mean - current_mean) <= epsilon - and _smaller_diff(candidate, best[0], run.submissions) + and smaller_diff(candidate, best[0], run.submissions) ) if better: best = (candidate.id, mean) @@ -463,8 +552,12 @@ def _mean_score_verdict(ballots: Sequence[Ballot], n_judges: int, run: Run) -> V ) -def _smaller_diff(candidate: Artifact, other_id: str, all_: Sequence[Artifact]) -> bool: - other = next((a for a in all_ if a.id == other_id), None) +def smaller_diff( + candidate: Artifact, other_id: str, candidates: Sequence[Artifact] +) -> bool: + """The smallest-diff tie-break: fewer files changed, then fewer + insertions, then lexicographic id — usable in any custom aggregation.""" + other = next((a for a in candidates if a.id == other_id), None) if other is None: return False return (candidate.files_changed, candidate.insertions, candidate.id) < ( @@ -474,7 +567,9 @@ def _smaller_diff(candidate: Artifact, other_id: str, all_: Sequence[Artifact]) ) -def _render_evidence(run: Run) -> str: +def render_evidence(run: Run) -> str: + """Render a folded run's submissions and verifications as the compact, + id-citable evidence block `judge_panel` grounds its judges in.""" lines = ["Submissions:"] for sub in run.submissions: lines.append( diff --git a/tests/test_patterns.py b/tests/test_patterns.py index 1e16beb..67922c2 100644 --- a/tests/test_patterns.py +++ b/tests/test_patterns.py @@ -4,7 +4,7 @@ import pytest -from h5i.orchestra import patterns +from h5i.orchestra import Artifact, Review, Verdict, patterns from mock_server import MockOrchestra, launch_conductor @@ -227,6 +227,104 @@ def ask(p): await c.close() +def test_merge_reviews_tags_reviewers_and_references_target(): + art = Artifact.from_raw(artifact_raw("claude")) + merged = patterns.merge_reviews( + [ + Review(reviewer="codex", target="claude", round=2, body="fix x"), + Review(reviewer="gemini", target="claude", round=2, body="fix y"), + ], + art, + ) + assert merged.reviewer == "codex+gemini" + assert merged.target == "claude" + assert merged.round == 2 + assert "[codex]\nfix x" in merged.body and "[gemini]\nfix y" in merged.body + assert merged.referenced_artifacts == (art.id,) + with pytest.raises(ValueError, match="at least one review"): + patterns.merge_reviews([], art) + + +async def test_ensemble_custom_approve_predicate(): + mock = MockOrchestra() + wire_basics(mock) + # "SHIP IT" fails the default APPROVE/LGTM convention — only the custom + # predicate keeps every author from being sent a revise turn. + mock.on("agent.review", lambda p: { + "reviewer": p["reviewer"], "target": p["artifact"]["owner_agent"], + "round": 1, "body": "SHIP IT", + }) + c = await launch_conductor(mock) + try: + agents = [await c.hire("claude"), await c.hire("codex")] + outcome = await patterns.ensemble( + c, "task", agents, rounds=3, + approve=lambda r: r.body.startswith("SHIP"), + ) + assert outcome.rounds_run == 1 + assert not mock.calls_to("agent.revise") + assert outcome.verdict is None # no verifier, no judge → no verdict + finally: + await c.close() + + +async def test_verify_and_judge_fallback_policy_and_none(): + mock = MockOrchestra() + wire_basics(mock) + c = await launch_conductor(mock) + try: + agent = await c.hire("claude") + artifact = await agent.work("task") + await c.freeze() + # Neither a verifier nor a judge → nothing runs, nothing is recorded. + assert await patterns.verify_and_judge(c, [artifact]) is None + assert not mock.calls_to("conductor.verify") + # A verifier without a judge falls back to the CLI finalize rule. + verdict = await patterns.verify_and_judge(c, [artifact], verify=["true"]) + assert verdict is not None + assert verdict.method == "tests_then_smallest_diff" + assert len(mock.calls_to("conductor.verify")) == 1 + finally: + await c.close() + + +async def test_judge_panel_custom_aggregate(): + mock = MockOrchestra() + wire_basics(mock) + s1 = artifact_raw("claude", id_="s1") + s2 = artifact_raw("codex", id_="s2") + run_raw = {"id": "testrun", "phase": "sealed_submit", + "submissions": [s1, s2], "verifications": []} + mock.on("conductor.status", lambda p: run_raw) + mock.on("agent.ask", lambda p: {"ballots": [ + {"artifact_id": "s1", "score": 9, "rationale": "solid", "cited_ids": ["s1"]}, + {"artifact_id": "s2", "score": 2, "rationale": "broken", "cited_ids": ["s2"]}, + ]}) + mock.on("conductor.judge_begin", lambda p: {"replayed": False, "token": "judge#1", "run": run_raw}) + mock.on("conductor.judge_commit", lambda p: p["verdict"]) + + def veto(ballots, n_judges, run): + survivors = {b.artifact_id for b in ballots} - { + b.artifact_id for b in ballots if b.score < 5 + } + return Verdict( + selected_submission=next(iter(survivors), None), + method=f"panel:veto({n_judges} judges)", + decided_by="test", + can_auto_apply=False, + reasons=("any score below 5 disqualifies",), + ) + + c = await launch_conductor(mock) + try: + judge = await c.hire("judge") + outcome = await patterns.judge_panel(c, "rubric", [judge], aggregate=veto) + assert outcome.verdict.selected_submission == "s1" + assert outcome.verdict.method == "panel:veto(1 judges)" + finally: + await c.close() + + async def test_debate_alternates_and_concludes(): mock = MockOrchestra() wire_basics(mock)