From 8d581458e30c3d8122532295e591d86ce9ff0fc9 Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Fri, 17 Jul 2026 04:54:25 -0400 Subject: [PATCH 01/21] docs: design elite decision protocol --- ...26-07-17-elite-decision-protocol-design.md | 69 +++++ .../2026-07-17-elite-decision-protocol.md | 258 ++++++++++++++++++ 2 files changed, 327 insertions(+) create mode 100644 docs/plans/2026-07-17-elite-decision-protocol-design.md create mode 100644 docs/plans/2026-07-17-elite-decision-protocol.md diff --git a/docs/plans/2026-07-17-elite-decision-protocol-design.md b/docs/plans/2026-07-17-elite-decision-protocol-design.md new file mode 100644 index 0000000..bb0459a --- /dev/null +++ b/docs/plans/2026-07-17-elite-decision-protocol-design.md @@ -0,0 +1,69 @@ +# Elite Decision Protocol Design + +## Outcome + +Add a first-class `elite` council mode optimized for consequential decisions. Elite runs +favor decision quality over latency and cost while preserving conclave's existing BYO-key, +partial-failure, auditability, and library-first guarantees. + +The protocol requires at least three successful responders. Three is the minimum that can +express a majority and a minority; four or five remain supported for higher-stakes use. +The threshold is an invariant, not a user setting, so `elite` cannot silently degrade into +an ordinary one- or two-model answer. + +## Protocol + +1. **Independent answer:** fan the original prompt to every member concurrently. Members do + not see one another's answers, preventing early anchoring and groupthink. +2. **Evidence audit:** each surviving member receives an anonymized Model A/B/C panel and + produces one council-wide critique. The prompt asks it to identify supported claims, + conflicts, hidden assumptions, and externally unverified claims. It must cite stable + answer IDs and must not invent sources. +3. **Revision:** each survivor receives its original answer, the anonymized panel, and all + anonymized critiques, then writes one complete revised answer to the original prompt. +4. **Decision:** the existing synthesis and auditable-verdict pipeline processes the revised + answers. The current `CouncilVerdict` remains canonical; elite does not create a competing + verdict schema. + +Each member phase is concurrent, making the protocol O(N) calls per phase rather than an +O(N-squared) pairwise debate. If fewer than three responses survive any member phase, the +protocol stops further calls and returns partial artifacts with an explicit failure reason. +It never raises for provider-side failure and never emits a final synthesis or verdict for an +incomplete elite run. + +## Public API + +- CLI: `conclave ask "..." --mode elite -c grok,gemini,claude` +- Async: `await Council(...).elite(prompt)` +- Sync: `Council(...).elite_sync(prompt)` +- Result: backward-compatible optional `CouncilResult.elite: EliteResult | None` + +`EliteResult` records the protocol version, required responders, completion status, failure +reason, initial answers, critiques, and revisions. On success, `CouncilResult.answers` holds +the successful revisions. On incomplete runs it holds the latest complete answer-to-original +prompt stage, while critiques remain under `result.elite`. + +Streaming is not supported initially because quality depends on completing and validating +each stage before the next begins. The CLI rejects `--mode elite --stream` explicitly. + +## Auditability and safety + +The manifest must cover initial, critique, and revision calls. Provider receipts gain a +backward-compatible optional `phase` field, while `providers_called` remains a unique member +list. Total latency and usage aggregate the recorded member calls across all three phases. +The manifest is rescanned before receiving `verified_no_secrets`. + +Model identities remain anonymized inside cross-member prompts. Answer IDs provide evidence +links without disclosing provider brands. The audit prompt distinguishes externally +unverified claims from disproven claims because conclave has no retrieval or tool-use layer. + +## Acceptance criteria + +- Three successful members complete all stages and produce the existing auditable verdict. +- Four or five configured members may still complete if failures leave at least three. +- Fewer than three successes at any phase returns an incomplete result without a verdict. +- Critiques and revisions retain attributable answer IDs without provider-name leakage. +- Manifest receipts expose every member phase and remain secret-free. +- Existing modes, cache behavior, output schemas, and public APIs remain compatible. +- Unit, CLI, manifest, cache, secret-safety, lint, formatting, and full test suites pass. + diff --git a/docs/plans/2026-07-17-elite-decision-protocol.md b/docs/plans/2026-07-17-elite-decision-protocol.md new file mode 100644 index 0000000..4460850 --- /dev/null +++ b/docs/plans/2026-07-17-elite-decision-protocol.md @@ -0,0 +1,258 @@ +# Elite Decision Protocol Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Add a three-stage, minimum-three-member Elite Decision Protocol that produces a stronger evidence-audited decision through conclave's existing auditable verdict. + +**Architecture:** Add `elite` as a first-class mode implemented by `run_elite`: independent fan-out, concurrent council-wide evidence critiques, concurrent revisions, then existing synthesis and verdict application. Store phase artifacts in a backward-compatible `EliteResult`, and extend manifest receipts with phase provenance so the full protocol is auditable without exposing provider identities or secrets. + +**Tech Stack:** Python 3.11+, asyncio, Pydantic v2, Typer, pytest/pytest-asyncio, Ruff. + +--- + +### Task 1: Define the result contract + +**Files:** +- Modify: `src/conclave/models.py` +- Modify: `src/conclave/__init__.py` +- Test: `tests/test_elite_mode.py` + +**Step 1: Write failing tests** + +Test that `EliteResult` defaults to protocol version `elite_v1`, requires three responders, +starts incomplete, serializes phase artifacts, and is publicly importable. Test that existing +`CouncilResult` construction still works with `elite is None`. + +**Step 2: Verify red** + +Run: `pytest tests/test_elite_mode.py -q` +Expected: collection/import failure because `EliteResult` does not exist. + +**Step 3: Implement minimally** + +Add `ELITE_PROTOCOL_VERSION = "elite_v1"` and `ELITE_MIN_RESPONDERS = 3`. Add a Pydantic +`EliteResult` containing `protocol_version`, `required_responders`, `completed`, +`failure_reason`, `initial_answers`, `critiques`, and `revisions`. Add +`elite: EliteResult | None = None` to `CouncilResult` and export the public symbols. + +**Step 4: Verify green** + +Run: `pytest tests/test_elite_mode.py -q` +Expected: result-contract tests pass. + +**Step 5: Commit** + +`git add src/conclave/models.py src/conclave/__init__.py tests/test_elite_mode.py && git commit -m "feat(elite): define protocol result contract"` + +### Task 2: Add evidence-audit and revision prompts + +**Files:** +- Modify: `src/conclave/prompts.py` +- Test: `tests/test_elite_mode.py` + +**Step 1: Write failing tests** + +Test prompt builders with three model answers. Assert stable Model A/B/C aliases, stable +answer IDs, no provider/model names, explicit supported/conflicting/externally-unverified +categories, no-invented-citations instruction, original answer inclusion in revision, and +deterministic output for the same input. + +**Step 2: Verify red** + +Run: `pytest tests/test_elite_mode.py -q` +Expected: failures because elite prompt builders are missing. + +**Step 3: Implement minimally** + +Add `ELITE_CRITIC_SYSTEM`, `ELITE_REVISION_SYSTEM`, and narrow builder functions that accept +the original prompt and `ModelAnswer` sequences. Reuse the established anonymization style; +represent panel entries using aliases plus answer IDs, never provider names. Keep the elite +protocol version separate from the synthesis prompt version. + +**Step 4: Verify green** + +Run: `pytest tests/test_elite_mode.py -q` +Expected: prompt tests pass. + +**Step 5: Commit** + +`git add src/conclave/prompts.py tests/test_elite_mode.py && git commit -m "feat(elite): add evidence audit and revision prompts"` + +### Task 3: Implement the three-stage mode + +**Files:** +- Modify: `src/conclave/modes.py` +- Test: `tests/test_elite_mode.py` + +**Step 1: Write failing tests** + +Add async tests using the repository's call seams for: independent initial fan-out; critique +calls only after three initials; revision calls only after three critiques; successful 4-to-3 +partial failure; initial, critique, and revision gate failures; correct stage artifacts; and +no calls after a failed gate. + +**Step 2: Verify red** + +Run: `pytest tests/test_elite_mode.py -q` +Expected: failures because `run_elite` is missing. + +**Step 3: Implement minimally** + +Implement `run_elite(council, prompt)` using the existing concurrent `fan_out` and provider +call seams. Centralize the strict success-count gate. Preserve provider-side errors in phase +artifacts, stop immediately below three successes, never raise, and set `result.answers` to +successful revisions only after the revision gate passes. + +**Step 4: Verify green** + +Run: `pytest tests/test_elite_mode.py -q` +Expected: mode tests pass. + +**Step 5: Commit** + +`git add src/conclave/modes.py tests/test_elite_mode.py && git commit -m "feat(elite): implement evidence audited decision flow"` + +### Task 4: Wire library synthesis and verdict behavior + +**Files:** +- Modify: `src/conclave/council.py` +- Test: `tests/test_elite_mode.py` +- Test: `tests/test_cache.py` + +**Step 1: Write failing tests** + +Test `elite` and `elite_sync`, final synthesis receiving revised answers, existing +`_apply_verdict` producing the canonical verdict only on completed runs, incomplete runs +having no synthesis/verdict, and cache keys/results remaining isolated from other modes. + +**Step 2: Verify red** + +Run: `pytest tests/test_elite_mode.py tests/test_cache.py -q` +Expected: failures because Council does not expose or dispatch elite. + +**Step 3: Implement minimally** + +Add async/sync public wrappers routed through `_cached_run`. On a completed `run_elite` +result, invoke existing `_synthesize` followed by `_apply_verdict`; skip both when incomplete. +Use the existing mode field in cache identity and preserve normal-mode behavior. + +**Step 4: Verify green** + +Run: `pytest tests/test_elite_mode.py tests/test_cache.py -q` +Expected: library and cache tests pass. + +**Step 5: Commit** + +`git add src/conclave/council.py tests/test_elite_mode.py tests/test_cache.py && git commit -m "feat(elite): expose elite council API"` + +### Task 5: Make every elite phase auditable + +**Files:** +- Modify: `src/conclave/manifest.py` +- Modify: `src/conclave/providers.py` +- Modify: `src/conclave/council.py` +- Test: `tests/test_manifest_all_modes.py` +- Test: `tests/test_secret_safety_matrix.py` + +**Step 1: Write failing tests** + +Test that successful elite manifests have initial/critique/revision receipts, unique +`providers_called`, aggregate usage for all recorded member phases, cache-hit manifests, +and `verified_no_secrets`. Inject secret-like provider errors and assert serialization stays +redacted. Test incomplete runs retain receipts for completed/attempted phases. + +**Step 2: Verify red** + +Run: `pytest tests/test_manifest_all_modes.py tests/test_secret_safety_matrix.py -q` +Expected: failures because receipts lack phase provenance and elite calls are absent. + +**Step 3: Implement minimally** + +Add optional `phase` to `ProviderExecutionReceipt` and `receipt_from_answer`. Build an elite +manifest from flattened phase artifacts, keeping `providers_called` unique and phase receipts +repeatable. Aggregate usage and latency using existing manifest conventions, record skipped +members without secrets, then rerun the existing secret-material scan before stamping safety. + +**Step 4: Verify green** + +Run: `pytest tests/test_manifest_all_modes.py tests/test_secret_safety_matrix.py -q` +Expected: manifest and secret-safety tests pass. + +**Step 5: Commit** + +`git add src/conclave/manifest.py src/conclave/providers.py src/conclave/council.py tests/test_manifest_all_modes.py tests/test_secret_safety_matrix.py && git commit -m "feat(elite): audit every protocol phase"` + +### Task 6: Add the CLI surface + +**Files:** +- Modify: `src/conclave/cli.py` +- Test: `tests/test_cli.py` +- Test: `tests/test_cli_verdict.py` + +**Step 1: Write failing tests** + +Test `--mode elite` dispatch, human completion summary, full JSON serialization, incomplete +exit code 1 with reason, and explicit rejection of `--stream`. Confirm all existing modes' +CLI tests remain unchanged. + +**Step 2: Verify red** + +Run: `pytest tests/test_cli.py tests/test_cli_verdict.py -q` +Expected: elite is rejected as an invalid mode. + +**Step 3: Implement minimally** + +Add elite to mode validation/help and dispatch to `Council.elite_sync`. Render a concise +protocol summary plus the existing verdict panel. JSON emits the complete `CouncilResult`. +Return exit 1 for incomplete elite runs and reject stream before provider calls. + +**Step 4: Verify green** + +Run: `pytest tests/test_cli.py tests/test_cli_verdict.py -q` +Expected: CLI tests pass. + +**Step 5: Commit** + +`git add src/conclave/cli.py tests/test_cli.py tests/test_cli_verdict.py && git commit -m "feat(elite): add quality first CLI mode"` + +### Task 7: Document behavior and verify the release candidate + +**Files:** +- Modify: `README.md` +- Modify: `docs/PRODUCT_DESIGN_DOCUMENT.md` +- Modify: `SYSTEM_CONTEXT_DIAGRAM.md` +- Modify: `DOCUMENTATION_INDEX.md` +- Modify: `CHANGELOG.md` + +**Step 1: Update existing docs** + +Document the quality-first intent, three-member invariant, exact call stages, partial-failure +semantics, latency/cost trade-off, CLI/library examples, manifest phase receipts, and lack of +streaming. Keep the three core docs under 500 lines where feasible and do not claim a release. + +**Step 2: Run focused verification** + +Run: `pytest tests/test_elite_mode.py tests/test_manifest_all_modes.py tests/test_secret_safety_matrix.py tests/test_cli.py tests/test_cli_verdict.py tests/test_cache.py -q` +Expected: all focused tests pass. + +**Step 3: Run full verification** + +Run: `pytest -q` +Expected: zero failures. + +Run: `ruff check .` +Expected: zero lint errors. + +Run: `ruff format --check .` +Expected: all files already formatted. + +**Step 4: Review safety and scope** + +Run: `git diff --check && git status --short && git diff --stat main...HEAD` +Expected: no whitespace errors; only planned code, tests, and existing documentation changed. +Confirm no `.env`, credentials, generated caches, release tags, or publishing changes exist. + +**Step 5: Commit** + +`git add README.md docs/PRODUCT_DESIGN_DOCUMENT.md SYSTEM_CONTEXT_DIAGRAM.md DOCUMENTATION_INDEX.md CHANGELOG.md && git commit -m "docs: specify elite decision protocol"` + From e133b8f8fda4d04a453d77ac699cf8582b23f570 Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Fri, 17 Jul 2026 04:56:28 -0400 Subject: [PATCH 02/21] feat(elite): define protocol result contract --- src/conclave/__init__.py | 6 +++++ src/conclave/models.py | 16 +++++++++++++ tests/test_elite_mode.py | 50 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 72 insertions(+) create mode 100644 tests/test_elite_mode.py diff --git a/src/conclave/__init__.py b/src/conclave/__init__.py index 7c37492..2f2cfa1 100644 --- a/src/conclave/__init__.py +++ b/src/conclave/__init__.py @@ -48,9 +48,12 @@ VerdictExtraction, ) from .models import ( + ELITE_MIN_RESPONDERS, + ELITE_PROTOCOL_VERSION, AdversarialResult, CouncilResult, DebateRound, + EliteResult, ModelAnswer, StreamEvent, TokenUsage, @@ -83,6 +86,9 @@ "TokenUsage", "DebateRound", "AdversarialResult", + "EliteResult", + "ELITE_PROTOCOL_VERSION", + "ELITE_MIN_RESPONDERS", "StreamEvent", "ConclaveConfig", "load_config", diff --git a/src/conclave/models.py b/src/conclave/models.py index 4856341..a0ef74b 100644 --- a/src/conclave/models.py +++ b/src/conclave/models.py @@ -15,6 +15,9 @@ ProviderVote, ) +ELITE_PROTOCOL_VERSION = "elite_v1" +ELITE_MIN_RESPONDERS = 3 + # NOTE: ``ModelHarnessManifest`` is imported at the BOTTOM of this module (just # before the ``model_rebuild()`` calls), not here. ``manifest`` imports # :class:`TokenUsage` from this module, so importing it before ``TokenUsage`` is @@ -84,6 +87,18 @@ def latency_ms(self) -> float: return self.latency_s * 1000.0 +class EliteResult(BaseModel): + """Phase artifacts and completion state for an elite protocol run.""" + + protocol_version: str = ELITE_PROTOCOL_VERSION + required_responders: int = Field(default=ELITE_MIN_RESPONDERS, ge=ELITE_MIN_RESPONDERS) + completed: bool = False + failure_reason: str | None = None + initial_answers: list[ModelAnswer] = Field(default_factory=list) + critiques: list[ModelAnswer] = Field(default_factory=list) + revisions: list[ModelAnswer] = Field(default_factory=list) + + class StreamEvent(BaseModel): """One incremental event from a streaming council run (issue #7). @@ -284,6 +299,7 @@ class CouncilResult(BaseModel): rounds: list[DebateRound] = Field(default_factory=list) adversarial: AdversarialResult | None = None vote: VoteResult | None = None + elite: EliteResult | None = None cached: bool = False converged: bool = False convergence_score: float | None = None diff --git a/tests/test_elite_mode.py b/tests/test_elite_mode.py new file mode 100644 index 0000000..3a30723 --- /dev/null +++ b/tests/test_elite_mode.py @@ -0,0 +1,50 @@ +"""Tests for the Elite Decision Protocol.""" + +import pytest +from pydantic import ValidationError + +from conclave import ( + ELITE_MIN_RESPONDERS, + ELITE_PROTOCOL_VERSION, + CouncilResult, + EliteResult, + ModelAnswer, +) + + +def test_elite_result_defaults_to_incomplete_v1_protocol() -> None: + result = EliteResult() + + assert ELITE_PROTOCOL_VERSION == "elite_v1" + assert ELITE_MIN_RESPONDERS == 3 + assert result.protocol_version == ELITE_PROTOCOL_VERSION + assert result.required_responders == ELITE_MIN_RESPONDERS + assert result.completed is False + assert result.failure_reason is None + + +def test_elite_result_rejects_fewer_than_three_required_responders() -> None: + with pytest.raises(ValidationError): + EliteResult(required_responders=2) + + +def test_elite_result_serializes_phase_artifacts() -> None: + initial = ModelAnswer(name="member-a", model_id="provider/model", answer="initial") + critique = ModelAnswer(name="member-b", model_id="provider/model", answer="critique") + revision = ModelAnswer(name="member-a", model_id="provider/model", answer="revision") + + serialized = EliteResult( + initial_answers=[initial], + critiques=[critique], + revisions=[revision], + ).model_dump() + + assert serialized["initial_answers"][0]["answer"] == "initial" + assert serialized["critiques"][0]["answer"] == "critique" + assert serialized["revisions"][0]["answer"] == "revision" + + +def test_existing_council_result_defaults_elite_to_none() -> None: + result = CouncilResult(prompt="Should we proceed?") + + assert result.elite is None From 6b8206e02499a4596a41a585014f08bf0604e6eb Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Fri, 17 Jul 2026 05:00:42 -0400 Subject: [PATCH 03/21] feat(elite): add evidence audit and revision prompts --- src/conclave/prompts.py | 62 +++++++++++++++++++++++++++ tests/test_elite_mode.py | 91 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 153 insertions(+) diff --git a/src/conclave/prompts.py b/src/conclave/prompts.py index 792c1c3..ff06994 100644 --- a/src/conclave/prompts.py +++ b/src/conclave/prompts.py @@ -8,6 +8,8 @@ from __future__ import annotations +from collections.abc import Sequence + from .models import ModelAnswer # Version identifier for the synthesis/judge prompt *set*. Bump this string @@ -59,6 +61,26 @@ "the material provided; do not invent positions." ) +ELITE_CRITIC_SYSTEM = ( + "You are an evidence auditor in an elite decision council. Independently " + "stress-test the anonymized answers against the original prompt. Organize " + "your audit into three explicit categories: SUPPORTED claims, CONFLICTING " + "claims, and EXTERNALLY UNVERIFIED claims. Distinguish evidence supplied in " + "the answers from assumptions that would require outside verification. Do " + "not invent citations, sources, quotations, measurements, or facts. Cite " + "the displayed answer IDs when identifying a claim. Preserve meaningful " + "minority positions and be precise about uncertainty." +) + +ELITE_REVISION_SYSTEM = ( + "You are revising your independent answer after an evidence audit by an " + "elite decision council. Use the anonymized initial panel and critiques to " + "correct unsupported claims, resolve conflicts where the supplied material " + "permits, and clearly mark externally unverified claims. Do not invent " + "citations, sources, quotations, measurements, or facts. Produce a complete " + "standalone answer to the original prompt, not a description of your edits." +) + VOTE_SYSTEM = ( "You are a voting member of a multi-model council. You will be given a " @@ -144,3 +166,43 @@ def judge_user(prompt: str, proposer: str, proposal_text: str, critique_blocks: f"CRITIQUES:\n\n{critique_blocks}\n\n" "Now issue your verdict and the strengthened final answer." ) + + +def _elite_artifact_block(answers: Sequence[ModelAnswer], phase: str) -> str: + """Render successful artifacts with stable position aliases and safe IDs.""" + parts: list[str] = [] + for index, answer in enumerate(answer for answer in answers if answer.ok): + alias = LETTERS[index] + answer_id = answer.answer_id or f"{phase}-{index + 1:03d}" + parts.append(f"### Model {alias} {phase} (Answer ID: {answer_id})\n{answer.answer}") + return "\n\n".join(parts) + + +def elite_critic_user(prompt: str, answers: Sequence[ModelAnswer]) -> str: + """Build an anonymized, evidence-audit prompt for an elite critic.""" + panel = _elite_artifact_block(answers, "initial answer") + return ( + f"Original prompt:\n{prompt}\n\n" + f"Anonymized initial panel:\n\n{panel}\n\n" + "Audit the panel using the required evidence categories." + ) + + +def elite_revision_user( + prompt: str, + original_answer: ModelAnswer, + answers: Sequence[ModelAnswer], + critiques: Sequence[ModelAnswer], +) -> str: + """Build an anonymized elite revision prompt with all prior artifacts.""" + original_id = original_answer.answer_id or "initial-original" + initial_panel = _elite_artifact_block(answers, "initial answer") + critique_panel = _elite_artifact_block(critiques, "critique") + return ( + f"Original prompt:\n{prompt}\n\n" + f"Your original answer (Answer ID: {original_id}):\n" + f"{original_answer.answer or ''}\n\n" + f"Anonymized initial panel:\n\n{initial_panel}\n\n" + f"Anonymized evidence critiques:\n\n{critique_panel}\n\n" + "Now provide your evidence-aware revised answer." + ) diff --git a/tests/test_elite_mode.py b/tests/test_elite_mode.py index 3a30723..b777cf9 100644 --- a/tests/test_elite_mode.py +++ b/tests/test_elite_mode.py @@ -10,6 +10,12 @@ EliteResult, ModelAnswer, ) +from conclave.prompts import ( + ELITE_CRITIC_SYSTEM, + ELITE_REVISION_SYSTEM, + elite_critic_user, + elite_revision_user, +) def test_elite_result_defaults_to_incomplete_v1_protocol() -> None: @@ -48,3 +54,88 @@ def test_existing_council_result_defaults_elite_to_none() -> None: result = CouncilResult(prompt="Should we proceed?") assert result.elite is None + + +def _elite_prompt_answers() -> list[ModelAnswer]: + return [ + ModelAnswer( + name="alpha-provider", + model_id="vendor-one/secret-model", + answer="Adopt option one because the measured risk is lower.", + answer_id="initial-001", + ), + ModelAnswer( + name="beta-provider", + model_id="vendor-two/hidden-model", + answer="Adopt option two because it preserves flexibility.", + answer_id="initial-002", + ), + ModelAnswer( + name="gamma-provider", + model_id="vendor-three/private-model", + answer="Delay the choice until the key assumption is tested.", + answer_id="initial-003", + ), + ] + + +def test_elite_critic_prompt_is_anonymized_evidence_audit() -> None: + answers = _elite_prompt_answers() + + built = elite_critic_user("Choose the strongest option.", answers) + + assert "Model A" in built + assert "Model B" in built + assert "Model C" in built + assert "initial-001" in built + assert "initial-002" in built + assert "initial-003" in built + for answer in answers: + assert answer.name not in built + assert answer.model_id not in built + assert "SUPPORTED" in ELITE_CRITIC_SYSTEM + assert "CONFLICTING" in ELITE_CRITIC_SYSTEM + assert "EXTERNALLY UNVERIFIED" in ELITE_CRITIC_SYSTEM + assert "Do not invent citations" in ELITE_CRITIC_SYSTEM + + +def test_elite_revision_prompt_includes_original_panel_and_critiques() -> None: + answers = _elite_prompt_answers() + critiques = [ + ModelAnswer( + name=f"critic-{index}", + model_id=f"critic-vendor/model-{index}", + answer=f"Evidence audit {index}", + answer_id=f"critique-{index:03d}", + ) + for index in range(1, 4) + ] + + built = elite_revision_user("Choose the strongest option.", answers[1], answers, critiques) + + assert "Your original answer" in built + assert answers[1].answer in built + assert "initial-002" in built + assert all(f"Model {letter}" in built for letter in "ABC") + assert all(critique.answer_id in built for critique in critiques) + for artifact in [*answers, *critiques]: + assert artifact.name not in built + assert artifact.model_id not in built + assert "Do not invent citations" in ELITE_REVISION_SYSTEM + + +def test_elite_prompt_builders_are_deterministic() -> None: + answers = _elite_prompt_answers() + critiques = [ + ModelAnswer( + name="critic-a", + model_id="vendor/model", + answer="Audit the evidence base.", + answer_id="critique-001", + ) + ] + + assert elite_critic_user("Decide.", answers) == elite_critic_user("Decide.", answers) + assert elite_revision_user("Decide.", answers[0], answers, critiques) == elite_revision_user( + "Decide.", answers[0], answers, critiques + ) From 77779f9f44657a94fbdd3ec7061dba1618d050e1 Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Fri, 17 Jul 2026 05:04:32 -0400 Subject: [PATCH 04/21] feat(elite): implement evidence audited decision flow --- src/conclave/modes.py | 114 ++++++++++++++++++++++- tests/test_elite_mode.py | 189 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 302 insertions(+), 1 deletion(-) diff --git a/src/conclave/modes.py b/src/conclave/modes.py index 4e1927a..db9cc3e 100644 --- a/src/conclave/modes.py +++ b/src/conclave/modes.py @@ -28,7 +28,14 @@ from . import prompts from .logging import get_logger -from .models import AdversarialResult, CouncilResult, DebateRound, ModelAnswer, VoteResult +from .models import ( + AdversarialResult, + CouncilResult, + DebateRound, + EliteResult, + ModelAnswer, + VoteResult, +) from .registry import key_present if TYPE_CHECKING: # avoid a circular import at runtime; only needed for typing @@ -37,6 +44,111 @@ logger = get_logger("modes") +async def run_elite(council: Council, prompt: str) -> CouncilResult: + """Run the three-stage Elite Decision Protocol without final synthesis. + + The protocol requires at least three successful responders after each of its + independent-answer, evidence-critique, and revision phases. Provider failures + remain attached to their phase artifacts; a failed gate stops later calls and + returns the latest fully qualified answers to the original prompt. + """ + members, skipped = council._available_members() + elite = EliteResult() + result = CouncilResult(prompt=prompt, mode="elite", skipped=skipped, elite=elite) + + def initial_messages(_name: str, _model_id: str) -> list[dict[str, str]]: + return [{"role": "user", "content": prompt}] + + elite.initial_answers = await council.fan_out(members, initial_messages) + initial_successes = _elite_gate(elite, "initial", elite.initial_answers) + result.answers = initial_successes + if elite.failure_reason is not None: + return result + + critique_members = _elite_surviving_members(members, initial_successes) + critique_messages = _elite_critique_messages_for(prompt, initial_successes) + elite.critiques = await council.fan_out(critique_members, critique_messages) + critique_successes = _elite_gate(elite, "critique", elite.critiques) + if elite.failure_reason is not None: + return result + + revision_members = _elite_surviving_members(critique_members, critique_successes) + initials_by_name = {answer.name: answer for answer in initial_successes} + revision_messages = _elite_revision_messages_for( + prompt, + initials_by_name, + initial_successes, + critique_successes, + ) + elite.revisions = await council.fan_out(revision_members, revision_messages) + revision_successes = _elite_gate(elite, "revision", elite.revisions) + if elite.failure_reason is not None: + return result + + result.answers = revision_successes + elite.completed = True + return result + + +def _elite_gate( + elite: EliteResult, + phase: str, + artifacts: list[ModelAnswer], +) -> list[ModelAnswer]: + """Return successful artifacts and record a strict responder-gate failure.""" + successful = [answer for answer in artifacts if answer.ok] + if len(successful) < elite.required_responders: + elite.failure_reason = ( + f"{phase} phase required {elite.required_responders} successful responders; " + f"got {len(successful)}" + ) + logger.warning(elite.failure_reason) + return successful + + +def _elite_surviving_members( + members: list[tuple[str, str]], + successful: list[ModelAnswer], +) -> list[tuple[str, str]]: + """Preserve council order while retaining only successful phase members.""" + names = {answer.name for answer in successful} + return [(name, model_id) for name, model_id in members if name in names] + + +def _elite_critique_messages_for(prompt: str, answers: list[ModelAnswer]): + """Build the shared evidence-audit messages for each surviving member.""" + messages = [ + {"role": "system", "content": prompts.ELITE_CRITIC_SYSTEM}, + {"role": "user", "content": prompts.elite_critic_user(prompt, answers)}, + ] + return lambda _name, _model_id: messages + + +def _elite_revision_messages_for( + prompt: str, + initials_by_name: dict[str, ModelAnswer], + initial_answers: list[ModelAnswer], + critiques: list[ModelAnswer], +): + """Build member-specific evidence-aware revision messages.""" + + def messages_for(name: str, _model_id: str) -> list[dict[str, str]]: + return [ + {"role": "system", "content": prompts.ELITE_REVISION_SYSTEM}, + { + "role": "user", + "content": prompts.elite_revision_user( + prompt, + initials_by_name[name], + initial_answers, + critiques, + ), + }, + ] + + return messages_for + + async def run_vote( council: Council, prompt: str, diff --git a/tests/test_elite_mode.py b/tests/test_elite_mode.py index b777cf9..574a106 100644 --- a/tests/test_elite_mode.py +++ b/tests/test_elite_mode.py @@ -6,16 +6,52 @@ from conclave import ( ELITE_MIN_RESPONDERS, ELITE_PROTOCOL_VERSION, + Council, CouncilResult, EliteResult, ModelAnswer, ) +from conclave.config import ConclaveConfig +from conclave.modes import run_elite from conclave.prompts import ( ELITE_CRITIC_SYSTEM, ELITE_REVISION_SYSTEM, elite_critic_user, elite_revision_user, ) +from tests.conftest import make_response + + +def _all_keys(monkeypatch) -> None: + for variable in ( + "XAI_API_KEY", + "GEMINI_API_KEY", + "ANTHROPIC_API_KEY", + "PERPLEXITY_API_KEY", + ): + monkeypatch.setenv(variable, "dummy-key") + + +def _elite_config() -> ConclaveConfig: + return ConclaveConfig( + models={ + "grok": "xai/grok-4.3", + "gemini": "gemini/gemini-2.5-pro", + "claude": "anthropic/claude-sonnet-4-6", + "perplexity": "perplexity/sonar-pro", + }, + councils={"default": ["grok", "gemini", "claude", "perplexity"]}, + synthesizer="claude", + ) + + +def _phase(messages) -> str: + system = next((message["content"] for message in messages if message["role"] == "system"), "") + if system == ELITE_CRITIC_SYSTEM: + return "critique" + if system == ELITE_REVISION_SYSTEM: + return "revision" + return "initial" def test_elite_result_defaults_to_incomplete_v1_protocol() -> None: @@ -139,3 +175,156 @@ def test_elite_prompt_builders_are_deterministic() -> None: assert elite_revision_user("Decide.", answers[0], answers, critiques) == elite_revision_user( "Decide.", answers[0], answers, critiques ) + + +async def test_run_elite_executes_three_stages_and_captures_artifacts( + monkeypatch, patch_call_model +) -> None: + _all_keys(monkeypatch) + calls: list[tuple[str, str]] = [] + + def handler(model_id, messages): + phase = _phase(messages) + calls.append((phase, model_id)) + return make_response(f"{phase} from {model_id}") + + patch_call_model(handler) + council = Council( + models=["grok", "gemini", "perplexity"], + config=_elite_config(), + extract_verdict=False, + ) + + result = await run_elite(council, "Choose the strongest option.") + + assert result.mode == "elite" + assert result.elite is not None + assert result.elite.completed is True + assert result.elite.failure_reason is None + assert len(result.elite.initial_answers) == 3 + assert len(result.elite.critiques) == 3 + assert len(result.elite.revisions) == 3 + assert [phase for phase, _model in calls] == ["initial"] * 3 + ["critique"] * 3 + [ + "revision" + ] * 3 + assert all(answer.ok for answer in result.answers) + assert all(answer.answer.startswith("revision") for answer in result.answers) + + +@pytest.mark.parametrize("failed_phase", ["initial", "critique", "revision"]) +async def test_run_elite_four_members_survives_one_failure_in_any_phase( + monkeypatch, patch_call_model, failed_phase +) -> None: + _all_keys(monkeypatch) + + def handler(model_id, messages): + phase = _phase(messages) + if phase == failed_phase and model_id == "gemini/gemini-2.5-pro": + raise RuntimeError(f"{phase} provider failure") + return make_response(f"{phase} from {model_id}") + + patch_call_model(handler) + council = Council( + models=["grok", "gemini", "claude", "perplexity"], + config=_elite_config(), + extract_verdict=False, + ) + + result = await run_elite(council, "Decide.") + + assert result.elite is not None + assert result.elite.completed is True + assert len(result.successful_answers) == 3 + assert all(answer.answer.startswith("revision") for answer in result.answers) + phase_artifacts = { + "initial": result.elite.initial_answers, + "critique": result.elite.critiques, + "revision": result.elite.revisions, + } + assert sum(not answer.ok for answer in phase_artifacts[failed_phase]) == 1 + + +async def test_run_elite_stops_after_failed_initial_gate(monkeypatch, patch_call_model) -> None: + _all_keys(monkeypatch) + phases: list[str] = [] + + def handler(model_id, messages): + phase = _phase(messages) + phases.append(phase) + if model_id == "gemini/gemini-2.5-pro": + raise RuntimeError("initial unavailable") + return make_response(f"{phase} from {model_id}") + + patch_call_model(handler) + council = Council( + models=["grok", "gemini", "perplexity"], + config=_elite_config(), + extract_verdict=False, + ) + + result = await run_elite(council, "Decide.") + + assert result.elite is not None + assert result.elite.completed is False + assert result.elite.failure_reason == "initial phase required 3 successful responders; got 2" + assert phases == ["initial"] * 3 + assert result.elite.critiques == [] + assert result.elite.revisions == [] + assert len(result.answers) == 2 + assert all(answer.answer.startswith("initial") for answer in result.answers) + + +async def test_run_elite_stops_after_failed_critique_gate(monkeypatch, patch_call_model) -> None: + _all_keys(monkeypatch) + phases: list[str] = [] + + def handler(model_id, messages): + phase = _phase(messages) + phases.append(phase) + if phase == "critique" and model_id == "gemini/gemini-2.5-pro": + raise RuntimeError("critic unavailable") + return make_response(f"{phase} from {model_id}") + + patch_call_model(handler) + council = Council( + models=["grok", "gemini", "perplexity"], + config=_elite_config(), + extract_verdict=False, + ) + + result = await run_elite(council, "Decide.") + + assert result.elite is not None + assert result.elite.completed is False + assert result.elite.failure_reason == "critique phase required 3 successful responders; got 2" + assert phases == ["initial"] * 3 + ["critique"] * 3 + assert len(result.elite.critiques) == 3 + assert result.elite.revisions == [] + assert len(result.answers) == 3 + assert all(answer.answer.startswith("initial") for answer in result.answers) + + +async def test_run_elite_stops_after_failed_revision_gate(monkeypatch, patch_call_model) -> None: + _all_keys(monkeypatch) + + def handler(model_id, messages): + phase = _phase(messages) + if phase == "revision" and model_id == "gemini/gemini-2.5-pro": + raise RuntimeError("revision unavailable") + return make_response(f"{phase} from {model_id}") + + patch_call_model(handler) + council = Council( + models=["grok", "gemini", "perplexity"], + config=_elite_config(), + extract_verdict=False, + ) + + result = await run_elite(council, "Decide.") + + assert result.elite is not None + assert result.elite.completed is False + assert result.elite.failure_reason == "revision phase required 3 successful responders; got 2" + assert len(result.elite.revisions) == 3 + assert len(result.answers) == 3 + assert all(answer.answer.startswith("initial") for answer in result.answers) From 3ec90e3c2b61dd431c9f6dbb7ee65639e0c08536 Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Fri, 17 Jul 2026 05:08:03 -0400 Subject: [PATCH 05/21] feat(elite): expose elite council API --- src/conclave/council.py | 22 +++++++ tests/test_cache.py | 31 ++++++++++ tests/test_elite_mode.py | 126 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 179 insertions(+) diff --git a/src/conclave/council.py b/src/conclave/council.py index 86d7b34..ed28c59 100644 --- a/src/conclave/council.py +++ b/src/conclave/council.py @@ -827,6 +827,24 @@ async def adversarial(self, prompt: str, proposer: str | None = None) -> Council proposer=proposer, ) + async def elite(self, prompt: str) -> CouncilResult: + """Run the evidence-audited Elite Decision Protocol. + + Completed runs synthesize the council's revised answers and apply the + canonical structured verdict. A run that fails any three-responder gate + returns its phase artifacts without synthesis or verdict extraction. + """ + from .modes import run_elite + + async def run() -> CouncilResult: + result = await run_elite(self, prompt) + if result.elite is not None and result.elite.completed: + await self._synthesize(result) + await self._apply_verdict(result) + return result + + return await self._cached_run(prompt, "elite", run) + async def aclose(self) -> None: """Close the shared pooled HTTP client. @@ -939,6 +957,10 @@ def adversarial_sync(self, prompt: str, proposer: str | None = None) -> CouncilR lambda: self.adversarial(prompt, proposer=proposer), "adversarial_sync" ) + def elite_sync(self, prompt: str) -> CouncilResult: + """Synchronous wrapper around :meth:`elite`.""" + return self._run_sync(lambda: self.elite(prompt), "elite_sync") + async def vote(self, prompt: str, choices: list[str]) -> CouncilResult: """Run a constrained-choice vote. See :func:`conclave.modes.run_vote`. diff --git a/tests/test_cache.py b/tests/test_cache.py index d6b7688..544960c 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -191,6 +191,37 @@ async def test_changing_mode_misses(monkeypatch, counting_call_model, cache_home assert r.cached is False +async def test_elite_cache_hit_preserves_artifacts_and_isolated_mode( + monkeypatch, counting_call_model, cache_home +): + _set_keys(monkeypatch) + council = Council( + models=["grok", "gemini", "perplexity"], + synthesizer="claude", + config=_config(), + cache=True, + extract_verdict=False, + ) + + live = await council.elite("same prompt") + calls_after_live = counting_call_model["n"] + cached = await council.elite("same prompt") + + assert live.cached is False + assert live.elite is not None + assert live.elite.completed is True + assert cached.cached is True + assert cached.elite == live.elite + assert counting_call_model["n"] == calls_after_live + + normal = await council.ask("same prompt") + + assert normal.cached is False + assert normal.elite is None + assert counting_call_model["n"] > calls_after_live + assert len(list(cache_home.glob("*.json"))) == 2 + + async def test_changing_model_id_misses(monkeypatch, counting_call_model, cache_home): """Same friendly name but a different resolved model id -> different key.""" _set_keys(monkeypatch) diff --git a/tests/test_elite_mode.py b/tests/test_elite_mode.py index 574a106..4ad101f 100644 --- a/tests/test_elite_mode.py +++ b/tests/test_elite_mode.py @@ -1,5 +1,7 @@ """Tests for the Elite Decision Protocol.""" +import json + import pytest from pydantic import ValidationError @@ -54,6 +56,39 @@ def _phase(messages) -> str: return "initial" +def _is_verdict_call(messages) -> bool: + return bool(messages) and messages[0].get("content", "").startswith( + "You are the verdict extractor" + ) + + +def _elite_verdict_json() -> str: + members = ["grok", "gemini", "perplexity"] + return json.dumps( + { + "verdict_applies": True, + "verdict_type": "decision", + "headline": "Proceed.", + "recommendation": "Choose the strongest option.", + "positions": [ + { + "label": "proceed", + "summary": "The revised council agrees.", + "providers": members, + "evidence_answer_ids": [], + } + ], + "provider_votes": [ + {"provider": member, "position_label": "proceed"} for member in members + ], + "minority_reports": [], + "conflicts": [], + "caveats": [], + "dissent_summary": None, + } + ) + + def test_elite_result_defaults_to_incomplete_v1_protocol() -> None: result = EliteResult() @@ -328,3 +363,94 @@ def handler(model_id, messages): assert len(result.elite.revisions) == 3 assert len(result.answers) == 3 assert all(answer.answer.startswith("initial") for answer in result.answers) + + +async def test_council_elite_synthesizes_revisions_and_applies_canonical_verdict( + monkeypatch, patch_call_model +) -> None: + _all_keys(monkeypatch) + synthesis_inputs: list[str] = [] + + def handler(model_id, messages): + if _is_verdict_call(messages): + return make_response(_elite_verdict_json()) + system = next( + (message["content"] for message in messages if message["role"] == "system"), "" + ) + if system.startswith("You are the synthesizer of a council"): + synthesis_inputs.append(messages[-1]["content"]) + return make_response("elite synthesis") + phase = _phase(messages) + return make_response(f"{phase} from {model_id}") + + patch_call_model(handler) + council = Council( + models=["grok", "gemini", "perplexity"], + config=_elite_config(), + ) + + result = await council.elite("Choose the strongest option.") + + assert result.elite is not None + assert result.elite.completed is True + assert result.synthesis == "elite synthesis" + assert len(synthesis_inputs) == 1 + assert "revision from xai/grok-4.3" in synthesis_inputs[0] + assert "initial from xai/grok-4.3" not in synthesis_inputs[0] + assert result.verdict is not None + assert result.consensus_score == result.verdict.consensus_score == 1.0 + assert result.provider_votes == result.verdict.provider_votes + + +async def test_council_elite_incomplete_skips_synthesis_and_verdict( + monkeypatch, patch_call_model +) -> None: + _all_keys(monkeypatch) + calls = {"synthesis": 0, "verdict": 0} + + def handler(model_id, messages): + if model_id == "gemini/gemini-2.5-pro": + raise RuntimeError("initial unavailable") + return make_response(f"initial from {model_id}") + + async def unexpected_synthesis(_result): + calls["synthesis"] += 1 + + async def unexpected_verdict(_result): + calls["verdict"] += 1 + + patch_call_model(handler) + council = Council( + models=["grok", "gemini", "perplexity"], + config=_elite_config(), + ) + monkeypatch.setattr(council, "_synthesize", unexpected_synthesis) + monkeypatch.setattr(council, "_apply_verdict", unexpected_verdict) + + result = await council.elite("Choose.") + + assert result.elite is not None + assert result.elite.completed is False + assert result.synthesis is None + assert result.verdict is None + assert calls == {"synthesis": 0, "verdict": 0} + + +def test_council_elite_sync_exposes_completed_protocol(monkeypatch, patch_call_model) -> None: + _all_keys(monkeypatch) + + def handler(model_id, messages): + return make_response(f"{_phase(messages)} from {model_id}") + + patch_call_model(handler) + council = Council( + models=["grok", "gemini", "perplexity"], + config=_elite_config(), + extract_verdict=False, + ) + + result = council.elite_sync("Choose.") + + assert result.elite is not None + assert result.elite.completed is True + assert result.synthesis is not None From f6c5451378b9536cf631f7b6be9e04fbdfac00c9 Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Fri, 17 Jul 2026 05:14:07 -0400 Subject: [PATCH 06/21] feat(elite): audit every protocol phase --- src/conclave/council.py | 70 +++++++++++++++++- src/conclave/manifest.py | 4 + src/conclave/providers.py | 8 +- tests/test_manifest_all_modes.py | 113 +++++++++++++++++++++++++++++ tests/test_secret_safety_matrix.py | 26 +++++++ 5 files changed, 217 insertions(+), 4 deletions(-) diff --git a/src/conclave/council.py b/src/conclave/council.py index ed28c59..39baae9 100644 --- a/src/conclave/council.py +++ b/src/conclave/council.py @@ -310,9 +310,16 @@ def _ensure_manifest(self, result: CouncilResult, mode: str) -> None: if result.manifest is not None: return members, skipped = self._available_members() - result.manifest = self._build_manifest( - mode=mode, members=members, skipped=skipped, answers=result.answers - ) + if mode == "elite" and result.elite is not None: + result.manifest = self._build_elite_manifest( + members=members, + skipped=skipped, + result=result, + ) + else: + result.manifest = self._build_manifest( + mode=mode, members=members, skipped=skipped, answers=result.answers + ) async def fan_out( self, @@ -450,6 +457,63 @@ def _build_manifest( manifest.secret_safety = verified_secret_safety(manifest) return manifest + def _build_elite_manifest( + self, + *, + members: list[tuple[str, str]], + skipped: list[str], + result: CouncilResult, + ) -> ModelHarnessManifest: + """Assemble a phase-complete manifest for an elite protocol run. + + Elite keeps every attempted member call in phase-specific artifact + collections. Flattening those collections here preserves repeated calls + as distinct receipts while provider membership remains unique. Failed + gates therefore retain the completed and attempted phases without + inventing receipts for phases that never ran. + """ + from . import __version__ + + elite = result.elite + phase_artifacts = ( + [] + if elite is None + else [ + ("initial", elite.initial_answers), + ("critique", elite.critiques), + ("revision", elite.revisions), + ] + ) + flattened = [answer for _phase, answers in phase_artifacts for answer in answers] + receipts = [ + receipt_from_answer( + answer, + temperature=self.temperature, + timeout=self.timeout, + phase=phase, + ) + for phase, answers in phase_artifacts + for answer in answers + ] + manifest = ModelHarnessManifest( + request_id=uuid4().hex, + conclave_version=__version__, + mode="elite", + providers_considered=list(self.requested_models), + providers_called=list(dict.fromkeys(name for name, _model_id in members)), + providers_skipped=[ + ProviderSkip(name=name, reason="no API key in environment") for name in skipped + ], + model_ids=list(dict.fromkeys(model_id for _name, model_id in members)), + generation_settings={"temperature": self.temperature, "timeout": self.timeout}, + receipts=receipts, + total_latency_ms=sum(answer.latency_ms for answer in flattened), + total_usage=self._sum_usage(flattened), + redacted_errors=[answer.error for answer in flattened if answer.error], + ) + manifest.secret_safety = verified_secret_safety(manifest) + return manifest + @staticmethod def _sum_usage(answers: list[ModelAnswer]) -> TokenUsage | None: """Sum token usage across answers, or ``None`` when none reported usage. diff --git a/src/conclave/manifest.py b/src/conclave/manifest.py index 280312d..5f1ca06 100644 --- a/src/conclave/manifest.py +++ b/src/conclave/manifest.py @@ -96,6 +96,9 @@ class ProviderExecutionReceipt(BaseModel): settings actually used, the latency, the token usage, and a redacted error. Attributes: + phase: Optional protocol phase provenance. ``None`` for legacy and + non-phased modes; elite receipts use ``initial``, ``critique``, or + ``revision``. name: Friendly council member name (e.g. ``"grok"``). provider: Provider prefix derived from ``model_id`` (e.g. ``"xai"``). model_id: Resolved provider-prefixed model id (e.g. ``"xai/grok-4.3"``). @@ -110,6 +113,7 @@ class ProviderExecutionReceipt(BaseModel): until CAC-02 structured output exists; defined now, populated later. """ + phase: str | None = None name: str provider: str model_id: str diff --git a/src/conclave/providers.py b/src/conclave/providers.py index 6974759..175c05e 100644 --- a/src/conclave/providers.py +++ b/src/conclave/providers.py @@ -32,7 +32,11 @@ def receipt_from_answer( - answer: ModelAnswer, *, temperature: float, timeout: float + answer: ModelAnswer, + *, + temperature: float, + timeout: float, + phase: str | None = None, ) -> ProviderExecutionReceipt: """Map a collected :class:`ModelAnswer` to a :class:`ProviderExecutionReceipt`. @@ -56,11 +60,13 @@ def receipt_from_answer( answer: The collected member answer (success or failure). temperature: The sampling temperature the council used for the call. timeout: The per-call timeout (seconds) the council used. + phase: Optional protocol phase provenance for phased modes. Returns: A :class:`ProviderExecutionReceipt` for this member. """ return ProviderExecutionReceipt( + phase=phase, name=answer.name, provider=provider_prefix(answer.model_id), model_id=answer.model_id, diff --git a/tests/test_manifest_all_modes.py b/tests/test_manifest_all_modes.py index 643956d..c6c329b 100644 --- a/tests/test_manifest_all_modes.py +++ b/tests/test_manifest_all_modes.py @@ -22,6 +22,8 @@ from conclave import Council from conclave.config import ConclaveConfig from conclave.manifest import SECRET_SAFETY_VERIFIED, ModelHarnessManifest +from conclave.models import ModelAnswer +from conclave.providers import receipt_from_answer from tests.conftest import make_response @@ -71,6 +73,117 @@ def _assert_verified_manifest(result, expected_mode: str) -> None: assert all(c in "0123456789abcdef" for c in manifest.request_id) +def _elite_phase(messages) -> str: + """Identify the elite phase from its system prompt.""" + system = _system_text(messages) + if "evidence auditor" in system: + return "critique" + if "revise your original answer" in system: + return "revision" + return "initial" + + +def test_receipt_phase_is_backward_compatible_and_optional() -> None: + """Existing callers get phase=None; elite callers can stamp provenance.""" + answer = ModelAnswer(name="grok", model_id="xai/grok-4.3", answer="A") + + normal = receipt_from_answer(answer, temperature=0.7, timeout=120.0) + elite = receipt_from_answer(answer, temperature=0.7, timeout=120.0, phase="critique") + + assert normal.phase is None + assert elite.phase == "critique" + + +async def test_elite_manifest_audits_every_phase(monkeypatch, patch_call_model): + """A complete elite run records all nine member calls and their usage.""" + _all_keys(monkeypatch) + monkeypatch.delenv("ANTHROPIC_API_KEY") + + def handler(model, messages, **kwargs): + return make_response(f"{_elite_phase(messages)} from {model}") + + patch_call_model(handler) + council = Council( + models=["grok", "gemini", "claude", "perplexity"], + synthesizer="claude", + config=_config(), + extract_verdict=False, + ) + + result = await council.elite("Choose the strongest option.") + + _assert_verified_manifest(result, "elite") + assert result.manifest.providers_called == ["grok", "gemini", "perplexity"] + assert len(result.manifest.providers_called) == len(set(result.manifest.providers_called)) + assert [(skip.name, skip.reason) for skip in result.manifest.providers_skipped] == [ + ("claude", "no API key in environment") + ] + assert [receipt.phase for receipt in result.manifest.receipts] == ( + ["initial"] * 3 + ["critique"] * 3 + ["revision"] * 3 + ) + assert result.manifest.total_usage is not None + assert result.manifest.total_usage.prompt_tokens == 45 + assert result.manifest.total_usage.completion_tokens == 63 + assert result.manifest.total_usage.total_tokens == 108 + + +async def test_incomplete_elite_manifest_keeps_attempted_phase_receipts( + monkeypatch, patch_call_model +): + """A failed critique gate retains initial and attempted critique receipts.""" + _all_keys(monkeypatch) + + def handler(model, messages, **kwargs): + phase = _elite_phase(messages) + if phase == "critique" and model == "gemini/gemini-2.5-pro": + raise RuntimeError("critic unavailable") + return make_response(f"{phase} from {model}") + + patch_call_model(handler) + council = Council( + models=["grok", "gemini", "perplexity"], + config=_config(), + extract_verdict=False, + ) + + result = await council.elite("Choose.") + + assert result.elite is not None and result.elite.completed is False + _assert_verified_manifest(result, "elite") + assert [receipt.phase for receipt in result.manifest.receipts] == ( + ["initial"] * 3 + ["critique"] * 3 + ) + assert result.manifest.total_usage is not None + assert result.manifest.total_usage.total_tokens == 60 + + +async def test_elite_cache_hit_preserves_phase_receipts(monkeypatch, patch_call_model, tmp_path): + """Elite phase provenance survives cache serialization and reload.""" + _all_keys(monkeypatch) + monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path)) + + def handler(model, messages, **kwargs): + return make_response(f"{_elite_phase(messages)} from {model}") + + patch_call_model(handler) + council = Council( + models=["grok", "gemini", "perplexity"], + config=_config(), + extract_verdict=False, + cache=True, + ) + + first = await council.elite("Choose.") + second = await council.elite("Choose.") + + assert first.cached is False + assert second.cached is True + _assert_verified_manifest(second, "elite") + assert [receipt.phase for receipt in second.manifest.receipts] == ( + ["initial"] * 3 + ["critique"] * 3 + ["revision"] * 3 + ) + + # --------------------------------------------------------------------------- # # Debate # --------------------------------------------------------------------------- # diff --git a/tests/test_secret_safety_matrix.py b/tests/test_secret_safety_matrix.py index 67e0d5a..8067329 100644 --- a/tests/test_secret_safety_matrix.py +++ b/tests/test_secret_safety_matrix.py @@ -350,6 +350,32 @@ async def echoing_401(url, headers, json_body, timeout): _assert_result_canary_free(result) +async def test_secret_safety_forced_error_elite(monkeypatch): + """Elite phase artifacts and receipts redact a provider error echoing a key.""" + _plant_all_keys(monkeypatch) + monkeypatch.setenv("CONCLAVE_CONFIG", "/nonexistent/conclave.yml") + + async def echoing_401(url, headers, json_body, timeout): + return 401, {"error": {"message": f"invalid api key: {PLANTED}"}} + + monkeypatch.setattr("conclave.transport.post_json", echoing_401) + council = Council( + models=["grok", "gemini", "perplexity"], + synthesizer="claude", + config=_config(), + ) + + result = await council.elite("audit prompt") + + assert result.elite is not None + assert result.elite.completed is False + assert result.elite.critiques == [] + assert result.elite.revisions == [] + assert all(not answer.ok for answer in result.elite.initial_answers) + _assert_result_canary_free(result) + assert [receipt.phase for receipt in result.manifest.receipts] == ["initial"] * 3 + + async def test_secret_safety_forced_error_streaming(monkeypatch, mock_stream_client): """1c (streaming): a mid-stream 401 echoing the canary leaks into no event/answer. From 977cad28f66224f7a0329f34aa31a62beaa39453 Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Fri, 17 Jul 2026 05:16:04 -0400 Subject: [PATCH 07/21] fix(elite): preserve verdict manifest provenance --- src/conclave/council.py | 1 + tests/test_elite_mode.py | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/src/conclave/council.py b/src/conclave/council.py index 39baae9..fa26c9e 100644 --- a/src/conclave/council.py +++ b/src/conclave/council.py @@ -904,6 +904,7 @@ async def run() -> CouncilResult: result = await run_elite(self, prompt) if result.elite is not None and result.elite.completed: await self._synthesize(result) + self._ensure_manifest(result, "elite") await self._apply_verdict(result) return result diff --git a/tests/test_elite_mode.py b/tests/test_elite_mode.py index 4ad101f..d128db7 100644 --- a/tests/test_elite_mode.py +++ b/tests/test_elite_mode.py @@ -14,6 +14,7 @@ ModelAnswer, ) from conclave.config import ConclaveConfig +from conclave.manifest import SECRET_SAFETY_VERIFIED from conclave.modes import run_elite from conclave.prompts import ( ELITE_CRITIC_SYSTEM, @@ -400,6 +401,10 @@ def handler(model_id, messages): assert result.verdict is not None assert result.consensus_score == result.verdict.consensus_score == 1.0 assert result.provider_votes == result.verdict.provider_votes + assert result.manifest is not None + assert result.manifest.verdict_extraction.model_id == "anthropic/claude-sonnet-4-6" + assert result.manifest.verdict_extraction.prompt_version is not None + assert result.manifest.secret_safety == SECRET_SAFETY_VERIFIED async def test_council_elite_incomplete_skips_synthesis_and_verdict( From 40c21e16fa3260fffb7f71b30c779631f5136477 Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Fri, 17 Jul 2026 05:20:00 -0400 Subject: [PATCH 08/21] feat(elite): add quality first CLI mode --- src/conclave/cli.py | 49 +++++++++++++-- tests/test_cli.py | 142 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 187 insertions(+), 4 deletions(-) diff --git a/src/conclave/cli.py b/src/conclave/cli.py index 565fe83..4d52bd9 100644 --- a/src/conclave/cli.py +++ b/src/conclave/cli.py @@ -4,7 +4,7 @@ * ``conclave ask "" --council grok,gemini,claude --mode synthesize`` Modes: ``synthesize`` (default), ``raw``, ``debate`` (``--rounds N``), - ``adversarial`` (``--proposer NAME``). + ``adversarial`` (``--proposer NAME``), ``vote``, and ``elite``. * ``conclave providers`` -- show which providers currently have a key (without ever printing key values). """ @@ -297,6 +297,36 @@ def _render_adversarial(result: CouncilResult) -> None: err_console.print(f"[yellow]No verdict: {adv.verdict_error}[/yellow]") +def _render_elite(result: CouncilResult) -> None: + """Render an elite run as a concise phase summary plus canonical verdict.""" + elite = result.elite + if elite is None: + err_console.print("[yellow]No elite protocol result produced.[/yellow]") + return + + def successful(answers) -> int: + return sum(answer.ok for answer in answers) + + required = elite.required_responders + summary = " · ".join( + ( + f"Initial {successful(elite.initial_answers)}/{required}", + f"Critiques {successful(elite.critiques)}/{required}", + f"Revisions {successful(elite.revisions)}/{required}", + ) + ) + console.print( + Panel( + summary, + title=f"[bold]ELITE DECISION PROTOCOL[/bold] ({elite.protocol_version})", + border_style="cyan", + ) + ) + _print_synthesis(result) + _print_verdict(result) + _print_verdict_absent_note(result) + + def _stream_to_terminal(council: Council, prompt: str, *, synthesize: bool) -> CouncilResult: """Render a live token stream to the terminal and return the final result. @@ -357,10 +387,11 @@ def on_event(event: StreamEvent) -> None: "debate": _render_debate, "adversarial": _render_adversarial, "vote": _render_vote, + "elite": _render_elite, } -_VALID_MODES = {"synthesize", "raw", "debate", "adversarial", "vote"} +_VALID_MODES = {"synthesize", "raw", "debate", "adversarial", "vote", "elite"} # Threshold used when --converge is passed without an explicit --converge-threshold. # High by design: an early stop should require answers that are nearly stable @@ -409,7 +440,7 @@ def ask( "synthesize", "--mode", "-m", - help="Run mode: synthesize | raw | debate | adversarial.", + help="Run mode: synthesize | raw | debate | adversarial | vote | elite.", ), synthesizer: str | None = typer.Option( None, "--synthesizer", "-s", help="Override the synthesizer/judge model name." @@ -534,6 +565,8 @@ def ask( elif mode_lower == "vote": choice_list = [ch.strip() for ch in (choices or "").split(",") if ch.strip()] result = c.vote_sync(prompt, choices=choice_list) + elif mode_lower == "elite": + result = c.elite_sync(prompt) else: result = c.ask_sync(prompt, synthesize=(mode_lower == "synthesize")) @@ -541,15 +574,23 @@ def ask( # purposes regardless of output format. We compute this once and apply the # same exit-code contract to both the JSON and human paths. no_usable_answers = not result.successful_answers + elite_incomplete = mode_lower == "elite" and ( + result.elite is None or not result.elite.completed + ) if as_json: # Always emit valid JSON to stdout so a consumer can parse the payload, # then signal failure via the exit code if nothing usable came back. console.print_json(json.dumps(_result_to_dict(result))) - if no_usable_answers: + if no_usable_answers or elite_incomplete: raise typer.Exit(code=1) return + if elite_incomplete: + reason = result.elite.failure_reason if result.elite is not None else "missing result" + err_console.print(f"[red]Elite protocol incomplete: {reason}[/red]") + raise typer.Exit(code=1) + if no_usable_answers: err_console.print( "[red]No usable council answers. Run 'conclave providers' to check keys.[/red]" diff --git a/tests/test_cli.py b/tests/test_cli.py index 0b14a0c..fa8771d 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -22,6 +22,8 @@ from conclave import cli from conclave.config import ConclaveConfig +from conclave.models import CouncilResult, EliteResult, ModelAnswer +from conclave.verdict import CouncilVerdict runner = CliRunner() @@ -102,6 +104,146 @@ def test_unknown_mode_exits_two(patch_cli_config): assert result.exit_code == 2 +def _elite_cli_result(*, completed: bool = True, failure_reason: str | None = None): + """Build a compact elite result for CLI-only behavior tests.""" + initials = [ + ModelAnswer(name=f"member-{i}", model_id=f"provider/model-{i}", answer=f"initial-{i}") + for i in range(1, 4) + ] + critiques = [ + ModelAnswer(name=f"member-{i}", model_id=f"provider/model-{i}", answer=f"critique-{i}") + for i in range(1, 4) + ] + revisions = [ + ModelAnswer(name=f"member-{i}", model_id=f"provider/model-{i}", answer=f"revision-{i}") + for i in range(1, 4) + ] + return CouncilResult( + prompt="Should we ship?", + mode="elite", + answers=revisions if completed else [], + synthesis="Ship with safeguards." if completed else None, + elite=EliteResult( + completed=completed, + failure_reason=failure_reason, + initial_answers=initials, + critiques=critiques if completed else [], + revisions=revisions if completed else [], + ), + verdict=( + CouncilVerdict( + verdict_type="decision", + headline="Ship deliberately.", + recommendation="Ship with safeguards.", + consensus_score=1.0, + consensus_label="unanimous", + ) + if completed + else None + ), + ) + + +def test_elite_mode_dispatches_and_renders_protocol_summary(monkeypatch, patch_cli_config): + """Elite dispatch uses elite_sync and renders the protocol plus verdict.""" + calls: list[str] = [] + + class FakeCouncil: + def __init__(self, **kwargs): + pass + + def elite_sync(self, prompt: str) -> CouncilResult: + calls.append(prompt) + return _elite_cli_result() + + monkeypatch.setattr(cli, "Council", FakeCouncil) + + result = runner.invoke(cli.app, ["ask", "Should we ship?", "--mode", "elite"]) + + assert result.exit_code == 0 + assert calls == ["Should we ship?"] + assert "ELITE DECISION PROTOCOL" in result.output + assert "elite_v1" in result.output + assert "Initial 3/3" in result.output + assert "Critiques 3/3" in result.output + assert "Revisions 3/3" in result.output + assert "VERDICT" in result.output + assert "Ship deliberately." in result.output + + +def test_elite_json_emits_full_council_result(monkeypatch, patch_cli_config): + """Elite JSON includes every protocol phase artifact.""" + + class FakeCouncil: + def __init__(self, **kwargs): + pass + + def elite_sync(self, prompt: str) -> CouncilResult: + return _elite_cli_result() + + monkeypatch.setattr(cli, "Council", FakeCouncil) + + result = runner.invoke(cli.app, ["ask", "Should we ship?", "--mode", "elite", "--json"]) + + assert result.exit_code == 0 + payload = json.loads(result.stdout) + assert payload["mode"] == "elite" + assert payload["elite"]["completed"] is True + assert len(payload["elite"]["initial_answers"]) == 3 + assert len(payload["elite"]["critiques"]) == 3 + assert len(payload["elite"]["revisions"]) == 3 + assert payload["verdict"]["headline"] == "Ship deliberately." + + +def test_incomplete_elite_exits_one_with_reason(monkeypatch, patch_cli_config): + """An incomplete protocol is a failed CLI run even with prior phase answers.""" + + class FakeCouncil: + def __init__(self, **kwargs): + pass + + def elite_sync(self, prompt: str) -> CouncilResult: + return _elite_cli_result( + completed=False, + failure_reason="critique phase requires at least 3 successful responders; got 2", + ) + + monkeypatch.setattr(cli, "Council", FakeCouncil) + + result = runner.invoke(cli.app, ["ask", "Should we ship?", "--mode", "elite"]) + + assert result.exit_code == 1 + assert "Elite protocol incomplete" in result.output + assert "critique phase requires at least 3 successful" in result.output + assert "responders; got 2" in result.output + + +def test_elite_stream_is_rejected_before_council_creation(monkeypatch, patch_cli_config): + """Elite streaming fails as usage before a provider-capable Council exists.""" + created = False + + class ForbiddenCouncil: + def __init__(self, **kwargs): + nonlocal created + created = True + + monkeypatch.setattr(cli, "Council", ForbiddenCouncil) + + result = runner.invoke(cli.app, ["ask", "Should we ship?", "--mode", "elite", "--stream"]) + + assert result.exit_code == 2 + assert created is False + assert "--stream" in result.output + assert "elite" in result.output + + +def test_ask_help_lists_elite_mode(): + """CLI help advertises elite as a supported mode.""" + result = runner.invoke(cli.app, ["ask", "--help"]) + assert result.exit_code == 0 + assert "elite" in result.output + + def test_unresolved_council_exits_two(patch_cli_config): """Usage error (empty council selector resolves to zero members) -> exit 2.""" result = runner.invoke(cli.app, ["ask", "hello", "--council", " , "]) From 4760db5584e8505c7c310b3c82a667e5fe4de7cc Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Fri, 17 Jul 2026 05:28:08 -0400 Subject: [PATCH 09/21] docs: specify elite decision protocol --- CHANGELOG.md | 20 +++++++++ DOCUMENTATION_INDEX.md | 26 ++++++------ README.md | 73 +++++++++++++++++++++++++++++---- SYSTEM_CONTEXT_DIAGRAM.md | 23 ++++++++--- docs/PRODUCT_DESIGN_DOCUMENT.md | 68 +++++++++++++++--------------- 5 files changed, 149 insertions(+), 61 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f49353..757ad67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- **Elite Decision Protocol (unreleased).** A quality-first `elite` mode for consequential + decisions: independent member answers → concurrent council-wide evidence audits → concurrent + member revisions → the existing synthesis and canonical auditable verdict. It is available + through `--mode elite`, `Council.elite()`, and `elite_sync()` in source; no version, tag, or + package publication is claimed here. +- **Fixed three-success phase gate.** Each of Elite's `initial`, `critique`, and `revision` + phases requires three successful responders. Larger councils may survive partial failures + while three remain. A failed gate stops later calls and returns an incomplete result with a + phase-specific reason, attempted artifacts, no synthesis/verdict, and CLI exit code 1. +- **Phase-auditable results.** `EliteResult` preserves initial answers, critiques, and revisions; + the manifest records a separate redacted receipt with `phase` provenance for every attempted + member call, aggregates phase usage/latency, and retains the existing secret-safety scan. + Elite is buffered-only: `--stream` is rejected before provider calls. The quality tradeoff is + explicit—up to `3N + 2` calls for N members versus the ordinary single-fan-out workflow. + ### Fixed - **`ModelHarnessManifest` now rides on *every* mode's result — a true invariant.** @@ -26,6 +43,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Documentation +- Added Elite CLI/library examples and reconciled the PDD, system diagram, README, and index + around its unreleased status, fixed gate, partial-failure behavior, phased receipts, and + latency/cost tradeoff. - Reconciled `README.md`, `SYSTEM_CONTEXT_DIAGRAM.md`, and the PDD so the manifest-on-every-result claim is now accurate, and documented the constrained-choice **`vote` mode** as **shipped** (CAC-09 / #3) rather than "absorbed by `provider_votes`" diff --git a/DOCUMENTATION_INDEX.md b/DOCUMENTATION_INDEX.md index ab476f9..c9f2cc3 100644 --- a/DOCUMENTATION_INDEX.md +++ b/DOCUMENTATION_INDEX.md @@ -7,7 +7,7 @@ the canonical authority spec on top of those. - **Repo:** `/Users/ernestprovo/dev/conclave/` - **Version:** 1.1.0 · **License:** MIT -- **Last updated:** 2026-06-21 +- **Last updated:** 2026-07-17 --- @@ -15,15 +15,15 @@ the canonical authority spec on top of those. | # | Doc | Path | Purpose | |---|-----|------|---------| -| 1 | **README** (project overview) | [`README.md`](README.md) | What conclave is, install, BYO-keys, CLI + library quickstart, config, test. The fast on-ramp. | -| 2 | **System Context Diagram** | [`SYSTEM_CONTEXT_DIAGRAM.md`](SYSTEM_CONTEXT_DIAGRAM.md) | Mermaid system-context view: user/consumer → CLI/library → Council → provider highway (httpx transport + adapter registry) → 9 providers; the v1.1 verdict pipeline (`extract_verdict` → deterministic `agreement` → `CouncilVerdict`) and the first-class secret-free `ModelHarnessManifest`; config + env-var key inputs; custom OpenAI-compatible endpoints; mcp-warden as dev-time consumer. | +| 1 | **README** (project overview) | [`README.md`](README.md) | Install, BYO-keys, six-mode CLI/library quickstart, and the unreleased quality-first Elite protocol. | +| 2 | **System Context Diagram** | [`SYSTEM_CONTEXT_DIAGRAM.md`](SYSTEM_CONTEXT_DIAGRAM.md) | Mermaid context for the provider highway, six modes, unreleased Elite gated phases, verdict pipeline, phased manifest receipts, and mcp-warden boundary. | | 3 | **Documentation Index** | [`DOCUMENTATION_INDEX.md`](DOCUMENTATION_INDEX.md) | This file. Master map of all docs + source layout. | ## Authority spec | Doc | Path | Purpose | |-----|------|---------| -| **Product Design Document** | [`docs/PRODUCT_DESIGN_DOCUMENT.md`](docs/PRODUCT_DESIGN_DOCUMENT.md) | **Canonical** product spec and roadmap, reframed in v1.1 to **the auditable multi-model council**. Problem & vision, personas, BYO-keys/key-handling security, council modes (synthesize/raw/debate/adversarial/vote — `vote` shipped as a constrained-choice ballot, CAC-09 / #3, complementary to the verdict's `provider_votes`), the v1.1 auditable verdict (§4a — CouncilResult v2, deterministic consensus, native structured output, the manifest), provider matrix, architecture, scope + non-goals, demand-gated v1.2 roadmap, the mcp-warden dev-time boundary, licensing & positioning, open questions. **When docs disagree, the PDD wins.** | +| **Product Design Document** | [`docs/PRODUCT_DESIGN_DOCUMENT.md`](docs/PRODUCT_DESIGN_DOCUMENT.md) | **Canonical** product spec: six modes including implemented-but-unreleased Elite, its fixed three-success phase gate and cost/latency tradeoff, the v1.1 auditable verdict, manifest, architecture, boundaries, and roadmap. **When docs disagree, the PDD wins.** | --- @@ -34,15 +34,15 @@ Package root: `src/conclave/` (installed as the `conclave` package; console scri | Module | Path | Responsibility | |--------|------|----------------| -| Package API | [`src/conclave/__init__.py`](src/conclave/__init__.py) | Public exports: `Council`, `CouncilResult`, `ModelAnswer`, `TokenUsage`, `StreamEvent`, `DebateRound`, `AdversarialResult`, `ConclaveConfig`, `load_config`, `aclose`, `guard_transport_logging`, `__version__`; **v1.1 verdict surface** — `CouncilVerdict`, `CouncilConflict`, `CouncilPosition`, `ProviderVote`, `MinorityReport`, `extract_verdict`, `VerdictSynthesisResult`, `VerdictExtractionModel`, `ModelHarnessManifest`, `ProviderExecutionReceipt`, `ProviderSkip`, `VerdictExtraction`, `verdict_json_schema`, `member_answer_json_schema`, `verdict_extraction_json_schema`, `VERDICT_SCHEMA_VERSION`, `VERDICT_EXTRACTION_PROMPT_VERSION`. | -| Council | [`src/conclave/council.py`](src/conclave/council.py) | Primary importable entry point. Reusable primitives (`fan_out`, `synthesize_blocks`) + the public mode API (`ask`/`debate`/`adversarial`, async + sync) + streaming (`ask_stream`/`stream_sync`, synthesize/raw). Runs **default-on** verdict extraction via `_apply_verdict` on both buffered + streaming paths (opt out with `Council(extract_verdict=False)`). | -| Modes | [`src/conclave/modes.py`](src/conclave/modes.py) | `debate` (multi-round, anonymized peers, drop-out) and `adversarial` (propose → refute → verdict) orchestration, built on `Council.fan_out`/`synthesize_blocks`. | +| Package API | [`src/conclave/__init__.py`](src/conclave/__init__.py) | Public exports include `Council`, result types, verdict/manifest surface, and unreleased `EliteResult` plus Elite protocol constants. | +| Council | [`src/conclave/council.py`](src/conclave/council.py) | Six-mode async/sync API; completed Elite runs synthesize revisions and apply the canonical verdict; incomplete runs retain artifacts without either. | +| Modes | [`src/conclave/modes.py`](src/conclave/modes.py) | Debate, adversarial, vote, and unreleased Elite orchestration; Elite gates initial/audit/revision at three successes. | | Verdict types | [`src/conclave/verdict.py`](src/conclave/verdict.py) | Public verdict/member Pydantic types (`CouncilVerdict`, `CouncilPosition`, `CouncilConflict`, `ProviderVote`, `MinorityReport`) + the LCD JSON Schemas (`verdict_json_schema`/`member_answer_json_schema`/`verdict_extraction_json_schema`) usable across all three native structured-output surfaces; `VERDICT_SCHEMA_VERSION`/`VERDICT_EXTRACTION_PROMPT_VERSION`. | | Agreement | [`src/conclave/agreement.py`](src/conclave/agreement.py) | Deterministic consensus: `consensus_score` (`position_cluster_ratio_v1` — largest cluster / positioned members; `None` for N<2) + `consensus_label` buckets. Pure arithmetic, no `difflib`, never LLM-emitted. | | Verdict synthesis | [`src/conclave/verdict_synthesis.py`](src/conclave/verdict_synthesis.py) | `extract_verdict` engine: one extraction call clustering stances, native `output_contract` enforcement + prompt-level fallback, validate → repair-once → graceful `verdict=None`; the three verdict-absent reasons; provenance on every return path. | -| Manifest | [`src/conclave/manifest.py`](src/conclave/manifest.py) | `ModelHarnessManifest` (first-class on every result) + `ProviderExecutionReceipt`/`ProviderSkip`/`VerdictExtraction` + `scan_for_secret_material()` → `secret_safety` stamp. No key values; `estimated_cost` left `None`. | +| Manifest | [`src/conclave/manifest.py`](src/conclave/manifest.py) | Secret-scanned manifest on every result; Elite receipts preserve `initial`/`critique`/`revision` phase provenance. | | Streaming | [`src/conclave/streaming.py`](src/conclave/streaming.py) | `stream_ask` — council-level streaming engine behind `Council.ask_stream`: concurrent member interleaving via an `asyncio.Queue`, optional synthesizer streaming, terminal `done` event with the full `CouncilResult` (synthesize/raw only). | -| Prompts | [`src/conclave/prompts.py`](src/conclave/prompts.py) | Role/template strings for debate + adversarial and the anonymized peer-block builder. | +| Prompts | [`src/conclave/prompts.py`](src/conclave/prompts.py) | Debate/adversarial/vote plus anonymized Elite evidence-audit and revision prompts. | | Providers | [`src/conclave/providers.py`](src/conclave/providers.py) | Async `call_model` (buffered) + `call_model_stream` (SSE) paths: resolve the adapter, read the key by name at call time, call transport, parse; latency/usage/redacted-error capture; never raises (partial text preserved on mid-stream failure). | | Transport | [`src/conclave/transport.py`](src/conclave/transport.py) | `post_json` + `stream_sse` — the single async httpx network boundary (buffered POST and `client.stream(...)` SSE) for the whole provider highway. | | Adapter registry | [`src/conclave/adapters/__init__.py`](src/conclave/adapters/__init__.py) | `resolve_adapter(model_id, config)` — provider registry + **extension seam** (one registration per family; config-only for OpenAI-compatible endpoints). | @@ -52,8 +52,8 @@ Package root: `src/conclave/` (installed as the `conclave` package; console scri | Gemini adapter | [`src/conclave/adapters/gemini.py`](src/conclave/adapters/gemini.py) | `GeminiAdapter` — native `generateContent`, OpenAI-role mapping, `usageMetadata`. | | Registry | [`src/conclave/registry.py`](src/conclave/registry.py) | Friendly-name → model-id defaults; provider → env-var mapping; key **presence** logic (never values). | | Config | [`src/conclave/config.py`](src/conclave/config.py) | Loads/merges `~/.conclave/config.yml` over defaults; resolves model ids and named/CSV councils; parses the `endpoints:` section (custom OpenAI-compatible providers). | -| Models | [`src/conclave/models.py`](src/conclave/models.py) | Pydantic result contract: `TokenUsage`, `ModelAnswer` (stable `answer_id`), `StreamEvent`, `DebateRound`, `AdversarialResult`, `CouncilResult` v2 (`mode`/`rounds`/`adversarial`/`synthesis_error`/`prompt_version` **plus** `verdict`/`consensus_score`/`consensus_method`/`consensus_label`/`conflicts`/`provider_votes`/`minority_reports`/`manifest`, all backward-compatible). Stable downstream surface. | -| CLI | [`src/conclave/cli.py`](src/conclave/cli.py) | `conclave ask` (synthesize/raw/debate/adversarial; `--rounds`/`--proposer`/`--stream`) + `conclave providers`; rich panels incl. the green `VERDICT ()` panel (consensus/conflicts/minority blocks, or a dim `No verdict: ` note), live `--stream` output, and `--json` carrying the full verdict + manifest; never prints key values. | +| Models | [`src/conclave/models.py`](src/conclave/models.py) | Stable Pydantic contract, including `EliteResult` phase artifacts and backward-compatible `CouncilResult.elite`. | +| CLI | [`src/conclave/cli.py`](src/conclave/cli.py) | Six-mode `conclave ask` plus `providers`; Elite adds a phase summary, full JSON, incomplete exit 1, and explicit stream rejection. | | Logging | [`src/conclave/logging.py`](src/conclave/logging.py) | Logger factory; stderr; verbosity via `CONCLAVE_LOG_LEVEL` (default `WARNING`). | ## Tests @@ -63,6 +63,7 @@ Package root: `src/conclave/` (installed as the `conclave` package; console scri | Council tests | [`tests/test_council.py`](tests/test_council.py) | Fan-out, partial failure, synthesis behavior. | | Synthesizer tests | [`tests/test_synthesizer.py`](tests/test_synthesizer.py) | Pins the synthesizer/judge contract: default + configurable (arg/config/CLI `--synthesizer`) selection; observable degradation (unkeyed/failed → `synthesis_error`/`verdict_error`, never silent) for synthesize, debate, and the adversarial judge; versioned synthesis prompt (`SYNTHESIS_PROMPT_VERSION` + `result.prompt_version`) with prompt-text + version pins. | | Modes tests | [`tests/test_modes.py`](tests/test_modes.py) | Debate multi-round flow, mid-round drop-out, peer anonymization; adversarial proposer/critic/verdict, proposal/critic failure paths, no-key judge, sync wrappers. | +| Elite tests | [`tests/test_elite_mode.py`](tests/test_elite_mode.py) | Three-phase gates, anonymized prompts, partial failures, synthesis/verdict behavior, cache and public API. | | Adapter tests | [`tests/test_adapters.py`](tests/test_adapters.py) | Per-adapter `build_request` + `parse_response` for openai-compat/anthropic/gemini: system-hoist, max_tokens, role mapping, usage parsing, empty/malformed/error-status raises. | | Provider highway tests | [`tests/test_providers.py`](tests/test_providers.py) | `resolve_adapter` (built-in prefixes, per-provider URLs, custom endpoints, unknown-prefix raise), end-to-end `call_model`, and `redact()` (bearer/`sk-`/env-var-value/`x-api-key` scrubbing; pre-redacted provider errors). | | Registry/config tests | [`tests/test_registry_config.py`](tests/test_registry_config.py) | Name resolution, key-presence logic, config merge. | @@ -81,7 +82,7 @@ Run: `pytest` (config in `pyproject.toml`, `asyncio_mode = "auto"`). |------|------|---------| | Packaging | [`pyproject.toml`](pyproject.toml) | hatchling build, deps (httpx, pydantic, rich, typer, pyyaml — no LLM SDK), dev extras, console script, pytest config. License: MIT. **PyPI distribution name `conclave-cli`** (the name `conclave` is an unrelated project); command + import stay `conclave`. | | Release runbook | [`RELEASING.md`](RELEASING.md) | Operator runbook: one-time PyPI OIDC Trusted-Publisher setup for `conclave-cli`, cut-a-release checklist (bump→tag→publish Release), post-release verification (Sigstore bundle, PEP 740 attestations), rollback/yank. | -| Changelog | [`CHANGELOG.md`](CHANGELOG.md) | Keep-a-Changelog history per release (SemVer). The `[Unreleased]` entry covers the manifest-on-every-result invariant fix and the `vote`-mode-shipped doc reconciliation; `[1.1.0]` covers the auditable council — CouncilResult v2, deterministic consensus, native structured output, the manifest; the 1.0.0 entry covers the distribution rename, key-leak hardening, synthesizer versioning, and release engineering. | +| Changelog | [`CHANGELOG.md`](CHANGELOG.md) | Keep-a-Changelog history. `[Unreleased]` includes Elite and does not claim a version or publication. | | Dev lockfile | [`requirements-dev.lock`](requirements-dev.lock) | Hash-pinned dev + runtime tree for reproducible installs/CI. Regenerate via `uv pip compile --universal --generate-hashes --python-version 3.11 --extra dev pyproject.toml -o requirements-dev.lock`. | | License | [`LICENSE`](LICENSE) | MIT License. Copyright (c) 2026 Ernest Provo. Matches the `pyproject.toml` license field. | | Security policy | [`SECURITY.md`](SECURITY.md) | BYO-keys vulnerability reporting policy: how to report, scope, and the key-handling guarantees consumers can rely on. | @@ -100,6 +101,7 @@ Run: `pytest` (config in `pyproject.toml`, `asyncio_mode = "auto"`). | Date | Change | |------|--------| +| 2026-07-17 | Documented the implemented-but-unreleased Elite Decision Protocol: fixed three-success gates, initial/audit/revision artifacts, existing final verdict, phased receipts, failure semantics, cost/latency tradeoff, and no streaming. | | 2026-07-13 | **Manifest-on-every-result invariant fix + doc reconciliation.** `ModelHarnessManifest` now attaches on all five modes (was `None` for `debate`/`adversarial`/`vote`) via a single-site fix at `Council._cached_run` → new `_ensure_manifest`; regression tests in `tests/test_manifest_all_modes.py`. Docs corrected so the manifest-on-every-result claim is accurate and the `vote` mode is documented as **shipped** (CAC-09 / #3), not "absorbed" — across README, `SYSTEM_CONTEXT_DIAGRAM.md`, PDD (§1/§3/§4a/§8/§9/§12), and `CHANGELOG.md` (`[Unreleased]`). Verdict scope unchanged (still `synthesize`/`ask`-only; not layered on `adversarial`). | | 2026-06-21 | **v1.1 docs pivot — the auditable multi-model council.** PDD reframed to the auditable-council wedge (new §4a: CouncilResult v2, deterministic `position_cluster_ratio_v1` consensus, native + fallback structured output, the verdict-optional rule, the secret-free `ModelHarnessManifest`); `vote` mode marked **absorbed** by `provider_votes` across PDD §1/§4/§8/§9/§12; demand-gated v1.2 "Operable Council" roadmap. System Context diagram gains the verdict pipeline + manifest (mermaid re-validated). README gains an "Auditable verdict" section (CLI panel + library example). `CHANGELOG.md` `[1.1.0]` added. v0.x mode-detail prose archived to [`docs/archive/pdd-v0.x-modes-detail.md`](docs/archive/pdd-v0.x-modes-detail.md) to keep the PDD under 500 lines. New modules documented: `verdict.py`, `agreement.py`, `verdict_synthesis.py`, `manifest.py`. | | 2026-06-14 | v1.0.0 release. Version bump 0.3.0 → 1.0.0; new `CHANGELOG.md` (Keep-a-Changelog). Integrates the three v1.0 PRs below. | diff --git a/README.md b/README.md index ab51f46..8a96ba9 100644 --- a/README.md +++ b/README.md @@ -12,9 +12,10 @@ It is **library-first** (the CLI is a thin shell over the same `Council` you imp returns **structured results** (per-model latency, token usage, and error capture), and is **partial-failure resilient** — one provider erroring never aborts the run. Keys are **bring-your-own**, referenced by environment-variable *name* only — never stored or -logged. It ships four modes: **synthesize** (merge answers into one), **raw** (no merge), +logged. It ships six modes: **synthesize** (merge answers into one), **raw** (no merge), **debate** (multi-round, members revise after seeing peers' anonymized answers), and -**adversarial** (propose → refute → verdict). conclave is intentionally lightweight — a +**adversarial** (propose → refute → verdict), **vote** (fixed-choice tally), and +**elite** (quality-first evidence audit and revision). conclave is intentionally lightweight — a small council primitive, not an agent framework. **The v1.1 wedge — the auditable council.** Every run also yields **a multi-model council @@ -23,8 +24,8 @@ verdict you can act on — structured, scored for agreement, fully auditable**: a deterministic `consensus_score` (arithmetic over the model's clustering, *never* an LLM-emitted number); and a redacted `ModelHarnessManifest` recording how the run executed and which model produced the disagreement analysis. The verdict is **default-on**, and the manifest -rides on **every** result — for all five modes (`synthesize`, `raw`, `debate`, `adversarial`, -`vote`), not just `ask`. A constrained-choice **`vote` mode** (`--mode vote --choices ...`) also +rides on **every** result — for all six modes (`synthesize`, `raw`, `debate`, `adversarial`, +`vote`, `elite`), not just `ask`. A constrained-choice **`vote` mode** (`--mode vote --choices ...`) also shipped in v1.1 (CAC-09 / #3) — distinct from the verdict's `provider_votes`, which score free-form agreement rather than tally a fixed ballot. @@ -125,17 +126,23 @@ conclave ask "Defend event sourcing for this ledger." \ conclave ask "Which datastore for this workload?" \ -c grok,gemini,claude --mode vote --choices "Postgres,DynamoDB,MongoDB" -# Machine-readable output (works for every mode; carries rounds/adversarial/vote too) +# Elite (unreleased): independent answers -> evidence audits -> revisions -> verdict +conclave ask "Should we adopt a service mesh for 8 services?" \ + -c grok,gemini,claude --mode elite + +# Machine-readable output (works for every mode; carries elite phase artifacts too) conclave ask "..." -c grok,perplexity --mode debate --json ``` -Mode flags at a glance: `--mode synthesize|raw|debate|adversarial|vote`. `--rounds N` +Mode flags at a glance: `--mode synthesize|raw|debate|adversarial|vote|elite`. `--rounds N` (default 2) is the *maximum* round count for `debate`; `--converge-threshold FLOAT` (or `--converge`/`--no-converge`) optionally stops a debate early once answers stabilize round-over-round (off by default — `--rounds` runs in full). `--proposer NAME` (default: first member) applies to `adversarial`. `--choices "A,B,C"` (two or more) is required for `vote`. `--synthesizer/-s` overrides the synthesizer *and* the -adversarial judge. Every mode's result carries the auditable `ModelHarnessManifest`. +adversarial judge and Elite's final synthesizer. Every mode's result carries the auditable +`ModelHarnessManifest`. Elite is currently **unreleased**; use it from this source branch until +a future release explicitly includes it. `--council` accepts either a comma-separated list of friendly names or the name of a council defined in your config (see below). The built-in `default` council @@ -152,6 +159,7 @@ Streaming and the non-streaming default produce the **same** final `CouncilResult`; `--stream` only changes how output is rendered. It is ignored with `--json` (which always emits the full structured payload), and on a cache hit the cached text is rendered in one shot rather than as a fake token stream. +`--stream` is rejected for `elite` before any provider call; Elite runs are buffered only. ## Quickstart (library) @@ -172,6 +180,55 @@ for answer in result.answers: print("SYNTHESIS:\n", result.synthesis) ``` +### Elite Decision Protocol (unreleased) + +Elite is the quality-first path for consequential decisions. It trades latency and provider +spend for a stronger answer: three concurrent member phases, followed by conclave's existing +synthesis and canonical structured verdict. + +1. **Initial:** members answer independently. +2. **Evidence audit:** every surviving member audits the anonymized panel for supported, + conflicting, and externally unverified claims. +3. **Revision:** survivors revise their own answer using the full anonymized panel and audits. +4. **Final:** conclave synthesizes successful revisions, then applies the existing auditable + verdict and deterministic consensus calculation. + +Each member phase has a fixed gate of **three successful responders**. A council larger than +three may lose members and continue while at least three remain. If any gate falls below three, +the protocol stops immediately: `result.elite.completed` is `False`, `failure_reason` names the +failed phase, later phases are not called, and synthesis/verdict extraction do not run. Attempted +answers and failures remain in `initial_answers`, `critiques`, or `revisions`; the CLI exits 1 +(and `--json` still emits the full result). This is stricter than the ordinary modes' best-effort +partial-failure behavior by design. + +```python +from conclave import Council + +council = Council( + models=["grok", "gemini", "claude", "perplexity"], + synthesizer="claude", +) + +result = council.elite_sync("Should we adopt a service mesh for 8 services?") +# or: result = await council.elite("Should we adopt a service mesh for 8 services?") + +if result.elite.completed: + print(result.synthesis) + print(result.verdict) +else: + print(result.elite.failure_reason) + +for receipt in result.manifest.receipts: + print(receipt.phase, receipt.name, receipt.error) # initial/critique/revision +``` + +Elite normally makes up to `3N + 2` calls for `N` members: three concurrent member phases, +one synthesis call, and one verdict-extraction call. Failures reduce later member calls, and +`Council(extract_verdict=False)` removes the final extraction call. The manifest keeps a +separate receipt for every attempted `initial`, `critique`, and `revision` call, aggregates +their latency/usage, preserves redacted failures, and remains secret-scanned. Elite does **not** +stream. + ## Auditable verdict On a decision- or review-style prompt with at least two responding members, conclave @@ -268,7 +325,7 @@ Event types: `member_delta` / `member_done` (per member, interleaved), A member that cannot stream, or any mid-stream failure, degrades gracefully — partial text is preserved and the error lands on that member's `ModelAnswer`, never raising (the same never-raises contract as `ask`). Streaming applies to -`synthesize`/`raw` only; `debate`/`adversarial` are not streamed. +`synthesize`/`raw` only; `debate`/`adversarial`/`vote`/`elite` are not streamed. ### Debate and adversarial modes diff --git a/SYSTEM_CONTEXT_DIAGRAM.md b/SYSTEM_CONTEXT_DIAGRAM.md index 2485389..2ea8668 100644 --- a/SYSTEM_CONTEXT_DIAGRAM.md +++ b/SYSTEM_CONTEXT_DIAGRAM.md @@ -6,7 +6,8 @@ environment-variable keys feed in, how requests reach the nine first-class provi conclave's own **provider highway** (an httpx transport + per-provider adapter registry — no LLM-SDK dependency), how the v1.1 **verdict pipeline** turns the member answers into a structured, agreement-scored, **auditable** verdict plus a redacted execution manifest, and -where the sibling **mcp-warden** project sits as a **dev-time** consumer. +how the implemented-but-unreleased **Elite Decision Protocol** adds gated evidence audit and +revision before that verdict. It also shows where **mcp-warden** sits as a dev-time consumer. > Authority note: behavioral details here are descriptive. The canonical spec is > [`docs/PRODUCT_DESIGN_DOCUMENT.md`](docs/PRODUCT_DESIGN_DOCUMENT.md). @@ -30,6 +31,7 @@ flowchart TB lib["Library API · from conclave import Council (__init__.py)"] council["Council orchestrator
fan_out · synthesize_blocks · skip-no-key (council.py)"] modes["Deliberation modes
debate · adversarial · vote (modes.py + prompts.py)"] + elite["Elite (UNRELEASED)
initial → evidence audit → revision
fixed 3-success gate each phase"] registry["Registry · name to model-id
key PRESENCE only, never values (registry.py)"] config["Config loader · custom endpoints (config.py)"] models["Result contract · CouncilResult v2
answers · verdict · consensus · manifest (models.py)"] @@ -46,7 +48,7 @@ flowchart TB extract["extract_verdict
ONE call · cluster stances · repair-once · never raises (verdict_synthesis.py)"] agree["agreement.position_cluster_ratio_v1
DETERMINISTIC arithmetic · no difflib · never LLM-emitted (agreement.py)"] verdictobj["CouncilVerdict
positions + evidence_answer_ids · conflicts · provider_votes · minority_reports (verdict.py)"] - manifest["ModelHarnessManifest
first-class · secret-free · extraction provenance (manifest.py)"] + manifest["ModelHarnessManifest
first-class · secret-free · phased receipts (manifest.py)"] end end @@ -76,6 +78,8 @@ flowchart TB lib --> council council --> modes modes -->|"reuse fan_out + synthesize_blocks"| council + council --> elite + elite -->|"3+ revisions: existing synthesis + verdict
under 3: stop incomplete"| council council --> provider provider --> adreg adreg --> oai @@ -96,7 +100,7 @@ flowchart TB transport --> together provider --> models - council -->|"member answers ready"| applyverdict + council -->|"member answers or completed Elite revisions"| applyverdict applyverdict -->|"synthesizer clusters stances
(ONE extraction call, output_contract)"| extract extract -->|"clustering"| agree agree -->|"consensus_score/label
(arithmetic over the clustering)"| verdictobj @@ -148,12 +152,19 @@ flowchart TB (`manifest.py`) rides on **every** result — first-class, not a debug flag — recording which model + prompt version produced the clustering (provenance) and stamping `secret_safety` only after the serialized manifest is scanned clean. This is enforced as a true invariant at - the single chokepoint `Council._cached_run` → `_ensure_manifest`: **all five modes** - (`synthesize`, `raw`, `debate`, `adversarial`, `vote`) funnel through it, so a result can + the single chokepoint `Council._cached_run` → `_ensure_manifest`: **all six modes** + (`synthesize`, `raw`, `debate`, `adversarial`, `vote`, `elite`) funnel through it, so a result can never escape without a manifest — including the zero-members early return and cache hits. A verdict is *optional*: open-ended prompts, fewer than two responding members, or extraction failure leave `verdict = None` with the synthesis and member answers intact and the reason recorded on the manifest. +- **Elite is quality-first and unreleased.** Its three concurrent member phases are independent + answers, council-wide anonymized evidence audits, and member revisions. Each phase requires + three successful responders. Larger councils may lose members and continue while three remain; + a failed gate stops later calls and returns an incomplete result with artifacts and phased + receipts for attempted work, but no synthesis or verdict. Completed runs feed revisions into + the existing synthesis and verdict pipeline. This can cost up to `3N + 2` calls and more + wall-clock time than ordinary modes; `--stream` is rejected because Elite is buffered only. - **Streaming shares the same boundary (PDD §9 #5).** A `--stream` run (and the library `Council.ask_stream` async generator) flows through a streaming sibling of the call path: `call_model_stream` (`providers.py`) → `transport.stream_sse` (`transport.py`, the single @@ -161,7 +172,7 @@ flowchart TB `parse_sse_event` (OpenAI-compat `data:`/`[DONE]` deltas; Anthropic named SSE events; Gemini `streamGenerateContent?alt=sse`). `streaming.py` interleaves members concurrently and emits `StreamEvent`s, ending with a `done` event whose `CouncilResult` matches the - non-streaming shape. Streaming covers `synthesize`/`raw` only; the never-raises + + non-streaming shape. Streaming covers `synthesize`/`raw` only; Elite has no streaming; the never-raises + `redact()` invariants hold identically, with partial text preserved on mid-stream failure. - **`resolve_adapter` is the extension seam.** Adding a *new provider family* is one registration in `adapters/__init__.py`; adding an *OpenAI-compatible endpoint* is diff --git a/docs/PRODUCT_DESIGN_DOCUMENT.md b/docs/PRODUCT_DESIGN_DOCUMENT.md index fa1109b..5c6876c 100644 --- a/docs/PRODUCT_DESIGN_DOCUMENT.md +++ b/docs/PRODUCT_DESIGN_DOCUMENT.md @@ -1,10 +1,11 @@ # conclave — Product Design Document -> **Status:** v1.0 stable (the BYO-keys multi-model council: synthesize/raw/debate/ -> adversarial, 9 providers, owned httpx provider highway, key-leak hardening, streaming, +> **Status:** v1.1 stable (the BYO-keys multi-model council: synthesize/raw/debate/ +> adversarial/vote, 9 providers, owned httpx provider highway, key-leak hardening, streaming, > cache). **v1.1 — the auditable council — SHIPPED:** every run now yields a structured, > agreement-scored, fully auditable **verdict** plus a redacted execution **manifest** (see -> §4a). This is the **canonical authority document** for conclave's product scope, design, +> §4a). The quality-first **Elite Decision Protocol is implemented but unreleased**. +> This is the **canonical authority document** for conclave's product scope, design, > and roadmap. When this document and any other doc disagree, this document wins. Code is the > source of truth for *current behavior*; this document marks anything not yet in code as > **Roadmap**. @@ -12,7 +13,7 @@ - **Repo:** `/Users/ernestprovo/dev/conclave/` - **License:** MIT - **Author:** Data Science & Engineering Experts, Inc. (DSE) -- **Last updated:** 2026-06-21 +- **Last updated:** 2026-07-17 --- @@ -47,10 +48,12 @@ structured, scored for agreement, fully auditable.** Every run yields a `Council exposing agreement, disagreement (`conflicts`), minority views (`minority_reports`), and per-provider votes (`provider_votes`); a deterministic `consensus_score` (arithmetic over the model's clustering, *never* an LLM-emitted number); and a redacted `ModelHarnessManifest` (how -the run executed + which model produced the analysis) riding on **every** result across all five +the run executed + which model produced the analysis) riding on **every** result across all six modes (one chokepoint, §4a). A constrained-choice **`vote` mode** also **shipped** (CAC-09 / #3, `--mode vote --choices "A,B,C"` → plurality winner/split) — distinct from `provider_votes`. +The next product-quality step is **Elite** (implemented, unreleased): independent answers → council-wide evidence audits → member revisions → the existing synthesis and auditable verdict. Elite requires three successful responders at every member phase and intentionally spends more calls and time to improve consequential decisions. + conclave's first real use was an **adversarial design review**: a council of Grok, Gemini, Perplexity, and Claude critiquing a security-tool strategy and catching flaws a single model missed. That origin is why the adversarial and debate modes are first-class — they @@ -126,6 +129,7 @@ output. The v1.1 verdict layer (§4a) sits on top of whichever mode produced the | **debate** | **BUILT (v0.2)** | N rounds (`--rounds`, default 2). Round 1 is an independent fan-out; rounds 2..N show each member its peers' **anonymized** prior-round answers (`Model A/B/C`) and ask it to revise or defend. A member that errors in a round drops out of later rounds; the debate continues with survivors. The synthesizer consolidates the final round. Exposed as `--mode debate` / `Council.debate()` / `debate_sync()`. | | **adversarial** | **BUILT (v0.2)** | Structured propose → refute → verdict. A `--proposer` (default: first member) answers; the remaining members are CRITICS explicitly prompted to refute it; the synthesizer acts as JUDGE, weighing proposal vs. critiques and issuing a verdict + strengthened answer. This is the mode conclave's origin story (the security design review) exercised by hand. Exposed as `--mode adversarial` / `Council.adversarial()` / `adversarial_sync()`. | | **vote** | **BUILT (v1.1, CAC-09 / #3)** | Constrained-choice ballot: each member sees a fixed labelled option set (`A, B, C, …`) and answers with one letter; responses are tallied to a plurality `winner` (or `split` on a tie) on `result.vote` (`VoteResult`). Exposed as `--mode vote --choices "A,B,C"` / `Council.vote()` / `vote_sync()`. Distinct from the verdict's `provider_votes`, which cluster free-form stances with evidence (§4a); `vote` tallies a fixed ballot. | +| **elite** | **IMPLEMENTED, UNRELEASED** | Quality-first decision protocol: independent fan-out → concurrent council-wide evidence audits → concurrent member revisions → existing synthesis and canonical verdict. Every member phase has a fixed three-success gate. Exposed in source as `--mode elite` / `Council.elite()` / `elite_sync()`; buffered only, no streaming. | **Mode algorithms (as built).** The step-by-step "as built" prose for synthesize / raw / debate / adversarial / vote (fan-out + partial-results, peer anonymization + drop-out, proposer → @@ -133,11 +137,15 @@ critic → judge, ballot tally) is landed history and lives in [`docs/archive/pdd-v0.x-modes-detail.md`](archive/pdd-v0.x-modes-detail.md). In brief: every mode fans out concurrently, captures each call as a `ModelAnswer` (answer **or** redacted error — `call_model` never raises), and survives partial failure; the deliberation modes extend -`CouncilResult` (`mode`, `rounds`, `adversarial`, `vote`) backward-compatibly so v0.1 +`CouncilResult` (`mode`, `rounds`, `adversarial`, `vote`, `elite`) backward-compatibly so v0.1 `answers`/`synthesis` consumers keep working. The mode *text* output is a generative, inherently stochastic reconciliation (load-bearing for the mcp-warden boundary, §10); the v1.1 verdict (§4a) adds a *deterministic* agreement number on top, by arithmetic over a clustering. +**Elite gate and partial-failure contract.** `required_responders` is fixed at 3. Councils may start larger, but only members successful in a phase advance. One failure in a four-member council is tolerated; fewer than three successes in `initial`, `critique`, or `revision` stops the protocol immediately. +The result is incomplete with a phase-specific `failure_reason`, no later calls, synthesis, or verdict; attempted phase artifacts and redacted failures remain serialized and the CLI exits 1. A completed run mirrors successful revisions to `answers`. +Compared with ordinary modes, Elite deliberately trades latency and cost for decision quality: up to `3N + 2` calls for N members (three member phases, synthesis, verdict extraction), or `3N + 1` with verdict extraction disabled. + --- ## 4a. The Auditable Verdict (v1.1) @@ -241,11 +249,11 @@ verdict appears identically in the buffered result and the streaming `done` even `secret_safety` stamp is re-run over the final content. **Cost:** default-on adds exactly **one** extra synthesizer call per run; the opt-out exists for cost-sensitive callers. -**Verdict scope across modes (deliberate).** Posture: *manifest everywhere, clustering verdict only where unambiguously additive.* The manifest rides on all five modes; the clustering verdict still runs on `synthesize`/`ask` only — deferred (not forced) on `debate`/`vote` (a future change may opt them in behind `extract_verdict`), and **intentionally NOT on `adversarial`**, which already emits a judge verdict the clustering verdict would double-count. +**Verdict scope across modes (deliberate).** Posture: *manifest everywhere, clustering verdict only where unambiguously additive.* The manifest rides on all six modes. The clustering verdict runs on `synthesize`/`ask` and on **completed Elite runs after synthesis of the revised answers**; it is deferred on `debate`/`vote` and intentionally not layered onto `adversarial`, which already emits a judge verdict. An incomplete Elite run has no synthesis or verdict. ### ModelHarnessManifest — first-class, secret-free The `ModelHarnessManifest` (`manifest.py`) rides on every `CouncilResult` — *not* behind a debug -flag, and now a **true invariant** enforced at one chokepoint: all five modes funnel through +flag, and now a **true invariant** enforced at one chokepoint: all six modes funnel through `Council._cached_run` → `_ensure_manifest`, which attaches the manifest on every returned result — including `debate`/`adversarial`/`vote` (built in `modes.py`), the zero-members early return, and cache hits (synthesize/raw builds its own richer one earlier). Pinned by @@ -258,6 +266,8 @@ error(redacted), schema_valid}`), `total_latency_ms`, `total_usage`, `schema_val prompt_version}` — the auditability hook — plus `verdict_type`, `consensus_method`, `verdict_absent_reason`). Two deliberate honesty choices: +For Elite, every attempted member call becomes a separate `initial`, `critique`, or `revision` receipt; repeated providers have repeated receipts while `providers_called` stays unique. Incomplete runs retain only attempted phases. Total usage/latency covers all recorded member phases; synthesis and verdict provenance retain their existing fields. + - **No invented pricing.** `estimated_cost` stays `None` (a wrong number in an audit receipt is worse than none); `pricing_snapshot_date` is the dated-estimate slot until a real pricing table exists. Usage (tokens) is recorded; cost is not guessed. @@ -312,16 +322,16 @@ consumers. The end-to-end flow — `CLI/Library → Council → call_model → a | Module | Responsibility | |--------|----------------| -| `council.py` | `Council` — primary importable entry point. Resolves names, partitions members, and exposes two reusable primitives: `fan_out` (the single concurrent + partial-failure call loop) and `synthesize_blocks` (the single synthesizer/judge call path). Hosts the public mode API: `ask`/`ask_sync` (synthesize/raw), `debate`/`debate_sync`, `adversarial`/`adversarial_sync`, `vote`/`vote_sync`. Sync wrappers guard against a running event loop. Every mode funnels through `_cached_run`, which enforces the manifest-on-every-result invariant via `_ensure_manifest`. Runs `_apply_verdict` (default-on; `extract_verdict=False` opts out) on the synthesize buffered + streaming paths after the manifest exists. | +| `council.py` | `Council` — primary importable entry point. Resolves names, partitions members, and exposes two reusable primitives: `fan_out` (the single concurrent + partial-failure call loop) and `synthesize_blocks` (the single synthesizer/judge call path). Hosts the async/sync APIs for all six modes, including unreleased `elite`/`elite_sync`. Every mode funnels through `_cached_run`; completed Elite runs synthesize revisions and then use `_apply_verdict`. | | `verdict.py` | Public verdict/member Pydantic types (`CouncilVerdict`, `CouncilPosition`, `CouncilConflict`, `ProviderVote`, `MinorityReport`) + the LCD JSON Schemas (`verdict_json_schema`/`member_answer_json_schema`/`verdict_extraction_json_schema`) usable across all three native structured-output surfaces; `VERDICT_SCHEMA_VERSION`. | | `agreement.py` | Deterministic consensus: `consensus_score` (`position_cluster_ratio_v1` — largest cluster / positioned members; `None` for N<2) + `consensus_label` buckets. Pure arithmetic, no `difflib`, never LLM-emitted. | | `verdict_synthesis.py` | `extract_verdict` engine: one extraction call (clusters stances, never emits a number), native `output_contract` enforcement + prompt-level fallback, validate → repair-once → graceful `verdict=None`; the three verdict-absent reasons; provenance on every return path. | -| `manifest.py` | `ModelHarnessManifest` (first-class on every result), `ProviderExecutionReceipt`/`ProviderSkip`/`VerdictExtraction`, and `scan_for_secret_material()` → `secret_safety` stamp. No key values; `estimated_cost` left `None`. | -| `modes.py` | Deliberation orchestration: `run_debate` (multi-round, anonymized peers, drop-out) and `run_adversarial` (propose → refute → verdict). Built entirely on `Council.fan_out` + `synthesize_blocks` — no duplicated concurrency or synthesizer code. | -| `prompts.py` | Role/template strings for debate and adversarial (member, critic, judge, debate-final system prompts) and the anonymized peer-block builder. Separates *what each role is told* from *when to call whom*. | +| `manifest.py` | `ModelHarnessManifest` (first-class on every result), phased `ProviderExecutionReceipt`/`ProviderSkip`/`VerdictExtraction`, and `scan_for_secret_material()` → `secret_safety` stamp. No key values; `estimated_cost` left `None`. | +| `modes.py` | Deliberation orchestration: `run_debate`, `run_adversarial`, `run_vote`, and unreleased `run_elite` (three gated member phases). Built on `Council.fan_out` + `synthesize_blocks`. | +| `prompts.py` | Role/template strings for debate, adversarial, vote, and Elite evidence-audit/revision prompts. Elite panel text uses stable Model A/B/C aliases and answer ids, never provider identities. | | `providers.py` | `call_model` (+ `call_model_stream`) — the single async call path: resolve adapter, read key *by name at call time*, call adapter+transport (with an optional `output_contract`), parse, capture latency/usage/redacted-error into a `ModelAnswer`; never raises for provider-side failures. | | `transport.py` | The single async network boundary: `post_json` (buffered) + `stream_sse` (issue #7) — the only two httpx call sites in the highway. | -| `streaming.py` | Streaming engine (issue #7): `stream_ask` interleaves members via an `asyncio.Queue`, optionally streams the synthesizer, ends with a `done` event whose `CouncilResult` (incl. verdict) matches the buffered shape. synthesize/raw only. | +| `streaming.py` | Streaming engine (issue #7): `stream_ask` interleaves members via an `asyncio.Queue`, optionally streams the synthesizer, ends with a `done` result. Synthesize/raw only; Elite explicitly rejects streaming. | | `adapters/__init__.py` | `resolve_adapter(model_id, config)` — the provider registry + **extension seam**: one registration per family; config-only for OpenAI-compatible endpoints. | | `adapters/base.py` | `ProviderAdapter` protocol, `OutputContract` (native-structured-output request), `ProviderError`, and `redact()` (error-string secret scrubber). | | `adapters/openai_compat.py` | `OpenAICompatAdapter` — openai/xai/perplexity/groq/deepseek/mistral/together + custom endpoints; per-provider completions URL (Perplexity no `/v1`; Groq under `/openai/v1`); `response_format` json_schema when an `output_contract` is set. | @@ -329,7 +339,7 @@ consumers. The end-to-end flow — `CLI/Library → Council → call_model → a | `adapters/gemini.py` | `GeminiAdapter` — native `generateContent` (role-map, `systemInstruction` hoist, `usageMetadata`); `responseSchema` for an `output_contract`. | | `registry.py` | Single source of truth for name→model-id defaults + provider→env-var mapping. Key *presence* only — never values. | | `config.py` | Loads/merges `~/.conclave/config.yml` over defaults; resolves model ids + named/CSV councils; parses `endpoints:`. Keys-free by construction. | -| `models.py` | Pydantic contract: `TokenUsage`, `ModelAnswer` (stable `answer_id`), `CouncilResult` v2 — adds top-level `verdict`/`consensus_score`/`consensus_method`/`consensus_label`/`conflicts`/`provider_votes`/`minority_reports`/`manifest` (all backward-compatible). The stable importable surface for downstream consumers. | +| `models.py` | Pydantic contract: `TokenUsage`, `ModelAnswer`, `EliteResult` (protocol state + three phase-artifact lists), and backward-compatible `CouncilResult` v2 with verdict/consensus/manifest fields. | | `cli.py` | `conclave ask` and `conclave providers`. Rich panels for humans (incl. the green `VERDICT ()` panel + consensus/conflicts/minority blocks, or a dim `No verdict: ` note when absent), `--json` for machines (carries verdict + manifest). Never prints key values. | | `logging.py` | One logger factory, stderr, verbosity via `CONCLAVE_LOG_LEVEL` (default `WARNING`). | @@ -383,9 +393,8 @@ Condensed history (v0.x mode-detail archived per §4, per-release changelog in ` - **Not an agent framework.** No tool-calling graphs, stateful agents, or orchestration DSL — we compete by being *small*. (Permanent.) - **Not a key manager / secrets vault.** conclave reads env vars; it does not provision, rotate, store, or proxy keys. (Permanent.) - **No hosted/proxied token path.** No conclave-operated endpoint that sees prompts or takes a margin — BYO-keys, direct-to-provider, always. (Permanent.) -- **No streaming for debate/adversarial/vote** (synthesize/raw streaming landed in v0.3, #7). +- **No streaming for debate/adversarial/vote/elite** (synthesize/raw streaming landed in v0.3, #7). - **No server mode** (possible Roadmap, §9; local HTTP spike #8 closed no-go — §9 item 6). -- ~~**No `vote` mode** yet.~~ **Shipped in v1.1** (CAC-09 / #3) — `--mode vote --choices "A,B,C"` tallies a fixed ballot (plurality winner or split). Complementary to the verdict's `provider_votes`, which cluster free-form stances with evidence (§4a); the two are not the same thing. --- @@ -394,8 +403,13 @@ Condensed history (v0.x mode-detail archived per §4, per-release changelog in ` `adversarial`/`debate` shipped in v0.2; streaming/cache/convergence in v1.0; the **auditable council shipped in v1.1** (§4a — the wedge). +### Elite quality track (implemented, unreleased) + +Elite is the owner-approved next product slice: stronger decisions before broader operability. Its source implementation is complete, but this document assigns no release number and claims no publication. Release requires normal verification, versioning, and publishing gates; the operability prove-it gate below does not block this quality track. + ### v1.2 — the "Operable Council" (DEMAND-GATED, not scheduled) -v1.2 is held behind a **prove-it gate**: put the auditable verdict in front of real users +The broader v1.2 operability substrate is held behind a **prove-it gate**: put the auditable +verdict and Elite workflow in front of real users first; observed pull authorizes the build. This substrate is demoted from the old "engagement-modes-first" plan and only builds if demand appears — **engagement modes** (regular/smart) on a generation-settings substrate; **thin task profiles** @@ -408,29 +422,13 @@ score is validated against real runs. ### Landed history (kept struck-through for traceability) -1. ~~**`vote` mode**~~ **SHIPPED in v1.1 (CAC-09 / #3)** — a real constrained-choice mode (`--mode vote --choices "A,B,C"` / `Council.vote()`): each member picks one labelled option, tallied to a plurality `winner`/`split` on `result.vote`. Complementary to the verdict's `provider_votes` (§4a), not the same thing. -2. ~~**Debate convergence/stop criteria**~~ **LANDED (#4)** — opt-in `converge_threshold` early-stop on round-over-round text stability (`difflib`); off by default; recorded on `CouncilResult.converged`/`convergence_score`. (NB: text-stability ≠ the verdict `consensus_score`, §4a.) -3. ~~**More first-class providers**~~ **LANDED (#5)** — `groq`/`deepseek`/`mistral`/`together` promoted to typed defaults (`registry.OPENAI_COMPAT_PROVIDERS`); no native adapter; aggregators excluded (§11). -4. ~~**Caching**~~ **LANDED (#6)** — opt-in result cache (`config.cache` / `--cache`), off by default, on-disk, secret-free SHA-256 key, corrupt = silent miss; hit on `CouncilResult.cached`. -5. ~~**Streaming**~~ **LANDED (#7)** — `synthesize`/`raw` streaming via `Council.ask_stream` + CLI `--stream` over `transport.stream_sse`; never-raises + partial text preserved; non-streaming default byte-for-byte unchanged. -6. **Local HTTP/server mode (open)** — a *local* server for convenience only; must not become a hosted token path. **Spike #8 (2026-06-09): no-go on HTTP** (`127.0.0.1` still carries DNS-rebinding/CSRF surface; the library already serves in-process); if cross-process access is wanted, prefer a thin **stdio MCP server**. Final disposition is the maintainer's. -7. ~~**Key-leak hardening**~~ **LANDED in v0.3** via `redact()` on every provider/transport error before `ModelAnswer.error` (§3). - -**Roadmap discipline:** items are reprioritized freely but not *removed* on a single data -point; completed items are **marked done in place** (struck through with a "LANDED" note). +Shipped items remain traceable in `CHANGELOG.md`: vote (CAC-09/#3), debate convergence (#4), four additional first-class providers (#5), caching (#6), streaming (#7), and key-leak hardening. The local HTTP spike (#8) remains a no-go; prefer a thin stdio MCP server if cross-process access gains demand. Roadmap items are reprioritized freely but not removed on one data point. --- ## 10. Downstream Boundary: conclave ↔ mcp-warden -**mcp-warden** (sibling MCP-server security gateway) **imports conclave as a DEV-TIME -dependency only** — adversarial design review of warden's strategy, taxonomy brainstorming. - -**mcp-warden will NOT use conclave as a RUNTIME dependency.** Security findings require -determinism and reproducibility; a stochastic council is the wrong tool for runtime -adjudication. The v1.1 verdict does **not** change this: its `consensus_score` is -deterministic *arithmetic*, but the *clustering* it scores is LLM-assisted, so the verdict -is auditable, not a reproducible gate (§8). This boundary is deliberate and load-bearing: +**mcp-warden** imports conclave for dev-time design review and taxonomy work only, never as a runtime dependency. Security findings require determinism; conclave remains stochastic even though its consensus arithmetic is deterministic over LLM-assisted clustering. This boundary is load-bearing: | | conclave (this project) | mcp-warden runtime | |---|---|---| From 75bd5c4f5670cb3ecf51bedbbec90fd25c92b839 Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Fri, 17 Jul 2026 05:29:29 -0400 Subject: [PATCH 10/21] docs: normalize elite plan whitespace --- docs/plans/2026-07-17-elite-decision-protocol-design.md | 1 - docs/plans/2026-07-17-elite-decision-protocol.md | 1 - 2 files changed, 2 deletions(-) diff --git a/docs/plans/2026-07-17-elite-decision-protocol-design.md b/docs/plans/2026-07-17-elite-decision-protocol-design.md index bb0459a..e45d3af 100644 --- a/docs/plans/2026-07-17-elite-decision-protocol-design.md +++ b/docs/plans/2026-07-17-elite-decision-protocol-design.md @@ -66,4 +66,3 @@ unverified claims from disproven claims because conclave has no retrieval or too - Manifest receipts expose every member phase and remain secret-free. - Existing modes, cache behavior, output schemas, and public APIs remain compatible. - Unit, CLI, manifest, cache, secret-safety, lint, formatting, and full test suites pass. - diff --git a/docs/plans/2026-07-17-elite-decision-protocol.md b/docs/plans/2026-07-17-elite-decision-protocol.md index 4460850..592eadc 100644 --- a/docs/plans/2026-07-17-elite-decision-protocol.md +++ b/docs/plans/2026-07-17-elite-decision-protocol.md @@ -255,4 +255,3 @@ Confirm no `.env`, credentials, generated caches, release tags, or publishing ch **Step 5: Commit** `git add README.md docs/PRODUCT_DESIGN_DOCUMENT.md SYSTEM_CONTEXT_DIAGRAM.md DOCUMENTATION_INDEX.md CHANGELOG.md && git commit -m "docs: specify elite decision protocol"` - From 68c61b4bf359e03e3481e0f7fd47cb6db0bec030 Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Fri, 17 Jul 2026 07:46:41 -0400 Subject: [PATCH 11/21] docs: reframe roadmap around decision quality proof --- DOCUMENTATION_INDEX.md | 9 +- docs/PRODUCT_DESIGN_DOCUMENT.md | 61 +++-- .../2026-07-17-decision-quality-roadmap.md | 223 ++++++++++++++++++ 3 files changed, 259 insertions(+), 34 deletions(-) create mode 100644 docs/plans/2026-07-17-decision-quality-roadmap.md diff --git a/DOCUMENTATION_INDEX.md b/DOCUMENTATION_INDEX.md index c9f2cc3..b3298f0 100644 --- a/DOCUMENTATION_INDEX.md +++ b/DOCUMENTATION_INDEX.md @@ -23,7 +23,13 @@ the canonical authority spec on top of those. | Doc | Path | Purpose | |-----|------|---------| -| **Product Design Document** | [`docs/PRODUCT_DESIGN_DOCUMENT.md`](docs/PRODUCT_DESIGN_DOCUMENT.md) | **Canonical** product spec: six modes including implemented-but-unreleased Elite, its fixed three-success phase gate and cost/latency tradeoff, the v1.1 auditable verdict, manifest, architecture, boundaries, and roadmap. **When docs disagree, the PDD wins.** | +| **Product Design Document** | [`docs/PRODUCT_DESIGN_DOCUMENT.md`](docs/PRODUCT_DESIGN_DOCUMENT.md) | **Canonical** product spec: six modes including implemented-but-unreleased Elite, its fixed three-success phase gate and cost/latency tradeoff, the v1.1 execution-traceable verdict, manifest, architecture, boundaries, and roadmap. **When docs disagree, the PDD wins.** | + +## Product plans + +| Doc | Path | Purpose | +|-----|------|---------| +| **Decision Quality Roadmap** | [`docs/plans/2026-07-17-decision-quality-roadmap.md`](docs/plans/2026-07-17-decision-quality-roadmap.md) | H0-H4 prove-it roadmap: Elite correctness, randomized ablations, source-grounded Decision Briefs, buyer pull, outcome learning, old-backlog disposition, demand gates, and kill criteria. | --- @@ -101,6 +107,7 @@ Run: `pytest` (config in `pyproject.toml`, `asyncio_mode = "auto"`). | Date | Change | |------|--------| +| 2026-07-17 | Reframed the product roadmap around empirically proven decision quality: narrowed current claims to execution traceability, identified answer IDs as internal provenance rather than external evidence, gated Elite merge on H0 correctness, and added H1-H4 evidence, buyer, and outcome gates. | | 2026-07-17 | Documented the implemented-but-unreleased Elite Decision Protocol: fixed three-success gates, initial/audit/revision artifacts, existing final verdict, phased receipts, failure semantics, cost/latency tradeoff, and no streaming. | | 2026-07-13 | **Manifest-on-every-result invariant fix + doc reconciliation.** `ModelHarnessManifest` now attaches on all five modes (was `None` for `debate`/`adversarial`/`vote`) via a single-site fix at `Council._cached_run` → new `_ensure_manifest`; regression tests in `tests/test_manifest_all_modes.py`. Docs corrected so the manifest-on-every-result claim is accurate and the `vote` mode is documented as **shipped** (CAC-09 / #3), not "absorbed" — across README, `SYSTEM_CONTEXT_DIAGRAM.md`, PDD (§1/§3/§4a/§8/§9/§12), and `CHANGELOG.md` (`[Unreleased]`). Verdict scope unchanged (still `synthesize`/`ask`-only; not layered on `adversarial`). | | 2026-06-21 | **v1.1 docs pivot — the auditable multi-model council.** PDD reframed to the auditable-council wedge (new §4a: CouncilResult v2, deterministic `position_cluster_ratio_v1` consensus, native + fallback structured output, the verdict-optional rule, the secret-free `ModelHarnessManifest`); `vote` mode marked **absorbed** by `provider_votes` across PDD §1/§4/§8/§9/§12; demand-gated v1.2 "Operable Council" roadmap. System Context diagram gains the verdict pipeline + manifest (mermaid re-validated). README gains an "Auditable verdict" section (CLI panel + library example). `CHANGELOG.md` `[1.1.0]` added. v0.x mode-detail prose archived to [`docs/archive/pdd-v0.x-modes-detail.md`](docs/archive/pdd-v0.x-modes-detail.md) to keep the PDD under 500 lines. New modules documented: `verdict.py`, `agreement.py`, `verdict_synthesis.py`, `manifest.py`. | diff --git a/docs/PRODUCT_DESIGN_DOCUMENT.md b/docs/PRODUCT_DESIGN_DOCUMENT.md index 5c6876c..53aa76d 100644 --- a/docs/PRODUCT_DESIGN_DOCUMENT.md +++ b/docs/PRODUCT_DESIGN_DOCUMENT.md @@ -3,7 +3,7 @@ > **Status:** v1.1 stable (the BYO-keys multi-model council: synthesize/raw/debate/ > adversarial/vote, 9 providers, owned httpx provider highway, key-leak hardening, streaming, > cache). **v1.1 — the auditable council — SHIPPED:** every run now yields a structured, -> agreement-scored, fully auditable **verdict** plus a redacted execution **manifest** (see +> agreement-scored, execution-traceable **verdict** plus a redacted execution **manifest** (see > §4a). The quality-first **Elite Decision Protocol is implemented but unreleased**. > This is the **canonical authority document** for conclave's product scope, design, > and roadmap. When this document and any other doc disagree, this document wins. Code is the @@ -43,8 +43,8 @@ response; v0.2 adds **council modes** — **debate** (multi-round) and **adversa deliberation. **The v1.1 wedge — the auditable council.** A synthesis paragraph is not enough to *act* on. -v1.1 makes the product identity precise: **a multi-model council verdict you can act on — -structured, scored for agreement, fully auditable.** Every run yields a `CouncilVerdict` +v1.1 makes the product identity precise: **a multi-model council verdict you can inspect — +structured, scored for agreement, and execution-traceable.** Every run yields a `CouncilVerdict` exposing agreement, disagreement (`conflicts`), minority views (`minority_reports`), and per-provider votes (`provider_votes`); a deterministic `consensus_score` (arithmetic over the model's clustering, *never* an LLM-emitted number); and a redacted `ModelHarnessManifest` (how @@ -52,7 +52,7 @@ the run executed + which model produced the analysis) riding on **every** result modes (one chokepoint, §4a). A constrained-choice **`vote` mode** also **shipped** (CAC-09 / #3, `--mode vote --choices "A,B,C"` → plurality winner/split) — distinct from `provider_votes`. -The next product-quality step is **Elite** (implemented, unreleased): independent answers → council-wide evidence audits → member revisions → the existing synthesis and auditable verdict. Elite requires three successful responders at every member phase and intentionally spends more calls and time to improve consequential decisions. +The next product-quality step is **Elite** (implemented, unreleased): independent answers → council-wide answer/claim audits → member revisions → the existing synthesis and execution-traceable verdict. Elite requires three successful responders at every member phase and intentionally spends more calls and time to improve consequential decisions. conclave's first real use was an **adversarial design review**: a council of Grok, Gemini, Perplexity, and Claude critiquing a security-tool strategy and catching flaws a single @@ -177,8 +177,8 @@ evidence_answer_ids}`; `CouncilConflict{topic, position_labels, summary, consens `ProviderVote{provider, position_label, confidence}`; `MinorityReport{providers, claim, evidence_answer_ids, why_it_matters}`. -**Evidence is the product, not a nicety.** Every clustered stance cites `evidence_answer_ids` -(the member `answer_id`s backing it) and every conflict names the positions in tension — a +**Answer provenance is the current product, not external evidence.** Every clustered stance +cites `evidence_answer_ids` (the member `answer_id`s backing it) and every conflict names the positions in tension — a conflict that just says "models disagreed about cost" without pointing at answers is a *failure*. `ProviderVote.confidence` is recorded but **never used in arithmetic**. @@ -200,14 +200,15 @@ The consensus number is **arithmetic over the model's clustering, never LLM-emit | `majority` | 0.5 < score < 0.75 | | `split` | score ≤ 0.5 (no majority); a 1-of-2 tie is `0.5` = `split`, never "50% consensus" | -**Why it is auditable, not theater.** The extraction schema carries *no* consensus field — +**Why it is execution-traceable, not ground truth.** The extraction schema carries *no* consensus field — `verdict_extraction_json_schema()` strips it and `VerdictExtractionModel` ignores extra keys, so a model that smuggles a number in is dropped by the validator. The module deliberately does **not** import `difflib`: text-similarity is the debate `convergence_score` (a *forbidden* consensus measure), never conflated with agreement. The single LLM-assisted step is the **semantic clustering** of stances; the number is reproducible arithmetic over it, each cluster cites its `evidence_answer_ids`, and the manifest records which model + prompt -version did the clustering — so the score is traceable. +version did the clustering — so the score is reproducible and traceable. Answer IDs point to +council outputs, not external sources; source grounding is Roadmap (§9). ### Verdict extraction + native structured output `extract_verdict(prompt, member_answers, *, synthesizer_name, synthesizer_model_id, @@ -403,26 +404,20 @@ Condensed history (v0.x mode-detail archived per §4, per-release changelog in ` `adversarial`/`debate` shipped in v0.2; streaming/cache/convergence in v1.0; the **auditable council shipped in v1.1** (§4a — the wedge). -### Elite quality track (implemented, unreleased) - -Elite is the owner-approved next product slice: stronger decisions before broader operability. Its source implementation is complete, but this document assigns no release number and claims no publication. Release requires normal verification, versioning, and publishing gates; the operability prove-it gate below does not block this quality track. - -### v1.2 — the "Operable Council" (DEMAND-GATED, not scheduled) -The broader v1.2 operability substrate is held behind a **prove-it gate**: put the auditable -verdict and Elite workflow in front of real users -first; observed pull authorizes the build. This substrate is demoted from the old -"engagement-modes-first" plan and only builds if demand appears — **engagement modes** -(regular/smart) on a generation-settings substrate; **thin task profiles** -(cheap/balanced/frontier/critic); **profile compilation + routing** -(`parallel_synthesize`, `sequential_fallback`, `cheap_then_smart`); a **capability cache** -with explicit refresh/discovery; `conclave doctor` + `providers` subcommands; a minimal -mock/replay transport; and a narrow eval harness. The `cheap_then_smart` soft-triggers (low -confidence / high disagreement) are gated behind **proven scoring** — flag-only until the -score is validated against real runs. - -### Landed history (kept struck-through for traceability) - -Shipped items remain traceable in `CHANGELOG.md`: vote (CAC-09/#3), debate convergence (#4), four additional first-class providers (#5), caching (#6), streaming (#7), and key-leak hardening. The local HTTP spike (#8) remains a no-go; prefer a thin stdio MCP server if cross-process access gains demand. Roadmap items are reprioritized freely but not removed on one data point. +The revised thesis is a **source-grounded, execution-traceable decision record with +empirically proven quality**. Current answer IDs identify model outputs, not external evidence; +"fully auditable" is therefore too broad until source grounding ships. Elite remains +implemented but unreleased on draft PR #51. + +The canonical roadmap is +[`docs/plans/2026-07-17-decision-quality-roadmap.md`](plans/2026-07-17-decision-quality-roadmap.md): +**H0** closes Elite correctness and wording gaps before merge; **H1** runs budget-matched, +randomized ablations with go/kill gates; **H2** adds a minimal Markdown evidence bundle, +claim ledger, deterministic citation-integrity checks, readiness tri-state, and Markdown/JSON +Decision Brief; **H3** proves onboarding, repeat use, and paid buyer pull; **H4** learns +escalation and roster weighting from outcomes. The plan also records retain/promote/retire +decisions for the old v1.2 backlog, commodity-versus-moat boundaries, demand-gated items, and +portfolio kill criteria. No horizon is a release commitment. --- @@ -456,13 +451,13 @@ chase LiteLLM (routing/budgets), Vercel AI SDK (provider abstraction), LangChain **Where we are *now* distinct** (re-anchored on what competitors have not replicated): -1. **The auditable verdict (the v1.1 wedge).** A council answer you can *act on*: structured +1. **The execution-traceable verdict (the v1.1 wedge).** A council answer you can inspect: structured positions, `conflicts` and `minority_reports` that cite `evidence_answer_ids`, `provider_votes`, a **deterministic** `consensus_score` (arithmetic over the model's clustering, never an LLM-emitted number), and a redacted `ModelHarnessManifest` recording which model + prompt version produced the disagreement analysis. Peers ship "structured - verdicts" as synthesizer *content*; conclave's verdict is a reproducible, evidence-cited, - provenance-stamped object. **This is now the most defensible claim.** + verdicts" as synthesizer *content*; conclave's current verdict is a reproducible, + answer-linked, provenance-stamped object. External source grounding remains Roadmap (§9). 2. **Owned, zero-LLM-SDK provider highway** — a single hand-owned httpx transport + adapter registry, no provider SDKs, no OpenRouter. Competitors lean on aggregators or vendor SDKs. 3. **Direct-keys / no-middleman + name-only key rigor** — never an aggregator, never a token @@ -471,9 +466,9 @@ chase LiteLLM (routing/budgets), Vercel AI SDK (provider abstraction), LangChain 4. **A telemetry-grade `CouncilResult` contract** — per-model latency + token usage + typed error capture as a *stable downstream contract* (the mcp-warden dev-time story). -We are the small, embeddable, **auditable** council primitive — not a LangGraph/AutoGen +We are the small, embeddable, **execution-traceable** council primitive — not a LangGraph/AutoGen rival. Against the direct peers (`llm-council-core`, `the-llm-council`) we differentiate on -the auditable verdict, the owned provider layer, the no-aggregator posture, and key rigor. +the execution-traceable verdict, the owned provider layer, the no-aggregator posture, and key rigor. --- diff --git a/docs/plans/2026-07-17-decision-quality-roadmap.md b/docs/plans/2026-07-17-decision-quality-roadmap.md new file mode 100644 index 0000000..70e0c2a --- /dev/null +++ b/docs/plans/2026-07-17-decision-quality-roadmap.md @@ -0,0 +1,223 @@ +# Conclave Decision Quality Roadmap + +**Status:** Approved product direction; implementation is gated by the evidence below +**Date:** 2026-07-17 +**Current product:** v1.1.0 stable; Elite implemented but unreleased on draft PR #51 + +## Thesis + +Conclave should become the smallest tool that produces a **source-grounded, +execution-traceable decision record whose quality is empirically proven**. The product is +not "more agents." It is a disciplined path from source material to competing claims, +critique, decision readiness, and an exportable brief, with receipts that let a reviewer +reconstruct what happened. + +That wording is intentionally narrower than the current "fully auditable" language. Today, +Conclave traces model execution and the model-assisted clustering behind a verdict. Its +`evidence_answer_ids` identify council answers, **not external evidence**. Until source +grounding and deterministic citation validation ship, the honest claim is +**execution-traceable**, not source-auditable or fully auditable. + +## Product doctrine + +- Quality claims require comparative evidence, not architectural intuition. +- A completed protocol is not necessarily a decision ready for use. +- Atomic, source-linked claims are the unit of quality; prose fluency is not. +- Preserve dissent, uncertainty, and failure states rather than forcing a verdict. +- Keep BYO keys, direct provider access, the owned transport, and a library-first API. +- Build the narrowest workflow that improves consequential decisions; integrate commodity + infrastructure instead of recreating it. + +## Horizon 0 — verify Elite correctness before merge + +Elite remains implemented but unreleased. Draft PR #51 must remain draft until these +correctness gaps are resolved and regression-tested: + +1. **Persistent identities.** Give each initial answer a stable identifier that survives + audit, revision, synthesis, serialization, and cache replay. Do not present those IDs as + external citations. +2. **Two states, not one.** Separate `protocol_completion` (all required phases executed) + from `decision_readiness`. Readiness is `ready`, `not_ready`, or `indeterminate`, with + machine-readable reasons. A completed run may still be `not_ready`. +3. **Full-call receipts.** Manifest totals must include member, audit, revision, synthesis, + repair, and verdict-extraction calls, with phase, latency, usage, errors, and prompt/schema + versions. Never imply completeness when final calls are absent from totals. +4. **Version-aware cache.** Cache identity must cover protocol, prompt, schema, model, + generation settings, source-bundle digest, and relevant config versions so incompatible + artifacts cannot replay as current. +5. **Custom config threading.** Every Elite phase, including synthesis and verdict + extraction, must receive the same resolved custom-endpoint/provider configuration. +6. **Correct language.** Rename "evidence audit" where necessary to "answer/claim audit" + until external sources exist. Product copy must say execution-traceable. Answer IDs prove + provenance within a run; they do not prove truth. + +**Merge gate:** all six items have tests, the complete suite and static checks pass, a +secret scan passes, documentation matches behavior, and PR review finds no false quality or +auditability claim. Otherwise keep Elite unreleased. + +## Horizon 1 — prove that the protocol improves decisions + +Run a budget-matched randomized ablation before claiming Elite is better. Compare at least +single-frontier-model, ordinary synthesis, adversarial, and Elite conditions using the same +task set and matched token or dollar budgets. Randomize condition order and blind human +graders to mode and provider identity. + +### Program stages + +1. **Pilot:** 20-30 diverse, answerable decision tasks to debug rubrics, establish grader + agreement, estimate variance, and set confirmatory sample size. Results are exploratory. +2. **Confirmatory:** preregister hypotheses, primary metric, exclusions, analysis, minimum + effect, and sample size; freeze prompts and scoring before running the held-out set. +3. **Shadow:** run the candidate protocol alongside real decisions without influencing them; + compare usefulness, unsupported claims, reversals, cost, latency, and reviewer effort. + +### Atomic metrics + +- supported-claim precision and material-claim source coverage; +- factual error and unsupported-claim rates, severity-weighted; +- recommendation correctness or expert preference on tasks with a defensible reference; +- calibrated readiness: error/reversal rate by `ready`/`not_ready`/`indeterminate`; +- conflict and minority-view recall against expert annotation; +- decision completeness and actionability on a fixed rubric; +- inter-rater reliability, run-to-run stability, latency, tokens, cost, and reviewer minutes. + +Report distributions and confidence intervals, not one composite "quality score." Cost and +latency are co-primary constraints, not footnotes. + +### Anti-gaming controls + +- held-out tasks and sources; no tuning on confirmatory cases; +- atomic claim scoring before holistic preference scoring; +- blinded, randomized outputs with length-normalized views; +- identical evidence access and budget ceilings across conditions; +- duplicate/leakage checks, adjudicated grader disagreements, and preserved raw scores; +- failure-inclusive analysis: timeouts, abstentions, invalid citations, and incomplete runs + remain in the denominator; +- publish prompt, roster, version, exclusions, and analysis artifacts with each result. + +### Go/kill gates + +Advance to productization only if Elite shows a statistically credible improvement on the +preregistered primary quality metric and no material regression in severe-error rate, +calibration, or reviewer effort, within the agreed cost/latency ceiling. Repeat on at least +two task families. Kill or redesign Elite if it does not beat the best budget-matched +baseline, gains disappear on held-out tasks, graders cannot agree on the rubric, or the same +quality is achieved by a simpler single-model critique/revision loop. + +## Horizon 2 — source-grounded decision records + +Build the minimum evidence product, in this order: + +1. **Markdown evidence bundle.** Accept local Markdown containing source blocks with stable + source IDs, title/origin metadata, and quoted or summarized content. No crawler, vector + database, or document platform in the first version. +2. **Integrity.** Canonicalize each source block and store a content digest. Record the bundle + digest in cache identity and the run manifest. +3. **Structured claim audit.** Emit atomic claims with disposition, materiality, supporting + and contradicting source IDs, exact citation spans, and audit rationale. +4. **Deterministic validation.** Verify that every cited source and span exists and that the + quoted text matches the digested bundle. This validates citation integrity, not semantic + truth; semantic entailment remains scored and explicitly probabilistic. +5. **Claim ledger.** Preserve claim lineage from initial answer through critique, revision, + synthesis, and final disposition, including unresolved conflicts and minority positions. +6. **Readiness tri-state.** Compute `ready`, `not_ready`, or `indeterminate` from explicit + policy and validated artifacts; never infer readiness from protocol completion alone. +7. **Decision Brief.** Export equivalent Markdown and JSON containing recommendation, + alternatives, assumptions, claim ledger, conflicts, minority report, source citations, + readiness, execution receipts, and limitations. + +**Go gate:** deterministic citation checks catch seeded broken citations, humans can trace +every material brief claim to a source or explicit unsupported status, and JSON/Markdown +round-trip without semantic drift. **Kill criterion:** if the minimal bundle adds process +cost without improving supported-claim precision or reviewer time in the confirmatory study, +do not expand ingestion. + +## Horizon 3 — prove buyer pull and repeat use + +Target skeptical engineering and security leaders making architecture, vendor, build/buy, +and risk decisions. Run small experiments rather than building a sales platform: + +- **Onboarding:** five fresh users install, configure three providers, run a supplied evidence + bundle, and export a brief; measure completion, time-to-first-brief, and support needed. +- **Buyer pilots:** 3-5 teams use Conclave on live but reversible decisions for 4-6 weeks; + compare their current review process, reviewer minutes, defects caught, and decision reuse. +- **Repeat use:** measure a second self-initiated decision within 30 days and whether briefs + are shared in an existing review workflow. +- **Paid design partners:** ask qualified pilots to pay for a defined support/evaluation + package before building collaboration or hosting features. + +**Go gate:** at least three teams complete pilots, two use it again without prompting, and +two accept a paid design-partner offer or equivalent written budget commitment. **Kill or +pivot:** fewer than two teams repeat after onboarding fixes, no buyer treats the brief as a +review artifact, or willingness to pay depends on hosted collaboration rather than decision +quality. Revisit the persona and workflow before adding features. + +## Horizon 4 — learn from outcomes + +Only after repeat use exists, add an opt-in outcome journal linking a decision record to later +results, reversals, and reviewer feedback. Use that evidence to learn, by task family: + +- when ordinary synthesis is enough and when escalation to Elite pays; +- which roster compositions improve quality per dollar/minute; +- whether provider weights help out of sample; +- which readiness signals predict reversals or severe errors. + +Escalation and roster weighting must start as offline recommendations, be evaluated on +held-out outcomes, retain an explicit manual override, and never hide minority views. Kill +learned weighting if lift does not replicate, creates provider monoculture, worsens +calibration, or cannot be explained from recorded evidence. + +## Previous v1.2 backlog disposition + +| Previous item | Disposition | Reason / gate | +|---|---|---| +| Narrow eval harness and mock/replay transport | **Promote to H1** | Required for reproducible ablations; keep narrow and artifact-based. | +| `conclave doctor` and provider diagnostics | **Retain in H3** | Valuable onboarding aid; implement only against observed setup failures. | +| Generation-settings substrate | **Retain** | Build the minimum needed for budget-matched experiments and receipts. | +| Thin profiles (`cheap`, `balanced`, `frontier`, `critic`) | **Retire as product presets for now** | Roster/profile names imply quality before evidence; experiments may use internal fixed conditions. | +| Regular/smart engagement modes | **Retire** | Product framing is decision quality, not engagement. | +| Profile compilation and generic routing | **Demand-gated** | Commodity gateway territory; add only for a proven decision workflow. | +| `cheap_then_smart` adaptive routing | **Promote concept to H4** | Learn escalation from outcomes; do not trigger on unvalidated confidence. | +| Capability cache/discovery | **Retain, demand-gated** | Add only when provider drift blocks pilots; require explicit refresh and provenance. | +| Local HTTP server | **Retire / no-go** | Prior spike found no need; keep library and CLI surfaces. | +| Thin stdio MCP | **Demand-gated** | Build only after repeated integration requests from pilots. | + +## Commodity versus moat + +**Commodity/integrate:** provider routing, generic model catalogs, hosted observability, +prompt management, vector search, crawlers, document conversion, workflow engines, dashboards, +and general eval platforms. Conclave should emit clean artifacts and interoperate with them. + +**Potential moat, still to be proven:** the evaluation corpus and atomic scoring method; +source/claim lineage across multi-model critique and revision; calibrated decision-readiness +signals; evidence about which deliberation protocol and roster works for which decision; +and trusted, portable Decision Briefs with deterministic integrity checks. The owned provider +highway and BYO-key rigor are valuable control and trust properties, but not sufficient moat. + +## Do not build without demonstrated demand + +- hosted multi-tenant SaaS, accounts, billing, collaboration, or a cloud prompt proxy; +- a general agent framework, gateway/router, provider SDK, or workflow engine; +- broad RAG, crawling, vector storage, or document-management infrastructure; +- a local review UI, IDE extension, stdio MCP, or OpenTelemetry exporter; +- generic dashboards, marketplace profiles, autonomous action execution, or security gates; +- streaming Elite, dynamic swarms, automatic model procurement, or opaque self-improvement. + +Any item above needs repeated pilot evidence, a named buyer/workflow, a measurable quality or +adoption hypothesis, a simpler-alternative analysis, and an explicit kill date. + +## Portfolio kill criteria + +Stop or materially reframe the Decision Quality strategy if, after H1 and corrected H3 +onboarding, any of these hold: + +- no protocol beats the strongest budget-matched baseline on held-out tasks; +- source grounding does not reduce severe unsupported claims or reviewer effort; +- quality gains require cost/latency buyers reject and adaptive escalation cannot recover it; +- users value model comparison as entertainment but do not use briefs in real decisions; +- pilots do not repeat or produce credible willingness to pay; +- outcome data is too sparse, biased, or sensitive to support safe learning; +- a simpler commodity tool matches the full decision record and measured quality. + +If killed, preserve Conclave as a small open-source council/evaluation primitive; do not chase +usage with unrelated platform features. From bff6bb1408fb6383590ea13f52a61ccd313dc32b Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Fri, 17 Jul 2026 08:06:39 -0400 Subject: [PATCH 12/21] docs: plan elite horizon zero hardening --- docs/plans/2026-07-17-elite-h0-hardening.md | 155 ++++++++++++++++++++ 1 file changed, 155 insertions(+) create mode 100644 docs/plans/2026-07-17-elite-h0-hardening.md diff --git a/docs/plans/2026-07-17-elite-h0-hardening.md b/docs/plans/2026-07-17-elite-h0-hardening.md new file mode 100644 index 0000000..e72e5fe --- /dev/null +++ b/docs/plans/2026-07-17-elite-h0-hardening.md @@ -0,0 +1,155 @@ +# Elite Horizon 0 Hardening Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Close every Horizon 0 correctness gap in DSE-690 so Elite is execution-traceable, configuration-consistent, cache-safe, and incapable of reporting a decision-ready success when required protocol phases fail. + +**Architecture:** Keep Elite inside the existing `Council` orchestration surface. Add stable Conclave-owned answer identity at the provider boundary, explicit readiness on `EliteResult`, a shared per-call receipt path that covers every phase, and a canonical versioned cache identity. Preserve backward-compatible `completed` semantics as protocol completion while making readiness a separate machine-readable contract. + +**Tech Stack:** Python 3.11+, Pydantic, asyncio, httpx, pytest, pytest-asyncio, Ruff, Typer. + +--- + +## Working rules + +- Implement each task red-green-refactor: add the smallest failing test, run it and confirm the expected failure, implement the minimum fix, then rerun focused tests. +- Use a fresh serialized subagent for each bounded implementation task; do not allow concurrent edits in this worktree. +- After each subagent commit, review the diff and independently rerun its focused verification before continuing. +- Use this sandbox-safe pytest form: + + ```bash + PYTHONPATH=src PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 pytest -p no:cacheprovider -p pytest_asyncio.plugin + ``` + +- Do not merge PR #51 or release a package. Horizon 0 ends at verified code, docs, Linear evidence, commit, and push. + +## Task 1: Persistent answer identity and accurate claim-audit language + +**Files:** + +- Modify: `src/conclave/models.py` +- Modify: `src/conclave/providers.py` +- Modify: `src/conclave/prompts.py` +- Modify: `src/conclave/modes.py` +- Modify: `src/conclave/verdict.py` +- Modify: `src/conclave/verdict_synthesis.py` +- Test: `tests/test_providers.py` +- Test: `tests/test_elite_mode.py` +- Test: `tests/test_output_contract_plumbing.py` + +1. Add failing tests proving every successful `call_model` answer receives a non-secret Conclave-owned stable ID and that IDs survive critique, revision, synthesis input, serialization, and reconstructed/cache-replayed results. +2. Add failing prompt/contract tests that use `claim audit`, `answer provenance`, or `within-run provenance`, and reject language implying answer IDs are external evidence. +3. Generate the ID deterministically from non-secret immutable response facts, with phase-specific derived IDs where a phase transforms an answer. Do not use provider names as the normal fallback. +4. Rename internal prompt text and docstrings from evidence audit to claim audit without breaking serialized field compatibility (`evidence_answer_ids` remains a compatibility field until a separately versioned schema migration). +5. Run focused tests and commit: + + ```bash + git add src/conclave/models.py src/conclave/providers.py src/conclave/prompts.py src/conclave/modes.py src/conclave/verdict.py src/conclave/verdict_synthesis.py tests/test_providers.py tests/test_elite_mode.py tests/test_output_contract_plumbing.py + git commit -m "fix(elite): preserve answer identity and claim provenance" + ``` + +## Task 2: Thread resolved configuration through every model call + +**Files:** + +- Modify: `src/conclave/council.py` +- Modify: `src/conclave/verdict_synthesis.py` +- Test: `tests/test_council.py` +- Test: `tests/test_elite_mode.py` +- Test: `tests/test_output_contract_plumbing.py` + +1. Add failing spies proving member fan-out, synthesis, verdict extraction, and verdict repair all receive the same resolved `ConclaveConfig` instance. +2. Extend verdict extraction with an optional config parameter and pass it to both initial and repair `call_model` invocations. +3. Pass `self.config` through `Council.fan_out`, `Council.synthesize_blocks`, and `_extract_verdict`. +4. Preserve public call compatibility by keeping new arguments keyword-only and optional where functions are public. +5. Run focused tests and commit `fix(config): thread resolved config through council calls`. + +## Task 3: Make cache identity protocol- and version-aware + +**Files:** + +- Modify: `src/conclave/cache.py` +- Modify: `src/conclave/council.py` +- Modify: `src/conclave/prompts.py` +- Modify: `src/conclave/verdict.py` +- Test: `tests/test_cache.py` +- Test: `tests/test_secret_safety_matrix.py` + +1. Add failing tests showing cache keys differ when protocol version, prompt version, verdict schema version, model roster, generation settings, verdict extraction flag, relevant resolved config, or evidence-bundle digest changes. +2. Define explicit protocol/prompt/schema version constants next to their owners and include them in one canonical secret-free cache identity document. +3. Add an optional `source_bundle_digest` cache-key input now so Horizon 2 source grounding cannot silently reuse ungrounded entries. +4. Hash only normalized, non-secret configuration identity; prove API keys and raw secret values cannot enter keys or payloads. +5. Preserve cache replay compatibility by treating old entries as misses rather than attempting unsafe migration. +6. Run focused tests and commit `fix(cache): version elite protocol identity`. + +## Task 4: Separate protocol completion from decision readiness + +**Files:** + +- Modify: `src/conclave/models.py` +- Modify: `src/conclave/modes.py` +- Modify: `src/conclave/council.py` +- Modify: `src/conclave/cli.py` +- Test: `tests/test_elite_mode.py` +- Test: `tests/test_cli.py` +- Test: `tests/test_cache.py` + +1. Add failing tests for `ready`, `not_ready`, and `indeterminate`, including machine-readable reasons. +2. Keep `completed` as the factual protocol-completion flag. Add `decision_readiness` and `readiness_reasons` to `EliteResult` with conservative defaults. +3. Define readiness deterministically: required successful synthesis and, when configured, a valid verdict are necessary for `ready`; partial/failed required phases become `not_ready`; insufficient or ambiguous evidence becomes `indeterminate`. +4. Change CLI success reporting and exit behavior so protocol completion alone cannot imply decision readiness. JSON must expose both dimensions. +5. Verify cache serialization/replay preserves the new fields and old cached shapes default conservatively. +6. Run focused tests and commit `fix(elite): separate completion from readiness`. + +## Task 5: Capture complete call receipts and accounting + +**Files:** + +- Modify: `src/conclave/models.py` +- Modify: `src/conclave/manifest.py` +- Modify: `src/conclave/providers.py` +- Modify: `src/conclave/council.py` +- Modify: `src/conclave/verdict_synthesis.py` +- Modify: `src/conclave/modes.py` +- Test: `tests/test_manifest_all_modes.py` +- Test: `tests/test_elite_mode.py` +- Test: `tests/test_output_contract_plumbing.py` + +1. Add failing tests asserting receipts cover member, audit, revision, synthesis, verdict extraction, and repair attempts, including failed calls. +2. Introduce a secret-free call receipt carrying phase, provider/model identity, attempt/outcome, latency, available usage/cost, error category, prompt version, and schema version. +3. Return or collect receipts through explicit values/callbacks; do not depend on global mutable state or logs. +4. Aggregate receipt latency/usage/cost into the manifest while preserving `None` when a provider does not report a trustworthy value. +5. Ensure prompt content, API keys, raw provider bodies, and exception chains never enter receipts. +6. Run focused tests plus the secret matrix and commit `feat(manifest): account for every elite model call`. + +## Task 6: Align product documentation with verified behavior + +**Files:** + +- Modify: `README.md` +- Modify: `docs/PRODUCT_DESIGN_DOCUMENT.md` +- Modify: `docs/DOCUMENTATION_INDEX.md` +- Modify: `docs/plans/2026-07-17-decision-quality-roadmap.md` +- Modify: other tracked docs returned by the claim scan + +1. Scan tracked prose for `fully auditable`, `evidence audit`, `consensus means truth`, and claims that answer IDs prove external facts. +2. Document the exact distinction among execution traceability, answer provenance, source grounding, protocol completion, and decision readiness. +3. Keep Horizon 1-4 claims explicitly prospective and preserve the roadmap's falsification/kill gates. +4. Run documentation/whitespace checks and commit `docs: align elite claims with verified guarantees`. + +## Task 7: Completion audit, Linear evidence, and remote synchronization + +1. Audit each DSE-690 Horizon 0 checkbox against code, tests, and documentation. Fix any gap before checking it off. +2. Run the complete verification gate: + + ```bash + PYTHONPATH=src PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 pytest -p no:cacheprovider -p pytest_asyncio.plugin + ruff check . + ruff format --check . + git diff --check origin/main...HEAD + git grep -nE 'sk_live|AKIA[0-9A-Z]{16}|BEGIN [A-Z ]*PRIVATE KEY|gh[pousr]_[A-Za-z0-9_]+' -- . ':!uv.lock' + ``` + +3. Update DSE-690 with checked acceptance items and a concise evidence comment containing commit SHA, test counts, PR #51, and any deliberately deferred Horizon 1-4 work. +4. Push `feat/elite-decision-protocol`, verify the remote SHA, and confirm PR #51 remains draft. +5. Stop. Do not merge or publish. From d860830a4fdf873d2144cc6f2a9bec7f45e71fb1 Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Fri, 17 Jul 2026 08:17:49 -0400 Subject: [PATCH 13/21] fix(elite): preserve answer identity and claim provenance --- src/conclave/council.py | 6 ++- src/conclave/models.py | 70 ++++++++++++++++++++++++-- src/conclave/modes.py | 32 ++++++++++-- src/conclave/prompts.py | 18 ++++--- src/conclave/providers.py | 4 +- src/conclave/verdict.py | 16 +++--- src/conclave/verdict_synthesis.py | 34 +++++++------ tests/test_elite_mode.py | 51 ++++++++++++++++++- tests/test_output_contract_plumbing.py | 33 +++++++++++- tests/test_providers.py | 21 ++++++++ 10 files changed, 244 insertions(+), 41 deletions(-) diff --git a/src/conclave/council.py b/src/conclave/council.py index fa26c9e..d7bb36a 100644 --- a/src/conclave/council.py +++ b/src/conclave/council.py @@ -714,7 +714,11 @@ async def _synthesize(self, result: CouncilResult) -> None: logger.warning(result.synthesis_error) return - blocks = "\n\n".join(f"### Answer from {a.name} ({a.model_id})\n{a.answer}" for a in usable) + blocks = "\n\n".join( + f"### Answer from {a.name} ({a.model_id})" + f"{f' (Answer ID: {a.answer_id})' if a.answer_id else ''}\n{a.answer}" + for a in usable + ) user_content = ( f"Original prompt:\n{result.prompt}\n\n" f"Council answers:\n\n{blocks}\n\n" diff --git a/src/conclave/models.py b/src/conclave/models.py index a0ef74b..50dd269 100644 --- a/src/conclave/models.py +++ b/src/conclave/models.py @@ -6,6 +6,10 @@ from __future__ import annotations +import hashlib +import json +from collections.abc import Iterable + from pydantic import BaseModel, Field from .verdict import ( @@ -18,6 +22,65 @@ ELITE_PROTOCOL_VERSION = "elite_v1" ELITE_MIN_RESPONDERS = 3 +_ANSWER_ID_VERSION = "answer_v1" + + +def stable_answer_id(name: str, model_id: str, answer: str) -> str: + """Return an opaque, deterministic identity for one successful answer. + + The digest is derived only from immutable, non-credential response facts. + Raw answer text and model metadata are never exposed in the identifier. + Latency and usage are intentionally excluded because they can change across + equivalent calls and cache reconstruction. + """ + identity = json.dumps( + { + "version": _ANSWER_ID_VERSION, + "name": name, + "model_id": model_id, + "answer": answer, + }, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + ).encode("utf-8") + return f"ca_{hashlib.sha256(identity).hexdigest()[:24]}" + + +def derive_phase_answer_id( + answer: ModelAnswer, + phase: str, + *, + parent_answer_ids: Iterable[str] = (), +) -> ModelAnswer: + """Assign an idempotent phase-specific identity to a successful artifact. + + Elite critiques and revisions transform earlier answers. Their IDs therefore + include the phase and the sorted IDs of the within-run parent artifacts while + remaining opaque and secret-free. Failed answers remain unidentified. + """ + if not answer.ok: + return answer + prefix = f"ca_{phase}_" + if answer.answer_id and answer.answer_id.startswith(prefix): + return answer + base_id = answer.answer_id or stable_answer_id( + answer.name, answer.model_id, answer.answer or "" + ) + identity = json.dumps( + { + "version": _ANSWER_ID_VERSION, + "phase": phase, + "base_answer_id": base_id, + "parent_answer_ids": sorted(parent_answer_ids), + }, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + answer.answer_id = f"{prefix}{hashlib.sha256(identity).hexdigest()[:24]}" + return answer + + # NOTE: ``ModelHarnessManifest`` is imported at the BOTTOM of this module (just # before the ``model_rebuild()`` calls), not here. ``manifest`` imports # :class:`TokenUsage` from this module, so importing it before ``TokenUsage`` is @@ -59,9 +122,10 @@ class ModelAnswer(BaseModel): latency_s: Wall-clock seconds for the call. usage: Token usage if reported by the provider. error: Error message if the call failed, else ``None``. - answer_id: Stable id assigned by conclave (not emitted by the model), used - to back ``evidence_answer_ids`` on the verdict's positions/conflicts - (CAC-01 result contract v2). ``None`` until conclave assigns it. + answer_id: Stable opaque id assigned by Conclave (not emitted by the + model). It provides within-run answer provenance and backs the legacy + compatibility field ``evidence_answer_ids``; it is not external + evidence that a claim is true. Failed calls keep ``None``. warnings: Non-fatal notes about this answer (e.g. structured-output repair applied). Empty by default. Distinct from ``error``, which marks the whole call as failed. diff --git a/src/conclave/modes.py b/src/conclave/modes.py index db9cc3e..2aa87c5 100644 --- a/src/conclave/modes.py +++ b/src/conclave/modes.py @@ -35,6 +35,7 @@ EliteResult, ModelAnswer, VoteResult, + derive_phase_answer_id, ) from .registry import key_present @@ -48,7 +49,7 @@ async def run_elite(council: Council, prompt: str) -> CouncilResult: """Run the three-stage Elite Decision Protocol without final synthesis. The protocol requires at least three successful responders after each of its - independent-answer, evidence-critique, and revision phases. Provider failures + independent-answer, claim-audit, and revision phases. Provider failures remain attached to their phase artifacts; a failed gate stops later calls and returns the latest fully qualified answers to the original prompt. """ @@ -60,6 +61,7 @@ def initial_messages(_name: str, _model_id: str) -> list[dict[str, str]]: return [{"role": "user", "content": prompt}] elite.initial_answers = await council.fan_out(members, initial_messages) + _elite_assign_phase_ids(elite.initial_answers, "initial") initial_successes = _elite_gate(elite, "initial", elite.initial_answers) result.answers = initial_successes if elite.failure_reason is not None: @@ -68,6 +70,11 @@ def initial_messages(_name: str, _model_id: str) -> list[dict[str, str]]: critique_members = _elite_surviving_members(members, initial_successes) critique_messages = _elite_critique_messages_for(prompt, initial_successes) elite.critiques = await council.fan_out(critique_members, critique_messages) + _elite_assign_phase_ids( + elite.critiques, + "critique", + parent_answer_ids=(answer.answer_id for answer in initial_successes), + ) critique_successes = _elite_gate(elite, "critique", elite.critiques) if elite.failure_reason is not None: return result @@ -81,6 +88,13 @@ def initial_messages(_name: str, _model_id: str) -> list[dict[str, str]]: critique_successes, ) elite.revisions = await council.fan_out(revision_members, revision_messages) + _elite_assign_phase_ids( + elite.revisions, + "revision", + parent_answer_ids=( + answer.answer_id for answer in [*initial_successes, *critique_successes] + ), + ) revision_successes = _elite_gate(elite, "revision", elite.revisions) if elite.failure_reason is not None: return result @@ -90,6 +104,18 @@ def initial_messages(_name: str, _model_id: str) -> list[dict[str, str]]: return result +def _elite_assign_phase_ids( + artifacts: list[ModelAnswer], + phase: str, + *, + parent_answer_ids=(), +) -> None: + """Attach opaque phase identities without exposing model-name fallbacks.""" + parents = [answer_id for answer_id in parent_answer_ids if answer_id] + for artifact in artifacts: + derive_phase_answer_id(artifact, phase, parent_answer_ids=parents) + + def _elite_gate( elite: EliteResult, phase: str, @@ -116,7 +142,7 @@ def _elite_surviving_members( def _elite_critique_messages_for(prompt: str, answers: list[ModelAnswer]): - """Build the shared evidence-audit messages for each surviving member.""" + """Build the shared claim-audit messages for each surviving member.""" messages = [ {"role": "system", "content": prompts.ELITE_CRITIC_SYSTEM}, {"role": "user", "content": prompts.elite_critic_user(prompt, answers)}, @@ -130,7 +156,7 @@ def _elite_revision_messages_for( initial_answers: list[ModelAnswer], critiques: list[ModelAnswer], ): - """Build member-specific evidence-aware revision messages.""" + """Build member-specific claim-audit-aware revision messages.""" def messages_for(name: str, _model_id: str) -> list[dict[str, str]]: return [ diff --git a/src/conclave/prompts.py b/src/conclave/prompts.py index ff06994..dc84ea4 100644 --- a/src/conclave/prompts.py +++ b/src/conclave/prompts.py @@ -62,18 +62,20 @@ ) ELITE_CRITIC_SYSTEM = ( - "You are an evidence auditor in an elite decision council. Independently " + "You are a claim auditor in an elite decision council. Independently " "stress-test the anonymized answers against the original prompt. Organize " "your audit into three explicit categories: SUPPORTED claims, CONFLICTING " "claims, and EXTERNALLY UNVERIFIED claims. Distinguish evidence supplied in " "the answers from assumptions that would require outside verification. Do " - "not invent citations, sources, quotations, measurements, or facts. Cite " - "the displayed answer IDs when identifying a claim. Preserve meaningful " + "not invent citations, sources, quotations, measurements, or facts. The " + "displayed answer IDs provide within-run answer provenance only; they do " + "not prove any claim against external sources. Cite those IDs when identifying " + "a claim. Preserve meaningful " "minority positions and be precise about uncertainty." ) ELITE_REVISION_SYSTEM = ( - "You are revising your independent answer after an evidence audit by an " + "You are revising your independent answer after a claim audit by an " "elite decision council. Use the anonymized initial panel and critiques to " "correct unsupported claims, resolve conflicts where the supplied material " "permits, and clearly mark externally unverified claims. Do not invent " @@ -179,12 +181,12 @@ def _elite_artifact_block(answers: Sequence[ModelAnswer], phase: str) -> str: def elite_critic_user(prompt: str, answers: Sequence[ModelAnswer]) -> str: - """Build an anonymized, evidence-audit prompt for an elite critic.""" + """Build an anonymized claim-audit prompt with within-run answer IDs.""" panel = _elite_artifact_block(answers, "initial answer") return ( f"Original prompt:\n{prompt}\n\n" f"Anonymized initial panel:\n\n{panel}\n\n" - "Audit the panel using the required evidence categories." + "Audit the panel using the required claim categories." ) @@ -203,6 +205,6 @@ def elite_revision_user( f"Your original answer (Answer ID: {original_id}):\n" f"{original_answer.answer or ''}\n\n" f"Anonymized initial panel:\n\n{initial_panel}\n\n" - f"Anonymized evidence critiques:\n\n{critique_panel}\n\n" - "Now provide your evidence-aware revised answer." + f"Anonymized claim audits:\n\n{critique_panel}\n\n" + "Now provide your claim-audit-aware revised answer." ) diff --git a/src/conclave/providers.py b/src/conclave/providers.py index 175c05e..f1b3005 100644 --- a/src/conclave/providers.py +++ b/src/conclave/providers.py @@ -24,7 +24,7 @@ from .config import ConclaveConfig, load_config from .logging import get_logger from .manifest import ProviderExecutionReceipt -from .models import ModelAnswer, TokenUsage +from .models import ModelAnswer, TokenUsage, stable_answer_id from .registry import provider_prefix from .transport import TransportError @@ -165,6 +165,7 @@ async def call_model( answer=text, latency_s=latency, usage=usage, + answer_id=stable_answer_id(name, model_id, text), ) except (ProviderError, TransportError) as exc: latency = time.perf_counter() - started @@ -329,6 +330,7 @@ async def call_model_stream( answer=text, latency_s=latency, usage=usage, + answer_id=stable_answer_id(name, model_id, text), ) except (ProviderError, TransportError) as exc: # Mid-stream failure: preserve any partial text already collected, set diff --git a/src/conclave/verdict.py b/src/conclave/verdict.py index 4b14200..8c62a1f 100644 --- a/src/conclave/verdict.py +++ b/src/conclave/verdict.py @@ -57,7 +57,7 @@ # audit can tell WHICH extractor wording produced a given clustering. Opaque # string; only equality/inequality is meaningful. Bump on any change to the # extraction system prompt in :mod:`conclave.verdict_synthesis`. -VERDICT_EXTRACTION_PROMPT_VERSION = "1" +VERDICT_EXTRACTION_PROMPT_VERSION = "2" class CouncilPosition(BaseModel): @@ -66,8 +66,10 @@ class CouncilPosition(BaseModel): The verdict-extraction step (CAC-05) clusters semantically-equivalent member positions into these. This is the element shape reused by both the verdict's ``positions`` and the human-readable side of a conflict. Every cluster carries - its ``providers`` and ``evidence_answer_ids`` so a human can verify each - assignment against the member's raw answer (DD-1 invariant 2). + its ``providers`` and ``evidence_answer_ids`` so a human can trace each + assignment to the member's raw answer (DD-1 invariant 2). Despite the legacy + compatibility name, these IDs are within-run answer provenance, not proof + against external sources. See DD-2 verdict schema. @@ -76,8 +78,9 @@ class CouncilPosition(BaseModel): summary: One-line human-readable summary of the clustered stance. providers: Provider names whose answers fall in this cluster (e.g. ``["anthropic", "openai"]``). Names only — never key material. - evidence_answer_ids: Stable ``answer_id`` values backing this cluster - (e.g. ``["anthropic-1", "openai-1"]``), for human verification. + evidence_answer_ids: Stable ``answer_id`` values tracing this cluster to + council answers, for within-run verification. The field name is kept + for serialized compatibility; the IDs are not external evidence. """ label: str @@ -134,7 +137,8 @@ class MinorityReport(BaseModel): Attributes: providers: Provider names holding the minority view. Names only. claim: The minority claim itself. - evidence_answer_ids: Stable ``answer_id`` values backing the claim. + evidence_answer_ids: Stable ``answer_id`` values tracing the claim to + council answers. Compatibility name only; not external evidence. why_it_matters: Optional rationale for surfacing the dissent despite it being a minority view. """ diff --git a/src/conclave/verdict_synthesis.py b/src/conclave/verdict_synthesis.py index e0fd617..52a6442 100644 --- a/src/conclave/verdict_synthesis.py +++ b/src/conclave/verdict_synthesis.py @@ -102,7 +102,9 @@ _EXTRACTION_SYSTEM = ( "You are the verdict extractor for an auditable multi-model council. You are " "given the original prompt and each council member's answer, labeled with a " - "stable evidence id. Produce ONE JSON object that conforms exactly to the " + "stable answer id. These IDs provide within-run answer provenance only; they " + "do not prove claims against external sources. Produce ONE JSON object that " + "conforms exactly to the " "provided JSON Schema and nothing else.\n\n" "Your job is to ADJUDICATE, not to re-answer:\n" "- Set verdict_applies=false when the prompt is open-ended generation (a poem, " @@ -111,8 +113,9 @@ "- Set verdict_type to 'decision' (a question with an answer), 'review' (an " "accept/revise/reject judgment), or 'synthesis' (open-ended consolidation).\n" "- Cluster the members into positions[]; each position lists the providers in " - "it and the evidence_answer_ids (the labels shown) backing it, so a human can " - "verify every assignment against the raw answer.\n" + "it and the evidence_answer_ids (the compatibility field containing the answer " + "IDs shown) tracing it, so a human can verify every assignment against the raw " + "answer.\n" "- Record one provider_vote per member that took a stance (position_label must " "match a positions[].label); omit a member that took no clean stance.\n" "- Add conflicts[] only when there are two or more positions in tension.\n\n" @@ -170,31 +173,30 @@ def _responding(member_answers: list[ModelAnswer]) -> list[ModelAnswer]: return [a for a in member_answers if a.ok and a.answer and a.answer.strip()] -def _evidence_label(answer: ModelAnswer, index: int) -> str: - """Return a stable evidence label for one responding member. +def _answer_label(answer: ModelAnswer, index: int) -> str: + """Return a stable within-run provenance label for one responding member. Prefers the conclave-assigned ``answer_id`` (which backs - ``evidence_answer_ids`` on the verdict's positions). Falls back to the member - ``name`` and then to a positional ``member-{index}`` so the model always has a - stable, citable handle even when ``answer_id`` is ``None`` (DD-2 positions must - cite evidence regardless). + ``evidence_answer_ids`` compatibility field on verdict positions). Successful + answers normally always carry this ID. The positional fallback deliberately + avoids exposing a provider/model name as a substitute identity. Args: answer: One responding member answer. index: Its 0-based position in the responding list (fallback id source). Returns: - A non-empty label string the model can cite as an evidence id. + A non-empty label string the model can cite as an answer id. """ - return answer.answer_id or answer.name or f"member-{index}" + return answer.answer_id or f"unidentified-answer-{index + 1}" def _build_messages(prompt: str, responders: list[ModelAnswer]) -> list[dict[str, str]]: """Build the extraction messages from the prompt + every responding answer. Each responding member's answer is included verbatim, labeled with its stable - evidence id (:func:`_evidence_label`) so the model can populate - ``evidence_answer_ids``. The LCD extraction schema is embedded in the user + answer id (:func:`_answer_label`) so the model can populate the compatibility + field ``evidence_answer_ids``. The LCD extraction schema is embedded in the user message (prompt-level structured output — see the module docstring). Messages are built from answer TEXT + evidence labels only; the synthesizer call routes through :func:`conclave.providers.call_model`, which redacts, so no key @@ -209,8 +211,8 @@ def _build_messages(prompt: str, responders: list[ModelAnswer]) -> list[dict[str """ blocks = [] for i, ans in enumerate(responders): - label = _evidence_label(ans, i) - blocks.append(f"### Member answer (evidence id: {label}) — from {ans.name}\n{ans.answer}") + label = _answer_label(ans, i) + blocks.append(f"### Member answer (answer id: {label}) — from {ans.name}\n{ans.answer}") answers_block = "\n\n".join(blocks) schema_json = json.dumps(verdict_extraction_json_schema(), indent=2) @@ -409,7 +411,7 @@ async def extract_verdict( ``"fewer than 2 responding members"`` and NO LLM call (consensus is undefined for N<2, DD-1 edge case). 2. **Extract.** Build the ``[system, user]`` messages from the prompt + every - responding answer (labeled by evidence id) with the LCD extraction schema + responding answer (labeled by within-run answer id) with the LCD extraction schema embedded, and make ONE :func:`conclave.providers.call_model` call. 3. **Validate → repair-once → fallback.** Parse JSON + validate via Pydantic. On failure, re-call ONCE with the stringified errors appended; if it still diff --git a/tests/test_elite_mode.py b/tests/test_elite_mode.py index d128db7..5e5141b 100644 --- a/tests/test_elite_mode.py +++ b/tests/test_elite_mode.py @@ -151,7 +151,7 @@ def _elite_prompt_answers() -> list[ModelAnswer]: ] -def test_elite_critic_prompt_is_anonymized_evidence_audit() -> None: +def test_elite_critic_prompt_is_anonymized_claim_audit() -> None: answers = _elite_prompt_answers() built = elite_critic_user("Choose the strongest option.", answers) @@ -169,6 +169,9 @@ def test_elite_critic_prompt_is_anonymized_evidence_audit() -> None: assert "CONFLICTING" in ELITE_CRITIC_SYSTEM assert "EXTERNALLY UNVERIFIED" in ELITE_CRITIC_SYSTEM assert "Do not invent citations" in ELITE_CRITIC_SYSTEM + assert "claim auditor" in ELITE_CRITIC_SYSTEM + assert "within-run answer provenance" in ELITE_CRITIC_SYSTEM + assert "evidence auditor" not in ELITE_CRITIC_SYSTEM def test_elite_revision_prompt_includes_original_panel_and_critiques() -> None: @@ -177,7 +180,7 @@ def test_elite_revision_prompt_includes_original_panel_and_critiques() -> None: ModelAnswer( name=f"critic-{index}", model_id=f"critic-vendor/model-{index}", - answer=f"Evidence audit {index}", + answer=f"Claim audit {index}", answer_id=f"critique-{index:03d}", ) for index in range(1, 4) @@ -247,6 +250,49 @@ def handler(model_id, messages): assert all(answer.answer.startswith("revision") for answer in result.answers) +async def test_run_elite_assigns_phase_specific_ids_and_preserves_them_round_trip( + monkeypatch, patch_call_model +) -> None: + _all_keys(monkeypatch) + + def handler(model_id, messages): + return make_response(f"{_phase(messages)} from {model_id}") + + patch_call_model(handler) + result = await run_elite( + Council( + models=["grok", "gemini", "perplexity"], + config=_elite_config(), + extract_verdict=False, + ), + "Decide.", + ) + + assert result.elite is not None + phase_artifacts = { + "initial": result.elite.initial_answers, + "critique": result.elite.critiques, + "revision": result.elite.revisions, + } + for phase, artifacts in phase_artifacts.items(): + assert all((answer.answer_id or "").startswith(f"ca_{phase}_") for answer in artifacts) + assert [answer.answer_id for answer in result.answers] == [ + answer.answer_id for answer in result.elite.revisions if answer.ok + ] + + reconstructed = CouncilResult.model_validate_json(result.model_dump_json()) + assert reconstructed.elite is not None + assert [answer.answer_id for answer in reconstructed.elite.initial_answers] == [ + answer.answer_id for answer in result.elite.initial_answers + ] + assert [answer.answer_id for answer in reconstructed.elite.critiques] == [ + answer.answer_id for answer in result.elite.critiques + ] + assert [answer.answer_id for answer in reconstructed.elite.revisions] == [ + answer.answer_id for answer in result.elite.revisions + ] + + @pytest.mark.parametrize("failed_phase", ["initial", "critique", "revision"]) async def test_run_elite_four_members_survives_one_failure_in_any_phase( monkeypatch, patch_call_model, failed_phase @@ -398,6 +444,7 @@ def handler(model_id, messages): assert len(synthesis_inputs) == 1 assert "revision from xai/grok-4.3" in synthesis_inputs[0] assert "initial from xai/grok-4.3" not in synthesis_inputs[0] + assert all(f"Answer ID: {answer.answer_id}" in synthesis_inputs[0] for answer in result.answers) assert result.verdict is not None assert result.consensus_score == result.verdict.consensus_score == 1.0 assert result.provider_votes == result.verdict.provider_votes diff --git a/tests/test_output_contract_plumbing.py b/tests/test_output_contract_plumbing.py index ee013a3..fc7a737 100644 --- a/tests/test_output_contract_plumbing.py +++ b/tests/test_output_contract_plumbing.py @@ -227,13 +227,31 @@ async def test_call_model_stream_without_contract_leaves_body_free_prose( assert "response_format" not in captured["body"] +async def test_call_model_stream_success_assigns_stable_answer_identity( + monkeypatch, mock_stream_client +) -> None: + """The streaming provider path shares the successful-answer ID contract.""" + monkeypatch.setenv("OPENAI_API_KEY", "sk-test") + monkeypatch.setenv("CONCLAVE_CONFIG", "/nonexistent/conclave.yml") + captured = {} + mock_stream_client(_openai_stream_handler(captured)) + + first = await _drain_stream("openai", "openai/gpt-4.1") + second = await _drain_stream("openai", "openai/gpt-4.1") + + assert first is not None and second is not None + assert first.ok and second.ok + assert first.answer_id == second.answer_id + assert (first.answer_id or "").startswith("ca_") + + # --------------------------------------------------------------------------- # # extract_verdict now requests native structured output (+ keeps the fallback) # --------------------------------------------------------------------------- # def _answer(name: str, text: str) -> ModelAnswer: - """A responding member answer with a stable evidence id.""" + """A responding member answer with a stable within-run answer id.""" return ModelAnswer(name=name, model_id=f"{name}/m", answer=text, answer_id=f"{name}-1") @@ -272,6 +290,19 @@ async def fake_call_model(name, model_id, messages, *, config=None, output_contr assert result.verdict is None +def test_verdict_prompt_describes_ids_as_answer_provenance_not_external_evidence() -> None: + """Compatibility field IDs trace council answers; they do not prove facts.""" + import conclave.verdict_synthesis as vs + + answers = [_answer("a", "yes"), _answer("b", "no")] + messages = vs._build_messages("decide?", answers) + rendered = "\n".join(message["content"] for message in messages) + + assert "within-run answer provenance" in rendered + assert "answer id:" in rendered + assert "evidence id:" not in rendered + + async def test_extract_verdict_still_degrades_gracefully_on_bad_json(monkeypatch, conclave_caplog): """Native contract is additive: bad JSON still yields verdict=None gracefully. diff --git a/tests/test_providers.py b/tests/test_providers.py index c35dd60..a85fa67 100644 --- a/tests/test_providers.py +++ b/tests/test_providers.py @@ -13,6 +13,7 @@ from __future__ import annotations +import re from pathlib import Path import pytest @@ -247,6 +248,26 @@ async def fake_post_json(url, headers, json_body, timeout): assert captured["headers"]["Authorization"] == "Bearer sk-test" +async def test_call_model_success_assigns_stable_conclave_answer_identity(monkeypatch): + """Successful responses receive deterministic, opaque Conclave-owned IDs.""" + monkeypatch.setenv("OPENAI_API_KEY", "sk-test") + monkeypatch.setenv("CONCLAVE_CONFIG", "/nonexistent/conclave.yml") + + async def fake_post_json(url, headers, json_body, timeout): + return 200, {"choices": [{"message": {"content": "sensitive answer text"}}]} + + monkeypatch.setattr("conclave.transport.post_json", fake_post_json) + messages = [{"role": "user", "content": "hi"}] + + first = await call_model("openai", "openai/gpt-4.1", messages) + second = await call_model("openai", "openai/gpt-4.1", messages) + + assert first.ok and second.ok + assert first.answer_id == second.answer_id + assert re.fullmatch(r"ca_[0-9a-f]{24}", first.answer_id or "") + assert "sensitive" not in (first.answer_id or "") + + async def test_call_model_transport_error_becomes_model_answer_error(monkeypatch): """A raised transport error is captured as a non-raising ModelAnswer.error.""" monkeypatch.setenv("OPENAI_API_KEY", "sk-test") From 635033698b6825adae1cdf7daf8e6d0700fe2b7f Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Fri, 17 Jul 2026 08:21:39 -0400 Subject: [PATCH 14/21] fix(config): thread resolved config through council calls --- src/conclave/council.py | 2 ++ tests/test_council.py | 58 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/src/conclave/council.py b/src/conclave/council.py index d7bb36a..e47b73b 100644 --- a/src/conclave/council.py +++ b/src/conclave/council.py @@ -347,6 +347,7 @@ async def fan_out( name, model_id, messages_for(name, model_id), + config=self.config, temperature=self.temperature, timeout=self.timeout, ) @@ -841,6 +842,7 @@ async def synthesize_blocks(self, system_prompt: str, user_content: str) -> Mode self.synthesizer, synth_id, messages, + config=self.config, temperature=self.temperature, timeout=self.timeout, ) diff --git a/tests/test_council.py b/tests/test_council.py index 2152860..ba55114 100644 --- a/tests/test_council.py +++ b/tests/test_council.py @@ -68,6 +68,60 @@ def handler(model, messages, **kwargs): assert result.synthesizer == "claude" +async def test_resolved_config_identity_reaches_every_model_call(monkeypatch): + """One resolved config instance reaches members, synthesis, and verdict repair.""" + _all_keys(monkeypatch) + + import conclave.council as council_mod + import conclave.verdict_synthesis as verdict_synthesis_mod + from conclave.models import ModelAnswer + + resolved_config = _config() + received: list[tuple[str, ConclaveConfig | None]] = [] + + async def config_spy( + name, + model_id, + messages, + *, + temperature=0.7, + timeout=120.0, + config=None, + **kwargs, + ): + system = messages[0]["content"] if messages else "" + if system.startswith("You are the verdict extractor"): + phase = "verdict_repair" if len(messages) == 3 else "verdict_extraction" + answer = "not valid json" + elif system == council_mod._SYNTH_SYSTEM: + phase = "synthesis" + answer = "merged" + else: + phase = "member" + answer = "member answer" + received.append((phase, config)) + return ModelAnswer(name=name, model_id=model_id, answer=answer) + + monkeypatch.setattr(council_mod, "call_model", config_spy) + monkeypatch.setattr(verdict_synthesis_mod, "call_model", config_spy) + + council = Council( + models=["grok", "gemini"], + synthesizer="claude", + config=resolved_config, + ) + await council.ask("Should we proceed?") + + assert [phase for phase, _config_value in received].count("member") == 2 + assert {phase for phase, _config_value in received} == { + "member", + "synthesis", + "verdict_extraction", + "verdict_repair", + } + assert all(config_value is resolved_config for _phase, config_value in received) + + async def test_concurrency_is_real(monkeypatch): """Members run concurrently: total time ~= slowest call, not the sum.""" _all_keys(monkeypatch) @@ -76,7 +130,9 @@ async def test_concurrency_is_real(monkeypatch): from conclave.models import ModelAnswer # Replace call_model with a coroutine that sleeps, to prove gather concurrency. - async def sleepy_call_model(name, model_id, messages, *, temperature=0.7, timeout=120.0): + async def sleepy_call_model( + name, model_id, messages, *, temperature=0.7, timeout=120.0, config=None + ): await asyncio.sleep(0.2) return ModelAnswer(name=name, model_id=model_id, answer=f"ok {model_id}") From 253f4a890d50394eabaa39fabb626fc2963842a2 Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Fri, 17 Jul 2026 08:33:13 -0400 Subject: [PATCH 15/21] fix(cache): version elite protocol identity --- src/conclave/cache.py | 245 ++++++++++++++++++++++++++++++++++------ src/conclave/council.py | 30 ++++- src/conclave/prompts.py | 5 + tests/test_cache.py | 214 ++++++++++++++++++++++++++++++++++- 4 files changed, 453 insertions(+), 41 deletions(-) diff --git a/src/conclave/cache.py b/src/conclave/cache.py index 873ca8f..f3563ad 100644 --- a/src/conclave/cache.py +++ b/src/conclave/cache.py @@ -11,9 +11,10 @@ Storage ======= Entries live one-per-file under ``$XDG_CACHE_HOME/conclave`` (falling back to -``~/.cache/conclave``). Each file is named ``.json`` and holds the -JSON serialization of a :class:`conclave.models.CouncilResult` (via -``model_dump(mode="json")``), which by construction carries no secrets. +``~/.cache/conclave``). Each file is named ``.json`` and holds a +versioned envelope around the JSON serialization of a +:class:`conclave.models.CouncilResult`. Unversioned/old-format envelopes are +misses, never migrated or replayed against current protocol semantics. Graceful degradation ==================== @@ -38,20 +39,54 @@ import json import os import re +from collections.abc import Mapping from pathlib import Path +from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit from pydantic import ValidationError from .logging import get_logger -from .models import CouncilResult +from .models import ELITE_PROTOCOL_VERSION, CouncilResult +from .prompts import ELITE_PROMPT_VERSION, SYNTHESIS_PROMPT_VERSION +from .verdict import ( + VERDICT_EXTRACTION_PROMPT_VERSION, + VERDICT_SCHEMA_VERSION, +) logger = get_logger("cache") # Bumped if the cache-key composition or stored schema changes incompatibly, so # old entries simply miss instead of being mis-served against new code. -_CACHE_VERSION = 1 +CACHE_FORMAT_VERSION = "2" _WHITESPACE = re.compile(r"\s+") +_SECRET_QUERY_PARTS = ( + "authorization", + "auth", + "credential", + "passwd", + "password", + "secret", + "signature", + "token", +) +_SECRET_QUERY_KEYS = { + "access_key", + "api_key", + "apikey", + "code", + "key", + "sig", +} +_SECRET_VALUE_MARKERS = ( + "akia", + "authorization:", + "bearer ", + "ghp_", + "sk-", + "sk_", + "x-api-key", +) def cache_dir() -> Path: @@ -74,6 +109,122 @@ def _normalize_prompt(prompt: str) -> str: return _WHITESPACE.sub(" ", prompt).strip() +def _digest(value: str) -> str: + """Return a one-way identity fingerprint without retaining ``value``.""" + return hashlib.sha256(value.encode("utf-8")).hexdigest() + + +def _endpoint_fingerprint(raw_url: str) -> str: + """Fingerprint output-affecting endpoint routing without credential material. + + Userinfo, fragments, and secret-like query parameters are deliberately + excluded. Safe query parameters (for example an API version) remain part of + the fingerprint because they can change model behavior. Only the digest is + returned; the normalized URL never enters an identity document or diagnostic. + """ + parsed = urlsplit(raw_url) + hostname = (parsed.hostname or "").lower() + try: + port = parsed.port + except ValueError: + # A malformed configured port will fail at request time, but cache + # identity construction must remain safe and deterministic first. + port = None + hostname = f"{hostname}:invalid-port" + if port is not None: + hostname = f"{hostname}:{port}" + safe_query = [] + for key, value in parse_qsl(parsed.query, keep_blank_values=True): + key_lower = key.lower() + value_lower = value.lower() + if key_lower in _SECRET_QUERY_KEYS or any( + part in key_lower for part in _SECRET_QUERY_PARTS + ): + continue + if any(marker in value_lower for marker in _SECRET_VALUE_MARKERS): + continue + safe_query.append((key, value)) + safe_query.sort() + normalized = urlunsplit( + ( + parsed.scheme.lower(), + hostname, + parsed.path.rstrip("/") or "/", + urlencode(safe_query), + "", + ) + ) + return _digest(normalized) + + +def build_identity( + *, + prompt: str, + mode: str, + members: list[tuple[str, str]], + synthesizer: str | None, + synthesizer_model_id: str | None, + temperature: float, + timeout: float = 120.0, + rounds: int | None = None, + proposer: str | None = None, + converge_threshold: float | None = None, + choices: list[str] | None = None, + extract_verdict: bool = True, + endpoint_urls: Mapping[str, str] | None = None, + source_bundle_digest: str | None = None, + cache_format_version: str = CACHE_FORMAT_VERSION, + protocol_version: str = ELITE_PROTOCOL_VERSION, + synthesis_prompt_version: str = SYNTHESIS_PROMPT_VERSION, + elite_prompt_version: str = ELITE_PROMPT_VERSION, + verdict_schema_version: str = VERDICT_SCHEMA_VERSION, + verdict_prompt_version: str = VERDICT_EXTRACTION_PROMPT_VERSION, +) -> dict[str, object]: + """Build the canonical, secret-free identity document for a council run. + + Raw endpoint URLs and source bundle values never enter the returned document; + only sanitized one-way fingerprints do. API keys are not accepted and no + environment value is read here. + """ + payload: dict[str, object] = { + "versions": { + "cache_format": cache_format_version, + "elite_protocol": protocol_version, + "synthesis_prompt": synthesis_prompt_version, + "elite_prompt": elite_prompt_version, + "verdict_schema": verdict_schema_version, + "verdict_extraction_prompt": verdict_prompt_version, + }, + "prompt_fingerprint": _digest(_normalize_prompt(prompt)), + "mode": mode, + # Pairs as lists so JSON round-trips; order preserved deliberately. + "members": [[name, model_id] for name, model_id in members], + "synthesizer": [synthesizer, synthesizer_model_id], + "generation": {"temperature": temperature, "timeout": timeout}, + "extract_verdict": extract_verdict, + "endpoint_fingerprints": { + prefix: _endpoint_fingerprint(url) + for prefix, url in sorted((endpoint_urls or {}).items()) + }, + # Re-hash the caller-supplied digest. This keeps even a malformed caller + # value out of the inspectable identity while preserving invalidation. + "source_bundle_fingerprint": ( + _digest(source_bundle_digest) if source_bundle_digest is not None else None + ), + "mode_params": {}, + } + mode_params = payload["mode_params"] + assert isinstance(mode_params, dict) + if mode == "debate": + mode_params["rounds"] = rounds + mode_params["converge_threshold"] = converge_threshold + if mode == "adversarial": + mode_params["proposer"] = proposer + if mode == "vote": + mode_params["choices"] = choices or [] + return payload + + def make_key( *, prompt: str, @@ -82,23 +233,34 @@ def make_key( synthesizer: str | None, synthesizer_model_id: str | None, temperature: float, + timeout: float = 120.0, rounds: int | None = None, proposer: str | None = None, converge_threshold: float | None = None, choices: list[str] | None = None, + extract_verdict: bool = True, + endpoint_urls: Mapping[str, str] | None = None, + source_bundle_digest: str | None = None, + cache_format_version: str = CACHE_FORMAT_VERSION, + protocol_version: str = ELITE_PROTOCOL_VERSION, + synthesis_prompt_version: str = SYNTHESIS_PROMPT_VERSION, + elite_prompt_version: str = ELITE_PROMPT_VERSION, + verdict_schema_version: str = VERDICT_SCHEMA_VERSION, + verdict_prompt_version: str = VERDICT_EXTRACTION_PROMPT_VERSION, ) -> str: - """Build the stable cache key (sha256 hex) for a council run. + """Build the stable SHA-256 cache key from canonical secret-free identity. - The key is a SHA-256 over a canonical JSON document of only output-affecting, - secret-free identity: + Identity covers: * normalized prompt, * run mode, * ordered ``(friendly_name, resolved_model_id)`` member pairs (order matters -- see module docstring), * synthesizer/judge friendly name + resolved model id, - * temperature, and mode params (``rounds`` + ``converge_threshold`` for - debate, ``proposer`` for adversarial) when they apply. + * generation settings and mode parameters, + * protocol/prompt/schema/cache-format versions, + * verdict extraction behavior, custom endpoint routing, and an optional + future source-bundle digest. Args: prompt: The raw user prompt (normalized internally). @@ -117,26 +279,28 @@ def make_key( Returns: A 64-char lowercase hex SHA-256 digest. Contains zero key material. """ - payload: dict[str, object] = { - "v": _CACHE_VERSION, - "prompt": _normalize_prompt(prompt), - "mode": mode, - # Pairs as lists so JSON round-trips; order preserved deliberately. - "members": [[name, model_id] for name, model_id in members], - "synthesizer": synthesizer, - "synthesizer_model_id": synthesizer_model_id, - "temperature": temperature, - } - if mode == "debate": - payload["rounds"] = rounds - # Early-stop changes how many rounds actually run and thus the output; - # it must distinguish a converged run from a fixed-rounds run. - payload["converge_threshold"] = converge_threshold - if mode == "adversarial": - payload["proposer"] = proposer - if mode == "vote": - payload["choices"] = choices or [] - + payload = build_identity( + prompt=prompt, + mode=mode, + members=members, + synthesizer=synthesizer, + synthesizer_model_id=synthesizer_model_id, + temperature=temperature, + timeout=timeout, + rounds=rounds, + proposer=proposer, + converge_threshold=converge_threshold, + choices=choices, + extract_verdict=extract_verdict, + endpoint_urls=endpoint_urls, + source_bundle_digest=source_bundle_digest, + cache_format_version=cache_format_version, + protocol_version=protocol_version, + synthesis_prompt_version=synthesis_prompt_version, + elite_prompt_version=elite_prompt_version, + verdict_schema_version=verdict_schema_version, + verdict_prompt_version=verdict_prompt_version, + ) canonical = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False) return hashlib.sha256(canonical.encode("utf-8")).hexdigest() @@ -170,7 +334,9 @@ def load(key: str) -> CouncilResult | None: try: data = json.loads(raw) - result = CouncilResult.model_validate(data) + if not isinstance(data, dict) or data.get("cache_format_version") != CACHE_FORMAT_VERSION: + return None + result = CouncilResult.model_validate(data.get("result")) except (json.JSONDecodeError, ValidationError, TypeError) as exc: logger.warning("corrupt cache entry %s: %s; treating as miss", path, exc) return None @@ -205,12 +371,17 @@ def store(key: str, result: CouncilResult) -> None: # conclave.providers, BEFORE it is placed on the result and therefore long # before it reaches this write. Member/synthesis answer TEXT is provider # content, never key material. The cache KEY (make_key) is composed solely - # of prompt + mode + member/synthesizer NAMES + model ids + params -- no - # env var name or value is read here. Net: no raw key (name or value) can - # reach a cache file or filename. Do not move any un-redacted capture into - # the result after this contract -- it would persist a secret to disk. - payload = result.model_dump(mode="json") - payload["cached"] = False + # of canonical secret-free identity and then SHA-256 hashed. Custom + # endpoint URLs are sanitized and fingerprinted; no env var or key value + # is read here. Net: no raw key or credential-bearing URL can reach a + # cache file or filename. Do not move any un-redacted capture into the + # result after this contract -- it would persist a secret to disk. + result_payload = result.model_dump(mode="json") + result_payload["cached"] = False + payload = { + "cache_format_version": CACHE_FORMAT_VERSION, + "result": result_payload, + } # Atomic-ish write: write to a temp sibling then replace, so a crash mid # write never leaves a half-written (corrupt) entry behind. tmp = path.with_suffix(".json.tmp") diff --git a/src/conclave/council.py b/src/conclave/council.py index e47b73b..7729fc0 100644 --- a/src/conclave/council.py +++ b/src/conclave/council.py @@ -61,8 +61,8 @@ from .config import ConclaveConfig, load_config from .logging import get_logger from .manifest import ModelHarnessManifest, ProviderSkip, verified_secret_safety -from .models import CouncilResult, ModelAnswer, StreamEvent, TokenUsage -from .prompts import SYNTHESIS_PROMPT_VERSION +from .models import ELITE_PROTOCOL_VERSION, CouncilResult, ModelAnswer, StreamEvent, TokenUsage +from .prompts import ELITE_PROMPT_VERSION, SYNTHESIS_PROMPT_VERSION from .providers import call_model, receipt_from_answer from .registry import key_present @@ -135,6 +135,10 @@ class Council: keys; you remain responsible for that band then. Consumers using the provider functions directly (without a ``Council``) can still call :func:`conclave.guard_transport_logging` themselves. + source_bundle_digest: Optional digest of a future source-grounding bundle. + When supplied, it participates in cache identity so grounded and + ungrounded Elite runs cannot collide. The value is re-hashed before + entering the canonical identity document. Example: >>> council = Council(models=["grok", "perplexity"], synthesizer="claude") @@ -152,6 +156,7 @@ def __init__( cache: bool | None = None, extract_verdict: bool = True, allow_transport_debug_logging: bool = False, + source_bundle_digest: str | None = None, ) -> None: self.config = config or load_config() self.requested_models = list(models) @@ -165,6 +170,11 @@ def __init__( # ``extract_verdict`` engine function. There is no per-call override -- # this constructor flag is the single resolution path (one opt-out). self.extract_verdict_enabled = extract_verdict + # Horizon-2 placeholder: callers with a grounded source bundle can bind + # its digest into cache identity now, before source retrieval ships. + # cache.build_identity re-hashes this value so malformed/raw input never + # appears in inspectable identity documents or diagnostics. + self.source_bundle_digest = source_bundle_digest # Default-on transport-logging guard (key-leak audit, RANK 6): drop # httpx/httpcore DEBUG records (the only band that emits the auth header) # so a process holding a real key cannot leak it via verbose transport @@ -214,6 +224,11 @@ def _cache_key( """ members, _skipped = self._available_members() synth_id = self.config.resolve_model_id(self.synthesizer) + used_prefixes = { + model_id.split("/", 1)[0] + for _name, model_id in [*members, (self.synthesizer, synth_id)] + if "/" in model_id + } return cache_mod.make_key( prompt=prompt, mode=mode, @@ -221,10 +236,21 @@ def _cache_key( synthesizer=self.synthesizer, synthesizer_model_id=synth_id, temperature=self.temperature, + timeout=self.timeout, rounds=rounds, proposer=proposer, converge_threshold=converge_threshold, choices=choices, + extract_verdict=self.extract_verdict_enabled, + endpoint_urls={ + prefix: endpoint.completions_url + for prefix, endpoint in self.config.endpoints.items() + if prefix in used_prefixes + }, + source_bundle_digest=self.source_bundle_digest, + protocol_version=ELITE_PROTOCOL_VERSION, + synthesis_prompt_version=SYNTHESIS_PROMPT_VERSION, + elite_prompt_version=ELITE_PROMPT_VERSION, ) async def _cached_run( diff --git a/src/conclave/prompts.py b/src/conclave/prompts.py index dc84ea4..569fe70 100644 --- a/src/conclave/prompts.py +++ b/src/conclave/prompts.py @@ -23,6 +23,11 @@ # opaque (a date-stamped tag); only equality/inequality is meaningful. SYNTHESIS_PROMPT_VERSION = "2026-06-29" +# Elite's prompt version is independent from the orchestration version owned by +# ``models.ELITE_PROTOCOL_VERSION``. Bump this whenever the critic or revision +# wording below changes; both versions are part of result-cache identity. +ELITE_PROMPT_VERSION = "1" + # Stable position-based labels used to anonymize peers in debate rounds 2..N. LETTERS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" diff --git a/tests/test_cache.py b/tests/test_cache.py index 544960c..d8efd66 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -26,7 +26,7 @@ import conclave.council as council_mod from conclave import Council from conclave import cache as cache_mod -from conclave.config import ConclaveConfig +from conclave.config import ConclaveConfig, CustomEndpoint from conclave.models import ModelAnswer from tests.conftest import make_response @@ -68,7 +68,9 @@ def counting_call_model(monkeypatch): """ counter = {"n": 0} - async def fake_call_model(name, model_id, messages, *, temperature=0.7, timeout=120.0): + async def fake_call_model( + name, model_id, messages, *, temperature=0.7, timeout=120.0, config=None + ): counter["n"] += 1 await asyncio.sleep(0) # Synthesizer call is the 2-message (system+user) one. @@ -375,6 +377,214 @@ def test_make_key_debate_converge_threshold_differs(monkeypatch): assert k_off == cache_mod.make_key(converge_threshold=None, **base) +@pytest.mark.parametrize( + ("override", "value"), + [ + ("cache_format_version", "next-cache-format"), + ("protocol_version", "next-elite-protocol"), + ("synthesis_prompt_version", "next-synthesis-prompt"), + ("elite_prompt_version", "next-elite-prompt"), + ("verdict_schema_version", "next-verdict-schema"), + ("verdict_prompt_version", "next-verdict-prompt"), + ("timeout", 45.0), + ("extract_verdict", False), + ("source_bundle_digest", "sha256:grounded-source-bundle"), + ], +) +def test_make_key_varies_for_every_protocol_identity_dimension(override, value): + """Every output-affecting protocol/version setting must invalidate reuse.""" + base = dict( + prompt="Should we ship?", + mode="elite", + members=[("grok", "xai/grok-4.3"), ("gemini", "gemini/gemini-2.5-pro")], + synthesizer="claude", + synthesizer_model_id="anthropic/claude-sonnet-4-6", + temperature=0.7, + timeout=120.0, + extract_verdict=True, + source_bundle_digest=None, + ) + changed = dict(base) + changed[override] = value + + assert cache_mod.make_key(**base) != cache_mod.make_key(**changed) + + +def test_cache_identity_covers_roster_mode_params_and_safe_custom_endpoint_config(): + """Roster, mode parameters, and endpoint routing all affect identity.""" + base = dict( + prompt="pick one", + mode="vote", + members=[("a", "custom/model-a")], + synthesizer="judge", + synthesizer_model_id="custom/judge-a", + temperature=0.2, + timeout=30.0, + extract_verdict=True, + choices=["A", "B"], + endpoint_urls={"custom": "https://gateway.example/v1/chat?api-version=2026-01"}, + ) + + assert cache_mod.make_key(**base) != cache_mod.make_key( + **{**base, "members": [("a", "custom/model-b")]} + ) + assert cache_mod.make_key(**base) != cache_mod.make_key(**{**base, "choices": ["A", "B", "C"]}) + assert cache_mod.make_key(**base) != cache_mod.make_key( + **{ + **base, + "endpoint_urls": {"custom": "https://other.example/v1/chat?api-version=2026-01"}, + } + ) + assert cache_mod.make_key(**base) != cache_mod.make_key( + **{ + **base, + "endpoint_urls": {"custom": "https://gateway.example/v1/chat?api-version=2027-01"}, + } + ) + + +def test_cache_identity_never_contains_or_depends_on_endpoint_credentials(): + """Credential-bearing URL components and API-key values are non-identity.""" + secret = "sk-CONCLAVE-ENDPOINT-SECRET-0123456789" + safe_url = "https://gateway.example/v1/chat?api-version=2026-01" + credentialed_url = ( + f"https://user:{secret}@gateway.example/v1/chat?" + f"api-version=2026-01&api_key={secret}&key={secret}&sig={secret}&" + f"code={secret}&access_token={secret}&auth={secret}&password={secret}&" + f"label={secret}#private-fragment" + ) + common = dict( + prompt="q", + mode="elite", + members=[("custom", "custom/model")], + synthesizer="custom", + synthesizer_model_id="custom/model", + temperature=0.7, + ) + + safe_identity = cache_mod.build_identity(**common, endpoint_urls={"custom": safe_url}) + credentialed_identity = cache_mod.build_identity( + **common, endpoint_urls={"custom": credentialed_url} + ) + blob = json.dumps(credentialed_identity, sort_keys=True) + + assert credentialed_identity == safe_identity + assert secret not in blob + assert credentialed_url not in blob + assert "api_key" not in blob + + +def test_cache_identity_fingerprints_prompt_and_source_inputs(): + """Potentially sensitive prompt/source content never appears in identity.""" + secret = "sk_FAKE_PROMPT_SECRET_0123456789" + identity = cache_mod.build_identity( + prompt=f"analyze {secret}", + mode="elite", + members=[("a", "x/1")], + synthesizer="a", + synthesizer_model_id="x/1", + temperature=0.7, + source_bundle_digest=f"malformed-digest-{secret}", + ) + + blob = json.dumps(identity, sort_keys=True) + assert secret not in blob + assert "analyze" not in blob + assert "malformed-digest" not in blob + + +def test_council_threads_resolved_identity_settings_into_cache_key(): + """Council cache identity includes resolved endpoint and runtime settings.""" + cfg = _config() + cfg.models["custom"] = "private/model-a" + cfg.endpoints["private"] = CustomEndpoint( + completions_url="https://gateway.example/v1/chat?api-version=2026-01", + env_var="PRIVATE_API_KEY", + ) + base = Council( + models=["custom"], + synthesizer="custom", + config=cfg, + timeout=30.0, + extract_verdict=True, + source_bundle_digest="bundle-a", + ) + base_key = base._cache_key("q", "elite") + + changed_timeout = Council( + models=["custom"], + synthesizer="custom", + config=cfg, + timeout=45.0, + extract_verdict=True, + source_bundle_digest="bundle-a", + ) + changed_verdict = Council( + models=["custom"], + synthesizer="custom", + config=cfg, + timeout=30.0, + extract_verdict=False, + source_bundle_digest="bundle-a", + ) + changed_source = Council( + models=["custom"], + synthesizer="custom", + config=cfg, + timeout=30.0, + extract_verdict=True, + source_bundle_digest="bundle-b", + ) + changed_cfg = cfg.model_copy(deep=True) + changed_cfg.endpoints[ + "private" + ].completions_url = "https://other.example/v1/chat?api-version=2026-01" + changed_endpoint = Council( + models=["custom"], + synthesizer="custom", + config=changed_cfg, + timeout=30.0, + extract_verdict=True, + source_bundle_digest="bundle-a", + ) + + assert base_key != changed_timeout._cache_key("q", "elite") + assert base_key != changed_verdict._cache_key("q", "elite") + assert base_key != changed_source._cache_key("q", "elite") + assert base_key != changed_endpoint._cache_key("q", "elite") + + +def test_malformed_endpoint_port_degrades_to_safe_deterministic_fingerprint(): + """A malformed configured port cannot crash cache identity construction.""" + raw_url = "https://gateway.example:not-a-port/v1/chat?api-version=2026-01" + identity = cache_mod.build_identity( + prompt="q", + mode="elite", + members=[("custom", "custom/model")], + synthesizer="custom", + synthesizer_model_id="custom/model", + temperature=0.7, + endpoint_urls={"custom": raw_url}, + ) + + blob = json.dumps(identity, sort_keys=True) + assert raw_url not in blob + assert len(identity["endpoint_fingerprints"]["custom"]) == 64 + + +def test_old_unversioned_cache_payload_is_a_miss(cache_home): + """Pre-envelope cache files are not replayed against a new protocol.""" + key = "legacy" + cache_home.mkdir(parents=True, exist_ok=True) + legacy = ModelAnswer(name="a", model_id="x/1", answer="old") + from conclave.models import CouncilResult + + old_result = CouncilResult(prompt="q", answers=[legacy]) + (cache_home / f"{key}.json").write_text(old_result.model_dump_json(), encoding="utf-8") + + assert cache_mod.load(key) is None + + async def test_cache_converge_vs_fixed_no_collision(cache_home, monkeypatch, patch_call_model): """End-to-end: a converged debate and a fixed debate get distinct cache entries. From 7c8fd0874c34e677ffbc1730eb5d91ce10c6b172 Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Fri, 17 Jul 2026 08:34:39 -0400 Subject: [PATCH 16/21] test: align fixtures with elite contracts --- tests/test_cli.py | 4 +++- tests/test_keyleak_audit.py | 4 +++- tests/test_manifest_all_modes.py | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index fa8771d..3d020c3 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -419,7 +419,9 @@ def test_cache_flag_serves_second_run_from_cache(monkeypatch, patch_cli_config, counter = {"n": 0} - async def fake_call_model(name, model_id, messages, *, temperature=0.7, timeout=120.0): + async def fake_call_model( + name, model_id, messages, *, temperature=0.7, timeout=120.0, config=None + ): counter["n"] += 1 return ModelAnswer(name=name, model_id=model_id, answer=f"ans-{model_id}") diff --git a/tests/test_keyleak_audit.py b/tests/test_keyleak_audit.py index 2e54fa9..275e8fd 100644 --- a/tests/test_keyleak_audit.py +++ b/tests/test_keyleak_audit.py @@ -511,7 +511,9 @@ async def test_fan_out_catch_all_error_is_redacted(monkeypatch): import conclave.council as council_mod - async def raising_call_model(name, model_id, messages, *, temperature=0.7, timeout=120.0): + async def raising_call_model( + name, model_id, messages, *, temperature=0.7, timeout=120.0, config=None + ): # Simulate an unexpected escape carrying the key in its text. raise RuntimeError(f"unexpected boom leaking {PLANTED}") diff --git a/tests/test_manifest_all_modes.py b/tests/test_manifest_all_modes.py index c6c329b..23cce66 100644 --- a/tests/test_manifest_all_modes.py +++ b/tests/test_manifest_all_modes.py @@ -76,7 +76,7 @@ def _assert_verified_manifest(result, expected_mode: str) -> None: def _elite_phase(messages) -> str: """Identify the elite phase from its system prompt.""" system = _system_text(messages) - if "evidence auditor" in system: + if "claim auditor" in system: return "critique" if "revise your original answer" in system: return "revision" From f326f4f952ce571624548f30e17da9607b41bed4 Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Fri, 17 Jul 2026 08:40:03 -0400 Subject: [PATCH 17/21] fix(elite): separate completion from readiness --- src/conclave/__init__.py | 2 + src/conclave/cli.py | 32 ++++++++++---- src/conclave/council.py | 37 +++++++++++++++- src/conclave/models.py | 7 ++- src/conclave/modes.py | 4 ++ tests/test_cache.py | 30 +++++++++++++ tests/test_cli.py | 56 ++++++++++++++++++++++- tests/test_elite_mode.py | 96 ++++++++++++++++++++++++++++++++++++++++ 8 files changed, 253 insertions(+), 11 deletions(-) diff --git a/src/conclave/__init__.py b/src/conclave/__init__.py index 2f2cfa1..97922bb 100644 --- a/src/conclave/__init__.py +++ b/src/conclave/__init__.py @@ -53,6 +53,7 @@ AdversarialResult, CouncilResult, DebateRound, + DecisionReadiness, EliteResult, ModelAnswer, StreamEvent, @@ -87,6 +88,7 @@ "DebateRound", "AdversarialResult", "EliteResult", + "DecisionReadiness", "ELITE_PROTOCOL_VERSION", "ELITE_MIN_RESPONDERS", "StreamEvent", diff --git a/src/conclave/cli.py b/src/conclave/cli.py index 4d52bd9..ec5b8eb 100644 --- a/src/conclave/cli.py +++ b/src/conclave/cli.py @@ -315,9 +315,12 @@ def successful(answers) -> int: f"Revisions {successful(elite.revisions)}/{required}", ) ) + readiness = f"Decision readiness: {elite.decision_readiness.upper()}" + if elite.readiness_reasons: + readiness += f" ({', '.join(elite.readiness_reasons)})" console.print( Panel( - summary, + f"{summary}\n{readiness}", title=f"[bold]ELITE DECISION PROTOCOL[/bold] ({elite.protocol_version})", border_style="cyan", ) @@ -574,22 +577,35 @@ def ask( # purposes regardless of output format. We compute this once and apply the # same exit-code contract to both the JSON and human paths. no_usable_answers = not result.successful_answers - elite_incomplete = mode_lower == "elite" and ( - result.elite is None or not result.elite.completed + elite_not_ready = mode_lower == "elite" and ( + result.elite is None or result.elite.decision_readiness != "ready" ) if as_json: # Always emit valid JSON to stdout so a consumer can parse the payload, # then signal failure via the exit code if nothing usable came back. console.print_json(json.dumps(_result_to_dict(result))) - if no_usable_answers or elite_incomplete: + if no_usable_answers or elite_not_ready: raise typer.Exit(code=1) return - if elite_incomplete: - reason = result.elite.failure_reason if result.elite is not None else "missing result" - err_console.print(f"[red]Elite protocol incomplete: {reason}[/red]") - raise typer.Exit(code=1) + if mode_lower == "elite": + if result.elite is None: + err_console.print("[red]Elite decision not ready: missing result[/red]") + raise typer.Exit(code=1) + _render_elite(result) + if not result.elite.completed: + reason = result.elite.failure_reason or ", ".join(result.elite.readiness_reasons) + err_console.print(f"[red]Elite protocol incomplete: {reason}[/red]") + raise typer.Exit(code=1) + if result.elite.decision_readiness != "ready": + reasons = ", ".join(result.elite.readiness_reasons) or "no reason recorded" + err_console.print( + "[red]Elite decision not ready: " + f"{result.elite.decision_readiness} ({reasons})[/red]" + ) + raise typer.Exit(code=1) + return if no_usable_answers: err_console.print( diff --git a/src/conclave/council.py b/src/conclave/council.py index 7729fc0..6917c04 100644 --- a/src/conclave/council.py +++ b/src/conclave/council.py @@ -937,11 +937,46 @@ async def run() -> CouncilResult: if result.elite is not None and result.elite.completed: await self._synthesize(result) self._ensure_manifest(result, "elite") - await self._apply_verdict(result) + if result.synthesis is None: + result.elite.decision_readiness = "not_ready" + result.elite.readiness_reasons = ["synthesis.failed"] + elif not self.extract_verdict_enabled: + result.elite.decision_readiness = "indeterminate" + result.elite.readiness_reasons = ["adjudication.disabled"] + else: + await self._apply_verdict(result) + self._set_elite_readiness(result) return result return await self._cached_run(prompt, "elite", run) + @staticmethod + def _set_elite_readiness(result: CouncilResult) -> None: + """Classify Elite readiness after required synthesis and adjudication.""" + elite = result.elite + if elite is None: + return + if result.verdict is not None: + elite.decision_readiness = "ready" + elite.readiness_reasons = [] + return + + absent_reason = ( + result.manifest.verdict_absent_reason if result.manifest is not None else None + ) + if absent_reason == "verdict extraction failed schema validation": + elite.decision_readiness = "not_ready" + elite.readiness_reasons = ["adjudication.verdict_extraction_failed"] + elif absent_reason == "fewer than 2 responding members": + elite.decision_readiness = "not_ready" + elite.readiness_reasons = ["adjudication.insufficient_responders"] + elif absent_reason == "open-ended prompt (no decision/review to adjudicate)": + elite.decision_readiness = "indeterminate" + elite.readiness_reasons = ["adjudication.open_ended"] + else: + elite.decision_readiness = "indeterminate" + elite.readiness_reasons = ["adjudication.unknown"] + async def aclose(self) -> None: """Close the shared pooled HTTP client. diff --git a/src/conclave/models.py b/src/conclave/models.py index 50dd269..5d0efd6 100644 --- a/src/conclave/models.py +++ b/src/conclave/models.py @@ -9,6 +9,7 @@ import hashlib import json from collections.abc import Iterable +from typing import Literal from pydantic import BaseModel, Field @@ -22,6 +23,8 @@ ELITE_PROTOCOL_VERSION = "elite_v1" ELITE_MIN_RESPONDERS = 3 +DecisionReadiness = Literal["ready", "not_ready", "indeterminate"] + _ANSWER_ID_VERSION = "answer_v1" @@ -152,12 +155,14 @@ def latency_ms(self) -> float: class EliteResult(BaseModel): - """Phase artifacts and completion state for an elite protocol run.""" + """Protocol completion and independently adjudicated decision readiness.""" protocol_version: str = ELITE_PROTOCOL_VERSION required_responders: int = Field(default=ELITE_MIN_RESPONDERS, ge=ELITE_MIN_RESPONDERS) completed: bool = False failure_reason: str | None = None + decision_readiness: DecisionReadiness = "indeterminate" + readiness_reasons: list[str] = Field(default_factory=lambda: ["adjudication.not_evaluated"]) initial_answers: list[ModelAnswer] = Field(default_factory=list) critiques: list[ModelAnswer] = Field(default_factory=list) revisions: list[ModelAnswer] = Field(default_factory=list) diff --git a/src/conclave/modes.py b/src/conclave/modes.py index 2aa87c5..133d1f0 100644 --- a/src/conclave/modes.py +++ b/src/conclave/modes.py @@ -101,6 +101,8 @@ def initial_messages(_name: str, _model_id: str) -> list[dict[str, str]]: result.answers = revision_successes elite.completed = True + elite.decision_readiness = "indeterminate" + elite.readiness_reasons = ["adjudication.pending"] return result @@ -128,6 +130,8 @@ def _elite_gate( f"{phase} phase required {elite.required_responders} successful responders; " f"got {len(successful)}" ) + elite.decision_readiness = "not_ready" + elite.readiness_reasons = [f"protocol.{phase}_responder_gate_failed"] logger.warning(elite.failure_reason) return successful diff --git a/tests/test_cache.py b/tests/test_cache.py index d8efd66..4923f52 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -212,6 +212,8 @@ async def test_elite_cache_hit_preserves_artifacts_and_isolated_mode( assert live.cached is False assert live.elite is not None assert live.elite.completed is True + assert live.elite.decision_readiness == "indeterminate" + assert live.elite.readiness_reasons == ["adjudication.disabled"] assert cached.cached is True assert cached.elite == live.elite assert counting_call_model["n"] == calls_after_live @@ -224,6 +226,34 @@ async def test_elite_cache_hit_preserves_artifacts_and_isolated_mode( assert len(list(cache_home.glob("*.json"))) == 2 +def test_current_cache_shape_without_readiness_defaults_indeterminate(cache_home): + """A legacy Elite payload in the current envelope can never replay as ready.""" + from conclave.models import CouncilResult, EliteResult + + key = "legacy-elite-readiness" + cache_home.mkdir(parents=True, exist_ok=True) + result = CouncilResult( + prompt="q", + mode="elite", + elite=EliteResult(completed=True), + ).model_dump(mode="json") + del result["elite"]["decision_readiness"] + del result["elite"]["readiness_reasons"] + envelope = { + "cache_format_version": cache_mod.CACHE_FORMAT_VERSION, + "result": result, + } + (cache_home / f"{key}.json").write_text(json.dumps(envelope), encoding="utf-8") + + cached = cache_mod.load(key) + + assert cached is not None + assert cached.elite is not None + assert cached.elite.completed is True + assert cached.elite.decision_readiness == "indeterminate" + assert cached.elite.readiness_reasons == ["adjudication.not_evaluated"] + + async def test_changing_model_id_misses(monkeypatch, counting_call_model, cache_home): """Same friendly name but a different resolved model id -> different key.""" _set_keys(monkeypatch) diff --git a/tests/test_cli.py b/tests/test_cli.py index 3d020c3..267db4a 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -104,7 +104,13 @@ def test_unknown_mode_exits_two(patch_cli_config): assert result.exit_code == 2 -def _elite_cli_result(*, completed: bool = True, failure_reason: str | None = None): +def _elite_cli_result( + *, + completed: bool = True, + failure_reason: str | None = None, + decision_readiness: str = "ready", + readiness_reasons: list[str] | None = None, +): """Build a compact elite result for CLI-only behavior tests.""" initials = [ ModelAnswer(name=f"member-{i}", model_id=f"provider/model-{i}", answer=f"initial-{i}") @@ -126,6 +132,8 @@ def _elite_cli_result(*, completed: bool = True, failure_reason: str | None = No elite=EliteResult( completed=completed, failure_reason=failure_reason, + decision_readiness=decision_readiness, + readiness_reasons=readiness_reasons or [], initial_answers=initials, critiques=critiques if completed else [], revisions=revisions if completed else [], @@ -167,6 +175,7 @@ def elite_sync(self, prompt: str) -> CouncilResult: assert "Initial 3/3" in result.output assert "Critiques 3/3" in result.output assert "Revisions 3/3" in result.output + assert "Decision readiness: READY" in result.output assert "VERDICT" in result.output assert "Ship deliberately." in result.output @@ -189,6 +198,8 @@ def elite_sync(self, prompt: str) -> CouncilResult: payload = json.loads(result.stdout) assert payload["mode"] == "elite" assert payload["elite"]["completed"] is True + assert payload["elite"]["decision_readiness"] == "ready" + assert payload["elite"]["readiness_reasons"] == [] assert len(payload["elite"]["initial_answers"]) == 3 assert len(payload["elite"]["critiques"]) == 3 assert len(payload["elite"]["revisions"]) == 3 @@ -206,6 +217,8 @@ def elite_sync(self, prompt: str) -> CouncilResult: return _elite_cli_result( completed=False, failure_reason="critique phase requires at least 3 successful responders; got 2", + decision_readiness="not_ready", + readiness_reasons=["protocol.critique_responder_gate_failed"], ) monkeypatch.setattr(cli, "Council", FakeCouncil) @@ -218,6 +231,47 @@ def elite_sync(self, prompt: str) -> CouncilResult: assert "responders; got 2" in result.output +@pytest.mark.parametrize("as_json", [False, True]) +@pytest.mark.parametrize( + ("decision_readiness", "reason"), + [ + ("indeterminate", "adjudication.disabled"), + ("not_ready", "synthesis.failed"), + ], +) +def test_elite_non_ready_state_exits_one_with_machine_reason( + monkeypatch, patch_cli_config, as_json, decision_readiness, reason +): + """Protocol completion cannot produce CLI success without adjudicated readiness.""" + + class FakeCouncil: + def __init__(self, **kwargs): + pass + + def elite_sync(self, prompt: str) -> CouncilResult: + return _elite_cli_result( + decision_readiness=decision_readiness, + readiness_reasons=[reason], + ) + + monkeypatch.setattr(cli, "Council", FakeCouncil) + args = ["ask", "Should we ship?", "--mode", "elite"] + if as_json: + args.append("--json") + + result = runner.invoke(cli.app, args) + + assert result.exit_code == 1 + if as_json: + payload = json.loads(result.stdout) + assert payload["elite"]["completed"] is True + assert payload["elite"]["decision_readiness"] == decision_readiness + assert payload["elite"]["readiness_reasons"] == [reason] + else: + assert f"Decision readiness: {decision_readiness.upper()}" in result.output + assert reason in result.output + + def test_elite_stream_is_rejected_before_council_creation(monkeypatch, patch_cli_config): """Elite streaming fails as usage before a provider-capable Council exists.""" created = False diff --git a/tests/test_elite_mode.py b/tests/test_elite_mode.py index 5e5141b..39184cf 100644 --- a/tests/test_elite_mode.py +++ b/tests/test_elite_mode.py @@ -99,6 +99,8 @@ def test_elite_result_defaults_to_incomplete_v1_protocol() -> None: assert result.required_responders == ELITE_MIN_RESPONDERS assert result.completed is False assert result.failure_reason is None + assert result.decision_readiness == "indeterminate" + assert result.readiness_reasons == ["adjudication.not_evaluated"] def test_elite_result_rejects_fewer_than_three_required_responders() -> None: @@ -239,6 +241,8 @@ def handler(model_id, messages): assert result.mode == "elite" assert result.elite is not None assert result.elite.completed is True + assert result.elite.decision_readiness == "indeterminate" + assert result.elite.readiness_reasons == ["adjudication.pending"] assert result.elite.failure_reason is None assert len(result.elite.initial_answers) == 3 assert len(result.elite.critiques) == 3 @@ -349,6 +353,8 @@ def handler(model_id, messages): assert result.elite is not None assert result.elite.completed is False assert result.elite.failure_reason == "initial phase required 3 successful responders; got 2" + assert result.elite.decision_readiness == "not_ready" + assert result.elite.readiness_reasons == ["protocol.initial_responder_gate_failed"] assert phases == ["initial"] * 3 assert result.elite.critiques == [] assert result.elite.revisions == [] @@ -379,6 +385,8 @@ def handler(model_id, messages): assert result.elite is not None assert result.elite.completed is False assert result.elite.failure_reason == "critique phase required 3 successful responders; got 2" + assert result.elite.decision_readiness == "not_ready" + assert result.elite.readiness_reasons == ["protocol.critique_responder_gate_failed"] assert phases == ["initial"] * 3 + ["critique"] * 3 assert len(result.elite.critiques) == 3 assert result.elite.revisions == [] @@ -407,6 +415,8 @@ def handler(model_id, messages): assert result.elite is not None assert result.elite.completed is False assert result.elite.failure_reason == "revision phase required 3 successful responders; got 2" + assert result.elite.decision_readiness == "not_ready" + assert result.elite.readiness_reasons == ["protocol.revision_responder_gate_failed"] assert len(result.elite.revisions) == 3 assert len(result.answers) == 3 assert all(answer.answer.startswith("initial") for answer in result.answers) @@ -440,6 +450,8 @@ def handler(model_id, messages): assert result.elite is not None assert result.elite.completed is True + assert result.elite.decision_readiness == "ready" + assert result.elite.readiness_reasons == [] assert result.synthesis == "elite synthesis" assert len(synthesis_inputs) == 1 assert "revision from xai/grok-4.3" in synthesis_inputs[0] @@ -483,6 +495,8 @@ async def unexpected_verdict(_result): assert result.elite is not None assert result.elite.completed is False + assert result.elite.decision_readiness == "not_ready" + assert result.elite.readiness_reasons == ["protocol.initial_responder_gate_failed"] assert result.synthesis is None assert result.verdict is None assert calls == {"synthesis": 0, "verdict": 0} @@ -506,3 +520,85 @@ def handler(model_id, messages): assert result.elite is not None assert result.elite.completed is True assert result.synthesis is not None + assert result.elite.decision_readiness == "indeterminate" + assert result.elite.readiness_reasons == ["adjudication.disabled"] + + +async def test_council_elite_synthesis_failure_is_not_ready(monkeypatch, patch_call_model) -> None: + _all_keys(monkeypatch) + + def handler(model_id, messages): + system = next( + (message["content"] for message in messages if message["role"] == "system"), "" + ) + if system.startswith("You are the synthesizer of a council"): + raise RuntimeError("synth unavailable") + return make_response(f"{_phase(messages)} from {model_id}") + + patch_call_model(handler) + result = await Council( + models=["grok", "gemini", "perplexity"], + config=_elite_config(), + ).elite("Choose.") + + assert result.elite is not None + assert result.elite.completed is True + assert result.synthesis is None + assert result.elite.decision_readiness == "not_ready" + assert result.elite.readiness_reasons == ["synthesis.failed"] + + +async def test_council_elite_failed_verdict_repair_is_not_ready( + monkeypatch, patch_call_model +) -> None: + _all_keys(monkeypatch) + + def handler(model_id, messages): + if _is_verdict_call(messages): + return make_response("not valid verdict json") + system = next( + (message["content"] for message in messages if message["role"] == "system"), "" + ) + if system.startswith("You are the synthesizer of a council"): + return make_response("elite synthesis") + return make_response(f"{_phase(messages)} from {model_id}") + + patch_call_model(handler) + result = await Council( + models=["grok", "gemini", "perplexity"], + config=_elite_config(), + ).elite("Choose.") + + assert result.elite is not None + assert result.elite.completed is True + assert result.elite.decision_readiness == "not_ready" + assert result.elite.readiness_reasons == ["adjudication.verdict_extraction_failed"] + + +async def test_council_elite_open_ended_verdict_is_indeterminate( + monkeypatch, patch_call_model +) -> None: + _all_keys(monkeypatch) + + def handler(model_id, messages): + if _is_verdict_call(messages): + payload = json.loads(_elite_verdict_json()) + payload["verdict_applies"] = False + return make_response(json.dumps(payload)) + system = next( + (message["content"] for message in messages if message["role"] == "system"), "" + ) + if system.startswith("You are the synthesizer of a council"): + return make_response("elite synthesis") + return make_response(f"{_phase(messages)} from {model_id}") + + patch_call_model(handler) + result = await Council( + models=["grok", "gemini", "perplexity"], + config=_elite_config(), + ).elite("Explore the topic.") + + assert result.elite is not None + assert result.elite.completed is True + assert result.elite.decision_readiness == "indeterminate" + assert result.elite.readiness_reasons == ["adjudication.open_ended"] From 3b97537978e2ec194c3bac26995481e3f390fb8a Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Fri, 17 Jul 2026 08:56:50 -0400 Subject: [PATCH 18/21] feat(manifest): account for every elite model call --- src/conclave/council.py | 126 +++++++++++---- src/conclave/manifest.py | 49 ++++-- src/conclave/providers.py | 48 +++++- src/conclave/streaming.py | 7 +- src/conclave/verdict_synthesis.py | 75 ++++++++- tests/test_manifest_all_modes.py | 207 ++++++++++++++++++++++++- tests/test_output_contract_plumbing.py | 22 ++- tests/test_secret_safety_matrix.py | 19 +++ tests/test_verdict_edge_cases.py | 13 +- 9 files changed, 502 insertions(+), 64 deletions(-) diff --git a/src/conclave/council.py b/src/conclave/council.py index 6917c04..241f499 100644 --- a/src/conclave/council.py +++ b/src/conclave/council.py @@ -60,7 +60,12 @@ from .adapters.base import redact from .config import ConclaveConfig, load_config from .logging import get_logger -from .manifest import ModelHarnessManifest, ProviderSkip, verified_secret_safety +from .manifest import ( + ModelHarnessManifest, + ProviderExecutionReceipt, + ProviderSkip, + verified_secret_safety, +) from .models import ELITE_PROTOCOL_VERSION, CouncilResult, ModelAnswer, StreamEvent, TokenUsage from .prompts import ELITE_PROMPT_VERSION, SYNTHESIS_PROMPT_VERSION from .providers import call_model, receipt_from_answer @@ -475,13 +480,8 @@ def _build_manifest( model_ids=[model_id for _name, model_id in members], generation_settings={"temperature": self.temperature, "timeout": self.timeout}, receipts=receipts, - total_latency_ms=sum(a.latency_ms for a in answers), - total_usage=self._sum_usage(answers), - redacted_errors=[a.error for a in answers if a.error], ) - # Stamp VERIFIED only when the serialized manifest is provably clean - # (the load-bearing CAC-04 acceptance criterion). - manifest.secret_safety = verified_secret_safety(manifest) + self._recompute_manifest_accounting(manifest) return manifest def _build_elite_manifest( @@ -511,13 +511,14 @@ def _build_elite_manifest( ("revision", elite.revisions), ] ) - flattened = [answer for _phase, answers in phase_artifacts for answer in answers] receipts = [ receipt_from_answer( answer, temperature=self.temperature, timeout=self.timeout, phase=phase, + protocol_version=ELITE_PROTOCOL_VERSION, + prompt_version=None if phase == "initial" else ELITE_PROMPT_VERSION, ) for phase, answers in phase_artifacts for answer in answers @@ -534,29 +535,48 @@ def _build_elite_manifest( model_ids=list(dict.fromkeys(model_id for _name, model_id in members)), generation_settings={"temperature": self.temperature, "timeout": self.timeout}, receipts=receipts, - total_latency_ms=sum(answer.latency_ms for answer in flattened), - total_usage=self._sum_usage(flattened), - redacted_errors=[answer.error for answer in flattened if answer.error], ) - manifest.secret_safety = verified_secret_safety(manifest) + self._recompute_manifest_accounting(manifest) return manifest @staticmethod - def _sum_usage(answers: list[ModelAnswer]) -> TokenUsage | None: - """Sum token usage across answers, or ``None`` when none reported usage. - - Returns ``None`` (not a zeroed :class:`~conclave.models.TokenUsage`) when - no member reported usage, so the manifest can distinguish "no usage data" - from "a real zero". - """ - reported = [a.usage for a in answers if a.usage is not None] - if not reported: - return None - return TokenUsage( - prompt_tokens=sum(u.prompt_tokens for u in reported), - completion_tokens=sum(u.completion_tokens for u in reported), - total_tokens=sum(u.total_tokens for u in reported), + def _recompute_manifest_accounting(manifest: ModelHarnessManifest) -> None: + """Recompute every manifest aggregate from its complete receipt ledger.""" + manifest.total_latency_ms = sum(receipt.latency_ms for receipt in manifest.receipts) + usages = [receipt.usage for receipt in manifest.receipts if receipt.usage is not None] + manifest.total_usage = ( + TokenUsage( + prompt_tokens=sum(usage.prompt_tokens for usage in usages), + completion_tokens=sum(usage.completion_tokens for usage in usages), + total_tokens=sum(usage.total_tokens for usage in usages), + ) + if usages + else None ) + # Cost is known only when every actual call carries a trustworthy priced + # value. A partially-known total would be misleading, so unknown remains + # None rather than being silently coerced to zero. + costs = [receipt.estimated_cost for receipt in manifest.receipts] + manifest.estimated_cost = ( + sum(cost for cost in costs if cost is not None) + if costs and all(cost is not None for cost in costs) + else None + ) + manifest.redacted_errors = [ + receipt.error for receipt in manifest.receipts if receipt.error is not None + ] + manifest.secret_safety = verified_secret_safety(manifest) + + def _append_manifest_receipts( + self, + result: CouncilResult, + receipts: list[ProviderExecutionReceipt], + ) -> None: + """Append explicit call receipts and refresh all derived manifest fields.""" + if result.manifest is None or not receipts: + return + result.manifest.receipts.extend(receipts) + self._recompute_manifest_accounting(result.manifest) async def _ask_uncached(self, prompt: str, synthesize: bool = True) -> CouncilResult: """The live ask path (no cache consultation). See :meth:`ask`. @@ -590,7 +610,23 @@ async def _ask_uncached(self, prompt: str, synthesize: bool = True) -> CouncilRe # raw mode (no synthesizer call) and is opt-out via the constructor # flag (resolved inside the helper). The no-members early return above # never reaches here, so a memberless run carries no verdict. - await self._synthesize(result) + synthesis_answer = await self._synthesize(result) + if synthesis_answer is not None: + self._append_manifest_receipts( + result, + [ + receipt_from_answer( + synthesis_answer, + temperature=self.temperature, + timeout=self.timeout, + phase="synthesis", + protocol_version=( + ELITE_PROTOCOL_VERSION if result.mode == "elite" else None + ), + prompt_version=SYNTHESIS_PROMPT_VERSION, + ) + ], + ) await self._apply_verdict(result) return result @@ -696,7 +732,7 @@ def _replay_cached(result: CouncilResult) -> list[StreamEvent]: events.append(StreamEvent(type="done", result=result)) return events - async def _synthesize(self, result: CouncilResult) -> None: + async def _synthesize(self, result: CouncilResult) -> ModelAnswer | None: """Run the synthesizer over the successful answers, mutating ``result``. This is the buffered (non-streaming) synthesize path; the streaming @@ -727,7 +763,7 @@ async def _synthesize(self, result: CouncilResult) -> None: if not usable: result.synthesis_error = "no successful member answers to synthesize" logger.warning(result.synthesis_error) - return + return None synth_id = self.config.resolve_model_id(self.synthesizer) result.synthesizer = self.synthesizer @@ -739,7 +775,7 @@ async def _synthesize(self, result: CouncilResult) -> None: "returning raw answers only" ) logger.warning(result.synthesis_error) - return + return None blocks = "\n\n".join( f"### Answer from {a.name} ({a.model_id})" @@ -756,8 +792,14 @@ async def _synthesize(self, result: CouncilResult) -> None: result.synthesis = answer.answer else: result.synthesis_error = answer.error + return answer - async def _apply_verdict(self, result: CouncilResult) -> None: + async def _apply_verdict( + self, + result: CouncilResult, + *, + record_receipts: bool = True, + ) -> None: """Run verdict extraction over the answers and hoist it onto ``result``. The SINGLE shared verdict-resolution path. Both the buffered @@ -822,8 +864,14 @@ async def _apply_verdict(self, result: CouncilResult) -> None: synthesizer_name=synthesizer_name, synthesizer_model_id=synth_id, config=self.config, + temperature=self.temperature, + timeout=self.timeout, + protocol_version=(ELITE_PROTOCOL_VERSION if result.mode == "elite" else None), ) + if record_receipts: + self._append_manifest_receipts(result, vsr.attempt_receipts) + result.verdict = vsr.verdict if vsr.verdict is not None: # Hoist the canonical verdict's values to the top-level mirrors. @@ -935,8 +983,22 @@ async def elite(self, prompt: str) -> CouncilResult: async def run() -> CouncilResult: result = await run_elite(self, prompt) if result.elite is not None and result.elite.completed: - await self._synthesize(result) + synthesis_answer = await self._synthesize(result) self._ensure_manifest(result, "elite") + if synthesis_answer is not None: + self._append_manifest_receipts( + result, + [ + receipt_from_answer( + synthesis_answer, + temperature=self.temperature, + timeout=self.timeout, + phase="synthesis", + protocol_version=ELITE_PROTOCOL_VERSION, + prompt_version=SYNTHESIS_PROMPT_VERSION, + ) + ], + ) if result.synthesis is None: result.elite.decision_readiness = "not_ready" result.elite.readiness_reasons = ["synthesis.failed"] diff --git a/src/conclave/manifest.py b/src/conclave/manifest.py index 5f1ca06..0bd3f4b 100644 --- a/src/conclave/manifest.py +++ b/src/conclave/manifest.py @@ -6,7 +6,7 @@ * WHAT ran — ``request_id``, ``conclave_version``, deliberation ``mode``, providers considered / called / **skipped (with reasons)**, the concrete - resolved model ids, the generation settings used, per-member execution + resolved model ids, the generation settings used, per-call execution receipts, total latency, and total token usage; * cost (carefully — Scope Plan §8) — token ``total_usage`` is always present; ``estimated_cost`` is left ``None`` (a wrong number inside an audit receipt is @@ -19,10 +19,10 @@ leaves them ``None``. **Secret-safety (Scope Plan §5 — non-negotiable).** Key VALUES never appear in a -manifest. Per-member errors are redacted upstream (in :mod:`conclave.providers`) -before they reach a receipt, and :func:`receipt_from_answer` re-applies -:func:`conclave.adapters.base.redact` belt-and-suspenders. After assembling the -manifest the council runs :func:`scan_for_secret_material` and stamps +manifest. Arbitrary provider errors are reduced to bounded categories before +they reach a receipt, so raw endpoints, bodies, prompts, and exception chains are +never retained. After assembling the manifest the council runs +:func:`scan_for_secret_material` and stamps ``secret_safety`` VERIFIED only when the serialized manifest is provably clean. This module deliberately does NOT import :mod:`conclave.models`; the dependency @@ -34,6 +34,8 @@ from __future__ import annotations +from typing import Literal + from pydantic import BaseModel, Field from .models import TokenUsage @@ -50,6 +52,16 @@ # redacted error string legitimately carries it and must not trip the scan. _FORBIDDEN_SUBSTRINGS = ("sk-", "bearer", "authorization", "api_key", "x-api-key") +ReceiptOutcome = Literal["success", "failed", "schema_invalid"] +ReceiptErrorCategory = Literal[ + "authentication", + "rate_limit", + "timeout", + "transport", + "provider_error", + "schema_validation", +] + class ProviderSkip(BaseModel): """One council member skipped before any call was made. @@ -93,12 +105,15 @@ class ProviderExecutionReceipt(BaseModel): Built from a returned :class:`~conclave.models.ModelAnswer` by :func:`receipt_from_answer` (a skipped member has a :class:`ProviderSkip` instead, never a receipt). Carries only non-secret, auditable facts: the - settings actually used, the latency, the token usage, and a redacted error. + settings actually used, the latency, the token usage, and a bounded error + category. Attributes: - phase: Optional protocol phase provenance. ``None`` for legacy and - non-phased modes; elite receipts use ``initial``, ``critique``, or - ``revision``. + phase: Optional protocol phase provenance. Elite also records synthesis, + verdict extraction, and repair calls. + attempt: One-based attempt number within a phase. + outcome: Bounded outcome such as ``success``, ``failed``, or + ``schema_invalid``. name: Friendly council member name (e.g. ``"grok"``). provider: Provider prefix derived from ``model_id`` (e.g. ``"xai"``). model_id: Resolved provider-prefixed model id (e.g. ``"xai/grok-4.3"``). @@ -106,22 +121,32 @@ class ProviderExecutionReceipt(BaseModel): (``{"temperature": ..., "timeout": ...}``). latency_ms: Wall-clock latency of the call in milliseconds. usage: Token usage if the provider reported it, else ``None``. - error: Redacted error message if the call failed, else ``None``. This - field NEVER holds key material — it is redacted upstream and again on - construction (belt-and-suspenders). + estimated_cost: Trustworthy per-call cost when a dated pricing source is + available. Conclave currently has no pricing table, so this stays + ``None`` rather than inventing a number. + error: Compatibility field containing only the bounded error category, + never raw provider text, URLs, bodies, prompts, or exception chains. + error_category: Secret-free bounded failure category. schema_valid: Whether the member's structured output validated. ``None`` until CAC-02 structured output exists; defined now, populated later. """ phase: str | None = None + attempt: int = Field(default=1, ge=1) + outcome: ReceiptOutcome = "success" name: str provider: str model_id: str generation_settings: dict[str, float] = Field(default_factory=dict) latency_ms: float = 0.0 usage: TokenUsage | None = None + estimated_cost: float | None = None error: str | None = None + error_category: ReceiptErrorCategory | None = None schema_valid: bool | None = None + protocol_version: str | None = None + prompt_version: str | None = None + schema_version: str | None = None class ModelHarnessManifest(BaseModel): diff --git a/src/conclave/providers.py b/src/conclave/providers.py index f1b3005..1ae0fe9 100644 --- a/src/conclave/providers.py +++ b/src/conclave/providers.py @@ -23,7 +23,7 @@ from .adapters.base import OutputContract, ProviderAdapter, redact from .config import ConclaveConfig, load_config from .logging import get_logger -from .manifest import ProviderExecutionReceipt +from .manifest import ProviderExecutionReceipt, ReceiptErrorCategory, ReceiptOutcome from .models import ModelAnswer, TokenUsage, stable_answer_id from .registry import provider_prefix from .transport import TransportError @@ -37,6 +37,13 @@ def receipt_from_answer( temperature: float, timeout: float, phase: str | None = None, + attempt: int = 1, + outcome: ReceiptOutcome | None = None, + error_category: ReceiptErrorCategory | None = None, + schema_valid: bool | None = None, + protocol_version: str | None = None, + prompt_version: str | None = None, + schema_version: str | None = None, ) -> ProviderExecutionReceipt: """Map a collected :class:`ModelAnswer` to a :class:`ProviderExecutionReceipt`. @@ -49,12 +56,10 @@ def receipt_from_answer( assigns them here). The ``provider`` prefix is derived from ``answer.model_id`` via - :func:`conclave.registry.provider_prefix`. The ``error`` is re-run through - :func:`redact` belt-and-suspenders: it is already redacted upstream (every - :class:`ModelAnswer.error` conclave produces is scrubbed), and ``redact`` is - idempotent and safe on clean text, so this re-application cannot leak a key - and cannot raise on the happy path -- preserving the redaction invariant even - on an unexpected error string. + :func:`conclave.registry.provider_prefix`. Arbitrary provider errors are + inspected only to choose a bounded category; the receipt stores that category + in both ``error`` (compatibility) and ``error_category``. It never retains raw + endpoint URLs, bodies, prompts, credentials, or exception chains. Args: answer: The collected member answer (success or failure). @@ -65,18 +70,45 @@ def receipt_from_answer( Returns: A :class:`ProviderExecutionReceipt` for this member. """ + has_error = answer.error is not None + category = error_category or (_receipt_error_category(answer.error) if has_error else None) + resolved_outcome = outcome or ("failed" if has_error else "success") return ProviderExecutionReceipt( phase=phase, + attempt=attempt, + outcome=resolved_outcome, name=answer.name, provider=provider_prefix(answer.model_id), model_id=answer.model_id, generation_settings={"temperature": temperature, "timeout": timeout}, latency_ms=answer.latency_ms, usage=answer.usage, - error=redact(answer.error) if answer.error else None, + # Keep the compatibility field, but store only the bounded category. Raw + # provider text can contain endpoints, response bodies, and exception + # chains even after credential-shaped values are redacted. + error=category, + error_category=category, + schema_valid=schema_valid, + protocol_version=protocol_version, + prompt_version=prompt_version, + schema_version=schema_version, ) +def _receipt_error_category(error: str) -> ReceiptErrorCategory: + """Reduce arbitrary provider error text to a bounded, secret-free category.""" + scrubbed = redact(error).lower() + if "timeout" in scrubbed or "timed out" in scrubbed: + return "timeout" + if any(token in scrubbed for token in ("401", "403", "unauthorized", "forbidden")): + return "authentication" + if "429" in scrubbed or "rate limit" in scrubbed: + return "rate_limit" + if any(token in scrubbed for token in ("connection", "network", "transport", "dns")): + return "transport" + return "provider_error" + + def _resolve_key(adapter: ProviderAdapter) -> str | None: """Read the active key VALUE for an adapter from the environment, or None. diff --git a/src/conclave/streaming.py b/src/conclave/streaming.py index fa27627..e140537 100644 --- a/src/conclave/streaming.py +++ b/src/conclave/streaming.py @@ -239,7 +239,12 @@ async def stream_ask( # no-op when disabled, never raises, and only attaches secret-free content. async for event in _stream_synthesis(council, result): yield event - await council._apply_verdict(result) + # Streaming synthesis has its own token transport and does not yet emit a + # synthesis receipt. Preserve the established streaming manifest as a + # member-call ledger rather than appending only half of the downstream + # call sequence. Complete receipt capture is currently the buffered/Elite + # contract. + await council._apply_verdict(result, record_receipts=False) yield StreamEvent(type="done", result=result) diff --git a/src/conclave/verdict_synthesis.py b/src/conclave/verdict_synthesis.py index 52a6442..a3073bc 100644 --- a/src/conclave/verdict_synthesis.py +++ b/src/conclave/verdict_synthesis.py @@ -69,11 +69,12 @@ from . import agreement from .adapters.base import OutputContract, redact from .logging import get_logger -from .manifest import VerdictExtraction +from .manifest import ProviderExecutionReceipt, VerdictExtraction from .models import ModelAnswer -from .providers import call_model +from .providers import call_model, receipt_from_answer from .verdict import ( VERDICT_EXTRACTION_PROMPT_VERSION, + VERDICT_SCHEMA_VERSION, CouncilConflict, CouncilVerdict, VerdictExtractionModel, @@ -148,11 +149,49 @@ class VerdictSynthesisResult(BaseModel): (``"fewer than 2 responding members"``, ``"open-ended prompt (no decision/review to adjudicate)"``, or ``"verdict extraction failed schema validation"``), or ``None`` when a verdict is present. + attempt_receipts: One secret-free receipt for each actual extraction or + repair call. The N<2 gate makes no call and therefore yields none. """ verdict: CouncilVerdict | None = None extraction: VerdictExtraction verdict_absent_reason: str | None = Field(default=None) + attempt_receipts: list[ProviderExecutionReceipt] = Field(default_factory=list) + + +def _verdict_attempt_receipt( + answer: ModelAnswer, + *, + phase: str, + attempt: int, + schema_valid: bool | None, + temperature: float, + timeout: float, + protocol_version: str | None, +) -> ProviderExecutionReceipt: + """Build one secret-free receipt for an extraction or repair attempt.""" + if answer.error is not None: + outcome = "failed" + error_category = None + elif schema_valid: + outcome = "success" + error_category = None + else: + outcome = "schema_invalid" + error_category = "schema_validation" + return receipt_from_answer( + answer, + temperature=temperature, + timeout=timeout, + phase=phase, + attempt=attempt, + outcome=outcome, + error_category=error_category, + schema_valid=schema_valid, + protocol_version=protocol_version, + prompt_version=VERDICT_EXTRACTION_PROMPT_VERSION, + schema_version=VERDICT_SCHEMA_VERSION, + ) def _responding(member_answers: list[ModelAnswer]) -> list[ModelAnswer]: @@ -401,6 +440,9 @@ async def extract_verdict( synthesizer_name: str, synthesizer_model_id: str, config=None, # noqa: ANN001 -- ConclaveConfig | None; untyped to avoid an import edge + temperature: float = 0.7, + timeout: float = 120.0, + protocol_version: str | None = None, ) -> VerdictSynthesisResult: """Extract a structured, auditable verdict from a council's member answers. @@ -489,12 +531,25 @@ async def extract_verdict( synthesizer_name, synthesizer_model_id, messages, + temperature=temperature, + timeout=timeout, config=config, output_contract=output_contract, ) # Step 3 — validate, then repair ONCE on failure, then fall back. extraction, errors = _parse_and_validate(answer) + attempt_receipts = [ + _verdict_attempt_receipt( + answer, + phase="verdict_extraction", + attempt=1, + schema_valid=None if answer.error is not None else extraction is not None, + temperature=temperature, + timeout=timeout, + protocol_version=protocol_version, + ) + ] if extraction is None: repair_messages = messages + [ { @@ -512,10 +567,23 @@ async def extract_verdict( synthesizer_name, synthesizer_model_id, repair_messages, + temperature=temperature, + timeout=timeout, config=config, output_contract=output_contract, ) extraction, errors = _parse_and_validate(retry) + attempt_receipts.append( + _verdict_attempt_receipt( + retry, + phase="verdict_repair", + attempt=2, + schema_valid=None if retry.error is not None else extraction is not None, + temperature=temperature, + timeout=timeout, + protocol_version=protocol_version, + ) + ) if extraction is None: # Repair exhausted — degrade gracefully (DD-2), never raise. @@ -524,6 +592,7 @@ async def extract_verdict( verdict=None, extraction=extraction_provenance, verdict_absent_reason=_REASON_EXTRACTION_FAILED, + attempt_receipts=attempt_receipts, ) # Step 4 — open-ended prompt → synthesis-only, no verdict (DD-2). @@ -532,6 +601,7 @@ async def extract_verdict( verdict=None, extraction=extraction_provenance, verdict_absent_reason=_REASON_OPEN_ENDED, + attempt_receipts=attempt_receipts, ) # Step 5 — compute consensus deterministically + assemble the verdict. @@ -540,4 +610,5 @@ async def extract_verdict( verdict=verdict, extraction=extraction_provenance, verdict_absent_reason=None, + attempt_receipts=attempt_receipts, ) diff --git a/tests/test_manifest_all_modes.py b/tests/test_manifest_all_modes.py index 23cce66..23b567d 100644 --- a/tests/test_manifest_all_modes.py +++ b/tests/test_manifest_all_modes.py @@ -21,9 +21,15 @@ from conclave import Council from conclave.config import ConclaveConfig -from conclave.manifest import SECRET_SAFETY_VERIFIED, ModelHarnessManifest -from conclave.models import ModelAnswer +from conclave.manifest import ( + SECRET_SAFETY_VERIFIED, + ModelHarnessManifest, + ProviderExecutionReceipt, +) +from conclave.models import ELITE_PROTOCOL_VERSION, ModelAnswer, TokenUsage +from conclave.prompts import ELITE_PROMPT_VERSION, SYNTHESIS_PROMPT_VERSION from conclave.providers import receipt_from_answer +from conclave.verdict import VERDICT_EXTRACTION_PROMPT_VERSION, VERDICT_SCHEMA_VERSION from tests.conftest import make_response @@ -94,6 +100,201 @@ def test_receipt_phase_is_backward_compatible_and_optional() -> None: assert elite.phase == "critique" +def test_receipt_audit_fields_are_additive_and_conservative() -> None: + """Old construction remains valid while new accounting fields default safely.""" + receipt = ProviderExecutionReceipt(name="grok", provider="xai", model_id="xai/grok-4.3") + + assert receipt.attempt == 1 + assert receipt.outcome == "success" + assert receipt.error_category is None + assert receipt.protocol_version is None + assert receipt.prompt_version is None + assert receipt.schema_version is None + assert receipt.estimated_cost is None + + +def test_receipt_empty_error_string_is_still_a_failed_call() -> None: + """ModelAnswer.error presence, not truthiness, determines call failure.""" + answer = ModelAnswer(name="grok", model_id="xai/grok-4.3", error="") + + receipt = receipt_from_answer(answer, temperature=0.7, timeout=120.0) + + assert receipt.outcome == "failed" + assert receipt.error_category == "provider_error" + assert receipt.error == "provider_error" + + +def _valid_elite_verdict_json() -> str: + return """{ + "verdict_applies": true, + "verdict_type": "decision", + "headline": "Choose A", + "recommendation": "Choose A", + "positions": [{ + "label": "A", "summary": "A wins", "providers": ["grok", "gemini", "perplexity"], + "evidence_answer_ids": [] + }], + "provider_votes": [ + {"provider": "grok", "position_label": "A"}, + {"provider": "gemini", "position_label": "A"}, + {"provider": "perplexity", "position_label": "A"} + ], + "minority_reports": [], "conflicts": [], "caveats": [], "dissent_summary": null + }""" + + +async def test_elite_manifest_captures_synthesis_and_verdict_repair_calls(monkeypatch) -> None: + """Every actual Elite model call has one versioned, correctly-accounted receipt.""" + _all_keys(monkeypatch) + verdict_attempt = 0 + calls: list[tuple[float, float]] = [] + + async def fake_call_model( + name, + model, + messages, + *, + temperature=0.7, + timeout=120.0, + config=None, + **kwargs, + ): + nonlocal verdict_attempt + calls.append((temperature, timeout)) + system = _system_text(messages) + if system.startswith("You are the verdict extractor"): + verdict_attempt += 1 + text = "not json" if verdict_attempt == 1 else _valid_elite_verdict_json() + elif system.startswith("You are the synthesizer of a council"): + text = "elite synthesis" + else: + text = f"{_elite_phase(messages)} from {model}" + return ModelAnswer( + name=name, + model_id=model, + answer=text, + latency_s=len(calls) / 1000, + usage=TokenUsage(prompt_tokens=5, completion_tokens=7, total_tokens=12), + ) + + monkeypatch.setattr("conclave.council.call_model", fake_call_model) + monkeypatch.setattr("conclave.verdict_synthesis.call_model", fake_call_model) + result = await Council( + models=["grok", "gemini", "perplexity"], + config=_config(), + temperature=0.25, + timeout=37.0, + ).elite("Choose the strongest option.") + + manifest = result.manifest + assert manifest is not None + assert [receipt.phase for receipt in manifest.receipts] == ( + ["initial"] * 3 + + ["critique"] * 3 + + ["revision"] * 3 + + ["synthesis", "verdict_extraction", "verdict_repair"] + ) + first_verdict, repair = manifest.receipts[-2:] + assert (first_verdict.attempt, first_verdict.outcome, first_verdict.schema_valid) == ( + 1, + "schema_invalid", + False, + ) + assert first_verdict.error_category == "schema_validation" + assert (repair.attempt, repair.outcome, repair.schema_valid) == (2, "success", True) + assert repair.error_category is None + assert all( + receipt.generation_settings == {"temperature": 0.25, "timeout": 37.0} + for receipt in manifest.receipts + ) + assert calls == [(0.25, 37.0)] * 12 + assert all(receipt.protocol_version == ELITE_PROTOCOL_VERSION for receipt in manifest.receipts) + assert all(receipt.prompt_version is None for receipt in manifest.receipts[:3]) + assert all(receipt.prompt_version == ELITE_PROMPT_VERSION for receipt in manifest.receipts[3:9]) + assert manifest.receipts[9].prompt_version == SYNTHESIS_PROMPT_VERSION + assert all( + receipt.prompt_version == VERDICT_EXTRACTION_PROMPT_VERSION + for receipt in manifest.receipts[-2:] + ) + assert all( + receipt.schema_version == VERDICT_SCHEMA_VERSION for receipt in manifest.receipts[-2:] + ) + assert manifest.total_usage is not None + assert manifest.total_usage.total_tokens == 12 * 12 + assert manifest.total_latency_ms == sum(r.latency_ms for r in manifest.receipts) + assert manifest.total_latency_ms == 78.0 + assert manifest.estimated_cost is None + + +async def test_elite_manifest_keeps_failed_synthesis_call_without_raw_error( + monkeypatch, patch_call_model +) -> None: + """A failed synthesis is counted, categorized, and stripped of raw provider detail.""" + _all_keys(monkeypatch) + + def handler(model, messages, **kwargs): + if _system_text(messages).startswith("You are the synthesizer of a council"): + raise RuntimeError("POST https://secret.example/v1 body={'token':'do-not-store'}") + return make_response(f"{_elite_phase(messages)} from {model}") + + patch_call_model(handler) + result = await Council( + models=["grok", "gemini", "perplexity"], + config=_config(), + ).elite("Choose.") + + assert result.manifest is not None + receipt = result.manifest.receipts[-1] + assert receipt.phase == "synthesis" + assert receipt.outcome == "failed" + assert receipt.error_category == "provider_error" + assert receipt.error == "provider_error" + assert "secret.example" not in result.manifest.model_dump_json() + assert "do-not-store" not in result.manifest.model_dump_json() + + +async def test_elite_manifest_keeps_failed_verdict_attempt_and_successful_repair( + monkeypatch, patch_call_model +) -> None: + """A provider-error extraction and its repair remain distinct receipts.""" + _all_keys(monkeypatch) + verdict_attempt = 0 + + def handler(model, messages, **kwargs): + nonlocal verdict_attempt + system = _system_text(messages) + if system.startswith("You are the verdict extractor"): + verdict_attempt += 1 + if verdict_attempt == 1: + raise RuntimeError("upstream provider unavailable") + return make_response(_valid_elite_verdict_json()) + if system.startswith("You are the synthesizer of a council"): + return make_response("elite synthesis") + return make_response(f"{_elite_phase(messages)} from {model}") + + patch_call_model(handler) + result = await Council( + models=["grok", "gemini", "perplexity"], + config=_config(), + ).elite("Choose.") + + assert result.manifest is not None + failed, repaired = result.manifest.receipts[-2:] + assert (failed.phase, failed.attempt, failed.outcome, failed.error_category) == ( + "verdict_extraction", + 1, + "failed", + "provider_error", + ) + assert failed.schema_valid is None + assert (repaired.phase, repaired.attempt, repaired.outcome, repaired.schema_valid) == ( + "verdict_repair", + 2, + "success", + True, + ) + + async def test_elite_manifest_audits_every_phase(monkeypatch, patch_call_model): """A complete elite run records all nine member calls and their usage.""" _all_keys(monkeypatch) @@ -180,7 +381,7 @@ def handler(model, messages, **kwargs): assert second.cached is True _assert_verified_manifest(second, "elite") assert [receipt.phase for receipt in second.manifest.receipts] == ( - ["initial"] * 3 + ["critique"] * 3 + ["revision"] * 3 + ["initial"] * 3 + ["critique"] * 3 + ["revision"] * 3 + ["synthesis"] ) diff --git a/tests/test_output_contract_plumbing.py b/tests/test_output_contract_plumbing.py index fc7a737..bc84b89 100644 --- a/tests/test_output_contract_plumbing.py +++ b/tests/test_output_contract_plumbing.py @@ -268,7 +268,16 @@ async def test_extract_verdict_passes_native_output_contract(monkeypatch): captured = {} - async def fake_call_model(name, model_id, messages, *, config=None, output_contract=None): + async def fake_call_model( + name, + model_id, + messages, + *, + temperature=0.7, + timeout=120.0, + config=None, + output_contract=None, + ): captured["output_contract"] = output_contract return ModelAnswer(name=name, model_id=model_id, answer="not json") @@ -312,7 +321,16 @@ async def test_extract_verdict_still_degrades_gracefully_on_bad_json(monkeypatch """ import conclave.verdict_synthesis as vs - async def fake_call_model(name, model_id, messages, *, config=None, output_contract=None): + async def fake_call_model( + name, + model_id, + messages, + *, + temperature=0.7, + timeout=120.0, + config=None, + output_contract=None, + ): return ModelAnswer(name=name, model_id=model_id, answer="not json") monkeypatch.setattr(vs, "call_model", fake_call_model) diff --git a/tests/test_secret_safety_matrix.py b/tests/test_secret_safety_matrix.py index 8067329..0b84fb1 100644 --- a/tests/test_secret_safety_matrix.py +++ b/tests/test_secret_safety_matrix.py @@ -47,6 +47,7 @@ scan_for_secret_material, ) from conclave.models import ModelAnswer +from conclave.providers import receipt_from_answer from tests.conftest import make_response # The synthesizer/extractor resolved id for the "claude" friendly name below. @@ -234,6 +235,24 @@ def _assert_result_canary_free(result) -> None: assert scan_for_secret_material(result.manifest) is True +def test_receipt_reduces_raw_provider_error_to_secret_free_category() -> None: + """Receipts never retain endpoint URLs, response bodies, or echoed credentials.""" + answer = ModelAnswer( + name="grok", + model_id="xai/grok-4.3", + error=f"RuntimeError: POST https://provider.example/v1 body={PLANTED}", + ) + + receipt = receipt_from_answer(answer, temperature=0.7, timeout=120.0) + payload = receipt.model_dump_json() + + assert receipt.outcome == "failed" + assert receipt.error_category == "provider_error" + assert receipt.error == "provider_error" + assert "provider.example" not in payload + assert PLANTED not in payload + + # --------------------------------------------------------------------------- # # 1a — buffered Council.ask(synthesize=True) WITH a real verdict. # --------------------------------------------------------------------------- # diff --git a/tests/test_verdict_edge_cases.py b/tests/test_verdict_edge_cases.py index e4aa64e..e280df7 100644 --- a/tests/test_verdict_edge_cases.py +++ b/tests/test_verdict_edge_cases.py @@ -268,12 +268,17 @@ def handler(model, messages, **kwargs): assert manifest.providers_skipped == [] assert manifest.providers_called == ["grok", "gemini", "perplexity"] # Its redacted error is recorded, and it has a receipt carrying that error. - assert any("provider 503" in e or "[REDACTED]" in e for e in manifest.redacted_errors) + assert manifest.redacted_errors == ["provider_error"] assert len(manifest.redacted_errors) == 1 perplexity_receipt = next(r for r in manifest.receipts if r.name == "perplexity") - assert perplexity_receipt.error is not None - # One receipt per attempted member (success and failure alike). - assert len(manifest.receipts) == 3 + assert perplexity_receipt.error == "provider_error" + assert perplexity_receipt.error_category == "provider_error" + # Every actual call is retained: three members, prose synthesis, verdict. + assert len(manifest.receipts) == 5 + assert [receipt.phase for receipt in manifest.receipts[-2:]] == [ + "synthesis", + "verdict_extraction", + ] assert manifest.secret_safety == SECRET_SAFETY_VERIFIED From b150da83b8cca028a7bf430e4fd946b151fa4055 Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Fri, 17 Jul 2026 09:04:29 -0400 Subject: [PATCH 19/21] docs: align elite claims with verified guarantees --- CHANGELOG.md | 23 ++++++--- DOCUMENTATION_INDEX.md | 10 ++-- README.md | 51 ++++++++++--------- SYSTEM_CONTEXT_DIAGRAM.md | 16 +++--- docs/PRODUCT_DESIGN_DOCUMENT.md | 34 ++++++------- .../2026-07-17-decision-quality-roadmap.md | 20 ++++---- ...26-07-17-elite-decision-protocol-design.md | 3 ++ .../2026-07-17-elite-decision-protocol.md | 3 ++ src/conclave/council.py | 2 +- 9 files changed, 90 insertions(+), 72 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 757ad67..66c45e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,19 +10,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - **Elite Decision Protocol (unreleased).** A quality-first `elite` mode for consequential - decisions: independent member answers → concurrent council-wide evidence audits → concurrent - member revisions → the existing synthesis and canonical auditable verdict. It is available + decisions: independent member answers → concurrent council-wide claim audits → concurrent + member revisions → the existing synthesis and canonical execution-traceable verdict. It is available through `--mode elite`, `Council.elite()`, and `elite_sync()` in source; no version, tag, or package publication is claimed here. - **Fixed three-success phase gate.** Each of Elite's `initial`, `critique`, and `revision` phases requires three successful responders. Larger councils may survive partial failures while three remain. A failed gate stops later calls and returns an incomplete result with a phase-specific reason, attempted artifacts, no synthesis/verdict, and CLI exit code 1. -- **Phase-auditable results.** `EliteResult` preserves initial answers, critiques, and revisions; +- **Execution-traceable results.** `EliteResult` preserves initial answers, critiques, and revisions; the manifest records a separate redacted receipt with `phase` provenance for every attempted - member call, aggregates phase usage/latency, and retains the existing secret-safety scan. + buffered Elite call through synthesis, extraction, and repair, and aggregates reported + usage/latency. Unknown cost remains `None` instead of being guessed. Elite is buffered-only: `--stream` is rejected before provider calls. The quality tradeoff is - explicit—up to `3N + 2` calls for N members versus the ordinary single-fan-out workflow. + explicit—normally up to `3N + 2` calls for N members, or `3N + 3` when repair is attempted. +- **Completion is not readiness.** `EliteResult.completed` records successful member phases; + `decision_readiness` separately reports `ready`, `not_ready`, or `indeterminate` with reasons. + The CLI exits 1 unless an Elite decision is `ready`. +- **Version-aware cache identity.** Cache keys cover the resolved roster, generation and mode + settings, extraction behavior, sanitized endpoint routing, optional source-bundle digest, and + cache/protocol/prompt/schema versions. Old incompatible envelopes are safe misses. ### Fixed @@ -54,9 +61,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [1.1.0] - 2026-06-21 -The **auditable council**. Every run now produces a structured, agreement-scored, -fully auditable verdict plus a redacted execution manifest, on top of the existing -synthesize/raw/debate/adversarial modes. The verdict is the product wedge: a +The **auditable council** (the historical v1.1 release name). Every run produces a structured, +agreement-scored, execution-traceable verdict plus a redacted execution manifest, on top of the +existing synthesize/raw/debate/adversarial modes. The verdict is the product wedge: a multi-model council answer you can act on, with the agreement number computed by reproducible arithmetic over the model's clustering — never an LLM-emitted figure. diff --git a/DOCUMENTATION_INDEX.md b/DOCUMENTATION_INDEX.md index b3298f0..82e7cb0 100644 --- a/DOCUMENTATION_INDEX.md +++ b/DOCUMENTATION_INDEX.md @@ -41,14 +41,14 @@ Package root: `src/conclave/` (installed as the `conclave` package; console scri | Module | Path | Responsibility | |--------|------|----------------| | Package API | [`src/conclave/__init__.py`](src/conclave/__init__.py) | Public exports include `Council`, result types, verdict/manifest surface, and unreleased `EliteResult` plus Elite protocol constants. | -| Council | [`src/conclave/council.py`](src/conclave/council.py) | Six-mode async/sync API; completed Elite runs synthesize revisions and apply the canonical verdict; incomplete runs retain artifacts without either. | -| Modes | [`src/conclave/modes.py`](src/conclave/modes.py) | Debate, adversarial, vote, and unreleased Elite orchestration; Elite gates initial/audit/revision at three successes. | +| Council | [`src/conclave/council.py`](src/conclave/council.py) | Six-mode async/sync API; Elite keeps protocol completion separate from `ready`/`not_ready`/`indeterminate` decision readiness. | +| Modes | [`src/conclave/modes.py`](src/conclave/modes.py) | Debate, adversarial, vote, and unreleased Elite orchestration; Elite gates initial/claim-audit/revision at three successes. | | Verdict types | [`src/conclave/verdict.py`](src/conclave/verdict.py) | Public verdict/member Pydantic types (`CouncilVerdict`, `CouncilPosition`, `CouncilConflict`, `ProviderVote`, `MinorityReport`) + the LCD JSON Schemas (`verdict_json_schema`/`member_answer_json_schema`/`verdict_extraction_json_schema`) usable across all three native structured-output surfaces; `VERDICT_SCHEMA_VERSION`/`VERDICT_EXTRACTION_PROMPT_VERSION`. | | Agreement | [`src/conclave/agreement.py`](src/conclave/agreement.py) | Deterministic consensus: `consensus_score` (`position_cluster_ratio_v1` — largest cluster / positioned members; `None` for N<2) + `consensus_label` buckets. Pure arithmetic, no `difflib`, never LLM-emitted. | | Verdict synthesis | [`src/conclave/verdict_synthesis.py`](src/conclave/verdict_synthesis.py) | `extract_verdict` engine: one extraction call clustering stances, native `output_contract` enforcement + prompt-level fallback, validate → repair-once → graceful `verdict=None`; the three verdict-absent reasons; provenance on every return path. | -| Manifest | [`src/conclave/manifest.py`](src/conclave/manifest.py) | Secret-scanned manifest on every result; Elite receipts preserve `initial`/`critique`/`revision` phase provenance. | +| Manifest | [`src/conclave/manifest.py`](src/conclave/manifest.py) | Secret-scanned manifest on every result; buffered Elite receipts cover all attempted phases through synthesis, extraction, and repair. Unknown cost remains `None`. | | Streaming | [`src/conclave/streaming.py`](src/conclave/streaming.py) | `stream_ask` — council-level streaming engine behind `Council.ask_stream`: concurrent member interleaving via an `asyncio.Queue`, optional synthesizer streaming, terminal `done` event with the full `CouncilResult` (synthesize/raw only). | -| Prompts | [`src/conclave/prompts.py`](src/conclave/prompts.py) | Debate/adversarial/vote plus anonymized Elite evidence-audit and revision prompts. | +| Prompts | [`src/conclave/prompts.py`](src/conclave/prompts.py) | Debate/adversarial/vote plus anonymized Elite claim-audit and revision prompts; answer IDs provide within-run provenance, not external grounding. | | Providers | [`src/conclave/providers.py`](src/conclave/providers.py) | Async `call_model` (buffered) + `call_model_stream` (SSE) paths: resolve the adapter, read the key by name at call time, call transport, parse; latency/usage/redacted-error capture; never raises (partial text preserved on mid-stream failure). | | Transport | [`src/conclave/transport.py`](src/conclave/transport.py) | `post_json` + `stream_sse` — the single async httpx network boundary (buffered POST and `client.stream(...)` SSE) for the whole provider highway. | | Adapter registry | [`src/conclave/adapters/__init__.py`](src/conclave/adapters/__init__.py) | `resolve_adapter(model_id, config)` — provider registry + **extension seam** (one registration per family; config-only for OpenAI-compatible endpoints). | @@ -108,7 +108,7 @@ Run: `pytest` (config in `pyproject.toml`, `asyncio_mode = "auto"`). | Date | Change | |------|--------| | 2026-07-17 | Reframed the product roadmap around empirically proven decision quality: narrowed current claims to execution traceability, identified answer IDs as internal provenance rather than external evidence, gated Elite merge on H0 correctness, and added H1-H4 evidence, buyer, and outcome gates. | -| 2026-07-17 | Documented the implemented-but-unreleased Elite Decision Protocol: fixed three-success gates, initial/audit/revision artifacts, existing final verdict, phased receipts, failure semantics, cost/latency tradeoff, and no streaming. | +| 2026-07-17 | Documented the implemented-but-unreleased Elite Decision Protocol: fixed three-success gates, initial/claim-audit/revision artifacts, existing final verdict, phased receipts, failure semantics, cost/latency tradeoff, and no streaming. | | 2026-07-13 | **Manifest-on-every-result invariant fix + doc reconciliation.** `ModelHarnessManifest` now attaches on all five modes (was `None` for `debate`/`adversarial`/`vote`) via a single-site fix at `Council._cached_run` → new `_ensure_manifest`; regression tests in `tests/test_manifest_all_modes.py`. Docs corrected so the manifest-on-every-result claim is accurate and the `vote` mode is documented as **shipped** (CAC-09 / #3), not "absorbed" — across README, `SYSTEM_CONTEXT_DIAGRAM.md`, PDD (§1/§3/§4a/§8/§9/§12), and `CHANGELOG.md` (`[Unreleased]`). Verdict scope unchanged (still `synthesize`/`ask`-only; not layered on `adversarial`). | | 2026-06-21 | **v1.1 docs pivot — the auditable multi-model council.** PDD reframed to the auditable-council wedge (new §4a: CouncilResult v2, deterministic `position_cluster_ratio_v1` consensus, native + fallback structured output, the verdict-optional rule, the secret-free `ModelHarnessManifest`); `vote` mode marked **absorbed** by `provider_votes` across PDD §1/§4/§8/§9/§12; demand-gated v1.2 "Operable Council" roadmap. System Context diagram gains the verdict pipeline + manifest (mermaid re-validated). README gains an "Auditable verdict" section (CLI panel + library example). `CHANGELOG.md` `[1.1.0]` added. v0.x mode-detail prose archived to [`docs/archive/pdd-v0.x-modes-detail.md`](docs/archive/pdd-v0.x-modes-detail.md) to keep the PDD under 500 lines. New modules documented: `verdict.py`, `agreement.py`, `verdict_synthesis.py`, `manifest.py`. | | 2026-06-14 | v1.0.0 release. Version bump 0.3.0 → 1.0.0; new `CHANGELOG.md` (Keep-a-Changelog). Integrates the three v1.0 PRs below. | diff --git a/README.md b/README.md index 8a96ba9..80b48f5 100644 --- a/README.md +++ b/README.md @@ -15,11 +15,11 @@ returns **structured results** (per-model latency, token usage, and error captur logged. It ships six modes: **synthesize** (merge answers into one), **raw** (no merge), **debate** (multi-round, members revise after seeing peers' anonymized answers), and **adversarial** (propose → refute → verdict), **vote** (fixed-choice tally), and -**elite** (quality-first evidence audit and revision). conclave is intentionally lightweight — a +**elite** (quality-first claim audit and revision). conclave is intentionally lightweight — a small council primitive, not an agent framework. -**The v1.1 wedge — the auditable council.** Every run also yields **a multi-model council -verdict you can act on — structured, scored for agreement, fully auditable**: a +**The v1.1 wedge — the execution-traceable council.** Every run also yields **a multi-model +council verdict you can inspect — structured, scored for agreement, and execution-traceable**: a `CouncilVerdict` exposing agreement, `conflicts`, `minority_reports`, and `provider_votes`; a deterministic `consensus_score` (arithmetic over the model's clustering, *never* an LLM-emitted number); and a redacted `ModelHarnessManifest` recording how the run executed and @@ -126,7 +126,7 @@ conclave ask "Defend event sourcing for this ledger." \ conclave ask "Which datastore for this workload?" \ -c grok,gemini,claude --mode vote --choices "Postgres,DynamoDB,MongoDB" -# Elite (unreleased): independent answers -> evidence audits -> revisions -> verdict +# Elite (unreleased): independent answers -> claim audits -> revisions -> verdict conclave ask "Should we adopt a service mesh for 8 services?" \ -c grok,gemini,claude --mode elite @@ -187,10 +187,10 @@ spend for a stronger answer: three concurrent member phases, followed by conclav synthesis and canonical structured verdict. 1. **Initial:** members answer independently. -2. **Evidence audit:** every surviving member audits the anonymized panel for supported, - conflicting, and externally unverified claims. +2. **Claim audit:** every surviving member audits the anonymized panel for supported, + conflicting, and externally unverified claims. This compares answers; it does not check sources. 3. **Revision:** survivors revise their own answer using the full anonymized panel and audits. -4. **Final:** conclave synthesizes successful revisions, then applies the existing auditable +4. **Final:** conclave synthesizes successful revisions, then applies the existing structured verdict and deterministic consensus calculation. Each member phase has a fixed gate of **three successful responders**. A council larger than @@ -212,27 +212,30 @@ council = Council( result = council.elite_sync("Should we adopt a service mesh for 8 services?") # or: result = await council.elite("Should we adopt a service mesh for 8 services?") -if result.elite.completed: +if result.elite.completed and result.elite.decision_readiness == "ready": print(result.synthesis) print(result.verdict) else: - print(result.elite.failure_reason) + print(result.elite.decision_readiness, result.elite.readiness_reasons) for receipt in result.manifest.receipts: - print(receipt.phase, receipt.name, receipt.error) # initial/critique/revision + print(receipt.phase, receipt.name, receipt.outcome) ``` Elite normally makes up to `3N + 2` calls for `N` members: three concurrent member phases, -one synthesis call, and one verdict-extraction call. Failures reduce later member calls, and -`Council(extract_verdict=False)` removes the final extraction call. The manifest keeps a -separate receipt for every attempted `initial`, `critique`, and `revision` call, aggregates -their latency/usage, preserves redacted failures, and remains secret-scanned. Elite does **not** -stream. +one synthesis call, and one verdict-extraction call; a schema repair can raise that to `3N + 3`. +`Council(extract_verdict=False)` removes extraction and leaves readiness `indeterminate`. The +secret-scanned manifest records every attempted member, synthesis, extraction, and repair call +and aggregates reported latency/usage; unknown cost remains `None`. Elite does **not** stream. -## Auditable verdict +`completed` means the three member phases ran successfully; it does not mean the output is ready +to use. `decision_readiness` is separately `ready`, `not_ready`, or `indeterminate`, with +machine-readable `readiness_reasons`; the CLI exits 1 unless readiness is `ready`. + +## Execution-traceable verdict On a decision- or review-style prompt with at least two responding members, conclave -adjudicates the answers into a structured, agreement-scored, auditable **verdict**. It is +adjudicates the answers into a structured, agreement-scored, execution-traceable **verdict**. It is **default-on** — no flag needed. From the CLI, a decision prompt renders a green `VERDICT` panel: @@ -297,8 +300,8 @@ run), construct with `Council(extract_verdict=False)`; then `result.verdict` sta members' stances; `consensus_score` is then deterministic arithmetic over that clustering (largest cluster / members with a position — no text-similarity, never model-emitted). Every cluster cites the member `answer_id`s backing it (`evidence_answer_ids`), and the manifest -records which model + prompt version did the clustering, so the score is reproducible and -traceable rather than a number to take on faith. +records which model + prompt version did the clustering, so the calculation is traceable. It +measures agreement over model-assisted clusters, not truth or external factual support. **The verdict is optional.** It is absent — `result.verdict is None`, with the synthesis and member answers still returned — for an open-ended/creative prompt, for fewer than two @@ -421,11 +424,11 @@ Then: `conclave ask "..." --council fast`. Repeated or eval runs can be served from an on-disk cache instead of re-calling the providers. It is **off by default** and **never persists API keys** — the -cache key is a SHA-256 over the normalized prompt, the ordered council members -(friendly name + resolved model id), the mode, the synthesizer/judge identity, -and the mode params (temperature, debate `rounds` + `converge_threshold`, -adversarial `proposer`). No key value or env-var name ever reaches the key or the -stored payload. +cache key is a SHA-256 over a canonical identity covering prompt, mode, ordered resolved +models, synthesizer/judge, generation/mode settings, verdict-extraction behavior, sanitized +custom-endpoint fingerprints, source-bundle digest, and cache/protocol/prompt/schema versions. +No key value or raw endpoint URL reaches the identity or stored payload; incompatible older +cache envelopes are misses, not migrated results. Enable it per run with `--cache` (or disable a config default with `--no-cache`): diff --git a/SYSTEM_CONTEXT_DIAGRAM.md b/SYSTEM_CONTEXT_DIAGRAM.md index 2ea8668..b3a7159 100644 --- a/SYSTEM_CONTEXT_DIAGRAM.md +++ b/SYSTEM_CONTEXT_DIAGRAM.md @@ -5,8 +5,8 @@ the system context: how a user (or a downstream consumer) drives conclave, how c environment-variable keys feed in, how requests reach the nine first-class providers through conclave's own **provider highway** (an httpx transport + per-provider adapter registry — no LLM-SDK dependency), how the v1.1 **verdict pipeline** turns the member answers into a -structured, agreement-scored, **auditable** verdict plus a redacted execution manifest, and -how the implemented-but-unreleased **Elite Decision Protocol** adds gated evidence audit and +structured, agreement-scored, **execution-traceable** verdict plus a redacted execution manifest, and +how the implemented-but-unreleased **Elite Decision Protocol** adds gated claim audit and revision before that verdict. It also shows where **mcp-warden** sits as a dev-time consumer. > Authority note: behavioral details here are descriptive. The canonical spec is @@ -31,7 +31,7 @@ flowchart TB lib["Library API · from conclave import Council (__init__.py)"] council["Council orchestrator
fan_out · synthesize_blocks · skip-no-key (council.py)"] modes["Deliberation modes
debate · adversarial · vote (modes.py + prompts.py)"] - elite["Elite (UNRELEASED)
initial → evidence audit → revision
fixed 3-success gate each phase"] + elite["Elite (UNRELEASED)
initial → claim audit → revision
fixed 3-success gate each phase"] registry["Registry · name to model-id
key PRESENCE only, never values (registry.py)"] config["Config loader · custom endpoints (config.py)"] models["Result contract · CouncilResult v2
answers · verdict · consensus · manifest (models.py)"] @@ -136,7 +136,7 @@ flowchart TB speaks native `generateContent` (OpenAI roles mapped, `systemInstruction` hoisted, `usageMetadata` parsed). Every adapter builds a request and hands it to the **single** network boundary — `transport.post_json` (`transport.py`), one async httpx call site. -- **The verdict pipeline is default-on and auditable (PDD §4a).** Once the council has the +- **The verdict pipeline is default-on and execution-traceable (PDD §4a).** Once the council has the member answers, `_apply_verdict` (`council.py`, run on both the buffered and streaming paths, *after* the manifest exists) drives `extract_verdict` (`verdict_synthesis.py`): a **single** extraction call asks the synthesizer model to *cluster* the members' stances — @@ -159,11 +159,13 @@ flowchart TB prompts, fewer than two responding members, or extraction failure leave `verdict = None` with the synthesis and member answers intact and the reason recorded on the manifest. - **Elite is quality-first and unreleased.** Its three concurrent member phases are independent - answers, council-wide anonymized evidence audits, and member revisions. Each phase requires + answers, council-wide anonymized claim audits, and member revisions. Answer IDs trace outputs + within the run; they are not external citations. Each phase requires three successful responders. Larger councils may lose members and continue while three remain; a failed gate stops later calls and returns an incomplete result with artifacts and phased - receipts for attempted work, but no synthesis or verdict. Completed runs feed revisions into - the existing synthesis and verdict pipeline. This can cost up to `3N + 2` calls and more + receipts for attempted work, but no synthesis or verdict. Completion is separate from decision + readiness (`ready`/`not_ready`/`indeterminate`). Completed runs feed revisions into the existing + synthesis and verdict pipeline. This normally costs up to `3N + 2` calls (`3N + 3` with repair) and more wall-clock time than ordinary modes; `--stream` is rejected because Elite is buffered only. - **Streaming shares the same boundary (PDD §9 #5).** A `--stream` run (and the library `Council.ask_stream` async generator) flows through a streaming sibling of the call path: diff --git a/docs/PRODUCT_DESIGN_DOCUMENT.md b/docs/PRODUCT_DESIGN_DOCUMENT.md index 53aa76d..7006eef 100644 --- a/docs/PRODUCT_DESIGN_DOCUMENT.md +++ b/docs/PRODUCT_DESIGN_DOCUMENT.md @@ -42,7 +42,7 @@ response; v0.2 adds **council modes** — **debate** (multi-round) and **adversa (propose → refute → verdict) — that turn a flat panel of opinions into structured deliberation. -**The v1.1 wedge — the auditable council.** A synthesis paragraph is not enough to *act* on. +**The v1.1 wedge — the historically named "auditable council."** A synthesis paragraph is not enough to *act* on. v1.1 makes the product identity precise: **a multi-model council verdict you can inspect — structured, scored for agreement, and execution-traceable.** Every run yields a `CouncilVerdict` exposing agreement, disagreement (`conflicts`), minority views (`minority_reports`), and @@ -58,7 +58,7 @@ conclave's first real use was an **adversarial design review**: a council of Gro Perplexity, and Claude critiquing a security-tool strategy and catching flaws a single model missed. That origin is why the adversarial and debate modes are first-class — they are now built, not a bolt-on. The product stays lightweight: a **library-first primitive -with structured, auditable results**, not an agent framework and not a general AI SDK — it +with structured, execution-traceable results**, not an agent framework and not a general AI SDK — it builds only what deepens the council wedge (the boundary vs. LiteLLM/Vercel/LangChain/ Helicone is in §11). @@ -129,7 +129,7 @@ output. The v1.1 verdict layer (§4a) sits on top of whichever mode produced the | **debate** | **BUILT (v0.2)** | N rounds (`--rounds`, default 2). Round 1 is an independent fan-out; rounds 2..N show each member its peers' **anonymized** prior-round answers (`Model A/B/C`) and ask it to revise or defend. A member that errors in a round drops out of later rounds; the debate continues with survivors. The synthesizer consolidates the final round. Exposed as `--mode debate` / `Council.debate()` / `debate_sync()`. | | **adversarial** | **BUILT (v0.2)** | Structured propose → refute → verdict. A `--proposer` (default: first member) answers; the remaining members are CRITICS explicitly prompted to refute it; the synthesizer acts as JUDGE, weighing proposal vs. critiques and issuing a verdict + strengthened answer. This is the mode conclave's origin story (the security design review) exercised by hand. Exposed as `--mode adversarial` / `Council.adversarial()` / `adversarial_sync()`. | | **vote** | **BUILT (v1.1, CAC-09 / #3)** | Constrained-choice ballot: each member sees a fixed labelled option set (`A, B, C, …`) and answers with one letter; responses are tallied to a plurality `winner` (or `split` on a tie) on `result.vote` (`VoteResult`). Exposed as `--mode vote --choices "A,B,C"` / `Council.vote()` / `vote_sync()`. Distinct from the verdict's `provider_votes`, which cluster free-form stances with evidence (§4a); `vote` tallies a fixed ballot. | -| **elite** | **IMPLEMENTED, UNRELEASED** | Quality-first decision protocol: independent fan-out → concurrent council-wide evidence audits → concurrent member revisions → existing synthesis and canonical verdict. Every member phase has a fixed three-success gate. Exposed in source as `--mode elite` / `Council.elite()` / `elite_sync()`; buffered only, no streaming. | +| **elite** | **IMPLEMENTED, UNRELEASED** | Quality-first decision protocol: independent fan-out → concurrent council-wide claim audits → concurrent member revisions → existing synthesis and canonical verdict. Every member phase has a fixed three-success gate. Exposed in source as `--mode elite` / `Council.elite()` / `elite_sync()`; buffered only, no streaming. | **Mode algorithms (as built).** The step-by-step "as built" prose for synthesize / raw / debate / adversarial / vote (fan-out + partial-results, peer anonymization + drop-out, proposer → @@ -143,15 +143,16 @@ stochastic reconciliation (load-bearing for the mcp-warden boundary, §10); the adds a *deterministic* agreement number on top, by arithmetic over a clustering. **Elite gate and partial-failure contract.** `required_responders` is fixed at 3. Councils may start larger, but only members successful in a phase advance. One failure in a four-member council is tolerated; fewer than three successes in `initial`, `critique`, or `revision` stops the protocol immediately. -The result is incomplete with a phase-specific `failure_reason`, no later calls, synthesis, or verdict; attempted phase artifacts and redacted failures remain serialized and the CLI exits 1. A completed run mirrors successful revisions to `answers`. -Compared with ordinary modes, Elite deliberately trades latency and cost for decision quality: up to `3N + 2` calls for N members (three member phases, synthesis, verdict extraction), or `3N + 1` with verdict extraction disabled. +The result is incomplete with a phase-specific `failure_reason`, no later calls, synthesis, or verdict; attempted phase artifacts and redacted failures remain serialized and the CLI exits 1. A completed run mirrors successful revisions to `answers`, but completion only means the three member phases passed. +Decision readiness is separate: `ready`, `not_ready`, or `indeterminate`, plus machine-readable `readiness_reasons`; synthesis or required-adjudication failure is not ready, and disabled or inapplicable adjudication is indeterminate. The CLI exits 1 unless readiness is `ready`. Elite normally uses up to `3N + 2` calls, `3N + 3` if verdict repair runs, or `3N + 1` with extraction disabled. +Cache identity is version-aware and secret-free: protocol/prompt/schema/cache versions, resolved model roster, generation and mode settings, extraction behavior, sanitized endpoint routing, and optional source-bundle digest all invalidate incompatible entries; old envelopes miss safely. --- -## 4a. The Auditable Verdict (v1.1) +## 4a. The Execution-Traceable Verdict (v1.1) The verdict layer turns a council run's answers into a structured, agreement-scored, -auditable adjudication — on top of any mode, default-on, never breaking the v0.1 surface +execution-traceable adjudication — on top of any mode, default-on, never breaking the v0.1 surface (every new field defaults to `None`/empty). ### CouncilResult v2 surface @@ -182,7 +183,7 @@ cites `evidence_answer_ids` (the member `answer_id`s backing it) and every confl conflict that just says "models disagreed about cost" without pointing at answers is a *failure*. `ProviderVote.confidence` is recorded but **never used in arithmetic**. -### Deterministic consensus — the auditability fix +### Deterministic consensus — the traceability fix The consensus number is **arithmetic over the model's clustering, never LLM-emitted** (`agreement.py`, method `position_cluster_ratio_v1`). @@ -264,14 +265,13 @@ and cache hits (synthesize/raw builds its own richer one earlier). Pinned by a `ProviderExecutionReceipt{name, provider, model_id, generation_settings, latency_ms, usage, error(redacted), schema_valid}`), `total_latency_ms`, `total_usage`, `schema_valid`, `redacted_errors`, and verdict-provenance slots (`verdict_extraction: VerdictExtraction{model_id, -prompt_version}` — the auditability hook — plus `verdict_type`, `consensus_method`, +prompt_version}` — the execution-trace hook — plus `verdict_type`, `consensus_method`, `verdict_absent_reason`). Two deliberate honesty choices: -For Elite, every attempted member call becomes a separate `initial`, `critique`, or `revision` receipt; repeated providers have repeated receipts while `providers_called` stays unique. Incomplete runs retain only attempted phases. Total usage/latency covers all recorded member phases; synthesis and verdict provenance retain their existing fields. +For buffered Elite, every attempted call becomes a receipt: `initial`, `critique`, `revision`, `synthesis`, `verdict_extraction`, and `verdict_repair` when repair is attempted. Receipts carry phase, attempt/outcome, provider/model identity, latency, available usage/cost, bounded error category, and prompt/schema/protocol versions; totals are recomputed from this complete ledger. Incomplete runs retain only calls actually attempted. -- **No invented pricing.** `estimated_cost` stays `None` (a wrong number in an audit receipt is - worse than none); `pricing_snapshot_date` is the dated-estimate slot until a real pricing table - exists. Usage (tokens) is recorded; cost is not guessed. +- **No invented pricing.** Unknown per-call or aggregate `estimated_cost` stays `None`; a total is + computed only when every actual call has trustworthy priced data. Usage is recorded when reported. - **Proven secret-safety.** `secret_safety` defaults to `unverified`, promoted to `verified_no_secrets` **only** after `scan_for_secret_material()` proves the serialized manifest free of forbidden substrings (`sk-`, `bearer`, `authorization`, `api_key`, `x-api-key`). Key @@ -327,9 +327,9 @@ consumers. The end-to-end flow — `CLI/Library → Council → call_model → a | `verdict.py` | Public verdict/member Pydantic types (`CouncilVerdict`, `CouncilPosition`, `CouncilConflict`, `ProviderVote`, `MinorityReport`) + the LCD JSON Schemas (`verdict_json_schema`/`member_answer_json_schema`/`verdict_extraction_json_schema`) usable across all three native structured-output surfaces; `VERDICT_SCHEMA_VERSION`. | | `agreement.py` | Deterministic consensus: `consensus_score` (`position_cluster_ratio_v1` — largest cluster / positioned members; `None` for N<2) + `consensus_label` buckets. Pure arithmetic, no `difflib`, never LLM-emitted. | | `verdict_synthesis.py` | `extract_verdict` engine: one extraction call (clusters stances, never emits a number), native `output_contract` enforcement + prompt-level fallback, validate → repair-once → graceful `verdict=None`; the three verdict-absent reasons; provenance on every return path. | -| `manifest.py` | `ModelHarnessManifest` (first-class on every result), phased `ProviderExecutionReceipt`/`ProviderSkip`/`VerdictExtraction`, and `scan_for_secret_material()` → `secret_safety` stamp. No key values; `estimated_cost` left `None`. | +| `manifest.py` | `ModelHarnessManifest` (first-class on every result), phased `ProviderExecutionReceipt`/`ProviderSkip`/`VerdictExtraction`, and `scan_for_secret_material()` → `secret_safety` stamp. No key values; unknown `estimated_cost` remains `None`. | | `modes.py` | Deliberation orchestration: `run_debate`, `run_adversarial`, `run_vote`, and unreleased `run_elite` (three gated member phases). Built on `Council.fan_out` + `synthesize_blocks`. | -| `prompts.py` | Role/template strings for debate, adversarial, vote, and Elite evidence-audit/revision prompts. Elite panel text uses stable Model A/B/C aliases and answer ids, never provider identities. | +| `prompts.py` | Role/template strings for debate, adversarial, vote, and Elite claim-audit/revision prompts. Elite panel text uses stable Model A/B/C aliases and answer ids, never provider identities. | | `providers.py` | `call_model` (+ `call_model_stream`) — the single async call path: resolve adapter, read key *by name at call time*, call adapter+transport (with an optional `output_contract`), parse, capture latency/usage/redacted-error into a `ModelAnswer`; never raises for provider-side failures. | | `transport.py` | The single async network boundary: `post_json` (buffered) + `stream_sse` (issue #7) — the only two httpx call sites in the highway. | | `streaming.py` | Streaming engine (issue #7): `stream_ask` interleaves members via an `asyncio.Queue`, optionally streams the synthesizer, ends with a `done` result. Synthesize/raw only; Elite explicitly rejects streaming. | @@ -390,7 +390,7 @@ Condensed history (v0.x mode-detail archived per §4, per-release changelog in ` - **Not a runtime adjudicator.** conclave is stochastic; it must not be a deterministic decision gate (§10). **Permanent** for synthesize/debate/adversarial — *and* for the v1.1 verdict: the verdict's *clustering* is LLM-assisted (stochastic), so even the deterministic - `consensus_score` is not a reproducible security gate. The number is auditable, not authoritative. + `consensus_score` is not a reproducible security gate. The calculation is traceable, not authoritative. - **Not an agent framework.** No tool-calling graphs, stateful agents, or orchestration DSL — we compete by being *small*. (Permanent.) - **Not a key manager / secrets vault.** conclave reads env vars; it does not provision, rotate, store, or proxy keys. (Permanent.) - **No hosted/proxied token path.** No conclave-operated endpoint that sees prompts or takes a margin — BYO-keys, direct-to-provider, always. (Permanent.) @@ -406,7 +406,7 @@ council shipped in v1.1** (§4a — the wedge). The revised thesis is a **source-grounded, execution-traceable decision record with empirically proven quality**. Current answer IDs identify model outputs, not external evidence; -"fully auditable" is therefore too broad until source grounding ships. Elite remains +source-auditable language is therefore too broad until source grounding ships. Elite remains implemented but unreleased on draft PR #51. The canonical roadmap is diff --git a/docs/plans/2026-07-17-decision-quality-roadmap.md b/docs/plans/2026-07-17-decision-quality-roadmap.md index 70e0c2a..adbe108 100644 --- a/docs/plans/2026-07-17-decision-quality-roadmap.md +++ b/docs/plans/2026-07-17-decision-quality-roadmap.md @@ -1,6 +1,6 @@ # Conclave Decision Quality Roadmap -**Status:** Approved product direction; implementation is gated by the evidence below +**Status:** Approved direction; Horizon 0 implemented on the draft branch, pending full gate review **Date:** 2026-07-17 **Current product:** v1.1.0 stable; Elite implemented but unreleased on draft PR #51 @@ -12,11 +12,11 @@ not "more agents." It is a disciplined path from source material to competing cl critique, decision readiness, and an exportable brief, with receipts that let a reviewer reconstruct what happened. -That wording is intentionally narrower than the current "fully auditable" language. Today, +That wording is intentionally narrower than source-auditable product language. Today, Conclave traces model execution and the model-assisted clustering behind a verdict. Its `evidence_answer_ids` identify council answers, **not external evidence**. Until source grounding and deterministic citation validation ship, the honest claim is -**execution-traceable**, not source-auditable or fully auditable. +**execution-traceable**, not source-auditable. ## Product doctrine @@ -30,16 +30,16 @@ grounding and deterministic citation validation ship, the honest claim is ## Horizon 0 — verify Elite correctness before merge -Elite remains implemented but unreleased. Draft PR #51 must remain draft until these -correctness gaps are resolved and regression-tested: +Elite remains implemented but unreleased. The draft branch now implements these contracts; +draft PR #51 must remain draft until the complete merge gate verifies them: -1. **Persistent identities.** Give each initial answer a stable identifier that survives +1. **Persistent identities.** Each initial answer has a stable identifier that survives claim audit, revision, synthesis, serialization, and cache replay. Do not present those IDs as external citations. -2. **Two states, not one.** Separate `protocol_completion` (all required phases executed) +2. **Two states, not one.** Separate `completed` (all required member phases succeeded) from `decision_readiness`. Readiness is `ready`, `not_ready`, or `indeterminate`, with machine-readable reasons. A completed run may still be `not_ready`. -3. **Full-call receipts.** Manifest totals must include member, audit, revision, synthesis, +3. **Full-call receipts.** Buffered Elite manifest totals include member, claim-audit, revision, synthesis, repair, and verdict-extraction calls, with phase, latency, usage, errors, and prompt/schema versions. Never imply completeness when final calls are absent from totals. 4. **Version-aware cache.** Cache identity must cover protocol, prompt, schema, model, @@ -47,8 +47,8 @@ correctness gaps are resolved and regression-tested: artifacts cannot replay as current. 5. **Custom config threading.** Every Elite phase, including synthesis and verdict extraction, must receive the same resolved custom-endpoint/provider configuration. -6. **Correct language.** Rename "evidence audit" where necessary to "answer/claim audit" - until external sources exist. Product copy must say execution-traceable. Answer IDs prove +6. **Correct language.** Use "answer/claim audit" until external sources exist. Product copy + must say execution-traceable. Answer IDs prove provenance within a run; they do not prove truth. **Merge gate:** all six items have tests, the complete suite and static checks pass, a diff --git a/docs/plans/2026-07-17-elite-decision-protocol-design.md b/docs/plans/2026-07-17-elite-decision-protocol-design.md index e45d3af..d04fa12 100644 --- a/docs/plans/2026-07-17-elite-decision-protocol-design.md +++ b/docs/plans/2026-07-17-elite-decision-protocol-design.md @@ -1,5 +1,8 @@ # Elite Decision Protocol Design +> **Historical design (superseded):** terminology and receipt scope below record the initial +> design. The current contracts are defined by the PDD and the H0 hardening plan. + ## Outcome Add a first-class `elite` council mode optimized for consequential decisions. Elite runs diff --git a/docs/plans/2026-07-17-elite-decision-protocol.md b/docs/plans/2026-07-17-elite-decision-protocol.md index 592eadc..12ec8c0 100644 --- a/docs/plans/2026-07-17-elite-decision-protocol.md +++ b/docs/plans/2026-07-17-elite-decision-protocol.md @@ -1,5 +1,8 @@ # Elite Decision Protocol Implementation Plan +> **Historical implementation plan (superseded):** terminology and receipt scope below record +> the initial build. The current contracts are defined by the PDD and H0 hardening plan. + > **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. **Goal:** Add a three-stage, minimum-three-member Elite Decision Protocol that produces a stronger evidence-audited decision through conclave's existing auditable verdict. diff --git a/src/conclave/council.py b/src/conclave/council.py index 241f499..3206315 100644 --- a/src/conclave/council.py +++ b/src/conclave/council.py @@ -972,7 +972,7 @@ async def adversarial(self, prompt: str, proposer: str | None = None) -> Council ) async def elite(self, prompt: str) -> CouncilResult: - """Run the evidence-audited Elite Decision Protocol. + """Run the answer/claim-audited Elite Decision Protocol. Completed runs synthesize the council's revised answers and apply the canonical structured verdict. A run that fails any three-responder gate From 23e8fce437172c040084d31dcea4ed5737ebae2a Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Fri, 17 Jul 2026 09:05:45 -0400 Subject: [PATCH 20/21] docs: tighten verdict and receipt guarantees --- README.md | 5 +++-- docs/PRODUCT_DESIGN_DOCUMENT.md | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 80b48f5..77bc29b 100644 --- a/README.md +++ b/README.md @@ -18,8 +18,9 @@ logged. It ships six modes: **synthesize** (merge answers into one), **raw** (no **elite** (quality-first claim audit and revision). conclave is intentionally lightweight — a small council primitive, not an agent framework. -**The v1.1 wedge — the execution-traceable council.** Every run also yields **a multi-model -council verdict you can inspect — structured, scored for agreement, and execution-traceable**: a +**The v1.1 wedge — the execution-traceable council.** Decision/review synthesis runs can yield +**a multi-model council verdict you can inspect — structured, scored for agreement, and +execution-traceable**: a `CouncilVerdict` exposing agreement, `conflicts`, `minority_reports`, and `provider_votes`; a deterministic `consensus_score` (arithmetic over the model's clustering, *never* an LLM-emitted number); and a redacted `ModelHarnessManifest` recording how the run executed and diff --git a/docs/PRODUCT_DESIGN_DOCUMENT.md b/docs/PRODUCT_DESIGN_DOCUMENT.md index 7006eef..2582094 100644 --- a/docs/PRODUCT_DESIGN_DOCUMENT.md +++ b/docs/PRODUCT_DESIGN_DOCUMENT.md @@ -262,8 +262,9 @@ and cache hits (synthesize/raw builds its own richer one earlier). Pinned by `tests/test_manifest_all_modes.py`. It records `request_id`, `conclave_version`, `mode`, `providers_considered/called/skipped` (each skip a `ProviderSkip{name, reason}`), `model_ids`, `generation_settings`, `receipts` (each -a `ProviderExecutionReceipt{name, provider, model_id, generation_settings, latency_ms, usage, -error(redacted), schema_valid}`), `total_latency_ms`, `total_usage`, `schema_valid`, +a `ProviderExecutionReceipt{phase, attempt, outcome, name, provider, model_id, +generation_settings, latency_ms, usage, error_category, schema_valid, versions}`), +`total_latency_ms`, `total_usage`, `schema_valid`, `redacted_errors`, and verdict-provenance slots (`verdict_extraction: VerdictExtraction{model_id, prompt_version}` — the execution-trace hook — plus `verdict_type`, `consensus_method`, `verdict_absent_reason`). Two deliberate honesty choices: From b72645806912f8407b195acdd70142288d31214e Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Sat, 18 Jul 2026 19:33:12 -0400 Subject: [PATCH 21/21] fix(cache): preserve exact prompt identity --- DOCUMENTATION_INDEX.md | 14 +++++----- README.md | 24 ++++++++-------- SYSTEM_CONTEXT_DIAGRAM.md | 16 ++++++----- docs/PRODUCT_DESIGN_DOCUMENT.md | 28 ++++++++++--------- .../2026-07-17-decision-quality-roadmap.md | 8 +++--- src/conclave/cache.py | 22 ++++----------- src/conclave/cli.py | 8 ++++-- src/conclave/verdict_synthesis.py | 9 +++--- tests/test_cache.py | 21 ++++++++++++-- tests/test_cli.py | 10 +++---- 10 files changed, 86 insertions(+), 74 deletions(-) diff --git a/DOCUMENTATION_INDEX.md b/DOCUMENTATION_INDEX.md index 82e7cb0..3228a29 100644 --- a/DOCUMENTATION_INDEX.md +++ b/DOCUMENTATION_INDEX.md @@ -15,15 +15,15 @@ the canonical authority spec on top of those. | # | Doc | Path | Purpose | |---|-----|------|---------| -| 1 | **README** (project overview) | [`README.md`](README.md) | Install, BYO-keys, six-mode CLI/library quickstart, and the unreleased quality-first Elite protocol. | -| 2 | **System Context Diagram** | [`SYSTEM_CONTEXT_DIAGRAM.md`](SYSTEM_CONTEXT_DIAGRAM.md) | Mermaid context for the provider highway, six modes, unreleased Elite gated phases, verdict pipeline, phased manifest receipts, and mcp-warden boundary. | +| 1 | **README** (project overview) | [`README.md`](README.md) | Install, BYO-keys, five released-mode CLI/library quickstart, and the source-only unreleased Elite protocol. | +| 2 | **System Context Diagram** | [`SYSTEM_CONTEXT_DIAGRAM.md`](SYSTEM_CONTEXT_DIAGRAM.md) | Mermaid context for the provider highway, five released modes, source-only Elite gated phases, verdict pipeline, phased manifest receipts, and mcp-warden boundary. | | 3 | **Documentation Index** | [`DOCUMENTATION_INDEX.md`](DOCUMENTATION_INDEX.md) | This file. Master map of all docs + source layout. | ## Authority spec | Doc | Path | Purpose | |-----|------|---------| -| **Product Design Document** | [`docs/PRODUCT_DESIGN_DOCUMENT.md`](docs/PRODUCT_DESIGN_DOCUMENT.md) | **Canonical** product spec: six modes including implemented-but-unreleased Elite, its fixed three-success phase gate and cost/latency tradeoff, the v1.1 execution-traceable verdict, manifest, architecture, boundaries, and roadmap. **When docs disagree, the PDD wins.** | +| **Product Design Document** | [`docs/PRODUCT_DESIGN_DOCUMENT.md`](docs/PRODUCT_DESIGN_DOCUMENT.md) | **Canonical** product spec: five released modes plus implemented-but-unreleased Elite, its fixed three-success phase gate and cost/latency tradeoff, the v1.1 execution-traceable verdict, manifest, architecture, boundaries, and roadmap. **When docs disagree, the PDD wins.** | ## Product plans @@ -41,11 +41,11 @@ Package root: `src/conclave/` (installed as the `conclave` package; console scri | Module | Path | Responsibility | |--------|------|----------------| | Package API | [`src/conclave/__init__.py`](src/conclave/__init__.py) | Public exports include `Council`, result types, verdict/manifest surface, and unreleased `EliteResult` plus Elite protocol constants. | -| Council | [`src/conclave/council.py`](src/conclave/council.py) | Six-mode async/sync API; Elite keeps protocol completion separate from `ready`/`not_ready`/`indeterminate` decision readiness. | +| Council | [`src/conclave/council.py`](src/conclave/council.py) | Five released async/sync modes plus source-only Elite; Elite keeps protocol completion separate from `ready`/`not_ready`/`indeterminate` decision readiness. | | Modes | [`src/conclave/modes.py`](src/conclave/modes.py) | Debate, adversarial, vote, and unreleased Elite orchestration; Elite gates initial/claim-audit/revision at three successes. | | Verdict types | [`src/conclave/verdict.py`](src/conclave/verdict.py) | Public verdict/member Pydantic types (`CouncilVerdict`, `CouncilPosition`, `CouncilConflict`, `ProviderVote`, `MinorityReport`) + the LCD JSON Schemas (`verdict_json_schema`/`member_answer_json_schema`/`verdict_extraction_json_schema`) usable across all three native structured-output surfaces; `VERDICT_SCHEMA_VERSION`/`VERDICT_EXTRACTION_PROMPT_VERSION`. | | Agreement | [`src/conclave/agreement.py`](src/conclave/agreement.py) | Deterministic consensus: `consensus_score` (`position_cluster_ratio_v1` — largest cluster / positioned members; `None` for N<2) + `consensus_label` buckets. Pure arithmetic, no `difflib`, never LLM-emitted. | -| Verdict synthesis | [`src/conclave/verdict_synthesis.py`](src/conclave/verdict_synthesis.py) | `extract_verdict` engine: one extraction call clustering stances, native `output_contract` enforcement + prompt-level fallback, validate → repair-once → graceful `verdict=None`; the three verdict-absent reasons; provenance on every return path. | +| Verdict synthesis | [`src/conclave/verdict_synthesis.py`](src/conclave/verdict_synthesis.py) | `extract_verdict` engine: one initial extraction call and at most one repair, native `output_contract` enforcement + prompt-level fallback, validate → repair-once → graceful `verdict=None`; the three verdict-absent reasons; provenance on every return path. | | Manifest | [`src/conclave/manifest.py`](src/conclave/manifest.py) | Secret-scanned manifest on every result; buffered Elite receipts cover all attempted phases through synthesis, extraction, and repair. Unknown cost remains `None`. | | Streaming | [`src/conclave/streaming.py`](src/conclave/streaming.py) | `stream_ask` — council-level streaming engine behind `Council.ask_stream`: concurrent member interleaving via an `asyncio.Queue`, optional synthesizer streaming, terminal `done` event with the full `CouncilResult` (synthesize/raw only). | | Prompts | [`src/conclave/prompts.py`](src/conclave/prompts.py) | Debate/adversarial/vote plus anonymized Elite claim-audit and revision prompts; answer IDs provide within-run provenance, not external grounding. | @@ -59,7 +59,7 @@ Package root: `src/conclave/` (installed as the `conclave` package; console scri | Registry | [`src/conclave/registry.py`](src/conclave/registry.py) | Friendly-name → model-id defaults; provider → env-var mapping; key **presence** logic (never values). | | Config | [`src/conclave/config.py`](src/conclave/config.py) | Loads/merges `~/.conclave/config.yml` over defaults; resolves model ids and named/CSV councils; parses the `endpoints:` section (custom OpenAI-compatible providers). | | Models | [`src/conclave/models.py`](src/conclave/models.py) | Stable Pydantic contract, including `EliteResult` phase artifacts and backward-compatible `CouncilResult.elite`. | -| CLI | [`src/conclave/cli.py`](src/conclave/cli.py) | Six-mode `conclave ask` plus `providers`; Elite adds a phase summary, full JSON, incomplete exit 1, and explicit stream rejection. | +| CLI | [`src/conclave/cli.py`](src/conclave/cli.py) | Five released `conclave ask` modes plus source-only Elite and `providers`; Elite exits 1 unless decision readiness is `ready`, emits full JSON before failure, and rejects streaming. | | Logging | [`src/conclave/logging.py`](src/conclave/logging.py) | Logger factory; stderr; verbosity via `CONCLAVE_LOG_LEVEL` (default `WARNING`). | ## Tests @@ -73,7 +73,7 @@ Package root: `src/conclave/` (installed as the `conclave` package; console scri | Adapter tests | [`tests/test_adapters.py`](tests/test_adapters.py) | Per-adapter `build_request` + `parse_response` for openai-compat/anthropic/gemini: system-hoist, max_tokens, role mapping, usage parsing, empty/malformed/error-status raises. | | Provider highway tests | [`tests/test_providers.py`](tests/test_providers.py) | `resolve_adapter` (built-in prefixes, per-provider URLs, custom endpoints, unknown-prefix raise), end-to-end `call_model`, and `redact()` (bearer/`sk-`/env-var-value/`x-api-key` scrubbing; pre-redacted provider errors). | | Registry/config tests | [`tests/test_registry_config.py`](tests/test_registry_config.py) | Name resolution, key-presence logic, config merge. | -| CLI tests | [`tests/test_cli.py`](tests/test_cli.py) | Typer `CliRunner`: exit-code contract (0 success / 1 zero-usable-answers / 2 usage error), `--json` payload + exit code, human renderers per mode, `providers` table never prints secrets, aclose lifecycle. | +| CLI tests | [`tests/test_cli.py`](tests/test_cli.py) | Typer `CliRunner`: exit-code contract (0 usable result and ready Elite / 1 zero-usable or non-ready Elite / 2 usage error), `--json` payload + exit code, human renderers per mode, `providers` table never prints secrets, aclose lifecycle. | | Transport tests | [`tests/test_transport.py`](tests/test_transport.py) | `post_json` via httpx `MockTransport`: success/error-status/non-JSON fallback, timeout & connect/HTTP errors → `TransportError` (key never leaks), client reuse/pooling, aclose idempotency. | | Streaming tests | [`tests/test_streaming.py`](tests/test_streaming.py) | Per-adapter SSE via `MockTransport` (openai-compat/anthropic/gemini): incremental chunks + assembled answer == concatenation == buffered result; mid-stream malformed-frame/connection-drop/non-2xx → error set with partial text preserved (never raises); key redaction in stream errors; buffered `ask()` never opens a stream; `Council.ask_stream` interleaving + terminal `done` shape; CLI `--stream` smoke + exit-code contract + debate rejection; `--stream` + cache one-shot replay. | | Logging tests | [`tests/test_logging.py`](tests/test_logging.py) | `CONCLAVE_LOG_LEVEL` resolution (default `WARNING`, case-insensitive, unknown → `WARNING`), factory contract, one-shot configuration. | diff --git a/README.md b/README.md index 77bc29b..59a3613 100644 --- a/README.md +++ b/README.md @@ -12,11 +12,11 @@ It is **library-first** (the CLI is a thin shell over the same `Council` you imp returns **structured results** (per-model latency, token usage, and error capture), and is **partial-failure resilient** — one provider erroring never aborts the run. Keys are **bring-your-own**, referenced by environment-variable *name* only — never stored or -logged. It ships six modes: **synthesize** (merge answers into one), **raw** (no merge), +logged. Published v1.1 ships five modes: **synthesize** (merge answers into one), **raw** (no merge), **debate** (multi-round, members revise after seeing peers' anonymized answers), and -**adversarial** (propose → refute → verdict), **vote** (fixed-choice tally), and -**elite** (quality-first claim audit and revision). conclave is intentionally lightweight — a -small council primitive, not an agent framework. +**adversarial** (propose → refute → verdict), and **vote** (fixed-choice tally). This source +branch also implements the unreleased **elite** quality-first claim-audit and revision mode. +conclave is intentionally lightweight — a small council primitive, not an agent framework. **The v1.1 wedge — the execution-traceable council.** Decision/review synthesis runs can yield **a multi-model council verdict you can inspect — structured, scored for agreement, and @@ -25,8 +25,7 @@ execution-traceable**: a a deterministic `consensus_score` (arithmetic over the model's clustering, *never* an LLM-emitted number); and a redacted `ModelHarnessManifest` recording how the run executed and which model produced the disagreement analysis. The verdict is **default-on**, and the manifest -rides on **every** result — for all six modes (`synthesize`, `raw`, `debate`, `adversarial`, -`vote`, `elite`), not just `ask`. A constrained-choice **`vote` mode** (`--mode vote --choices ...`) also +rides on **every** released-mode result; source-only Elite results carry it too. A constrained-choice **`vote` mode** (`--mode vote --choices ...`) also shipped in v1.1 (CAC-09 / #3) — distinct from the verdict's `provider_votes`, which score free-form agreement rather than tally a fixed ballot. @@ -135,14 +134,16 @@ conclave ask "Should we adopt a service mesh for 8 services?" \ conclave ask "..." -c grok,perplexity --mode debate --json ``` -Mode flags at a glance: `--mode synthesize|raw|debate|adversarial|vote|elite`. `--rounds N` +Published v1.1 mode flags are `--mode synthesize|raw|debate|adversarial|vote`; this source +branch additionally exposes `--mode elite`. `--rounds N` (default 2) is the *maximum* round count for `debate`; `--converge-threshold FLOAT` (or `--converge`/`--no-converge`) optionally stops a debate early once answers stabilize round-over-round (off by default — `--rounds` runs in full). `--proposer NAME` (default: first member) applies to `adversarial`. `--choices "A,B,C"` (two or more) is required for `vote`. `--synthesizer/-s` overrides the synthesizer *and* the -adversarial judge and Elite's final synthesizer. Every mode's result carries the auditable -`ModelHarnessManifest`. Elite is currently **unreleased**; use it from this source branch until +adversarial judge and, on this source branch, Elite's final synthesizer. Every released mode's +result carries the auditable `ModelHarnessManifest`; source-only Elite results do too. Elite is +currently **unreleased**; use it from this source branch until a future release explicitly includes it. `--council` accepts either a comma-separated list of friendly names or the name @@ -293,8 +294,9 @@ print(result.manifest.verdict_extraction.model_id) # which model produced the print(result.manifest.secret_safety) # "verified_no_secrets" ``` -To opt out (e.g. for cost-sensitive runs — the verdict adds one extra synthesizer call per -run), construct with `Council(extract_verdict=False)`; then `result.verdict` stays `None`. +To opt out (e.g. for cost-sensitive runs — the verdict adds one initial synthesizer call and, +when schema repair is needed, one retry), construct with `Council(extract_verdict=False)`; +then `result.verdict` stays `None`. `conclave ask ... --json` already carries the full structured `verdict` and `manifest`. **How the consensus number stays honest.** The single LLM-assisted step is *clustering* the diff --git a/SYSTEM_CONTEXT_DIAGRAM.md b/SYSTEM_CONTEXT_DIAGRAM.md index b3a7159..b43b7eb 100644 --- a/SYSTEM_CONTEXT_DIAGRAM.md +++ b/SYSTEM_CONTEXT_DIAGRAM.md @@ -45,7 +45,7 @@ flowchart TB end subgraph verdictpipe["Verdict pipeline (v1.1 · default-on)"] applyverdict["_apply_verdict
buffered + streaming (council.py)"] - extract["extract_verdict
ONE call · cluster stances · repair-once · never raises (verdict_synthesis.py)"] + extract["extract_verdict
1 initial call + up to 1 repair · cluster stances · never raises (verdict_synthesis.py)"] agree["agreement.position_cluster_ratio_v1
DETERMINISTIC arithmetic · no difflib · never LLM-emitted (agreement.py)"] verdictobj["CouncilVerdict
positions + evidence_answer_ids · conflicts · provider_votes · minority_reports (verdict.py)"] manifest["ModelHarnessManifest
first-class · secret-free · phased receipts (manifest.py)"] @@ -101,7 +101,7 @@ flowchart TB provider --> models council -->|"member answers or completed Elite revisions"| applyverdict - applyverdict -->|"synthesizer clusters stances
(ONE extraction call, output_contract)"| extract + applyverdict -->|"synthesizer clusters stances
(initial call + optional repair, output_contract)"| extract extract -->|"clustering"| agree agree -->|"consensus_score/label
(arithmetic over the clustering)"| verdictobj extract -->|"positions · conflicts · provider_votes"| verdictobj @@ -138,9 +138,10 @@ flowchart TB network boundary — `transport.post_json` (`transport.py`), one async httpx call site. - **The verdict pipeline is default-on and execution-traceable (PDD §4a).** Once the council has the member answers, `_apply_verdict` (`council.py`, run on both the buffered and streaming - paths, *after* the manifest exists) drives `extract_verdict` (`verdict_synthesis.py`): a - **single** extraction call asks the synthesizer model to *cluster* the members' stances — - not to re-answer, and crucially **not to emit a number**. That clustering feeds + paths, *after* the manifest exists) drives `extract_verdict` (`verdict_synthesis.py`): an + initial extraction call asks the synthesizer model to *cluster* the members' stances — + not to re-answer, and crucially **not to emit a number**. Invalid output permits one repair + call. The validated clustering feeds `agreement.position_cluster_ratio_v1` (`agreement.py`), which computes the `consensus_score` as pure **deterministic arithmetic** (largest cluster / positioned members; no `difflib`, never model-emitted). The assembled `CouncilVerdict` (`verdict.py`) carries positions with @@ -152,8 +153,9 @@ flowchart TB (`manifest.py`) rides on **every** result — first-class, not a debug flag — recording which model + prompt version produced the clustering (provenance) and stamping `secret_safety` only after the serialized manifest is scanned clean. This is enforced as a true invariant at - the single chokepoint `Council._cached_run` → `_ensure_manifest`: **all six modes** - (`synthesize`, `raw`, `debate`, `adversarial`, `vote`, `elite`) funnel through it, so a result can + the single chokepoint `Council._cached_run` → `_ensure_manifest`: all five released modes + (`synthesize`, `raw`, `debate`, `adversarial`, `vote`) plus source-only `elite` funnel through + it, so a result can never escape without a manifest — including the zero-members early return and cache hits. A verdict is *optional*: open-ended prompts, fewer than two responding members, or extraction failure leave `verdict = None` diff --git a/docs/PRODUCT_DESIGN_DOCUMENT.md b/docs/PRODUCT_DESIGN_DOCUMENT.md index 2582094..d57a616 100644 --- a/docs/PRODUCT_DESIGN_DOCUMENT.md +++ b/docs/PRODUCT_DESIGN_DOCUMENT.md @@ -48,8 +48,9 @@ structured, scored for agreement, and execution-traceable.** Every run yields a exposing agreement, disagreement (`conflicts`), minority views (`minority_reports`), and per-provider votes (`provider_votes`); a deterministic `consensus_score` (arithmetic over the model's clustering, *never* an LLM-emitted number); and a redacted `ModelHarnessManifest` (how -the run executed + which model produced the analysis) riding on **every** result across all six -modes (one chokepoint, §4a). A constrained-choice **`vote` mode** also **shipped** (CAC-09 / #3, +the run executed + which model produced the analysis) riding on **every** released-mode result; +source-only Elite results carry it too (one chokepoint, §4a). A constrained-choice **`vote` +mode** also **shipped** (CAC-09 / #3, `--mode vote --choices "A,B,C"` → plurality winner/split) — distinct from `provider_votes`. The next product-quality step is **Elite** (implemented, unreleased): independent answers → council-wide answer/claim audits → member revisions → the existing synthesis and execution-traceable verdict. Elite requires three successful responders at every member phase and intentionally spends more calls and time to improve consequential decisions. @@ -213,10 +214,11 @@ council outputs, not external sources; source grounding is Roadmap (§9). ### Verdict extraction + native structured output `extract_verdict(prompt, member_answers, *, synthesizer_name, synthesizer_model_id, -config=None) -> VerdictSynthesisResult(verdict, extraction, verdict_absent_reason)` -(`verdict_synthesis.py`) makes **one** extraction call asking the synthesizer model to -*cluster* stances (not to re-answer, not to emit a number), validates, repairs once, falls -back gracefully — never raises. +config=None, temperature=0.7, timeout=120.0, protocol_version=None) -> +VerdictSynthesisResult(verdict, extraction, verdict_absent_reason, attempt_receipts)` +(`verdict_synthesis.py`) makes one initial extraction call asking the synthesizer model to +*cluster* stances (not to re-answer, not to emit a number), validates, and makes at most one +repair call before falling back gracefully — never raises. It builds an `OutputContract(schema=verdict_extraction_json_schema(), schema_name="VerdictExtraction", strict=True)` (CAC-06-PLUMB threaded `output_contract` @@ -248,14 +250,14 @@ council's product. Opt out with `Council(extract_verdict=False)` (then `result.v (`self.extract_verdict_enabled`), no per-call override. Buffered (`ask`) and streaming (`stream_ask`) both run the same `_apply_verdict` helper *after* the manifest exists, so the verdict appears identically in the buffered result and the streaming `done` event and the -`secret_safety` stamp is re-run over the final content. **Cost:** default-on adds exactly -**one** extra synthesizer call per run; the opt-out exists for cost-sensitive callers. +`secret_safety` stamp is re-run over the final content. **Cost:** default-on adds one initial +synthesizer call and at most one schema-repair retry; the opt-out exists for cost-sensitive callers. -**Verdict scope across modes (deliberate).** Posture: *manifest everywhere, clustering verdict only where unambiguously additive.* The manifest rides on all six modes. The clustering verdict runs on `synthesize`/`ask` and on **completed Elite runs after synthesis of the revised answers**; it is deferred on `debate`/`vote` and intentionally not layered onto `adversarial`, which already emits a judge verdict. An incomplete Elite run has no synthesis or verdict. +**Verdict scope across modes (deliberate).** Posture: *manifest everywhere, clustering verdict only where unambiguously additive.* The manifest rides on all five released modes and on source-only Elite. The clustering verdict runs on `synthesize`/`ask` and on **completed Elite runs after synthesis of the revised answers**; it is deferred on `debate`/`vote` and intentionally not layered onto `adversarial`, which already emits a judge verdict. An incomplete Elite run has no synthesis or verdict. ### ModelHarnessManifest — first-class, secret-free The `ModelHarnessManifest` (`manifest.py`) rides on every `CouncilResult` — *not* behind a debug -flag, and now a **true invariant** enforced at one chokepoint: all six modes funnel through +flag, and now a **true invariant** enforced at one chokepoint: all five released modes plus source-only Elite funnel through `Council._cached_run` → `_ensure_manifest`, which attaches the manifest on every returned result — including `debate`/`adversarial`/`vote` (built in `modes.py`), the zero-members early return, and cache hits (synthesize/raw builds its own richer one earlier). Pinned by @@ -324,10 +326,10 @@ consumers. The end-to-end flow — `CLI/Library → Council → call_model → a | Module | Responsibility | |--------|----------------| -| `council.py` | `Council` — primary importable entry point. Resolves names, partitions members, and exposes two reusable primitives: `fan_out` (the single concurrent + partial-failure call loop) and `synthesize_blocks` (the single synthesizer/judge call path). Hosts the async/sync APIs for all six modes, including unreleased `elite`/`elite_sync`. Every mode funnels through `_cached_run`; completed Elite runs synthesize revisions and then use `_apply_verdict`. | +| `council.py` | `Council` — primary importable entry point. Resolves names, partitions members, and exposes two reusable primitives: `fan_out` (the single concurrent + partial-failure call loop) and `synthesize_blocks` (the single synthesizer/judge call path). Hosts the async/sync APIs for five released modes plus unreleased `elite`/`elite_sync`. Every source mode funnels through `_cached_run`; completed Elite runs synthesize revisions and then use `_apply_verdict`. | | `verdict.py` | Public verdict/member Pydantic types (`CouncilVerdict`, `CouncilPosition`, `CouncilConflict`, `ProviderVote`, `MinorityReport`) + the LCD JSON Schemas (`verdict_json_schema`/`member_answer_json_schema`/`verdict_extraction_json_schema`) usable across all three native structured-output surfaces; `VERDICT_SCHEMA_VERSION`. | | `agreement.py` | Deterministic consensus: `consensus_score` (`position_cluster_ratio_v1` — largest cluster / positioned members; `None` for N<2) + `consensus_label` buckets. Pure arithmetic, no `difflib`, never LLM-emitted. | -| `verdict_synthesis.py` | `extract_verdict` engine: one extraction call (clusters stances, never emits a number), native `output_contract` enforcement + prompt-level fallback, validate → repair-once → graceful `verdict=None`; the three verdict-absent reasons; provenance on every return path. | +| `verdict_synthesis.py` | `extract_verdict` engine: one initial extraction call and at most one repair (clusters stances, never emits a number), native `output_contract` enforcement + prompt-level fallback, validate → repair-once → graceful `verdict=None`; the three verdict-absent reasons; provenance on every return path. | | `manifest.py` | `ModelHarnessManifest` (first-class on every result), phased `ProviderExecutionReceipt`/`ProviderSkip`/`VerdictExtraction`, and `scan_for_secret_material()` → `secret_safety` stamp. No key values; unknown `estimated_cost` remains `None`. | | `modes.py` | Deliberation orchestration: `run_debate`, `run_adversarial`, `run_vote`, and unreleased `run_elite` (three gated member phases). Built on `Council.fan_out` + `synthesize_blocks`. | | `prompts.py` | Role/template strings for debate, adversarial, vote, and Elite claim-audit/revision prompts. Elite panel text uses stable Model A/B/C aliases and answer ids, never provider identities. | @@ -408,7 +410,7 @@ council shipped in v1.1** (§4a — the wedge). The revised thesis is a **source-grounded, execution-traceable decision record with empirically proven quality**. Current answer IDs identify model outputs, not external evidence; source-auditable language is therefore too broad until source grounding ships. Elite remains -implemented but unreleased on draft PR #51. +implemented but unreleased on open PR #51, pending final review and merge. The canonical roadmap is [`docs/plans/2026-07-17-decision-quality-roadmap.md`](plans/2026-07-17-decision-quality-roadmap.md): diff --git a/docs/plans/2026-07-17-decision-quality-roadmap.md b/docs/plans/2026-07-17-decision-quality-roadmap.md index adbe108..fd4e95b 100644 --- a/docs/plans/2026-07-17-decision-quality-roadmap.md +++ b/docs/plans/2026-07-17-decision-quality-roadmap.md @@ -1,8 +1,8 @@ # Conclave Decision Quality Roadmap -**Status:** Approved direction; Horizon 0 implemented on the draft branch, pending full gate review +**Status:** Approved direction; Horizon 0 implemented on open PR #51, pending final gate review **Date:** 2026-07-17 -**Current product:** v1.1.0 stable; Elite implemented but unreleased on draft PR #51 +**Current product:** v1.1.0 stable; Elite implemented but unreleased on open PR #51 ## Thesis @@ -30,8 +30,8 @@ grounding and deterministic citation validation ship, the honest claim is ## Horizon 0 — verify Elite correctness before merge -Elite remains implemented but unreleased. The draft branch now implements these contracts; -draft PR #51 must remain draft until the complete merge gate verifies them: +Elite remains implemented but unreleased. Open PR #51 implements these contracts and must +remain unmerged until the complete merge gate verifies them: 1. **Persistent identities.** Each initial answer has a stable identifier that survives claim audit, revision, synthesis, serialization, and cache replay. Do not present those IDs as diff --git a/src/conclave/cache.py b/src/conclave/cache.py index f3563ad..94a3a17 100644 --- a/src/conclave/cache.py +++ b/src/conclave/cache.py @@ -3,7 +3,7 @@ This is the §9 #4 roadmap item: an opt-in cache keyed on ``(prompt, council, mode, model ids)`` so repeated or eval runs are cheap. It is **off by default** and **never persists key material** -- the cache key and the -stored payload are derived solely from the normalized prompt, the ordered council +stored payload are derived solely from the exact prompt content, the ordered council member friendly-names + resolved model ids, the run mode, the synthesizer/judge identity, and the mode parameters that affect output. No environment variable is read here; no key value reaches the key string or the on-disk artifact. @@ -38,7 +38,6 @@ import hashlib import json import os -import re from collections.abc import Mapping from pathlib import Path from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit @@ -57,9 +56,7 @@ # Bumped if the cache-key composition or stored schema changes incompatibly, so # old entries simply miss instead of being mis-served against new code. -CACHE_FORMAT_VERSION = "2" - -_WHITESPACE = re.compile(r"\s+") +CACHE_FORMAT_VERSION = "3" _SECRET_QUERY_PARTS = ( "authorization", "auth", @@ -100,15 +97,6 @@ def cache_dir() -> Path: return base / "conclave" -def _normalize_prompt(prompt: str) -> str: - """Collapse runs of whitespace and strip ends for a stable prompt key. - - Two prompts that differ only in incidental whitespace should hit the same - cache entry; semantic content is otherwise preserved verbatim. - """ - return _WHITESPACE.sub(" ", prompt).strip() - - def _digest(value: str) -> str: """Return a one-way identity fingerprint without retaining ``value``.""" return hashlib.sha256(value.encode("utf-8")).hexdigest() @@ -195,7 +183,7 @@ def build_identity( "verdict_schema": verdict_schema_version, "verdict_extraction_prompt": verdict_prompt_version, }, - "prompt_fingerprint": _digest(_normalize_prompt(prompt)), + "prompt_fingerprint": _digest(prompt), "mode": mode, # Pairs as lists so JSON round-trips; order preserved deliberately. "members": [[name, model_id] for name, model_id in members], @@ -252,7 +240,7 @@ def make_key( Identity covers: - * normalized prompt, + * exact prompt content, * run mode, * ordered ``(friendly_name, resolved_model_id)`` member pairs (order matters -- see module docstring), @@ -263,7 +251,7 @@ def make_key( future source-bundle digest. Args: - prompt: The raw user prompt (normalized internally). + prompt: The exact raw user prompt. mode: ``"synthesize" | "raw" | "debate" | "adversarial"``. members: Ordered ``(friendly_name, resolved_model_id)`` pairs actually run. synthesizer: Synthesizer/judge friendly name (``None`` when not applicable). diff --git a/src/conclave/cli.py b/src/conclave/cli.py index ec5b8eb..bde351b 100644 --- a/src/conclave/cli.py +++ b/src/conclave/cli.py @@ -512,9 +512,11 @@ def ask( Exit codes: - * 0 -- the run produced at least one usable member answer. - * 1 -- the run completed but produced zero usable member answers (e.g. no - council member had an API key, or every member failed). Under ``--json`` + * 0 -- the run produced at least one usable member answer and, for Elite, + ``decision_readiness`` is ``ready``. + * 1 -- the run produced zero usable member answers (e.g. no council member + had an API key, or every member failed), or an Elite result is missing or + has ``decision_readiness`` ``not_ready``/``indeterminate``. Under ``--json`` the full JSON result is still emitted to stdout first, so a script can both parse the payload and detect the failure via the non-zero exit code. * 2 -- a usage/config error (unknown mode, or no members resolved). diff --git a/src/conclave/verdict_synthesis.py b/src/conclave/verdict_synthesis.py index a3073bc..608f1ef 100644 --- a/src/conclave/verdict_synthesis.py +++ b/src/conclave/verdict_synthesis.py @@ -5,8 +5,9 @@ plus ``00_SCOPE_PLAN.md`` §4 (the three honesty corrections). This is a self-contained, council-agnostic engine. Given the prompt and the -council members' raw answers, it asks ONE synthesizer model to produce a -structured *judgment* (the clustering of stances, the conflicts, the votes), then +council members' raw answers, it asks a synthesizer model to produce a structured +*judgment* (the clustering of stances, the conflicts, the votes), with at most one +repair call when the initial output is invalid, then computes the consensus number **itself, deterministically**, from that clustering — the model never emits the consensus score. CAC-06 wires :func:`extract_verdict` into ``council.ask``; this module does not wire itself. @@ -454,7 +455,7 @@ async def extract_verdict( undefined for N<2, DD-1 edge case). 2. **Extract.** Build the ``[system, user]`` messages from the prompt + every responding answer (labeled by within-run answer id) with the LCD extraction schema - embedded, and make ONE :func:`conclave.providers.call_model` call. + embedded, and make the initial :func:`conclave.providers.call_model` call. 3. **Validate → repair-once → fallback.** Parse JSON + validate via Pydantic. On failure, re-call ONCE with the stringified errors appended; if it still fails (or the extractor errored / returned empty), return ``verdict=None`` @@ -515,7 +516,7 @@ async def extract_verdict( verdict_absent_reason=_REASON_TOO_FEW, ) - # Step 2 — one extraction call. Request native structured output (capable + # Step 2 — initial extraction call. Request native structured output (capable # providers enforce the schema) AND keep the embedded schema in the messages as # the belt-and-suspenders fallback for providers without strict support. Built # once and reused for the initial call and the repair retry below; schema_name diff --git a/tests/test_cache.py b/tests/test_cache.py index 4923f52..612c29c 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -254,6 +254,21 @@ def test_current_cache_shape_without_readiness_defaults_indeterminate(cache_home assert cached.elite.readiness_reasons == ["adjudication.not_evaluated"] +def test_previous_cache_format_payload_is_a_miss(cache_home): + """Version 2 identities cannot replay after exact-prompt keying ships.""" + from conclave.models import CouncilResult + + key = "version-two-entry" + cache_home.mkdir(parents=True, exist_ok=True) + envelope = { + "cache_format_version": "2", + "result": CouncilResult(prompt="q", mode="raw").model_dump(mode="json"), + } + (cache_home / f"{key}.json").write_text(json.dumps(envelope), encoding="utf-8") + + assert cache_mod.load(key) is None + + async def test_changing_model_id_misses(monkeypatch, counting_call_model, cache_home): """Same friendly name but a different resolved model id -> different key.""" _set_keys(monkeypatch) @@ -369,7 +384,7 @@ def test_make_key_is_deterministic_and_order_sensitive(): assert len(k1) == 64 # sha256 hex -def test_make_key_normalizes_whitespace(): +def test_make_key_preserves_exact_prompt_whitespace(): common = dict( mode="raw", members=[("a", "x/1")], @@ -377,8 +392,8 @@ def test_make_key_normalizes_whitespace(): synthesizer_model_id=None, temperature=0.7, ) - assert cache_mod.make_key(prompt="a b\n c", **common) == cache_mod.make_key( - prompt=" a b c ", **common + assert cache_mod.make_key(prompt="a\n b", **common) != cache_mod.make_key( + prompt="a b", **common ) diff --git a/tests/test_cli.py b/tests/test_cli.py index 267db4a..2984dfc 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -3,11 +3,11 @@ These exercise ``conclave.cli.ask`` through Typer's ``CliRunner`` (no real keys, no network). Two concerns are pinned here: -* **Exit-code contract (#17).** A run that produces zero *usable* member answers - exits non-zero (code 1) on both the human and ``--json`` paths, and under - ``--json`` the full JSON payload is still emitted to stdout so a script can - parse the result *and* detect the failure via the exit code. A run with at - least one usable answer exits 0. +* **Exit-code contract (#17).** A run that produces zero *usable* member answers, + or an Elite run whose ``decision_readiness`` is not ``ready``, exits non-zero + (code 1) on both the human and ``--json`` paths. Under ``--json`` the full JSON + payload is still emitted to stdout so a script can parse the result *and* + detect the failure via the exit code. Other runs with a usable answer exit 0. * **Pooled-client lifecycle (#20).** The synchronous council wrappers close the shared httpx client when the run completes, so ``transport.aclose()`` is actually invoked and no client leaks past the CLI command.