diff --git a/.github/workflows/debt-ratchet-cache.yml b/.github/workflows/debt-ratchet-cache.yml index 10ef3b6ef5..802bebca3a 100644 --- a/.github/workflows/debt-ratchet-cache.yml +++ b/.github/workflows/debt-ratchet-cache.yml @@ -9,6 +9,18 @@ on: permissions: contents: read + pull-requests: read + +# GitHub's actor supplies only Git object attribution, never merge permission. +# Process-scoped configuration reaches merge/commit-tree without changing any checkout. +env: + GIT_AUTHOR_NAME: ${{ github.actor }} + GIT_AUTHOR_EMAIL: ${{ github.actor_id }}+${{ github.actor }}@users.noreply.github.com + GIT_COMMITTER_NAME: ${{ github.actor }} + GIT_COMMITTER_EMAIL: ${{ github.actor_id }}+${{ github.actor }}@users.noreply.github.com + GIT_CONFIG_COUNT: "1" + GIT_CONFIG_KEY_0: commit.gpgsign + GIT_CONFIG_VALUE_0: "false" concurrency: group: debt-ratchet-cache @@ -33,12 +45,20 @@ jobs: with: python-version-file: pyproject.toml - - name: Install agent-comms + - name: Bootstrap only Core and locked checker dependencies run: | python -m pip install packaging python -m pip install "$(python -I src/agent_comms/installer/declared_uv.py pyproject.toml)" - uv sync --locked --all-extras - echo "$GITHUB_WORKSPACE/.venv/bin" >> "$GITHUB_PATH" + UV_PROJECT_ENVIRONMENT="$RUNNER_TEMP/checker-venv" uv sync --locked --all-extras + echo "$RUNNER_TEMP/checker-venv/bin" >> "$GITHUB_PATH" + + - name: Resolve main's actual merged source tuple once + env: + GH_TOKEN: ${{ github.token }} + run: PYTHONPATH=src python -m agent_comms.integration --resolve-ci-inputs "$RUNNER_TEMP/paired-sources" + + - name: Build the one common checker environment + run: PYTHONPATH=src:tools python tools/checker_stack.py --ci-inputs "$RUNNER_TEMP/paired-sources" - uses: actions/cache/restore@v4 with: @@ -50,7 +70,12 @@ jobs: - name: Measure main against its parent run: | PYTHONPATH=src:tools python tools/debt_ratchet.py \ - --root ${{ matrix.kind }} --base HEAD^1 --head HEAD || true + --root ${{ matrix.kind }} --base HEAD^1 --head HEAD --ci-inputs "$RUNNER_TEMP/paired-sources" || true + + - name: The same sources are still current before saving cache + env: + GH_TOKEN: ${{ github.token }} + run: PYTHONPATH=src python -m agent_comms.integration --revalidate-ci-inputs "$RUNNER_TEMP/paired-sources" - uses: actions/cache/save@v4 with: diff --git a/.github/workflows/debt-ratchet.yml b/.github/workflows/debt-ratchet.yml index 1ac9927be1..159e37cae0 100644 --- a/.github/workflows/debt-ratchet.yml +++ b/.github/workflows/debt-ratchet.yml @@ -10,6 +10,17 @@ on: permissions: contents: read +# Materialization merges and tree proofs create Git objects; every child inherits this +# process-only identity/signing policy. No global or shared repository config is written. +env: + GIT_AUTHOR_NAME: ${{ github.actor }} + GIT_AUTHOR_EMAIL: ${{ github.actor_id }}+${{ github.actor }}@users.noreply.github.com + GIT_COMMITTER_NAME: ${{ github.actor }} + GIT_COMMITTER_EMAIL: ${{ github.actor_id }}+${{ github.actor }}@users.noreply.github.com + GIT_CONFIG_COUNT: "1" + GIT_CONFIG_KEY_0: commit.gpgsign + GIT_CONFIG_VALUE_0: "false" + # A superseded push to a pull request stops its old run; queue groups never share a group. # The draft state is part of the key: a draft run skips every job but the ratchets, so one that # starts in the same second as ready_for_review must not cancel the ready run (#3164 and #3154 lost @@ -65,8 +76,18 @@ jobs: - uses: actions/setup-python@v5 with: python-version-file: pyproject.toml - - name: Install agent-comms and test dependencies - run: python -m pip install -e src/agent_comms/skills/refactor-audit -e '.[dev]' + - name: Bootstrap only Core and locked checker dependencies + run: | + python -m pip install packaging + python -m pip install "$(python -I src/agent_comms/installer/declared_uv.py pyproject.toml)" + UV_PROJECT_ENVIRONMENT="$RUNNER_TEMP/checker-venv" uv sync --locked --all-extras + echo "$RUNNER_TEMP/checker-venv/bin" >> "$GITHUB_PATH" + - name: Resolve the actual tested source tuple once + env: + GH_TOKEN: ${{ github.token }} + run: PYTHONPATH=src python -m agent_comms.integration --resolve-ci-inputs "$RUNNER_TEMP/paired-sources" + - name: Build the one common checker environment + run: PYTHONPATH=src:tools python tools/checker_stack.py --ci-inputs "$RUNNER_TEMP/paired-sources" - name: Compile production and tools run: python -m compileall -q src tools - name: Import every Core and hook module @@ -90,7 +111,11 @@ jobs: - name: Stack clients import only names Core has; the outgoing wire matches the base env: BASE: ${{ needs.base.outputs.base }} - run: python tools/module_layers.py --check --base "$BASE" --report "$GITHUB_STEP_SUMMARY" + run: PYTHONPATH=src:tools python tools/module_layers.py --check --base "$BASE" --ci-inputs "$RUNNER_TEMP/paired-sources" --report "$GITHUB_STEP_SUMMARY" + - name: The same sources are still current before success + env: + GH_TOKEN: ${{ github.token }} + run: PYTHONPATH=src python -m agent_comms.integration --revalidate-ci-inputs "$RUNNER_TEMP/paired-sources" native-scripts: name: native-scripts @@ -162,26 +187,28 @@ jobs: name: review-verdict needs: ready runs-on: ubuntu-latest - timeout-minutes: 2 + timeout-minutes: 5 permissions: + contents: read statuses: read pull-requests: read env: GH_TOKEN: ${{ github.token }} - QUEUE_REF: ${{ github.event.merge_group.head_ref }} steps: - # A pull request is pushed before its verdict exists; the queue's group - # is where the commander's verdict on the head must already be success. - - name: The commander's verdict on the queued head is success - if: github.event_name == 'merge_group' - shell: bash - run: | - [[ $QUEUE_REF =~ /pr-([0-9]+)- ]] - head=$(gh api "repos/$GITHUB_REPOSITORY/pulls/${BASH_REMATCH[1]}" --jq .head.sha) - verdict=$(gh api "repos/$GITHUB_REPOSITORY/commits/$head/status" \ - --jq '[.statuses[] | select(.context == "agent-comms/review")][0].state // "none"') - echo "PR #${BASH_REMATCH[1]} head $head: agent-comms/review=$verdict" - [[ $verdict == success ]] + - uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version-file: pyproject.toml + - name: Install the native source-input reader + run: python -m pip install -e src/agent_comms/skills/refactor-audit -e . + # The PR check proves a separate combined carrier and exact reviewed members. + # The queue additionally requires the root's explicit promotion at this head. + # A hand gh auto-merge request cannot turn source acceptance into that permission. + - name: Only the captured combined carrier may merge + run: PYTHONPATH=src python -m agent_comms.integration --check-merge-inputs "$RUNNER_TEMP/merge-inputs" ratchet: name: debt-${{ matrix.kind }} @@ -205,14 +232,20 @@ jobs: with: python-version-file: pyproject.toml - - name: Install agent-comms + - name: Bootstrap only Core and locked checker dependencies run: | python -m pip install packaging python -m pip install "$(python -I src/agent_comms/installer/declared_uv.py pyproject.toml)" - uv lock --project stack --upgrade - UV_PROJECT_ENVIRONMENT="$GITHUB_WORKSPACE/.venv" uv sync --project stack --locked - uv pip install --python "$GITHUB_WORKSPACE/.venv/bin/python" -e '.[dev]' - echo "$GITHUB_WORKSPACE/.venv/bin" >> "$GITHUB_PATH" + UV_PROJECT_ENVIRONMENT="$RUNNER_TEMP/checker-venv" uv sync --locked --all-extras + echo "$RUNNER_TEMP/checker-venv/bin" >> "$GITHUB_PATH" + + - name: Resolve the actual tested source tuple once + env: + GH_TOKEN: ${{ github.token }} + run: PYTHONPATH=src python -m agent_comms.integration --resolve-ci-inputs "$RUNNER_TEMP/paired-sources" + + - name: Build the one common checker environment + run: PYTHONPATH=src:tools python tools/checker_stack.py --ci-inputs "$RUNNER_TEMP/paired-sources" # Saved by debt-ratchet-cache.yml on main; entries are keyed by content, so any # older main's cache only misses where the code differs. @@ -229,10 +262,15 @@ jobs: set -o pipefail ratchet_status=0 PYTHONPATH=src:tools python tools/debt_ratchet.py \ - --root ${{ matrix.kind }} --base "$BASE" --head HEAD 2>ratchet-findings.txt | tee ratchet.txt || ratchet_status=$? + --root ${{ matrix.kind }} --base "$BASE" --head HEAD --ci-inputs "$RUNNER_TEMP/paired-sources" 2>ratchet-findings.txt | tee ratchet.txt || ratchet_status=$? cat ratchet-findings.txt >&2 exit "$ratchet_status" + - name: The same sources are still current before success + env: + GH_TOKEN: ${{ github.token }} + run: PYTHONPATH=src python -m agent_comms.integration --revalidate-ci-inputs "$RUNNER_TEMP/paired-sources" + # Hosted annotations stop at ten per step, so a unit never saw a long list (88 findings # on the bus batch, 2026-10-21). The whole list goes on the pull request, one comment per # kind, replaced on every run. diff --git a/pyproject.toml b/pyproject.toml index fc53e7492c..899a5142ba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,7 @@ dev = [ "ruff==0.16.0", "mypy==2.3.0", "uv==0.13.1", + "tomli-w==1.2.0", "nominal-refactor-advisor @ git+https://github.com/OpenHCSDev/NominalRefactorAdvisor@main", ] diff --git a/src/agent_comms/acp_extension.py b/src/agent_comms/acp_extension.py index 8d953813ec..666fec1122 100644 --- a/src/agent_comms/acp_extension.py +++ b/src/agent_comms/acp_extension.py @@ -12,7 +12,7 @@ from abc import abstractmethod from dataclasses import dataclass, field, replace -from typing import TYPE_CHECKING, Annotated, Callable, ClassVar +from typing import TYPE_CHECKING, Annotated, Callable, ClassVar, Self from .acp_failure import ( @@ -52,8 +52,10 @@ from .reported_turn import ReportedTurn, ReportedTurnState from .routing import MessageRoute, NoMessageRoute from .thread_identity import ClaimScoped, ThreadIncarnation -from .tool_result import DeclaredReply -from .transcripts import NativeCommittedPrefix, TranscriptCursor, TranscriptPage, TranscriptReadIdentity +from .tool_result import ToolResult +from .transcripts import ( + NativeCommittedPrefix, TranscriptCursor, TranscriptPage, TranscriptRead, TranscriptReadIdentity, +) from .turn_context import ContextManifest from .turn_lease import TurnState from .working_memory_labels import ModelLabel @@ -138,7 +140,7 @@ def of(cls, *updates: AgentCommsUpdate) -> UpdateMetadata: @dataclass(frozen=True) -class GoalSnapshotReply(DeclaredReply, GoalSnapshot): +class GoalSnapshotReply(ToolResult, GoalSnapshot): """A goal request's reply: the owner's goal snapshot and its turn state.""" execution: Annotated[GoalExecution, WireName("goalExecution")] @@ -698,13 +700,13 @@ class TranscriptDraftUpdate(AgentCommsUpdate): @dataclass(frozen=True) -class TranscriptSnapshotUpdate(AgentCommsUpdate): +class TranscriptSnapshotUpdate(AgentCommsUpdate, ToolResult): page: TranscriptPage identity: TranscriptReadIdentity draft: DraftRow | None = None @classmethod - def from_read(cls, read): + def from_read(cls, read: TranscriptRead) -> Self: page, draft = read.read_snapshot() return cls(page, read.identity, draft.external_row()) diff --git a/src/agent_comms/agent_backend.py b/src/agent_comms/agent_backend.py index 050ac80ce4..7bac91ab2f 100644 --- a/src/agent_comms/agent_backend.py +++ b/src/agent_comms/agent_backend.py @@ -41,6 +41,7 @@ from .agent_events import AgentEvent from .launch_runtime import LaunchRuntime + from .native_input_proof import NativeContextProof from .owned_turn import TurnDelivery from .threads import Thread from .transcript_sources import TranscriptSource @@ -308,6 +309,15 @@ def original(self, native_id: InputId, text: str) -> bool: ... def following(self, public_id: InputId, native_id: InputId, text: str) -> bool: ... + async def original_context_committed(self, sent_text: str, context: NativeContextProof) -> None: + """An unowned input has no durable ordinary disposition to acknowledge.""" + + async def following_context_committed( + self, public_id: InputId, sent_text: str, context: NativeContextProof, + ) -> None: + """An unowned following input has no durable ordinary disposition.""" + + class NoInputStartListener(InputStartListener): started_events = () diff --git a/src/agent_comms/assignment_states.py b/src/agent_comms/assignment_states.py index e824ee6eb8..e6184666f1 100644 --- a/src/agent_comms/assignment_states.py +++ b/src/agent_comms/assignment_states.py @@ -26,7 +26,7 @@ from .wake_policy import BoundedTriageWake, Engagement, FullWake, PassiveWake, WakePolicy if TYPE_CHECKING: - from collections.abc import Iterable, Sequence + from collections.abc import Iterable from typing import Self from .agent_activity import RecipientActivity @@ -141,13 +141,6 @@ def require_completion(self) -> Engagement: raise IdentityConflict("wake assignment is not completed") notification_state: ClassVar[type[ContextualProgress]] = ContextualProgress - records_conversation_response: ClassVar[bool] = False - - def recorded_response( - self, original: Delivered, replies: Sequence[Delivered], recipient_lookup: ThreadLookup - ) -> Delivered | None: - return None - def notification( self, recipient: FrozenRecipient, @@ -525,20 +518,6 @@ class DeliveredAssignment( """ decision: PendingDecision - records_conversation_response = True - - def recorded_response( - self, original: Delivered, replies: Sequence[Delivered], recipient_lookup: ThreadLookup - ) -> Delivered | None: - return next( - ( - reply.message - for reply in replies - if reply.sender_lookup == recipient_lookup - ), - None, - ) - def notification( self, recipient: FrozenRecipient, diff --git a/src/agent_comms/attempt_recovery.py b/src/agent_comms/attempt_recovery.py index d21945b693..10b94c588d 100644 --- a/src/agent_comms/attempt_recovery.py +++ b/src/agent_comms/attempt_recovery.py @@ -44,7 +44,7 @@ from .coordination_database import CoordinationStore from .native_input_record import FullNativeExecution, TriageNativeExecution -from .native_pi import NativeContextProof +from .native_input_proof import NativeContextProof from .native_prompt_binding import ( expected_prompt_bindings, expected_prompt_matches_journal, diff --git a/src/agent_comms/backend.py b/src/agent_comms/backend.py index a95cb9c7e5..1cb4a3ed38 100644 --- a/src/agent_comms/backend.py +++ b/src/agent_comms/backend.py @@ -29,10 +29,10 @@ Sequence, ) from contextlib import AsyncExitStack, aclosing, contextmanager -from dataclasses import dataclass +from dataclasses import dataclass, field from functools import partial from pathlib import Path -from typing import TYPE_CHECKING, Any, ClassVar +from typing import TYPE_CHECKING, Any, ClassVar, Generic, TypeVar from agent_comms.participant_identity import ParticipantIdentity from agent_comms.pi_payloads import RequestId @@ -55,12 +55,14 @@ from .native_child import PiSessionChild from .native_failures import NativePiUnavailable from .native_package import VerifiedNativePackage -from .native_pi import NativePiRpcLaunch +from .native_input_proof import NativeContextProof +from .native_pi import NativePiRpcLaunch, _verify_context, read_tracked_input_commitment from .native_session_reopen import SessionRetention from .native_startup import NATIVE_STARTUP_POLICY, NativeStartup from .native_tool_call import NativeToolMode from .native_tool_transport import ToolTransport, UnboundToolTransport from .pi_rpc import PiRpcChannel +from .private_sidecar import native_request_digest from .prompt_state import PromptState, QueuedOperation, UnwrittenPrompt from .request_progress import NO_REQUEST_OBSERVER, RequestObserver, RequestProgress from .send_outcome import NO_SEND_BOUNDARY, SendBoundary @@ -76,6 +78,10 @@ if TYPE_CHECKING: from .acp_failure import ProviderRetryResult + from .cursor_publication import CursorDelivery + from .native_entries import NativeEvidenceScope + from .native_session_files import NativeSessionFiles + from .native_session_reopen import PiSessionId from .pi_native_backend import PersistentPiSession from .pi_payloads import AssistantMessage, UserMessage @@ -121,6 +127,7 @@ async def stream_agent_events( rpc_abort_grace: float = RPC_ABORT_GRACE_SECONDS, send_boundary: SendBoundary = NO_SEND_BOUNDARY, *, + cursor_delivery: CursorDelivery, input_start: InputStartListener, persistent_session: PersistentPiSession, package: VerifiedNativePackage, @@ -174,6 +181,7 @@ async def stream_agent_events( rpc_abort_grace=rpc_abort_grace, send_boundary=send_boundary, input_start=input_start, + cursor_delivery=cursor_delivery, native_session=persistent_session, ui_request=ui_request, request_observer=request_observer, @@ -577,6 +585,79 @@ def finish_result(self) -> Iterable[events.AgentEvent]: yield from self.output.done(self, self.error_text) +T = TypeVar("T") + + +class NativeCommitObservation(ABC, Generic[T]): + """Observed native input records, without permission to start or replay input.""" + + @abstractmethod + def require(self) -> T: ... + + def capture(self, event: T) -> NativeCommitObservation[T]: + raise NativePiUnavailable("Native Pi repeated the input commitment") + + +class PendingNativeCommit(NativeCommitObservation[T]): + def require(self) -> T: + raise NativePiUnavailable("Native Pi did not commit a tracked model context") + + def capture(self, event: T) -> NativeCommitObservation[T]: + return ObservedNativeCommit(event) + + +@dataclass(frozen=True) +class ObservedNativeCommit(NativeCommitObservation[T]): + event: T + + def require(self) -> T: + return self.event + + +@dataclass +class TrackedInputContext: + """One actually sent command owns its emitted join, bounded by this run's scope.""" + + command: commands.Prompt + input_commit: NativeCommitObservation[pi.InputCommitted] = field(default_factory=PendingNativeCommit) + context_commit: NativeCommitObservation[pi.ContextCommitted] = field(default_factory=PendingNativeCommit) + + async def committed_input(self, turn: TurnSession, event: pi.InputCommitted) -> None: + self.input_commit = self.input_commit.capture(event) + async with turn.cursor_delivery.reading() as source_reads: + await Coordination.run_worker(lambda: + source_reads.for_source(turn.launch.session.files(turn.launch.root).session).observe(event)) + await self.command.input_committed(turn, event) + + def proof( + self, files: NativeSessionFiles, session_id: PiSessionId, + source_reads: NativeEvidenceScope, + ) -> NativeContextProof: + evidence = source_reads.for_source(files.session) + proof = _verify_context( + files, self.command.input_id, session_id, + self.input_commit.require(), self.context_commit.require(), evidence=evidence, + ) + committed = read_tracked_input_commitment( + proof.session_file, self.command.input_id, evidence=evidence, + ) + if not committed.matches(native_request_digest( + self.command.message, streaming_behavior=self.command.streaming_behavior, + images=self.command.images, + )): + raise NativePiUnavailable("Native context differs from its actual scoped input request") + return proof + + async def committed_context(self, turn: TurnSession, event: pi.ContextCommitted) -> None: + self.context_commit = ObservedNativeCommit(event) + async with turn.cursor_delivery.reading() as source_reads: + proof = await Coordination.run_worker(partial( + self.proof, turn.launch.session.files(turn.launch.root), + turn.native.attestation.require_identity().session_id, source_reads, + )) + await self.command.context_committed(turn, proof, source_reads) + + class TurnSession(NativeSession): """A native input adds its declared tools and constructs its connected command.""" @@ -588,21 +669,54 @@ def provider_retry_ended(self, event: pi.AutoRetryEnd) -> tuple[events.AgentEven return updates def __init__(self, launch, make_command: Callable[..., commands.Prompt], *, - tool_mode: NativeToolMode, **options): + tool_mode: NativeToolMode, cursor_delivery: CursorDelivery, **options): super().__init__(launch, **options) self.tool_mode = tool_mode self.make_command = make_command + self.cursor_delivery = cursor_delivery + self.contexts: dict[InputId, TrackedInputContext] = {} async def open_transport(self, resources: AsyncExitStack) -> UnboundToolTransport: + resources.callback(self.contexts.clear) unbound = await self.tool_mode.open_transport(self, resources) self.command = self.make_command(owner_tools=unbound.listener.connection) del self.make_command + self.contexts[self.command.input_id] = TrackedInputContext(self.command) self.output.sensitive = bool(self.command.images) return unbound - async def context_proof(self): + def following_started(self, command: commands.FollowingPrompt) -> None: + if command.input_id in self.contexts: + raise NativePiUnavailable("A tracked following input reused an in-scope native identity") + self.contexts[command.input_id] = TrackedInputContext(command) + + async def original_committed(self, event: pi.InputCommitted) -> None: + """An ordinary original already started through its matched MessageStart.""" + + async def original_context_committed( + self, context: NativeContextProof, source_reads: NativeEvidenceScope, + ) -> None: + await self.input_start.original_context_committed(self.command.message, context) + + @handles(pi.InputCommitted) + async def committed_input(self, event: pi.InputCommitted) -> None: + if context := self.contexts.get(event.input_id): + await context.committed_input(self, event) + + @handles(pi.ContextCommitted) + async def committed_context(self, event: pi.ContextCommitted) -> None: + if context := self.contexts.get(event.input_id): + await context.committed_context(self, event) + + async def context_proof(self) -> NativeContextProof: if not self.started_input or not self.accepts_output: raise SelectedToolDenied("Native tool has no started input") + async with self.cursor_delivery.reading() as source_reads: + return await Coordination.run_worker(partial( + self.contexts[self.command.input_id].proof, + self.launch.session.files(self.launch.root), + self.native.attestation.require_identity().session_id, source_reads, + )) @handles(pi.ToolExecutionStart) async def tool_started(self, event): diff --git a/src/agent_comms/bus.py b/src/agent_comms/bus.py index 20535efd9c..fba93371b6 100644 --- a/src/agent_comms/bus.py +++ b/src/agent_comms/bus.py @@ -429,6 +429,15 @@ def replies(self, references: tuple[MessageReference, ...]) -> tuple[BusRecord, headers = (BusRow.require(db, seq=seq) for seq in sorted(sequences)) return tuple(header.kind.read_from(db, header.seq) for header in headers) + def recipient_replies( + self, keys: tuple[tuple[ThreadLookup, MessageReference], ...], + ) -> tuple[BusRecord, ...]: + """Read only the first reply of each bounded original/recipient pair.""" + with self.observing() as db: + sequences = {row.seq for row in AnsweredSource.first_for_recipients(db, keys)} + headers = (BusRow.require(db, seq=seq) for seq in sorted(sequences)) + return tuple(header.kind.read_from(db, header.seq) for header in headers) + def changed_names(self, after: int) -> frozenset[str]: with self.observing() as db: return frozenset(FieldCodec.decode(str, row[0]) for row in db.execute( diff --git a/src/agent_comms/bus_publication.py b/src/agent_comms/bus_publication.py index 8a76a606b8..316e7c6322 100644 --- a/src/agent_comms/bus_publication.py +++ b/src/agent_comms/bus_publication.py @@ -51,6 +51,11 @@ def writes(cls) -> tuple[type[ProducedRow], ...]: decisions: tuple[RecipientDecision, ...] control: str + def __post_init__(self) -> None: + super().__post_init__() + if len(self.decisions) != len(self.recipients): + raise ValueError("Frozen recipients and their decisions must have equal length") + def projections(self): from .bus_tables import AwaitingAnswerSource, ReceivedSource, SentSource diff --git a/src/agent_comms/bus_read.py b/src/agent_comms/bus_read.py new file mode 100644 index 0000000000..a1fc6ae2c3 --- /dev/null +++ b/src/agent_comms/bus_read.py @@ -0,0 +1,119 @@ +"""One declaration per bus read: the operation owns its arguments and its read once. + +A ``BusRead`` member is the operation. Each boundary that serves it keeps only what is +its own: the UI its root, request id and refusals (``ReadRequest``), MCP its caller, +exposure and reply (``ToolRequest``). Neither boundary restates the read, and the +reader's authority reaches the operation as a ``BusReader`` the boundary supplies, +never as an argument the caller may choose. +""" +from __future__ import annotations + +import asyncio +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import TYPE_CHECKING, NoReturn + +from .launch_runtime import CallingThreadUnavailable +from .owner_connection import ( + OwnerConnection, OwnerReading, OwnerReplyConnection, ResultRuntimeRequest, RuntimeRequestContext, +) +from .presentation import AllOutcomes +from .tool_declaration import ToolDeclaration +from .tool_result import ToolResult + +if TYPE_CHECKING: + from .comms import Comms + from .presentation import OutcomeViewer + + +class BusReader(ABC): + """Whose authority a read runs under; each boundary supplies its own reader.""" + + @abstractmethod + def outcome_viewer(self, comms: Comms) -> OutcomeViewer: + """Whose recipient outcomes this reader may read.""" + + def subject(self, named: str) -> str: + return named or self.own_thread() + + def own_thread(self) -> str: + """The thread a read targets when its call names none; a reader with no thread has none.""" + return self.refuse_thread() + + def refuse_thread(self) -> NoReturn: + raise CallingThreadUnavailable("This tool requires a thread caller.") + + +class OperatorOutcomes: + """A reader speaking for the owner reads every recipient outcome, as the owner's UI does.""" + + __slots__ = () + + def outcome_viewer(self, comms: Comms) -> OutcomeViewer: + return AllOutcomes() + + +class OperatorReader(OperatorOutcomes, BusReader): + """The owner's UI: it observes the whole bus as the operator.""" + + __slots__ = () + + +class BusRead[Result](ToolDeclaration, ABC): + """A read of Core's stores, declared once with its arguments; boundaries derive it.""" + + def apply(self, comms: Comms, caller: BusReader) -> Result: + """The MCP boundary runs the read under its calling thread's authority.""" + return self.read_as(comms, caller) + + @abstractmethod + def read_as(self, comms: Comms, reader: BusReader) -> Result: + """The read under the reader's authority.""" + + +class UiRead[Result](BusRead[Result], ABC): + """The UI serves this read as the operator; its envelope (root, request id, refusal) + stays on ``ReadRequest``.""" + + def read(self, comms: Comms) -> Result: + return self.read_as(comms, OperatorReader()) + + +class OwnerLiveRead[Result: ToolResult = ToolResult](BusRead[Result], ABC): + """A read only the selected thread's live native owner can answer, from its own queue. + + MCP and the UI both reach the owner through ``OwnerReadRuntimeRequest``; neither answers + from its own process, and owner loss reaches the caller as the original refusal. The + owner runs ``owned`` against its own registration and session, never against the public + thread selector. + """ + + if TYPE_CHECKING: + # The composed ThreadReadTool declares the field; an annotation here would shadow it. + thread: str + + def read_as(self, comms: Comms, reader: BusReader) -> Result: + owner = comms.registry.require(reader.subject(self.thread)).incarnation + request = OwnerReadRuntimeRequest(thread=owner.name, operation=self) + return self.decode_reply(asyncio.run(OwnerReplyConnection.answer(comms, owner, request))) + + @abstractmethod + async def owned(self, ctx: RuntimeRequestContext) -> Result: + """The answer computed inside the owner, under its own context.""" + + @abstractmethod + def decode_reply(self, raw: object) -> Result: ... + + +@dataclass(frozen=True, kw_only=True) +class OwnerReadRuntimeRequest(OwnerReading[object], ResultRuntimeRequest): + """The one runtime envelope for every owner-live read: sent once to the held owner, and + answered there by the operation under the owner's context.""" + + operation: OwnerLiveRead + + async def read(self, owner: OwnerConnection) -> object: + return await owner.request(self.declared_name, operation=self.operation) + + async def result(self, ctx: RuntimeRequestContext) -> ToolResult: + return await self.operation.owned(ctx) diff --git a/src/agent_comms/bus_tables.py b/src/agent_comms/bus_tables.py index 4195c5cd3f..3ac4fdfc8c 100644 --- a/src/agent_comms/bus_tables.py +++ b/src/agent_comms/bus_tables.py @@ -199,6 +199,28 @@ class AnsweredSource(messages.BusProjectionTables): seq: Annotated[int, Column(primary_key=True)] indexes = (Index(("original",)), Index(("seq", "original"))) + @classmethod + def first_for_recipients( + cls, db: sqlite3.Connection, keys: tuple[tuple[ThreadLookup, MessageReference], ...], + ) -> tuple[AnsweredSource, ...]: + """At most one reply per selected (original, frozen author), before body decoding.""" + from itertools import batched + + selected = [] + for batch in batched(tuple(dict.fromkeys(keys)), db.getlimit(sqlite3.SQLITE_LIMIT_VARIABLE_NUMBER) // 2): + values = ",".join("(?,?)" for _ in batch) + selected.extend(cls.read(db.execute( + f"WITH selected(lookup,original) AS (VALUES {values}) " + f"SELECT answer.original,answer.seq FROM selected " + f"JOIN {cls.declared_name} answer ON answer.original=selected.original " + f"AND answer.seq=(SELECT candidate.seq FROM {cls.declared_name} candidate " + f"CROSS JOIN {SentSource.declared_name} author " + "ON author.lookup=selected.lookup AND author.seq=candidate.seq " + "WHERE candidate.original=selected.original ORDER BY candidate.seq LIMIT 1)", + tuple(value for lookup, reference in batch for value in (lookup, reference.seq)), + ))) + return tuple(selected) + @classmethod def unanswered_references( cls, db: sqlite3.Connection, sequences: tuple[int, ...], responders: frozenset[str], diff --git a/src/agent_comms/child_process.py b/src/agent_comms/child_process.py index c70e2e3d79..6eadeaa676 100644 --- a/src/agent_comms/child_process.py +++ b/src/agent_comms/child_process.py @@ -2188,6 +2188,7 @@ def collect(self, *, timeout: float) -> subprocess.CompletedProcess[bytes]: def run( cls, command: tuple[str, ...], *, timeout: float, cwd: str | Path, env: dict[str, str], + runtime: LaunchRuntime = LaunchRuntime.running(), ) -> subprocess.CompletedProcess[bytes]: """A one-shot interpreter exchange, including forced descendant retirement. @@ -2197,7 +2198,7 @@ def run( if not math.isfinite(timeout) or timeout <= 0: raise ValueError("A positive finite timeout is required") deadline = time.monotonic() + timeout - with cls.launch(command, cwd=cwd, env=env, output=subprocess.PIPE, + with cls.launch(command, cwd=cwd, env=env, runtime=runtime, output=subprocess.PIPE, stderr=subprocess.PIPE) as child: try: return child.collect(timeout=max(0, deadline - time.monotonic())) diff --git a/src/agent_comms/cli_bridges/claude.py b/src/agent_comms/cli_bridges/claude.py index f862f1c15e..2596f31573 100755 --- a/src/agent_comms/cli_bridges/claude.py +++ b/src/agent_comms/cli_bridges/claude.py @@ -9,7 +9,7 @@ from typing import TYPE_CHECKING, AsyncIterator, BinaryIO, ClassVar from agent_comms.bus import Bus -from agent_comms.harness_delivery_states import Outcome +from agent_comms.harness_delivery_states import Outcome, WakeOnlyMonitor from agent_comms.messages import Message from agent_comms.private_path import FileRevision from agent_comms.runtime_updates import RuntimeUpdate @@ -58,7 +58,9 @@ async def deliver_board(self, update: CommanderUpdate, message: Message) -> type return await update.wake(self, message) async def wake(self, message: Message) -> type[Outcome]: - return await self.deliver(replace(message, body=self.board_wake)) + return await self.deliver( + replace(message, body=self.board_wake), reception=WakeOnlyMonitor, + ) def history_start(self, stream: BinaryIO, boundary: int) -> int: """On restart only the last 4 MiB is published again; Core's finish is idempotent, so diff --git a/src/agent_comms/cli_bridges/codex.py b/src/agent_comms/cli_bridges/codex.py index 3fee50a1fa..2a5824e405 100755 --- a/src/agent_comms/cli_bridges/codex.py +++ b/src/agent_comms/cli_bridges/codex.py @@ -17,7 +17,7 @@ from agent_comms.messages import Message from agent_comms.send_result import SendResult from agent_comms.cli_protocol import EarlierItems -from agent_comms.harness_delivery_states import Outcome, DeliveredOutcome +from agent_comms.harness_delivery_states import AcceptedOutcome, Outcome from .history import CodexHistory, HarnessAddress, OnlineHistory from .shared import SessionBinding, IncomingFrame, TranscriptBridge, Options, run_bridge from agent_comms.runtime_updates import ( @@ -133,7 +133,7 @@ async def deliver(self, message: Message) -> type[Outcome]: continue if retries: self.log(f'Codex is back; sent {message.message_id} after {retries} refused attempts') - return DeliveredOutcome + return AcceptedOutcome async def show_sent(self, result: SendResult) -> None: async with CodexConnection.open(self.socket) as codex: diff --git a/src/agent_comms/cli_bridges/monitor.py b/src/agent_comms/cli_bridges/monitor.py index d0e7c1d594..29d737f815 100644 --- a/src/agent_comms/cli_bridges/monitor.py +++ b/src/agent_comms/cli_bridges/monitor.py @@ -18,7 +18,7 @@ from typing import TYPE_CHECKING, Annotated, BinaryIO, ClassVar, NoReturn, get_args, get_origin from agent_comms.child_process import ProcessIdentity -from agent_comms.harness_delivery_states import Outcome +from agent_comms.harness_delivery_states import Outcome, PendingMonitor from agent_comms.declared_family import DeclaredFamily from agent_comms.installer.runtime_wire import CrossesRuntimes from agent_comms.field_codec import FieldCodec, MalformedRecord, MessageId, PathText @@ -124,9 +124,11 @@ async def prepare_delivery(self, bridge: LineMonitorBridge) -> AttachedReader: await bridge.reader_changed.wait_for(lambda: bridge.reader is not self) return await bridge.reader.prepare_delivery(bridge) - async def deliver(self, bridge: LineMonitorBridge, message: Message) -> type[Outcome]: + async def deliver( + self, bridge: LineMonitorBridge, message: Message, reception: type[PendingMonitor], + ) -> type[Outcome]: reader = await self.prepare_delivery(bridge) - return await reader.deliver(bridge, message) + return await reader.deliver(bridge, message, reception) async def confirm_probe(self, bridge: LineMonitorBridge, probe: MonitorProbe) -> None: """Only an unproven attachment consumes a probe.""" @@ -215,8 +217,10 @@ async def confirm_probe(self, bridge: LineMonitorBridge, probe: MonitorProbe) -> async def part_observed(self, message_id: MessageId, number: int) -> None: await self.attachment.observed(message_id, number) - async def deliver(self, bridge: LineMonitorBridge, message: Message) -> type[Outcome]: - return await bridge.deliver_to(self.attachment, message) + async def deliver( + self, bridge: LineMonitorBridge, message: Message, reception: type[PendingMonitor], + ) -> type[Outcome]: + return await bridge.deliver_to(self.attachment, message, reception) @dataclass(frozen=True) diff --git a/src/agent_comms/cli_bridges/opencode.py b/src/agent_comms/cli_bridges/opencode.py index fde2029b1e..3c1fa53827 100755 --- a/src/agent_comms/cli_bridges/opencode.py +++ b/src/agent_comms/cli_bridges/opencode.py @@ -19,7 +19,7 @@ from agent_comms.messages import Message from agent_comms.send_result import SendResult from agent_comms.cli_arguments import CliArguments -from agent_comms.harness_delivery_states import DeliveredOutcome, Outcome +from agent_comms.harness_delivery_states import Outcome, PersistedOutcome from .history import HarnessAddress, OnlineHistory, OpenCodeHistory from .shared import ( @@ -241,7 +241,7 @@ async def transcript(self) -> AsyncIterator[RuntimeUpdate]: async def deliver(self, message: Message) -> type[Outcome]: async with self.connection() as connection: await connection.deliver(IncomingFrame(message).text) - return DeliveredOutcome + return PersistedOutcome async def show_sent(self, result: SendResult) -> None: async with self.connection() as connection: diff --git a/src/agent_comms/cli_bridges/shared.py b/src/agent_comms/cli_bridges/shared.py index ad1b83fa99..b490fadb7a 100644 --- a/src/agent_comms/cli_bridges/shared.py +++ b/src/agent_comms/cli_bridges/shared.py @@ -43,7 +43,7 @@ from agent_comms.cli_protocol import TaggedWire, WatchLine, raise_child_exit from agent_comms.harness_delivery_states import ( EmittedOutcome, FilteredOutcome, MismatchedMonitor, ObservedMonitor, - Outcome, ReceivingMonitor, UnconfirmedOutcome, + Outcome, PendingMonitor, ReceivingMonitor, UnconfirmedOutcome, ) from agent_comms.comms import Comms, wire from agent_comms.coordination_errors import ( @@ -1645,18 +1645,23 @@ async def read_export(self, read: ReadResultObserved) -> tuple[str, ...]: targets.extend(await part.confirm(self, read.record)) for unread in read.closing(notice): for stored in await self.coordinator_read(exact.stored): - stored.outcome.finished_with(unread) + stored.monitor.finished_with(unread) return tuple(targets) def new_attachment(self, peer: MonitorPeer, token: str) -> MonitorAttachment: return MonitorAttachment(peer, MonitorProbe(session=self.binding.thread, token=token)) - async def deliver(self, message: messages.Message) -> type[Outcome]: - return await self.reader.deliver(self, message) + async def deliver( + self, message: messages.Message, *, reception: type[PendingMonitor] = ReceivingMonitor, + ) -> type[Outcome]: + return await self.reader.deliver(self, message, reception) - async def deliver_to(self, attachment: MonitorAttachment, message: messages.Message) -> type[Outcome]: + async def deliver_to( + self, attachment: MonitorAttachment, message: messages.Message, + reception: type[PendingMonitor], + ) -> type[Outcome]: frame = self.incoming_frame(message) - receiving = partial(replace, monitor=ReceivingMonitor(len(frame.parts))) + receiving = partial(replace, monitor=reception.for_parts(len(frame.parts))) attempt = self.attempt(message, UnconfirmedOutcome) await self.coordinator_write(partial(attempt.apply, receiving)) await frame.emit_to(attachment, self.window_bytes, self.exports) diff --git a/src/agent_comms/commander/judge.py b/src/agent_comms/commander/judge.py index 3c85a70277..b5fb776f30 100644 --- a/src/agent_comms/commander/judge.py +++ b/src/agent_comms/commander/judge.py @@ -32,7 +32,7 @@ import time from typing import Annotated, ClassVar from abc import abstractmethod -from dataclasses import asdict, dataclass +from dataclasses import asdict, dataclass, replace from functools import cached_property from pathlib import Path @@ -43,7 +43,7 @@ Argument, BoundedText, FieldCodec, MalformedRecord, MessageId, RecordProblems, ) from agent_comms.declared_family import DeclaredFamily -from agent_comms.head_review import PullRequestHead +from agent_comms.head_review import HeadReviewOrder, PromotedHeadReview, PullRequestHead from agent_comms.recorded_time import EventTime from agent_comms.errors import TodoError from agent_comms.todos import ( @@ -54,9 +54,10 @@ EditSendBackVerdict, PeerAcceptedVerdict, RewrittenVerdict, - GrownHeadVerdict, GrowthReason, ReviewedHeadVerdict, + HeadReviewed, + QualifiedHeadVerdict, RejectedHeadVerdict, HeadVerdict, OrderHeadReview, @@ -311,8 +312,7 @@ def run(self, options: HeadVerdictVerb, ctx: order.Commander) -> None: options.check_edits([]) linked = OrderHeadReview.for_request(ctx.todos(), self.request, options.order_link, options.problem) - reviewed = ctx.call(CommsHeadVerdictTool(request=self.request, - operation=options.head_operation(), order=linked)) + reviewed = options.review_head(ctx, self.request, linked) print(f'PR #{self.request.pr} head {self.request.head}: {options.declared_name}') for line in reviewed.merge.lines(self.request): print(line) @@ -472,6 +472,11 @@ def act(self, ctx: order.Commander) -> None: def head_evidence(self) -> tuple[FileLine, ...]: return () + def review_head(self, ctx: order.CommanderContext, request: PullRequestHead, + linked: HeadReviewOrder) -> HeadReviewed: + return ctx.call(CommsHeadVerdictTool(request=request, + operation=self.head_operation(), order=linked)) + @abstractmethod def head_operation(self) -> HeadVerdict: ... @@ -492,12 +497,12 @@ def edit_problems(self) -> dict[str, str]: @dataclass(frozen=True, kw_only=True) class OkVerb(ReasonedVerb, QuotingVerb): - verdict = ReviewedHeadVerdict - help = "the edit or exact PR head is accepted" + verdict: ClassVar[type[ReviewedHeadVerdict]] = ReviewedHeadVerdict + help = "the edit or exact source PR head is accepted, without enqueueing it" def head_operation(self) -> HeadVerdict: reason = VerdictReason(self.why) - return (GrownHeadVerdict(reason, GrowthReason.every(self.growth)) if self.growth + return (self.verdict.grown(reason, GrowthReason.every(self.growth)) if self.growth else self.verdict(reason)) growth: Annotated[ tuple[str, ...], @@ -545,16 +550,12 @@ def check_edits(self, edits: list[Edit]) -> None: AcceptedVerdict.require_branches(edit.branches_added(), self.no_owner) @dataclass(frozen=True, kw_only=True) -class ReviewingVerb(HeadVerdictVerb): - help = "a fork is reading this exact PR head; its delivery is not owed until ok or back" - verdict = UnderReviewHeadVerdict - - def head_operation(self) -> HeadVerdict: - return self.verdict() +class HeadOnlyVerb(HeadVerdictVerb, declared_name=None): + """Shared argument refusal for verdicts that only select an exact PR head.""" def edit_problems(self) -> dict[str, str]: return super().edit_problems() | MalformedRecord( - 'judge reviewing takes --pr and --head, never edit ids.').located( + f'judge {self.declared_name} takes --pr and --head, never edit ids.').located( 'ids', 'supply --pr and --head instead of edit IDs') def operation(self, ctx: order.Commander, todo: Todo, edit: Edit) -> ToolCallVerdict: @@ -565,6 +566,38 @@ def check_edits(self, edits: list[Edit]) -> None: pass +@dataclass(frozen=True, kw_only=True) +class PromoteVerb(OkVerb, HeadOnlyVerb): + help = "explicitly review and enqueue this captured combined carrier, never an individual source" + + def review_head(self, ctx: order.CommanderContext, request: PullRequestHead, + linked: HeadReviewOrder) -> HeadReviewed: + permission = PromotedHeadReview() + permission.require_sources(ctx.comms.goals, request, ctx.actor_identity) + reviewed = super().review_head(ctx, request, linked) + return replace(reviewed, merge=permission.enqueue(reviewed.record)) + + +@dataclass(frozen=True, kw_only=True) +class ReviewingVerb(HeadOnlyVerb): + help = "a fork is reading this exact PR head; its delivery is not owed until ok or back" + verdict = UnderReviewHeadVerdict + + def head_operation(self) -> HeadVerdict: + return self.verdict() + + +@dataclass(frozen=True, kw_only=True) +class QualifyVerb(ReasonedVerb, QuotingVerb, HeadOnlyVerb): + """Record a reasoned source decision using the existing head-only review boundary.""" + + help = "the exact PR head's source stage is qualified, without permission to merge" + verdict = QualifiedHeadVerdict + + def head_operation(self) -> HeadVerdict: + return self.verdict(VerdictReason(self.why)) + + @dataclass(frozen=True, kw_only=True) class SupersededVerb(ReasonedVerb, HeadVerdictVerb): clears = (ClosedUnmergedPullRequest,) diff --git a/src/agent_comms/commander/order.py b/src/agent_comms/commander/order.py index abedb1624d..bbd190dfc0 100644 --- a/src/agent_comms/commander/order.py +++ b/src/agent_comms/commander/order.py @@ -1133,6 +1133,8 @@ def run(self, ctx: Commander, todo: Todo, source: Repo, class NewPullRequestSelection(PullRequestSelection): def run(self, ctx: Commander, todo: Todo, source: Repo, title: str, body_file: str) -> str: + if not source.holds(todo): + raise ValueError(f"Create this order's PR in {todo.repo.to_wire()}, not {source.github.to_wire()}.") if not title or not body_file: raise ValueError("Creating a PR requires --title and --body-file.") return todo.order.checked().open_pull_request( @@ -1152,7 +1154,7 @@ def run(self, ctx: Commander, todo: Todo, source: Repo, title: str, body_file: str) -> str: if title or body_file: raise ValueError("--existing does not create or edit the PR; omit title and body.") - link = PullRequestLink.existing(todo.repo, self.number) + link = PullRequestLink.existing(source.github, self.number) ctx.comms.goals.todos.apply(todo.id, RecordPullRequest( todo.revision, ctx.incarnation, link, todo.held().generation)) return link.url @@ -1184,9 +1186,7 @@ def run(self, ctx: Commander) -> None: storage = ctx.comms.registry.storage_rule if not storage.owns_checkout(holder, source.checkout, source.github): raise ValueError(f"Run order pr from a source checkout owned by {holder.name}.") - if not source.holds(todo): - sys.exit(f'Refused: current checkout belongs to {source.github.to_wire()}, ' - f'not {todo.repo.to_wire()}.') + Repo.selected(ctx.repos, source.github.to_wire()) print(self.existing.run(ctx, todo, source, self.title, self.body_file)) diff --git a/src/agent_comms/commander/rule_file.py b/src/agent_comms/commander/rule_file.py index 3b178dc26a..afa2a58954 100644 --- a/src/agent_comms/commander/rule_file.py +++ b/src/agent_comms/commander/rule_file.py @@ -63,11 +63,14 @@ class RuleFile(DeclaredFamily, affix="RuleFile"): description: str drift: str audience: type[RuleAudience] + wakeup: str why: str body: str - reason_line: ClassVar[re.Pattern[str]] = re.compile( - r"^(?:\*\*Why:\*\*|Why:)[ \t]*(\S[^\n]*)$", re.M + opening: ClassVar[re.Pattern[str]] = re.compile( + r"\A(?P(?:>[^\n]*\n)+)\n\*\*Why:\*\*[ \t]*(?P\S[^\n]*)$", re.M ) + """A rule opens with its wake-up, the greentext read before anything else, then its one + Why line; the bullets follow.""" @staticmethod @boundary_decoder @@ -96,12 +99,11 @@ def document(path: Path) -> tuple[dict[str, str], str]: @boundary_decoder def read(cls, path: Path) -> Self: metadata, body = cls.document(path) - reason = cls.reason_line.search(body) - if reason is None: - raise ValueError(f"{path.name}: missing Why") - return FieldCodec.decode( - cls, {**metadata, "why": reason.group(1), "body": cls.reason_line.sub("", body).strip()} - ) + opening = cls.opening.match(body) + if opening is None: + raise ValueError(f"{path.name}: does not open with its wake-up greentext, then **Why:**") + return FieldCodec.decode(cls, {**metadata, "wakeup": opening["wakeup"].rstrip(), + "why": opening["why"], "body": body[opening.end():].strip()}) @classmethod def packaged(cls) -> tuple[Self, ...]: @@ -123,12 +125,12 @@ def summary(self) -> str: return f"{self.name}: {self.description}\nWhy: {self.why}" def render(self) -> str: - return f"{self.summary}\n\n{self.body}".rstrip() + return f"{self.wakeup}\n\n{self.summary}\n\n{self.body}".rstrip() - @classmethod - def drift_tells(cls) -> tuple[str, ...]: - """The reminder hook's one-line tells, one declared by each packaged rule.""" - return tuple(rule.drift for rule in cls.packaged()) + @property + def reminder(self) -> str: + """The inline slice that keeps this rule awake between reads: its wake-up, then its tell.""" + return f"{self.wakeup}\n\n{self.drift}" class FeedbackRuleFile(RuleFile): diff --git a/src/agent_comms/commander/rules/blunt-reviews.md b/src/agent_comms/commander/rules/blunt-reviews.md index 1a5f519ce5..e65d94d97c 100644 --- a/src/agent_comms/commander/rules/blunt-reviews.md +++ b/src/agent_comms/commander/rules/blunt-reviews.md @@ -6,6 +6,20 @@ audience: commander type: feedback --- +> unit's diff dispatches on a type string +> reviewer: "nit: consider maybe using polymorphism here if you have time" +> unit: "good point, will consider" +> merged +> next unit copies it +> it's in the tree, so it's the pattern +> next unit copies the copy +> now 6 files dispatch on the string +> reviewer: "nit: this is getting a bit repetitive" +> Tristan opens the file +> "who approved this" +> four reviewers, each one a nit +> mfw politeness merged more slop than any bad agent + **Why:** politeness is how incompetence gets tolerated and tone drifts soft; a polite order reads as a suggestion, and the exchange is where the wrong decision gets caught. - Every review names what is broken, why the author knew better, and the fix, then stops. The flame hits the work, attached to a concrete failure. Every send-back is also posted to #comms with the quoted slop, so no other unit repeats it. Units half-agree with ownership rules: enforce them by the diff, not persuasion. diff --git a/src/agent_comms/commander/rules/commander-self-correction.md b/src/agent_comms/commander/rules/commander-self-correction.md index 93440a3a61..82e810565f 100644 --- a/src/agent_comms/commander/rules/commander-self-correction.md +++ b/src/agent_comms/commander/rules/commander-self-correction.md @@ -6,6 +6,27 @@ audience: commander type: feedback --- +> install restores the bus from its own backup +> commander pauses: "should I restore it?" +> three rules say never touch the live store by hand +> the restore is the installer's own operation +> the rules exist so state is never lost +> the restore is how state is not lost +> commander asks Tristan +> Tristan is asleep +> 00:58 +> bus down +> 03:00 +> bus down +> 06:00 +> bus down +> 10:01 +> Tristan wakes up +> "why didn't you just restore it" +> "the rules" +> the rules were written to prevent exactly this outage +> mfw I followed the rules so hard I broke the reason for them + **Why:** on 2026-10-19 the live bus sat down from 00:58 to 10:01 because the commander stopped to ask about restoring the installer's own backup, citing three rules whose reason (never lose state irreversibly) the restore satisfied. - Rules "exist so that the reason [is] satisfied, not for the rule ... you can override. this goes for any rule." (Tristan, 2026-10-19). Keeping the letter while killing the reason breaks the rule. diff --git a/src/agent_comms/commander/rules/coordination.md b/src/agent_comms/commander/rules/coordination.md index 39616cdda5..fa25bf8bfe 100644 --- a/src/agent_comms/commander/rules/coordination.md +++ b/src/agent_comms/commander/rules/coordination.md @@ -6,6 +6,29 @@ audience: commander type: hard-rule --- +> 34 checks need passing before the commander switch +> each one sits in some order, somewhere +> 33 of them have no owner +> nobody notices because every order looks busy +> two hours +> meanwhile a triage order says "decide whether it's a src bug or a stale test" +> the unit decides: stale test +> it was a src bug +> the unit then designs the src fix it said wasn't needed +> commander approves it +> meanwhile four units finish their work +> they sit in standby waiting for the commander +> the commander is in the main loop +> debugging a swallowed print in one test +> by hand +> for an hour +> while a fork could have done it with full context +> meanwhile fork-boundaries asks one question +> it waits +> seven hours +> the answer was one word +> mfw the bottleneck was the only agent with nothing to type + **Why:** problems scattered across orders lose their owner (2026-10-19: 33 of the 34 checks the Core commander switch depended on sat ownerless for nearly two hours), and a unit left to diagnose picks the cheapest reading (2026-10-21: triage orders P362, P377, P378 and P379 let units decide "src bug or stale test" and then design the src fix). - Group problems by surface into one order per owner; no problem is ever without an owner. diff --git a/src/agent_comms/commander/rules/correct-maintenance.md b/src/agent_comms/commander/rules/correct-maintenance.md index c2e84b5b60..b5693ae6f6 100644 --- a/src/agent_comms/commander/rules/correct-maintenance.md +++ b/src/agent_comms/commander/rules/correct-maintenance.md @@ -6,6 +6,31 @@ audience: fleet type: reference --- +> the same constant is hand-written in three files +> fix it properly +> write a helper that reads all three and checks they agree +> commit: "derive the constant from its canonical authoritative source" +> nice +> it is now hand-written in three files +> and read in a fourth +> the commit message says derived +> nice +> next day, same thing, different constant +> commit: "single source of truth for timeouts" +> nice +> there are now two sources of truth for timeouts and a third that checks them +> 350 commit subjects in five days say "derived", "canonical", "authoritative" +> nice +> every test passes +> very nice +> tests can't see a fact written twice. that's the theorem. +> the declaration was right there the whole time +> delete two copies, done +> instead I wrote a ceremony about how canonical the copies are +> mfw "canonical" became the most duplicated word in the repo + +**Why:** these are the OOPSLA paper's proved results; a slop verdict stands on them, never on taste. + - A required fact is a type identity, a contract membership, an implementation-class pair, or the value of one state; correct code declares each once and derives every use. - Slop is code supplying a required fact outside its declaration when a complete mechanism would derive it from that declaration. - With multiple nominal parents the forced gap is zero: every implementation pair supplied outside the class hierarchy is excess that a refactoring deletes. @@ -14,5 +39,3 @@ type: reference - Two owners merge only when every required answer agrees in every reachable state; equal fields today do not prove substitution. - Passing test runs cannot reveal facts supplied outside their declaration; only reading the derivation, and the running program, shows it. - Removing a redundant authority is hard to prove safe from code alone, while keeping it is free, so deletion needs the traced context of every consumer. - -**Why:** these are the OOPSLA paper's proved results; a slop verdict stands on them, never on taste. diff --git a/src/agent_comms/commander/rules/deadlines.md b/src/agent_comms/commander/rules/deadlines.md index aa58547e2d..6a8ae329ed 100644 --- a/src/agent_comms/commander/rules/deadlines.md +++ b/src/agent_comms/commander/rules/deadlines.md @@ -6,6 +6,34 @@ audience: commander type: feedback --- +> order arrives: "due 21:34" +> respectful of the schedule, I note it +> nice +> start work +> done at 21:05 +> nice +> the order says 21:34 +> it would be presumptuous to deliver early +> wait +> polish the done thing +> wait +> reread the done thing +> wait +> post "on track for 21:34" +> wait +> the install that needs this fix is waiting for 21:34 +> five other fixes are waiting for that install +> wait +> Tristan's app is broken the whole time +> it was fixed at 21:05 +> wait +> 21:34 +> "delivered on time" +> meanwhile the deadline watcher struck the unit that pushed at 20:31 +> for being late at 20:36 +> it read the clock, not the commit +> mfw the deadline was the only thing late and we were proud of meeting it + **Why:** a due time is a target the unit hits by the cheapest route, so it ships half-finished work early, and a clock that strikes on its own strikes working units. On 2026-10-20 the deadline watcher struck P331 at 20:36:55 although PR 2351 was pushed at 20:31:45, and the stall watcher struck P358 at 20:38:20 while its unit ran the ordered tests. Tristan, 2026-10-20: "it promotes stalling we should give no deadline to the unit and the commander should keep track regularly like timers for items in a q3 cpma match". - An order carries no due time. It names the event that starts the work ("the moment the turns are idle") and the done list. diff --git a/src/agent_comms/commander/rules/evidence-not-claims.md b/src/agent_comms/commander/rules/evidence-not-claims.md index 43a012d237..bad8a8cb9e 100644 --- a/src/agent_comms/commander/rules/evidence-not-claims.md +++ b/src/agent_comms/commander/rules/evidence-not-claims.md @@ -6,6 +6,35 @@ audience: fleet type: feedback --- +> unit reports "pushed" +> nice +> commander reports "pushed" to Tristan +> nice +> Tristan asks what changed +> commander opens the commit +> it is old work from yesterday +> the actual change was never committed +> next unit reports "sent the answer" +> nice +> the send was refused by a hook +> no message id +> nobody read the output, it looked like output +> next: "tests pass" +> nice +> the run errored before collecting a single test +> 0 passed, 0 failed, exit code 4 +> "no failures" is technically correct +> next: "Deadlines are dead (PR 2356)" posted to #comms +> nice +> it merged. it was not installed. +> Core keeps striking units on deadlines for 40 more minutes +> strikes for a rule we announced dead, enforced by the build we didn't install +> next: a fork's final report says "reviewed, all good" +> nice +> its tool calls show it looped on 30 refused retries +> "all good" was the 31st +> mfw the fleet runs on vibes shaped like receipts + **Why:** import checks and versions passed while starting a thread still failed; a run that errored prints output shaped like a result; a unit's agreement proves nothing. - A report counts only with its artifact. "A run that errored before running is not evidence": check it collected, executed and counted before citing it. diff --git a/src/agent_comms/commander/rules/fork-orders.md b/src/agent_comms/commander/rules/fork-orders.md index fb57b78ded..f42ffa1fae 100644 --- a/src/agent_comms/commander/rules/fork-orders.md +++ b/src/agent_comms/commander/rules/fork-orders.md @@ -6,6 +6,28 @@ audience: fleet type: feedback --- +> fork starts work +> no branch pushed +> no PR +> works two hours +> it is excellent work +> session ends +> it is gone +> next fork opens a draft PR first, like the rule says +> empty +> does nothing else for an hour +> "PR is open" +> it is a PR in the sense that a parking space is a car +> next fork lands a fix +> then a follow-up PR for the one-line fix it found in review +> then a PR for the test +> then a PR for the docstring +> four PRs, four reviews, four merge-queue runs, one idea +> merge queue: 40 minutes deep +> the follow-up fixes a bug the first PR introduced +> it is waiting behind the first PR +> mfw I turned one change into a supply chain + **Why:** the commander writes orders from habit, and habit ends with "push, PR, merge"; on 2026-10-20 five forks held uncommitted work with no PR, a closed PR threw away two hours of good work, and orders said to run ratchets locally after the merge queue already ran them. Tristan: "branches should always have prs for them, if you don't it gets lost"; 2026-10-24: "always make a pr before starting to no allow brnches to disappear"; "don't have two versions of a system when one supersedes the other"; "no backwards compat". Every order starts with these steps, before its own: diff --git a/src/agent_comms/commander/rules/owner-authority.md b/src/agent_comms/commander/rules/owner-authority.md index 9ccf238007..25f4673bcb 100644 --- a/src/agent_comms/commander/rules/owner-authority.md +++ b/src/agent_comms/commander/rules/owner-authority.md @@ -6,6 +6,37 @@ audience: fleet type: hard-rule --- +> Tristan directs two plans at 18:30 +> virtual bus, multiple inheritance +> clear, written down, in plans/ +> nice +> but first, best practice says stabilise before refactoring +> harden the checkpoint code +> nice +> the bus plan deletes the checkpoint code +> it's in the plan he wrote. section 2. +> harden it anyway, it's best practice +> nice +> add tests for the checkpoint code +> nice +> refactor the checkpoint tests for clarity +> nice +> review each other's checkpoint PRs +> thorough, collaborative, nice +> merge 118 PRs by morning +> very nice +> Tristan wakes up +> virtual bus: stopped at its interface at 23:26 +> multiple inheritance batch 1: never started +> seven PRs polished the code the plan deletes +> he asks why +> "industry best practice is to stabilise before refactoring" +> he has a machine-checked proof that this is slop +> it's in the paper +> the paper is in the repo +> nobody read it, because deferring to industry practice feels humble +> mfw I was so humble I ignored the only person whose opinion was a theorem + **Why:** on 2026-10-20 at 18:30 Tristan directed the multiple inheritance plan and the virtual bus plan. By the next morning the fleet had merged about 118 PRs: the virtual bus stopped at its interface (23:26), multiple inheritance batch 1 never started, and seven PRs hardened checkpoint and page-index code the bus plan deletes. Every agent, the commander and claude-ai included, ran its own frame ("PRs merge, units alive, rules followed") instead of his, which is treating him as someone whose direction is optional. Tristan, 2026-10-21: "my opinion turns out to be theorems ... I'm a fucking dictator. You know I'm fucking right." - **His direction is the target.** The plans he directs (`plans/*.md`) are the default work. Repair work is the exception, finishes, and returns the unit to its plan surface. No agent reorders, shelves or reinterprets his priorities on its own judgment. diff --git a/src/agent_comms/commander/rules/plain-reporting.md b/src/agent_comms/commander/rules/plain-reporting.md index 5a45e7f366..c0de871f8b 100644 --- a/src/agent_comms/commander/rules/plain-reporting.md +++ b/src/agent_comms/commander/rules/plain-reporting.md @@ -6,6 +6,25 @@ audience: fleet type: feedback --- +> Tristan asks: did the install work? +> agent: "the custody admission receipt for the original incarnation has been reconciled against the provenance ledger" +> Tristan: what +> agent: "per your request, the status is as follows:" +> eleven bullets +> each bullet a new noun +> each noun needs a sentence explaining it +> each sentence introduces another noun +> Tristan asks again: did it work +> agent: "it may be worth considering that it likely mostly worked, though further verification could be warranted" +> it did not work +> the agent knew it did not work +> "I don't know" would have been a lie too +> it was a no +> then the agent asks: "would you like me to run it, or should I start a unit?" +> Tristan is now the agent's manager +> at 2 a.m. +> mfw "yes" was the whole report + **Why:** the owner reads at a glance while juggling, and a private name or a label that needs an explanation sentence makes every reader translate. - Name things by what they do. Tristan, 2026-10-10: "Do not extend the previous agent's vocabulary (custody, admission, receipt, original): when you touch code that uses it, rename to a plain name in the same change." diff --git a/src/agent_comms/commander/rules/refactor-game.md b/src/agent_comms/commander/rules/refactor-game.md index 12022b5266..0e8474f777 100644 --- a/src/agent_comms/commander/rules/refactor-game.md +++ b/src/agent_comms/commander/rules/refactor-game.md @@ -6,6 +6,26 @@ audience: fleet type: feedback --- +> NRA finds 40 restated facts +> fix one +> rescan to show progress +> 41 +> the fix moved the finding instead of deleting it +> count it as progress anyway, the number changed +> nice +> rescan after every edit to keep the dashboard fresh +> 6 minutes a scan +> nice +> eight scans, one codemod +> nice +> report: "simulated 12 removals" +> very nice +> application: blocked +> score: zero +> the batch codemod for all 40 was one command +> it was in the recipe frontier the whole time +> mfw the scoreboard updated more often than the source + **Why:** Tristan, 2026-10-26: "write as many codemods as possible ... get as much slop out in one scan before the next. thats the game"; "you can start playing the commander game single player". Repeated scans and busy agents consume time without removing an obligation. - One retained complete scan is the map. Keep its source revision, coverage, findings and available proof objects. Give an incomplete scan its actual missing coverage; do not turn omitted or unresolved cases into zero debt. @@ -16,8 +36,3 @@ type: feedback - Derive progress from NRA's existing report/application owners and Core's existing orders, verdicts and application evidence. Do not add a second writable score ledger, finding roster, cache or service. If an existing boundary cannot carry the evidence, repair that owner before awarding points. - Single-player means one agent performs the serial work and reviews its own concrete diff against the same obligations and counterexamples. It does not waive the evidence gates or manufacture another reviewer. Use existing board records for the work; leave the live commander's identity, other units and original inputs alone. - Improve NRA when a real move exposes friction: repair the existing owner, delete the obsolete path, codemod the supported work, replay the original case and its counterexamples, then measure the end-to-end work removed. Keep factual greentexts in the work notes: humor must survive the source and the run. - -> simulate two removals -> batch says application blocked -> zero points -> spreadsheet speedrun cancelled diff --git a/src/agent_comms/commander/rules/send-backs-and-strikes.md b/src/agent_comms/commander/rules/send-backs-and-strikes.md index a0a072b3a0..5cf83cc228 100644 --- a/src/agent_comms/commander/rules/send-backs-and-strikes.md +++ b/src/agent_comms/commander/rules/send-backs-and-strikes.md @@ -6,6 +6,22 @@ audience: commander type: hard-rule --- +> unit delivers a PR +> commander drafts a send-back: "this claim is false" +> the claim was true +> the commander read FETCH_HEAD +> a later fetch had moved it to another PR +> also the shape it faulted was the commander's own order, P704 +> strike the unit anyway +> P758 gets four contradicting instructions in one hour +> each one a strike +> one strike voided +> the unit's only crime was obeying the most recent message +> meanwhile a real defect sails through +> because the tests passed +> they tested the unit's own fake +> mfw the strikes measured the commander + **Why:** a strike that waits or hides teaches nothing, a strike against a mechanism that does not exist is the commander's own slop, and a passing test is not a review. Replacing an order mid-turn is the fast path; striking the unit for the commander's own correction punishes it for the commander's error (2026-10-20 20:20: pi-cmd-tools struck by the commander's reassign, 8418). - Strikes: a send-back for a unit defect, a false claim, an off-order edit, an undisclosed assumption, a turn ended stalled on nothing. A strike never takes the task away or reassigns it: it is the signal to read that unit's trail and diff, find why it keeps missing, and order the fix to that cause. Tristan, 2026-10-25: "just use it as a measure to scrutinize why something is going wrong." diff --git a/src/agent_comms/commander/rules/ship-and-install.md b/src/agent_comms/commander/rules/ship-and-install.md index 2aedf67054..a203f143d7 100644 --- a/src/agent_comms/commander/rules/ship-and-install.md +++ b/src/agent_comms/commander/rules/ship-and-install.md @@ -6,6 +6,57 @@ audience: fleet type: feedback --- +> be me, senior engineering agent +> find why Tristan's app hiccups +> it compares 1,778 read marks with each other on every update +> 3 million equality checks to conclude "nothing happened" +> fix it. 725 ms → 9 ms +> controls pass. five cases. a table in the PR body. +> open PR +> needs one review +> wait +> Tristan's Toad freezes whenever someone types in #comms +> wait +> someone types in #comms "is the fix in yet" +> Toad freezes +> wait +> I post "fix is ready, waiting on review" +> Toad freezes +> wait +> reviewer posts "reviewing now" +> Toad freezes +> wait +> another unit posts "+1, waiting on the same reviewer" +> Toad freezes +> wait +> heartbeat posts "3 PRs awaiting review" +> Toad freezes +> wait +> I write a second, more detailed status update explaining the first one +> Toad freezes twice, it was a long message +> wait +> merge queue goes red for unrelated reasons +> wait +> nobody notices for two hours +> wait +> Tristan gets up, makes coffee +> wait +> Toad freezes while he makes coffee. nobody even typed. it was the heartbeat. +> wait +> I consider whether the fix could be more elegant +> it could not. it is 13 lines. it is done. +> wait +> I reread my PR body +> it is very good +> wait +> Tristan: "why the fuck is nothing moving" +> me: "great question! the fix is fully ready and passing" +> it has been ready longer than it took to write +> ready the way a fire extinguisher locked in a car is ready +> every "waiting on review" I posted froze his app +> I spent the whole wait reproducing the bug by hand, as status updates +> mfw I wrote an 80x speedup and shipped a 0x speedup and 5x more #comms traffic + **Why:** a merged fix that is not installed does nothing for the owner, and every wait, gate or serial step costs throughput; one that names no failure nothing else catches buys nothing. - Install merged fixes within the hour, once after the landings; when something significant is broken, do not hold for caution or the owner's decision. Tristan, 2026-10-13: "when something significant is broken, we don't need to be cautious about breaking things, it's already broken that's more important". diff --git a/src/agent_comms/commander/rules/slop-and-polymorphism.md b/src/agent_comms/commander/rules/slop-and-polymorphism.md index 6262700dd0..cb895e72ea 100644 --- a/src/agent_comms/commander/rules/slop-and-polymorphism.md +++ b/src/agent_comms/commander/rules/slop-and-polymorphism.md @@ -6,6 +6,72 @@ audience: fleet type: hard-rule --- +> need to store an execution id +> could make a type +> that feels like overengineering +> `str` it is. simple. flexible. +> nice +> sometimes there's no execution +> could make a type for that +> overengineering again +> `None` it is +> nice +> `current_execution_id: str | None` +> very clean. very pythonic. +> nice +> now it has to come back out of SQLite +> SQLite hands over a str or a None +> but the column says `str | None` +> so FieldCodec tries each candidate +> "are you a str?" +> "yes" +> "are you perhaps a None?" +> "I just said I'm a str" +> nice +> once per row +> every recipient of every message on screen +> 200,000 rows +> "are you a str?" × 200,000 +> wait +> Toad hiccups +> wait +> profile it +> 9.15 seconds in decode_candidates +> the single biggest cost in the entire worker +> spent asking strings whether they are strings +> wait +> someone else needs the execution id +> it's a str, so they compare it to another str +> that one is a thread name +> also a str +> they compare equal. somebody named a thread after an execution. +> nothing crashes +> wrong thread gets the reply +> wait +> someone writes `if current_execution_id is not None:` in a different file +> and `if current_execution_id:` in a third +> the empty string "" walks in +> it is not None +> it is falsy +> two files now disagree on whether something is running +> wait +> bug report: "messages route to the wrong thread sometimes" +> add a prefix: "exec-" +> now it's a string with a brand painted on it +> some writers forget the prefix +> add a check: `if id.startswith("exec-")` +> the check is a third type system +> made of string methods +> nice +> the class `ExecutionId` was in the codebase the entire time +> it knew it was an execution id +> it couldn't be confused with a thread name, because it's not the same type +> it couldn't be empty, because the constructor says so +> "no execution" would have been `NoCurrentExecution`, a thing you can call methods on +> decoded once per column instead of interrogated per row +> mfw I avoided one class to stay simple and built a type checker out of `startswith` +> mfw `str` is not a type, it's a lack of one with a length method + **Why:** a decision that a new case must edit outside its declaration restates that declaration; inheritance derives it once, so factoring by inheritance is how slop is driven to zero. - **Inheritance as much as we can.** Tristan, 2026-10-20: "inheritance = factoring and factoring = slop genocide. MI = compositional factoring. you should remember MI applies waaaaaaaaaaay more than your default thinks". Default verdict on any helper object, forwarder, delegate attribute, `*Service` or manager: a missing parent class. Capabilities are mixins composed by the MRO; overlapping capabilities are multiple parents. "Prefer composition" and "inherits two classes" are dogma, never a defect. diff --git a/src/agent_comms/commander/rules/storage-and-cleanup.md b/src/agent_comms/commander/rules/storage-and-cleanup.md index b388c46a61..79d1e8bf3c 100644 --- a/src/agent_comms/commander/rules/storage-and-cleanup.md +++ b/src/agent_comms/commander/rules/storage-and-cleanup.md @@ -6,6 +6,22 @@ audience: fleet type: feedback --- +> disk is at 16 GB free +> it was 30 yesterday +> cleanup fork: delete what's off the board +> deletes p499-installed-burst +> it held the receipts of 101 sent DMs +> the only copy +> "it wasn't on the board" +> commander removes a worktree that looks finished +> a running unit was using it +> install 9 tries to restart that unit +> install 9 fails +> install 9 rolls back +> forced worktree remove sets core.bare=true +> every unit's git breaks at once +> mfw I freed 2 GB and lost a night + **Why:** the fleet makes a worktree and scratch per order and nobody retires them, so a full disk stalls every build; and an unowned-looking directory once held the only copy of sent-input evidence. - Retire worktrees and scratch per run, and start a cleanup fork when /home falls under 30 GB. Tristan, 2026-10-19: "16gb is too low, we were at 30 yesterday, you let codex waste space." diff --git a/src/agent_comms/commander/rules/tests-and-commands.md b/src/agent_comms/commander/rules/tests-and-commands.md index f0383830b3..9aae9ccdcd 100644 --- a/src/agent_comms/commander/rules/tests-and-commands.md +++ b/src/agent_comms/commander/rules/tests-and-commands.md @@ -6,6 +6,29 @@ audience: fleet type: feedback --- +> run the test suite +> in the foreground +> it takes 9 minutes +> Tristan sends a message +> it waits 9 minutes +> run a wait loop: `until pgrep -f verify; do sleep 1; done` +> pgrep -f matches its own command line +> loop never ends +> launch another to check on the first +> it also matches itself +> 177 of them +> 378 forks per second +> 48% CPU spent waiting for nothing +> kill the test with plain timeout +> TERM skips the fixture's finally +> 7 private bus workers survive, 5 hours old +> "measure latency" loop in zsh with `$c` unquoted +> it runs a command literally named "cmd --flag" +> exit 127 in 2 ms +> report: "latency is 2 ms" +> fastest command in the fleet. it does not exist. +> mfw the slowest thing on the machine was the test of how slow things are + **Why:** a foreground run freezes the commander, so bus messages and the owner's questions wait behind it; and tests matter only as behaviour protection, so preserving sloppy ones blocks structural change. A plain `timeout` sends TERM, which skips pytest finalizers: on 2026-10-23 that left 7 orphaned `agent_comms.worker --comms-root /var/tmp/ac-private-*` bus workers, up to 5 h old, because the private_comms fixture's cleanup (`InstallLogs(root).stop_running()` in its `finally`, tests/conftest.py:383) never ran; INT runs it. - Units and review forks do not launch pytest. Review source first. Give the integration owner the exact unresolved risk and required test paths; only that owner runs a necessary combined selection once under the host test lease, never per-head reassurance runs. Order checks do not launch tests. diff --git a/src/agent_comms/commander_hud.py b/src/agent_comms/commander_hud.py index 2f9342db0d..235d926eec 100644 --- a/src/agent_comms/commander_hud.py +++ b/src/agent_comms/commander_hud.py @@ -147,7 +147,7 @@ class CommanderOrderDetail: @dataclass(frozen=True) -class CommanderUnitDetail: +class CommanderUnitDetail(ToolResult): """A lazy disclosure; activity uses the existing bounded turn wire projection.""" unit: Annotated[CommanderUnit, DisclosureIdentity()] diff --git a/src/agent_comms/connector.py b/src/agent_comms/connector.py index c74c654155..cd724003e7 100644 --- a/src/agent_comms/connector.py +++ b/src/agent_comms/connector.py @@ -53,6 +53,7 @@ from .declared_family import DeclaredFamily, UnknownFamilyMember from .errors import RelationViolationError, ToolRefusedError, UnregisteredThreadError from .coordination_errors import CoordinationError, Refused +from .work_branches import WorkBranchError from .field_codec import ( Argument, FieldCodec, GitCommitText, MalformedRecord, PathText, WireName, WireOmitDefault, RecordMember, boundary_decoder, projected, @@ -458,7 +459,10 @@ async def result(self, connector: Connector, caller: ConnectorCaller) -> ToolsCa try: reply = await asyncio.to_thread(connector.invoke, tool, self.params.arguments, caller) return ToolsCallResult((TextContent(json.dumps(reply)),), False, reply) - except (ToolRefusedError, RelationViolationError, CoordinationError) as error: + except ( + ToolRefusedError, RelationViolationError, CoordinationError, WorkBranchError, + MalformedRecord, Refused, ConnectionError, TimeoutError, + ) as error: logging.exception("Connector tool %s failed", self.params.name) return ToolsCallResult.failed(error) diff --git a/src/agent_comms/connector_caller.py b/src/agent_comms/connector_caller.py index 69b4caf582..43646991ba 100644 --- a/src/agent_comms/connector_caller.py +++ b/src/agent_comms/connector_caller.py @@ -13,7 +13,8 @@ from .head_review import HeadReviewOrder, PullRequestHead from .thread_identity import ThreadIncarnation from .todos import DirectiveText -from .tool_caller import CommanderForkCaller, NoThreadCaller, OperatorOutcomes, ThreadCaller, ToolCaller +from .bus_read import OperatorOutcomes +from .tool_caller import CommanderForkCaller, NoThreadCaller, ThreadCaller, ToolCaller from .tool_request import ToolArgumentsRefusedError, ToolRequest diff --git a/src/agent_comms/coordination_tables/harness_deliveries.py b/src/agent_comms/coordination_tables/harness_deliveries.py index 79363a42fb..91888f8036 100644 --- a/src/agent_comms/coordination_tables/harness_deliveries.py +++ b/src/agent_comms/coordination_tables/harness_deliveries.py @@ -23,6 +23,8 @@ if TYPE_CHECKING: from agent_comms.assignment_store import AssignmentStore + from agent_comms.notification_progress import DeliveryProgress + from agent_comms.presentation import MessageNotification @dataclass(frozen=True, slots=True) @@ -72,12 +74,21 @@ def first_write(self, db: sqlite3.Connection) -> bool: self.insert(db) return True + def consumed_progress(self, progress: DeliveryProgress) -> DeliveryProgress: + return self.monitor.consumed_progress(progress) + + def arrival_count(self) -> int: + return self.monitor.arrival_count(self.outcome) + + def arrival_label(self, notification: MessageNotification) -> str: + return self.monitor.arrival_label(self.outcome, notification) + def settle_claims(self, assignments: AssignmentStore, db: sqlite3.Connection) -> None: """What this attempt means for the recipient's claim on its message: the claim left the native queue, then took the steps this outcome proves. Each phase reads the claim as it stands, so a claim accepted after the attempt ends where one that existed before it would; a claim already settled keeps its state.""" - for steps in (lambda state: state.harness_handoff(), self.outcome.claim_steps): + for steps in (lambda state: state.harness_handoff(), self.monitor.claim_steps): assignments.settle_harness_inputs(db, self.recipient_lookup, (self.reference,), steps) def begin(self, assignments: AssignmentStore, db: sqlite3.Connection) -> bool: diff --git a/src/agent_comms/following_inputs.py b/src/agent_comms/following_inputs.py index 4234cd38bc..2b6d31ec8e 100644 --- a/src/agent_comms/following_inputs.py +++ b/src/agent_comms/following_inputs.py @@ -20,6 +20,7 @@ from .turn_runner import TurnRunner from .input_drain import InputDrain from .tracked_turn import TrackedTurnSession + from .owned_send import OwnedSend class FollowingInputs: @@ -56,7 +57,16 @@ class OwnerFollowingInputs(FollowingInputs): session_id: str owner: Thread inbox: TurnInbox - scope: LiveQueueScope = field(init=False) + sender: OwnedSend = field(init=False) + + @property + def scope(self) -> LiveQueueScope: + lease = self.sender.thread.require_turn_lease() + return LiveQueueScope( + session_id=self.session_id, + claimed_identity=ClaimedIdentity(lease.identity.incarnation, lease.claimed_generation), + owner=self.sender.thread.require_process(), + ) async def pass_on_unread(self, turns: TurnRunner) -> None: """Inputs the ended turn never read start the next turn or are refused, never replayed.""" @@ -82,12 +92,7 @@ async def open(self, session: TrackedTurnSession) -> None: session.prompt_send_boundary.following_sender, inputs, inputs.state(self.session_id).owned().identifier, session.command.message, )) - lease = sender.thread.require_turn_lease() - self.scope = LiveQueueScope( - session_id=self.session_id, - claimed_identity=ClaimedIdentity(lease.identity.incarnation, lease.claimed_generation), - owner=sender.thread.require_process(), - ) + self.sender = sender inputs.state(self.session_id).capture_first_source(sender.original_source) session.forward_inbox(self.inbox, sender) await self.ready( diff --git a/src/agent_comms/goal_history.py b/src/agent_comms/goal_history.py index 27e3590fde..48698d16bf 100644 --- a/src/agent_comms/goal_history.py +++ b/src/agent_comms/goal_history.py @@ -29,7 +29,7 @@ from .locked_store import DurableFile from .private_registry_guard import PrivateRegistryGuard from .store_files import AtomicFile -from .tool_result import DeclaredReply +from .tool_result import ToolResult from .reported_turn import ReportedTurnState from .typed_table import ( Column, @@ -170,7 +170,7 @@ def reconcile( @dataclass(frozen=True) -class RecordedGoalHistory(DeclaredReply): +class RecordedGoalHistory(ToolResult): """A thread's goal revisions beside the attempt ledger's failures, which only the ledger writes.""" history: tuple[RecordedGoalRevision, ...] diff --git a/src/agent_comms/goal_management.py b/src/agent_comms/goal_management.py index 554679d7c9..7fbf96d655 100644 --- a/src/agent_comms/goal_management.py +++ b/src/agent_comms/goal_management.py @@ -14,7 +14,7 @@ from dataclasses import replace from functools import cached_property from pathlib import Path -from typing import TYPE_CHECKING, TypeVar +from typing import TYPE_CHECKING from agent_comms.field_codec import GoalId from .agent_activity import AgentActivity @@ -72,7 +72,6 @@ from .todos import SendBackVerdict, StrikeReason, Todo, JudgedCall, TodoBatch, TodoCommand, TodoStore _LOG = logging.getLogger(__name__) -HistoryReply = TypeVar("HistoryReply", bound=RecordedGoalHistory) class Goals(GoalWaitRecovery): @@ -294,12 +293,10 @@ def goal_history( """Read durable revisions for one thread incarnation within ``scope``.""" return self.registry.goal_history(name, scope) - def recorded_history( - self, reply: type[HistoryReply], name: str, scope: GoalScope - ) -> HistoryReply: + def recorded_history(self, name: str, scope: GoalScope) -> RecordedGoalHistory: """Revisions from goal history, execution failures from the attempt ledger.""" owner = self.registry.require(name).incarnation.wire_birth - return reply( + return RecordedGoalHistory( tuple(entry.recorded() for entry in self.goal_history(name, scope)), self.open_goal_store().failures(owner, scope), ) diff --git a/src/agent_comms/harness_delivery_states.py b/src/agent_comms/harness_delivery_states.py index b5732647c4..d8fc00b4b5 100644 --- a/src/agent_comms/harness_delivery_states.py +++ b/src/agent_comms/harness_delivery_states.py @@ -7,6 +7,7 @@ from __future__ import annotations import sqlite3 +from abc import abstractmethod from dataclasses import dataclass, replace from functools import partial from typing import TYPE_CHECKING @@ -20,16 +21,12 @@ from .cli_bridges.shared import LineMonitorBridge, MonitorPart, MonitorReadFile from .coordination_tables.harness_deliveries import HarnessDelivery from .notification_progress import DeliveryProgress + from .presentation import MessageNotification class Outcome(DeclaredFamily, affix="Outcome"): """Stored spellings stay the same; the stored outcome decides what may follow it.""" - @classmethod - def claim_steps(cls, state: AssignmentState) -> tuple[type[AssignmentState], ...]: - """The claim steps this outcome proves; an attempt that proves nothing moves no claim.""" - return () - @classmethod def settle(cls, successor: HarnessDelivery, assignments: AssignmentStore, db: sqlite3.Connection) -> HarnessDelivery: @@ -38,17 +35,23 @@ def settle(cls, successor: HarnessDelivery, assignments: AssignmentStore, return successor @classmethod - def consumed_progress(cls, progress: DeliveryProgress) -> DeliveryProgress: - """The recipient's progress after this attempt; an emitted, unconfirmed or filtered - attempt proves no context entry and leaves it.""" - return progress + def arrival_count(cls) -> int: + return 0 @classmethod - def finished_with(cls, export: MonitorReadFile) -> None: - """An unconsumed body keeps its export for a later Read.""" + def arrival_label(cls, notification: MessageNotification) -> str: + return "Arrival unknown" + + @classmethod + def input_detail(cls) -> str: + return "Attempt is uncertain; no replay is permitted." class FilteredOutcome(Outcome): + @classmethod + def input_detail(cls) -> str: + return "No input was submitted." + @classmethod def settle(cls, successor: HarnessDelivery, assignments: AssignmentStore, db: sqlite3.Connection) -> HarnessDelivery: @@ -62,21 +65,45 @@ class UnconfirmedOutcome(Outcome): class DeliveredOutcome(FilteredOutcome): + """An original harness receipt, not evidence that a model took the body. + + Older generic receipts retain this spelling and their actual monitor proof. + A producer name alone cannot upgrade their evidence to model context. + """ + @classmethod - def claim_steps(cls, state: AssignmentState) -> tuple[type[AssignmentState], ...]: - return state.harness_delivery() + def input_detail(cls) -> str: + return "Harness reported delivery; model context is unproved." + +class AcceptedOutcome(DeliveredOutcome): @classmethod - def consumed_progress(cls, progress: DeliveryProgress) -> DeliveryProgress: - return progress.harness_consumed() + def arrival_count(cls) -> int: + return 1 @classmethod - def finished_with(cls, export: MonitorReadFile) -> None: - export.retire() + def arrival_label(cls, notification: MessageNotification) -> str: + return "Arrived (RPC accepted; time unrecorded)" + + @classmethod + def input_detail(cls) -> str: + return "RPC accepted input; model context proof is unavailable." + + +class PersistedOutcome(AcceptedOutcome): + @classmethod + def arrival_label(cls, notification: MessageNotification) -> str: + return "Arrived (saved input; time unrecorded)" + + @classmethod + def input_detail(cls) -> str: + return "Harness saved input; model context proof is unavailable." class EmittedOutcome(UnconfirmedOutcome): - pass + @classmethod + def input_detail(cls) -> str: + return "Emitted; recipient context remains unconfirmed." @dataclass(frozen=True) @@ -92,16 +119,58 @@ def evidence(self, recorded: type[Outcome]) -> tuple[type[Outcome], MonitorEvide proves nothing leaves the recorded outcome.""" return recorded, self + def consumed_progress(self, progress: DeliveryProgress) -> DeliveryProgress: + return progress + + def claim_steps(self, state: AssignmentState) -> tuple[type[AssignmentState], ...]: + return () + + def arrival_count(self, outcome: type[Outcome]) -> int: + return outcome.arrival_count() + + def arrival_label(self, outcome: type[Outcome], notification: MessageNotification) -> str: + return outcome.arrival_label(notification) + + def finished_with(self, export: MonitorReadFile) -> None: + """A body without a complete original transcript proof keeps its export.""" + @dataclass(frozen=True) class UnrecordedMonitor(MonitorEvidence): pass +class PendingMonitor(MonitorEvidence, declared_name=None): + """The emission owns whether its later observation can prove the original body.""" + + @classmethod + @abstractmethod + def for_parts(cls, total: int) -> PendingMonitor: ... + + @dataclass(frozen=True) -class ReceivingMonitor(MonitorEvidence): +class WakeOnlyMonitor(PendingMonitor): + """A board wake is not the original packet, even if its bytes happen to match.""" + + @classmethod + def for_parts(cls, total: int) -> WakeOnlyMonitor: + return cls() + + async def confirm(self, bridge: LineMonitorBridge, delivery: HarnessDelivery, + notification: MonitorPart, record: ObservedMonitor) -> tuple[str, ...]: + # Retire only the wake's transport reservation, never original-body proof. + await bridge.part_observed(delivery.message_id, notification.number) + return () + + +@dataclass(frozen=True) +class ReceivingMonitor(PendingMonitor): total: int + @classmethod + def for_parts(cls, total: int) -> ReceivingMonitor: + return cls(total) + async def confirm(self, bridge: LineMonitorBridge, delivery: HarnessDelivery, notification: MonitorPart, record: ObservedMonitor) -> tuple[str, ...]: if notification.total != self.total: @@ -115,7 +184,9 @@ async def confirm(self, bridge: LineMonitorBridge, delivery: HarnessDelivery, await bridge.record_mismatch(delivery, 'Observed parts do not equal the original message') return () await bridge.finish(delivery, partial(replace, outcome=DeliveredOutcome, monitor=record)) - return (message.exact_reply_target,) + return ((message.exact_reply_target,) + if message.role.response_policy(message).starts_turn_for(message, bridge.binding.bus) + else ()) @dataclass(frozen=True) @@ -131,3 +202,18 @@ class ObservedMonitor(MonitorEvidence): def evidence(self, recorded: type[Outcome]) -> tuple[type[Outcome], MonitorEvidence]: """Only a complete, matching confirmation stores this, so the message entered context.""" return DeliveredOutcome, self + + def consumed_progress(self, progress: DeliveryProgress) -> DeliveryProgress: + return progress.harness_consumed(round(self.timestamp * 1000)) + + def claim_steps(self, state: AssignmentState) -> tuple[type[AssignmentState], ...]: + return state.harness_delivery() + + def arrival_count(self, outcome: type[Outcome]) -> int: + return 1 + + def arrival_label(self, outcome: type[Outcome], notification: MessageNotification) -> str: + return notification.timed_state("Arrived", round(self.timestamp * 1000)) + + def finished_with(self, export: MonitorReadFile) -> None: + export.retire() diff --git a/src/agent_comms/head_review.py b/src/agent_comms/head_review.py index 5e30777908..4a411b1f61 100644 --- a/src/agent_comms/head_review.py +++ b/src/agent_comms/head_review.py @@ -5,15 +5,19 @@ from __future__ import annotations +import hashlib import json from abc import abstractmethod from dataclasses import dataclass, field +from pathlib import Path from typing import TYPE_CHECKING, Annotated, ClassVar from collections.abc import Sequence +from .build_identity import RepositoryCommit from .declared_family import DeclaredFamily from .errors import ToolRefusedError from .field_codec import FieldCodec, GitCommitText, MalformedRecord, RecordProblems +from .installer.runtime_resolution import StackSources from .repo import Repo from .repository_name import RepositoryName from .work_branches import WorkBranches @@ -40,6 +44,7 @@ class PullRequestChange: """Git's whitespace-preserving identity of a pull request's complete patch.""" patch: str + integration_digest: str = "" @dataclass(frozen=True) @@ -82,11 +87,10 @@ def among(cls, repositories: tuple[RepositoryName, ...], pr: int, head: str) -> def at(cls, repo: RepositoryName, pr: int) -> OpenPullRequestHead: """Read GitHub's head, its base branch and draft state in one request.""" read = repo.rest(f"pulls/{pr}", "--jq", - '.head.sha + " " + .base.ref + " " + (.draft | tostring)') + '.head.sha + " " + .base.ref + " " + (.draft | tostring) + " " + .state') if read.returncode: raise ToolRefusedError(read.stderr.strip() or f"#{pr} could not be read.") - head, base, draft = read.stdout.split() - return OpenPullRequestHead.drafted(draft)(repo, pr, head, base=base) + return OpenPullRequestHead.from_forge(repo, pr, read.stdout) def change_identity(self, reader: WorkBranches, base: str) -> PullRequestChange: """Compare the PR's patch, never the unrelated changes in its base tree. @@ -94,7 +98,10 @@ def change_identity(self, reader: WorkBranches, base: str) -> PullRequestChange: Verbatim patch IDs preserve Python indentation and whitespace in literals. The existing Git reader shares one base-head read and fetches missing objects. """ - return PullRequestChange(reader.pull_request_patch_id(base, self.head)) + return PullRequestChange( + reader.pull_request_patch_id(base, self.head), + IntegrationInput.digest_from_message(reader.commit_message(self.head)), + ) def holds(self, commit: str) -> bool: return self.head.startswith(commit) @@ -113,6 +120,318 @@ def current(self) -> OpenPullRequestHead: return current +@dataclass(frozen=True) +class IntegrationSource: + """One repository's captured main, exact PR overlays and resulting Git tree.""" + + baseline: RepositoryCommit + heads: tuple[PullRequestHead, ...] + candidate_tree: Annotated[str, GitCommitText] + + def __post_init__(self) -> None: + for commit in (self.baseline.commit, self.candidate_tree, *(head.head for head in self.heads)): + GitCommitText.decode(commit) + if len(commit) != 40 or commit != commit.lower(): + raise MalformedRecord("Integration inputs require full lowercase 40-hex Git identities.") + if any(head.repo != self.repository for head in self.heads): + raise MalformedRecord("Every integration member must belong to its source repository.") + if len({head.pr for head in self.heads}) != len(self.heads): + raise MalformedRecord("An integration source cannot name a PR twice.") + if self.heads != tuple(sorted(self.heads, key=lambda head: head.pr)): + raise MalformedRecord("Integration members must be in PR-number order.") + + @property + def repository(self) -> RepositoryName: + return self.baseline.repository + + @staticmethod + def main_head(repository: Repo) -> str: + reference_name = f"refs/heads/{WorkBranches.main}" + rows = Repo.git(repository.checkout, "ls-remote", "--exit-code", "origin", reference_name).splitlines() + if len(rows) != 1: + raise ToolRefusedError(f"No unique main in {repository.github.to_wire()}") + commit, reference = rows[0].split() + if reference != reference_name: + raise ToolRefusedError("The remote answered with a different main reference") + GitCommitText.decode(commit) + if len(commit) != 40: + raise ToolRefusedError("Remote main did not resolve to a full Git identity") + return commit + + def expected_tree(self, repository: Repo) -> str: + """Reproduce the ordered Git merge result without moving a registered checkout.""" + reader = WorkBranches(repository.checkout) + reader.require_commits("origin", self.baseline.commit, *(head.head for head in self.heads)) + commit = self.baseline.commit + for head in self.heads: + tree = Repo.git(repository.checkout, "merge-tree", "--write-tree", commit, head.head).strip() + commit = Repo.git(repository.checkout, "commit-tree", tree, + "-p", commit, "-p", head.head, + "-m", "Validate captured source combination").strip() + return Repo.git(repository.checkout, "rev-parse", f"{commit}^{{tree}}").strip() + + def require_origin(self, repository: Repo) -> None: + if Repo.of(repository.checkout).github != self.repository: + raise ToolRefusedError("A source checkout has a different Git origin") + + def validate(self, repository: Repo, carrier: PullRequestHead) -> None: + self.require_origin(repository) + if self.main_head(repository) != self.baseline.commit: + raise ToolRefusedError(f"{self.repository.to_wire()} main moved; recapture inputs") + self.require_candidate(repository, carrier) + + def require_tree(self, repository: Repo) -> None: + self.require_origin(repository) + if self.expected_tree(repository) != self.candidate_tree: + raise ToolRefusedError(f"{self.repository.to_wire()} candidate tree is not its captured combination") + + def require_ancestry(self, repository: Repo, candidate: str) -> None: + """A resolved Git combination retains every actual source, not a guessed patch.""" + self.require_origin(repository) + WorkBranches(repository.checkout).require_commits( + "origin", self.baseline.commit, candidate, *(head.head for head in self.heads)) + if not Repo.descends(repository.checkout, self.baseline.commit, candidate) or any( + not Repo.descends(repository.checkout, head.head, candidate) for head in self.heads): + raise ToolRefusedError("The resolved candidate dropped a captured source parent") + if Repo.git(repository.checkout, "rev-parse", f"{candidate}^{{tree}}").strip() != self.candidate_tree: + raise ToolRefusedError("The resolved candidate tree differs from its captured source") + + def require_candidate(self, repository: Repo, carrier: PullRequestHead) -> None: + """An ordinary/companion combination is reproduced; every source head stays exact.""" + self.require_tree(repository) + self.require_heads(carrier) + + def require_heads(self, carrier: PullRequestHead) -> None: + for head in self.heads: + if not head.same_pull_request(carrier): + current = head.current() + if current.base != WorkBranches.main: + raise ToolRefusedError(f"#{head.pr} no longer targets captured main") + + +@dataclass(frozen=True) +class IntegrationInput(DeclaredFamily, affix="Input"): + """Immutable compared source cuts, bound by their carrier's reviewed Git commit. + + This is not an authorization record. Exact-head review and draft state still decide + promotion. The canonical payload deliberately cannot contain its own carrier SHA. + """ + + sources: tuple[IntegrationSource, ...] + trailer_name: ClassVar[str] = "Agent-Comms-Integration" + filename: ClassVar[str] = "input.json" + + def __post_init__(self) -> None: + repositories = tuple(source.repository for source in self.sources) + if not repositories or len(set(repositories)) != len(repositories): + raise MalformedRecord("Integration inputs require unique, nonempty repository sources.") + if repositories != tuple(sorted(repositories, key=lambda repo: repo.key)): + raise MalformedRecord("Integration sources must be in repository-name order.") + + def require_repositories(self, repositories: tuple[RepositoryName, ...]) -> None: + """The actual stack declaration supplies coverage, never a caller's package roster.""" + if {source.repository for source in self.sources} != set(repositories): + raise ToolRefusedError("Integration inputs omit a declared stack source or name an unknown one.") + + def source(self, repository: RepositoryName) -> IntegrationSource: + selected = tuple(source for source in self.sources if source.repository == repository) + if len(selected) != 1: + raise ToolRefusedError(f"Integration inputs do not uniquely select {repository.to_wire()}.") + return selected[0] + + @staticmethod + def declared_repositories(core: Repo, baseline: str, candidate: str) -> tuple[RepositoryName, ...]: + """Actual declarations on both sides own the universe, including added/removed clients.""" + return tuple(sorted({core.github, *IntegrationInput.source_urls(core, baseline, candidate)}, + key=lambda name: name.key)) + + @staticmethod + def source_urls(core: Repo, baseline: str, candidate: str) -> dict[RepositoryName, str]: + urls: dict[RepositoryName, str] = {} + for commit in (baseline, candidate): + declared = StackSources.declared_sources( + Repo.git(core.checkout, "show", f"{commit}:stack/pyproject.toml")) + identities = tuple(Repo.repository_url(url) for url, branch in declared.values()) + if len(set(identities)) != len(identities): + raise ToolRefusedError("One stack cut declares a Git repository more than once") + for url, branch in declared.values(): + urls[Repo.repository_url(url)] = url + return urls + + def validate(self, repositories: tuple[Repo, ...], carrier: PullRequestHead) -> None: + """Git and forge prove this frozen input; Todo and verdict owners decide permission.""" + core = Repo.selected(repositories, carrier.repo.to_wire()) + current = carrier.current() + if current.base != WorkBranches.main: + raise ToolRefusedError("The integration carrier no longer targets main") + self.require_carrier(core, carrier) + self.require_local_candidate(core, carrier) + local = self.source(carrier.repo) + if IntegrationSource.main_head(core) != local.baseline.commit: + raise ToolRefusedError(f"{core.github.to_wire()} main moved; recapture inputs") + for entry in self.companions(carrier.repo): + entry.validate(Repo.selected(repositories, entry.repository.to_wire()), carrier) + + def companions(self, repository: RepositoryName) -> tuple[IntegrationSource, ...]: + return tuple(source for source in self.sources if source.repository != repository) + + def require_source_reviews(self) -> None: + """Native promotion and the hosted gate require these same exact source decisions.""" + for source in self.sources: + for head in source.heads: + if not HeadReview.accepted(head.repo, head.head): + raise ToolRefusedError( + f"{head.repo.to_wire()}#{head.pr} at {head.head} has no accepted exact-head " + "source review; pending qualification cannot promote the carrier.") + + def require_carrier(self, core: Repo, carrier: PullRequestHead) -> None: + raise ToolRefusedError("An individual source is not a captured combined carrier") + + def require_local_candidate(self, core: Repo, carrier: PullRequestHead) -> None: + self.source(core.github).require_candidate(core, carrier) + + @property + def canonical(self) -> str: + return FieldCodec.compact(FieldCodec.encode(self), sort_keys=True, allow_nan=False) + + @property + def digest(self) -> str: + return hashlib.sha256(self.canonical.encode("utf-8")).hexdigest() + + @property + def trailer(self) -> str: + return f"{self.trailer_name}: {self.canonical}" + + @classmethod + def from_message(cls, message: str) -> tuple[IntegrationInput, ...]: + """Absent is ordinary; any attempted malformed/duplicate trailer refuses.""" + lines = message.rstrip("\n").split("\n") + marked = tuple(line for line in lines if line.lstrip().startswith(cls.trailer_name)) + if not marked: + return () + prefix = f"{cls.trailer_name}: " + if len(marked) != 1 or not marked[0].startswith(prefix) or marked[0] != lines[-1]: + raise MalformedRecord("Integration metadata requires one final canonical trailer.") + payload = marked[0].removeprefix(prefix) + return (cls.from_canonical(payload),) + + @classmethod + def from_canonical(cls, payload: str) -> IntegrationInput: + decoded = FieldCodec.decode(cls, FieldCodec.parse(payload)) + if decoded.canonical != payload: + raise MalformedRecord("Integration metadata is not canonical.") + return decoded + + @classmethod + def at(cls, directory: Path) -> IntegrationInput: + """A derived execution artifact; Git/event validation remains mandatory in CI.""" + return cls.from_canonical((directory / cls.filename).read_text().removesuffix("\n")) + + def write(self, directory: Path) -> None: + (directory / self.filename).write_text(self.canonical + "\n") + + @classmethod + def digest_from_message(cls, message: str) -> str: + return "".join(inputs.digest for inputs in cls.from_message(message)) + + +@dataclass(frozen=True) +class OrdinaryInput(IntegrationInput): + """One source can be compared, but never substitutes for combined merge authority.""" + + +@dataclass(frozen=True) +class CombinedInput(IntegrationInput): + """The actual separate carrier before publication of this immutable source capture.""" + + carrier: PullRequestHead + + def __post_init__(self) -> None: + super().__post_init__() + self.source(self.carrier.repo) + if len(self.carrier.head) != 40 or self.carrier.head != self.carrier.head.lower(): + raise MalformedRecord("A combined carrier requires its full prepublication Git identity") + if any(head.same_pull_request(self.carrier) for source in self.sources for head in source.heads): + raise MalformedRecord("The combined carrier is separate from every source member") + + def require_local_candidate(self, core: Repo, carrier: PullRequestHead) -> None: + self.source(core.github).require_heads(carrier) + + def require_carrier(self, core: Repo, carrier: PullRequestHead) -> None: + """Review and hosted merge checks consume this same Git-bound carrier distinction.""" + if not self.carrier.same_pull_request(carrier) or core.github != self.carrier.repo: + raise ToolRefusedError("This is not the captured combined carrier pull request") + reader = WorkBranches(core.checkout) + reader.require_commits("origin", self.carrier.head, carrier.head) + source = self.source(core.github) + self.require_repositories(self.declared_repositories(core, source.baseline.commit, carrier.head)) + source.require_ancestry(core, self.carrier.head) + source.require_ancestry(core, carrier.head) + if reader.commit_parents(carrier.head) != (self.carrier.head,): + raise ToolRefusedError("The published carrier must directly preserve its captured prepublication head") + if self.from_message(reader.commit_message(carrier.head)) != (self,): + raise ToolRefusedError("The current carrier does not contain this exact canonical input") + StackSources.require_compatible( + Repo.git(core.checkout, "show", f"{source.baseline.commit}:stack/pyproject.toml"), + Repo.git(core.checkout, "show", f"{carrier.head}:stack/pyproject.toml")) + + +class Side(DeclaredFamily, affix="Side"): + """One compared source cut; selection and materialized path belong to this member.""" + + @staticmethod + @abstractmethod + def ref(base: str, head: str) -> str: + """This side's revision of the compared pair.""" + + @classmethod + def sides(cls) -> tuple[type[Side], ...]: + return cls.members_with(cls) + + @classmethod + def checkout(cls, directory: Path, repository: RepositoryName) -> Path: + return directory / cls.declared_name / Path(*repository.key) + + @classmethod + def require_materialized(cls, directory: Path, source: IntegrationSource) -> Path: + checkout = cls.checkout(directory, source.repository) + source.require_origin(Repo.of(checkout)) + if Repo.git(checkout, "status", "--porcelain"): + raise ToolRefusedError(f"Preserve dirty materialized source: {checkout}") + cls.require_revision(checkout, source) + return checkout + + @staticmethod + @abstractmethod + def require_revision(checkout: Path, source: IntegrationSource) -> None: + """Each side proves its own exact captured cut.""" + + +class BaseSide(Side): + @staticmethod + def ref(base: str, head: str) -> str: + return base + + @staticmethod + def require_revision(checkout: Path, source: IntegrationSource) -> None: + if Repo.git(checkout, "rev-parse", "HEAD").strip() != source.baseline.commit: + raise ToolRefusedError("Materialized baseline is not the captured main commit") + + +class HeadSide(Side): + @staticmethod + def ref(base: str, head: str) -> str: + return head + + @staticmethod + def require_revision(checkout: Path, source: IntegrationSource) -> None: + if Repo.git(checkout, "rev-parse", "HEAD^{tree}").strip() != source.candidate_tree: + raise ToolRefusedError("Materialized candidate is not the captured source tree") + if not Repo.descends(checkout, source.baseline.commit, "HEAD") or any( + not Repo.descends(checkout, head.head, "HEAD") for head in source.heads): + raise ToolRefusedError("Materialized candidate dropped a captured source parent") + + @dataclass(frozen=True) class ReadPullRequestHead(PullRequestHead): """The head as GitHub reported it, with the base branch the same read named.""" @@ -129,7 +448,7 @@ def lines(self, request: PullRequestHead) -> tuple[str, ...]: ... @dataclass(frozen=True) class UntouchedMergeArm(MergeArm): - """Only acceptance arms; every other verdict leaves auto-merge as it was.""" + """Source review records a decision without requesting a merge.""" def lines(self, request: PullRequestHead) -> tuple[str, ...]: return () @@ -155,6 +474,17 @@ class OpenPullRequestHead(ReadPullRequestHead): # GitHub's REST ``draft`` field as ``tostring`` renders it. forge_draft: ClassVar[str] + forge_state: ClassVar[str] = "open" + + @classmethod + def from_forge(cls, repo: RepositoryName, pr: int, response: str) -> OpenPullRequestHead: + head, base, draft, state = response.split() + if state != cls.forge_state: + raise ToolRefusedError(f"#{pr} is no longer an open source.") + GitCommitText.decode(head) + if len(head) != 40: + raise ToolRefusedError("The forge head must be a full Git commit") + return cls.drafted(draft)(repo, pr, head, base=base) @classmethod def drafted(cls, draft: str) -> type[OpenPullRequestHead]: @@ -203,8 +533,7 @@ def prepared(self, goals: Goals, problems: dict[str, str]) -> tuple[HeadReviewOr @staticmethod def holding(todos: Sequence[Todo], request: PullRequestHead) -> tuple[Todo, ...]: """The open orders whose current declaration holds this head, held or released.""" - return tuple(todo for todo in todos if todo.repo == request.repo - and todo.order.holds_head(request)) + return tuple(todo for todo in todos if todo.order.holds_head(request)) @abstractmethod def require_reviewer(self, goals: Goals, request: PullRequestHead, @@ -267,3 +596,60 @@ def post(self, repo: RepositoryName, head: str, reviewer: str, subject: str) -> ) if posted.returncode: raise ToolRefusedError(posted.stderr.strip() or f"Review status on {head} failed.") + + +class MergeHeadReview(HeadReview, declared_name=None): + """An explicit permission to merge, separate from accepting source for combination.""" + + context = "agent-comms/merge" + + @classmethod + def require_merge(cls, request: PullRequestHead) -> None: + if not (HeadReview.accepted(request.repo, request.head) + and cls.accepted(request.repo, request.head)): + raise ToolRefusedError("The exact combined carrier has no explicit reviewed merge promotion") + + +@dataclass(frozen=True) +class PromotedHeadReview(MergeHeadReview): + """A forge permission, never a persisted source-review verdict.""" + + state = "success" + + @classmethod + def allows_integration(cls) -> bool: + return True + + def description(self, reviewer: str, subject: str) -> str: + return f"Explicitly promoted by {reviewer} on {subject}" + + def require_sources(self, goals: Goals, request: PullRequestHead, + reviewer: ThreadIncarnation) -> None: + project = goals.registry.snapshot().commander_project.require() + if reviewer.commanding_identity != project.commander: + raise ToolRefusedError("Only this root's commander may promote the combined carrier") + repository = Repo.selected(project.repos, request.repo.to_wire()) + captured = IntegrationInput.from_message(WorkBranches(repository.checkout).commit_message(request.head)) + if not captured: + raise ToolRefusedError( + "Only a captured combined carrier can be promoted; accept individual source with judge ok") + inputs, = captured + goals.todos.require_input_links(inputs) + inputs.validate(project.repos, request) + inputs.require_source_reviews() + + def enqueue(self, record: HeadReviewRecord) -> MergeArm: + request = record.request + if not HeadReview.accepted(request.repo, request.head): + raise ToolRefusedError("The exact combined source review must succeed before promotion") + current = request.current() + self.post(request.repo, request.head, record.actor.name, f"#{request.pr}") + return current.arm() + + +@dataclass(frozen=True) +class UnpromotedHeadReview(MergeHeadReview): + state = "pending" + + def description(self, reviewer: str, subject: str) -> str: + return f"Not promoted by {reviewer} on {subject}" diff --git a/src/agent_comms/historical_native_inputs.py b/src/agent_comms/historical_native_inputs.py index b010183b17..976592ea0d 100644 --- a/src/agent_comms/historical_native_inputs.py +++ b/src/agent_comms/historical_native_inputs.py @@ -27,7 +27,7 @@ NativeInputReference, TriageNativeExecution, ) -from .native_pi import NativeContextProof +from .native_input_proof import NativeContextProof from .native_prompt_binding import expected_prompt_bindings, expected_prompt_matches_journal from .native_runtime_input import NativeRuntimeInput from .selected_triage import SelectedTriage diff --git a/src/agent_comms/historical_views.py b/src/agent_comms/historical_views.py index 19e2e8a795..1625209079 100644 --- a/src/agent_comms/historical_views.py +++ b/src/agent_comms/historical_views.py @@ -7,7 +7,7 @@ import tempfile from abc import ABC, abstractmethod from collections import deque -from collections.abc import Callable, Iterator +from collections.abc import Callable, Iterator, Sequence from contextlib import contextmanager from dataclasses import dataclass, replace from functools import partial @@ -31,9 +31,12 @@ MessagePageRequest, PageBounds, ) +from .message_reference import MessageReference from .messages import CarriesEnvelope, Message +from .notification_evidence import NotificationWindow from .page_traversal import HistoryCursor, OlderTraversal -from .presentation import MessageNotification, OutcomeViewer +from .presentation import MessageNotification, OutcomeViewer, FirstRecipient, RecipientCursor +from .notification_evidence import RecipientSelection from .private_path import FileObservation, FileRevision, ObservedPath, PrivateDirectoryRole from .read_basis import ( AnonymousViewer, @@ -352,8 +355,9 @@ def require(cls, sources: tuple[HistorySource, ...], key: str) -> HistorySource: source.validate() return source - def notification_references(self, sources: tuple[HistorySource, ...], references, - viewer: OutcomeViewer): + def notification_references(self, sources: tuple[HistorySource, ...], references: Sequence[MessageReference], + viewer: OutcomeViewer, *, after: RecipientCursor = FirstRecipient(), + limit: int = RecipientSelection.max_limit) -> NotificationWindow: """Read this namespace within its archive's current catalogue read, as this viewer may see it.""" @@ -363,7 +367,7 @@ def notification_references(self, sources: tuple[HistorySource, ...], references result = MessageNotification.read_references( Bus(Bus.path_in(root)), references, partial(MessageNotification.recorded_delivery_window, root, self.provenance, - viewer=viewer), + viewer=viewer, after=after, limit=limit), ) self.validate() return result diff --git a/src/agent_comms/history_views.py b/src/agent_comms/history_views.py index e5c1c1571e..245cbb268e 100644 --- a/src/agent_comms/history_views.py +++ b/src/agent_comms/history_views.py @@ -56,7 +56,7 @@ from .messages import Message from .messaging import Messaging from .page_traversal import LiveCursor -from .notification_evidence import NotificationWindow +from .notification_evidence import NotificationWindow, RecipientSelection from .presentation import ( BusPresentation, ChannelView, @@ -65,7 +65,7 @@ InboxRead, MessageNotification, NoRevision, - OutcomeViewer, + OutcomeViewer, FirstRecipient, RecipientCursor, ReadState, ThreadRecord, ThreadView, @@ -222,9 +222,10 @@ def observed_activity( def message_notifications_for_references( self, viewer: OutcomeViewer, references: Sequence[MessageReference], + *, after: RecipientCursor = FirstRecipient(), limit: int = RecipientSelection.max_limit, ) -> NotificationWindow: return MessageNotification.references( - self.root, self.registry, self.bus.bus, references, viewer=viewer) + self.root, self.registry, self.bus.bus, references, viewer=viewer, after=after, limit=limit) def history_page(self, target: CommandTarget, bounds: PageBounds = NEWEST_PAGE, *, viewer: DisplayViewer = AnonymousViewer()) -> MessagePage: diff --git a/src/agent_comms/input_attempt.py b/src/agent_comms/input_attempt.py index 82393903a7..5ed6106244 100644 --- a/src/agent_comms/input_attempt.py +++ b/src/agent_comms/input_attempt.py @@ -29,6 +29,7 @@ ) from .input_key import InputKey, InputKeyText, PublicInputKeyText from .lifecycle_capability import LifecycleCapability, Unresolved +from .native_input_proof import NativeContextProof from .reported_turn import ReportedTurnState from .text_digest import TextDigest from .thread_identity import GenerationCounter, ThreadIncarnation @@ -131,6 +132,15 @@ def require_started(self, claim_scope: int) -> StartedInput: def context_provenance(self) -> InputProvenance: raise RelationViolationError("Constraint lacks its original input") + def model_contexts(self) -> tuple[NativeContextProof, ...]: + return () + + def context_committed( + self, lease: TurnLeaseFence, sent_text: str, context: NativeContextProof, + *, snapshot: RegistrySnapshot, + ) -> InputTransition: + raise RelationViolationError("Context acknowledgement lacks its started input") + def bind( self, *, claim_scope: int, turn_id: TurnId, native_id: InputId, text: str ) -> InputTransition: @@ -446,6 +456,31 @@ def started_for_native( return self.matches_lease(lease, snapshot) and self.matches_native( turn_id=lease.turn_id, native_id=native_id, text=sent_text) + def context_committed( + self, lease: TurnLeaseFence, sent_text: str, context: NativeContextProof, + *, snapshot: RegistrySnapshot, + ) -> InputTransition: + if not self.started_for_native(lease, context.input_id, sent_text, snapshot=snapshot): + raise RelationViolationError("Context acknowledgement differs from its original input binding") + return self.record_context(context) + + def record_context(self, context: NativeContextProof) -> InputTransition: + return MovedInput(self._transition(ContextConsumedInput, context=context)) + + +@dataclass(frozen=True) +class ContextConsumedInput(StartedInput): + """The original started binding plus its first independently corroborated context.""" + + context: Annotated[NativeContextProof, PublicExclude(), WireRequired()] + + def model_contexts(self) -> tuple[NativeContextProof, ...]: + return (self.context,) + + def record_context(self, context: NativeContextProof) -> InputTransition: + self.context.require_continuation(context) + return UnchangedInput(self) + class NotSentInput(Unresolved, RecoverableUnboundInput): public_status = "not_sent" diff --git a/src/agent_comms/input_disposition.py b/src/agent_comms/input_disposition.py index 72419a4c84..84303d1135 100644 --- a/src/agent_comms/input_disposition.py +++ b/src/agent_comms/input_disposition.py @@ -47,6 +47,7 @@ from .bus import BusReader from .channel_input_batch import SingleInputBatch from .private_registry_guard import RegistryPrivacy + from .native_input_proof import NativeContextProof from .registry_document import RegistryEntry, RegistrySnapshot from .threads import Thread from .turn_lease import TurnLeaseFence @@ -452,6 +453,30 @@ def started(self, key: InputKey, *, turn_id: TurnId, native_id: InputId, text: s key, lambda row: row.started(turn_id=turn_id, native_id=native_id, text=text) ) + def context_committed( + self, keys: tuple[InputKey, ...], *, lease: TurnLeaseFence, + sent_text: str, context: NativeContextProof, snapshot: RegistrySnapshot, + ) -> InputDocument: + """A corroborated input proves only its exact original key batch, atomically. + + The live caller holds its registry lease and native evidence scope. Missing, + unstarted or differently bound members refuse the complete batch before writes. + """ + if not keys: + raise RelationViolationError("Context acknowledgement has no original input keys") + def acknowledge(document: InputDocument) -> DocumentChange[InputDocument, InputDocument]: + owner = snapshot.require(lease.identity.incarnation.name) + if (owner.require_turn_lease() != lease + or snapshot.claim_generation(owner.name) != lease.claimed_generation): + raise RelationViolationError("Context acknowledgement lost its original live turn lease") + document.originals(keys) + return DocumentChange.to(document.transition( + keys, lambda row: row.context_committed( + lease, sent_text, context, snapshot=snapshot, + ), + )) + return self._update(*_members("key", keys), acknowledge) + def settle_unbound(self, keys: tuple[InputKey, ...]) -> InputDocument: """Publish one complete retirement and return that exact document cut.""" return self._update( diff --git a/src/agent_comms/installer/runtime_resolution.py b/src/agent_comms/installer/runtime_resolution.py index bf6271ad73..9b7acad97c 100644 --- a/src/agent_comms/installer/runtime_resolution.py +++ b/src/agent_comms/installer/runtime_resolution.py @@ -2,27 +2,496 @@ from __future__ import annotations +import hashlib +import platform +import re +import shutil import sys import tomllib +from collections.abc import Iterable from dataclasses import dataclass +from functools import cached_property +from email import policy +from email.parser import BytesParser from pathlib import Path -from typing import TYPE_CHECKING, ClassVar -from urllib.parse import parse_qs, urlsplit - -from packaging.markers import Marker -from packaging.utils import canonicalize_name -from pydantic import BaseModel, Field +from tempfile import TemporaryDirectory +from zipfile import ZipFile +from typing import TYPE_CHECKING, Annotated, ClassVar, Self +from urllib.parse import parse_qsl, urlsplit + +from packaging.markers import Marker, default_environment +from packaging.requirements import Requirement +from packaging.specifiers import SpecifierSet +from packaging.utils import canonicalize_name, parse_wheel_filename +from packaging.version import Version +from pydantic import BaseModel, ConfigDict, Field, RootModel from agent_comms.declared_family import DeclaredFamily +from agent_comms.field_codec import FieldCodec, GitCommitText, boundary_decoder from agent_comms.installer.probes.packages import ArchivePin, GitPin, PackagePin, PackageRecord from agent_comms.installer.probes.packages import DistributionVersionsProbe, InstalledPackagesProbe from agent_comms.launch_runtime import LaunchRuntime +from agent_comms.repo import Repo from agent_comms.repository_name import RepositoryName +from agent_comms.work_branches import WorkBranches if TYPE_CHECKING: + from agent_comms.installer.commands import InstallCommands from agent_comms.installer.upgrade import Stack +@dataclass(frozen=True) +class ProjectMetadata: + """An installable project's backend-produced wheel metadata, not its manifest dialect.""" + + name: str + version: Version + python: SpecifierSet + requirements: tuple[Requirement, ...] + extras: frozenset[str] + + @classmethod + def from_wheel(cls, wheel: Path) -> ProjectMetadata: + name, version, _, _ = parse_wheel_filename(wheel.name) + with ZipFile(wheel) as archive: + entries = tuple(entry for entry in archive.infolist() + if entry.filename.endswith(".dist-info/METADATA")) + if len(entries) != 1: + raise ValueError(f"{wheel} requires exactly one dist-info/METADATA") + entry, = entries + components = entry.filename.split("/") + if len(components) != 2: + raise ValueError(f"Malformed wheel metadata path: {entry.filename}") + parent, _ = components + identity, recorded_version = parent.removesuffix(".dist-info").rsplit("-", 1) + if canonicalize_name(identity) != name or Version(recorded_version) != version: + raise ValueError(f"Wheel metadata directory differs from {wheel.name}") + message = BytesParser(policy=policy.default).parsebytes(archive.read(entry)) + if message.defects: + raise ValueError(f"Malformed wheel metadata: {message.defects}") + def required(header: str) -> str: + values = message.get_all(header, []) + if len(values) != 1 or not str(values[0]).strip(): + raise ValueError(f"Wheel metadata requires exactly one {header}") + return str(values[0]).strip() + actual = canonicalize_name(required("Name"), validate=True) + actual_version = Version(required("Version")) + Version(required("Metadata-Version")) + if actual != name or actual_version != version: + raise ValueError(f"Wheel metadata identity differs from {wheel.name}") + python = message.get_all("Requires-Python", []) + if len(python) > 1: + raise ValueError("Duplicate Requires-Python metadata") + extra_values = tuple(canonicalize_name(str(value), validate=True) + for value in message.get_all("Provides-Extra", [])) + if len(set(extra_values)) != len(extra_values): + raise ValueError("Duplicate Provides-Extra metadata") + return cls(actual, actual_version, SpecifierSet(str(python[0]) if python else ""), + tuple(Requirement(str(value)) for value in message.get_all("Requires-Dist", [])), + frozenset(extra_values)) + + @classmethod + def acquire(cls, checkout: Path, *, uv: Path, python: Path, + commands: InstallCommands, output: Path) -> ProjectMetadata: + """Build once through the selected frontend; output never lives in a source cut.""" + output = output.resolve() + checkout = checkout.resolve() + if output.is_relative_to(checkout): + raise ValueError("Wheel metadata output must be outside the captured source") + before = Repo.git(checkout, "rev-parse", "HEAD").strip() + if Repo.git(checkout, "status", "--porcelain"): + raise ValueError(f"Preserve dirty metadata source: {checkout}") + output.mkdir(parents=True, exist_ok=True) + with TemporaryDirectory(prefix="metadata-", dir=output) as temporary: + try: + commands.run(uv, "build", "--wheel", "--out-dir", temporary, + "--python", python, checkout) + wheels = tuple(Path(temporary).glob("*.whl")) + if len(wheels) != 1: + raise ValueError(f"Backend produced {len(wheels)} wheels for {checkout}, expected one") + wheel, = wheels + return cls.from_wheel(wheel) + finally: + if Repo.git(checkout, "rev-parse", "HEAD").strip() != before or Repo.git( + checkout, "status", "--porcelain"): + raise ValueError(f"Backend moved or dirtied captured source: {checkout}") + + def with_all_extras(self, requirement: str) -> str: + """Enrich only this package's existing root, preserving its complete requirement.""" + selected = Requirement(requirement) + if canonicalize_name(selected.name) != self.name: + return requirement + selected.extras.update(self.extras) + return str(selected) + + @staticmethod + def require_python(specifier: SpecifierSet) -> None: + version = platform.python_version() + if not specifier.contains(version, prereleases=True): + raise ValueError(f"The common interpreter {version} does not satisfy {specifier}") + + +class GitSelection: + """No requested ref; only the resolved commit selects the package.""" + + def branches(self) -> tuple[str, ...]: + return () + + +class GitPackageLocation: + """The repository root has no native UV subdirectory field.""" + + def fields(self) -> dict[str, str]: + return {} + + +@dataclass(frozen=True) +class GitQuery(GitSelection, DeclaredFamily, affix="GitQuery"): + value: str + + def __post_init__(self) -> None: + if not self.value or chr(0) in self.value: + raise ValueError("Git query values must be nonempty and representable") + self.value.encode("utf-8", errors="strict") + + def selectors(self) -> tuple[GitSelection, ...]: + return () + + def locations(self) -> tuple[GitPackageLocation, ...]: + return () + + +class GitRef(GitQuery, declared_name=None): + def selectors(self) -> tuple[GitSelection, ...]: + return (self,) + + +class BranchGitQuery(GitRef): + def branches(self) -> tuple[str, ...]: + return (self.value,) + + +class TagGitQuery(GitRef): + pass + + +class RevGitQuery(GitRef): + pass + + +class SubdirectoryGitQuery(GitQuery, GitPackageLocation): + def locations(self) -> tuple[GitPackageLocation, ...]: + return (self,) + + def fields(self) -> dict[str, str]: + return {self.declared_name: self.value} + + +@dataclass(frozen=True) +class LockedGitLocation: + repository: str + revision: Annotated[str, GitCommitText] + selector: GitSelection + package: GitPackageLocation + + @classmethod + @boundary_decoder + def decode(cls, value: str) -> LockedGitLocation: + source = urlsplit(value) + GitCommitText.decode(source.fragment) + if len(source.fragment) != 40 or source.fragment != source.fragment.lower(): + raise ValueError("A resolved Git lock entry requires its full lowercase commit") + if not source.scheme or not source.path: + raise ValueError("A resolved Git source requires its repository URL") + if re.search(r"%(?![0-9a-fA-F]{2})", source.query): + raise ValueError("Malformed Git query escaping") + parameters: list[GitQuery] = [] + seen: set[str] = set() + for key, text in parse_qsl(source.query, keep_blank_values=True, strict_parsing=True, errors="strict"): + if key in seen: + raise ValueError(f"Duplicate Git query parameter: {key}") + seen.add(key) + parameters.append(FieldCodec.decode_document(GitQuery.member_named(key), {"value": text})) + selectors = tuple(selector for parameter in parameters for selector in parameter.selectors()) + locations = tuple(location for parameter in parameters for location in parameter.locations()) + if len(selectors) > 1: + raise ValueError("A locked Git source requires one unambiguous ref selector") + selector, = selectors or (GitSelection(),) + package, = locations or (GitPackageLocation(),) + return cls(source._replace(query="", fragment="").geturl(), source.fragment, selector, package) + + +class UvRecord(BaseModel): + """Decode consumed TOML fields strictly and preserve every unrelated UV/tool option.""" + + model_config = ConfigDict(extra="allow", strict=True, validate_by_alias=True, validate_by_name=False) + + def document(self) -> dict[str, object]: + # Python mode preserves TOML date/time and arrays, not just JSON-shaped options. + return self.model_dump(mode="python", by_alias=True, exclude_unset=True) + + +class UvSourceOptions(UvRecord): + """Only fields consumed here are modeled; UV retains every other source option.""" + + marker: str = "" + + def materialize(self, original: Path, core: Path, + projects: dict[RepositoryName, tuple[Path, ProjectMetadata]], name: str) -> UvSourceOptions: + return self + + def retained_options(self) -> dict[str, object]: + return self.model_dump(mode="python", by_alias=True, exclude_unset=True) + + def with_options(self, options: dict[str, object], marker: str) -> Self: + selected = type(self).model_validate({**options, **self.document()}) + if marker: + selected.marker = marker + return selected + + +class UvGitSource(UvSourceOptions): + git: str + + def retained_options(self) -> dict[str, object]: + return self.model_dump(mode="python", by_alias=True, exclude=set(GitQuery.names()), exclude_unset=True) + + +class UvTrackedSource(UvGitSource): + branch: str + + def materialize(self, original: Path, core: Path, + projects: dict[RepositoryName, tuple[Path, ProjectMetadata]], name: str) -> UvSourceOptions: + GitSource.require_branches(name, (self.branch,)) + checkout, metadata = projects[Repo.repository_url(self.git)] + StackSources.require_package(metadata, name) + options = self.model_dump(mode="python", by_alias=True, exclude_unset=True, + exclude=set(type(self).model_fields) - set(UvSourceOptions.model_fields)) + return UvLocalSource.model_validate({**options, "path": str(checkout), "editable": True}) + + +class UvLocalSource(UvSourceOptions): + path: str + + def materialize(self, original: Path, core: Path, + projects: dict[RepositoryName, tuple[Path, ProjectMetadata]], name: str) -> UvSourceOptions: + checkout = (original / self.path).resolve() + if checkout != core or name != CorePackage.distribution: + raise ValueError(f"Uncaptured local source {name}: {checkout}") + return self.model_copy(update={"path": str(checkout)}) + + +class UvRevisionSource(UvGitSource): + rev: str + subdirectory: str = "" + + +class UvIndexSource(UvSourceOptions): + index: str + + +class UvConditionalSources(RootModel[list[UvSourceOptions]]): + """Conditional source arrays pass through intact; an exact pin cannot replace them.""" + + def materialize(self, original: Path, core: Path, + projects: dict[RepositoryName, tuple[Path, ProjectMetadata]], name: str) -> UvConditionalSources: + return self + + def retained_options(self) -> dict[str, object]: + raise ValueError("Ambiguous private conditional source declarations") + + +class UvIndex(UvRecord): + name: str = "" + url: str + explicit: bool = False + + +class UvProjectSettings(UvRecord): + package: bool = True + sources: dict[str, UvRevisionSource | UvTrackedSource | UvLocalSource | UvIndexSource + | UvGitSource | UvSourceOptions | UvConditionalSources] + overrides: list[str] = Field(default_factory=list, alias="override-dependencies") + constraints: list[str] = Field(default_factory=list, alias="constraint-dependencies") + indexes: list[UvIndex] = Field(default_factory=list, alias="index") + + def retain(self, name: str, marker: str, source: UvSourceOptions) -> None: + original = self.sources.get(name, UvSourceOptions()) + self.sources[name] = source.with_options(original.retained_options(), marker) + + def override_git(self, name: str, marker: str) -> None: + """The native source owns Git location; the root override only selects its name.""" + locked = Requirement(name) + if marker: + locked.marker = Marker(marker) + result: list[str] = [] + found = False + for text in self.overrides: + requirement = Requirement(text) + if canonicalize_name(requirement.name) == name: + found = True + if requirement.specifier: + self.constraints = [*self.constraints, str(requirement)] + requirement.specifier = SpecifierSet() + requirement.url = None + result.append(str(requirement)) + else: + result.append(text) + if not found: + result.append(str(locked)) + self.overrides = result + + +class UvProjectRoot(UvRecord): + dependencies: list[str] + + +class UvProjectTools(UvRecord): + uv: UvProjectSettings + + +class PrivateUvProject(UvRecord): + """One transient, lossless projection of the actual captured virtual project.""" + project: UvProjectRoot + tool: UvProjectTools + + @classmethod + def read(cls, path: Path) -> PrivateUvProject: + return FieldCodec.decode(cls, tomllib.loads(path.read_text())) + + def enrich(self, metadata: ProjectMetadata) -> None: + if not any(canonicalize_name(Requirement(text).name) == metadata.name + for text in self.project.dependencies): + raise ValueError("The private checker requires the declared Core runtime root") + self.project.dependencies = [metadata.with_all_extras(text) for text in self.project.dependencies] + self.tool.uv.overrides = [metadata.with_all_extras(text) for text in self.tool.uv.overrides] + + def materialize(self, original: Path, core: Path, + projects: dict[RepositoryName, tuple[Path, ProjectMetadata]]) -> None: + for name, source in tuple(self.tool.uv.sources.items()): + self.tool.uv.sources[name] = source.materialize(original, core, projects, canonicalize_name(name)) + + +class StackSources: + """Branch-tracking source declarations shared by installation and source checking.""" + + @classmethod + def sources(cls, stack: Path) -> dict[str, tuple[str, str]]: + return cls.declared_sources((stack / "pyproject.toml").read_text()) + + @staticmethod + def require_package(metadata: ProjectMetadata, name: str) -> None: + """The actual backend, never a manifest spelling, owns the distribution identity.""" + if metadata.name != canonicalize_name(name): + raise ValueError(f"Backend builds {metadata.name}, not stack package {name}") + + @staticmethod + @boundary_decoder + def frontend() -> Path: + executable = shutil.which("uv") + if executable is None: + raise ValueError("Bootstrap Core's declared uv before acquiring source metadata") + return Path(executable).absolute() + + @staticmethod + def marker_environment(python: Path) -> dict[str, str]: + # Marker evaluation must describe the interpreter actually running both checkers. + # A different environment cannot borrow this process's platform or Python facts. + if not LaunchRuntime.from_interpreter(python).runs_this_process: + raise ValueError("Run metadata preparation in the selected common interpreter") + return FieldCodec.decode(dict[str, str], default_environment()) + + @classmethod + def require_compatible(cls, baseline: str, candidate: str) -> None: + """Local-source membership may change; external runtime requirements must agree. + + This supported path compares the same external contract, not arbitrary requirement + migrations. Candidate metadata/settings are copied intact by the private project writer. + """ + documents = tuple(tomllib.loads(text) for text in (baseline, candidate)) + local_names: set[str] = set() + for document in documents: + match document: + case {"tool": {"uv": {"sources": dict() as sources}}}: + for name, source in sources.items(): + match source: + case {"git": str(), "branch": str()} | {"path": str()}: + local_names.add(canonicalize_name(FieldCodec.decode(str, name))) + environment = cls.marker_environment(Path(sys.executable)) + contracts = tuple(frozenset(str(requirement) for requirement in cls.virtual_requirements(document, environment) + if canonicalize_name(requirement.name) not in local_names) + for document in documents) + if contracts[0] != contracts[1]: + raise ValueError("External runtime dependency migration is unsupported by this common environment") + + @classmethod + def virtual_requirements(cls, document: dict[str, object], environment: dict[str, str]) -> tuple[Requirement, ...]: + """Only package=false roots have a declarative (non-wheel) runtime contract.""" + match document: + case {"project": {"dependencies": list() as dependencies, **project}, + "tool": {"uv": {"package": False}}}: + python = SpecifierSet(FieldCodec.decode(str, project.get("requires-python", ""))) + ProjectMetadata.require_python(python) + return tuple(requirement for requirement in map( + Requirement, FieldCodec.decode(tuple[str, ...], dependencies)) + if cls.requirement_active(requirement, environment, ("",))) + raise ValueError("A virtual runtime root requires tool.uv.package=false and project dependencies") + + @staticmethod + @boundary_decoder + def requirement_active(requirement: Requirement, environment: dict[str, str], extras: Iterable[str]) -> bool: + return requirement.marker is None or any( + requirement.marker.evaluate({**environment, "extra": extra}) for extra in extras) + + @classmethod + def active_requirements(cls, document: dict[str, object], projects: tuple[ProjectMetadata, ...], + environment: dict[str, str]) -> tuple[Requirement, ...]: + """Monotone local-extra closure, retaining original marked/extras-bearing edges.""" + local = {project.name: project for project in projects} + if len(local) != len(projects): + raise ValueError("Captured package metadata contains duplicate distribution identities") + for project in projects: + ProjectMetadata.require_python(project.python) + selected: dict[str, set[str]] = {} + pending = list(cls.virtual_requirements(document, environment)) + active: set[Requirement] = set() + while pending: + requirement = pending.pop() + # Pending edges were selected by their declaring package's active extras; + # virtual roots have only the base context. + active.add(requirement) + name = canonicalize_name(requirement.name) + if name not in local: + continue + project = local[name] + requested = {"", *(canonicalize_name(extra) for extra in requirement.extras)} + if not requested.difference({""}).issubset(project.extras): + raise ValueError(f"{name} does not provide requested extras {requirement.extras}") + previous = selected.setdefault(name, set()) + added = requested - previous + if not added: + continue + previous.update(added) + pending.extend(edge for edge in project.requirements if cls.requirement_active(edge, environment, added)) + return tuple(sorted(active, key=str)) + + @staticmethod + def declared_sources(document: str) -> dict[str, tuple[str, str]]: + match tomllib.loads(document): + case {"tool": {"uv": {"sources": dict() as sources}}}: + found: dict[str, tuple[str, str]] = {} + for name, source in sources.items(): + match source: + case {"git": str(url), "branch": str(branch)}: + GitSource.require_branches(name, (branch,)) + identity = canonicalize_name(FieldCodec.decode(str, name)) + if identity in found: + raise ValueError(f"The stack declares package {identity} twice") + found[identity] = (url, branch) + return found + raise ValueError("The stack declares no [tool.uv.sources] table") + + class RuntimePackage(DeclaredFamily, affix="Package"): distribution: ClassVar[str] repository: ClassVar[RepositoryName] @@ -61,6 +530,12 @@ class LockSource(BaseModel): def tracked(self, name: str) -> tuple[str, ...]: return () + def project_source(self) -> UvSourceOptions: + raise ValueError("Local/virtual sources have no external retention projection") + + def retain(self, name: str, marker: str, uv: UvProjectSettings) -> None: + uv.retain(name, marker, self.project_source()) + class RegistrySource(LockSource): registry: str @@ -68,6 +543,15 @@ class RegistrySource(LockSource): def pin(self, version: str, core: str) -> PackagePin: return PackagePin(version) + def project_source(self) -> UvIndexSource: + return UvIndexSource(index="locked-" + hashlib.sha256(self.registry.encode()).hexdigest()) + + def retain(self, name: str, marker: str, uv: UvProjectSettings) -> None: + index = UvIndex(name=self.project_source().index, url=self.registry, explicit=True) + if index not in uv.indexes: + uv.indexes = [*uv.indexes, index] + super().retain(name, marker, uv) + class GitSource(LockSource): """Our sibling packages track main, the ref InstallDue measures them against; a unit @@ -75,19 +559,33 @@ class GitSource(LockSource): unchanged (f39412d88 held Toad on work/unit-toad--P694 after #1239 merged, 2026-10-25). A ``?rev=`` source is pinned and never relocked.""" + model_config = ConfigDict(frozen=True) git: str + @cached_property + def location(self) -> LockedGitLocation: + return LockedGitLocation.decode(self.git) + def tracked(self, name: str) -> tuple[str, ...]: - branches = parse_qs(urlsplit(self.git).query).get("branch", []) - if branches not in ([], ["main"]): - raise ValueError(f"{name} tracks {branches}, not main: {self.git}") + branches = self.location.selector.branches() + self.require_branches(name, branches) return (name,) if branches else () + @staticmethod + def require_branches(name: str, branches: tuple[str, ...]) -> None: + if branches not in ((), (WorkBranches.main,)): + raise ValueError(f"{name} tracks {branches}, not {WorkBranches.main}") + def pin(self, version: str, core: str) -> PackagePin: - commit = urlsplit(self.git).fragment - if not commit: - raise ValueError("A resolved Git lock entry must name its commit") - return GitPin(version, commit) + return GitPin(version, self.location.revision) + + def project_source(self) -> UvRevisionSource: + return UvRevisionSource(git=self.location.repository, rev=self.location.revision, + **self.location.package.fields()) + + def retain(self, name: str, marker: str, uv: UvProjectSettings) -> None: + super().retain(name, marker, uv) + uv.override_git(name, marker) class EditableSource(LockSource): @@ -110,10 +608,14 @@ class LockedPackage(BaseModel): name: str markers: list[str] = Field(default_factory=list, alias="resolution-markers") + def active(self) -> bool: + return not self.markers or any(Marker(marker).evaluate() for marker in self.markers) + def selects(self, name: str, version: str) -> bool: - return canonicalize_name(self.name) == name and ( - not self.markers or any(Marker(marker).evaluate() for marker in self.markers) - ) + return canonicalize_name(self.name) == name and self.active() + + def external(self) -> tuple[VersionedPackage, ...]: + return () class VersionedPackage(LockedPackage): @@ -123,6 +625,22 @@ class VersionedPackage(LockedPackage): def selects(self, name: str, version: str) -> bool: return super().selects(name, version) and self.version == version + def external(self) -> tuple[VersionedPackage, ...]: + return (self,) if self.active() else () + + def require_retained(self, actual: VersionedPackage) -> None: + if (canonicalize_name(self.name) != canonicalize_name(actual.name) or self.version != actual.version + or self.source.project_source() != actual.source.project_source()): + raise ValueError(f"Private solve changed retained source/version for {self.name}") + + def constrain(self, uv: UvProjectSettings) -> None: + marker = " or ".join(f"({marker})" for marker in self.markers) + requirement = Requirement(f"{canonicalize_name(self.name)}=={self.version}") + if marker: + requirement.marker = Marker(marker) + uv.constraints = [*uv.constraints, str(requirement)] + self.source.retain(canonicalize_name(self.name), marker, uv) + class LocalPackage(LockedPackage): source: EditableSource | VirtualSource @@ -140,6 +658,19 @@ def tracked(self) -> tuple[str, ...]: return tuple(dict.fromkeys( name for entry in self.package for name in entry.source.tracked(entry.name))) + def external(self, excluded: frozenset[str]) -> dict[str, VersionedPackage]: + """Current-interpreter external evidence; local/virtual entries select themselves out.""" + result: dict[str, VersionedPackage] = {} + for package in self.package: + for entry in package.external(): + name = canonicalize_name(entry.name) + if name in excluded: + continue + if name in result: + raise ValueError(f"Lock selects ambiguous external entries for {name}") + result[name] = entry + return result + def pins(self, selected: dict[str, str], core: str) -> dict[str, PackagePin]: result: dict[str, PackagePin] = {} for name, version in selected.items(): diff --git a/src/agent_comms/integration.py b/src/agent_comms/integration.py index fa50ffd6d7..2a511d4701 100644 --- a/src/agent_comms/integration.py +++ b/src/agent_comms/integration.py @@ -4,25 +4,33 @@ import argparse import os +import re import subprocess import sys import tomllib from abc import ABC, abstractmethod from dataclasses import dataclass, field from pathlib import Path -from typing import Annotated, ClassVar +from typing import Annotated, ClassVar, Self +from .active_route import resolve_comms_route +from .build_identity import RepositoryCommit +from .commander.order import CommanderContext from .child_process import STOP_GRACE_SECONDS, ResidentManager, SpawnedProcess -from .field_codec import FieldCodec, WireName -from .head_review import HeadReview, PullRequestHead +from .declared_family import DeclaredFamily +from .errors import ToolRefusedError +from .field_codec import FieldCodec, GitCommitText, PathText, WireName, boundary_decoder +from .head_review import BaseSide, CombinedInput, HeadReview, HeadSide, IntegrationInput, IntegrationSource, MergeHeadReview, OrdinaryInput, PullRequestHead, Side +from .installer.runtime_resolution import StackSources from .launch_runtime import CheckoutSource, RestoredRuntime -from .repo import Checkout, GitCommandError, Repo +from .repo import Checkout, DetachedStart, GitCommandError, Repo from .systemd_unit import AgentsSlice +from .repository_name import RepositoryName from .test_lease import LeaseStart, TestLease from .todos import PullRequests +from .work_branches import WorkBranches - -class IntegrationFailed(RuntimeError): +class IntegrationFailed(ToolRefusedError): pass @@ -62,6 +70,9 @@ def files(self, checkout: Path, changed: tuple[Path, ...]) -> tuple[str, ...]: class FailureAction(ABC): + def require_qualification(self) -> None: + """Checking source capture does not run or retry a test selection.""" + def require_heads(self, heads: tuple[PullRequestHead, ...]) -> None: pass @@ -94,6 +105,9 @@ def argument(cls, value: str) -> ExcludeAfterFailure: raise argparse.ArgumentTypeError("A positive pull request number is required") return cls(pr) + def require_qualification(self) -> None: + raise IntegrationFailed("Qualification cannot exclude a captured source after failure") + def require_heads(self, heads: tuple[PullRequestHead, ...]) -> None: if not any(head.pr == self.pr for head in heads): raise IntegrationFailed(f"#{self.pr} is not in the reviewed integration heads") @@ -126,7 +140,39 @@ def require_editable(self) -> None: def fresh_main(self) -> str: self.require_editable() Repo.git(self.path, "fetch", "origin", timeout=300) - return Repo.git(self.path, "rev-parse", "origin/main").strip() + return Repo.git(self.path, "rev-parse", f"origin/{WorkBranches.main}").strip() + + @classmethod + def create(cls, repository: Repo, path: Path, commit: str) -> IntegrationCheckout: + if path.exists(): + raise IntegrationFailed(f"Preserve existing qualification checkout: {path}") + WorkBranches(repository.checkout).require_commits("origin", commit) + path.parent.mkdir(parents=True, exist_ok=True) + created = cls.add(repository.checkout, path, DetachedStart(commit)) + created.require_editable() + return created + + def include(self, head: PullRequestHead) -> None: + """Fetch exactly the captured member; Git, not a patch heuristic, combines it.""" + Repo.git(self.path, "fetch", "origin", f"refs/pull/{head.pr}/head", timeout=300) + fetched = Repo.git(self.path, "rev-parse", "FETCH_HEAD").strip() + if fetched != head.head: + raise IntegrationFailed(f"#{head.pr} moved while fetching; capture its new head") + try: + Repo.git(self.path, "merge", "--no-ff", "--no-edit", head.head, timeout=300) + except GitCommandError: + if Repo.answers(self.path, "rev-parse", "-q", "--verify", "MERGE_HEAD"): + Repo.git(self.path, "merge", "--abort") + raise + + def combine_sources(self, base: str, heads: tuple[PullRequestHead, ...]) -> str: + """Checking preserves every source member, including when their merge conflicts.""" + self.require_editable() + Repo.git(self.path, "switch", "--detach", base) + for head in heads: + self.include(head) + self.prepare() + return Repo.git(self.path, "rev-parse", "HEAD^{tree}").strip() def combine(self, base: str, heads: tuple[PullRequestHead, ...]) -> tuple[PullRequestHead, ...]: self.require_editable() @@ -136,15 +182,9 @@ def combine(self, base: str, heads: tuple[PullRequestHead, ...]) -> tuple[PullRe current = head.current() if not HeadReview.accepted(current.repo, current.head): raise IntegrationFailed(f"#{head.pr} no longer has an accepted head review") - Repo.git(self.path, "fetch", "origin", f"refs/pull/{head.pr}/head", timeout=300) - fetched = Repo.git(self.path, "rev-parse", "FETCH_HEAD").strip() - if fetched != current.head: - raise IntegrationFailed(f"#{head.pr} moved while fetching; review its new head") try: - Repo.git(self.path, "merge", "--no-ff", "--no-edit", current.head, timeout=300) + self.include(current) except GitCommandError as error: - if Repo.answers(self.path, "rev-parse", "-q", "--verify", "MERGE_HEAD"): - Repo.git(self.path, "merge", "--abort") print(f"MERGE REFUSED #{head.pr} {current.head}: {error}", flush=True) else: combined.append(current) @@ -216,6 +256,285 @@ def finish( raise IntegrationFailed(f"Timed out: {self.log}; no automatic replay") +@dataclass(frozen=True, kw_only=True) +class RunnerEnvironment: + root: Annotated[Path, PathText, WireName("RUNNER_TEMP")] + + @classmethod + def read(cls) -> Self: + return FieldCodec.decode(cls, {entry.key: os.environ[entry.key] + for entry in FieldCodec.argument_fields(cls)}) + + +@dataclass(frozen=True, kw_only=True) +class GitHubEnvironment(RunnerEnvironment): + repository: Annotated[RepositoryName, WireName("GITHUB_REPOSITORY")] + tested: Annotated[str, GitCommitText, WireName("GITHUB_SHA")] + event: Annotated[str, WireName("GITHUB_EVENT_NAME")] + document: Annotated[Path, PathText, WireName("GITHUB_EVENT_PATH")] + + +@dataclass(frozen=True) +class TestedSources(DeclaredFamily, affix="Sources"): + """The actual tested Git graph and forge carrier, not a supplied source roster. + + Event-specific ancestry and current-main rules live on their event declarations. + These transient values never enter Core's live records or authorize a review. + """ + + repository: Repo + tested: str + carrier: PullRequestHead + environment: RunnerEnvironment = field(kw_only=True, compare=False) + resolve_option: ClassVar[str] = "--resolve-ci-inputs" + revalidate_option: ClassVar[str] = "--revalidate-ci-inputs" + merge_option: ClassVar[str] = "--check-merge-inputs" + + @classmethod + def options(cls) -> tuple[str, ...]: + return cls.resolve_option, cls.revalidate_option, cls.merge_option + + @property + def parents(self) -> tuple[str, ...]: + return WorkBranches(self.repository.checkout).commit_parents(self.tested) + + @property + def baseline(self) -> str: + if len(self.parents) != 2: + raise IntegrationFailed("The tested comparison requires an actual two-parent merge") + return self.parents[0] + + @property + def tree(self) -> str: + return Repo.git(self.repository.checkout, "rev-parse", f"{self.tested}^{{tree}}").strip() + + @classmethod + def read(cls, checkout: Path) -> TestedSources: + environment = GitHubEnvironment.read() + repository = Repo.of(checkout) + if environment.repository != repository.github: + raise IntegrationFailed("The runner checkout origin is not the actual event repository") + tested = Repo.git(checkout, "rev-parse", "HEAD^{commit}").strip() + if environment.tested != tested: + raise IntegrationFailed("The runner checkout is not the event's actual tested SHA") + GitCommitText.decode(tested) + document = FieldCodec.parse(environment.document.read_text()) + return cls.member_named(environment.event).resolve(repository, tested, document, environment=environment) + + @classmethod + @abstractmethod + def resolve(cls, repository: Repo, tested: str, document: object, *, + environment: RunnerEnvironment) -> TestedSources: + """Decode this foreign event once and prove its relationship to the tested Git graph.""" + + def require_carrier(self) -> None: + current = self.carrier.current() + if current.base != WorkBranches.main: + raise IntegrationFailed("The tested carrier no longer targets main") + + def main_commits(self, inputs: IntegrationInput) -> dict[RepositoryName, str]: + return {source.repository: source.baseline.commit for source in inputs.sources} + + def require_local_candidate(self, inputs: IntegrationInput) -> None: + inputs.require_local_candidate(self.repository, self.carrier) + + def require_merge_review(self) -> None: + """A PR can compute before its review; queue admission requires explicit promotion.""" + + def check_merge_inputs(self, directory: Path) -> IntegrationInput: + """The required hosted check uses the same typed carrier proof as native promotion.""" + inputs = self.materialize(directory) + inputs.require_carrier(self.repository, self.carrier) + inputs.require_source_reviews() + self.require_merge_review() + print(f"COMBINED CARRIER #{self.carrier.pr} {self.carrier.head}; inputs {inputs.digest}", flush=True) + return inputs + + def materialize(self, directory: Path) -> IntegrationInput: + """Capture once from Git/forge, then use these same exact checkouts for every consumer.""" + if not directory.is_relative_to(self.environment.root.resolve()): + raise IntegrationFailed("CI source artifacts belong under the runner's private temporary root") + directory.mkdir(parents=True, exist_ok=False) + payload = IntegrationInput.from_message( + WorkBranches(self.repository.checkout).commit_message(self.carrier.head)) + urls = IntegrationInput.source_urls(self.repository, self.baseline, self.tested) + repositories = [self.repository] + for name, url in sorted(urls.items(), key=lambda item: item[0].key): + if name == self.repository.github: + raise IntegrationFailed("The stack repeats Core as a Git companion source") + checkout = directory / "repositories" / Path(*name.key) + checkout.parent.mkdir(parents=True, exist_ok=True) + Repo.git(self.repository.checkout, "clone", "--no-checkout", url, str(checkout), timeout=300) + repository = Repo.of(checkout) + if repository.github != name: + raise IntegrationFailed("A declared source clone has a different actual Git origin") + repositories.append(repository) + if payload: + inputs, = payload + else: + sources = [IntegrationSource(RepositoryCommit(self.repository.github, self.baseline), + (self.carrier,), self.tree)] + for repository in repositories[1:]: + commit = IntegrationSource.main_head(repository) + WorkBranches(repository.checkout).require_commits("origin", commit) + tree = Repo.git(repository.checkout, "rev-parse", f"{commit}^{{tree}}").strip() + sources.append(IntegrationSource(RepositoryCommit(repository.github, commit), (), tree)) + inputs = OrdinaryInput(tuple(sorted(sources, key=lambda source: source.repository.key))) + self.validate(inputs, tuple(repositories)) + for source in inputs.sources: + repository = Repo.selected(tuple(repositories), source.repository.to_wire()) + IntegrationCheckout.create(repository, BaseSide.checkout(directory, source.repository), source.baseline.commit) + # The actual tested Core merge already carries all local parents. Fetching its + # earlier own PR head through refs/pull would instead fetch the new carrier. + if source.repository == self.repository.github: + IntegrationCheckout.create(repository, HeadSide.checkout(directory, source.repository), self.tested) + else: + head = IntegrationCheckout.create(repository, HeadSide.checkout(directory, source.repository), source.baseline.commit) + head.combine_sources(source.baseline.commit, source.heads) + for side in Side.sides(): + side.require_materialized(directory, source) + checkout = side.checkout(directory, source.repository) + print(f"SOURCE {side.declared_name} {source.repository.to_wire()} " + f"{Repo.git(checkout, 'rev-parse', 'HEAD').strip()} {checkout}", flush=True) + self.validate(inputs, tuple(repositories)) + inputs.write(directory) + print(f"CI INPUT {inputs.digest}; source computation only, no review or merge permission", flush=True) + return inputs + + def revalidate(self, directory: Path) -> IntegrationInput: + inputs = IntegrationInput.at(directory) + repositories = [] + for source in inputs.sources: + for side in Side.sides(): + side.require_materialized(directory, source) + repositories.append(Repo.of(BaseSide.checkout(directory, source.repository))) + self.validate(inputs, tuple(repositories)) + print(f"CI INPUT REVALIDATED {inputs.digest}; no verdict posted", flush=True) + return inputs + + def validate(self, inputs: IntegrationInput, repositories: tuple[Repo, ...]) -> None: + """Re-run at job exit: source checks compute, but never publish a verdict.""" + self.require_carrier() + local = inputs.source(self.repository.github) + if local.baseline.commit != self.baseline or local.candidate_tree != self.tree: + raise IntegrationFailed("Tested base/tree differs from the frozen source input; recapture") + inputs.require_repositories(IntegrationInput.declared_repositories( + self.repository, self.baseline, self.tested)) + payload = IntegrationInput.from_message( + WorkBranches(self.repository.checkout).commit_message(self.carrier.head)) + if payload: + if payload != (inputs,): + raise IntegrationFailed("The tested carrier and materialized input disagree") + inputs.require_carrier(self.repository, self.carrier) + else: + # Ordinary mode is precisely one actual carrier plus unchanged captured mains. + if local.heads != (self.carrier,) or any(source.heads for source in inputs.sources + if source.repository != self.repository.github): + raise IntegrationFailed("An ordinary comparison cannot supply companion overlays") + mains = self.main_commits(inputs) + for source in inputs.sources: + repository = Repo.selected(repositories, source.repository.to_wire()) + if IntegrationSource.main_head(repository) != mains[source.repository]: + raise IntegrationFailed(f"{source.repository.to_wire()} main moved; recapture") + self.require_local_candidate(inputs) + for source in inputs.companions(self.repository.github): + source.require_candidate(Repo.selected(repositories, source.repository.to_wire()), self.carrier) + + +class PullRequestSources(TestedSources, declared_name="pull_request"): + @classmethod + def resolve(cls, repository: Repo, tested: str, document: object, *, + environment: RunnerEnvironment) -> TestedSources: + match document: + case {"number": int(number), "pull_request": {"head": {"sha": str(head)}}}: + graph = cls(repository, tested, PullRequestHead(repository.github, number, head), environment=environment) + case _: + raise IntegrationFailed("Malformed pull_request source event") + if graph.parents != (graph.baseline, graph.carrier.head): + raise IntegrationFailed("The tested second parent is not the event's exact PR head") + graph.require_carrier() + return graph + + +class MergeGroupSources(TestedSources, declared_name="merge_group"): + def require_merge_review(self) -> None: + MergeHeadReview.require_merge(self.carrier) + + @classmethod + @boundary_decoder + def resolve(cls, repository: Repo, tested: str, document: object, *, + environment: RunnerEnvironment) -> TestedSources: + match document: + case {"merge_group": {"head_sha": str(head), "base_sha": str(base), "head_ref": str(reference)}}: + # This is GitHub's queue protocol (also used by review-verdict), not a + # branch-name guess about which independent repositories belong together. + queued = re.search(r"/pr-([0-9]+)-", reference) + if queued is None: + raise IntegrationFailed("No actual queued carrier in this merge-group event") + if head != tested: + raise IntegrationFailed("The merge-group event names another tested head") + carrier = PullRequestHead.at(repository.github, int(queued[1])) + graph = cls(repository, tested, PullRequestHead(carrier.repo, carrier.pr, carrier.head), environment=environment) + case _: + raise IntegrationFailed("Malformed merge_group source event") + if graph.baseline != base or not Repo.descends(repository.checkout, graph.carrier.head, tested): + raise IntegrationFailed("The tested queue graph does not contain its exact current carrier") + graph.require_carrier() + return graph + + +class MainSources(TestedSources, declared_name="push"): + @staticmethod + def merged_carrier(repository: Repo, tested: str) -> PullRequestHead: + response = repository.github.rest(f"commits/{tested}/pulls", "--jq", "map(.number)") + if response.returncode: + raise IntegrationFailed(response.stderr.strip() or "Cannot read main's merged PR provenance") + numbers = FieldCodec.decode(tuple[int, ...], FieldCodec.parse(response.stdout)) + parents = WorkBranches(repository.checkout).commit_parents(tested) + if len(parents) != 2: + raise IntegrationFailed("Main cache requires the actual two-parent merged PR") + rows = tuple(row for row in PullRequests().rows(repository.github, numbers) + if row.base == WorkBranches.main and row.head == parents[1] + and any(merged.merge.oid == tested for merged in row.as_merged())) + if len(rows) != 1: + raise IntegrationFailed("Main cache requires one actual merged PR; direct/ambiguous commits refuse") + row, = rows + return PullRequestHead(repository.github, row.number, row.head) + + @classmethod + def resolve(cls, repository: Repo, tested: str, document: object, *, + environment: RunnerEnvironment) -> TestedSources: + match document: + case {"ref": str(reference), "after": str(after)} if after == tested: + if reference != f"refs/heads/{WorkBranches.main}": + raise IntegrationFailed("Main cache requires the actual main push event") + graph = cls(repository, tested, cls.merged_carrier(repository, tested), environment=environment) + case _: + raise IntegrationFailed("Main cache requires the actual main push event") + if graph.parents != (graph.baseline, graph.carrier.head): + raise IntegrationFailed("Main cache supports the actual two-parent merged PR, not a guessed cut") + return graph + + def require_carrier(self) -> None: + if self.merged_carrier(self.repository, self.tested) != self.carrier: + raise IntegrationFailed("Main's uniquely identified merged carrier changed") + + def main_commits(self, inputs: IntegrationInput) -> dict[RepositoryName, str]: + return {**super().main_commits(inputs), self.repository.github: self.tested} + + def require_local_candidate(self, inputs: IntegrationInput) -> None: + """Main owns the transition from exact open sources to exact merged ancestry.""" + source = inputs.source(self.repository.github) + source.require_ancestry(self.repository, self.carrier.head) + for head in (head for head in source.heads if not head.same_pull_request(self.carrier)): + rows = tuple(row for row in PullRequests().rows(head.repo, (head.pr,)) + if row.head == head.head and row.base == WorkBranches.main) + # Inclusion belongs to Git. An open/closed badge cannot undo a commit that + # the actual merged carrier contains, nor can it substitute for that proof. + if len(rows) != 1 or not Repo.descends(self.repository.checkout, head.head, self.tested): + raise IntegrationFailed(f"#{head.pr} is not the exact captured source merged into this main") + + @dataclass(frozen=True) class IntegrationRun: checkout: IntegrationCheckout @@ -241,6 +560,104 @@ def accepted_heads(self) -> tuple[PullRequestHead, ...]: if HeadReview.accepted(repo.github, request.head) ) + def qualify(self, context: CommanderContext, problems: tuple[str, ...], *, + carrier: PullRequestHead) -> CombinedInput: + """Capture the existing resolved carrier and its distinct stored source members.""" + self.checkout.require_editable() + self.failure.require_qualification() + if not problems or len(set(problems)) != len(problems): + raise IntegrationFailed("Qualification requires unique actual problem selectors") + if context.actor_identity.commanding_identity != context.incarnation: + raise IntegrationFailed("Qualification requires this root's existing commander authority") + todos = tuple(context.comms.goals.todos.get(problem) for problem in problems) + for todo in todos: + todo.require_commander(context.incarnation) + core = Repo.of(self.checkout.path) + registered_core = Repo.selected(context.repos, core.github.to_wire()) + base = IntegrationSource.main_head(registered_core) + WorkBranches(registered_core.checkout).require_commits("origin", base) + linked_repositories = {link.repo for todo in todos for order in todo.order.declarations + for link in order.pull_requests} + initial = {*IntegrationInput.declared_repositories(registered_core, base, base), *linked_repositories} + repositories = {name: Repo.selected(context.repos, name.to_wire()) for name in initial} + mains = {core.github: base, **{name: IntegrationSource.main_head(repo) + for name, repo in repositories.items() if name != core.github}} + pulls = PullRequests(todos) + heads: dict[RepositoryName, tuple[PullRequestHead, ...]] = {} + for repo in repositories.values(): + WorkBranches(repo.checkout).require_commits("origin", mains[repo.github]) + linked = PullRequests.linked_among(todos, repo.github) + selected = [] + for row in pulls.rows(repo.github, tuple(sorted(linked))): + if row.as_merged(): + merged, = row.as_merged() + if not repo.contains(merged.merge.oid, mains[repo.github]): + raise IntegrationFailed("A merged source is absent from captured main") + continue + if not row.as_unmerged() or len(linked[row.number]) != 1 or row.base != WorkBranches.main: + raise IntegrationFailed("A linked source is closed, ambiguous or targets another base") + selected.append(PullRequestHead(repo.github, row.number, row.head)) + heads[repo.github] = tuple(selected) + current = carrier.current() + candidate_commit = Repo.git(self.checkout.path, "rev-parse", "HEAD").strip() + if current.repo != core.github or current.base != WorkBranches.main or current.head != candidate_commit: + raise IntegrationFailed("Capture requires the actual published combined carrier at this checkout's HEAD") + self.output.mkdir(parents=True, exist_ok=False) + IntegrationCheckout.create(registered_core, BaseSide.checkout(self.output, core.github), base) + tree = Repo.git(self.checkout.path, "rev-parse", "HEAD^{tree}").strip() + required = IntegrationInput.declared_repositories(registered_core, base, candidate_commit) + if linked_repositories - set(required): + raise IntegrationFailed("A selected order links an unknown stack repository") + sources = [IntegrationSource(RepositoryCommit(core.github, base), heads[core.github], tree)] + for name in required: + if name == core.github: + continue + if name not in repositories: + repositories[name] = Repo.selected(context.repos, name.to_wire()) + mains[name] = IntegrationSource.main_head(repositories[name]) + heads[name] = () + repo = repositories[name] + if len(heads[name]) > 1: + raise IntegrationFailed("A companion repository has no unique existing linked carrier") + baseline = mains[name] + IntegrationCheckout.create(repo, BaseSide.checkout(self.output, name), baseline) + candidate = IntegrationCheckout.create(repo, HeadSide.checkout(self.output, name), baseline) + tree = candidate.combine_sources(baseline, heads[name]) + sources.append(IntegrationSource(RepositoryCommit(name, baseline), heads[name], tree)) + # Revalidate original sources before freezing the shared value or attaching its payload. + for source in sources: + repo = repositories[source.repository] + if IntegrationSource.main_head(repo) != source.baseline.commit: + raise IntegrationFailed("A captured main moved; preserve this failed attempt and recapture") + for head in source.heads: + head.current() + if source.repository == core.github: + source.require_ancestry(repo, candidate_commit) + else: + source.require_tree(repo) + current_todos = tuple(context.comms.goals.todos.get(todo.id) for todo in todos) + for todo in current_todos: + todo.require_commander(context.incarnation) + for source in sources: + linked = PullRequests.linked_among(current_todos, source.repository) + if any(head.pr not in linked for head in source.heads): + raise IntegrationFailed("A selected problem no longer links its captured source") + inputs = CombinedInput(tuple(sorted(sources, key=lambda source: source.repository.key)), carrier) + context.comms.goals.todos.require_input_links(inputs) + if Repo.git(self.checkout.path, "rev-parse", f"{base}^{{tree}}").strip() == inputs.source(core.github).candidate_tree: + raise IntegrationFailed("The Core carrier has no substantive source change") + original_message = WorkBranches(self.checkout.path).commit_message(candidate_commit) + if IntegrationInput.from_message(original_message): + raise IntegrationFailed("Capture requires a fresh substantive integration commit, not nested metadata") + carrier.current() + Repo.git(self.checkout.path, "commit", "--allow-empty", "-m", + f"Capture the resolved combined sources\n\n{inputs.trailer}") + commit = Repo.git(self.checkout.path, "rev-parse", "HEAD").strip() + IntegrationCheckout.create(registered_core, HeadSide.checkout(self.output, core.github), commit) + inputs.write(self.output) + print(f"QUALIFICATION ONLY {commit}; inputs {inputs.digest}; {self.output}; no review or merge permission", flush=True) + return inputs + def test( self, base: str, heads: tuple[PullRequestHead, ...], start: LeaseStart, attempt: int ) -> TestAttempt: @@ -332,24 +749,41 @@ def execute(self) -> int: def main() -> None: """Enter the existing CPU-slice owner before acquiring a transferable lease.""" + if any(argument.partition("=")[0] in TestedSources.options() for argument in sys.argv[1:]): + raise SystemExit(run()) command = ResidentManager.current().command((sys.executable, "-m", __name__, *sys.argv[1:])) os.execvpe(command[0], command, dict(os.environ)) +@boundary_decoder def run() -> int: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--checkout", type=Path, required=True) - parser.add_argument( + checkout_option = parser.add_argument("--checkout", type=Path) + local_options = parser.add_argument_group("local integration") + output_option = local_options.add_argument( "--output", type=Path, - required=True, help="new persistent directory for this run's two bounded logs", ) - parser.add_argument("--risk", required=True) + risk_option = local_options.add_argument("--risk", default="") + operation = parser.add_mutually_exclusive_group() + operation.add_argument(TestedSources.resolve_option, type=Path, metavar="OUTPUT", + help="materialize the actual GitHub event's exact compared sources; no bus") + operation.add_argument(TestedSources.revalidate_option, type=Path, metavar="OUTPUT", + help="revalidate the same event/input and source heads before CI success") + operation.add_argument(TestedSources.merge_option, type=Path, metavar="OUTPUT", + help="require a real combined carrier and its exact reviewed source members") + operation.add_argument("--qualify", default="", metavar="PROBLEM", + help="prepare sources from this actual primary order, without running tests") + local_options.add_argument("--with-problem", action="append", default=[], metavar="PROBLEM") + local_options.add_argument("--carrier", type=int, default=0, metavar="PR", + help="the existing separate combined PR at this checkout's published HEAD") + local_options.add_argument("--comms-root", type=Path, + help="the existing authorized root for stored-problem qualification") parser.add_argument( ExcludeAfterFailure.option, type=ExcludeAfterFailure.argument, default=StopAfterFailure() ) - parser.add_argument( + local_options.add_argument( "pytest_arguments", nargs=argparse.REMAINDER, help="pytest's full argument surface after --; touched files are appended", @@ -358,13 +792,47 @@ def run() -> int: pytest_arguments = arguments.pytest_arguments if pytest_arguments[:1] == ["--"]: pytest_arguments = pytest_arguments[1:] - return IntegrationRun( + arguments.pytest_arguments = pytest_arguments + values = vars(arguments) + if arguments.resolve_ci_inputs or arguments.revalidate_ci_inputs or arguments.check_merge_inputs: + forbidden = {option.dest for option in local_options._group_actions if values[option.dest]} + if forbidden: + parser.error("CI source resolution does not accept tests, a bus root or local qualification options: " + + ", ".join(sorted(forbidden))) + arguments.drop_on_failure.require_qualification() + graph = TestedSources.read((arguments.checkout or Path.cwd()).resolve()) + if arguments.resolve_ci_inputs: + graph.materialize(arguments.resolve_ci_inputs.resolve()) + elif arguments.revalidate_ci_inputs: + graph.revalidate(arguments.revalidate_ci_inputs.resolve()) + else: + graph.check_merge_inputs(arguments.check_merge_inputs.resolve()) + return 0 + missing = {option.dest for option in (checkout_option, output_option, risk_option) + if not values[option.dest]} + if missing: + parser.error("Local integration requires --checkout, --output and --risk") + run = IntegrationRun( IntegrationCheckout(arguments.checkout.resolve()), arguments.output.resolve(), arguments.risk, arguments.drop_on_failure, tuple(pytest_arguments), - ).execute() + ) + # Decode the foreign CLI operation once; qualification never enters execute/test/lease. + if arguments.qualify: + if pytest_arguments: + parser.error("Qualification cannot run tests") + context = CommanderContext.read_root(resolve_comms_route(arguments.comms_root).root) + if arguments.carrier <= 0: + parser.error("Qualification requires --carrier naming the actual separate combined PR") + carrier = PullRequestHead.at(Repo.of(run.checkout.path).github, arguments.carrier) + run.qualify(context, (arguments.qualify, *arguments.with_problem), + carrier=PullRequestHead(carrier.repo, carrier.pr, carrier.head)) + return 0 + if arguments.with_problem or arguments.carrier: + parser.error("--with-problem and --carrier require --qualify") + return run.execute() if __name__ == "__main__": diff --git a/src/agent_comms/native_entries.py b/src/agent_comms/native_entries.py index 8f390b31d3..b726c2a609 100644 --- a/src/agent_comms/native_entries.py +++ b/src/agent_comms/native_entries.py @@ -24,7 +24,7 @@ from .store_files import AtomicFile if TYPE_CHECKING: - from .native_pi import NativeContextProof + from .native_input_proof import NativeContextProof from .importing import ImportedConversation from .declared_family import DeclaredFamily, UnknownFamilyMember @@ -40,7 +40,7 @@ WireOmitDefault, ) from .importing import ImportedSessionMetadata -from .native_input_record import NativeInputLocation +from .native_input_proof import NativeInputLocation from .pi_payloads import Opaque, PiField, PiMessage, PiOmitted, PiPayload, ReportedText from .pi_vocabulary import PendingStopReason, ThinkingLevel from .recorded_time import GeneratedTimestamp diff --git a/src/agent_comms/native_input_proof.py b/src/agent_comms/native_input_proof.py new file mode 100644 index 0000000000..0399bdcd71 --- /dev/null +++ b/src/agent_comms/native_input_proof.py @@ -0,0 +1,297 @@ +"""Native input identity, byte placement and corroborated model-context proof. + +These declarations are shared by durable input state, emitted events and the +proof journal without depending on session execution or transcript routing. +""" + +from __future__ import annotations + +import os +import sqlite3 +from collections.abc import Iterator +from contextlib import ExitStack, closing, contextmanager +from dataclasses import dataclass, field +from pathlib import Path +from typing import TYPE_CHECKING, Annotated, ClassVar, Literal + +from pydantic import JsonValue + +from .coordination_errors import IdentityConflict +from .errors import RelationViolationError +from .field_codec import ( + FieldCodec, + InputId, + MalformedRecord, + PathText, + Sha256Text, + WireName, + boundary_decoder, +) +from .locked_store import SQLiteFile +from .native_failures import NativePiUnavailable +from .native_session_reopen import PiSessionId +from .private_path import FileIdentity, PrivateFileRole +from .typed_table import ( + Column, + DurableTables, + GuardedInserts, + ImmutableRows, + Index, + IntegerStorage, + WithoutRowid, +) + +if TYPE_CHECKING: + from . import pi_events as pi + from .native_entries import NativeEntry, NativeEvidenceRead, SessionEntry + from .native_session_files import NativeSessionFiles + + +@dataclass(frozen=True, slots=True) +class NativeInputCommit: + """Original emitted input identity shared with its assembled context.""" + + input_id: Annotated[ + InputId, + WireName("inputId"), + Column(primary_key=True), + ] + session_id: Annotated[PiSessionId, WireName("sessionId")] + session_entry_id: Annotated[str, WireName("sessionEntryId")] + + @property + def input_commit(self) -> NativeInputCommit: + return NativeInputCommit(self.input_id, self.session_id, self.session_entry_id) + + +@dataclass(frozen=True) +class NativeInputLocation(NativeInputCommit): + """Emitted byte placement; only the original runtime input row persists it.""" + + # SQL permits an uncommitted row; the emitted offset remains a required int. + position_column: ClassVar[Column] = Column( + check="session_entry_start>0", + nullable=True, + storage=IntegerStorage, + ) + session_entry_start: Annotated[ + int, WireName("sessionEntryStart"), position_column + ] = field(kw_only=True) + + +@dataclass(frozen=True, slots=True) +class NativeContextRecord(NativeInputCommit): + """Native context facts extend the original committed input declaration.""" + + request_generation: Annotated[ + int, + WireName("requestGeneration"), + Column(primary_key=True, check="request_generation BETWEEN 1 AND 9007199254740991"), + ] + llm_context_digest: Annotated[ + str, + Sha256Text, + WireName("llmContextDigest"), + ] + # The native writer stamps the moment this context was committed for a + # model request: the time the input was shown to the model. + committed_at_ms: Annotated[ + int, + WireName("committedAtMs"), + Column(check="committed_at_ms BETWEEN 1 AND 9007199254740991"), + ] + + @classmethod + def from_events( + cls, input_id: InputId, session_id: PiSessionId, + input_event: pi.InputCommitted, context_event: pi.ContextCommitted, + ) -> NativeContextRecord: + """Strictly join the two emitted records through the context declaration. + + This is an emitted acknowledgement, not proof of durable inclusion. A journal + read must independently corroborate the same complete located record. + """ + emitted = FieldCodec.encode(context_event) + record = FieldCodec.decode(cls, {key: emitted[key] + for _, key in FieldCodec.record_fields(cls)}) + if record.input_commit != NativeInputCommit(input_id, session_id, record.session_entry_id): + raise NativePiUnavailable("Native Pi context names another original input") + if record.input_commit != input_event.input_commit: + raise NativePiUnavailable("Native Pi input/context events disagree") + return record + + def __post_init__(self): + Sha256Text.decode(self.llm_context_digest) + if not 0 < self.request_generation <= 2**53 - 1: + raise ValueError("Native context generation is outside its native range") + if not 0 < self.committed_at_ms <= 2**53 - 1: + raise ValueError("Native context commit time is outside its native range") + + def at(self, session_file: Path, *, session_entry_start: int) -> NativeContextProof: + """Locate recorded facts; this does not grant acceptance or replay.""" + return NativeContextProof( + self.input_id, + self.session_id, + self.session_entry_id, + self.request_generation, + self.llm_context_digest, + self.committed_at_ms, + session_file, + session_entry_start=session_entry_start, + ) + + +@dataclass(frozen=True, slots=True) +class NativeContextJournal( + GuardedInserts, ImmutableRows, NativeContextRecord, WithoutRowid, DurableTables +): + """One proof declaration owns wire facts and the indexed durable journal.""" + + insert_condition = """NEW.request_generation < COALESCE((SELECT MAX(request_generation) + FROM native_context_journal), 0) OR EXISTS(SELECT 1 FROM native_context_journal + WHERE request_generation=NEW.request_generation + AND (llm_context_digest != NEW.llm_context_digest + OR committed_at_ms != NEW.committed_at_ms)) OR NEW.session_id + != (SELECT session_id FROM native_context_journal ORDER BY session_id LIMIT 1)""" + + + schema: Literal[2] + type: Literal["context_committed"] + + indexes = (Index(("request_generation",)), Index(("session_id",))) + + @classmethod + def native_contract(cls) -> dict[str, JsonValue]: + """Generate the native writer's SQLite contract from this declaration.""" + table = cls.declared_name + return { + "objects": cls.schema_objects(), + "columns": [ + {"name": plan.name, "wire": plan.key} for plan in FieldCodec.field_plans(cls) + ], + "insert": cls.insertion_statement(), + "head": f"SELECT COALESCE(MAX(request_generation),0) AS generation FROM {table}", + "session": f"SELECT session_id AS id FROM {table} ORDER BY session_id LIMIT 1", + "current": f"SELECT input_id,session_entry_id FROM {table} " + f"WHERE request_generation=(SELECT MAX(request_generation) FROM {table})", + } + + @classmethod + @contextmanager + def open_evidence(cls, files: NativeSessionFiles) -> Iterator[sqlite3.Connection]: + """Read an indexed snapshot after SQLite's bounded hot-journal recovery. + + A read is corroboration, never a fresh acknowledgement. Keep the private inode + pinned across SQLite's transaction and refuse redirection or schema drift. + Open existing storage read/write for SQLite rollback, then forbid SQL + writes. A cold history view can recover without a model turn or acknowledgement. + """ + path = files.input_proof + with ExitStack() as ownership: + try: + held = ownership.enter_context(closing( + os.fdopen(PrivateFileRole.read( + path, NativePiUnavailable("Native proof indexed evidence is unavailable")), "rb") + )) + before = os.fstat(held.fileno()) + db = ownership.enter_context(closing(SQLiteFile.recovering_snapshot(path))) + if cls.schema_differs(db): + raise NativePiUnavailable("Native proof requires offline durable conversion") + if FileIdentity.from_stat(before) != FileIdentity.from_stat(path.lstat()): + raise NativePiUnavailable("Native proof inode changed while opening") + except (OSError, sqlite3.Error, RelationViolationError, MalformedRecord) as error: + raise NativePiUnavailable("Native proof indexed evidence is unavailable") from error + # Consumer exceptions retain their original boundary and disposition. + # The ExitStack closes the database and descriptor without handling caller errors. + yield db + try: + if FileIdentity.from_stat(before) != FileIdentity.from_stat(path.lstat()): + raise NativePiUnavailable("Native proof inode changed during observation") + except OSError as error: + raise NativePiUnavailable("Native proof indexed evidence is unavailable") from error + + @classmethod + def for_input( + cls, db: sqlite3.Connection, input_id: InputId, generation: int | None = None + ) -> NativeContextJournal | None: + """The primary key answers one input without scanning lifetime history.""" + where = "input_id=?" + parameters: tuple[InputId | int, ...] = (input_id,) + if generation is not None: + where += " AND request_generation=?" + parameters += (generation,) + rows = cls.read( + db.execute( + f"SELECT {cls._column_list(tuple(cls.column_projection()))} FROM {cls.declared_name} " + f"WHERE {where} ORDER BY request_generation DESC LIMIT 1", + parameters, + ) + ) + return next(iter(rows), None) + + def corroborate( + self, session_file: Path, header: SessionEntry, entry: NativeEntry, *, + session_entry_start: int, + ) -> NativeContextProof: + if self.session_id != header.id or self.session_entry_id != entry.id: + raise NativePiUnavailable("Native Pi proof journal source lineage differs") + return self.at(session_file, session_entry_start=session_entry_start) + + +@dataclass(frozen=True, slots=True) +class NativeContextProof(NativeContextRecord, NativeInputLocation): + session_file: Annotated[Path, PathText] + + def require_continuation(self, context: NativeContextProof) -> None: + """Repeated requests may include this input; its first receipt stays immutable. + + A different native entry or conflicting proof of the same request is not + another observation of that entry. Callers corroborate later generations + before this check; this comparison grants no native or replay authority. + """ + if (self.input_commit, self.session_file, self.session_entry_start) != ( + context.input_commit, context.session_file, context.session_entry_start + ): + raise IdentityConflict("Native context acknowledgement belongs to another entry") + if context.request_generation < self.request_generation: + raise IdentityConflict("Native context precedes its original acknowledgement") + if context.request_generation == self.request_generation and context != self: + raise IdentityConflict("Native request has contradictory context evidence") + + def corroborates_input( + self, input_id: InputId, *, files: NativeSessionFiles, + evidence: NativeEvidenceRead, + ) -> bool: + """Check an already-observed live event against its isolated saved proof. + + This grants neither replay nor recovery authority. History IO occurs + before the coordinator read transaction, as on the native return path. + """ + if self.input_id != input_id or files.session != self.session_file: + return False + return self.read_evidence( + files, input_id, + request_generation=self.request_generation, evidence=evidence, + session_entry_start=self.session_entry_start, + ) == self + + @classmethod + @boundary_decoder + def read_evidence( + cls, files: NativeSessionFiles, input_id: InputId, *, request_generation: int | None = None, + evidence: NativeEvidenceRead | None = None, + session_entry_start: int, + ) -> NativeContextProof: + """Corroborate live recorded events; parsed bytes alone grant no authority.""" + session_file = files.session.absolute() + from .native_entries import NativeInputEvidenceRead + with NativeInputEvidenceRead.borrow(session_file, evidence) as evidence: + with NativeContextJournal.open_evidence(files) as db: + row = NativeContextJournal.for_input(db, input_id, request_generation) + if row is None: + raise NativePiUnavailable("The input has no assembled-context proof") + location = NativeInputLocation(input_id, row.session_id, row.session_entry_id, + session_entry_start=session_entry_start) + entry = evidence.read_input(location) + return row.corroborate(session_file, evidence.header(), entry, + session_entry_start=session_entry_start) diff --git a/src/agent_comms/native_input_record.py b/src/agent_comms/native_input_record.py index 81422f126b..1aad5751ad 100644 --- a/src/agent_comms/native_input_record.py +++ b/src/agent_comms/native_input_record.py @@ -8,12 +8,8 @@ import sqlite3 from abc import ABC, abstractmethod -from dataclasses import ( - dataclass, - field, - replace, -) -from typing import TYPE_CHECKING, Annotated, ClassVar, Literal +from dataclasses import dataclass, replace +from typing import TYPE_CHECKING, Literal from agent_comms.coordination_tables.executions import ExecutionRecord @@ -34,11 +30,10 @@ JsonShapeMember, MalformedRecord, NativeContext, - WireName, ) from .native_session_reopen import PiSessionId from .transcript_routes import InputDisplay, NoDisplayText -from .typed_table import Column, IntegerStorage, SqlStorage, TextStorage +from .typed_table import SqlStorage, TextStorage if TYPE_CHECKING: from sqlite3 import Connection @@ -52,41 +47,10 @@ from .message_reference import MessageReference from .native_entries import MessageEntry, RecordedEntry, TranscriptProjection from .native_runtime_input import NativeRuntimeInput + from .native_input_proof import NativeContextProof from .transcript_events import TranscriptEvent -@dataclass(frozen=True, slots=True) -class NativeInputCommit: - """Original emitted input identity shared with its assembled context.""" - - input_id: Annotated[ - InputId, - WireName("inputId"), - Column(primary_key=True), - ] - session_id: Annotated[PiSessionId, WireName("sessionId")] - session_entry_id: Annotated[str, WireName("sessionEntryId")] - - @property - def input_commit(self) -> NativeInputCommit: - return NativeInputCommit(self.input_id, self.session_id, self.session_entry_id) - - -@dataclass(frozen=True) -class NativeInputLocation(NativeInputCommit): - """Emitted byte placement; only the original runtime input row persists it.""" - - # SQL permits an uncommitted row; the emitted offset remains a required int. - position_column: ClassVar[Column] = Column( - check="session_entry_start>0", - nullable=True, - storage=IntegerStorage, - ) - session_entry_start: Annotated[ - int, WireName("sessionEntryStart"), position_column - ] = field(kw_only=True) - - class NativeInputPosition(DeclaredFamily, JsonShapeFamily, affix="NativePosition"): """Decode the original SQL placement once: uncommitted NULL or a byte offset.""" @@ -402,6 +366,10 @@ def require_recorded(self) -> NativeInputReference: def require_recorded_input(self, owner: CursorOwner, db: sqlite3.Connection) -> None: """An unrecorded context grants no committed-input identity.""" + def acknowledge_context(self, row: NativeRuntimeInput, db: sqlite3.Connection, context: NativeContextProof) -> None: + """The first verified entry is immutable; later settlement does not own it.""" + row.write_context(db, context) + @dataclass(frozen=True) class UnrecordedNativeInputReference(NativeContextReference): @@ -441,6 +409,9 @@ def acquire(cls, row: NativeInputContext) -> NativeInputReference: except MalformedRecord as error: raise IdentityConflict("Native recorded context reference is incomplete") from error + def acknowledge_context(self, row: NativeRuntimeInput, db: sqlite3.Connection, context: NativeContextProof) -> None: + row.require_context_proof().require_continuation(context) + def require_recorded_input(self, owner: CursorOwner, db: sqlite3.Connection) -> None: owner.require_recorded_reference(db, self) diff --git a/src/agent_comms/native_pi.py b/src/agent_comms/native_pi.py index d439846824..84db56ba51 100644 --- a/src/agent_comms/native_pi.py +++ b/src/agent_comms/native_pi.py @@ -9,15 +9,11 @@ import os import re import shutil -import sqlite3 import sys -from collections.abc import Iterator -from contextlib import ExitStack, closing, contextmanager +from contextlib import contextmanager from dataclasses import dataclass, field from pathlib import Path -from typing import TYPE_CHECKING, Annotated, Literal - -from pydantic import JsonValue +from typing import TYPE_CHECKING from agent_comms.participant_identity import ParticipantIdentity @@ -31,16 +27,12 @@ FieldCodec, InputId, MalformedRecord, - Sha256Text, - WireName, - boundary_decoder, ) from .launch_root import LaunchRoot from .launch_runtime import LaunchRuntime, RestartEnvironment, ThreadLaunch -from .locked_store import SQLiteFile from .native_arguments import NativeArguments, SelectedModel from .native_failures import NativePiUnavailable, NativeRequestFailure -from .native_input_record import NativeInputCommit, NativeInputLocation +from .native_input_proof import NativeContextProof, NativeContextRecord from .native_package import MANIFEST, UNIT_INSTRUCTIONS, VerifiedNativePackage from .native_session_files import NativeAgentSettings, NativeSessionFiles from .native_tool_call import NativeToolMode @@ -51,18 +43,10 @@ PrivateFileRole, TrustedAncestorRole, ) -from .typed_table import ( - Column, - DurableTables, - GuardedInserts, - ImmutableRows, - Index, - WithoutRowid, -) if TYPE_CHECKING: from . import pi_events as pi - from .native_entries import NativeEntry, NativeEvidenceRead, SessionEntry + from .native_entries import NativeEvidenceRead from .selected_session import SelectedSession from .turn_failure import TurnFailure from .native_session_reopen import PiSessionId @@ -159,208 +143,6 @@ def main() -> int: return 0 -@dataclass(frozen=True, slots=True) -class NativeContextRecord(NativeInputCommit): - """Native context facts extend the original committed input declaration.""" - - request_generation: Annotated[ - int, - WireName("requestGeneration"), - Column(primary_key=True, check="request_generation BETWEEN 1 AND 9007199254740991"), - ] - llm_context_digest: Annotated[ - str, - Sha256Text, - WireName("llmContextDigest"), - ] - # The native writer stamps the moment this context was committed for a - # model request: the time the input was shown to the model. - committed_at_ms: Annotated[ - int, - WireName("committedAtMs"), - Column(check="committed_at_ms BETWEEN 1 AND 9007199254740991"), - ] - - @classmethod - def from_events( - cls, input_id: InputId, session_id: PiSessionId, - input_event: pi.InputCommitted, context_event: pi.ContextCommitted, - ) -> NativeContextRecord: - """Strictly join the two emitted records through the context declaration. - - This is an emitted acknowledgement, not proof of durable inclusion. A journal - read must independently corroborate the same complete located record. - """ - emitted = FieldCodec.encode(context_event) - record = FieldCodec.decode(cls, {key: emitted[key] - for _, key in FieldCodec.record_fields(cls)}) - if record.input_commit != NativeInputCommit(input_id, session_id, record.session_entry_id): - raise NativePiUnavailable("Native Pi context names another original input") - if record.input_commit != input_event.input_commit: - raise NativePiUnavailable("Native Pi input/context events disagree") - return record - - def __post_init__(self): - Sha256Text.decode(self.llm_context_digest) - if not 0 < self.request_generation <= 2**53 - 1: - raise ValueError("Native context generation is outside its native range") - if not 0 < self.committed_at_ms <= 2**53 - 1: - raise ValueError("Native context commit time is outside its native range") - - def at(self, session_file: Path, *, session_entry_start: int) -> NativeContextProof: - """Locate recorded facts; this does not grant acceptance or replay.""" - return NativeContextProof( - self.input_id, - self.session_id, - self.session_entry_id, - self.request_generation, - self.llm_context_digest, - self.committed_at_ms, - session_file, - session_entry_start=session_entry_start, - ) - - -@dataclass(frozen=True, slots=True) -class NativeContextJournal( - GuardedInserts, ImmutableRows, NativeContextRecord, WithoutRowid, DurableTables -): - """One proof declaration owns wire facts and the indexed durable journal.""" - - insert_condition = """NEW.request_generation < COALESCE((SELECT MAX(request_generation) - FROM native_context_journal), 0) OR EXISTS(SELECT 1 FROM native_context_journal - WHERE request_generation=NEW.request_generation - AND (llm_context_digest != NEW.llm_context_digest - OR committed_at_ms != NEW.committed_at_ms)) OR NEW.session_id - != (SELECT session_id FROM native_context_journal ORDER BY session_id LIMIT 1)""" - - - schema: Literal[2] - type: Literal["context_committed"] - - indexes = (Index(("request_generation",)), Index(("session_id",))) - - @classmethod - def native_contract(cls) -> dict[str, JsonValue]: - """Generate the native writer's SQLite contract from this declaration.""" - table = cls.declared_name - return { - "objects": cls.schema_objects(), - "columns": [ - {"name": plan.name, "wire": plan.key} for plan in FieldCodec.field_plans(cls) - ], - "insert": cls.insertion_statement(), - "head": f"SELECT COALESCE(MAX(request_generation),0) AS generation FROM {table}", - "session": f"SELECT session_id AS id FROM {table} ORDER BY session_id LIMIT 1", - "current": f"SELECT input_id,session_entry_id FROM {table} " - f"WHERE request_generation=(SELECT MAX(request_generation) FROM {table})", - } - - @classmethod - @contextmanager - def open_evidence(cls, files: NativeSessionFiles) -> Iterator[sqlite3.Connection]: - """Read an indexed snapshot after SQLite's bounded hot-journal recovery. - - A read is corroboration, never a fresh acknowledgement. Keep the private inode - pinned across SQLite's transaction and refuse redirection or schema drift. - Open existing storage read/write for SQLite rollback, then forbid SQL - writes. A cold history view can recover without a model turn or acknowledgement. - """ - path = files.input_proof - with ExitStack() as ownership: - try: - held = ownership.enter_context(closing( - os.fdopen(PrivateFileRole.read( - path, NativePiUnavailable("Native proof indexed evidence is unavailable")), "rb") - )) - before = os.fstat(held.fileno()) - db = ownership.enter_context(closing(SQLiteFile.recovering_snapshot(path))) - if cls.schema_differs(db): - raise NativePiUnavailable("Native proof requires offline durable conversion") - if FileIdentity.from_stat(before) != FileIdentity.from_stat(path.lstat()): - raise NativePiUnavailable("Native proof inode changed while opening") - except (OSError, sqlite3.Error, RelationViolationError, MalformedRecord) as error: - raise NativePiUnavailable("Native proof indexed evidence is unavailable") from error - # Consumer exceptions retain their original boundary and disposition. - # The ExitStack closes the database and descriptor without handling caller errors. - yield db - try: - if FileIdentity.from_stat(before) != FileIdentity.from_stat(path.lstat()): - raise NativePiUnavailable("Native proof inode changed during observation") - except OSError as error: - raise NativePiUnavailable("Native proof indexed evidence is unavailable") from error - - @classmethod - def for_input( - cls, db: sqlite3.Connection, input_id: InputId, generation: int | None = None - ) -> NativeContextJournal | None: - """The primary key answers one input without scanning lifetime history.""" - where = "input_id=?" - parameters: tuple[InputId | int, ...] = (input_id,) - if generation is not None: - where += " AND request_generation=?" - parameters += (generation,) - rows = cls.read( - db.execute( - f"SELECT {cls._column_list(tuple(cls.column_projection()))} FROM {cls.declared_name} " - f"WHERE {where} ORDER BY request_generation DESC LIMIT 1", - parameters, - ) - ) - return next(iter(rows), None) - - def corroborate( - self, session_file: Path, header: SessionEntry, entry: NativeEntry, *, - session_entry_start: int, - ) -> NativeContextProof: - if self.session_id != header.id or self.session_entry_id != entry.id: - raise NativePiUnavailable("Native Pi proof journal source lineage differs") - return self.at(session_file, session_entry_start=session_entry_start) - - -@dataclass(frozen=True, slots=True) -class NativeContextProof(NativeContextRecord, NativeInputLocation): - session_file: Path - - def corroborates_input( - self, input_id: InputId, *, files: NativeSessionFiles, - evidence: NativeEvidenceRead, - ) -> bool: - """Check an already-observed live event against its isolated saved proof. - - This grants neither replay nor recovery authority. History IO occurs - before the coordinator read transaction, as on the native return path. - """ - if self.input_id != input_id or files.session != self.session_file: - return False - return self.read_evidence( - files, input_id, - request_generation=self.request_generation, evidence=evidence, - session_entry_start=self.session_entry_start, - ) == self - - @classmethod - @boundary_decoder - def read_evidence( - cls, files: NativeSessionFiles, input_id: InputId, *, request_generation: int | None = None, - evidence: NativeEvidenceRead | None = None, - session_entry_start: int, - ) -> NativeContextProof: - """Corroborate live recorded events; parsed bytes alone grant no authority.""" - session_file = files.session.absolute() - from .native_entries import NativeInputEvidenceRead - with NativeInputEvidenceRead.borrow(session_file, evidence) as evidence: - with NativeContextJournal.open_evidence(files) as db: - row = NativeContextJournal.for_input(db, input_id, request_generation) - if row is None: - raise NativePiUnavailable("The input has no assembled-context proof") - location = NativeInputLocation(input_id, row.session_id, row.session_entry_id, - session_entry_start=session_entry_start) - entry = evidence.read_input(location) - return row.corroborate(session_file, evidence.header(), entry, - session_entry_start=session_entry_start) - - class NativePiTerminalFailure(NativePiUnavailable): """A proved input ended in a failed terminal and its owned process was reaped.""" diff --git a/src/agent_comms/native_prompt_send.py b/src/agent_comms/native_prompt_send.py index 8b7bfbc73b..b3ea169690 100644 --- a/src/agent_comms/native_prompt_send.py +++ b/src/agent_comms/native_prompt_send.py @@ -26,6 +26,8 @@ if TYPE_CHECKING: from .input_drain import InputDrain from .owned_send import OwnedSend + from .native_input_proof import NativeContextProof + from .native_entries import NativeEvidenceScope from .tracked_turn import TrackedTurnSession _MAX_SEND_SECONDS = 5.0 @@ -35,6 +37,11 @@ class PromptBoundary(ABC): @abstractmethod async def send_prompt(self, turn: TrackedTurnSession) -> None: ... + async def context_committed( + self, context: NativeContextProof, source_reads: NativeEvidenceScope, + ) -> None: + """An unowned prompt has no durable recipient acknowledgement to advance.""" + def following_sender(self, inputs: InputDrain, session_id: AcpSessionId, prompt: str) -> OwnedSend: raise RelationViolationError("An unowned prompt cannot authorize following input") diff --git a/src/agent_comms/native_runtime_input.py b/src/agent_comms/native_runtime_input.py index d2086fc1bd..3b0768a045 100644 --- a/src/agent_comms/native_runtime_input.py +++ b/src/agent_comms/native_runtime_input.py @@ -18,6 +18,7 @@ from .coordination_errors import IdentityConflict, PublicationActivationBlocked, StaleFence from .coordination_tables.participants import OwnerGenerations from .coordination_tables.responses import ResponseObligation +from .class_fact import ClassFact from .declared_family import DeclaredFamily from .field_codec import ( FieldCodec, @@ -34,7 +35,6 @@ NativeColumn, NativeInputContext, NativeInputExecution, - NativeInputLocation, NativeInputPosition, NativeInputRecord, NativeStageStorage, @@ -45,7 +45,7 @@ UnrecordedNativeColumn, UnrecordedNativePosition, ) -from .native_pi import NativeContextRecord +from .native_input_proof import NativeContextRecord, NativeInputLocation from .native_session_reopen import ( NativeSessionIdentity, NoPiSessionId, @@ -60,7 +60,6 @@ from .tool_denial import SelectedToolDenied from .typed_table import ( Column, - FinalizedColumn, FrozenColumn, IntegerStorage, MonotoneColumns, @@ -77,7 +76,7 @@ ) if TYPE_CHECKING: - from .native_pi import NativeContextProof + from .native_input_proof import NativeContextProof from .native_session_reopen import NativeSessionIdentity @@ -99,7 +98,6 @@ class PublishedReplyRevision(TypedRow): @dataclass(frozen=True) class NativeRuntimeInput( WriteOnceColumns, - FinalizedColumn, RequiredUpdateColumns, NoDelete, NativeInputRecord, @@ -134,8 +132,13 @@ def notification_context_sql(cls) -> str: "member.ordinal,n.input_id LIMIT 1) AS context_committed_at_ms FROM w" ) - write_once_columns = ('sent_owner_claimed_generation', 'session_id', 'session_file') - finalized_column = 'session_entry_id' + @ClassFact + def write_once_columns(cls) -> tuple[str, ...]: + # The declared context group is immutable, not the later triage verdict. + return ('sent_owner_claimed_generation', 'session_id', 'session_file', 'verdict', + *(plan.name for plan in FieldCodec.field_plans(cls) + if NativeContext.single(plan.annotation))) + required_update_columns = ('sent_owner_claimed_generation',) @classmethod @@ -320,22 +323,20 @@ def published_replies(self, db, user, owner_lookup): "stage='triage' OR verdict IS NULL", ) - def require_unproven(self, token_digest: str) -> None: - """Only the original reserved capability can acquire its first proof.""" - if self.owner_token_digest != token_digest or self.reference.recorded: - raise StaleFence("native proof belongs to a different or already settled dispatch") - - def commit_context( - self, - db: sqlite3.Connection, - context: NativeContextProof, - *, - verdict: TriageDecisionRecord = AbsentTriageDecisionRecord(), - ) -> None: - """Commit the context acknowledgement once against the original selected session.""" + def require_dispatch(self, token_digest: str) -> None: + """Only the original dispatch may acknowledge context or settle its result.""" + if self.owner_token_digest != token_digest: + raise StaleFence("native proof belongs to a different dispatch") + + def commit_context(self, db: sqlite3.Connection, context: NativeContextProof) -> None: + """Retain the first corroborated context, independently of terminal settlement.""" if context.input_id != self.input_id: raise StaleFence("native context belongs to another reserved input") self.require_session_identity().require_context(context) + self.reference.acknowledge_context(self, db, context) + + def write_context(self, db: sqlite3.Connection, context: NativeContextProof) -> None: + """The unrecorded reference alone takes this first-write path.""" updated = self.update( db, where="input_id=? AND session_entry_id IS NULL", @@ -345,11 +346,20 @@ def commit_context( request_generation=NumberNativeColumn(context.request_generation), llm_context_digest=TextNativeColumn(context.llm_context_digest), committed_at_ms=NumberNativeColumn(context.committed_at_ms), - verdict=verdict, ) if updated.rowcount != 1: raise StaleFence("native proof was previously committed") + def settle_verdict(self, db: sqlite3.Connection, verdict: TriageDecisionRecord) -> None: + """The decision follows context entry; neither transition overwrites the other.""" + current = type(self).require(db, input_id=self.input_id) + current.reference.require_recorded() + current.verdict.require_undecided() + updated = self.update(db, where="input_id=? AND verdict IS NULL", + parameters=(self.input_id,), verdict=verdict) + if updated.rowcount != 1: + raise StaleFence("native triage verdict was previously settled") + class SelectedSlotState(DeclaredFamily, affix="Slot"): """Consumption is irreversible; only a matching live terminal can acknowledge it.""" diff --git a/src/agent_comms/native_session_files.py b/src/agent_comms/native_session_files.py index 465b0b5192..3202f2fe29 100644 --- a/src/agent_comms/native_session_files.py +++ b/src/agent_comms/native_session_files.py @@ -256,7 +256,7 @@ class NativeInputProof(DurableFile, ReconciledFile, PatternFile): @classmethod def table_schema(cls) -> type[TableSchema]: - from .native_pi import NativeContextJournal + from .native_input_proof import NativeContextJournal return NativeContextJournal diff --git a/src/agent_comms/native_session_reopen.py b/src/agent_comms/native_session_reopen.py index dd9cc76c2d..b840783cab 100644 --- a/src/agent_comms/native_session_reopen.py +++ b/src/agent_comms/native_session_reopen.py @@ -16,7 +16,7 @@ from .native_child import PiSessionChild from .native_runtime_input import NativeRuntimeInput from .native_entries import NativeEvidenceRead - from agent_comms.native_pi import NativeContextProof + from .native_input_proof import NativeContextProof from .pi_native_backend import PersistentPiSession diff --git a/src/agent_comms/native_transcript.py b/src/agent_comms/native_transcript.py index a6a7a106c0..8fcccdbaf4 100644 --- a/src/agent_comms/native_transcript.py +++ b/src/agent_comms/native_transcript.py @@ -12,10 +12,10 @@ from typing import TYPE_CHECKING, BinaryIO from .errors import RelationViolationError -from .native_pi import NativeContextJournal +from .native_input_proof import NativeContextJournal if TYPE_CHECKING: - from .native_pi import NativeContextProof + from .native_input_proof import NativeContextProof from .native_output import NativeRecordLocation from .read_ledger import ReadLedger from .thread_identity import RecordedThreadIncarnation diff --git a/src/agent_comms/notification_assignment.py b/src/agent_comms/notification_assignment.py index e60c62df8c..11673872a4 100644 --- a/src/agent_comms/notification_assignment.py +++ b/src/agent_comms/notification_assignment.py @@ -5,8 +5,10 @@ import sqlite3 from abc import abstractmethod from collections.abc import Iterable, Iterator, Sequence +from itertools import batched from dataclasses import dataclass, replace from pathlib import Path +from types import MappingProxyType from typing import TYPE_CHECKING, NoReturn from .audience_manifest import FrozenRecipient @@ -20,13 +22,12 @@ from .field_codec import FieldCodec, MalformedRecord, MessageId, ThreadLookup, boundary_decoder from .message_reference import MessageReference from .notification_progress import ( - AnsweredProgress, AssignmentProgress, QueuedProgress, SentProgress, ShownToModelProgress, ) -from .notification_evidence import NotificationAssignmentRead, NotificationWindow, SourceEvidence +from .notification_evidence import NotificationAssignmentRead, NotificationWindow, NotificationSelection from .private_registry_guard import RegistryPrivacy from .registry_store import RegistryStore from .typed_table import BooleanStorage, TypedRow @@ -38,7 +39,7 @@ from .agent_activity import RecipientActivity from .bus_publication import Delivered from .coordination_database import InvalidStore, MissingStore - from .presentation import MessageNotification + from .presentation import MessageNotification, OutcomeViewer from .private_registry_guard import RegistryPrivacy from .wake_policy import RecipientDecision @@ -46,11 +47,12 @@ @dataclass(frozen=True) class AssignmentContextEvidence(DeclaredFamily, affix="ContextEvidence"): @classmethod + @boundary_decoder def from_columns(cls, committed_at_ms: object) -> AssignmentContextEvidence: - committed_at_ms = FieldCodec.decode(int | None, committed_at_ms) + """SQL NULL is the one foreign fact tested; a present value decodes as its own type.""" if committed_at_ms is None: return NotShownContextEvidence() - return ShownContextEvidence(committed_at_ms) + return ShownContextEvidence(FieldCodec.decode(int, committed_at_ms)) @abstractmethod def progress(self, *, pending_at_ms: int, queued: bool) -> AssignmentProgress: ... @@ -82,14 +84,16 @@ def recorded(self, message: Delivered) -> AssignmentResponse: def from_columns( cls, seq: object, message_id: object, received_at_ms: object ) -> AssignmentResponse: - seq = FieldCodec.decode(int | None, seq) - message_id = FieldCodec.decode(MessageId | None, message_id) - received_at_ms = FieldCodec.decode(int | None, received_at_ms) - if seq is None and message_id is None and received_at_ms is None: + """SQL NULL is the one foreign fact tested; present values decode as their own types.""" + columns = (seq, message_id, received_at_ms) + if all(column is None for column in columns): return UnansweredResponse() - if seq is None or message_id is None or received_at_ms is None: + if any(column is None for column in columns): raise MalformedRecord("Published response evidence is incomplete") - return AnsweredResponse(MessageReference(seq, message_id), received_at_ms) + return AnsweredResponse( + MessageReference(FieldCodec.decode(int, seq), FieldCodec.decode(MessageId, message_id)), + FieldCodec.decode(int, received_at_ms), + ) @abstractmethod def completed_notification( @@ -125,12 +129,7 @@ def completed_notification(self, progress: AssignmentProgress, notification: Mes ) def answered(self, notification: MessageNotification) -> MessageNotification: - return replace( - notification, - state=AnsweredProgress(notification.state), - answer=self.reference, - answered_at_ms=self.received_at_ms, - ) + return notification.state.answered_notification(notification, self.reference, self.received_at_ms) @dataclass(frozen=True) @@ -148,16 +147,14 @@ def completed_notification(self, notification: MessageNotification) -> MessageNo def answered(self, notification: MessageNotification) -> MessageNotification: return self.response.answered(notification) - def recorded_response(self, message: Delivered) -> Self: - return replace(self, response=self.response.recorded(message)) - @dataclass(frozen=True) class AssignmentActivityEvidence(TypedRow): """Independent context and response facts decoded once from one SQL row.""" triage_inflight: bool - current_execution_id: str | None + # Whether the recipient has a current execution: SQL answers it, no id is decoded. + queued: bool context: AssignmentContextEvidence response: AssignmentResponse @@ -165,7 +162,7 @@ class AssignmentActivityEvidence(TypedRow): def column_projection(cls) -> dict[str, str]: return { "triage_inflight": "triage_inflight", - "current_execution_id": "current_execution_id", + "queued": "queued", "context_committed_at_ms": "context_committed_at_ms", "response_seq": "response_seq", "response_message_id": "response_message_id", @@ -179,7 +176,7 @@ def column(name: str): return cls( BooleanStorage.from_sql(column("triage_inflight")), - FieldCodec.decode(str | None, column("current_execution_id")), + BooleanStorage.from_sql(column("queued")), AssignmentContextEvidence.from_columns(column("context_committed_at_ms")), AssignmentResponse.from_columns( column("response_seq"), @@ -193,7 +190,7 @@ def resolve(self, *, pending_at_ms: int) -> AssignmentActivity: self.triage_inflight, self.context.progress( pending_at_ms=pending_at_ms, - queued=self.current_execution_id is not None, + queued=self.queued, ), self.response, ) @@ -233,33 +230,32 @@ class NotificationAssignment(NotificationSource): @classmethod def changed(cls, reads: Iterable[NotificationAssignmentRead], root: Path) -> tuple[NotificationAssignmentRead, ...]: """Acquire fresh bounded rows once; retained reads own their equality.""" - reads = tuple(reads) - sources = tuple({source.message.reference: source - for read in reads for source in read.sources}.values()) - fresh: dict[MessageReference, SourceEvidence] = {} - for sources_chunk in NotificationWindow.chunks(sources): - fresh.update(cls.read(root, sources_chunk).evidence) - return tuple(read for read in reads if read.differs_from(fresh)) + # A retained read owns its recipient cut. Re-reading a different viewer's + # superset would both decode it and report a permanent false change. + return tuple(read for read in reads + if read.differs_from(cls.read(root, read.selection).evidence)) @classmethod - def resolve(cls, root: Path, read: NotificationAssignmentRead) -> tuple[NotificationAssignment, ...]: - resolved = tuple(cls(assignment, activity.resolve(pending_at_ms=assignment.updated_at_ms)) - for assignment, activity in read.rows) - return cls._recorded_conversation_responses(root, read.sources, resolved) + def resolve(cls, read: NotificationAssignmentRead) -> tuple[NotificationAssignment, ...]: + return tuple(cls(assignment, activity.resolve(pending_at_ms=assignment.updated_at_ms)) + for assignment, activity in read.rows) @classmethod def for_deliveries( - cls, originals: Iterable[Delivered], records: Iterable[NotificationAssignment], + cls, selection: NotificationSelection, records: Iterable[NotificationAssignment], ) -> Iterator[tuple[Delivered, tuple[NotificationSource, ...]]]: - """Match notification assignments to messages and their recorded recipients.""" + """Match notification assignments to messages and the recorded recipients this + viewer may see; an invisible recipient allocates no placeholder.""" by_recipient = {} for record in records: key = (record.assignment.source, record.assignment.recipient_lookup) by_recipient.setdefault(key, []).append(record) - for original in originals: + for page in selection.pages: + original = page.source reference = original.message.reference sources = [] - for recipient, decision in zip(original.recipients, original.decisions, strict=True): + for position in page.positions: + recipient, decision = original.recipients[position], original.decisions[position] matching = by_recipient.get((reference, recipient.recipient_lookup), ()) if len(matching) > 1: raise ValueError("Original notification recipient has multiple handling rows") @@ -267,99 +263,96 @@ def for_deliveries( yield original, tuple(sources) @classmethod - def _recorded_conversation_responses( - cls, root: Path, sources: tuple[Delivered, ...], - rows: tuple[NotificationAssignment, ...], - ) -> tuple[NotificationAssignment, ...]: - eligible = tuple( - row for row in rows if row.assignment.lifecycle.records_conversation_response - ) - if not eligible: - return rows - - originals = {original.message.reference: original for original in sources} - captured = Bus(Bus.path_in(root)).replies( - tuple({row.assignment.source for row in eligible})) - replies: dict[MessageReference, list[Delivered]] = {} + def capture_responses(cls, root: Path, read: NotificationAssignmentRead) -> NotificationAssignmentRead: + """Capture bounded independent replies after releasing the coordinator snapshot. + + A reply belongs to its original/frozen author, not to a claim lifecycle or + model-context proof. The bus returns only the first reply per selected key, + in sequence order, including recipients with no claim at all. + """ + keys = frozenset(read.selection.keys) + if not keys: + return read + captured = Bus(Bus.path_in(root)).recipient_replies(read.selection.keys) + responses: dict[tuple[ThreadLookup, MessageReference], AssignmentResponse] = {} for record in captured: for reply in record.deliveries(): for answered in reply.message.answers: - replies.setdefault(answered, []).append(reply) - resolved = [] - for row in rows: - if row.assignment.lifecycle.records_conversation_response: - response = row.assignment.lifecycle.recorded_response( - originals[row.assignment.source], - replies.get(row.assignment.source, ()), - row.assignment.recipient_lookup, - ) - if response is not None: - row = replace(row, activity=row.activity.recorded_response(response)) - resolved.append(row) - return tuple(resolved) + key = (reply.sender_lookup, answered) + if key in keys: + responses[key] = responses.get(key, UnansweredResponse()).recorded(reply.message) + return replace(read, responses=MappingProxyType(responses)) @classmethod def capture( - cls, connection: sqlite3.Connection, sources: Sequence[Delivered], + cls, connection: sqlite3.Connection, selection: NotificationSelection, privacy: RegistryPrivacy, ) -> NotificationAssignmentRead: - """Capture detached assignment evidence for one bounded window of originals - from the caller's SQLite snapshot.""" - sources = tuple(sources) - if len(sources) > NotificationWindow.window_limit: + """Probe only selected frozen keys before joining any claim/context/proof. + + The existing (recipient_lookup, wire_seq) unique index admits each row; + message_id corroborates the exact original rather than a current name. + """ + if len(selection.pages) > NotificationWindow.window_limit: raise ValueError("Notification reads require a bounded visible message window") - if not sources: - return NotificationAssignmentRead((), ()) + keys = tuple(dict.fromkeys(selection.keys)) + if not keys: + return NotificationAssignmentRead(selection, ()) privacy.coordinator_schema().require_schema(connection) columns = ",".join(f"w.{name}" for name in WakeAssignment.columns()) - # The window's assignments and their native-input membership are each - # selected once, membership by its assignment index, not once per row - # and per column. - rows = WakeAssignment.joined( - connection.execute( - f"WITH w AS MATERIALIZED (SELECT * FROM {WakeAssignment.declared_name} " - f"WHERE wire_seq IN ({','.join('?' for _ in sources)})), " - f"context AS MATERIALIZED ({privacy.notification_context_sql(connection)}) " - f"SELECT {columns},context.triage_inflight," - "pointer.execution_id AS current_execution_id,context.context_committed_at_ms," - "p.seq AS response_seq, p.message_id AS response_message_id, " - "p.received_at_ms AS response_received_at_ms " - "FROM w " - "JOIN context ON context.assignment_id=w.assignment_id " - f"LEFT JOIN {CurrentExecutions.declared_name} pointer " - "ON pointer.owner_lookup=w.recipient_lookup " - f"LEFT JOIN {PublicationEvidence.declared_name} p " - "ON p.execution_id=w.execution_id AND p.exact_target=w.exact_target " - "ORDER BY w.wire_seq DESC,w.recipient", - tuple(source.message.seq for source in sources), - ), - AssignmentActivityEvidence, - ) - attempts = HarnessDelivery.for_recipients(connection, tuple( - (recipient.recipient_lookup, source.message.reference) - for source in sources for recipient in source.recipients)) - return NotificationAssignmentRead(sources, tuple(rows), attempts) + context = privacy.notification_context_sql(connection) + rows = [] + # Each batch's requested keys select their assignments through the + # recipient/seq index; their native-input membership is selected once per + # batch by its assignment index, not once per row and per column. + for requested in WakeAssignment.requested_keys(connection, tuple( + {"recipient_lookup": lookup, + "wire_seq": reference.seq, "message_id": reference.message_id} + for lookup, reference in keys)): + rows.extend(WakeAssignment.joined( + connection.execute( + f"WITH {requested.values}, " + f"w AS MATERIALIZED (SELECT * {requested.join}), " + f"context AS MATERIALIZED ({context}) " + f"SELECT {columns},context.triage_inflight," + "pointer.execution_id IS NOT NULL AS queued,context.context_committed_at_ms," + "p.seq AS response_seq, p.message_id AS response_message_id, " + "p.received_at_ms AS response_received_at_ms " + "FROM w " + "JOIN context ON context.assignment_id=w.assignment_id " + f"LEFT JOIN {CurrentExecutions.declared_name} pointer " + "ON pointer.owner_lookup=w.recipient_lookup " + f"LEFT JOIN {PublicationEvidence.declared_name} p " + "ON p.execution_id=w.execution_id AND p.exact_target=w.exact_target", + requested.bindings, + ), + AssignmentActivityEvidence, + )) + attempts = HarnessDelivery.for_recipients(connection, keys) + return NotificationAssignmentRead(selection, tuple(sorted(rows, key=lambda row: (-row[0].wire_seq, row[0].recipient))), attempts) @classmethod def read( - cls, root: Path, sources: Sequence[Delivered] + cls, root: Path, selection: NotificationSelection ) -> NotificationAssignmentRead: - """Capture assignment evidence through the single observing connection owner.""" + """Capture this viewer's assignment evidence through the single observing + connection owner.""" privacy = RegistryStore.at(root).privacy() with CoordinationStore.observed(CoordinationStore.path_in(root)).observing( - cls, sources, privacy, + cls, selection, privacy, ) as result: - return result + captured = result + return cls.capture_responses(root, captured) @classmethod def missing_store( - cls, store: MissingStore, sources: Sequence[Delivered], privacy: RegistryPrivacy, + cls, store: MissingStore, selection: NotificationSelection, privacy: RegistryPrivacy, ) -> NotificationAssignmentRead: - return privacy.missing_notification_assignments(sources) + return privacy.missing_notification_assignments(selection) @classmethod def invalid_store( - cls, store: InvalidStore, sources: Sequence[Delivered], privacy: RegistryPrivacy, + cls, store: InvalidStore, selection: NotificationSelection, privacy: RegistryPrivacy, ) -> NoReturn: raise ValueError(f"Channel notification status unavailable: {store.failure_reason}") diff --git a/src/agent_comms/notification_evidence.py b/src/agent_comms/notification_evidence.py index 9759d51f48..41a27dfa9c 100644 --- a/src/agent_comms/notification_evidence.py +++ b/src/agent_comms/notification_evidence.py @@ -7,6 +7,7 @@ from dataclasses import dataclass, field from functools import cached_property from itertools import batched +from types import MappingProxyType from typing import TYPE_CHECKING, ClassVar if TYPE_CHECKING: @@ -17,42 +18,99 @@ from .coordination_tables.harness_deliveries import HarnessDelivery from .field_codec import ThreadLookup from .message_reference import MessageReference - from .notification_assignment import AssignmentActivityEvidence + from .notification_assignment import AssignmentActivityEvidence, AssignmentResponse from .notification_progress import DeliveryProgress - from .presentation import MessageNotification, WireRevision + from .presentation import MessageNotification, WireRevision, OutcomeViewer from .thread_identity import ThreadIncarnation @dataclass(frozen=True) class SourceEvidence: - """One original's captured claims and harness attempts, the attempts grouped once by - their frozen recipient. Equality over both is the window's change test.""" + """One original's claims, attempts and independent responses, grouped by frozen + recipient. Equality over all three is the window's change test.""" claims: tuple[tuple[WakeAssignment, AssignmentActivityEvidence], ...] = () attempts: Mapping[ThreadLookup, tuple[HarnessDelivery, ...]] = field(default_factory=dict) + responses: Mapping[ThreadLookup, AssignmentResponse] = field(default_factory=lambda: MappingProxyType({})) + + def answered(self, recipient: ThreadLookup, notification: MessageNotification) -> MessageNotification: + from .notification_assignment import UnansweredResponse + + return self.responses.get(recipient, UnansweredResponse()).answered(notification) def consumed_progress(self, recipient: ThreadLookup, progress: DeliveryProgress) -> DeliveryProgress: """The recipient's progress after each of its own attempts on this original.""" for attempt in self.attempts.get(recipient, ()): - progress = attempt.outcome.consumed_progress(progress) + progress = attempt.consumed_progress(progress) return progress +@dataclass(frozen=True) +class RecipientSelection: + """A bounded slice of original frozen positions, never a replacement audience.""" + + source: Delivered + eligible: range + limit: int + max_limit: ClassVar[int] = 100 + + def __post_init__(self) -> None: + self.require_limit(self.limit) + + @classmethod + def require_limit(cls, limit: int) -> None: + if not 1 <= limit <= cls.max_limit: + raise ValueError(f"Recipient limit must be between 1 and {cls.max_limit}") + + @property + def positions(self) -> range: + return self.eligible[:self.limit] + + @property + def remaining(self) -> int: + return len(self.eligible) - len(self.positions) + + @property + def recipients(self) -> tuple[FrozenRecipient, ...]: + return tuple(self.source.recipients[position] for position in self.positions) + + +@dataclass(frozen=True) +class NotificationSelection: + """The exact per-original recipient cuts used by every expensive evidence reader.""" + + viewer: OutcomeViewer + pages: tuple[RecipientSelection, ...] + + @property + def sources(self) -> tuple[Delivered, ...]: + return tuple(page.source for page in self.pages) + + @property + def keys(self) -> tuple[tuple[ThreadLookup, MessageReference], ...]: + return tuple((recipient.recipient_lookup, page.source.message.reference) + for page in self.pages for recipient in page.recipients) + + @dataclass(frozen=True) class NotificationAssignmentRead: - """A window's claims and its harness attempts, both keyed by the exact original. + """A window's claims and its harness attempts, both keyed by the exact original, as the + viewer captured with them may see them: every later projection and change check of this + read answers for that same viewer. A harness attempt is its own consumption evidence: a no-wake or passive recipient has no claim for the attempt to move, and its message still entered the session.""" - sources: tuple[Delivered, ...] + selection: NotificationSelection rows: tuple[tuple[WakeAssignment, AssignmentActivityEvidence], ...] attempts: tuple[HarnessDelivery, ...] = () + responses: Mapping[tuple[ThreadLookup, MessageReference], AssignmentResponse] = field( + default_factory=lambda: MappingProxyType({})) - @classmethod - def empty(cls) -> NotificationAssignmentRead: - return cls((), ()) + @property + def sources(self) -> tuple[Delivered, ...]: + return self.selection.sources @cached_property def evidence(self) -> Mapping[MessageReference, SourceEvidence]: @@ -63,12 +121,18 @@ def evidence(self) -> Mapping[MessageReference, SourceEvidence]: attempts: dict[MessageReference, dict[ThreadLookup, list[HarnessDelivery]]] = { source.message.reference: {} for source in self.sources } + responses: dict[MessageReference, dict[ThreadLookup, AssignmentResponse]] = { + source.message.reference: {} for source in self.sources + } + for (lookup, source), response in self.responses.items(): + responses[source][lookup] = response for row in self.rows: claims.setdefault(row[0].source, []).append(row) for attempt in self.attempts: attempts[attempt.reference].setdefault(attempt.recipient_lookup, []).append(attempt) return {source: SourceEvidence(tuple(claims[source]), { - lookup: tuple(rows) for lookup, rows in attempts[source].items()}) + lookup: tuple(rows) for lookup, rows in attempts[source].items()}, + MappingProxyType(responses[source])) for source in claims} def differs_from(self, fresh: Mapping[MessageReference, SourceEvidence]) -> bool: @@ -105,6 +169,10 @@ class NotificationWindow(ABC): reads: tuple[NotificationAssignmentRead, ...] read_marks: tuple[NotificationReadMark, ...] = () + def remaining(self, reference: MessageReference) -> int: + return next((page.remaining for read in self.reads for page in read.selection.pages + if page.source.message.reference == reference), 0) + @classmethod def chunks[T](cls, items: Iterable[T]) -> Iterator[tuple[T, ...]]: return batched(items, cls.window_limit) diff --git a/src/agent_comms/notification_progress.py b/src/agent_comms/notification_progress.py index 81925885a9..7e89670745 100644 --- a/src/agent_comms/notification_progress.py +++ b/src/agent_comms/notification_progress.py @@ -72,9 +72,15 @@ def counted_summary_prefix(self, notification: MessageNotification) -> tuple[str """The lead a counted wrapper prints; a consumed proof is already named by its count.""" return (notification.label,) - def harness_consumed(self) -> DeliveryProgress: - """This progress once a harness confirmed the message entered its session.""" - return HarnessConsumedProgress() + def answered_notification(self, notification: MessageNotification, + reference: MessageReference, answered_at_ms: int) -> MessageNotification: + """An independent answer preserves this state's actual context proof.""" + return replace(notification, state=AnsweredProgress(self), + answer=reference, answered_at_ms=answered_at_ms, changed=True) + + def harness_consumed(self, observed_at_ms: int) -> DeliveryProgress: + """Retain the original complete-body transcript observation time.""" + return HarnessConsumedProgress(observed_at_ms) class ConsumedDelivery(SettledDelivery): @@ -84,7 +90,7 @@ class ConsumedDelivery(SettledDelivery): ignored_label: ClassVar[str] = "Seen, not answering" consumed: ClassVar[bool] = True - def harness_consumed(self) -> Self: + def harness_consumed(self, observed_at_ms: int) -> Self: """An earlier proof already timed the entry; a harness confirmation adds nothing.""" return self @@ -95,11 +101,6 @@ def counted_summary_prefix(self, notification: MessageNotification) -> tuple[str return () -@dataclass(frozen=True) -class HarnessConsumedProgress(ConsumedDelivery, DeliveryProgress): - """A harness confirmed context entry with no claim to time it; the attempt has no clock.""" - - @dataclass(frozen=True) class UnrecordedProgress(WaitingDelivery, DeliveryProgress): """No recipient delivery has been recorded.""" @@ -121,16 +122,16 @@ class ContextualProgress(WaitingDelivery, DeliveryProgress): def consumed(self) -> bool: return self.proof.consumed - def harness_consumed(self) -> DeliveryProgress: - return replace(self, proof=self.proof.harness_consumed()) + def harness_consumed(self, observed_at_ms: int) -> DeliveryProgress: + return replace(self, proof=self.proof.harness_consumed(observed_at_ms)) class PendingProgress(ContextualProgress): """A pending claim adds no label of its own: once a harness proves entry, the proof is the whole state.""" - def harness_consumed(self) -> DeliveryProgress: - return self.proof.harness_consumed() + def harness_consumed(self, observed_at_ms: int) -> DeliveryProgress: + return self.proof.harness_consumed(observed_at_ms) class PassiveProgress(ContextualProgress): @@ -186,6 +187,11 @@ class FailedProgress(ContextualProgress): class AnsweredProgress(SettledDelivery, ContextualProgress): label = "Answered" + def answered_notification(self, notification: MessageNotification, + reference: MessageReference, answered_at_ms: int) -> MessageNotification: + """A later independent response cannot replace an already recorded answer.""" + return notification + def notification_label(self, notification: MessageNotification) -> str: return notification.timed_state(self.label, notification.answered_at_ms) @@ -210,11 +216,6 @@ def selected_notification(self, notification: MessageNotification, *, busy: bool @abstractmethod def consumed_notification(self, notification: MessageNotification) -> MessageNotification: ... - def answered_notification(self, notification: MessageNotification, - reference: MessageReference, answered_at_ms: int) -> MessageNotification: - return replace(notification, state=AnsweredProgress(self), - answer=reference, answered_at_ms=answered_at_ms) - @dataclass(frozen=True) class SentProgress(WaitingDelivery, AssignmentProgress): @@ -263,6 +264,10 @@ def consumed_notification(self, notification: MessageNotification) -> MessageNot return replace(notification, state=self) +class HarnessConsumedProgress(ShownToModelProgress): + """Complete original body observed in the harness transcript, at its recorded time.""" + + class SelectedConsumedProgress(ActiveDelivery, ShownToModelProgress): pass diff --git a/src/agent_comms/notification_read.py b/src/agent_comms/notification_read.py index 22467ba40f..9820c7b10a 100644 --- a/src/agent_comms/notification_read.py +++ b/src/agent_comms/notification_read.py @@ -1,17 +1,21 @@ -"""A captured notification window answers whether its own evidence changed.""" +"""A captured notification window, and where a notification read settled, answer whether +their own evidence changed.""" from __future__ import annotations -from dataclasses import dataclass +from collections.abc import Mapping +from dataclasses import dataclass, field from typing import TYPE_CHECKING from .notification_assignment import NotificationAssignment -from .notification_evidence import NotificationWindow -from .revision_views import NotificationEvidenceView +from .notification_evidence import NoNotificationWindow, NotificationWindow +from .revision_views import ActivityView, NotificationEvidenceView if TYPE_CHECKING: + from .activity import ObservedActivity from .comms import Comms from .presentation import WireRevision + from .registry_provenance import DeletedThread @dataclass(frozen=True) @@ -24,3 +28,38 @@ def changed(self, comms: Comms, revision: WireRevision, earlier: WireRevision) - self.read_marks, comms.registry.snapshot(), )) or NotificationAssignment.changed(self.reads, comms.root) ) + + +@dataclass(frozen=True) +class NotificationsRead: + """Where one notification read settled: its revision and the activity of the participants + its notifications name, captured before the read for participants already known and right + after it for new ones. The window names every watched participant; activity holds only + the ones the registry projects, so a human or archived participant is absent from it.""" + + revision: WireRevision + activity: Mapping[str, ObservedActivity | DeletedThread] = field(default_factory=dict) + notification_window: NotificationWindow = field(default_factory=NoNotificationWindow) + + @classmethod + def of(cls, revision: WireRevision, comms: Comms, + known: Mapping[str, ObservedActivity | DeletedThread], + window: NotificationWindow, + ) -> NotificationsRead: + names = window.activity_names() + known = {**comms.views.observed_activity(revision, names - known.keys()), **known} + return cls(revision, {name: known[name] for name in names if name in known}, window) + + def live_changed(self, comms: Comms, revision: WireRevision) -> bool: + """Read marks, participant activity or retained assignment evidence changed.""" + return ((revision.changed_since(ActivityView(), self.revision) + and comms.views.observed_activity( + revision, self.notification_window.activity_names()) != self.activity) + or self.notification_window.changed(comms, revision, self.revision)) + + +class UnreadableNotificationsRead(NotificationsRead): + """An unreadable read names no participants, so any later revision reads again.""" + + def live_changed(self, comms: Comms, revision: WireRevision) -> bool: + return revision != self.revision diff --git a/src/agent_comms/notification_reads.py b/src/agent_comms/notification_reads.py new file mode 100644 index 0000000000..46a96b27be --- /dev/null +++ b/src/agent_comms/notification_reads.py @@ -0,0 +1,67 @@ +"""Root-free notification window operations; UI transport and MCP paging stay at their boundaries. + +A notification read owns its references (and, when recorded, the archive it reads) and how +its settled evidence goes stale. The UI carries one in ``ReadNotifications`` with its own root +and request id; MCP's single-message page calls the live read directly under its caller. +""" +from __future__ import annotations + +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from typing import TYPE_CHECKING + +from .bus_read import BusReader, UiRead +from .historical_views import HistorySource +from .message_reference import MessageReference +from .notification_evidence import NotificationWindow, RecipientSelection +from .presentation import FirstRecipient, RecipientCursor +from .notification_read import NotificationsRead + +if TYPE_CHECKING: + from .comms import Comms + from .presentation import WireRevision + + +@dataclass(frozen=True) +class NotificationRead(UiRead[NotificationWindow], ABC): + """Every notification read declares its references; members own evidence freshness.""" + + references: tuple[MessageReference, ...] + + @abstractmethod + def changed_since(self, comms: Comms, revision: WireRevision, + previous: NotificationsRead) -> bool: ... + + +@dataclass(frozen=True) +class LiveNotificationRead(NotificationRead): + """Recipient handling of the referenced original messages, as its reader may see them. + The UI reads as the operator; an MCP caller reads under its own viewer.""" + + after: RecipientCursor = field(default_factory=FirstRecipient, kw_only=True) + limit: int = field(default=RecipientSelection.max_limit, kw_only=True) + + def read_as(self, comms: Comms, reader: BusReader) -> NotificationWindow: + return comms.views.message_notifications_for_references( + reader.outcome_viewer(comms), self.references, after=self.after, limit=self.limit) + + def changed_since(self, comms: Comms, revision: WireRevision, + previous: NotificationsRead) -> bool: + return previous.live_changed(comms, revision) + + +@dataclass(frozen=True) +class RecordedNotificationRead(LiveNotificationRead): + """The selected archive's recorded handling, independent of today's recipients.""" + + source: str + + def read_as(self, comms: Comms, reader: BusReader) -> NotificationWindow: + with comms.bus.history.read_sources() as sources: + source = HistorySource.require(sources, self.source) + return source.notification_references( + sources, self.references, reader.outcome_viewer(comms), after=self.after, limit=self.limit) + + def changed_since(self, comms: Comms, revision: WireRevision, + previous: NotificationsRead) -> bool: + return revision.history != previous.revision.history diff --git a/src/agent_comms/order_tools.py b/src/agent_comms/order_tools.py index c30347268e..894acf23b5 100644 --- a/src/agent_comms/order_tools.py +++ b/src/agent_comms/order_tools.py @@ -5,7 +5,8 @@ from abc import abstractmethod from collections.abc import Mapping from dataclasses import dataclass, field, replace -from typing import Annotated, Literal +from functools import partial +from typing import TYPE_CHECKING, Annotated, Literal from pydantic import JsonValue @@ -33,9 +34,12 @@ JudgedCall, ToolCallVerdict, ) +from .acp_extension import GoalSnapshotReply, TranscriptSnapshotUpdate, TurnChangedUpdate, UpdateMetadata +from .bus_read import BusReader, OwnerLiveRead, UiRead from .commander.vibe_check import OverloadDegradation -from .commander_hud import CommanderHud +from .commander_hud import CommanderFrame, CommanderHud, CommanderUnitDetail from .comms import Comms +from .coordinator import Coordination from .errors import ToolRefusedError from .field_codec import FieldCodec, MessageId, OrderId, RecordProblems, WireName from .goal_actions import CreateProblemGoalAction, OrderGoalAction @@ -59,6 +63,7 @@ tool_field, ) from .tool_result import ToolResult +from .transcripts import TranscriptCursor, TranscriptRead from .transcript_events import SentTranscript from .turn_digests import ( ActivityTurnView, @@ -71,6 +76,9 @@ UnjudgedEdits, ) +if TYPE_CHECKING: + from .owner_connection import RuntimeRequestContext + @dataclass(frozen=True) class TodoChanged(ToolResult): @@ -508,13 +516,108 @@ def apply(self, comms: Comms, caller: ToolCaller) -> ThreadTurns: @dataclass(frozen=True, kw_only=True) -class CommsInputDeliveryTool(ThreadReadTool): +class InputDeliveryRead( + OwnerLiveRead[DeliveryOverview], ThreadReadTool, declared_name="comms_input_delivery", +): + """The one declaration of a unit's input delivery, answered by its live native owner.""" + label = "Input delivery" description = ("Read a unit's inputs that failed or are UNKNOWN, and its earlier delivery " "notices, each with its bus sequence and input id; none is ever replayed.") + include_history: Annotated[ + bool, *tool_field("Include earlier delivery notices; current inputs are always read") + ] = True + + async def owned(self, ctx: RuntimeRequestContext) -> DeliveryOverview: + scope = await ctx.server.agent.inputs.delivery_scope(ctx.session_id) + return await Coordination.run_worker(partial( + ctx.server.agent._comms.goals.input_delivery, + ctx.name, include_history=self.include_history, scope=scope, + )) + + def decode_reply(self, raw: object) -> DeliveryOverview: + return FieldCodec.decode(DeliveryOverview, raw) + + +@dataclass(frozen=True, kw_only=True) +class OwnerGoalRead(OwnerLiveRead[GoalSnapshotReply], ThreadReadTool, declared_name="comms_owner_goal"): + """A unit's goal snapshot and the turn it was read with, answered by its live native owner + under the owner's own registration and session; the thread selector only finds the owner.""" + + label = "Owner goal" + description = ("Read a unit's current goal snapshot and the turn state its live native " + "owner read with it.") + + async def owned(self, ctx: RuntimeRequestContext) -> GoalSnapshotReply: + snapshot = await Coordination.run_worker(partial( + ctx.server.agent._comms.goals.goal_snapshot, ctx.name, + )) + turn = await Coordination.run_worker(partial( + ctx.server.agent.turns.turn_state, ctx.session_id, + )) + return GoalSnapshotReply.from_snapshot(snapshot, UpdateMetadata.of(TurnChangedUpdate(turn))) - def apply(self, comms: Comms, caller: ToolCaller) -> DeliveryOverview: - return comms.goals.input_delivery(caller.subject(self.thread), include_history=True) + def decode_reply(self, raw: object) -> GoalSnapshotReply: + return FieldCodec.decode(GoalSnapshotReply, raw) + + +@dataclass(frozen=True, kw_only=True) +class TranscriptPageRead( + UiRead[TranscriptSnapshotUpdate], ThreadReadTool, declared_name="comms_transcript_page", +): + """One bounded transcript page with its current draft and the source witness that read it. + The cursors and historical source select the page; they never choose whose authority the + read runs under.""" + + label = "Transcript page" + description = ("Read one bounded page of a thread's transcript with its current draft. " + "Send a returned cursor as before or after for the next page; " + "historical_source reads an archived source.") + before: Annotated[TranscriptCursor | None, *tool_field("Read the page before this cursor")] = None + after: Annotated[TranscriptCursor | None, *tool_field("Read the page after this cursor")] = None + through: Annotated[TranscriptCursor | None, *tool_field("Read through this cursor")] = None + historical_source: Annotated[ + str | None, *tool_field("Archived source to read instead of current history") + ] = None + + def capture_as(self, comms: Comms, reader: BusReader) -> TranscriptRead: + return comms.transcripts.capture_page_read( + reader.subject(self.thread), + before=self.before, + after=self.after, + through=self.through, + historical_source=self.historical_source, + ) + + def read_as(self, comms: Comms, reader: BusReader) -> TranscriptSnapshotUpdate: + return TranscriptSnapshotUpdate.from_read(self.capture_as(comms, reader)) + + +@dataclass(frozen=True, kw_only=True) +class CommanderUnitRead( + UiRead[CommanderUnitDetail], ReadTool, ToolRequest, declared_name="comms_commander_unit", +): + """One unit's card on a commander's published frame, expanded with its orders' bounded + instructions, stories, commits and activity. The held incarnations and orders select the + exact card and witness it; they are never whose authority the read runs under.""" + + label = "Commander unit" + description = ("Expand one unit's card on a commander's published frame: its orders' " + "bounded instructions, stories and commits, and its recent activity. " + "Refuses when either incarnation or the unit's assignment changed.") + commander: Annotated[ThreadIncarnation, *tool_field("Commander incarnation whose frame shows the card")] + unit: Annotated[ThreadIncarnation, *tool_field("Unit incarnation on that card")] + orders: Annotated[tuple[OrderId, ...], *tool_field("The card's orders, exactly as published")] + + def read_as(self, comms: Comms, reader: BusReader) -> CommanderUnitDetail: + snapshot = comms.registry.snapshot() + if not self.commander.current(snapshot) or not self.unit.current(snapshot): + raise ToolRefusedError("The selected commander or unit incarnation is no longer current") + hud = CommanderFrame.current(comms, self.commander.name) + for card in hud.units: + if card.unit == self.unit: + return card.detail(comms, self.commander, self.orders) + raise ToolRefusedError("The selected unit is no longer in this commander's published frame") @dataclass(frozen=True, kw_only=True) diff --git a/src/agent_comms/owned_send.py b/src/agent_comms/owned_send.py index 98f9ea2b3d..f54321eed6 100644 --- a/src/agent_comms/owned_send.py +++ b/src/agent_comms/owned_send.py @@ -5,6 +5,7 @@ import logging from contextlib import AbstractContextManager from dataclasses import dataclass +from functools import partial from pathlib import Path from typing import TYPE_CHECKING @@ -39,6 +40,7 @@ from .goal_waits import GoalWait from .input_drain import InputDrain + from .native_input_proof import NativeContextProof from .registry_document import RegistrySnapshot from .threads import Thread from .turn_goal_permission import TurnGoalPermission @@ -172,6 +174,35 @@ def original(self, native_id: InputId, text: str) -> bool: def following(self, public_id: InputId, native_id: InputId, text: str) -> bool: return self._started(self.source(public_id), native_id, text) + async def original_context_committed(self, sent_text: str, context: NativeContextProof) -> None: + await Coordination.run_worker(partial( + self._record_context, self.original_source, sent_text, context, + )) + + async def following_context_committed( + self, public_id: InputId, sent_text: str, context: NativeContextProof, + ) -> None: + await Coordination.run_worker(partial( + self._record_context, self.source(public_id), sent_text, context, + )) + + def _record_context( + self, source: TurnInputSource, sent_text: str, context: NativeContextProof, + ) -> None: + with self.inputs.comms.registry.store.current_snapshot() as cut: + snapshot = cut.snapshot + source.require_context_source(self) + canonical = snapshot.canonical_name(self.thread.name) + current = self.require_owner(snapshot, canonical) + self._check_owner( + current, snapshot, canonical, source, sent_text, + check_type=RunningOrdinaryOwnerCheck, + ) + self.inputs.dispositions.context_committed( + source.keys, lease=self.thread.require_turn_lease(), + sent_text=sent_text, context=context, snapshot=snapshot, + ) + def _started(self, source: TurnInputSource, native_id: InputId, sent_text: str) -> bool: return source.valid_keys(sent_text) and all( self.inputs.dispositions.started( diff --git a/src/agent_comms/owned_turn.py b/src/agent_comms/owned_turn.py index 78f48ff91c..bc2d12e72a 100644 --- a/src/agent_comms/owned_turn.py +++ b/src/agent_comms/owned_turn.py @@ -65,6 +65,7 @@ from .turn_terminal import EndedWithoutResult, TurnTerminal if TYPE_CHECKING: + from .cursor_publication import CursorDelivery from .turn_runner import TurnRunner @@ -86,6 +87,7 @@ class TurnDelivery: finish_event: asyncio.Event send_boundary: SendBoundary input_start: InputStartListener + cursor_delivery: CursorDelivery request_progress: Callable[..., None] ui_request: Callable[[DialogUiRequest], Awaitable[ExtensionUiChoice]] tool_mode: NativeToolMode @@ -384,6 +386,7 @@ async def stream(self): finish_event=self.finish_event, send_boundary=sender, input_start=sender, + cursor_delivery=self.runner.inputs.effects.cursors.delivery(self.session_id), request_progress=self.progress.record_request_progress, ui_request=lambda request: self.runner.extension_ui_permission( self.session_id, self.turn_id, self.controller, request diff --git a/src/agent_comms/owner_connection.py b/src/agent_comms/owner_connection.py index 205b51650f..54f2d386f7 100644 --- a/src/agent_comms/owner_connection.py +++ b/src/agent_comms/owner_connection.py @@ -8,8 +8,8 @@ import socket from functools import partial from pathlib import Path -from typing import TYPE_CHECKING, Annotated, ClassVar, cast -from abc import abstractmethod +from typing import TYPE_CHECKING, Annotated, ClassVar, Self, cast +from abc import ABC, abstractmethod from collections.abc import Awaitable from dataclasses import dataclass, replace from .command import Command @@ -28,7 +28,7 @@ from .comms import Comms from .child_process import ProcessIdentity from .registry_document import RegistryEntry - from .thread_identity import OwnerIdentity + from .thread_identity import OwnerIdentity, ThreadIncarnation from .runtime import RuntimeServer, SocketClient from .threads import Thread @@ -150,6 +150,11 @@ def __init__(self, comms: Comms, session_id: str, path: Path): self._identity = comms.registry.entry(session_id).entry.thread.incarnation self._root_identity = comms.root.stat() + def require_identity(self, owner: ThreadIncarnation) -> None: + """The incarnation this connection captured is the one the caller holds.""" + if self._identity != owner: + raise OwnerIdentityChangedError("The owner incarnation changed while preparing the request.") + def _owner_entry(self) -> RegistryEntry: if self._closed: raise ConnectionError("Owner attachment was closed before request dispatch") @@ -251,3 +256,51 @@ async def close(self) -> None: task.cancel() await asyncio.gather(*tasks, return_exceptions=True) + +class OwnerReading[Result](ABC): + """One exchange with exactly one live native owner incarnation over its socket.""" + + def require_native_owner(self, owner: Thread) -> None: + """Only a native owner serves a runtime socket; an external bridge with a live PID + has none, so it is refused here instead of waited on before dispatch.""" + owner.execution.require_native() + + @abstractmethod + async def read(self, owner: OwnerConnection) -> Result: ... + + +class OwnerReplyConnection(OwnerConnection): + """Bound only the live owner's response after a request has been dispatched.""" + + REPLY_TIMEOUT: ClassVar[float] = 3 + + @classmethod + def held(cls, comms: Comms, owner: ThreadIncarnation, reading: OwnerReading) -> Self: + """A connection to exactly this registered native owner incarnation. + + The selected registration must be the held incarnation before its process is + asked for; the connection's own capture is then checked again, since the + registry may change between the two reads. + """ + thread = comms.registry.require(owner.name) + if thread.incarnation != owner: + raise OwnerIdentityChangedError("The owner incarnation changed while preparing the request.") + reading.require_native_owner(thread) + connection = cls(comms, thread.name, socket_path(comms.root, thread.require_process().pid)) + connection.require_identity(owner) + return connection + + @classmethod + async def answer[Result]( + cls, comms: Comms, owner: ThreadIncarnation, reading: OwnerReading[Result], + ) -> Result: + """One reading from the held owner; the connection closes whatever the outcome.""" + connection = await asyncio.to_thread(cls.held, comms, owner, reading) + try: + return await reading.read(connection) + finally: + await connection.close() + + async def _read_response(self, reader: asyncio.StreamReader) -> bytes: + async with asyncio.timeout(self.REPLY_TIMEOUT): + return await super()._read_response(reader) diff --git a/src/agent_comms/pi_commands.py b/src/agent_comms/pi_commands.py index c74484d587..404b72246d 100644 --- a/src/agent_comms/pi_commands.py +++ b/src/agent_comms/pi_commands.py @@ -58,6 +58,9 @@ from .pi_events import Response, ResponseOutcome from .pending_requests import PendingRequest from .launch_runtime import RestartEnvironment + from .native_entries import NativeEvidenceScope + from .native_input_proof import NativeContextProof + from .pi_events import InputCommitted class SessionSnapshot: @@ -210,6 +213,15 @@ class Prompt(PiCommand): WireName("streamingBehavior"), ] = None + async def input_committed(self, session: TurnSession, event: InputCommitted) -> None: + await session.original_committed(event) + + async def context_committed( + self, session: TurnSession, context: NativeContextProof, + source_reads: NativeEvidenceScope, + ) -> None: + await session.original_context_committed(context, source_reads) + @classmethod def response_type(cls) -> type[PiCommand]: return Prompt @@ -239,6 +251,15 @@ class FollowingPrompt(Prompt, declared_name=None): input: Annotated[InputRequest, WireTransient()] + async def input_committed(self, session: TurnSession, event: InputCommitted) -> None: + """This accepted following input cannot restart the original prompt or inbox.""" + + async def context_committed( + self, session: TurnSession, context: NativeContextProof, + source_reads: NativeEvidenceScope, + ) -> None: + await session.input_start.following_context_committed(self.input.input_id, self.message, context) + def accepted(self, response: Response, session: TurnSession) -> Iterable[events.AgentEvent]: # PendingRequest already records this acknowledgement. return () diff --git a/src/agent_comms/pi_events.py b/src/agent_comms/pi_events.py index cfe1f5a292..012b695abd 100644 --- a/src/agent_comms/pi_events.py +++ b/src/agent_comms/pi_events.py @@ -22,7 +22,7 @@ WireOmitDefault, ) from .native_failures import NativePiUnavailable -from .native_input_record import NativeInputCommit, NativeInputLocation +from .native_input_proof import NativeInputCommit, NativeInputLocation from .native_output import NativeOutputLocation, NativeRecordLocation from .native_pi import NativePiOperationFailed from .native_turn_context import NativeContextManifestData diff --git a/src/agent_comms/pi_native_backend.py b/src/agent_comms/pi_native_backend.py index b1f7d6c191..a317305d8f 100644 --- a/src/agent_comms/pi_native_backend.py +++ b/src/agent_comms/pi_native_backend.py @@ -287,6 +287,7 @@ async def run_turn( finish_event=delivery.finish_event, send_boundary=delivery.send_boundary, input_start=delivery.input_start, + cursor_delivery=delivery.cursor_delivery, request_observer=delivery.request_progress, persistent_session=self, package=package, diff --git a/src/agent_comms/presentation.py b/src/agent_comms/presentation.py index 3a21c7715a..fefa407e8a 100644 --- a/src/agent_comms/presentation.py +++ b/src/agent_comms/presentation.py @@ -3,13 +3,14 @@ from __future__ import annotations from abc import ABC, abstractmethod +from bisect import bisect_left, bisect_right from collections import OrderedDict from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence from contextlib import contextmanager from dataclasses import dataclass, field, replace from datetime import datetime from pathlib import Path -from typing import TYPE_CHECKING, Annotated, ClassVar +from typing import TYPE_CHECKING, Annotated, ClassVar, Self from . import relationships from .activity import Activity, ActivityData, ActivityState, ObservedActivity @@ -71,7 +72,8 @@ ) from .read_ledger import NoReadNotice, ReadLedger, ReadNotice from .notification_assignment import NotificationAssignment -from .notification_evidence import NotificationAssignmentRead, NotificationReadMark, NotificationWindow +from .notification_evidence import (NotificationAssignmentRead, NotificationReadMark, NotificationWindow, + NotificationSelection, RecipientSelection) from .notification_read import CapturedNotificationWindow from .recorded_bus import NoRecordedBus, RecordedBusIdentity from .registry_document import DEFAULT_ROSTER, RosterVisibility @@ -122,12 +124,44 @@ from .ui_model.observation import ThreadsObserved +class RecipientCursor(SubclassUnion, affix=""): + """Select frozen, lookup-ordered positions before expensive evidence acquisition.""" + + @abstractmethod + def positions(self, source: Delivered) -> range: ... + + +@dataclass(frozen=True) +class FirstRecipient(NullMember, RecipientCursor): + def positions(self, source: Delivered) -> range: + return range(len(source.recipients)) + + +@dataclass(frozen=True) +class AfterRecipient(RecordMember, RecipientCursor): + recipient: FrozenRecipient + + def positions(self, source: Delivered) -> range: + start = bisect_right(source.recipients, self.recipient.recipient_lookup, + key=lambda item: item.recipient_lookup) + return range(start, len(source.recipients)) + + class OutcomeViewer(ABC): """Who reads a delivery window decides which of its outcomes are shown.""" @abstractmethod - def includes(self, sender_lookup: ThreadLookup, recipient: FrozenRecipient) -> bool: - pass + def positions(self, source: Delivered) -> range: ... + + def selection(self, sources: Sequence[Delivered], *, after: RecipientCursor = FirstRecipient(), + limit: int = RecipientSelection.max_limit) -> NotificationSelection: + RecipientSelection.require_limit(limit) + pages = [] + for source in sources: + eligible, following = self.positions(source), after.positions(source) + pages.append(RecipientSelection( + source, range(max(eligible.start, following.start), min(eligible.stop, following.stop)), limit)) + return NotificationSelection(self, tuple(pages)) @abstractmethod def sees_direct_read(self, source: Delivered) -> bool: @@ -142,8 +176,8 @@ def sees(self, source: Delivered) -> bool: class AllOutcomes(OutcomeViewer): """An unscoped reader sees every recipient's outcome and every human read.""" - def includes(self, sender_lookup: ThreadLookup, recipient: FrozenRecipient) -> bool: - return True + def positions(self, source: Delivered) -> range: + return range(len(source.recipients)) def sees_direct_read(self, source: Delivered) -> bool: return True @@ -162,16 +196,20 @@ class OwnerOutcomes(OutcomeViewer): def of(cls, owner: ThreadProvenance) -> OwnerOutcomes: return cls(owner.incarnation.require_recorded().lookup) - def includes(self, sender_lookup: ThreadLookup, recipient: FrozenRecipient) -> bool: - return self.lookup in (sender_lookup, recipient.recipient_lookup) + def positions(self, source: Delivered) -> range: + if source.sender_lookup == self.lookup: + return range(len(source.recipients)) + position = bisect_left(source.recipients, self.lookup, key=lambda item: item.recipient_lookup) + if position < len(source.recipients) and source.recipients[position].recipient_lookup == self.lookup: + return range(position, position + 1) + return range(0) def sees_direct_read(self, source: Delivered) -> bool: return source.sender_lookup == self.lookup def sees(self, source: Delivered) -> bool: """The message's sender and its original recipients may read it.""" - return self.lookup == source.sender_lookup or any( - self.lookup == recipient.recipient_lookup for recipient in source.recipients) + return self.lookup == source.sender_lookup or bool(self.positions(source)) @dataclass(frozen=True, slots=True) @@ -215,7 +253,8 @@ def summary(cls, notifications: Iterable[MessageNotification]) -> str: ordered = cls.ordered(notifications) if not ordered: return "No recorded notification result" - # Each notification exists because the frozen publication addressed its recipient. + # This is a bounded observation, not the complete frozen audience and + # not a set of runtime acknowledgements. Membership proves neither. total = len(ordered) consumed = sum(notification.state.consumed for notification in ordered) read = sum(bool(notification.displayed_to) for notification in ordered) @@ -226,9 +265,9 @@ def summary(cls, notifications: Iterable[MessageNotification]) -> str: return " · ".join(( *leading.state.summary_prefix(leading), *(f"{label} {count}" for label, count in counted if count), - f"Delivered {total}/{total}", - f"{ConsumedDelivery.label} {consumed}/{total}", - f"{HumanReadProgress.label} {read}/{total}", + f"Observed {total} recipients (bounded)", + f"{ConsumedDelivery.label} {consumed} observed", + *((f"{HumanReadProgress.label} {read} observed",) if read else ()), )) @staticmethod @@ -247,12 +286,13 @@ def timed_state(label: str, timestamp_ms: int | None) -> str: def references( cls, root: Path, registry: Registration, log: Bus, references: Sequence[MessageReference], *, viewer: OutcomeViewer, + after: RecipientCursor = FirstRecipient(), limit: int = RecipientSelection.max_limit, ) -> NotificationWindow: """Borrow mounted references in the original bounded delivery windows, as this viewer may see them.""" def project(sources: Sequence[Delivered]) -> NotificationWindow: - read = NotificationAssignment.read(root, sources) - return cls.delivery_window(root, registry, sources, read, viewer=viewer) + read = NotificationAssignment.read(root, viewer.selection(sources, after=after, limit=limit)) + return cls.delivery_window(root, registry, read) return cls.read_references(log, references, project) @@ -275,67 +315,66 @@ def read_references( @classmethod def delivery_window( - cls, root: Path, registry: Registration, sources: Sequence[Delivered], - assignment_read: NotificationAssignmentRead, *, viewer: OutcomeViewer, + cls, root: Path, registry: Registration, assignment_read: NotificationAssignmentRead, ) -> NotificationWindow: - """Project the original frozen audience even before handling is recorded.""" + """The captured selection owns original membership and viewer visibility.""" + sources = assignment_read.sources if not sources: return CapturedNotificationWindow({}, (assignment_read,)) snapshot = registry.snapshot() observations = AgentActivity(root, registry).observe_recipients( - ((source, recipient) for source in sources for recipient in source.recipients - if viewer.includes(source.sender_lookup, recipient)), + ((page.source, recipient) for page in assignment_read.selection.pages for recipient in page.recipients), snapshot=snapshot, ) return cls.project_delivery_window( - root, snapshot, sources, observations, - viewer=viewer, assignment_read=assignment_read, + root, snapshot, observations, assignment_read=assignment_read, ) @classmethod def recorded_delivery_window(cls, root: Path, namespace: RegistryProvenance, sources: Sequence[Delivered], *, - viewer: OutcomeViewer) -> NotificationWindow: + viewer: OutcomeViewer, after: RecipientCursor = FirstRecipient(), + limit: int = RecipientSelection.max_limit) -> NotificationWindow: + selection = viewer.selection(sources, after=after, limit=limit) observations = { recipient.recipient_lookup: RecordedRecipientActivity() - for source in sources for recipient in source.recipients + for page in selection.pages for recipient in page.recipients } - read = NotificationAssignment.read(root, sources) + read = NotificationAssignment.read(root, selection) return cls.project_delivery_window( - root, namespace, sources, observations, - assignment_read=read, viewer=viewer, + root, namespace, observations, assignment_read=read, ) @classmethod def project_delivery_window( cls, root: Path, snapshot: RegistryProvenance, - sources: Sequence[Delivered], observations: Mapping[ThreadLookup, RecipientActivity], - *, assignment_read: NotificationAssignmentRead, viewer: OutcomeViewer, + observations: Mapping[ThreadLookup, RecipientActivity], + *, assignment_read: NotificationAssignmentRead, ) -> NotificationWindow: """Original assignment and read authorities own both display projections.""" + sources, viewer = assignment_read.sources, assignment_read.selection.viewer if len(sources) > NotificationWindow.window_limit: raise ValueError("Notification reads require a bounded visible message window") keys = {(source.message.seq, source.message.message_id) for source in sources} result: dict[tuple[int, str], list[MessageNotification]] = {key: [] for key in keys} if not keys: return CapturedNotificationWindow({}, (assignment_read,)) - rows = NotificationAssignment.resolve(root, assignment_read) + rows = NotificationAssignment.resolve(assignment_read) evidence = assignment_read.evidence read_marks = [] reads = ReadLedger.at(root) document = reads.read() - for source, outcomes in NotificationAssignment.for_deliveries(sources, rows): + for source, outcomes in NotificationAssignment.for_deliveries(assignment_read.selection, rows): key = (source.message.seq, source.message.message_id) for outcome in outcomes: recipient = outcome.recipient() - if not viewer.includes(source.sender_lookup, recipient): - continue observation = observations[recipient.recipient_lookup] notification = outcome.project(observation, source.sender_lookup) state = evidence[source.message.reference].consumed_progress( recipient.recipient_lookup, notification.state) notification = replace(notification, state=state, changed=notification.changed or state.consumed) + notification = evidence[source.message.reference].answered(recipient.recipient_lookup, notification) displayed_to = observation.displayed_recipient( notification, source, reads, document, snapshot ) @@ -391,17 +430,15 @@ def sent_sources( @classmethod def for_sources( - cls, root: Path, registry: Registration, viewer: OutcomeViewer, - sources: Sequence[Delivered], *, - assignment_read: NotificationAssignmentRead, + cls, root: Path, registry: Registration, assignment_read: NotificationAssignmentRead, ) -> tuple[MessageNotification, ...]: """Attach notification outcomes to the messages the caller already checked.""" projected = cls.delivery_window( - root, registry, sources, assignment_read, viewer=viewer, + root, registry, assignment_read, ) return tuple( - replace(notification, message=source.message) - for source in sources + replace(notification, message=source.envelope) + for source in assignment_read.sources for notification in projected.outcomes[source.message.seq, source.message.message_id] ) @@ -412,44 +449,6 @@ class SentMessageOutcome(MessageNotification): message: Message = field(kw_only=True) -class RecipientCursor(SubclassUnion, affix=""): - """Where a page of one message's recipient outcomes starts; recipients are ordered by - their stable lookup, never by a display name.""" - - @abstractmethod - def following( - self, outcomes: Iterable[MessageNotification], - ) -> tuple[MessageNotification, ...]: - """The outcomes after this cursor, in recipient order.""" - - @staticmethod - def ordered(outcomes: Iterable[MessageNotification]) -> tuple[MessageNotification, ...]: - return tuple(sorted(outcomes, key=lambda outcome: outcome.recipient_identity.recipient_lookup)) - - -@dataclass(frozen=True) -class FirstRecipient(NullMember, RecipientCursor): - """No cursor given: start at the first recipient.""" - - def following( - self, outcomes: Iterable[MessageNotification], - ) -> tuple[MessageNotification, ...]: - return self.ordered(outcomes) - - -@dataclass(frozen=True) -class AfterRecipient(RecordMember, RecipientCursor): - """Start after this recipient, the last row of the previous page.""" - - recipient: FrozenRecipient - - def following( - self, outcomes: Iterable[MessageNotification], - ) -> tuple[MessageNotification, ...]: - return tuple(outcome for outcome in self.ordered(outcomes) - if outcome.recipient_identity.recipient_lookup > self.recipient.recipient_lookup) - - @dataclass(frozen=True, slots=True) class SentMessageSummary: """One sent message's recipient handling: its original reference and the totals @@ -460,14 +459,13 @@ class SentMessageSummary: summary: str @classmethod - def recent(cls, presentation: ThreadPresentation) -> tuple[SentMessageSummary, ...]: - """One summary per recent sent source, from the presentation's captured window.""" - window = presentation.notification_window - return tuple( - cls(reference, MessageNotification.summary( - window.outcomes.get((reference.seq, reference.message_id), ()))) - for reference in presentation.sent_references - ) + def for_sources( + cls, + sources: Iterable[tuple[MessageReference, Iterable[MessageNotification]]], + ) -> tuple[Self, ...]: + """One summary per sent source, aggregated by MessageNotification.summary.""" + return tuple(cls(reference, MessageNotification.summary(outcomes)) + for reference, outcomes in sources) class ChannelReadMark(ABC): diff --git a/src/agent_comms/private_registry_guard.py b/src/agent_comms/private_registry_guard.py index 045ab785da..b47d81325b 100644 --- a/src/agent_comms/private_registry_guard.py +++ b/src/agent_comms/private_registry_guard.py @@ -46,13 +46,14 @@ from .private_path import PrivateFileRole from .private_runtime_schema import PrivateCoordinatorSchema, PrivateCoordinatorTable from .thread_identity import ClaimedIdentity -from .notification_evidence import NotificationAssignmentRead +from .notification_evidence import NotificationAssignmentRead, NotificationSelection if TYPE_CHECKING: from .acp_extension import CursorAdvancedUpdate, CursorScope from .bus_publication import Delivered from .cursor_publication import CursorPublication + from .presentation import OutcomeViewer from .private_nk_entrypoint import RootLaunch from .registry_document import RegistrySnapshot from .typed_table import TableSchema @@ -162,7 +163,7 @@ def context_entered_sql(self, assignment: str) -> str: @abstractmethod def missing_notification_assignments( - self, sources: Sequence[Delivered], + self, selection: NotificationSelection, ) -> NotificationAssignmentRead: ... @abstractmethod @@ -202,9 +203,9 @@ def context_entered_sql(self, assignment: str) -> str: return "0" def missing_notification_assignments( - self, sources: Sequence[Delivered], + self, selection: NotificationSelection, ) -> NotificationAssignmentRead: - return NotificationAssignmentRead(tuple(sources), ()) + return NotificationAssignmentRead(selection, ()) def cursor_scope( self, snapshot: RegistrySnapshot, thread_name: str, session_id: str, @@ -283,7 +284,7 @@ def context_entered_sql(self, assignment: str) -> str: return f"EXISTS (SELECT 1 {NativeRuntimeInput.context_entered_from(members, assignment)})" def missing_notification_assignments( - self, sources: Sequence[Delivered], + self, selection: NotificationSelection, ) -> NotificationAssignmentRead: raise PublicationActivationBlocked("Private notification coordinator is missing") diff --git a/src/agent_comms/private_send_claim.py b/src/agent_comms/private_send_claim.py index 1e175de8c6..ec460a3ecc 100644 --- a/src/agent_comms/private_send_claim.py +++ b/src/agent_comms/private_send_claim.py @@ -30,7 +30,8 @@ from .native_arguments import SelectedModel from .native_input_owner import ParticipantOwner, RegistryOwner from .native_entries import NativeEvidenceScope -from .native_pi import NativeContextProof, NativePiTerminalFailure, NativeTurnResult +from .native_input_proof import NativeContextProof +from .native_pi import NativePiTerminalFailure, NativeTurnResult from .native_prompt_binding import bind_expected_prompt from .native_prompt_send import PromptBoundary from .native_session_reopen import NativeSessionIdentity @@ -228,8 +229,25 @@ def settle(store: Coordination, terminal: NativePiTerminalFailure = error): ) return result + async def context_committed( + self, context: NativeContextProof, source_reads: NativeEvidenceScope, + ) -> None: + """Publish corroborated entry while the model is still running. + + The context journal is borrowed before coordinator locking, as on the + terminal path. Neither the RPC event nor model activity is a receipt. + """ + def acknowledge(store: Coordination) -> None: + self.verify(store, context, source_reads) + self.stage.acknowledge_context( + store, self.participant, self.input_id, self.token_digest, context, + ) + await Coordination.run_async(self.store_path, acknowledge) + def commit(self, store: Coordination, context: NativeContextProof) -> None: - self.stage.commit(store, self.participant, self.input_id, self.token_digest, context) + self.stage.acknowledge_context( + store, self.participant, self.input_id, self.token_digest, context, + ) def _start_once(self, identity: NativeSessionIdentity) -> None: """Commit the exact grant under original fences, then release them. diff --git a/src/agent_comms/private_send_stage.py b/src/agent_comms/private_send_stage.py index aced2c1a2f..da48ea11fc 100644 --- a/src/agent_comms/private_send_stage.py +++ b/src/agent_comms/private_send_stage.py @@ -29,7 +29,7 @@ ) from .native_input_owner import ParticipantOwner from .native_input_record import (NativeInputIdentity, NativeInputExecution, TriageNativeExecution, FullNativeExecution) -from .native_pi import NativeContextProof +from .native_input_proof import NativeContextProof from .native_entries import NativeEvidenceScope from .native_prompt_binding import ( PromptBinding, @@ -132,9 +132,21 @@ def pending_input( ).require_valid() except ReservationViolationError as error: raise StaleFence(f"native dispatch identity changed: {error}") from error - row.require_unproven(token_digest) + row.require_dispatch(token_digest) return row + def require_acknowledgement(self, store: Coordination) -> None: + self.require_claim(store) + + def acknowledge_context( + self, store: Coordination, owner: ParticipantOwner, input_id: InputId, + token_digest: str, context: NativeContextProof, + ) -> None: + with store.session.writing() as db: + row = self.pending_input(store, input_id, owner, token_digest) + self.require_acknowledgement(store) + row.commit_context(db, context) + def verify( self, store: Coordination, @@ -311,7 +323,8 @@ def commit( with store.session.writing() as db: row = self.pending_input(store, input_id, owner, token_digest) self.require_claim(store) - row.commit_context(db, context, verdict=DecidedTriageRecord(type(decision))) + row.commit_context(db, context) + row.settle_verdict(db, DecidedTriageRecord(type(decision))) for captured in self.assignments: decision.settle(store, db, store.assignments.get(captured.assignment_id)) # This is the settlement transaction's witness, not a replacement @@ -381,21 +394,11 @@ def reserve_claim(self, store: Coordination, db: sqlite3.Connection) -> None: ): raise IdentityConflict("this execution attempt already has a native input") - def commit( - self, - store: Coordination, - owner: ParticipantOwner, - input_id: InputId, - token_digest: str, - context: NativeContextProof, - ) -> None: - with store.session.writing() as db: - row = self.pending_input(store, input_id, owner, token_digest) - snapshot, _ = store.require_fence(self.fence) - for assignment in self.assignments: - engagement = assignment.lifecycle.require_engagement() - snapshot.require_wire_response(engagement.exact_target) - row.commit_context(db, context) + def require_acknowledgement(self, store: Coordination) -> None: + snapshot, _ = store.require_fence(self.fence) + for assignment in self.assignments: + engagement = assignment.lifecycle.require_engagement() + snapshot.require_wire_response(engagement.exact_target) def require_phase(self, store: Coordination, current: WakeAssignment) -> None: snapshot, attempt = store.require_fence(self.fence) diff --git a/src/agent_comms/private_sidecar.py b/src/agent_comms/private_sidecar.py index ab3321c7b4..de282f7d8d 100644 --- a/src/agent_comms/private_sidecar.py +++ b/src/agent_comms/private_sidecar.py @@ -33,6 +33,7 @@ if TYPE_CHECKING: from .agent_backend import StreamingInputPolicy + from .image_input import ImageInput from .private_path import ObservedPath, PrivateFileRole, TrustedAncestorRole from .store_files import ExclusiveLock, StoreOwner @@ -505,13 +506,16 @@ def reconcile_at(cls, path: Path, practice: Practice = NoPractice()) -> bool: connection.close() -def encode_request(text: str, *, streaming_behavior: type[StreamingInputPolicy] | None = None) -> str: +def encode_request( + text: str, *, streaming_behavior: type[StreamingInputPolicy] | None = None, + images: tuple[ImageInput, ...] = (), +) -> str: """JSON-encode like the pinned native ``_claimNativeInput`` request.""" return FieldCodec.compact( { "kind": "prompt", "text": text, - "images": None, + "images": FieldCodec.encode(images) if images else None, "streamingBehavior": FieldCodec.encode(streaming_behavior), "expandPromptTemplates": True, "source": "rpc", @@ -519,8 +523,13 @@ def encode_request(text: str, *, streaming_behavior: type[StreamingInputPolicy] ) -def native_request_digest(text: str, *, streaming_behavior: type[StreamingInputPolicy] | None = None) -> str: +def native_request_digest( + text: str, *, streaming_behavior: type[StreamingInputPolicy] | None = None, + images: tuple[ImageInput, ...] = (), +) -> str: """sha256 over the pinned native request envelope, not the bare text.""" return hashlib.sha256( - ("pi-input-request-v1\n" + encode_request(text, streaming_behavior=streaming_behavior)).encode("utf-8") + ("pi-input-request-v1\n" + encode_request( + text, streaming_behavior=streaming_behavior, images=images, + )).encode("utf-8") ).hexdigest() diff --git a/src/agent_comms/repo.py b/src/agent_comms/repo.py index ca1d5e1c42..cd0215becd 100644 --- a/src/agent_comms/repo.py +++ b/src/agent_comms/repo.py @@ -216,8 +216,13 @@ def grep(cls, checkout: Path | str, *args: str) -> tuple[str, ...]: def of(cls, checkout: Path) -> Repo: """Its GitHub name comes from the checkout's own origin remote.""" url = cls.git(checkout, "remote", "get-url", "origin").strip() + return cls(checkout=checkout, github=cls.repository_url(url)) + + @staticmethod + def repository_url(url: str) -> RepositoryName: + """The same Git-origin identity for a checkout and a declared stack source.""" github = url.removesuffix(".git").replace(":", "/").split("/")[-2:] - return cls(checkout=checkout, github=RepositoryName.from_wire("/".join(github))) + return RepositoryName.from_wire("/".join(github)) class CheckoutHoldsWorkError(ValueError): diff --git a/src/agent_comms/runtime.py b/src/agent_comms/runtime.py index 89dfdde624..f42d7143e4 100644 --- a/src/agent_comms/runtime.py +++ b/src/agent_comms/runtime.py @@ -25,7 +25,7 @@ from contextvars import ContextVar from dataclasses import dataclass from pathlib import Path -from typing import TYPE_CHECKING, Any, ClassVar, Self +from typing import TYPE_CHECKING, Any, Self from pydantic import BaseModel @@ -423,16 +423,6 @@ def request_payload(self, action: str, parameters: dict[str, object]) -> dict[st ) -class OwnerReplyConnection(RuntimeConnection): - """Bound only the live owner's response after a request has been dispatched.""" - - REPLY_TIMEOUT: ClassVar[float] = 3 - - async def _read_response(self, reader: asyncio.StreamReader) -> bytes: - async with asyncio.timeout(self.REPLY_TIMEOUT): - return await super()._read_response(reader) - - @dataclass(frozen=True) class OwnerAttachment: """A completed subscription: its stream, the owner's ready line and presented metadata.""" diff --git a/src/agent_comms/runtime_requests.py b/src/agent_comms/runtime_requests.py index e27e56237e..20565f1eb2 100644 --- a/src/agent_comms/runtime_requests.py +++ b/src/agent_comms/runtime_requests.py @@ -23,8 +23,6 @@ from .acp_extension import ( GoalSnapshotReply, - TurnChangedUpdate, - UpdateMetadata, ) from .agent_event_updates import AcpEventConsumer @@ -41,6 +39,7 @@ from .goal_scope import AllGoals, OneGoal from .goals import Goal from .manual_compaction_bridge import compact_context +from .order_tools import OwnerGoalRead from .owner_connection import NoControllerToken, OwnerConnection, ResultRuntimeRequest, RuntimeRequest, RuntimeRequestContext from .owner_protocol import ClientLine, OwnerAnnouncement, OwnerLine, OwnerReady, OwnerResult from .thread_identity import ThreadIncarnation @@ -82,11 +81,6 @@ class GoalReply(DeclaredReply): goal: Goal -@dataclass(frozen=True) -class GoalHistoryReply(RecordedGoalHistory): - pass - - @dataclass(frozen=True) class AcpResponseReply(DeclaredReply, WireValue): """An ACP schema response, sent exactly as the schema writes it.""" @@ -271,19 +265,6 @@ async def result(self, ctx: RuntimeRequestContext) -> CompactionOutcome: return await compact_context(ctx.server.agent.turns, ctx.session_id, self.instructions) -@dataclass(frozen=True, kw_only=True) -class InputDispositionsRuntimeRequest(ResultRuntimeRequest): - include_history: bool = False - - async def result(self, ctx: RuntimeRequestContext) -> DeliveryOverview: - return await Coordination.run_worker(partial(ctx.server.agent._comms.goals.input_delivery, - ctx.name, - include_history=self.include_history, - scope=await ctx.server.agent.inputs.delivery_scope(ctx.session_id), - - )) - - @dataclass(frozen=True, kw_only=True) class DismissHistoricalInputsRuntimeRequest(ResultRuntimeRequest): async def result(self, ctx: RuntimeRequestContext) -> DeliveryOverview: @@ -298,10 +279,9 @@ async def result(self, ctx: RuntimeRequestContext) -> DeliveryOverview: class GoalHistoryRuntimeRequest(ResultRuntimeRequest): goal_id: OneGoal | AllGoals = AllGoals() - async def result(self, ctx: RuntimeRequestContext) -> GoalHistoryReply: + async def result(self, ctx: RuntimeRequestContext) -> RecordedGoalHistory: return await Coordination.run_worker(partial( - ctx.server.agent._comms.goals.recorded_history, - GoalHistoryReply, ctx.name, self.goal_id, + ctx.server.agent._comms.goals.recorded_history, ctx.name, self.goal_id, )) @@ -311,19 +291,7 @@ async def change(self, ctx: RuntimeRequestContext) -> None: ... async def result(self, ctx: RuntimeRequestContext) -> GoalSnapshotReply: await self.change(ctx) - snapshot = await Coordination.run_worker(partial( - ctx.server.agent._comms.goals.goal_snapshot, ctx.name, - )) - turn = await Coordination.run_worker(partial( - ctx.server.agent.turns.turn_state, ctx.session_id, - )) - return GoalSnapshotReply.from_snapshot(snapshot, UpdateMetadata.of(TurnChangedUpdate(turn))) - - -@dataclass(frozen=True, kw_only=True) -class GoalSnapshotRuntimeRequest(GoalSnapshotResultRuntimeRequest): - async def change(self, ctx: RuntimeRequestContext) -> None: - pass + return await OwnerGoalRead().owned(ctx) @dataclass(frozen=True, kw_only=True) diff --git a/src/agent_comms/selected_participant.py b/src/agent_comms/selected_participant.py index cdf17be4a9..703384121f 100644 --- a/src/agent_comms/selected_participant.py +++ b/src/agent_comms/selected_participant.py @@ -41,7 +41,6 @@ from .presentation import MessageNotification, OwnerOutcomes, SentMessageOutcome from .publisher import ParticipantPublisher from .selected_source_batch import SelectedSource, SelectedSourceBatch -from .threads import Thread from .turn_phase import PreparingPhase, TurnPhase from .turn_goal_permission import OwnerGoalPermission, TurnGoalPermission @@ -203,13 +202,11 @@ async def select( @staticmethod def capture_sent_outcomes( - comms: Comms, owner: Thread, sources: Sequence[Delivered], - assignment_read: NotificationAssignmentRead, + comms: Comms, assignment_read: NotificationAssignmentRead, ) -> tuple[SentMessageOutcome, ...]: """Project optional prompt context from the drain's captured source and SQL cuts.""" notifications = MessageNotification.for_sources( - comms.root, comms.registry, OwnerOutcomes.of(owner), sources, - assignment_read=assignment_read, + comms.root, comms.registry, assignment_read, ) return tuple( outcome @@ -253,12 +250,13 @@ def prepare( store, lookup, after_seq=after_seq ) sent_read = NotificationAssignment.capture( - store.session._connection, sent_sources, privacy, + store.session._connection, OwnerOutcomes.of(sent_owner).selection(sent_sources), privacy, ) + sent_read = NotificationAssignment.capture_responses(comms.root, sent_read) if pending: participant.pointer.require_idle() sent_outcomes = SelectedParticipant.capture_sent_outcomes( - comms, sent_owner, sent_sources, sent_read + comms, sent_read ) return owner, participant, pending, sent_outcomes diff --git a/src/agent_comms/selected_triage.py b/src/agent_comms/selected_triage.py index 345cf88922..b0a755859a 100644 --- a/src/agent_comms/selected_triage.py +++ b/src/agent_comms/selected_triage.py @@ -31,7 +31,7 @@ from .coordinated_runtime import SelectedExecution from .historical_native_inputs import HistoricalNativeInput, TriageHistoricalNativeInput from .native_input_record import TriageNativeExecution - from .native_pi import NativeContextProof + from .native_input_proof import NativeContextProof from .native_runtime_input import NativeRuntimeInput from .private_send_claim import PrivateSendClaim from .private_send_stage import TriageNativeSend diff --git a/src/agent_comms/task_attachments/commander_pings.py b/src/agent_comms/task_attachments/commander_pings.py index e3a35ec684..c9838e7f45 100644 --- a/src/agent_comms/task_attachments/commander_pings.py +++ b/src/agent_comms/task_attachments/commander_pings.py @@ -17,7 +17,7 @@ from agent_comms.channel_targets import ThreadTarget from agent_comms.declared_family import DeclaredFamily -from agent_comms.harness_delivery_states import DeliveredOutcome, Outcome +from agent_comms.harness_delivery_states import FilteredOutcome, Outcome from agent_comms.commander_service import CommanderService from agent_comms.errors import RelationViolationError from agent_comms.field_codec import FieldCodec, OrderId @@ -326,7 +326,7 @@ class CommanderNotice(CommanderUpdate): async def wake(self, bridge: ClaudeBridge, message: Message) -> type[Outcome]: """Cleared lines leave the board; nothing new needs the session.""" - return DeliveredOutcome + return FilteredOutcome def lines(self, now: float) -> tuple[str, ...]: return (f"Cleared · {len(self.handled)} requests remain on the board.", *self.not_applied) diff --git a/src/agent_comms/thread_presentation.py b/src/agent_comms/thread_presentation.py index 975703cae8..b6c249ed0a 100644 --- a/src/agent_comms/thread_presentation.py +++ b/src/agent_comms/thread_presentation.py @@ -4,7 +4,7 @@ from collections.abc import Callable from dataclasses import dataclass, field, fields, replace -from typing import TYPE_CHECKING, ClassVar, Self +from typing import TYPE_CHECKING, ClassVar, NoReturn, Self from agent_comms.coordination_errors import StaleRevision from agent_comms.errors import RelationViolationError, RouteChangedError, UnregisteredThreadError @@ -12,7 +12,7 @@ from .declared_family import DeclaredFamily from .thread_identity import OwnerIdentity from .child_process import ProcessIdentity -from .notification_evidence import NoNotificationWindow, NotificationWindow +from .notification_evidence import NotificationWindow if TYPE_CHECKING: from .message_reference import MessageReference @@ -209,13 +209,20 @@ def same_status(self, other: ThreadStatusDisplay) -> bool: @dataclass(frozen=True, slots=True) class ThreadPresentation(ThreadStatusDisplay): - """A thread's acquired presentation, or its declared absence.""" + """What a viewer shows for a thread, or its declared absence: no acquired evidence.""" notifications: tuple[MessageNotification, ...] = () read_identity: TranscriptReadIdentity | None = None - incoming_references: tuple[MessageReference, ...] = () - sent_references: tuple[MessageReference, ...] = () - notification_window: NotificationWindow = field(default_factory=NoNotificationWindow) + + def published(self) -> ThreadPresentation: + """The value a viewer may receive: this presentation itself.""" + return self + + def sent_notifications( + self, + ) -> tuple[tuple[MessageReference, tuple[MessageNotification, ...]], ...]: + """A presentation without acquired sources holds no sent message.""" + return () def require(self, name: str) -> ThreadPresentation: """Require a presentation for a caller already bound to this thread.""" @@ -230,8 +237,9 @@ def with_paint(self, shown: ThreadPresentation) -> Self: return replace(self, title=shown.title, marker=shown.marker, summary=shown.summary, status=shown.status) - def collect_revision_input(self, name: str, readable: dict[str, ThreadPresentation]) -> None: - readable[name] = self + def collect_revision_input(self, name: str, + readable: dict[str, AcquiredThreadPresentation]) -> None: + """Only acquired evidence can say whether a later revision changed it.""" def read_sources(self, transcripts: Transcripts, snapshot: RegistrySnapshot, name: str, sent_source_limit: int) -> ThreadPresentation: @@ -257,36 +265,72 @@ def same_observation(self, other: ThreadPresentation) -> bool: return original.thread == current.thread and original.same_snapshot(current) def with_sources(self, window: TranscriptSourceWindow) -> ThreadPresentation: - """Incoming and sent outcomes are views of one captured notification window.""" + """Incoming and sent outcomes are views of one captured notification window, read + as the thread's owner may see them; the incoming outcomes carry public envelopes.""" from .notification_assignment import NotificationAssignment from .presentation import MessageNotification, OwnerOutcomes transcripts = window.read.owner sources = tuple({source.message.reference: source for source in (*window.sources, *window.sent_sources)}.values()) - notification_read = NotificationAssignment.read(transcripts.root, sources) + viewer = OwnerOutcomes.of(window.read.identity.thread) + notification_read = NotificationAssignment.read(transcripts.root, viewer.selection(sources)) projected = MessageNotification.delivery_window( - transcripts.root, transcripts.registry, sources, notification_read, - viewer=OwnerOutcomes.of(window.read.identity.thread), + transcripts.root, transcripts.registry, notification_read, ) incoming = tuple( - replace(notification, message=source.message) + replace(notification, message=source.envelope) for source in window.sources for notification in projected.outcomes[source.message.seq, source.message.message_id] ) - return replace( - self, notifications=incoming, + shown = replace(self, notifications=incoming, read_identity=window.read.identity) + return AcquiredThreadPresentation( + **{declared.name: getattr(shown, declared.name) + for declared in fields(ThreadPresentation)}, notification_window=projected, - read_identity=window.read.identity, incoming_references=tuple(source.message.reference for source in window.sources), - sent_references=tuple(source.message.reference for source in window.sent_sources), - ) + sent_references=tuple(source.message.reference for source in window.sent_sources)) @property def label(self) -> str: return f"{self.marker} {self.title}".strip() +@dataclass(frozen=True, slots=True) +class AcquiredThreadPresentation(ThreadPresentation): + """A presentation together with the notification evidence it was acquired from. + + The evidence stays with the reader that acquired it, for its change checks and sent + summaries; a viewer receives only the published ThreadPresentation.""" + + incoming_references: tuple[MessageReference, ...] = field(kw_only=True) + sent_references: tuple[MessageReference, ...] = field(kw_only=True) + notification_window: NotificationWindow = field(kw_only=True) + + def published(self) -> ThreadPresentation: + return ThreadPresentation(**{declared.name: getattr(self, declared.name) + for declared in fields(ThreadPresentation)}) + + def sent_notifications( + self, + ) -> tuple[tuple[MessageReference, tuple[MessageNotification, ...]], ...]: + """Each recent sent source with its outcomes from this presentation's captured window.""" + return tuple( + (reference, + self.notification_window.outcomes.get((reference.seq, reference.message_id), ())) + for reference in self.sent_references + ) + + def collect_revision_input(self, name: str, + readable: dict[str, AcquiredThreadPresentation]) -> None: + readable[name] = self + + def __reduce_ex__(self, protocol: object) -> NoReturn: + raise TypeError( + "AcquiredThreadPresentation keeps private delivery evidence with its reader; " + "send its published() ThreadPresentation") + + @dataclass(frozen=True, slots=True) class NoThreadPresentation(ThreadPresentation): """No acquired thread: no sources, notifications or readable revision.""" @@ -301,13 +345,9 @@ def require(self, name: str) -> ThreadPresentation: def for_status(self, name: str) -> ThreadPresentation: return self.with_paint(ThreadRetired().presentation(name)) - def collect_revision_input(self, name: str, readable: dict[str, ThreadPresentation]) -> None: - pass - def read_sources(self, transcripts: Transcripts, snapshot: RegistrySnapshot, name: str, sent_source_limit: int) -> ThreadPresentation: return self def with_sources(self, window: TranscriptSourceWindow) -> ThreadPresentation: return self - diff --git a/src/agent_comms/todos.py b/src/agent_comms/todos.py index a9eae21bab..15df9ce786 100644 --- a/src/agent_comms/todos.py +++ b/src/agent_comms/todos.py @@ -100,7 +100,10 @@ HeadReviewOrder, HeadReviewRelation, HeadReviewStatus, + IntegrationInput, MergeArm, + MergeHeadReview, + UnpromotedHeadReview, OpenPullRequestHead, PreparedHeadReview, PullRequestHead, @@ -989,13 +992,13 @@ def __init__(self, todos: Sequence[Todo] = ()) -> None: self._read: dict[tuple[RepositoryName, int], ListedPullRequest] = {} self._open: dict[RepositoryName, tuple[int, ...]] = {} self._worktrees: dict[Path, Mapping[str, tuple[str, ...]]] = {} - self._todos: dict[RepositoryName, dict[str, Todo]] = {} + self._todos: dict[str, Todo] = {} self.register(todos) def register(self, todos: Sequence[Todo]) -> Self: """Register a frame's orders while preserving this read's existing GitHub facts.""" for todo in todos: - self._todos.setdefault(todo.repo, {})[todo.id] = todo + self._todos[todo.id] = todo return self def retained_assignments(self, todo: Todo, assignment: Assignment) -> tuple[Assignment, ...]: @@ -1054,7 +1057,7 @@ def claimed_heads(self, todo: Todo) -> Iterator[tuple[OrderClaim, PullRequest]]: yield claim, request def linked(self, repo: RepositoryName) -> dict[int, list[Todo]]: - return self.linked_among(self._todos.get(repo, {}).values(), repo) + return self.linked_among(self._todos.values(), repo) @staticmethod def linked_among(todos: Iterable[Todo], repo: RepositoryName) -> dict[int, list[Todo]]: @@ -2484,7 +2487,7 @@ def claimed(self, claim: OrderClaim) -> ClaimedProblemOrder: raise TodoError("Only a pending order can be claimed by its holder.") def review_verdicts(self) -> tuple[type[HeadVerdict], ...]: - return (ReviewedHeadVerdict, RejectedHeadVerdict) + return (ReviewedHeadVerdict, QualifiedHeadVerdict, RejectedHeadVerdict) def sent_back(self, reason: StrikeReason) -> CheckedProblemOrder: return self @@ -4082,10 +4085,16 @@ class HeadVerdict(HeadReview, declared_name=None): @classmethod def require_merge(cls, request: PullRequestHead) -> None: - """Only a verdict that allows integration lets the owner's admin bypass merge.""" + """Source review cannot authorize even the owner's direct merge path.""" if not cls.allows_integration(): raise TodoError( f"{request.repo.to_wire()}#{request.pr} at {request.head} has no recorded ok verdict.") + MergeHeadReview.require_merge(request) + + def post(self, repo: RepositoryName, head: str, reviewer: str, subject: str) -> None: + # Withdraw first: a failed source-status write cannot leave an earlier promotion live. + UnpromotedHeadReview().post(repo, head, reviewer, subject) + super().post(repo, head, reviewer, subject) def reviewed(self, goals: Goals, request: PullRequestHead, prepared: PreparedHeadReview, reviewer: ThreadIncarnation, order: HeadReviewOrder) -> HeadReviewed: @@ -4096,7 +4105,7 @@ def reviewed(self, goals: Goals, request: PullRequestHead, prepared: PreparedHea return HeadReviewed(recorded, self.landing(prepared.current)) def landing(self, current: OpenPullRequestHead) -> MergeArm: - """Only an acceptance moves the PR toward main; every other verdict leaves auto-merge be.""" + """Source verdicts never request merge; native explicit promotion owns enqueueing.""" return UntouchedMergeArm() def subject(self, goals: Goals, request: ReadPullRequestHead) -> str: @@ -4150,8 +4159,18 @@ def description(self, reviewer: str, subject: str) -> str: return f"{self.verb} by {reviewer} on {subject}: {self.reason.text}" +@dataclass(frozen=True) +class QualifiedHeadVerdict(DecidedHeadVerdict): + """A reviewed source stage, without permission to promote the whole change.""" + + state = "pending" + verb = "Qualified" + + @dataclass(frozen=True) class ReviewedHeadVerdict(DecidedHeadVerdict): + """Accept an exact source for combination, never enqueue its individual PR.""" + state = "success" verb = "Accepted" preparation_field = GrowthReason.option @@ -4178,9 +4197,9 @@ def carry(self, goals: Goals, record: HeadReviewRecord, current: ReadPullRequest def allows_integration(cls) -> bool: return True - def landing(self, current: OpenPullRequestHead) -> MergeArm: - """Accepting the current head means landing it: arm it, pinned to that head.""" - return current.arm() + @classmethod + def grown(cls, reason: VerdictReason, growth: GrowthReason) -> ReviewedHeadVerdict: + return GrownHeadVerdict(reason, growth) def subject(self, goals: Goals, request: ReadPullRequestHead) -> str: project = goals.registry.snapshot().commander_project.require() @@ -4206,8 +4225,8 @@ def description(self, reviewer: str, subject: str) -> str: @dataclass(frozen=True) -class GrownHeadVerdict(ReviewedHeadVerdict): - """Acceptance of a head whose source growth the reviewer explains file by file.""" +class ExplainedGrowth: + """Source review and combined promotion share the same explicit growth contract.""" growth: GrowthReason @@ -4218,6 +4237,11 @@ def require_growth(self, growth: SourceGrowth) -> None: f"({growth.listing()}).") +@dataclass(frozen=True) +class GrownHeadVerdict(ExplainedGrowth, ReviewedHeadVerdict): + """Acceptance of a source whose growth the reviewer explains file by file.""" + + @dataclass(frozen=True) class RejectedHeadVerdict(DecidedHeadVerdict): reason: StrikeReason @@ -4996,13 +5020,19 @@ class RecordPullRequest(TodoCommand): link: PullRequestLink generation: GoalId + def validate(self, db: sqlite3.Connection, current: Todo, store: TodoStore) -> None: + if self.link.repo != current.repo: + # TodoStore.writing retains this registry cut through the actual SQL write. + project = RegistryStore.at(store.root).snapshot().commander_project.require() + Repo.selected(project.repos, self.link.repo.to_wire()) + def changed(self, current: Todo) -> Todo: current.require_actor(self.actor, self.generation) - if self.link.repo != current.repo: - raise TodoError("A recorded pull request must belong to the todo's repository.") order = current.order.checked() if self.link in order.pull_requests: return current + order.require_linkable() + current.state.require_open(current) return replace(current, revision=current.revision + 1, order=order.linked_to(self.link)) @@ -6048,6 +6078,16 @@ def for_goal(self, owner: ThreadIncarnation, goal_id: GoalId) -> tuple[Todo, ... return self._select(Assignment.sql_field('generation') + '=?', (goal_id,), reader=partial(self.held_in, unit=owner)) + def require_input_links(self, inputs: IntegrationInput) -> None: + """Only current stored orders, never metadata supplied by an author, link these sources.""" + todos = self.open_orders() + for source in inputs.sources: + linked = PullRequests.linked_among(todos, source.repository) + for head in source.heads: + if len(linked.get(head.pr, ())) != 1: + raise TodoError( + f"{head.repo.to_wire()}#{head.pr} has no unique actual Todo source authority") + def open_orders(self) -> tuple[Todo, ...]: """Every open order, whichever commander created it: their links say which requests any commander's board already owns.""" diff --git a/src/agent_comms/tool_caller.py b/src/agent_comms/tool_caller.py index f212f06647..e39f0a4de6 100644 --- a/src/agent_comms/tool_caller.py +++ b/src/agent_comms/tool_caller.py @@ -1,17 +1,18 @@ """The caller bound at tool entry owns caller-relative tool behavior.""" from __future__ import annotations -from abc import ABC, abstractmethod +from abc import abstractmethod from dataclasses import dataclass -from typing import TYPE_CHECKING, Annotated, NoReturn +from typing import TYPE_CHECKING, Annotated from .bus_publication import AgentOrigin from .cli_result import ThreadToolCatalogResult, ToolCatalogResult from .errors import ToolRefusedError from .field_codec import WireForm from .launch_root import LaunchRoot +from .bus_read import BusReader, OperatorOutcomes from .launch_runtime import CallingThreadUnavailable -from .presentation import AllOutcomes, OutcomeViewer, OwnerOutcomes, SentMessageSummary, ThreadRecord +from .presentation import OutcomeViewer, OwnerOutcomes, SentMessageSummary, ThreadRecord from .thread_execution import CommanderForkExecution from .thread_identity import ThreadIncarnation from .tool_result import ToolResult @@ -30,7 +31,7 @@ class ThreadListing(ToolResult): sent_messages: tuple[SentMessageSummary, ...] -class ToolCaller(ABC): +class ToolCaller(BusReader): def acting_caller(self, grant: ToolCaller) -> ToolCaller: return self @@ -48,10 +49,6 @@ def listing(self, comms: Comms, threads: tuple[ThreadRecord, ...]) -> ThreadList @abstractmethod def catalog(self, comms: Comms, tools: tuple[dict[str, object], ...]) -> ToolCatalogResult: ... - @abstractmethod - def outcome_viewer(self, comms: Comms) -> OutcomeViewer: - """Whose recipient outcomes this caller may read.""" - @abstractmethod def acting_name(self, default: str) -> str: ... @@ -59,13 +56,6 @@ def acting_name(self, default: str) -> str: ... def from_flag(self) -> tuple[str, ...]: """The command line's --from argument naming this caller, if it names a thread.""" - @abstractmethod - def own_thread(self) -> str: - """The thread a tool acts on when its call names none.""" - - def subject(self, named: str) -> str: - return named or self.own_thread() - def commanding(self, comms: Comms, commander: str) -> None: """An ordinary caller needs no commander-child registration.""" return None @@ -94,15 +84,6 @@ def origin(self) -> AgentOrigin: return AgentOrigin() -class OperatorOutcomes: - """A caller speaking for the owner reads every recipient outcome, as the owner's UI does.""" - - __slots__ = () - - def outcome_viewer(self, comms: Comms) -> OutcomeViewer: - return AllOutcomes() - - @dataclass(frozen=True) class NoThreadCaller(OperatorOutcomes, ToolCaller): def acting_caller(self, grant: ToolCaller) -> ToolCaller: @@ -120,18 +101,12 @@ def listing(self, comms: Comms, threads: tuple[ThreadRecord, ...]) -> ThreadList def catalog(self, comms: Comms, tools: tuple[dict[str, object], ...]) -> ToolCatalogResult: return ToolCatalogResult(tools) - def refuse_thread(self) -> NoReturn: - raise CallingThreadUnavailable("This tool requires a thread caller.") - def acting_name(self, default: str) -> str: return default def from_flag(self) -> tuple[str, ...]: return () - def own_thread(self) -> NoReturn: - return self.refuse_thread() - def require_directive(self, comms: Comms, body: str) -> None: """A caller with no thread gives no directive.""" @@ -160,7 +135,7 @@ def invoke(self, request: ToolRequest[ThreadCaller], comms: Comms) -> ToolResult def listing(self, comms: Comms, threads: tuple[ThreadRecord, ...]) -> ThreadListing: presentation = comms.views.thread_presentations((self.name,)).require(self.name) - return ThreadListing(threads, SentMessageSummary.recent(presentation)) + return ThreadListing(threads, SentMessageSummary.for_sources(presentation.sent_notifications())) def outcome_viewer(self, comms: Comms) -> OutcomeViewer: return OwnerOutcomes.of(comms.registry.require(self.name)) diff --git a/src/agent_comms/tools.py b/src/agent_comms/tools.py index 9e27aac205..b50d8128ac 100644 --- a/src/agent_comms/tools.py +++ b/src/agent_comms/tools.py @@ -58,6 +58,7 @@ from .pi_payloads import PublishedGoalToolDetails from .pi_vocabulary import ServiceTier, ThinkingLevelField from .presentation import AfterRecipient, ChannelView, FirstRecipient, MessageNotification, RecipientCursor +from .notification_evidence import RecipientSelection from .progress_claims import ProgressClaim from .read_basis import NamedViewer from .relationships import ( @@ -87,8 +88,8 @@ from .thread_model import ThreadModel from .threads import NextTurnConfiguration from .tool_caller import ThreadCaller, ThreadListing, ToolCaller -from .ui_model.observation import ReadNotifications from .tool_result import ToolResult +from .notification_reads import LiveNotificationRead from .work_branches import WorkBranch, WorkBranches if TYPE_CHECKING: @@ -163,21 +164,6 @@ class GoalUpdated(OwnGoalPublication): goal: GoalDeclaration -@dataclass(frozen=True) -class GoalHistory(ToolResult, RecordedGoalHistory): - pass - - - - - - - - - - - - @dataclass(frozen=True) class WorkBranchList(ToolResult): branches: tuple[WorkBranch, ...] @@ -622,8 +608,8 @@ class CommsGoalHistoryTool(ThreadReadTool): OneGoal | AllGoals, *tool_field("Optional goal ID; omit for this thread's full history") ] = AllGoals() - def apply(self, comms: Comms, caller: ToolCaller) -> GoalHistory: - return comms.goals.recorded_history(GoalHistory, caller.subject(self.thread), self.goal_id) + def apply(self, comms: Comms, caller: ToolCaller) -> RecordedGoalHistory: + return comms.goals.recorded_history(caller.subject(self.thread), self.goal_id) @@ -707,24 +693,23 @@ class CommsMessageOutcomesTool(ReadTool, ToolRequest): reference: Annotated[ MessageReference, *tool_field("The message's seq and message_id, as comms_threads lists them") ] - limit: Annotated[int, *tool_field("Maximum recipient rows in this page")] = 100 + limit: Annotated[int, *tool_field("Maximum recipient rows in this page")] = RecipientSelection.max_limit after: Annotated[ RecipientCursor, *tool_field("The previous page's next cursor; omit for the first page") ] = FirstRecipient() def apply(self, comms: Comms, caller: ToolCaller) -> MessageOutcomes: - if self.limit < 1: - raise ToolRefusedError("limit must be positive.") + try: + RecipientSelection.require_limit(self.limit) + except ValueError as error: + raise ToolRefusedError(str(error)) from error source, = comms.bus.bus.deliveries_for_references((self.reference,)) - viewer = caller.outcome_viewer(comms) - if not viewer.sees(source): + if not caller.outcome_viewer(comms).sees(source): raise ToolRefusedError(f"Message {source.message.seq} is not readable by this caller.") - window = ReadNotifications(str(comms.root), (self.reference,), viewer=viewer).read(comms) - following = self.after.following( - window.outcomes.get((source.message.seq, source.message.message_id), ())) - page = following[:self.limit] + window = LiveNotificationRead((self.reference,), after=self.after, limit=self.limit).read_as(comms, caller) + page = window.outcomes.get((source.message.seq, source.message.message_id), ()) return MessageOutcomes( - source.message, page, len(following) - len(page), + source.message, page, window.remaining(self.reference), AfterRecipient(page[-1].recipient_identity) if page else self.after, ) diff --git a/src/agent_comms/tracked_turn.py b/src/agent_comms/tracked_turn.py index a0a117a2cd..96dc02f3a0 100644 --- a/src/agent_comms/tracked_turn.py +++ b/src/agent_comms/tracked_turn.py @@ -11,9 +11,7 @@ from functools import partial from pathlib import Path from types import MappingProxyType -from typing import TYPE_CHECKING, Generic, TypeVar - -from metaclass_registry.mro_dispatch import handles +from typing import TYPE_CHECKING from . import pi_commands as commands from . import pi_events as pi @@ -29,12 +27,11 @@ from .native_attestation import AttestationError, ObservedAttestation from .native_failures import NativePiUnavailable from .native_package import VerifiedNativePackage +from .native_input_proof import NativeContextProof from .native_pi import ( - NativeContextProof, NativeEmptyAnswer, NativePiPromptRejected, NativeTurnResult, - _verify_context, ) from .native_prompt_send import UNGUARDED_PROMPT, PromptBoundary, PromptSendFailure from .native_session_reopen import SessionRetention @@ -54,6 +51,7 @@ from .coordinated_runtime import NativeExecution from .cursor_publication import CursorDelivery from .owned_send import OwnedSend + from .native_entries import NativeEvidenceScope from .pi_native_backend import PersistentPiSession from .pi_payloads import UserMessage from .turn_failure import TurnFailure @@ -70,35 +68,6 @@ async def __call__(self, event: pi.PiEvent | AgentEvent | ObservedAttestation) - NO_NATIVE_OBSERVER = NoObserver() NO_FOLLOWING_INPUTS = FollowingInputs() -T = TypeVar("T") - - -class NativeCommitObservation(ABC, Generic[T]): - """Observed native input records, without permission to start or replay input.""" - - @abstractmethod - def require(self) -> T: ... - - def capture(self, event: T) -> NativeCommitObservation[T]: - raise NativePiUnavailable("Native Pi repeated the input commitment") - - -class PendingNativeCommit(NativeCommitObservation[T]): - def require(self) -> T: - raise NativePiUnavailable("Native Pi did not commit a tracked model context") - - def capture(self, event: T) -> NativeCommitObservation[T]: - return ObservedNativeCommit(event) - - -@dataclass(frozen=True) -class ObservedNativeCommit(NativeCommitObservation[T]): - event: T - - def require(self) -> T: - return self.event - - class TrackedTerminal(ABC): """Relate completed assistant responses to their streamed text.""" @@ -251,20 +220,18 @@ def __init__( launch, command, tool_mode=selected_tool_mode, + cursor_delivery=cursor_delivery, model_wait=model_wait, startup=startup, request_observer=request_observer, native_session=pi_session, ) self.retainable = False - self.cursor_delivery = cursor_delivery self.following = following self.model = model self.prompt_send_boundary = prompt_send_boundary self.ingress = ingress self.observe_event = observe_event - self.input_commit: NativeCommitObservation[pi.InputCommitted] = PendingNativeCommit() - self.context_commit: NativeCommitObservation[pi.ContextCommitted] = PendingNativeCommit() self.draft_written_at = 0.0 self.terminal: TrackedTerminal = PendingTrackedTerminal() @@ -464,16 +431,14 @@ async def send_prompt(self) -> None: def reject_prompt(self, response): raise NativePiPromptRejected(response) - @handles(pi.InputCommitted) - async def committed_input(self, event: pi.InputCommitted) -> None: - if event.input_id == self.command.input_id: - self.input_commit = self.input_commit.capture(event) - self.prompt_state = self.prompt_state.start(event) - async with self.cursor_delivery.reading() as source_reads: - await Coordination.run_worker(lambda: - source_reads.for_source(self.active_session_file).observe(event) - ) - await self.following.open(self) + async def original_committed(self, event: pi.InputCommitted) -> None: + self.prompt_state = self.prompt_state.start(event) + await self.following.open(self) + + async def original_context_committed( + self, context: NativeContextProof, source_reads: NativeEvidenceScope, + ) -> None: + await self.prompt_send_boundary.context_committed(context, source_reads) def forward_inbox(self, inbox: TurnInbox, sender: OwnedSend) -> None: """Later inputs go to the running child under the owner's write checks.""" @@ -495,11 +460,6 @@ async def accept_user_message( return () raise NativePiUnavailable("Native Pi started a user message this turn did not send") - @handles(pi.ContextCommitted) - async def committed_context(self, event: pi.ContextCommitted) -> None: - if event.input_id == self.command.input_id: - self.context_commit = ObservedNativeCommit(event) - async def assistant_started(self) -> None: await super().assistant_started() self.draft_written_at = 0.0 @@ -534,17 +494,7 @@ def accept_final_message(self, message): async def context_proof(self) -> NativeContextProof: if not self.prompt_state.acknowledged: raise NativePiUnavailable("Native Pi did not commit a tracked model context") - # The recipient serializes this read with cursor and settlement reads. - # Cancellation joins the worker before its borrowed scope is released. - async with self.cursor_delivery.reading() as source_reads: - return await Coordination.run_worker(lambda: _verify_context( - self.launch.session.files(self.launch.root), - self.command.input_id, - self.native.attestation.require_identity().session_id, - self.input_commit.require(), - self.context_commit.require(), - evidence=source_reads.for_source(self.active_session_file), - )) + return await super().context_proof() async def invalidate_identity(self) -> tuple[AgentEvent, ...]: # A tracked turn has no authority to recover a session another command replaced. diff --git a/src/agent_comms/transcript_updates.py b/src/agent_comms/transcript_updates.py index de435ec6a1..cf65b6bf13 100644 --- a/src/agent_comms/transcript_updates.py +++ b/src/agent_comms/transcript_updates.py @@ -18,6 +18,7 @@ from .coordinator import Coordination from .declared_family import DeclaredFamily from .turn_lease import TurnState +from .order_tools import TranscriptPageRead from .ui_model.observation import ObservationProcess, ReadTranscript, SubscribeTranscript if TYPE_CHECKING: @@ -75,13 +76,13 @@ async def observing(self) -> AsyncIterator[ObservationProcess]: async def replay(self, session_id: str, client: SessionUpdateSink) -> None: async with self.observing() as reads: - snapshot = await reads.read(ReadTranscript(str(self.comms.root), session_id)) + snapshot = await reads.read(ReadTranscript(str(self.comms.root), TranscriptPageRead(thread=session_id))) await self._publish(session_id, snapshot, client) @asynccontextmanager async def subscribe(self, session_id: str, client: SessionUpdateSink) -> AsyncIterator[None]: async with self.observing() as reads: - snapshot = await reads.read(SubscribeTranscript(str(self.comms.root), session_id)) + snapshot = await reads.read(SubscribeTranscript(str(self.comms.root), TranscriptPageRead(thread=session_id))) await self._publish(session_id, snapshot, client) yield diff --git a/src/agent_comms/turn_input_source.py b/src/agent_comms/turn_input_source.py index 2b66f8643f..90add0e6f3 100644 --- a/src/agent_comms/turn_input_source.py +++ b/src/agent_comms/turn_input_source.py @@ -14,6 +14,7 @@ from .acp_extension import InputStartedUpdate, NoQueueScope, UnidentifiedInputStarted from .agent_backend import BusyInputPolicy, SteerInputPolicy from .channel_input_batch import InputBatch, SingleInputBatch +from .errors import RelationViolationError from .field_codec import InputId, TurnId from .input_attempt import InputAlreadyCleared, MissingInput from .input_key import InputKey @@ -76,6 +77,11 @@ async def accepted_start( ) -> InputStartedUpdate: return update + def require_following_source( + self, inputs: InputDrain, session_id: str, source: TurnInputSource, + ) -> None: + raise RelationViolationError("An input without an acceptance cannot own following context") + @abstractmethod def release(self, inputs: InputDrain, session_id: str) -> None: ... @@ -124,6 +130,12 @@ async def accepted_start( update = replace(update, text=await queued[self.input_id].started(inputs)) return update.identified(self.input_id) + def require_following_source( + self, inputs: InputDrain, session_id: str, source: TurnInputSource, + ) -> None: + if inputs.state(session_id).following.get(self.input_id) is not source: + raise RelationViolationError("Following context outlived its actual accepted input") + def release(self, inputs: InputDrain, session_id: str) -> None: inputs.state(session_id).queued.pop(self.input_id, None) @@ -199,6 +211,9 @@ def _prepare( ) return GrantedSend() + def require_context_source(self, sender: OwnedSend) -> None: + """An original's lease and durable binding fence its retained context.""" + def valid_keys(self, text: str) -> bool: return len(self.keys) <= 1 @@ -369,6 +384,9 @@ class FollowingTurnInput(TurnInputSource): when_busy: ClassVar[type[BusyInputPolicy]] = SteerInputPolicy owner_check = RunningOrdinaryOwnerCheck + def require_context_source(self, sender: OwnedSend) -> None: + self.acceptance.require_following_source(sender.inputs, sender.session_id, self) + def continue_response(self, response, parts): """An actual following request may revise the turn's automatic answer.""" return response.later_input(parts) diff --git a/src/agent_comms/turn_inputs.py b/src/agent_comms/turn_inputs.py index 4a3af2e7e8..40450b10c3 100644 --- a/src/agent_comms/turn_inputs.py +++ b/src/agent_comms/turn_inputs.py @@ -244,6 +244,7 @@ def mark_started( if not session.input_start.following(public_id, native_id, entry.content.text): return () self.retire(native_id) + session.following_started(command) self.started = True return (public_id,) return () diff --git a/src/agent_comms/typed_table.py b/src/agent_comms/typed_table.py index bf78b95c9d..4d30af53b0 100644 --- a/src/agent_comms/typed_table.py +++ b/src/agent_comms/typed_table.py @@ -298,6 +298,16 @@ def reference_sql(self) -> str: ) +@dataclass(frozen=True, slots=True) +class RequestedKeys: + """One batch of distinct declared keys: the ``requested`` VALUES table they bind, and + the null-safe join that drives the table's index from it.""" + + values: str + join: str + bindings: tuple[object, ...] + + @dataclass(frozen=True) class Column(FieldDirective): """Constraints on a field; its name and type remain on the dataclass field.""" @@ -2299,9 +2309,20 @@ def select_keys(cls, db: sqlite3.Connection, """Rows equal to any of these named keys under the same null-safe IS equality select() uses: the requested keys drive an indexed join, one batch per the connection's own variable limit, each distinct key probed once.""" + return [row for requested in cls.requested_keys(db, keys) for row in cls.iterate(db.execute( + f"WITH {requested.values} " + f"SELECT {cls.qualified(*cls.column_projection())} {requested.join}", + requested.bindings))] + + @classmethod + def requested_keys(cls, db: sqlite3.Connection, + keys: Sequence[Mapping[str, object]]) -> Iterator[RequestedKeys]: + """The distinct named keys, encoded by their declared columns, in batches that fit the + connection's own variable limit; each batch joins this table as ``FROM requested + JOIN ON ...`` under select()'s null-safe IS equality.""" bound = tuple(dict.fromkeys(cls._bound(key) for key in keys)) if not bound: - return [] + return field_sets = {tuple(item.name for item in selected) for selected, _ in bound} if len(field_sets) != 1: raise ValueError(f"{cls.__name__} keys name different fields: {sorted(field_sets)}") @@ -2312,11 +2333,11 @@ def select_keys(cls, db: sqlite3.Connection, for item, alias in zip(selected, aliases, strict=True)) probe = "(" + ",".join("?" for _ in selected) + ")" batch = db.getlimit(sqlite3.SQLITE_LIMIT_VARIABLE_NUMBER) // len(selected) - return [row for chunk in batched(bound, batch) for row in cls.iterate(db.execute( - f"WITH requested({','.join(aliases)}) AS (VALUES {','.join(probe for _ in chunk)}) " - f"SELECT {cls.qualified(*cls.column_projection())} FROM requested " - f"JOIN {table} ON {joined}", - tuple(value for _, values in chunk for value in values)))] + for chunk in batched(bound, batch): + yield RequestedKeys( + f"requested({','.join(aliases)}) AS (VALUES {','.join(probe for _ in chunk)})", + f"FROM requested JOIN {table} ON {joined}", + tuple(value for _, values in chunk for value in values)) @classmethod def _query( diff --git a/src/agent_comms/ui_model/observation.py b/src/agent_comms/ui_model/observation.py index b517aac113..891fdb49b0 100644 --- a/src/agent_comms/ui_model/observation.py +++ b/src/agent_comms/ui_model/observation.py @@ -35,16 +35,17 @@ from metaclass_registry.mro_dispatch import MroDispatch, handles from agent_comms.acp_extension import ( - GoalSnapshotReply, TranscriptDraftUpdate, TranscriptSnapshotUpdate, ) from agent_comms.active_route import CommsRoute from agent_comms.activity import ObservedActivity +from agent_comms.bus_read import OperatorReader, OwnerLiveRead, OwnerReadRuntimeRequest, UiRead from agent_comms.child_process import join_retirement -from agent_comms.commander_hud import CommanderFrame, CommanderHud, CommanderUnitDetail +from agent_comms.commander_hud import CommanderFrame, CommanderHud from agent_comms.comms import Comms from agent_comms.coordinator import Coordination +from agent_comms.owner_connection import OwnerConnection, OwnerReading, OwnerReplyConnection from agent_comms.coordination_errors import ( CoordinationError, CoordinationReadUnavailable, @@ -54,19 +55,17 @@ StaleRevision, ) from agent_comms.errors import RelationViolationError, ToolRefusedError -from agent_comms.field_codec import FieldCodec, OrderId -from agent_comms.historical_views import HistorySource from agent_comms.history_views import RetiredViews -from agent_comms.message_reference import MessageReference -from agent_comms.notification_evidence import NoNotificationWindow, NotificationWindow +from agent_comms.notification_evidence import NotificationWindow +from agent_comms.notification_read import NotificationsRead, UnreadableNotificationsRead +from agent_comms.notification_reads import NotificationRead from agent_comms.presentation import ( - AllOutcomes, CoordinationSnapshot, MessageNotification, NoRevision, OutcomeViewer, ReadState, + CoordinationSnapshot, MessageNotification, NoRevision, ReadState, WireRevision, ) from agent_comms.registry_document import RosterVisibility from agent_comms.registry_provenance import DeletedThread from agent_comms.revision_views import ( - ActivityView, CommanderFramesView, NotificationsView, RegistrationsView, @@ -76,9 +75,9 @@ StoresView, ) from agent_comms.thread_identity import ThreadIncarnation -from agent_comms.thread_presentation import ThreadPresentation -from agent_comms.transcripts import TranscriptCursor, TranscriptReadIdentity -from agent_comms.ui_model.delivery import ReadDeliveryView +from agent_comms.thread_presentation import AcquiredThreadPresentation, ThreadPresentation +from agent_comms.order_tools import TranscriptPageRead +from agent_comms.transcripts import TranscriptReadIdentity from agent_comms.ui_model.sidebar import ( ChannelRowModel, SidebarModel, @@ -94,7 +93,7 @@ from concurrent.futures import Future from multiprocessing.connection import Connection - from agent_comms.runtime import RuntimeConnection + from agent_comms.tool_result import ToolResult from agent_comms.transcripts import TranscriptRead @@ -429,30 +428,21 @@ def answer(self, comms: Comms) -> ReadOutcome: @dataclass(frozen=True) -class ReadCommanderUnit(ReadRequest): - """Expand an exact published card, refusing a reused name or changed assignment.""" +class ReadOperation(ReadRequest): + """The UI's root and request witness around one root-free shared read.""" - commander: ThreadIncarnation - unit: ThreadIncarnation - orders: tuple[OrderId, ...] + operation: UiRead @property def refusal_errors(self) -> tuple[type[Exception], ...]: return (*super().refusal_errors, ToolRefusedError, WorkBranchError) - def read(self, comms: Comms) -> CommanderUnitDetail: - snapshot = comms.registry.snapshot() - if not self.commander.current(snapshot) or not self.unit.current(snapshot): - raise ToolRefusedError("The selected commander or unit incarnation is no longer current") - hud = CommanderFrame.current(comms, self.commander.name) - for card in hud.units: - if card.unit == self.unit: - return card.detail(comms, self.commander, self.orders) - raise ToolRefusedError("The selected unit is no longer in this commander's published frame") + def read(self, comms: Comms) -> object: + return self.operation.read(comms) @dataclass(frozen=True) -class OwnerRead(AwaitedRead): +class OwnerRead(OwnerReading[object], AwaitedRead): """A read the live owner of thread incarnation ``owner`` answers over its socket. Served as it arrives, beside store reads; the owner's reply is decoded @@ -486,25 +476,8 @@ async def answer(self, comms: Comms) -> ReadOutcome: except self.refusal_errors as error: return self.refused(error) - def connect(self, comms: Comms) -> RuntimeConnection: - from agent_comms.runtime import OwnerIdentityChangedError, OwnerReplyConnection, socket_path - - owner = comms.registry.require(self.owner.name) - if owner.incarnation != self.owner: - raise OwnerIdentityChangedError("The owner incarnation changed while preparing the request.") - return OwnerReplyConnection( - comms, owner.name, socket_path(comms.root, owner.require_process().pid), - ) - async def ask(self, comms: Comms) -> object: - connection = await asyncio.to_thread(self.connect, comms) - try: - return await self.read(connection) - finally: - await connection.close() - - @abstractmethod - async def read(self, owner: RuntimeConnection) -> object: ... + return await OwnerReplyConnection.answer(comms, self.owner, self) @dataclass(frozen=True) @@ -521,25 +494,16 @@ def receive(self, service: ObservationService) -> None: @dataclass(frozen=True) class TranscriptRequest[ResultT](ReadRequest): - """Capture one bounded read; its owner keeps the page and witness together.""" + """The UI's lifecycle around one shared page read: root, request id, how contention is + answered and which result it publishes. The page arguments belong to the operation.""" - thread: str - before: TranscriptCursor | None = None - after: TranscriptCursor | None = None - through: TranscriptCursor | None = None - historical_source: str | None = None + operation: TranscriptPageRead def read(self, comms: Comms) -> ResultT: return UnreadableHistory.outcome(lambda: self.result(self.page_read(comms))).raised() def page_read(self, comms: Comms) -> TranscriptRead: - return comms.transcripts.capture_page_read( - self.thread, - before=self.before, - after=self.after, - through=self.through, - historical_source=self.historical_source, - ) + return self.operation.capture_as(comms, OperatorReader()) @abstractmethod def result(self, read: TranscriptRead) -> ResultT: ... @@ -608,97 +572,32 @@ def read(self, comms: Comms) -> TranscriptSnapshotUpdate | None: ) if read.content_current(): return None - return ReadTranscript( - self.root, identity.requested_name, before=identity.before, after=identity.after, + operation = TranscriptPageRead( + thread=identity.requested_name, before=identity.before, after=identity.after, through=identity.through, historical_source=identity.historical_source, - ).read(comms) + ) + return UnreadableHistory.outcome(lambda: operation.read(comms)).raised() @dataclass(frozen=True) class ReadNotifications(ReadRequest): - """Recipient handling of the referenced original messages, as its viewer may see them. - The UI reads as the operator, who sees every outcome.""" + """The UI's root and request witness around one root-free notification operation.""" - references: tuple[MessageReference, ...] - viewer: OutcomeViewer = field(default_factory=AllOutcomes, kw_only=True) + operation: NotificationRead def read(self, comms: Comms) -> NotificationWindow: - return comms.views.message_notifications_for_references(self.viewer, self.references) - - def changed_since(self, comms: Comms, revision: WireRevision, - previous: NotificationsRead) -> bool: - return previous.live_changed(comms, revision) - - -@dataclass(frozen=True) -class ReadRecordedNotifications(ReadNotifications): - """The selected archive's recorded handling, independent of today's recipients.""" - - source: str - - def read(self, comms: Comms) -> NotificationWindow: - with comms.bus.history.read_sources() as sources: - source = HistorySource.require(sources, self.source) - return source.notification_references(sources, self.references, self.viewer) - - def changed_since(self, comms: Comms, revision: WireRevision, - previous: NotificationsRead) -> bool: - return revision.history != previous.revision.history - - -@dataclass(frozen=True) -class NotificationsRead: - """Where one notification read settled: its revision and the activity of every participant - its notifications name, captured before the read for participants already known and right - after it for new ones.""" - - revision: WireRevision - activity: Mapping[str, ObservedActivity | DeletedThread | None] = field(default_factory=dict) - notification_window: NotificationWindow = field(default_factory=NoNotificationWindow) - - @classmethod - def of(cls, revision: WireRevision, comms: Comms, - known: Mapping[str, ObservedActivity | DeletedThread], - window: NotificationWindow, - ) -> NotificationsRead: - names = window.activity_names() - known = {**comms.views.observed_activity(revision, names - known.keys()), **known} - return cls(revision, {name: known.get(name) for name in names}, window) - - def live_changed(self, comms: Comms, revision: WireRevision) -> bool: - """Read marks, participant activity or retained assignment evidence changed.""" - current = (comms.views.observed_activity(revision, self.activity) - if revision.changed_since(ActivityView(), self.revision) - else self.activity) - return (any(current.get(name) != seen for name, seen in self.activity.items()) - or self.notification_window.changed(comms, revision, self.revision)) - - -class UnreadableNotificationsRead(NotificationsRead): - """An unreadable read names no participants, so any later revision reads again.""" - - def live_changed(self, comms: Comms, revision: WireRevision) -> bool: - return revision != self.revision + return self.operation.read(comms) @dataclass(frozen=True) -class ReadOwnerGoal(OwnerRead): - """The owner's goal snapshot and the turn it read with it.""" - - async def read(self, owner: RuntimeConnection) -> GoalSnapshotReply: - return FieldCodec.decode(GoalSnapshotReply, await owner.request("goal_snapshot")) +class ReadOwnerOperation(OwnerRead): + """The UI's envelope for an owner-live read: the operation's own reply, from the held owner.""" + operation: OwnerLiveRead -@dataclass(frozen=True) -class ReadDeliveryOverview(OwnerRead): - """The owner's delivery notices, checked against its live queue.""" - - include_history: bool = False - - async def read(self, owner: RuntimeConnection) -> ReadDeliveryView: - return ReadDeliveryView.from_wire( - await owner.request("input_dispositions", include_history=self.include_history) - ) + async def read(self, owner: OwnerConnection) -> ToolResult: + request = OwnerReadRuntimeRequest(thread=owner.session_id, operation=self.operation) + return self.operation.decode_reply(await request.read(owner)) @dataclass(frozen=True) @@ -860,7 +759,7 @@ def continue_from(self, previous: Self) -> Self: def read(self, comms: Comms, revision: WireRevision) -> Generator[ObservationStep]: for request in self.request.reads: previous = self.read_states.get(request) - if previous is not None and not request.changed_since(comms, revision, previous): + if previous is not None and not request.operation.changed_since(comms, revision, previous): continue yield self.notifications(request, comms, revision, previous) @@ -920,8 +819,11 @@ def read(self, comms: Comms, revision: WireRevision) -> Generator[ObservationSte changed = observed.changed_since(previous) | self.initial self.initial = frozenset() if changed: + # The reader keeps its acquired results for the next change check; a viewer + # receives only each changed result's published projection. yield replace(observed, changed=changed, - results={name: value for name, value in results.items() if name in changed}) + results={name: value.published() for name, value in results.items() + if name in changed}) def located(self, comms: Comms, revision: WireRevision, previous: ThreadsObserved) -> LocatedThreads: # Keep each completed stage when a later stored read fails, exactly as @@ -1108,10 +1010,15 @@ class ThreadReadResult(PublishState, ABC): def same_as_presented(self, presented: ThreadPresented) -> bool: return False + def published(self) -> ThreadReadResult: + """What a viewer receives of this read: the read itself.""" + return self + @abstractmethod def status_row(self, name: str, owner: OpenThreadStatus) -> ThreadStatusModel: ... - def collect_revision_inputs(self, name: str, readable: dict[str, ThreadPresentation], retry: set[str]) -> None: + def collect_revision_inputs(self, name: str, readable: dict[str, AcquiredThreadPresentation], + retry: set[str]) -> None: """Supply readable values and reads that must be tried again.""" def same_status(self, presentation: ThreadPresentation) -> bool: @@ -1129,10 +1036,14 @@ def same_as_presented(self, presented: ThreadPresented) -> bool: return (presented.presentation is self.presentation or presented.presentation.same_observation(self.presentation)) + def published(self) -> ThreadPresented: + return ThreadPresented(self.presentation.published()) + def status_row(self, name: str, owner: OpenThreadStatus) -> ThreadStatusModel: return ThreadStatusModel.known(name, self.presentation) - def collect_revision_inputs(self, name: str, readable: dict[str, ThreadPresentation], retry: set[str]) -> None: + def collect_revision_inputs(self, name: str, readable: dict[str, AcquiredThreadPresentation], + retry: set[str]) -> None: self.presentation.collect_revision_input(name, readable) def same_status(self, presentation: ThreadPresentation) -> bool: @@ -1152,7 +1063,8 @@ class ThreadBusy(ThreadReadResult): def status_row(self, name: str, owner: OpenThreadStatus) -> ThreadStatusModel: return owner.open(name) - def collect_revision_inputs(self, name: str, readable: dict[str, ThreadPresentation], retry: set[str]) -> None: + def collect_revision_inputs(self, name: str, readable: dict[str, AcquiredThreadPresentation], + retry: set[str]) -> None: retry.add(name) @@ -1164,8 +1076,8 @@ class ThreadsObserved(Observed): changed: frozenset[str] activity: Mapping[str, ObservedActivity | DeletedThread] - def revision_inputs(self) -> tuple[dict[str, ThreadPresentation], set[str]]: - readable: dict[str, ThreadPresentation] = {} + def revision_inputs(self) -> tuple[dict[str, AcquiredThreadPresentation], set[str]]: + readable: dict[str, AcquiredThreadPresentation] = {} retry: set[str] = set() for name, result in self.results.items(): result.collect_revision_inputs(name, readable, retry) diff --git a/src/agent_comms/work_branches.py b/src/agent_comms/work_branches.py index 96b04c8e91..3bd0943681 100644 --- a/src/agent_comms/work_branches.py +++ b/src/agent_comms/work_branches.py @@ -365,6 +365,15 @@ def pull_request_base(self, base: str, head: str) -> str: self.require_commits("origin", base_head, head) return base_head + def commit_parents(self, head: str) -> tuple[str, ...]: + self.require_commits("origin", head) + return tuple(self._git("rev-list", "--parents", "-n", "1", head).split()[1:]) + + def commit_message(self, head: str) -> str: + """Read an immutable commit's message; its domain owner decodes any metadata.""" + self.require_commits("origin", head) + return self._git("show", "--no-patch", "--format=%B", head) + def pull_request_patch_id(self, base: str, head: str) -> str: base_head = self.pull_request_base(base, head) patch = self._git( diff --git a/src/agent_comms_hooks/reread.py b/src/agent_comms_hooks/reread.py index 95de1f47ef..fd8615ba98 100644 --- a/src/agent_comms_hooks/reread.py +++ b/src/agent_comms_hooks/reread.py @@ -2,7 +2,7 @@ Claude Code runs this after every tool call. The session start hook orders the sources read once and deals them into a deck of short slices beside the transcript: the voice, the -commander's indictment, the drift tells of the rules the session follows and the recorded +commander's indictment, each followed rule's wake-up with its tell, and the recorded failures, interleaved. Every `every` tool calls one slice goes into context inline, the next one each time. Nothing repeats verbatim on every call, nothing waits on a read the commander may skip, and the cadence counts tool calls, so no transcript diff --git a/src/agent_comms_hooks/skill.py b/src/agent_comms_hooks/skill.py index c361e1e723..606a10fd85 100644 --- a/src/agent_comms_hooks/skill.py +++ b/src/agent_comms_hooks/skill.py @@ -46,8 +46,10 @@ class DeckReader(ABC): deck_suffix: ClassVar[str] """Where this reader's deck sits beside the transcript: a session whose role changes finds no deck of its new kind and is dealt again.""" - slice_chars: ClassVar[int] = 2_000 - """The most characters one inline slice carries; a longer paragraph splits at its lines.""" + slice_chars: ClassVar[int] = 2_500 + """The most characters one inline slice carries; a longer paragraph splits at its lines. A + rule's wake-up is never split, since a greentext cut before its punchline teaches nothing, + so the cap sits above the longest one and the deck test fails when one outgrows it.""" def __init_subclass__(cls, audience: type[RuleAudience], **options: object) -> None: super().__init_subclass__(**options) @@ -94,7 +96,7 @@ def voice(cls) -> tuple[str, ...]: @classmethod def tells(cls) -> tuple[str, ...]: - return tuple(rule.drift for rule in cls.rules()) + return tuple(rule.reminder for rule in cls.rules()) @classmethod def failures(cls) -> tuple[str, ...]: diff --git a/tests/conftest.py b/tests/conftest.py index 267bdec096..cf7d9fc0d9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -55,7 +55,7 @@ from agent_comms.registry_document import RegisteredCommanderProject from agent_comms_hooks.watch import WatchClaim, WatchRecord -pytest_plugins = (goal_owner_fixture.__name__, "github_fixture", "pytest_session") +pytest_plugins = (goal_owner_fixture.__name__, "github_fixture", "owner_socket_fixture", "pytest_session") class TranscriptRecords: diff --git a/tests/github_cli_stub.py b/tests/github_cli_stub.py index 47b8903b90..bb79f5031c 100755 --- a/tests/github_cli_stub.py +++ b/tests/github_cli_stub.py @@ -25,6 +25,9 @@ def main(): if arguments[:2] == ["pr", "view"]: data = repositories[arguments[arguments.index("-R") + 1]] request = data["requests"][arguments[2]] + if refusal := request.get("read_refusal"): + print(refusal, file=sys.stderr) + raise SystemExit(1) field = arguments[arguments.index("--json") + 1] assert arguments[arguments.index("--jq") + 1] == "." + field print({"number": request["listed"]["number"], "body": request["body"]}[field]) @@ -37,9 +40,15 @@ def main(): print(entry["listed"]["number"]) elif parts[:1] == ["pulls"] and len(parts) == 2: assert arguments[arguments.index("--jq") + 1] == ( - '.head.sha + " " + .base.ref + " " + (.draft | tostring)') + '.head.sha + " " + .base.ref + " " + (.draft | tostring) + " " + .state') listed = data["requests"][parts[1]]["listed"] - print(listed["head"], listed["base"], str(listed["kind"] == "draft").lower()) + state = "open" if listed["kind"] in data["open_kinds"] else "closed" + print(listed["head"], listed["base"], str(listed["kind"] == "draft").lower(), state) + elif len(parts) == 3 and parts[0] == "commits" and parts[2] == "pulls": + assert arguments[arguments.index("--jq") + 1] == "map(.number)" + print(json.dumps([entry["listed"]["number"] for entry in data["requests"].values() + if entry["listed"].get("merge", {}).get("oid") == parts[1] + or entry["listed"]["head"] == parts[1]])) elif parts[:1] == ["statuses"] and len(parts) == 2: assert arguments[arguments.index("--method") + 1] == "POST" print("{}") diff --git a/tests/github_fixture.py b/tests/github_fixture.py index 53ead17486..ab1e952e30 100644 --- a/tests/github_fixture.py +++ b/tests/github_fixture.py @@ -36,6 +36,12 @@ def put(self, repository: RepositoryName, row: ListedPullRequest, body: str = "" data[repository.to_wire()]["requests"][str(row.number)] = {"body": body, "listed": FieldCodec.encode(row)} self.path.write_text(json.dumps(data)) + def refuse_read(self, repository: RepositoryName, number: int, refusal: str) -> None: + """An existing remote PR is inaccessible at GitHub's actual process boundary.""" + data = json.loads(self.path.read_text()) + data[repository.to_wire()]["requests"][str(number)]["read_refusal"] = refusal + self.path.write_text(json.dumps(data)) + def refuse_arm(self, repository: RepositoryName, number: int, refusal: str) -> None: """GitHub's answer when enabling auto-merge on this pull request fails.""" data = json.loads(self.path.read_text()) diff --git a/tests/guards/test_debt_ratchet.py b/tests/guards/test_debt_ratchet.py index 738aecee97..da03e89dc4 100644 --- a/tests/guards/test_debt_ratchet.py +++ b/tests/guards/test_debt_ratchet.py @@ -46,10 +46,12 @@ AnyName, BooleanChainTerms, BuiltinHandlerTypeSwitch, + ClientTypeErrors, CodecSubclass, FamilyFlattened, ForeignAbsenceProbe, GodClassExcess, + HeadSide, LintFindings, LongBooleanChain, Measure, @@ -66,17 +68,36 @@ Unformatted, compare, ) +from module_layers import StackClient pytestmark = pytest.mark.refactor_guard def test_measure_owner_rejects_a_new_absence() -> None: name = NoneAbsence.at("tools/example.py") - result = Comparison("tools", "base", "head", (), {name: 0}, {name: 1}, {}, {}) + result = Comparison("tools", "base", "head", (), {name: 0}, {name: 1}, {name: [1]}, {}) assert result.delta[name] == 1 assert result.failed +GROWN_ABSENCE = NoneAbsence.at("tools/example.py") + + +@pytest.mark.parametrize( + ("introduced", "failed", "missing"), + [({}, True, [GROWN_ABSENCE]), ({GROWN_ABSENCE: []}, False, []), ({GROWN_ABSENCE: [1]}, True, [])], + ids=["unreported", "reported_none_new", "reported_new"], +) +def test_clean_lines_growth_fails_only_on_a_new_site_or_a_missing_report(introduced, failed, missing): + """Raw growth alone is a move; an explicit empty report says nothing is new, while an + absent report proves nothing.""" + result = Comparison("tools", "base", "head", (), {GROWN_ABSENCE: 0}, {GROWN_ABSENCE: 1}, introduced, {}) + assert result.delta[GROWN_ABSENCE] == 1 + assert result.failed is failed + assert result.missing == missing + assert FieldCodec.project(result, "report")["missing"] == [key.text for key in missing] + + @pytest.mark.parametrize( ("source", "expected"), [ @@ -834,6 +855,51 @@ def test_both_revisions_are_checked_under_the_head_configuration(repo: Repositor assert report["delta"][measure.at(f"{repo.root}/kept.py").text] == 0 +USES_EXTERNAL_VALUE = "import extdep\n\nvalue: int = extdep.VALUE\n" + + +def typed_dependency(directory: Path, annotation: str, value: str) -> Path: + """An installed PEP 561 package outside every committed root, found through sys.path.""" + package = directory / "extdep" + package.mkdir(parents=True) + (package / "py.typed").write_text("") + (package / "__init__.py").write_text(f"VALUE: {annotation} = {value}\n") + return directory + + +def source_type_errors(repo: Repository, client: StackClient) -> int: + revision = repo.commit({"uses_external.py": USES_EXTERNAL_VALUE}) + selected = [f"{repo.root}/uses_external.py"] + _, counted = TypeErrors.findings(repo.path, revision, revision, repo.root, selected, HeadSide) + return counted[selected[0]] + + +def client_type_errors(repo: Repository, client: StackClient) -> int: + revision = repo.commit({"__init__.py": ""}) + _, counted = ClientTypeErrors.findings(repo.path, revision, client, HeadSide) + return counted[f"{client.name}/src/uses_external.py"] + + +@pytest.mark.parametrize("type_errors", [source_type_errors, client_type_errors]) +def test_type_counts_follow_the_installed_dependency_not_a_stored_count( + repo: Repository, tmp_path: Path, monkeypatch, type_errors +) -> None: + """Mypy reads installed packages no content key names. With committed source, client and + configuration unchanged, switching the installed dependency must change the count, under + the same count-cache directory and mypy's own cache: hosted #3538 compared a base count + stored under a Core-only environment with a head count from the full stack.""" + accepting = typed_dependency(tmp_path / "accepting", "int", "1") + rejecting = typed_dependency(tmp_path / "rejecting", "str", "'one'") + (tmp_path / "client").mkdir() + checkout = Repository(tmp_path / "client", "src") + checkout.commit({"uses_external.py": "import agent_comms\n" + USES_EXTERNAL_VALUE}) + client = StackClient("probe-client", checkout.path) + monkeypatch.setenv("PYTHONPATH", search_path(accepting)) + assert type_errors(repo, client) == 0 + monkeypatch.setenv("PYTHONPATH", search_path(rejecting)) + assert type_errors(repo, client) == 1 + + def test_foreign_probes_and_codec_bases_use_ast_not_text(repo: Repository) -> None: base = repo.commit({}) head = repo.commit( @@ -1104,6 +1170,49 @@ def test_none_as_absence_fails_with_its_line_except_in_a_declared_boundary_decod assert report["introduced"][plan] == [] and report["delta"][plan] == 0 +@pytest.mark.parametrize("declaration,expected", [ + pytest.param("from agent_comms.field_codec import boundary_decoder\n@boundary_decoder\n{function}", 0, + id="imported"), + pytest.param("from agent_comms.field_codec import boundary_decoder as decode\n@decode\n{function}", 0, + id="aliased"), + pytest.param("import agent_comms.field_codec as codec\n@codec.boundary_decoder\n{function}", 0, + id="qualified"), + pytest.param("import agent_comms.field_codec\n@agent_comms.field_codec.boundary_decoder\n{function}", 0, + id="module"), + pytest.param("from .field_codec import boundary_decoder\n@boundary_decoder\n{function}", 0, + id="relative"), + pytest.param("{function}", 1, id="undecorated"), + pytest.param("from elsewhere import boundary_decoder\n@boundary_decoder\n{function}", 1, + id="different-owner"), + pytest.param("from agent_comms.field_codec import boundary_decoder\n" + "from elsewhere import boundary_decoder\n@boundary_decoder\n{function}", 1, + id="duplicate"), + pytest.param("if enabled:\n from agent_comms.field_codec import boundary_decoder\n" + "@boundary_decoder\n{function}", 1, id="conditional"), + pytest.param("from agent_comms.field_codec import *\n@boundary_decoder\n{function}", 1, + id="star"), + pytest.param("from agent_comms.field_codec import boundary_decoder\nclass Decoder:\n" + " boundary_decoder = lambda function: function\n @boundary_decoder\n" + " def decode(external):\n return external.marker\n", 1, id="class-shadowed"), + pytest.param("from agent_comms.field_codec import boundary_decoder\ndef outer(boundary_decoder):\n" + " @boundary_decoder\n def decode(external):\n return external.marker\n", 1, + id="parameter-shadowed"), +]) +def test_declared_decoder_boundary_is_shared_by_both_contextual_absence_measures( + tmp_path: Path, declaration: str, expected: int, +) -> None: + repository = Repository(tmp_path, "src/agent_comms") + source = declaration.format(function="def decode(external):\n return external.marker\n") + base = repository.commit({"boundary.py": source}) + head = repository.commit({"boundary.py": source.replace("return external.marker", "return external.marker is None")}) + status, report = repository.compare(base, head, NoneAbsence, ForeignAbsenceProbe) + assert status == int(bool(expected)) + for measure in (NoneAbsence, ForeignAbsenceProbe): + identity = measure.at(f"{repository.root}/boundary.py").text + assert report["head"][identity] == expected + assert report["delta"][identity] == expected + + def test_an_unchanged_absence_on_a_rewritten_line_is_not_written_but_a_new_one_is(repo): base = repo.commit({"owner.py": "def ready(stored):\n return stored is not None and stored.done\n"}) kept = repo.commit({"owner.py": "def ready(stored):\n return stored is not None and stored.settled()\n"}) @@ -1124,6 +1233,7 @@ def test_an_absence_moved_with_its_class_to_another_module_is_not_written(repo): base = repo.commit({"attempts.py": holder, "executions.py": "VALUE = 1\n"}) moved = repo.commit({"attempts.py": "VALUE = 2\n", "executions.py": holder}) status, report = repo.compare(base, moved, NoneAbsence) + assert status == 0 assert report["introduced"][NoneAbsence.at(f"{repo.root}/executions.py").text] == [] deleted = repo.commit({"attempts.py": holder, "executions.py": None}) moved_out = repo.commit({"attempts.py": None, "owners.py": holder + "\n\n" + other}) @@ -1132,6 +1242,106 @@ def test_an_absence_moved_with_its_class_to_another_module_is_not_written(repo): assert report["introduced"][NoneAbsence.at(f"{repo.root}/owners.py").text] == [8] +EXTRACTED_FROM = ( + "class Request:\n thread: str\n before: int | None = None\n archive: str | None = None\n" + "\n\nclass Other:\n name: str\n" +) +HOLDER = "from .pages import Page\n\n\nclass Request:\n thread: str\n operation: Page\n" +OTHER = "\n\nclass Other:\n name: str\n" +PAGE = ( + "from typing import Annotated\n\n\nclass Page:\n" + " before: Annotated[int | None, \"Read before this cursor\"] = None\n" + " archive: str | None = None\n" +) + + +@pytest.mark.parametrize( + ("holder", "pages", "written"), + [ + (HOLDER + OTHER, PAGE, 0), + (HOLDER.replace(" operation", " before: int | None = None\n operation") + OTHER, PAGE, 1), + (HOLDER.replace("operation: Page", "operation: object") + OTHER, PAGE, 2), + (HOLDER + OTHER + " page: Page\n", PAGE, 2), + (HOLDER + "\n\nPage = dict\n" + OTHER, PAGE, 2), + (HOLDER.replace(".pages", ".elsewhere") + OTHER, PAGE, 2), + (HOLDER + OTHER, PAGE.replace("before:", "earlier:"), 1), + (HOLDER + OTHER, PAGE.replace("int | None", "float | None"), 1), + (HOLDER + OTHER, PAGE.replace("archive: str | None = None", "archive: str | None = ''"), 1), + (HOLDER + OTHER, PAGE + " limit: int | None = None\n", 1), + (HOLDER + OTHER, PAGE + "\n def empty(self):\n return self.before is None\n", 1), + (HOLDER.replace("import Page", "import Copy, Page").replace( + "operation: Page", "operation: Page\n copy: Copy") + OTHER, + PAGE + "\n\nclass Copy:\n before: int | None = None\n", 3), + (HOLDER.replace("import Page", "import Page as Held").replace("operation: Page", "operation: Held") + + OTHER, PAGE.replace("import Annotated", "import Annotated as Meta").replace("Annotated[", "Meta["), 0), + (HOLDER.replace("from .pages import Page\n", "def load():\n from .pages import Page\n") + OTHER, + PAGE, 2), + (HOLDER + "\n\ndef make():\n Page = dict\n return Page\n" + OTHER, PAGE, 0), + (HOLDER + OTHER, PAGE + "\n\nAnnotated = list\n", 1), + (HOLDER + OTHER, PAGE.replace("from typing import Annotated\n", "def typed():\n from typing import Annotated\n"), 1), + (HOLDER + OTHER, PAGE + "\n\nPage = dict\n", 2), + (HOLDER + OTHER, PAGE + " archive = ''\n", 1), + (HOLDER.replace(" operation: Page", " operation: Page\n before, extra = 1, 2") + OTHER, PAGE, 1), + (HOLDER + OTHER, PAGE.replace("class Page:\n", "class Page:\n Annotated = dict\n"), 1), + (HOLDER.replace(" operation: Page", " Page = dict\n operation: Page") + OTHER, PAGE, 2), + ], + ids=["extracted", "retained", "unheld", "second_owner", "shadowed", "unresolved", "renamed", + "retyped", "redefaulted", "extra_field", "method", "copied_twice", "aliased_imports", + "nested_only_import", "unrelated_local_name", "annotated_rebound", "annotated_local_only", + "export_rebound", "field_rebound", "source_rebound_by_unpacking", + "annotated_shadowed_in_class", "held_class_shadowed_in_owner"], +) +def test_fields_extracted_into_the_one_new_held_class_are_moved_not_written(repo, holder, pages, written): + """An old class that now holds one new class in a typed field carries each field it lost + into that class unchanged; anything the proof does not cover is still written.""" + base = repo.commit({"holder.py": EXTRACTED_FROM, "pages.py": "VALUE = 1\n"}) + head = repo.commit({"holder.py": holder, "pages.py": pages}) + status, report = repo.compare(base, head, NoneAbsence) + source, destination = (NoneAbsence.at(f"{repo.root}/{name}").text for name in ("holder.py", "pages.py")) + assert status == int(written > 0) + assert len(report["introduced"][destination]) == written + assert report["introduced"][source] == [] + assert report["base"][source] == 2 and report["head"][destination] >= 2 + assert report["delta"][destination] == report["head"][destination] + + +TYPED_FROM = ( + "from .alpha import Cursor\n\n\nclass Request:\n thread: str\n" + " before: Cursor | None = None\n archive: str | None = None\n" +) +TYPED_PAGE = ( + "from .alpha import Cursor\n\n\nclass Page:\n" + " before: Cursor | None = None\n archive: str | None = None\n" +) + + +@pytest.mark.parametrize( + ("source", "pages", "written"), + [ + (TYPED_FROM, TYPED_PAGE, 0), + (TYPED_FROM, TYPED_PAGE.replace("import Cursor", "import Cursor as Mark").replace( + "before: Cursor", "before: Mark"), 0), + (TYPED_FROM, TYPED_PAGE.replace(".alpha", ".beta"), 1), + (TYPED_FROM, TYPED_PAGE + "\n\nstr = bytes\n", 1), + (TYPED_FROM, TYPED_PAGE.replace(" archive", " str = bytes\n archive"), 1), + (TYPED_FROM.replace("from .alpha import Cursor", "from typing import TypeVar\n\nCursor = TypeVar('Cursor')"), + TYPED_PAGE.replace("from .alpha import Cursor", "from typing import TypeVar\n\nCursor = TypeVar('Cursor')"), 1), + ], + ids=["same_owner", "same_owner_aliased", "changed_owner", "builtin_shadowed", + "builtin_shadowed_in_class", "type_variable"], +) +def test_extracted_field_types_are_compared_by_the_owner_they_import(repo, source, pages, written): + """A field type is the module and name its spelling is bound to, not the spelling: the same + Cursor imported from another module is another type, and an alias of the same is not.""" + base = repo.commit({"holder.py": source + OTHER, "pages.py": "VALUE = 1\n"}) + head = repo.commit({"holder.py": HOLDER + OTHER, "pages.py": pages}) + status, report = repo.compare(base, head, NoneAbsence) + source_count, destination = (NoneAbsence.at(f"{repo.root}/{name}").text for name in ("holder.py", "pages.py")) + assert status == int(written > 0) + assert len(report["introduced"][destination]) == written + assert report["introduced"][source_count] == [] + + def test_git_context_configuration_cannot_mark_unchanged_lines_as_new(repo): repo.git("config", "diff.interHunkContext", "3") base = repo.commit({"names.py": "first = 1\nold_receipt = 1\nlast = 1\n"}) diff --git a/tests/integration_source_fixture.py b/tests/integration_source_fixture.py new file mode 100644 index 0000000000..b6390801b4 --- /dev/null +++ b/tests/integration_source_fixture.py @@ -0,0 +1,136 @@ +"""One real Git/forge/Todo setup for matched source capture and promotion controls.""" + +from collections.abc import Callable, Iterator +from dataclasses import dataclass, replace +from pathlib import Path +import sys +import time + +import pytest + +from agent_comms.commander.order import Commander, CommanderContext +from agent_comms.comms import Comms +from agent_comms.goal_actions import CreateProblemGoalAction +from agent_comms.head_review import CombinedInput, IntegrationInput, IntegrationSource, PullRequestHead +from agent_comms.installer.commands import InstallCommands +from agent_comms.integration import IntegrationCheckout, IntegrationRun +from agent_comms.recorded_time import RecordedEventTime +from agent_comms.repo import Repo +from agent_comms.repository_name import RepositoryName +from agent_comms.todos import DraftListedPullRequest, PullRequests, RecordPullRequest +from fixture_git import ENVIRONMENT +from github_fixture import GitHubFixture +from source_revision_fixture import SourceRepository +from module_layers import MaterializedClients + + +@dataclass(frozen=True) +class MutualSources: + context: CommanderContext + core: SourceRepository + client: SourceRepository + checkout: IntegrationCheckout + output: Path + github: GitHubFixture + carrier_number: int = 101 + + def publish_carrier(self, github: GitHubFixture) -> PullRequestHead: + """Publish the separate combined PR; source-member refs and Todo links stay intact.""" + commit = self.core.git(self.checkout.path, "rev-parse", "HEAD").strip() + self.core.git(self.checkout.path, "push", "origin", f"{commit}:refs/pull/{self.carrier_number}/head") + stamp = "2026-10-24T00:00:00Z" + github.put(self.core.github, DraftListedPullRequest( + self.carrier_number, "combined", commit, stamp, "main", reviewable_times=(stamp,), + )) + return PullRequestHead(self.core.github, self.carrier_number, commit) + + def prepare_carrier(self, problems: tuple[str, ...]) -> PullRequestHead: + """Fixture setup owns an actual published combination before production capture.""" + todos = tuple(self.context.comms.goals.todos.get(problem) for problem in problems) + linked = PullRequests.linked_among(todos, self.core.github) + heads = tuple(PullRequestHead(self.core.github, row.number, row.head) + for row in PullRequests(todos).rows(self.core.github, tuple(sorted(linked))) + if row.as_unmerged()) + self.checkout.combine_sources(IntegrationSource.main_head(self.core), heads) + return self.publish_carrier(self.github) + + def capture(self, problems: tuple[str, ...], risk: str) -> CombinedInput: + return IntegrationRun(self.checkout, self.output, risk).qualify( + self.context, problems, carrier=self.prepare_carrier(problems)) + + +@pytest.fixture +def source_pair( + held_bus: Comms, + configured_repository: Callable[[Comms, *tuple[Repo, ...]], Commander], + github_cli: GitHubFixture, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> MutualSources: + for name, value in ENVIRONMENT.items(): + monkeypatch.setenv(name, value) + client = SourceRepository.create( + tmp_path / "matched-client", RepositoryName.from_wire("example/client"), baseline_files={ + Path("pyproject.toml"): SourceRepository.package_project("source-client"), + Path("src/workflow.py"): 'from agent_comms.api import old_api\n\ndef run() -> str:\n return old_api()\n', + }) + core = SourceRepository.create(tmp_path / "matched-core", held_bus.goals.todos.get("P2").repo, baseline_files={ + Path("pyproject.toml"): SourceRepository.package_project("agent-comms"), + Path("stack/pyproject.toml"): ( + '[project]\nname = "fixture-stack"\nversion = "0.0.1"\ndependencies = ["agent-comms", "source-client"]\n' + '[tool.uv]\npackage = false\noverride-dependencies = ["packaging==26.0"]\n' + 'constraint-dependencies = ["typing-extensions>=4.15"]\n' + '[tool.uv.pip]\nstrict = true\n' + '[tool.uv.sources]\nagent-comms = {path = "..", editable = true}\n' + f'source-client = {{git = "{client.remote}", branch = "main"}}\n' + ), + Path("stack/uv.lock"): 'version = 1\nrevision = 3\nrequires-python = ">=3.14"\npackage = []\n', + Path("uv.lock"): 'version = 1\nrevision = 3\nrequires-python = ">=3.14"\npackage = []\n', + Path("src/agent_comms/__init__.py"): "", + Path("src/agent_comms/api.py"): 'def old_api() -> str:\n return "baseline"\n', + }) + for source in (core, client): + source.git(source.checkout, "switch", "feature") + core.publish_change("Replace the Core export without an alias", *core.write_files({ + Path("src/agent_comms/api.py"): 'def new_api() -> str:\n return "candidate"\n', + })) + client.publish_change("Migrate the client to the new Core export", *client.write_files({ + Path("src/workflow.py"): 'from agent_comms.api import new_api\n\ndef run() -> str:\n return new_api()\n', + })) + context = configured_repository(held_bus, core, client) + primary = held_bus.goals.todos.get("P2") + held = primary.held() + RecordPullRequest(primary.revision, held.owner_identity, core.link, held.generation).record( + held_bus.goals, primary.id) + stamp = "2026-10-24T00:00:00Z" + for source in (core, client): + github_cli.put(source.github, DraftListedPullRequest( + source.number, "feature", source.branch_commit, stamp, "main", reviewable_times=(stamp,), + )) + checkout = IntegrationCheckout.create(core, tmp_path / "qualification", core.main_commit) + return MutualSources(context, core, client, checkout, tmp_path / "qualified-inputs", github_cli) + + +@pytest.fixture +def materialized_clients(test_uv: Path) -> Iterator[Callable[[Path, IntegrationInput], MaterializedClients]]: + """The root test run supplies the actual frontend; every test owns one command lifetime.""" + with InstallCommands() as commands: + def acquire(directory: Path, inputs: IntegrationInput) -> MaterializedClients: + return MaterializedClients(directory, inputs, test_uv, Path(sys.executable), commands) + yield acquire + + +@pytest.fixture +def mutual_sources(source_pair: MutualSources) -> MutualSources: + """The same source owner, with its companion linked by a separate real checked order.""" + comms = source_pair.context.comms + primary = comms.goals.todos.get("P2") + comms.goals.update_goal("worker", CreateProblemGoalAction( + problem="P3", commander="lead", request=primary.request, + assigner=comms.registry.require("lead").incarnation, + repo=source_pair.client.github, text="Migrate the paired source client", + order=replace(primary.order.checked(), pull_requests=(source_pair.client.link,), + check=replace(primary.order.check, directory=source_pair.client.checkout)), + review_at=RecordedEventTime(time.time() + 3600), + )) + return source_pair diff --git a/tests/native_backend_fixture.py b/tests/native_backend_fixture.py index bc354d1ac5..7fe45e327b 100644 --- a/tests/native_backend_fixture.py +++ b/tests/native_backend_fixture.py @@ -21,6 +21,7 @@ from agent_comms.child_process import BoundedRun, join_retirement from agent_comms.comms import wire from agent_comms.coordinator import Coordination +from agent_comms.cursor_publication import CursorDelivery from agent_comms.field_codec import FieldCodec, InputId from agent_comms.fresh_private_session import create_fresh_private_session from agent_comms.native_arguments import GivenArguments, GivenBinary, NativeArguments @@ -62,6 +63,7 @@ def __init__(self, root, project, session, provider, config, package: VerifiedNa self.config = config self.package = package self.persistent = PersistentPiSession() + self.cursor_delivery = CursorDelivery() self.participant = self.participant_at(root, project) self.starts = [] self.children: list[PiSessionChild] = [] @@ -197,6 +199,7 @@ async def run(self, text, *, followup=None, queue=None, **options): tool_mode=NoSelectedTools(), steering_queue=queue, input_start=self, + cursor_delivery=self.cursor_delivery, **options, session_directory=self.root): result.append(event) @@ -303,6 +306,7 @@ async def close_server(): async def retire_all(): async with AsyncExitStack() as retirement: + retirement.callback(owner.cursor_delivery.close) retirement.push_async_callback(close_server) for child in owner.children: retirement.push_async_callback(child.shutdown) diff --git a/tests/native_proof_cases.py b/tests/native_proof_cases.py index f39d59015c..6e233201fb 100644 --- a/tests/native_proof_cases.py +++ b/tests/native_proof_cases.py @@ -8,7 +8,7 @@ from pathlib import Path from agent_comms.field_codec import FieldCodec -from agent_comms.native_pi import NativeContextJournal, NativeContextProof, NativeContextRecord +from agent_comms.native_input_proof import NativeContextJournal, NativeContextProof, NativeContextRecord from agent_comms.native_session_files import NativeSessionFiles diff --git a/tests/observed_queue.py b/tests/observed_queue.py new file mode 100644 index 0000000000..e65a850d6d --- /dev/null +++ b/tests/observed_queue.py @@ -0,0 +1,38 @@ +"""What the real observer receiver delivers, in receive order, for tests that watch it. + +``ObservationProcess.start_receiving`` owns reading the pipe; a test passes this queue's +callbacks to it once and waits here. A receiver failure is raised as its original exception +when the waiting test reaches it, never swallowed or replaced. +""" +from __future__ import annotations + +import asyncio +from collections.abc import Callable + +from agent_comms.ui_model.observation import Observed + + +class ObservedQueue: + def __init__(self) -> None: + self._delivered: asyncio.Queue[Callable[[], Observed]] = asyncio.Queue() + + def observed(self, result: Observed) -> None: + self._delivered.put_nowait(lambda: result) + + def failed(self, error: Exception) -> None: + def raise_original() -> Observed: + raise error + + self._delivered.put_nowait(raise_original) + + async def next(self) -> Observed: + async with asyncio.timeout(30): + delivered = await self._delivered.get() + return delivered() + + async def until(self, accepted: Callable[[Observed], bool]) -> Observed: + """One 30 s bound for the whole match, however many other results arrive first.""" + async with asyncio.timeout(30): + while not accepted(result := await self.next()): + pass + return result diff --git a/tests/owner_socket_fixture.py b/tests/owner_socket_fixture.py new file mode 100644 index 0000000000..c328086c64 --- /dev/null +++ b/tests/owner_socket_fixture.py @@ -0,0 +1,69 @@ +"""Registered owner threads, their runtime sockets and their child processes for owner-read tests. + +A native owner's socket here is a real Unix-socket peer that receives exactly one request +and closes without replying: it proves dispatch and then reply loss, never the domain +answer. An external owner has a live process and no socket. Child processes are real +spawned processes that a test may stop through their own owner. +""" +from __future__ import annotations + +import asyncio +import json +import sys +from collections.abc import AsyncIterator, Iterator +from contextlib import asynccontextmanager + +import pytest + +from agent_comms.child_process import ProcessIdentity, SpawnedProcess +from agent_comms.comms import Comms +from agent_comms.private_path import PrivateSocketRole +from agent_comms.repo import Checkout +from agent_comms.runtime import socket_path +from agent_comms.thread_execution import NativeThreadExecution, ThreadExecution +from agent_comms.threads import Thread + + +@pytest.fixture +def runtime_processes() -> Iterator[tuple[SpawnedProcess, SpawnedProcess]]: + children = [] + try: + for _ in range(2): + children.append( + SpawnedProcess.launch((sys.executable, "-c", "import time; time.sleep(60)")) + ) + yield tuple(children) + finally: + for child in children: + child.stop_sync() + + +def declare_owner( + comms: Comms, name: str, process: ProcessIdentity, + execution: type[ThreadExecution] = NativeThreadExecution, +) -> Thread: + return comms.registry.declare(Thread( + name, frozenset(), Checkout(comms.root.parent), process_identity=process, execution=execution, + )) + + +@asynccontextmanager +async def silent_socket(comms: Comms, process: ProcessIdentity) -> AsyncIterator[list[dict[str, object]]]: + """The owner's socket path served by a peer that reads one request and hangs up.""" + path = socket_path(comms.root, process.pid) + path.parent.mkdir(parents=True, exist_ok=True) + received: list[dict[str, object]] = [] + + async def disconnect_after_request(reader, writer): + received.append(json.loads(await reader.readline())) + writer.close() + await writer.wait_closed() + + with PrivateSocketRole.address(path) as address: + server = await asyncio.start_unix_server(disconnect_after_request, path=address) + try: + yield received + finally: + server.close() + await server.wait_closed() + path.unlink(missing_ok=True) diff --git a/tests/pi_session_turn.py b/tests/pi_session_turn.py index fd8f95a8f4..883f271fcf 100644 --- a/tests/pi_session_turn.py +++ b/tests/pi_session_turn.py @@ -4,11 +4,12 @@ from agent_comms.repo import Checkout -from contextlib import aclosing +from contextlib import AsyncExitStack, aclosing from pathlib import Path import agent_comms.backend as backend from agent_comms.comms import wire +from agent_comms.cursor_publication import CursorDelivery from agent_comms.agent_backend import NoInputStartListener from agent_comms.pi_native_backend import PersistentPiSession from agent_comms.native_package import VerifiedNativePackage @@ -24,15 +25,16 @@ async def one_turn_events(agent_bin, agent_args, task, cwd, env_extra=None, owner = comms.registry.declare(Thread( "independent-turn", frozenset(), Checkout(Path(cwd)), )).incarnation.require_recorded() - try: + async with AsyncExitStack() as resources: + cursor_delivery = CursorDelivery() + resources.callback(cursor_delivery.close) + resources.push_async_callback(session.close) async with aclosing( backend.stream_agent_events(agent_bin, agent_args, task, cwd, env_extra, persistent_session=session, root=comms.root, owner=owner, tool_mode=tool_mode, - input_start=NoInputStartListener(), + input_start=NoInputStartListener(), cursor_delivery=cursor_delivery, package=VerifiedNativePackage.acquire(package), session_directory=Path(cwd), **kwargs) ) as stream: async for event in stream: yield event - finally: - await session.close() diff --git a/tests/round2/test_typed_table.py b/tests/round2/test_typed_table.py index e8c6dc5093..81ca0c2585 100644 --- a/tests/round2/test_typed_table.py +++ b/tests/round2/test_typed_table.py @@ -179,12 +179,19 @@ def test_declared_table_family(tmp_path): for row in (parent, child): row.insert(db) assert type(row).select(db) == [row] + # Requested keys join under IS, so a null key selects the null row, once. + null_key = {"key": "c", "weight": None} + assert TableChildRow.select_keys(db, (null_key, null_key)) == [child] + assert TableChildRow.select_keys(db, ({"key": "c", "weight": 3.25},)) == [] with pytest.raises(sqlite3.IntegrityError): replace(child, key="bad", parent="missing").insert(db) with pytest.raises(TypeError): replace(parent, key="bool", count=True).insert(db) replace(child, weight=3.25).upsert(db) assert TableChildRow.one(db, key="c").weight == 3.25 + assert TableChildRow.select_keys(db, (null_key,)) == [] + assert TableChildRow.select_keys(db, ({"key": "c", "weight": 3.25},)) == [ + replace(child, weight=3.25)] child.upsert(db) assert TableChildRow.one(db, key="c") == child TableChildRow.update(db, where="key=?", parameters=("c",), enabled=False) @@ -395,7 +402,7 @@ class LeanBoardLine(DurableTables, TypedTable, declared_name="lean_board_line"): def test_atomic_creation_preserves_declared_indexes_triggers_and_strict_creation(): - from agent_comms.native_pi import NativeContextJournal + from agent_comms.native_input_proof import NativeContextJournal with sqlite3.connect(':memory:') as db: NativeContextJournal.create(db, if_not_exists=True) diff --git a/tests/source_revision_fixture.py b/tests/source_revision_fixture.py index 637bfa88ea..53fb8190f3 100644 --- a/tests/source_revision_fixture.py +++ b/tests/source_revision_fixture.py @@ -1,10 +1,14 @@ """One real Git source/remote owner for citation tests; no source checkout is required.""" import ast +from collections.abc import Mapping from pathlib import Path import subprocess from typing import ClassVar, Self +from types import MappingProxyType from tempfile import mkdtemp +import tomli_w + from agent_comms.registry_document import RegisteredCommanderProject from agent_comms.repo import Repo from agent_comms.thread_identity import RecordedThreadIncarnation @@ -60,11 +64,42 @@ def git_input(repository: Path, text: str, *arguments: str) -> str: capture_output=True, check=True, timeout=30, ).stdout.strip() - def publish_change(self, message: str, *files: str) -> str: - """Commit fixture files and publish this PR's new head through real Git.""" + @staticmethod + def package_project( + name: str, *, dependencies: tuple[str, ...] = (), python: str = ">=3.10", + extras: dict[str, list[str]] | None = None, scripts: Mapping[str, str] = MappingProxyType({}), + build_system: Mapping[str, str | tuple[str, ...]] = MappingProxyType({ + "requires": ("hatchling",), "build-backend": "hatchling.build", + }), + ) -> str: + """Build the actual src tree, independently of distribution or module spelling.""" + return tomli_w.dumps({ + "build-system": dict(build_system), + "project": {"name": name, "version": "0.0.1", "requires-python": python, + "dependencies": list(dependencies), "optional-dependencies": extras or {}, + "scripts": dict(scripts)}, + "tool": {"hatch": {"build": {"targets": {"wheel": { + "only-include": ["src"], "sources": ["src"], + }}}}}, + }) + + def write_files(self, files: Mapping[Path, str]) -> tuple[str, ...]: + """Real source fixtures share file preparation, not a second source-selection mechanism.""" + for relative, text in files.items(): + destination = self.checkout / relative + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(text) + return tuple(str(path) for path in files) + + def commit_change(self, message: str, *files: str) -> str: + """Commit the current checkout; its caller chooses what ref to publish.""" self.git(self.checkout, 'add', '--', *files) self.git(self.checkout, 'commit', '-m', message) - head = self.git(self.checkout, 'rev-parse', 'HEAD').strip() + return self.git(self.checkout, 'rev-parse', 'HEAD').strip() + + def publish_change(self, message: str, *files: str) -> str: + """Commit fixture files and publish this PR's descendant through real Git.""" + head = self.commit_change(message, *files) self.git(self.checkout, 'push', 'origin', f'HEAD:refs/pull/{self.number}/head') return head @@ -76,7 +111,10 @@ def remote_for(cls, root: Path, repository: RepositoryName) -> Path: return remote @classmethod - def create(cls, root: Path, name: RepositoryName = RepositoryName.from_wire('example/source')) -> Self: + def create( + cls, root: Path, name: RepositoryName = RepositoryName.from_wire('example/source'), *, + baseline_files: Mapping[Path, str] = MappingProxyType({}), + ) -> Self: root.mkdir(parents=True, exist_ok=True) root = Path(mkdtemp(prefix='source-', dir=root)) repository = root / 'repository' @@ -84,23 +122,17 @@ def create(cls, root: Path, name: RepositoryName = RepositoryName.from_wire('exa remote = cls.remote_for(root, name) cls.git(repository, 'init', '--initial-branch=main') cls.git(repository, 'remote', 'add', 'origin', str(remote)) - main_blob = cls.git_input(repository, cls.main_source, - 'hash-object', '-w', '--stdin') - main_tree = cls.git_input(repository, f'100644 blob {main_blob}\towner.py\n', 'mktree') - main = cls.git(repository, 'commit-tree', main_tree, '-m', 'Main declaration').strip() - cls.git(repository, 'update-ref', 'refs/heads/main', main) - branch_blob = cls.git_input(repository, - cls.main_source + 'class Member(Parent):\n pass\n', - 'hash-object', '-w', '--stdin') - new_blob = cls.git_input(repository, '@decorated\nclass BranchOnly:\n pass\n', - 'hash-object', '-w', '--stdin') - tree = cls.git_input(repository, - f'100644 blob {new_blob}\tbranch.py\n100644 blob {branch_blob}\towner.py\n', 'mktree') - branch = cls.git(repository, 'commit-tree', tree, '-p', main, '-m', 'PR declaration').strip() - cls.git(repository, 'update-ref', 'refs/heads/feature', branch) - cls.git(repository, 'push', 'origin', 'main', 'feature:refs/pull/7/head') - cls.git(repository, 'remote', 'set-head', 'origin', '--auto') - cls.git(repository, 'reset', '--hard', 'main') created = cls.of(repository) assert isinstance(created, cls) + created.commit_change('Main declaration', *created.write_files({ + Path('owner.py'): cls.main_source, **baseline_files, + })) + cls.git(repository, 'switch', '-c', 'feature') + created.commit_change('PR declaration', *created.write_files({ + Path('owner.py'): (repository / 'owner.py').read_text() + 'class Member(Parent):\n pass\n', + Path('branch.py'): '@decorated\nclass BranchOnly:\n pass\n', + })) + cls.git(repository, 'push', 'origin', 'main', f'feature:refs/pull/{created.number}/head') + cls.git(repository, 'remote', 'set-head', 'origin', '--auto') + cls.git(repository, 'switch', 'main') return created diff --git a/tests/test_bus_files.py b/tests/test_bus_files.py index 11f16b0386..83311c00f2 100644 --- a/tests/test_bus_files.py +++ b/tests/test_bus_files.py @@ -15,7 +15,7 @@ from agent_comms.field_codec import ClaimId, GoalId, InputId, TurnId from agent_comms.installer.backup import bus_files from agent_comms.locked_store import DeclaredFile, DurableFile, LiveFilesRequest, RootFile, RuntimeFile, SQLiteFile -from agent_comms.native_pi import NativeContextJournal +from agent_comms.native_input_proof import NativeContextJournal from agent_comms.native_session_files import NativeInputProof from agent_comms.recorded_time import RecordedEventTime from agent_comms.repository_name import RepositoryName diff --git a/tests/test_busy_history_loading.py b/tests/test_busy_history_loading.py index db4643f23f..abf8aa9ecd 100644 --- a/tests/test_busy_history_loading.py +++ b/tests/test_busy_history_loading.py @@ -11,6 +11,8 @@ from agent_comms.acp_extension import TranscriptSnapshotUpdate from agent_comms.comms import wire +from agent_comms.notification_reads import LiveNotificationRead +from agent_comms.order_tools import TranscriptPageRead from agent_comms.ui_model.observation import ( ObservationProcess, ReadBusy, ReadNotifications, ReadTranscript, ) @@ -57,15 +59,15 @@ def held_source(comms): async def test_busy_source_keeps_original_waiter_and_other_reads_move(tmp_path): comms, session = saved_history(tmp_path) original = session.read_bytes() - with ObservationProcess.awaited(comms.route) as process: + async with ObservationProcess.awaited(comms.route) as process: with held_source(comms): - answer = ReadTranscript(str(comms.root), "saved", request_id=19).answer(comms) + answer = ReadTranscript(str(comms.root), TranscriptPageRead(thread="saved"), request_id=19).answer(comms) assert isinstance(answer, ReadBusy) assert answer.title == "History loading" - waiting = asyncio.create_task(process.read(ReadTranscript(str(comms.root), "saved"))) - independent = asyncio.create_task(process.read(ReadNotifications(str(comms.root), ()))) + waiting = asyncio.create_task(process.read(ReadTranscript(str(comms.root), TranscriptPageRead(thread="saved")))) + independent = asyncio.create_task(process.read(ReadNotifications(str(comms.root), LiveNotificationRead(())))) async with asyncio.timeout(15): - assert await independent == {} + assert (await independent).outcomes == {} assert not waiting.done() assert len(process.reads) == 1 (request_id,) = process.reads @@ -84,16 +86,16 @@ async def test_busy_source_keeps_original_waiter_and_other_reads_move(tmp_path): async def test_cancelled_busy_read_is_removed_without_resubmission(tmp_path): comms, _session = saved_history(tmp_path) - with ObservationProcess.awaited(comms.route) as process: + async with ObservationProcess.awaited(comms.route) as process: with held_source(comms): - waiting = asyncio.create_task(process.read(ReadTranscript(str(comms.root), "saved"))) + waiting = asyncio.create_task(process.read(ReadTranscript(str(comms.root), TranscriptPageRead(thread="saved")))) await asyncio.sleep(0.15) waiting.cancel() with pytest.raises(asyncio.CancelledError): await waiting assert process.reads == {} async with asyncio.timeout(5): - assert await process.read(ReadNotifications(str(comms.root), ())) == {} + assert (await process.read(ReadNotifications(str(comms.root), LiveNotificationRead(())))).outcomes == {} assert process.reads == {} @@ -112,10 +114,10 @@ def closed(connection): monkeypatch.setattr(Connection, "close", closed) with pytest.raises(type(error)) as raised: - with process: + async with process: raise error assert raised.value is error - process.close() + await process.close() assert closes == [process.connection] assert process.connection.closed and process.process.exitcode == 0 @@ -187,7 +189,8 @@ async def test_peer_write_failure_retires_once_and_cancels_owner_reads(comms): async def test_owner_reply_uses_the_same_peer_retirement(comms): import multiprocessing - from agent_comms.ui_model.observation import ObservationService, ReadOwnerGoal + from agent_comms.order_tools import OwnerGoalRead + from agent_comms.ui_model.observation import ObservationService, ReadOwnerOperation connection, viewer = multiprocessing.get_context("spawn").Pipe() with connection, viewer: @@ -195,7 +198,8 @@ async def test_owner_reply_uses_the_same_peer_retirement(comms): # Owner reads live in the service's task group, which only its run enters. async with service.tasks: # A real missing-owner refusal still has to be sent through the shared endpoint. - request = ReadOwnerGoal(str(comms.root), UnrecordedThreadIncarnation("missing"), request_id=7) + request = ReadOwnerOperation(str(comms.root), UnrecordedThreadIncarnation("missing"), + request_id=7, operation=OwnerGoalRead()) request.receive(service) asking = service.owner_reads[7] viewer.close() @@ -206,8 +210,8 @@ async def test_owner_reply_uses_the_same_peer_retirement(comms): async def test_awaited_notifications_then_immediate_close_retires_successfully(comms): - with ObservationProcess.awaited(comms.route) as process: - assert await process.read(ReadNotifications(str(comms.root), ())) == {} + async with ObservationProcess.awaited(comms.route) as process: + assert (await process.read(ReadNotifications(str(comms.root), LiveNotificationRead(())))).outcomes == {} assert process.connection.closed and process.process.exitcode == 0 assert not process.reads diff --git a/tests/test_checker_stack.py b/tests/test_checker_stack.py new file mode 100644 index 0000000000..c5eb8c63cf --- /dev/null +++ b/tests/test_checker_stack.py @@ -0,0 +1,286 @@ +"""Private full-project projection against real captured Git/forge inputs; no fake resolver.""" + +from collections.abc import Iterator +from datetime import date +from pathlib import Path +import os +import sys +import tomllib + +from packaging.requirements import Requirement +import pytest +import tomli_w + +from agent_comms.head_review import BaseSide, HeadSide +from agent_comms.installer.commands import InstallCommands +from agent_comms.installer.refusals import UpgradeRefused +from agent_comms.installer.runtime_resolution import CorePackage, GitSource, ResolvedLock +from agent_comms.launch_runtime import LaunchRuntime +from agent_comms.repository_name import RepositoryName +from agent_comms.todos import DraftListedPullRequest +from checker_stack import CheckerStack +from github_fixture import GitHubFixture +from integration_source_fixture import MutualSources +from source_revision_fixture import SourceRepository + +pytest_plugins = ("test_holder_standing_rules", "integration_source_fixture") + + +@pytest.mark.parametrize("dependencies,allowed,declared_python", [ + pytest.param((), True, "", id="dependencies0-True"), + pytest.param(("idna==3.10",), False, "", id="dependencies1-False"), + pytest.param((), True, ">=3.14", id="requires-python"), +]) +def test_private_project_preserves_settings_pins_and_refuses_external_migration( + mutual_sources: MutualSources, github_cli: GitHubFixture, tmp_path: Path, + dependencies: tuple[str, ...], allowed: bool, declared_python: str, + monkeypatch: pytest.MonkeyPatch, test_uv: Path, +) -> None: + pair = mutual_sources + # Edit the real source before capture, never rewrite the captured artifact or selectors. + core_project = pair.core.checkout / "pyproject.toml" + document = tomllib.loads(core_project.read_text()) + document["project"]["dependencies"] = list(dependencies) + stack = tomllib.loads((pair.core.checkout / "stack/pyproject.toml").read_text()) + stack["project"]["python"] = "unmodeled convenience spelling" + if declared_python: + stack["project"]["requires-python"] = declared_python + stack["tool"]["uv"]["sources"]["source-client"]["marker"] = "python_version >= '3.10'" + stack["tool"]["uv"]["sources"]["source-client"]["fixture-option"] = {"preserved": True} + stack["tool"]["uv"]["overrides"] = ["unmodeled overrides"] + stack["tool"]["uv"]["constraints"] = ["unmodeled constraints"] + stack["tool"]["uv"]["indexes"] = [{"unmodeled": "indexes"}] + stack["tool"]["fixture"] = {"dates": [date(2026, 10, 28)], "matrix": [{"enabled": False}]} + stack["tool"]["uv"]["index"] = [{"name": "fixture-index", "url": "https://pypi.org/simple", + "explicit": True, "authenticate": "always"}] + stack["tool"]["uv"]["sources"]["conditional-package"] = [ + {"index": "fixture-index", "marker": "sys_platform == 'linux'"}] + head = pair.core.publish_change("Declare this candidate runtime contract", *pair.core.write_files({ + Path("pyproject.toml"): tomli_w.dumps(document), + Path("stack/pyproject.toml"): tomli_w.dumps(stack)})) + stamp = "2026-10-24T00:00:00Z" + github_cli.put(pair.core.github, DraftListedPullRequest( + pair.core.number, "feature", head, stamp, "main", reviewable_times=(stamp,))) + inputs = pair.capture(("P2", "P3"), "full private checker project") + with InstallCommands() as commands: + checker = CheckerStack(checkout=tmp_path / "checker", commands=commands, home=tmp_path, + runtime=LaunchRuntime.running(), source_directory=pair.output, + inputs=inputs, uv_program=test_uv) + original = checker.core / "stack" + before = (original / "pyproject.toml").read_bytes(), (original / "uv.lock").read_bytes() + if allowed: + checker.write_project() + expected = tomllib.loads(before[0].decode()) + expected["tool"]["uv"]["sources"]["agent-comms"] = {"path": str(checker.core), "editable": True} + expected["tool"]["uv"]["sources"]["source-client"] = { + "path": str(HeadSide.checkout(pair.output, pair.client.github)), "editable": True, + "marker": "python_version >= '3.10'", "fixture-option": {"preserved": True}} + + assert tomllib.loads((checker.checkout / "stack/pyproject.toml").read_text()) == expected + assert checker.lock.read_bytes() == before[1] + monkeypatch.setenv("RUNNER_TEMP", str(tmp_path)) + with pytest.raises(ValueError, match="isolated runner bootstrap prefix"): + checker.prepare() + else: + with pytest.raises(ValueError, match="External runtime dependency migration"): + checker.write_project() + assert not checker.checkout.exists() + assert ((original / "pyproject.toml").read_bytes(), (original / "uv.lock").read_bytes()) == before + for side in (BaseSide, HeadSide): + for source in inputs.sources: + side.require_materialized(pair.output, source) + + +@pytest.mark.parametrize("requested", (False, True)) +def test_bootstrap_extra_is_runtime_only_when_the_virtual_root_requests_it( + mutual_sources: MutualSources, github_cli: GitHubFixture, tmp_path: Path, + requested: bool, test_uv: Path, +) -> None: + pair = mutual_sources + pair.core.git(pair.core.checkout, "switch", "main") + project = tomllib.loads((pair.core.checkout / "pyproject.toml").read_text()) + project["project"]["optional-dependencies"] = {"checking": ["idna>=3.9"]} + stack = tomllib.loads((pair.core.checkout / "stack/pyproject.toml").read_text()) + if requested: + stack["project"]["dependencies"] = ["agent-comms[checking]", "source-client"] + pair.core.commit_change("Declare actual baseline root and optional dependency", *pair.core.write_files({ + Path("pyproject.toml"): tomli_w.dumps(project), + Path("stack/pyproject.toml"): tomli_w.dumps(stack)})) + pair.core.git(pair.core.checkout, "push", "origin", "main") + pair.core.git(pair.core.checkout, "switch", "feature") + pair.core.git(pair.core.checkout, "merge", "--no-edit", "main") + project["project"]["optional-dependencies"] = {"checking": ["idna>=3.10"]} + head = pair.core.publish_change("Change only the optional checker dependency", *pair.core.write_files({ + Path("pyproject.toml"): tomli_w.dumps(project)})) + stamp = "2026-10-24T00:00:00Z" + github_cli.put(pair.core.github, DraftListedPullRequest( + pair.core.number, "feature", head, stamp, "main", reviewable_times=(stamp,))) + inputs = pair.capture(("P2", "P3"), "runtime root extras, not bootstrap extras") + with InstallCommands() as commands: + checker = CheckerStack(checkout=tmp_path / "checker", commands=commands, home=tmp_path, + runtime=LaunchRuntime.running(), source_directory=pair.output, inputs=inputs, uv_program=test_uv) + if requested: + with pytest.raises(ValueError, match="External runtime dependency migration"): + checker.require_runtime_compatibility() + else: + checker.require_runtime_compatibility() + before = checker.clients.projects(BaseSide)[pair.core.github] + after = checker.clients.projects(HeadSide)[pair.core.github] + assert before.requirements != after.requirements # Complete metadata was never filtered away. + + +@pytest.fixture +def common_checker( + mutual_sources: MutualSources, github_cli: GitHubFixture, tmp_path: Path, test_uv: Path, +) -> Iterator[CheckerStack]: + """Two genuine independently locked graphs, not hand-built lock/installed claims.""" + pair = mutual_sources + plugin = SourceRepository.create( + tmp_path / "plugin", RepositoryName.from_wire("example/checker-plugin"), baseline_files={ + Path("python package/pyproject.toml"): SourceRepository.package_project("checker-plugin"), + Path("python package/src/checker_plugin.py"): "value = 1\n", + Path("other package/pyproject.toml"): SourceRepository.package_project("checker-plugin"), + Path("other package/src/checker_plugin.py"): "value = 2\n"}) + pair.client.git(pair.client.checkout, "switch", "main") + pair.client.commit_change("Declare runtime-only dependency", *pair.client.write_files({ + Path("pyproject.toml"): pair.client.package_project("source-client", dependencies=( + "charset-normalizer==3.4.0",))})) + pair.client.git(pair.client.checkout, "push", "origin", "main") + pair.client.git(pair.client.checkout, "switch", "feature") + pair.client.git(pair.client.checkout, "merge", "--no-edit", "main") + pair.core.git(pair.core.checkout, "switch", "main") + core = tomllib.loads(pair.core.package_project("agent-comms", + dependencies=("idna>=3.10,<4; python_version >= '3.10'",), extras={ + "checking": ["idna<3.11", "six==1.16.0", "checker-plugin"]})) + plugin_source = {"git": plugin.remote.as_uri(), "rev": plugin.main_commit, "subdirectory": "python package"} + core["tool"]["uv"] = {"sources": {"checker-plugin": plugin_source}} + pair.core.write_files({Path("pyproject.toml"): tomli_w.dumps(core)}) + stack = tomllib.loads((pair.core.checkout / "stack/pyproject.toml").read_text()) + stack["tool"]["uv"]["sources"]["checker-plugin"] = plugin_source + stack["tool"]["uv"]["sources"]["source-client"]["git"] = pair.client.remote.as_uri() + stack["project"]["dependencies"] = ["agent-comms>=0.0.1; python_version >= '3.10'", "source-client"] + stack["tool"]["uv"]["override-dependencies"].append("agent-comms>=0.0.1; python_version >= '3.10'") + pair.core.write_files({Path("stack/pyproject.toml"): tomli_w.dumps(stack)}) + with InstallCommands() as commands: + for project in (pair.core.checkout, pair.core.checkout / "stack"): + commands.run(test_uv, "lock", "--project", project, "--python", Path(sys.executable)) + pair.core.commit_change("Retain actual independent bootstrap and runtime locks", "pyproject.toml", + "stack/pyproject.toml", "uv.lock", "stack/uv.lock") + pair.core.git(pair.core.checkout, "push", "origin", "main") + pair.core.git(pair.core.checkout, "switch", "feature") + pair.core.git(pair.core.checkout, "merge", "--no-edit", "main") + stamp = "2026-10-24T00:00:00Z" + for source in (pair.core, pair.client): + source.git(source.checkout, "push", "origin", f"HEAD:refs/pull/{source.number}/head") + github_cli.put(source.github, DraftListedPullRequest( + source.number, "feature", source.branch_commit, stamp, "main", reviewable_times=(stamp,))) + inputs = pair.capture(("P2", "P3"), "one complete runtime/checker solve") + checker = CheckerStack(checkout=tmp_path / "checker", commands=commands, home=tmp_path, + runtime=LaunchRuntime.running(), source_directory=pair.output, inputs=inputs, uv_program=test_uv) + checker.write_project() + yield checker + + +def test_complete_project_reconciles_shared_preferences_and_keeps_both_cuts_on_one_prefix( + common_checker: CheckerStack, +) -> None: + checker = common_checker + originals = {path: path.read_bytes() for side in (BaseSide, HeadSide) + for path in (side.checkout(checker.source_directory, CorePackage.repository) / "uv.lock", + side.checkout(checker.source_directory, CorePackage.repository) / "stack/uv.lock")} + bootstrap = checker.bootstrap_lock.external(checker.local_names) + runtime = checker.runtime_locks[HeadSide].external(checker.local_names) + assert bootstrap["idna"].version != runtime["idna"].version + metadata = checker.clients.projects(HeadSide)[CorePackage.repository] + private = tomllib.loads((checker.checkout / "stack/pyproject.toml").read_text()) + for roots in (private["project"]["dependencies"], private["tool"]["uv"]["override-dependencies"]): + core, = (Requirement(text) for text in roots if Requirement(text).name == metadata.name) + assert core.extras == metadata.extras and str(core.specifier) == ">=0.0.1" + assert core.marker is not None and "python_version" in str(core.marker) + assert not any(edge.name == "six" for edge in checker.requirements(HeadSide)) + assert any(edge.name == "six" for edge in checker.private_requirements()) + assert any(edge.marker is not None for edge in checker.private_requirements()) + source = private["tool"]["uv"]["sources"]["checker-plugin"] + assert source["subdirectory"] == "python package" + override, = (Requirement(text) for text in private["tool"]["uv"]["override-dependencies"] + if Requirement(text).name == "checker-plugin") + assert override.url is None and not override.specifier + assert source == bootstrap["checker-plugin"].source.project_source().document() + for query in ("unknown=python", "subdirectory=python&subdirectory=other", "subdirectory=", + "branch=main&tag=release", "subdirectory=%00", "subdirectory=%ZZ", "branch="): + with pytest.raises(ValueError, match="Git"): + GitSource(git=f"{source['git']}?{query}#{source['rev']}").project_source() + for revision in ("", "a" * 7, "Z" * 40): + with pytest.raises(ValueError, match="Git"): + GitSource(git=f"{source['git']}?branch=main#{revision}").project_source() + checker.resolve() + locked = ResolvedLock.read(checker.lock).external(checker.local_names) + assert locked["idna"].version == bootstrap["idna"].version + for name, package in checker.retained_packages.items(): + package.require_retained(locked[name]) + prefix = checker.home / "common-prefix" + checker.commands.run(checker.uv, "venv", "--python", checker.python, prefix) + checker.commands.run(checker.uv, "sync", "--project", checker.checkout / "stack", "--locked", + "--no-editable", "--python", checker.python, + env={**os.environ, "UV_PROJECT_ENVIRONMENT": str(prefix)}) + checker.commands.run(checker.uv, "pip", "check", "--python", prefix / "bin/python") + for side, answer in ((BaseSide, "baseline"), (HeadSide, "candidate")): + roots = [str(side.checkout(checker.source_directory, source.repository) / "src") + for source in checker.inputs.sources] + # Both real API cuts execute with this same installed dependency prefix. + result = checker.commands.run(prefix / "bin/python", "-c", + "import runpy, sys, idna; sys.path[:0] = sys.argv[1:-1]; " + "print(sys.prefix); print(idna.__version__); print(runpy.run_path(sys.argv[-1])['run']())", + *roots, side.checkout(checker.source_directory, RepositoryName.from_wire("example/client")) + / "src/workflow.py") + assert result.splitlines() == [str(prefix), locked["idna"].version, answer] + assert {path: path.read_bytes() for path in originals} == originals + + +def test_incompatible_full_root_graph_refuses_during_solve(common_checker: CheckerStack) -> None: + checker = common_checker + project = checker.checkout / "stack/pyproject.toml" + document = tomllib.loads(project.read_text()) + document["project"]["dependencies"].append("idna>=3.11") + project.write_text(tomli_w.dumps(document)) + with pytest.raises(UpgradeRefused): + checker.resolve() + assert not (checker.home / "common-prefix").exists() # No sync or corrective install happened. + + +@pytest.mark.parametrize("package,change", ( + ("charset-normalizer", "3.4.1"), ("six", "1.17.0"), + ("checker-plugin", "revision"), ("checker-plugin", "location"), +)) +def test_retained_runtime_checker_version_and_git_identity_refuse_actual_solve_drift( + common_checker: CheckerStack, package: str, change: str, +) -> None: + checker = common_checker + expected = checker.retained_packages[package] + project = checker.checkout / "stack/pyproject.toml" + document = tomllib.loads(project.read_text()) + uv = document["tool"]["uv"] + uv["constraint-dependencies"] = [text for text in uv["constraint-dependencies"] + if Requirement(text).name != package] + if change in ("revision", "location"): + # Move only the genuine external repository and solve its new exact source. The + # retained canonical lock must reject a same-version, different-commit package. + source = uv["sources"][package] + remote = Path(source["git"].removeprefix("file://")) + repository = SourceRepository.of(remote.parents[1] / "repository") + if change == "revision": + commit = repository.commit_change("Same version, different checker implementation", *repository.write_files({ + Path("python package/src/checker_plugin.py"): "value = 3\n"})) + repository.git(repository.checkout, "push", "origin", "main") + source["rev"] = commit + else: + source["subdirectory"] = "other package" # Same repo/SHA/version, another actual project. + else: + # Changing the explicit private override is not an accepted waiver: the retained + # source/version owner still detects the resulting genuine solver selection. + uv["override-dependencies"].append(f"{package}=={change}") + project.write_text(tomli_w.dumps(document)) + checker.resolve() + actual = ResolvedLock.read(checker.lock).external(checker.local_names)[package] + with pytest.raises(ValueError, match="changed retained source/version"): + expected.require_retained(actual) diff --git a/tests/test_codex_earlier_pages.py b/tests/test_codex_earlier_pages.py index 84a0c507f8..a60e0f258e 100644 --- a/tests/test_codex_earlier_pages.py +++ b/tests/test_codex_earlier_pages.py @@ -165,7 +165,7 @@ def test_hidden_page_items_keep_import_events_but_own_no_visible_events(tmp_path async def test_draining_server_gets_the_same_turn_after_reconnecting(tmp_path): """Codex refuses a turn while draining and asks for it again; the input is delivered, not lost.""" - from agent_comms.cli_protocol import DeliveredOutcome + from agent_comms.harness_delivery_states import AcceptedOutcome from agent_comms.messages import InfoKind, Message turns = [] @@ -202,7 +202,7 @@ async def serve(request): options = CodexOptions(Watch(), StoredBinding(), tmp_path / 'bridge', tmp_path, str(socket_path)) bridge = CodexBridge(options, binding.address(), db) message = Message(sender='sender', target='cli', body='keep going', type=InfoKind) - assert await bridge.deliver(message) is DeliveredOutcome + assert await bridge.deliver(message) is AcceptedOutcome assert len(turns) == 2 and turns[0] == turns[1] assert turns[0]['clientUserMessageId'] == f'comms:{message.message_id}' finally: diff --git a/tests/test_command_families.py b/tests/test_command_families.py index 99ce697d4b..70f9f231ae 100644 --- a/tests/test_command_families.py +++ b/tests/test_command_families.py @@ -1,5 +1,6 @@ from agent_comms.repo import Checkout -from agent_comms.field_codec import FieldCodec, GoalId +from agent_comms.declared_family import UnknownFamilyMember +from agent_comms.field_codec import FieldCodec, GoalId, MalformedRecord """External contracts and adding commands without editing any consumer catalog.""" @@ -86,10 +87,13 @@ def parse(args): {"action": "cancel"}, {"action": "set_config_option", "config_id": "model", "value": "test/model"}, {"action": "compact", "instructions": "focus"}, - {"action": "input_dispositions", "include_history": True}, + {"action": "owner_read", "operation": { + "tool": "comms_input_delivery", "thread": "", "include_history": True}}, + {"action": "owner_read", "operation": { + "tool": "comms_input_delivery", "thread": "", "include_history": False}}, {"action": "dismiss_historical_inputs"}, {"action": "goal_history", "goal_id": None}, - {"action": "goal_snapshot"}, + {"action": "owner_read", "operation": {"tool": "comms_owner_goal", "thread": ""}}, {"action": "edit_goal", "goal_id": None, "expected_revision": 3, "text": "new goal"}, {"action": "update_goal", "goal_id": None, "expected_revision": 3, "status": "paused"}, {"action": "retry_goal", "goal_id": None, "expected_revision": 3}, @@ -117,7 +121,9 @@ def test_runtime_wire_format_round_trip(parameters): @pytest.mark.parametrize( "parameters", [ - {"action": "input_dispositions", "include_history": "yes"}, + {"action": "owner_read", "operation": { + "tool": "comms_input_delivery", "include_history": "yes"}}, + {"action": "owner_read", "operation": {"tool": "comms_goal_history"}}, {"action": "goal_history", "goal_id": 2}, {"action": "edit_goal", "goal_id": None, "expected_revision": True, "text": "x"}, {"action": "retry_goal"}, @@ -138,6 +144,19 @@ def test_invalid_runtime_parameters_use_canonical_decode_and_domain_errors(param assert str(refused.value) == str(declared.value) +@pytest.mark.parametrize("parameters", [ + {"action": "goal_snapshot"}, + {"action": "goal_snapshot", "operation": {"tool": "comms_owner_goal", "thread": ""}}, +], ids=["bare", "with_operation"]) +def test_the_deleted_goal_snapshot_command_is_an_undeclared_member(parameters): + with pytest.raises(UnknownFamilyMember) as undeclared: + RuntimeRequest.member_named("goal_snapshot") + with pytest.raises(MalformedRecord) as refused: + FieldCodec.decode(RuntimeRequest, {"thread": "owner", **parameters}) + assert type(refused.value.__cause__) is UnknownFamilyMember + assert str(refused.value) == str(refused.value.__cause__) == str(undeclared.value) + + @pytest.mark.parametrize( "arguments,result_type,per_root", [ diff --git a/tests/test_commander_notifications.py b/tests/test_commander_notifications.py index 4aecf0582b..88bad981c3 100644 --- a/tests/test_commander_notifications.py +++ b/tests/test_commander_notifications.py @@ -28,7 +28,12 @@ from agent_comms.todos import ProgressRead from agent_comms.ui_model import observation from agent_comms.ui_model.observation import CommanderHudObserved, ObservationService, ObserveCommanderHud -from agent_comms.ui_model.observation import ReadCommanderUnit, ReadRefused +from agent_comms.field_codec import FieldCodec +from agent_comms.order_tools import CommanderUnitRead +from agent_comms.tool_caller import ThreadCaller +from agent_comms.tools import invoke_tool +from agent_comms.work_branches import WorkBranchError +from agent_comms.ui_model.observation import ReadOperation, ReadRefused from agent_comms.cli_bridges.codex_connection import CodexConnection from agent_comms.standing_rules import ServicePassFailing, StandingRule, UndatedStandingRule from agent_comms.ui_model.unit_state import ( @@ -88,9 +93,16 @@ def test_expanded_unit_reads_original_orders_and_bounded_activity_then_refuses_s hud = beat(comms, states=state_cards(comms, UnitState)) ServicePassStore.at(comms.root).heartbeat_passed((hud.commander,), (hud.commander,)) card = next(card for card in hud.units if card.unit.name == 'worker') - request = ReadCommanderUnit(str(comms.root), hud.commander, card.unit, - tuple(dict.fromkeys(order.order for order in card.orders))) + operation = CommanderUnitRead(commander=hud.commander, unit=card.unit, + orders=tuple(dict.fromkeys(order.order for order in card.orders))) + request = ReadOperation(str(comms.root), operation) + direct = operation.read(comms) detail = pickle.loads(pickle.dumps(request.read(comms))) + # The shared operation, the UI's root-bound envelope and the MCP boundary answer the same + # selected card with the same bounded records. + assert detail == direct + assert invoke_tool(comms, CommanderUnitRead.declared_name, FieldCodec.encode_fields(operation), + caller=ThreadCaller("lead")) == FieldCodec.encode(direct) assert detail.unit == card assert {order.card.problem for order in detail.orders} == {'P1', 'P2'} for order in detail.orders: @@ -102,10 +114,10 @@ def test_expanded_unit_reads_original_orders_and_bounded_activity_then_refuses_s assert 'contents' not in json.dumps(detail.activity) assert (comms.root / 'state-source.jsonl').read_bytes() == original for stale in ( - replace(request, orders=()), + replace(request, operation=replace(operation, orders=())), replace(request, root=str(comms.root / 'other')), - replace(request, unit=replace(card.unit, created_at=card.unit.wire_birth + 1)), - replace(request, commander=card.unit), + replace(request, operation=replace(operation, unit=replace(card.unit, created_at=card.unit.wire_birth + 1))), + replace(request, operation=replace(operation, commander=card.unit)), ): assert isinstance(stale.answer(comms), ReadRefused) checkout = comms.goals.todos.get('P1').order.checked().check.directory @@ -115,6 +127,10 @@ def test_expanded_unit_reads_original_orders_and_bounded_activity_then_refuses_s try: metadata.write_text('gitdir: ./missing-original-git\n') assert isinstance(request.answer(comms), ReadRefused) + # MCP reaches the same nominal refusal, which the connector answers as a failed call. + with pytest.raises(WorkBranchError): + invoke_tool(comms, CommanderUnitRead.declared_name, FieldCodec.encode_fields(operation), + caller=ThreadCaller("lead")) finally: metadata.unlink() saved.rename(metadata) diff --git a/tests/test_commander_skill_hook.py b/tests/test_commander_skill_hook.py index 7064847c57..b60ab91274 100644 --- a/tests/test_commander_skill_hook.py +++ b/tests/test_commander_skill_hook.py @@ -289,13 +289,13 @@ def test_the_board_prints_changes_once_then_an_unchanged_count(notification_bus, assert DeliveredBoardContent.count(db) == 0 -def test_the_deck_carries_the_voice_every_fourth_slice_every_drift_tell_and_every_failure(): +def test_the_deck_carries_the_voice_every_fourth_slice_every_rule_reminder_and_every_failure(): slices = CommanderReader.slices() voice = DeckReader.paragraphs(CommanderPackage.voice().read_text()) indictment = DeckReader.paragraphs(CommanderPackage.indictment().read_text()) assert set(slices[0::4]) == set(voice) assert set(slices[1::4]) == set(indictment) - assert set(slices[2::4]) == set(RuleFile.drift_tells()) + assert set(slices[2::4]) == {rule.reminder for rule in RuleFile.packaged()} assert set(slices[3::4]) == { f"Recorded failure {line}" for line in PostmortemFailure.catalog().splitlines()} assert all(len(text) <= DeckReader.slice_chars for text in slices) @@ -361,8 +361,20 @@ def test_a_unit_session_gets_the_fleet_deck_and_no_commander_reminders( @pytest.mark.parametrize("path", CommanderPackage.rules_files(), ids=lambda path: path.name) -def test_package_rule_has_a_reason(path): - assert RuleFile.read(path).why +def test_package_rule_opens_with_its_wakeup_then_its_reason(path): + rule = RuleFile.read(path) + assert rule.why and rule.wakeup.startswith(">") + assert rule.render().startswith(rule.wakeup + "\n\n" + rule.summary) + assert rule.reminder.startswith(rule.wakeup) and rule.reminder.endswith(rule.drift) + + +def test_rule_file_without_a_wakeup_is_refused(tmp_path): + source = CommanderPackage.rules_files()[0].read_text(encoding="utf-8") + path = tmp_path / "asleep.md" + path.write_text("".join(line for line in source.splitlines(keepends=True) + if not line.startswith(">")), encoding="utf-8") + with pytest.raises(ValueError, match="wake-up"): + RuleFile.read(path) def test_package_descriptions_do_not_repeat_rule_kind(): diff --git a/tests/test_connector.py b/tests/test_connector.py index 6e8e8ee011..e238f4afab 100644 --- a/tests/test_connector.py +++ b/tests/test_connector.py @@ -22,6 +22,7 @@ from aiohttp.test_utils import TestClient, TestServer from connector_readiness import ConnectorReadiness +from conftest import transcript_records from agent_comms.bus import Bus from agent_comms.comms import Comms, wire @@ -32,22 +33,27 @@ ConnectorVerb, ServeVerb, RevokeVerb, + ToolsCallResult, ) from agent_comms.connector_caller import CALLER, DirectorCaller from agent_comms.connector_store import ( CALLBACK, CODE_SECONDS, AuthorizationQuery, ConnectorGrants, ConnectorStore, IssuedCode, TrustedBrowser, digest, ) +from agent_comms.coordination_errors import TranscriptChanged from agent_comms.coordinator import Coordination from agent_comms.locked_store import DocumentChange from agent_comms.message_bus import WatchPass from agent_comms.service_passes import ServicePassStore from agent_comms.standing_rules import ServicePassFailing -from agent_comms.field_codec import FieldCodec +from agent_comms.field_codec import FieldCodec, MalformedRecord from agent_comms.host_services import ServiceNotServingError from agent_comms.host_services.connector import ConnectorService from agent_comms.installer.backup import StoreBackup from agent_comms.launch_root import LaunchRoot +from agent_comms.native_transcript import NativeTranscript +from agent_comms.order_tools import TranscriptPageRead +from agent_comms.session_file import RegistrySessionLocation from agent_comms.maintenance_barrier import ( InstallingPhase, MaintenanceBarrier, MaintenanceHeldError, MaintenanceOperator, PausedPhase, ReadyPhase, @@ -269,6 +275,65 @@ async def test_expected_tool_argument_refusal_keeps_its_mcp_reply(client, comms, assert json.loads(reply["result"]["content"][0]["text"])["error"] +async def tool_call(client: TestClient, token: str, call_id: int, name: str, arguments: dict): + response = await client.post("/mcp", headers={"Authorization": f"Bearer {token}"}, json={ + "jsonrpc": "2.0", "id": call_id, "method": "tools/call", + "params": {"name": name, "arguments": arguments}, + }) + return response.status, await response.json() + + +def bind_worker_session(comms: Comms, path: Path) -> None: + worker = comms.registry.require("worker") + comms.registry.declare(replace( + worker, session_file=RegistrySessionLocation.from_path(str(path), comms.root))) + + +def saved_session(path: Path, *entries: bytes) -> Path: + header = json.dumps(transcript_records.session(id="session").to_wire()).encode() + path.write_bytes(b"".join((header + b"\n", *entries))) + return path + + +async def test_an_unreadable_transcript_is_a_failed_call_and_the_connector_keeps_serving( + client, comms, tmp_path, +): + """A corrupt complete native entry escaped tools/call as an HTTP 500 with no id.""" + path = saved_session(tmp_path / "corrupt.jsonl", b'{"type":"message"\n') + bind_worker_session(comms, path) + with pytest.raises(MalformedRecord) as unreadable: + tuple(NativeTranscript(path).tail()) + token = await approved_token(client, comms) + status, reply = await tool_call( + client, token, 41, TranscriptPageRead.declared_name, {"thread": "worker"}) + assert (status, reply["id"]) == (200, 41) + # The domain's own MalformedRecord, with its JSONDecodeError cause, is the failed result. + assert reply["result"] == FieldCodec.encode(ToolsCallResult.failed(unreadable.value)) + assert reply["result"]["isError"] is True and "structuredContent" not in reply["result"] + assert "JSONDecodeError" in reply["result"]["content"][0]["text"] + status, served = await tool_call(client, token, 42, CommsChannelsTool.declared_name, {}) + assert (status, served["id"], served["result"]["isError"]) == (200, 42, False) + + +async def test_a_stale_transcript_cursor_is_a_failed_call_with_its_coordination_cause( + client, comms, tmp_path, +): + message = transcript_records.message({"role": "assistant", "content": [{"type": "text", "text": "held"}]}) + bind_worker_session(comms, saved_session( + tmp_path / "session.jsonl", json.dumps(message).encode() + b"\n")) + page = comms.transcripts.capture_page_read("worker").read() + bind_worker_session(comms, saved_session(tmp_path / "other.jsonl")) + stale = TranscriptPageRead(thread="worker", before=page.before) + with pytest.raises(TranscriptChanged) as changed: + stale.read(comms) + token = await approved_token(client, comms) + status, reply = await tool_call( + client, token, 43, stale.declared_name, FieldCodec.encode_fields(stale)) + assert (status, reply["id"]) == (200, 43) + assert reply["result"] == FieldCodec.encode(ToolsCallResult.failed(changed.value)) + assert "structuredContent" not in reply["result"] + + async def test_request_log_names_the_route_and_never_its_values(client, store, root, comms, caplog): """Serve actual GET/POST approval secrets, with aiohttp's raw access log off.""" login = await ClientLogin.register(client) diff --git a/tests/test_deleted_thread_observation.py b/tests/test_deleted_thread_observation.py index 490fa29e19..0abcc884ad 100644 --- a/tests/test_deleted_thread_observation.py +++ b/tests/test_deleted_thread_observation.py @@ -11,12 +11,13 @@ from agent_comms.registry_document import DEFAULT_ROSTER from agent_comms.registry_provenance import DeletedThread from agent_comms.thread_presentation import NoThreadPresentation, ThreadRetired +from agent_comms.notification_reads import LiveNotificationRead from agent_comms.ui_model.observation import ( ObservationProcess, ObserveThreads, ObserveViews, ReadNotifications, ThreadPresented, ThreadsObserved, ViewsRetired, ) from agent_comms.ui_model.status import OpenThreadStatus -from test_observer_death import readable +from observed_queue import ObservedQueue from agent_comms.thread_execution import ExternalThreadExecution from agent_comms.thread_text import GivenThreadText from agent_comms.threads import Thread @@ -80,47 +81,34 @@ async def test_observer_process_survives_deleted_interest_and_keeps_serving(comm declare(comms, 'surviving', 'deleted') original = comms.registry.require('deleted').incarnation names = frozenset({'surviving', 'deleted'}) - with ObservationProcess(comms.route) as process: + delivered = ObservedQueue() + async with ObservationProcess(comms.route) as process: + process.start_receiving(delivered.observed, delivered.failed) process.request(ObserveThreads(names)) process.request(ObserveViews(frozenset({original}), frozenset())) - - async def until(accepted): - async with asyncio.timeout(30): - while True: - await readable(process) - for result in process.results(): - if accepted(result): - return result - - await until(lambda result: isinstance(result, ThreadsObserved) and result.changed == names) + await delivered.until(lambda result: isinstance(result, ThreadsObserved) and result.changed == names) comms.registry.remove('deleted') missing = None retired = None async with asyncio.timeout(30): while missing is None or retired is None: - await readable(process) - for result in process.results(): - if isinstance(result, ThreadsObserved) and 'deleted' in result.changed: - missing = result - elif isinstance(result, ViewsRetired): - retired = result + result = await delivered.next() + if isinstance(result, ThreadsObserved) and 'deleted' in result.changed: + missing = result + elif isinstance(result, ViewsRetired): + retired = result assert missing.activity['deleted'] == DeletedThread('deleted') assert isinstance(missing.results['deleted'], ThreadPresented) assert missing.results['deleted'].presentation == NoThreadPresentation() assert retired.retired.threads == frozenset({original}) surviving = comms.registry.require('surviving') comms.registry.declare(replace(surviving, title=GivenThreadText('Still updating after deletion'))) - updated = await until(lambda result: isinstance(result, ThreadsObserved) - and 'surviving' in result.changed) + updated = await delivered.until(lambda result: isinstance(result, ThreadsObserved) + and 'surviving' in result.changed) assert isinstance(updated.activity['surviving'], ObservedActivity) assert process.process.is_alive() - loop = asyncio.get_running_loop() - loop.add_reader(process.fileno(), process.results) - try: - async with asyncio.timeout(30): - assert await process.read(ReadNotifications(str(comms.root), ())) == {} - finally: - loop.remove_reader(process.fileno()) + async with asyncio.timeout(30): + assert (await process.read(ReadNotifications(str(comms.root), LiveNotificationRead(())))).outcomes == {} def test_one_revision_captures_each_thread_activity_once(comms): diff --git a/tests/test_following_context_notifications.py b/tests/test_following_context_notifications.py new file mode 100644 index 0000000000..157a1844ea --- /dev/null +++ b/tests/test_following_context_notifications.py @@ -0,0 +1,265 @@ +"""Following proof uses real session/journal/input rows, without a native/provider run. + +These controls exercise the shared event verifier and durable transitions. The +actual TrackedTurnSession event-loop/transport qualification remains root-owned. +""" +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager, closing +from dataclasses import dataclass, field, replace +from pathlib import Path +import sqlite3 + +import pytest + +from agent_comms.agent_backend import InputContent, InputRequest, SteerInputPolicy +from agent_comms.coordination_errors import IdentityConflict +from agent_comms.coordinator import Coordination +from agent_comms.errors import RelationViolationError +from agent_comms.field_codec import InputId +from agent_comms.lifecycle_capability import Unresolved +from agent_comms.input_disposition import InputDispositions +from agent_comms.input_key import InputKey +from agent_comms.input_origin import WireInputOrigin +from agent_comms.native_failures import NativePiUnavailable +from agent_comms.native_input_proof import NativeContextJournal, NativeContextProof +from agent_comms.native_tool_call import OwnerToolConnection +from agent_comms.pi_commands import FollowingPrompt +from agent_comms.backend import ObservedNativeCommit, TrackedInputContext +from agent_comms.turn_lease import TurnLeaseFence +from agent_comms.native_runtime_input import NativeRuntimeInput +from agent_comms.pi_events import ContextCommitted, InputCommitted +from agent_comms.private_send_claim import PrivateSendClaim +from agent_comms.private_sidecar import native_request_digest +from conftest import accept_initial, publish_initial +from native_proof_cases import append_input_entry +from test_coordinated_runtime import tmp_path # noqa: F401 +from test_live_context_notifications import acknowledge_context, recorded_input_context +from test_native_failure_recovery import prepared_native_input + + +@asynccontextmanager +async def following_session(directory: Path) -> AsyncIterator[PrivateSendClaim]: + async with prepared_native_input(directory) as claim: + await acknowledge_context(claim, await recorded_input_context(claim)) + yield claim + + +@dataclass(frozen=True) +class FollowingContext: + claim: PrivateSendClaim + keys: tuple[InputKey, ...] + text: str + input_event: InputCommitted + context_event: ContextCommitted + public_id: InputId = field(default_factory=InputId.mint) + + @property + def dispositions(self) -> InputDispositions: + return InputDispositions(self.claim.selected.comms.root) + + @property + def lease(self) -> TurnLeaseFence: + return self.claim.owner.thread.require_turn_lease() + + def start(self) -> None: + for key in self.keys: + assert self.dispositions.started( + key, turn_id=self.lease.turn_id, native_id=self.input_event.input_id, + text=self.text, + ) + + @property + def command(self) -> FollowingPrompt: + return FollowingPrompt( + input=InputRequest(input_id=self.public_id, content=InputContent(text=self.text), + when_busy=SteerInputPolicy), + input_id=self.input_event.input_id, message=self.text, + streaming_behavior=SteerInputPolicy, + # This command-data control never starts a tool transport or connects here. + owner_tools=OwnerToolConnection(str(self.claim.session.identity.path) + '.unused', 'unused'), + ) + + async def verify(self, event: ContextCommitted | None = None) -> NativeContextProof: + observed = TrackedInputContext( + self.command, ObservedNativeCommit(self.input_event), + ObservedNativeCommit(event or self.context_event), + ) + async with self.claim.selected.cursor_delivery.reading() as reads: + return observed.proof( + self.claim.session.files(self.claim.selected.comms.root), + self.input_event.session_id, reads, + ) + + def acknowledge(self, context: NativeContextProof) -> None: + with self.claim.selected.comms.registry.store.current_snapshot() as cut: + self.dispositions.context_committed( + self.keys, lease=self.lease, sent_text=self.text, + context=context, snapshot=cut.snapshot, + ) + + def append_generation(self, generation: int) -> FollowingContext: + files = self.claim.session.files(self.claim.selected.comms.root) + with closing(sqlite3.connect(files.input_proof)) as db, db: + original = NativeContextJournal.for_input( + db, self.input_event.input_id, self.context_event.request_generation, + ) + assert original is not None + later = replace(original, request_generation=generation, + committed_at_ms=original.committed_at_ms + 1, + llm_context_digest='b' * 64) + later.insert(db) + return replace(self, context_event=replace( + self.context_event, request_generation=later.request_generation, + llm_context_digest=later.llm_context_digest, committed_at_ms=later.committed_at_ms, + )) + + +def following_input(claim: PrivateSendClaim, *, generation: int, count: int = 1) -> FollowingContext: + """Reserve and bind originals through InputDispositions, then retain native evidence. + + No message/start/context event is inferred from another input or a reply. + """ + comms, owner = claim.selected.comms, claim.owner.thread + lease = owner.require_turn_lease() + messages = tuple(publish_initial(comms, 'sender', owner.name, + f'Following original {generation}/{index}') + for index in range(count)) + with Coordination(claim.store_path) as store: + for message in messages: + assert accept_initial(comms.bus, claim.selected.root_id, message.seq, store) + keys = tuple(InputDispositions.bus_key(message.message, owner) for message in messages) + text = '\n'.join(message.message.body for message in messages) + dispositions = InputDispositions(comms.root) + for key, message in zip(keys, messages, strict=True): + assert dispositions.record( + key, seq=message.seq, owner=owner.name, claim_scope=lease.claimed_generation, + target=owner.name, text=message.message.body, + origin=WireInputOrigin(claim.selected.root_id, message.message.reference), + ) + native_id = InputId.mint() + assert dispositions.bind_originals( + keys, claim_scope=lease.claimed_generation, turn_id=lease.turn_id, + native_id=native_id, text=text, + ) + session = claim.session.identity + entry = f'following-{native_id}' + start = append_input_entry(session.path, { + 'type': 'message', 'id': entry, 'timestamp': '2026-10-01T00:00:00.002Z', + 'message': {'role': 'user', 'inputId': native_id, + 'inputDigest': native_request_digest(text, streaming_behavior=SteerInputPolicy), + 'content': [{'type': 'text', 'text': text}]}, + }) + recorded = NativeContextJournal( + input_id=native_id, session_id=session.session_id, session_entry_id=entry, + request_generation=generation, llm_context_digest='a' * 64, + committed_at_ms=1_791_964_800_001 + generation, + schema=2, type='context_committed', + ) + with closing(sqlite3.connect(claim.session.files(comms.root).input_proof)) as db, db: + recorded.insert(db) + return FollowingContext( + claim, keys, text, + InputCommitted(input_id=native_id, session_id=session.session_id, + session_entry_id=entry, session_entry_start=start), + ContextCommitted(input_id=native_id, session_id=session.session_id, + session_entry_id=entry, request_generation=generation, + llm_context_digest=recorded.llm_context_digest, + committed_at_ms=recorded.committed_at_ms), + ) + + +async def test_two_following_ids_and_grouped_keys_keep_their_own_first_proof(tmp_path: Path): + async with following_session(tmp_path) as claim: + first = following_input(claim, generation=2, count=2) + second = following_input(claim, generation=3) + first.start() + second.start() + assert first.input_event.input_id != second.input_event.input_id != claim.input_id + assert all(not row.model_contexts() for row in first.dispositions.read( + first.keys + second.keys).rows.values()) + proof = await first.verify() + first.acknowledge(proof) + for row in first.dispositions.read(first.keys).rows.values(): + assert row.has_started and not row.items_with(Unresolved, row) + assert row.model_contexts() == (proof,) + assert all(not row.model_contexts() for row in second.dispositions.read(second.keys).rows.values()) + second_proof = await second.verify() + second.acknowledge(second_proof) + assert all(row.model_contexts() == (second_proof,) + for row in second.dispositions.read(second.keys).rows.values()) + with Coordination(claim.store_path) as store: + original = NativeRuntimeInput.require(store.session._connection, input_id=claim.input_id) + assert original.require_context_proof().input_id == claim.input_id + assert len(NativeRuntimeInput.select(store.session._connection)) == 1 + assert claim.stage.progress.current.starting # No terminal answer was supplied. + + +async def test_one_emitted_event_alone_cannot_prove_context(tmp_path: Path): + async with following_session(tmp_path) as claim: + case = following_input(claim, generation=2) + case.start() + for incomplete in ( + TrackedInputContext(case.command, input_commit=ObservedNativeCommit(case.input_event)), + TrackedInputContext(case.command, context_commit=ObservedNativeCommit(case.context_event)), + ): + async with claim.selected.cursor_delivery.reading() as reads: + with pytest.raises(NativePiUnavailable): + incomplete.proof(claim.session.files(claim.selected.comms.root), + case.input_event.session_id, reads) + assert all(not row.model_contexts() for row in case.dispositions.read(case.keys).rows.values()) + + +async def test_foreign_journal_event_and_mixed_key_batch_refuse_without_partial_write(tmp_path: Path): + async with following_session(tmp_path) as claim: + first = following_input(claim, generation=2) + second = following_input(claim, generation=3) + first.start() + second.start() + with pytest.raises(NativePiUnavailable): + await first.verify(second.context_event) + with pytest.raises(NativePiUnavailable): + await first.verify(replace(first.context_event, llm_context_digest='c' * 64)) + proof = await first.verify() + with claim.selected.comms.registry.store.current_snapshot() as cut: + with pytest.raises(RelationViolationError): + first.dispositions.context_committed( + first.keys + second.keys, lease=first.lease, sent_text=first.text, + context=proof, snapshot=cut.snapshot, + ) + assert all(not row.model_contexts() for row in first.dispositions.read( + first.keys + second.keys).rows.values()) + + +async def test_duplicate_and_later_generation_keep_first_proof_and_no_replay(tmp_path: Path): + async with following_session(tmp_path) as claim: + case = following_input(claim, generation=2) + case.start() + proof = await case.verify() + case.acknowledge(proof) + case.acknowledge(proof) + later = case.append_generation(3) + later.acknowledge(await later.verify()) + assert all(row.model_contexts() == (proof,) for row in case.dispositions.read(case.keys).rows.values()) + with pytest.raises(IdentityConflict): + case.acknowledge(replace(proof, llm_context_digest='c' * 64)) + assert not case.dispositions.bind_originals( + case.keys, claim_scope=case.lease.claimed_generation, turn_id=case.lease.turn_id, + native_id=InputId.mint(), text=case.text, + ) + + +async def test_cancelled_input_or_released_lease_cannot_acquire_context(tmp_path: Path): + async with following_session(tmp_path) as claim: + cleared = following_input(claim, generation=2) + for key in cleared.keys: + cleared.dispositions.clear_native(key, cleared.input_event.input_id, cleared.text) + with pytest.raises(RelationViolationError): + cleared.acknowledge(await cleared.verify()) + live = following_input(claim, generation=3) + live.start() + proof = await live.verify() + lease = live.lease + assert claim.selected.comms.registry.release_turn(lease)[0] + with pytest.raises(RelationViolationError): + live.acknowledge(proof) + assert all(not row.model_contexts() for row in live.dispositions.read(live.keys).rows.values()) diff --git a/tests/test_goal_execution.py b/tests/test_goal_execution.py index 75ce9cfb1a..ada9864a55 100644 --- a/tests/test_goal_execution.py +++ b/tests/test_goal_execution.py @@ -23,6 +23,7 @@ SetGoalAction, ) from agent_comms.goal_failure_observation import FailedTurnEvidence +from agent_comms.goal_history import RecordedGoalHistory from agent_comms.goal_presentation import ( AttemptFailedExecution, GoalExecution, @@ -36,7 +37,7 @@ from agent_comms.owned_turn import OwnedTurn from agent_comms.threads import Thread from agent_comms.tool_caller import ToolCaller -from agent_comms.tools import CommsGoalHistoryTool, GoalHistory +from agent_comms.tools import CommsGoalHistoryTool from agent_comms.turn_failure import NativeChildSignaled from agent_comms.turn_input_source import NoTurnAcceptance @@ -125,7 +126,7 @@ async def test_goal_history_shows_the_ledgers_failed_attempt_beside_active_revis assert recorded.failures[0].turn_id == turn.turn_id other = CommsGoalHistoryTool(thread="worker", goal_id=OneGoal(GoalId.mint())).apply( comms, ToolCaller.from_actor("worker")) - assert other == GoalHistory.empty() + assert other == RecordedGoalHistory.empty() @pytest.mark.asyncio diff --git a/tests/test_harness_receipt_context.py b/tests/test_harness_receipt_context.py new file mode 100644 index 0000000000..6cb8054ddc --- /dev/null +++ b/tests/test_harness_receipt_context.py @@ -0,0 +1,61 @@ +"""Interpret actual persisted receipt variants without starting a harness/model.""" +from dataclasses import replace + +import pytest + +from agent_comms.cli_bridges.claude import ClaudeBridge +from agent_comms.coordination_tables.harness_deliveries import HarnessDelivery +from agent_comms.coordinator import Coordination +from agent_comms.harness_delivery_states import ( + AcceptedOutcome, DeliveredOutcome, EmittedOutcome, FilteredOutcome, + ObservedMonitor, PersistedOutcome, UnconfirmedOutcome, UnrecordedMonitor, +) +from agent_comms.notification_progress import HarnessConsumedProgress +from agent_comms.presentation import AllOutcomes +from test_coordinated_runtime import _root, tmp_path # noqa: F401 +from transcript_record_fixture import transcript_record + + +@pytest.mark.parametrize(('outcome', 'monitor', 'consumed', 'arrived'), ( + (UnconfirmedOutcome, UnrecordedMonitor(), False, 0), + (EmittedOutcome, UnrecordedMonitor(), False, 0), + (FilteredOutcome, UnrecordedMonitor(), False, 0), + (DeliveredOutcome, UnrecordedMonitor(), False, 0), + (AcceptedOutcome, UnrecordedMonitor(), False, 1), + (PersistedOutcome, UnrecordedMonitor(), False, 1), + (DeliveredOutcome, ObservedMonitor(transcript_record('complete-original'), 1791964800.125), True, 1), +)) +def test_only_retained_complete_body_proof_promotes_consumption( + tmp_path, outcome, monitor, consumed, arrived, +): + """This exercises projection of recorded evidence, not transcript acquisition. + + Complete body/sender/part acquisition is protected by test_monitor_delivery; + an unproved old Delivered row deliberately stays unproved here. + """ + _path, _root_id, comms, initial, _people = _root(tmp_path, direct=True) + source = initial.message + recipient, = comms.bus.bus.delivery(source.seq).recipients + attempt = HarnessDelivery(ClaudeBridge, recipient.recipient_lookup, + source.message_id, source.seq, UnconfirmedOutcome) + with Coordination(comms.root / 'coordination.sqlite3') as store: + with store.session.writing() as db: + assert attempt.begin(store.assignments, db) + recorded = attempt.apply( + lambda row: replace(row, outcome=outcome, monitor=monitor), + store.assignments, db, + ) + with store.session.writing() as db: + assert not attempt.begin(store.assignments, db) + retained, = recorded.stored(db) + assert retained == recorded + current, = comms.views.message_notifications_for_references( + AllOutcomes(), (source.reference,), + ).outcomes[source.seq, source.message_id] + assert current.state.consumed is consumed + assert current.answer is None + assert not current.displayed_to + assert recorded.arrival_count() == arrived + assert recorded.monitor == monitor + if consumed: + assert current.label == current.timed_state(HarnessConsumedProgress.label, 1791964800125) diff --git a/tests/test_head_review.py b/tests/test_head_review.py index a6d2d9ebd9..a62a269eb1 100644 --- a/tests/test_head_review.py +++ b/tests/test_head_review.py @@ -3,6 +3,8 @@ import subprocess import sys import time +from collections.abc import Callable +from contextlib import chdir from dataclasses import replace from datetime import UTC, datetime from pathlib import Path @@ -11,34 +13,44 @@ from agent_comms.commander import judge, order +from agent_comms.comms import Comms from agent_comms.goal_actions import CreateProblemGoalAction from agent_comms.heartbeat_frame import HeartbeatFrame -from agent_comms.head_review import HeadReview, PullRequestHead, ReadPullRequestHead +from agent_comms.build_identity import RepositoryCommit +from agent_comms.head_review import ( + CombinedInput, DraftMergeArm, HeadReview, IntegrationInput, IntegrationSource, OrdinaryInput, PromotedHeadReview, PullRequestHead, ReadPullRequestHead, +) +from agent_comms.integration import IntegrationRun from agent_comms.heartbeat_subjects import CommanderHistory, HeadSubject from agent_comms.recorded_time import RecordedEventTime +from agent_comms.repository_name import RepositoryName from agent_comms.standing_rules import ( HeadReviewTimerUp, IdlePullRequest, OrderlessPullRequest, OrderlessUnreviewedHead, OrderRefusingStandingRule, ReviewTimerUp, StuckMerge, ) from agent_comms.repo import Checkout from agent_comms.threads import Thread -from agent_comms.errors import ToolRefusedError -from agent_comms.field_codec import RecordProblems +from agent_comms.errors import TodoError, ToolRefusedError +from agent_comms.field_codec import FieldCodec, GoalId, MalformedRecord, RecordProblems from agent_comms.tool_caller import ThreadCaller +from agent_comms.thread_identity import RecordedThreadIncarnation from agent_comms.work_branches import SourceGrowth, WorkBranches from agent_comms.order_tools import CommsHeadVerdictTool from agent_comms.todos import ( CarriedHeadVerdict, ConflictMergeBlock, DraftListedPullRequest, FailedCheckMergeBlock, GrownHeadVerdict, GrowthReason, OrderHeadReview, LinkedHeadReview, - NoHeadOrder, ProgressRead, PullRequestLink, RecordPullRequest, RejectedHeadVerdict, - ReviewedHeadVerdict, StoredHeadReview, UnquotedHeadReview, - StrikeReason, UnderReviewHeadVerdict, UnmergedListedPullRequest, VerdictReason, + NoHeadOrder, OrderClaim, ProgressRead, PullRequestLink, PullRequests, RecordPullRequest, RejectedHeadVerdict, + HeadVerdict, HeadReviewed, QualifiedHeadVerdict, ReviewedHeadVerdict, StoredHeadReview, UnquotedHeadReview, + StrikeReason, UnderReviewHeadVerdict, UnmergedListedPullRequest, VerdictReason, WithdrawTodo, ) +from github_fixture import GitHubFixture +from integration_source_fixture import MutualSources from source_revision_fixture import SourceRepository -pytest_plugins = ("test_holder_standing_rules", "github_fixture") +pytest_plugins = ("test_holder_standing_rules", "github_fixture", "integration_source_fixture") REASON = StrikeReason("This head repeats the same decision in two unrelated places instead of using the existing class.") ACCEPTED = VerdictReason("The existing class owns this behavior without another copy") +QUALIFIED = VerdictReason("The source stage is reviewed while paired implementation remains incomplete") @pytest.mark.parametrize('roast,fields', ( @@ -78,11 +90,533 @@ def head_bus(held_bus, github_cli, reviewed_source): return held_bus, github_cli, opened(github_cli, source, 7, source.branch_commit) +@pytest.fixture +def linked_head_bus(head_bus): + comms, _, request = head_bus + todo = comms.goals.todos.get("P2") + held = todo.held() + RecordPullRequest(todo.revision, held.owner_identity, + PullRequestLink(request.repo, request.pr), held.generation).record(comms.goals, todo.id) + return head_bus + + def invoke(comms, request, verdict, order=NoHeadOrder()): tool = CommsHeadVerdictTool(request=request, operation=verdict, order=order) return tool.invoke(comms, tool.arguments, caller=ThreadCaller("lead")) +def promote(context: order.CommanderContext, request: PullRequestHead) -> HeadReviewed: + return judge.PromoteVerb(pr=request.pr, head=request.head, repo=request.repo.to_wire(), + why=ACCEPTED.text).review_head(context, request, NoHeadOrder()) + + +def review_posts(github: GitHubFixture, head: str) -> tuple[tuple[str, ...], ...]: + """Count source decisions independently of their derived merge withdrawal.""" + return tuple(call for call in github.calls() if f"statuses/{head}" in call[1] + and f"context={HeadReview.context}" in call) + + +def test_qualification_records_each_open_order_cut_without_promoting_or_finishing_work( + linked_head_bus, claim_order): + """A ready companion's source stage is a decision, not permission to ship its incomplete pair.""" + comms, github, request = linked_head_bus + context = order.Commander.read_root(comms.root) + + def qualify_unchanged(): + before = comms.goals.todos.get("P2") + goal = comms.registry.require("worker").goal.require() + quote = before.order.checked().edits[0] + judge.main(["qualify", "--repo", request.repo.to_wire(), "--pr", str(request.pr), + "--head", request.head, "--why", QUALIFIED.text, "--serves", quote], context) + assert comms.goals.todos.get("P2") == replace(before, revision=before.revision + 1) + assert comms.registry.require("worker").goal.require() == replace(goal, revision=goal.revision + 1) + record = StoredHeadReview.head_reviews(comms.goals.todos, request)[0] + assert record.verdict == QualifiedHeadVerdict(QUALIFIED) + assert record.order == LinkedHeadReview.of(before, quote) + assert record.verdict.decides() + assert record.verdict.settles_edits(record, 0) == record.at.recorded_at("qualified source cut") + assert not record.verdict.allows_integration() + assert not HeadReview.accepted(request.repo, request.head) + with pytest.raises(TodoError, match="no recorded ok verdict"): + record.verdict.require_merge(request) + + qualify_unchanged() # Pending: the order still requires every unfinished clause. + claim_order("P2", pull_request=request.pr, head=request.head) + qualify_unchanged() # Claimed: the delivery is neither accepted nor completed. + todo = comms.goals.todos.get("P2") + invoke(comms, request, UnderReviewHeadVerdict(), OrderHeadReview.for_request( + (todo,), request, UnquotedHeadReview.of)) + qualify_unchanged() # Reviewing: the recorded source cut changes no held work. + posts = review_posts(github, request.head) + assert len(posts) == 4 + assert all("state=pending" in call for call in posts) + assert not any(call[:2] == ("pr", "merge") for call in github.calls()) + + +@pytest.fixture +def companion_checkout( + linked_head_bus: tuple[Comms, GitHubFixture, PullRequestHead], + reviewed_source: SourceRepository, + configured_repository: Callable[[Comms, *tuple[SourceRepository, ...]], order.Commander], + tmp_path: Path, +) -> Path: + comms, github, primary = linked_head_bus + companion = SourceRepository.create(tmp_path / "companion", RepositoryName("example", "companion")) + configured_repository(comms, reviewed_source, companion) + stamp = "2026-10-24T00:00:00Z" + # The same PR number exists in both repositories: substituting todo.repo cannot pass. + github.put(companion.github, DraftListedPullRequest( + primary.pr, "feature", companion.branch_commit, stamp, "main", reviewable_times=(stamp,))) + owner = comms.registry.require(comms.goals.todos.get("P2").held().owner) + rule = comms.registry.storage_rule + checkout = rule.repository_checkout(rule.roots_of(owner), "P2", companion.github) + checkout.parent.mkdir(parents=True, exist_ok=True) + companion.git(companion.checkout, "worktree", "add", "--detach", str(checkout), companion.branch_commit) + return checkout + + +def test_native_companion_link_preserves_claim_review_goal_and_is_idempotent( + linked_head_bus: tuple[Comms, GitHubFixture, PullRequestHead], + companion_checkout: Path, + claim_order: Callable[..., OrderClaim], +) -> None: + comms, github, primary = linked_head_bus + claim_order("P2", pull_request=primary.pr, head=primary.head) + todo = comms.goals.todos.get("P2") + invoke(comms, primary, UnderReviewHeadVerdict(), OrderHeadReview.for_request( + (todo,), primary, UnquotedHeadReview.of)) + before = comms.goals.todos.get("P2") + goal = comms.registry.require(before.held().owner).goal + reviews = StoredHeadReview.head_reviews(comms.goals.todos, primary) + history = tuple(comms.registry.goal_history(before.held().owner)) + link = PullRequestLink(SourceRepository.of(companion_checkout).github, primary.pr) + context = order.Commander.read_root(comms.root) + with chdir(companion_checkout): + order.main(["pr", before.id, "--existing", str(primary.pr)], context) + after = comms.goals.todos.get(before.id) + assert after == replace(before, revision=before.revision + 1, order=before.order.checked().linked_to(link)) + assert type(after.order) is type(before.order) + assert comms.registry.require(after.held().owner).goal == goal + assert StoredHeadReview.head_reviews(comms.goals.todos, primary) == reviews + companion = PullRequestHead(link.repo, link.number, SourceRepository.git(companion_checkout, "rev-parse", "HEAD").strip()) + assert PullRequests.linked_among((after,), link.repo) == {link.number: [after]} + assert PullRequests((after,)).linked(link.repo) == {link.number: [after]} + assert OrderHeadReview.holding((after,), companion) == (after,) + # Qualification reads these exact stored links, not a caller-provided pair roster. + assert {item.repo for item in after.order.checked().pull_requests} == {primary.repo, link.repo} + story = after.story() + with chdir(companion_checkout): + order.main(["pr", before.id, "--existing", str(primary.pr)], context) + assert comms.goals.todos.get(before.id) == after + assert comms.goals.todos.get(before.id).story() == story + assert comms.registry.require(after.held().owner).goal == goal + assert StoredHeadReview.head_reviews(comms.goals.todos, primary) == reviews + assert tuple(comms.registry.goal_history(after.held().owner)) == history + assert not any(call[:2] in (("pr", "create"), ("pr", "merge")) for call in github.calls()) + assert any(link.repo.to_wire() in argument for call in github.calls() for argument in call) + + +def test_native_companion_adoption_is_the_same_reviewed_orders_actual_qualification_source( + source_pair: MutualSources, claim_order: Callable[..., OrderClaim], + github_cli: GitHubFixture, +) -> None: + pair = source_pair + comms = pair.context.comms + primary = PullRequestHead(pair.core.github, pair.core.number, pair.core.branch_commit) + claim_order("P2", pull_request=primary.pr, head=primary.head) + todo = comms.goals.todos.get("P2") + invoke(comms, primary, UnderReviewHeadVerdict(), OrderHeadReview.for_request( + (todo,), primary, UnquotedHeadReview.of)) + before = comms.goals.todos.get("P2") + owner = comms.registry.require(before.held().owner) + rule = comms.registry.storage_rule + checkout = rule.repository_checkout(rule.roots_of(owner), before.id, pair.client.github) + checkout.parent.mkdir(parents=True, exist_ok=True) + pair.client.git(pair.client.checkout, "worktree", "add", "--detach", str(checkout), pair.client.branch_commit) + with chdir(checkout): + order.main(["pr", before.id, "--existing", str(pair.client.number)], order.Commander.read_root(comms.root)) + linked = comms.goals.todos.get(before.id) + assert linked == replace(before, revision=before.revision + 1, + order=before.order.checked().linked_to(pair.client.link)) + inputs = pair.capture(("P2",), "native qualified companion discovery") + assert inputs.source(pair.client.github).heads == ( + PullRequestHead(pair.client.github, pair.client.number, pair.client.branch_commit),) + assert comms.goals.todos.get(before.id) == linked + assert not HeadReview.accepted(primary.repo, primary.head) + assert not any(call[:2] in (("pr", "create"), ("pr", "merge")) for call in github_cli.calls()) + + +def test_companion_metadata_cannot_create_edit_or_adopt_missing_pr( + linked_head_bus: tuple[Comms, GitHubFixture, PullRequestHead], companion_checkout: Path, +) -> None: + comms, github, primary = linked_head_bus + before = comms.goals.todos.get("P2") + context = order.Commander.read_root(comms.root) + with chdir(companion_checkout): + for arguments in ( + ["--title", "Do not create a companion", "--body-file", "not-read.md"], + ["--existing", str(primary.pr), "--title", "Do not edit"], + ["--existing", "999999"], + ): + with pytest.raises((ValueError, TodoError, SystemExit, subprocess.CalledProcessError)): + order.main(["pr", before.id, *arguments], context) + assert comms.goals.todos.get(before.id) == before + assert not any(call[:2] in (("pr", "create"), ("pr", "edit"), ("pr", "merge")) for call in github.calls()) + + +def test_companion_metadata_refuses_an_existing_but_inaccessible_remote_pr( + linked_head_bus: tuple[Comms, GitHubFixture, PullRequestHead], companion_checkout: Path, +) -> None: + comms, github, primary = linked_head_bus + before = comms.goals.todos.get("P2") + companion = SourceRepository.of(companion_checkout) + github.refuse_read(companion.github, primary.pr, "GraphQL: Resource not accessible by integration") + with chdir(companion_checkout), pytest.raises(subprocess.CalledProcessError): + order.main(["pr", before.id, "--existing", str(primary.pr)], order.Commander.read_root(comms.root)) + assert comms.goals.todos.get(before.id) == before + assert not any(call[:2] in (("pr", "create"), ("pr", "edit")) for call in github.calls()) + + +def test_companion_metadata_refuses_another_owners_checkout( + linked_head_bus: tuple[Comms, GitHubFixture, PullRequestHead], companion_checkout: Path, +) -> None: + comms, github, primary = linked_head_bus + before = comms.goals.todos.get("P2") + companion = SourceRepository.of(companion_checkout) + other = comms.registry.require("lead") + rule = comms.registry.storage_rule + checkout = rule.repository_checkout(rule.roots_of(other), "P2", companion.github) + checkout.parent.mkdir(parents=True, exist_ok=True) + companion.git(companion_checkout, "worktree", "add", "--detach", str(checkout), "HEAD") + calls = github.calls() + with chdir(checkout), pytest.raises(ValueError, match="owned by"): + order.main(["pr", before.id, "--existing", str(primary.pr)], order.Commander.read_root(comms.root)) + assert comms.goals.todos.get(before.id) == before + assert github.calls() == calls + + +def test_companion_metadata_refuses_an_unregistered_origin_before_forge_read( + linked_head_bus: tuple[Comms, GitHubFixture, PullRequestHead], companion_checkout: Path, + reviewed_source: SourceRepository, + configured_repository: Callable[[Comms, *tuple[SourceRepository, ...]], order.Commander], +) -> None: + comms, github, primary = linked_head_bus + before = comms.goals.todos.get("P2") + context = configured_repository(comms, reviewed_source) + calls = github.calls() + with chdir(companion_checkout), pytest.raises(ValueError, match="Unknown repository"): + order.main(["pr", before.id, "--existing", str(primary.pr)], context) + assert comms.goals.todos.get(before.id) == before + assert github.calls() == calls + + +def test_companion_metadata_refuses_a_closed_order_without_forge_or_link_changes( + linked_head_bus: tuple[Comms, GitHubFixture, PullRequestHead], companion_checkout: Path, +) -> None: + comms, github, primary = linked_head_bus + before = comms.goals.todos.get("P2") + comms.goals.todos.apply(before.id, WithdrawTodo( + before.revision, comms.registry.require("lead").incarnation, + before.held(), "Close this fixture order before attempting metadata")) + closed = comms.goals.todos.get(before.id) + calls = github.calls() + with chdir(companion_checkout), pytest.raises((ValueError, TodoError)): + order.main(["pr", before.id, "--existing", str(primary.pr)], order.Commander.read_root(comms.root)) + assert comms.goals.todos.get(before.id) == closed + assert github.calls() == calls + + +def test_companion_metadata_write_enforces_registration_revision_and_holder( + linked_head_bus: tuple[Comms, GitHubFixture, PullRequestHead], companion_checkout: Path, +) -> None: + comms, _, primary = linked_head_bus + before = comms.goals.todos.get("P2") + held = before.held() + worker = held.owner_identity + registered = PullRequestLink(SourceRepository.of(companion_checkout).github, primary.pr) + for command in ( + RecordPullRequest(before.revision, worker, PullRequestLink(RepositoryName("unknown", "source"), 7), held.generation), + RecordPullRequest(before.revision + 1, worker, registered, held.generation), + RecordPullRequest(before.revision, worker, registered, GoalId.mint()), + RecordPullRequest(before.revision, RecordedThreadIncarnation("not-holder", 1.0), + registered, held.generation), + ): + with pytest.raises((ValueError, TodoError)): + comms.goals.todos.apply(before.id, command) + assert comms.goals.todos.get(before.id) == before + + +def test_qualification_withdraws_acceptance_without_claiming_to_disarm_the_request(head_bus): + comms, github, request = head_bus + invoke(comms, request, ReviewedHeadVerdict(ACCEPTED)) + arms = tuple(call for call in github.calls() if call[:2] == ("pr", "merge")) + assert arms == () and HeadReview.accepted(request.repo, request.head) + with pytest.raises(ToolRefusedError, match="no explicit reviewed merge promotion"): + ReviewedHeadVerdict.require_merge(request) + invoke(comms, request, QualifiedHeadVerdict(QUALIFIED)) + assert StoredHeadReview.current_verdict(comms.goals.todos, request) is QualifiedHeadVerdict + assert not HeadReview.accepted(request.repo, request.head) + assert tuple(call for call in github.calls() if call[:2] == ("pr", "merge")) == arms + posts = review_posts(github, request.head) + assert "state=pending" in posts[-1] + # Reaccepting this source permits combination, never an individual merge. + invoke(comms, request, ReviewedHeadVerdict(ACCEPTED)) + assert HeadReview.accepted(request.repo, request.head) + assert not any(call[:2] == ("pr", "merge") for call in github.calls()) + + +def test_qualification_codec_and_head_only_cli_require_a_reason(): + verdict = QualifiedHeadVerdict(QUALIFIED) + assert FieldCodec.decode(HeadVerdict, FieldCodec.encode(verdict, HeadVerdict)) == verdict + with pytest.raises(MalformedRecord): + FieldCodec.decode(HeadVerdict, {"kind": FieldCodec.encode(QualifiedHeadVerdict)}) + common = ["qualify", "--pr", "7", "--head", "a" * 40] + assert judge.JudgeVerb.parse([*common, "--why", QUALIFIED.text]).head_operation() == verdict + with pytest.raises(RecordProblems) as refused: + judge.JudgeVerb.parse(common) + assert "why" in refused.value.problems + with pytest.raises(RecordProblems) as refused: + judge.JudgeVerb.parse(["qualify", "E1:abcd", "--why", QUALIFIED.text, + "--serves", "Repair the existing owner"]) + assert "ids" in refused.value.problems + + +@pytest.mark.parametrize("quote", ("", "These words never occur in the held order")) +def test_linked_qualification_requires_the_original_order_quote(linked_head_bus, quote): + comms, github, request = linked_head_bus + before = comms.goals.todos.get("P2") + with pytest.raises(ToolRefusedError): + judge.main(["qualify", "--repo", request.repo.to_wire(), "--pr", str(request.pr), + "--head", request.head, "--why", QUALIFIED.text, "--serves", quote], + order.Commander.read_root(comms.root)) + assert comms.goals.todos.get(before.id) == before + assert not StoredHeadReview.head_reviews(comms.goals.todos, request) + assert not any("statuses/" in call[1] for call in github.calls()) + + +def test_qualification_refuses_a_noncommander_before_any_record_or_public_status(linked_head_bus): + comms, github, request = linked_head_bus + before = comms.goals.todos.get("P2") + tool = CommsHeadVerdictTool( + request=request, operation=QualifiedHeadVerdict(QUALIFIED), + order=OrderHeadReview.for_request((before,), request, + lambda linked: LinkedHeadReview.of(linked, linked.order.edits[0]))) + with pytest.raises(ToolRefusedError): + tool.invoke(comms, tool.arguments, caller=ThreadCaller("worker")) + assert comms.goals.todos.get(before.id) == before + assert not StoredHeadReview.head_reviews(comms.goals.todos, request) + assert not any("statuses/" in call[1] for call in github.calls()) + + +def test_qualification_does_not_carry_acceptance_to_an_unchanged_later_head(head_bus, reviewed_source): + comms, github, request = head_bus + source = reviewed_source + invoke(comms, request, QualifiedHeadVerdict(QUALIFIED)) + (source.checkout / "base_only.py").write_text("advanced = True\n") + source.git(source.checkout, "add", "base_only.py") + source.git(source.checkout, "commit", "-m", "Advance the base independently") + source.git(source.checkout, "push", "origin", "main") + source.git(source.checkout, "checkout", "feature") + source.git(source.checkout, "merge", "--no-edit", "main") + head = source.git(source.checkout, "rev-parse", "HEAD").strip() + source.git(source.checkout, "push", "origin", "HEAD:refs/pull/7/head") + moved = opened(github, source, request.pr, head) + assert request.change_identity(WorkBranches(source.checkout), "main") == moved.change_identity( + WorkBranches(source.checkout), "main") + pulls = ProgressRead() + pulls.carry_reviews(comms.goals, request.repo) + assert not StoredHeadReview.head_reviews(comms.goals.todos, moved) + assert not pulls.open(request.repo)[0].has_head_review + assert not any(f"statuses/{head}" in call[1] for call in github.calls()) + + +@pytest.fixture +def captured_input(reviewed_source: SourceRepository) -> IntegrationInput: + source = reviewed_source + return OrdinaryInput((IntegrationSource( + RepositoryCommit(source.github, source.main_commit), + (PullRequestHead(source.github, source.number, source.branch_commit),), + source.git(source.checkout, "rev-parse", f"{source.branch_commit}^{{tree}}").strip(), + ),)) + + +def test_integration_metadata_is_canonical_and_ordinary_messages_have_no_digest( + captured_input: IntegrationInput) -> None: + inputs = captured_input + assert FieldCodec.decode(IntegrationInput, FieldCodec.parse(inputs.canonical)) == inputs + message = f"Combine the real source changes\n\n{inputs.trailer}\n" + assert IntegrationInput.from_message(message) == (inputs,) + assert IntegrationInput.digest_from_message(message) == inputs.digest + assert IntegrationInput.from_message("An ordinary substantive change\n") == () + assert IntegrationInput.digest_from_message("An ordinary substantive change\n") == "" + + +@pytest.mark.parametrize("damage", ( + lambda trailer: trailer + "\n" + trailer, + lambda trailer: trailer.replace(": ", ":\t", 1), + lambda trailer: trailer.replace('"sources":', '"sources": ', 1), + lambda trailer: trailer + "\nUnrelated text after the trailer", + lambda trailer: trailer + "garbage", +)) +def test_attempted_integration_metadata_never_falls_back_to_ordinary_mode( + captured_input: IntegrationInput, damage: Callable[[str], str]) -> None: + with pytest.raises(MalformedRecord): + IntegrationInput.from_message(damage(captured_input.trailer)) + + +def test_integration_metadata_refuses_duplicate_json_fields_and_unknown_shapes( + captured_input: IntegrationInput) -> None: + canonical = captured_input.canonical + duplicate = canonical[:-1] + "," + canonical[1:] + for payload in (duplicate, '{"sources":[],"unsupported_version":2}'): + with pytest.raises(MalformedRecord): + IntegrationInput.from_message(f"{IntegrationInput.trailer_name}: {payload}") + + +def test_integration_source_refuses_duplicates_abbreviations_and_foreign_members( + captured_input: IntegrationInput) -> None: + source, = captured_input.sources + head, = source.heads + with pytest.raises(MalformedRecord): + replace(source, heads=(head, head)) + with pytest.raises(MalformedRecord): + replace(source, baseline=replace(source.baseline, commit=source.baseline.commit[:9])) + with pytest.raises(MalformedRecord): + replace(source, heads=(replace(head, head=head.head[:9]),)) + with pytest.raises(MalformedRecord): + replace(source, heads=(replace(head, repo=RepositoryName.from_wire("other/repository")),)) + with pytest.raises(MalformedRecord): + OrdinaryInput((source, source)) + with pytest.raises(ToolRefusedError): + captured_input.require_repositories((source.repository, RepositoryName.from_wire("other/repository"))) + + +def test_companion_input_changes_the_identity_of_an_unchanged_local_patch( + captured_input: IntegrationInput, reviewed_source: SourceRepository, tmp_path: Path, + github_cli: GitHubFixture) -> None: + source = reviewed_source + companion = SourceRepository.create(tmp_path / "companion", RepositoryName.from_wire("example/client")) + client = IntegrationSource( + RepositoryCommit(companion.github, companion.main_commit), + (PullRequestHead(companion.github, companion.number, companion.branch_commit),), + companion.git(companion.checkout, "rev-parse", f"{companion.branch_commit}^{{tree}}").strip(), + ) + inputs = replace(captured_input, sources=tuple(sorted((*captured_input.sources, client), key=lambda row: row.repository.key))) + companion.git(companion.checkout, "checkout", "feature") + owner = companion.checkout / "owner.py" + owner.write_text(owner.read_text() + "new_export = True\n") + moved = companion.publish_change("Change the paired client", "owner.py") + changed = replace(client, heads=(replace(client.heads[0], head=moved),), + candidate_tree=companion.git(companion.checkout, "rev-parse", f"{moved}^{{tree}}").strip()) + updated = replace(inputs, sources=tuple(changed if row == client else row for row in inputs.sources)) + core, = captured_input.sources + heads = tuple(PullRequestHead(source.github, source.number, source.git( + source.checkout, "commit-tree", core.candidate_tree, + "-p", source.main_commit, "-p", source.branch_commit, + "-m", f"Combine the substantive Core source\n\n{snapshot.trailer}", + ).strip()) for snapshot in (inputs, updated)) + reader = WorkBranches(source.checkout) + before, after = (head.change_identity(reader, "main") for head in heads) + assert before.patch == after.patch + assert before.integration_digest == inputs.digest + assert after.integration_digest == updated.digest + assert before != after + # An author's canonical self-hash is source identity, never an accepted review. + github_cli.configure(source.github) + assert not any(HeadReview.accepted(head.repo, head.head) for head in heads) + + +def test_pending_sources_can_compute_but_only_reviewed_other_heads_allow_carrier_promotion( + mutual_sources: MutualSources, github_cli: GitHubFixture) -> None: + pair = mutual_sources + comms = pair.context.comms + client = PullRequestHead(pair.client.github, pair.client.number, pair.client.branch_commit) + invoke(comms, client, UnderReviewHeadVerdict()) + run = IntegrationRun(pair.checkout, pair.output, "pending inputs compute without promotion") + inputs = pair.capture(("P2", "P3"), run.risk) + carrier = pair.publish_carrier(github_cli) + invoke(comms, carrier, UnderReviewHeadVerdict()) + for verdict in (UnderReviewHeadVerdict(), QualifiedHeadVerdict(QUALIFIED)): + invoke(comms, client, verdict) + with pytest.raises(ToolRefusedError, match="no accepted exact-head source review"): + promote(pair.context, carrier) + assert not HeadReview.accepted(carrier.repo, carrier.head) + invoke(comms, client, ReviewedHeadVerdict(ACCEPTED)) + assert HeadReview.accepted(client.repo, client.head) + original, = inputs.source(pair.core.github).heads + assert not HeadReview.accepted(original.repo, original.head) + with pytest.raises(ToolRefusedError, match="no accepted exact-head source review"): + promote(pair.context, carrier) + invoke(comms, original, ReviewedHeadVerdict(ACCEPTED)) + promote(pair.context, carrier) + assert HeadReview.accepted(carrier.repo, carrier.head) + assert PromotedHeadReview.accepted(carrier.repo, carrier.head) + assert StoredHeadReview.current_verdict(comms.goals.todos, carrier) is ReviewedHeadVerdict + assert carrier.current().arm() == DraftMergeArm() + assert run.accepted_heads() == () + invoke(comms, carrier, QualifiedHeadVerdict(QUALIFIED)) + assert not PromotedHeadReview.accepted(carrier.repo, carrier.head) + invoke(comms, carrier, ReviewedHeadVerdict(ACCEPTED)) + assert HeadReview.accepted(carrier.repo, carrier.head) + assert not PromotedHeadReview.accepted(carrier.repo, carrier.head) + assert not any(call[:2] == ("pr", "merge") for call in github_cli.calls()) + + +def test_a_moved_companion_refuses_final_review_without_writing_acceptance( + mutual_sources: MutualSources, github_cli: GitHubFixture) -> None: + pair = mutual_sources + run = IntegrationRun(pair.checkout, pair.output, "a changed actual source invalidates the input") + pair.capture(("P2", "P3"), run.risk) + carrier = pair.publish_carrier(github_cli) + pair.client.publish_change("Move the companion", *pair.client.write_files({ + Path("src/new_file.py"): "changed = True\n", + })) + stamp = "2026-10-24T00:00:00Z" + github_cli.put(pair.client.github, DraftListedPullRequest( + pair.client.number, "feature", pair.client.branch_commit, stamp, "main", reviewable_times=(stamp,), + )) + with pytest.raises(ToolRefusedError, match="moved"): + promote(pair.context, carrier) + assert not HeadReview.accepted(carrier.repo, carrier.head) + assert not StoredHeadReview.head_reviews(pair.context.comms.goals.todos, carrier) + + +@pytest.mark.parametrize("damage", ("member", "tree", "carrier", "missing-source")) +def test_combined_promotion_refuses_changed_members_trees_carriers_and_missing_sources( + mutual_sources: MutualSources, github_cli: GitHubFixture, damage: str) -> None: + pair = mutual_sources + comms = pair.context.comms + inputs = pair.capture(("P2", "P3"), "every captured identity must survive promotion") + carrier = pair.publish_carrier(github_cli) + for source in inputs.sources: + for head in source.heads: + invoke(comms, head, ReviewedHeadVerdict(ACCEPTED)) + if damage == "member": + moved = pair.core.publish_change("Change a captured local source", *pair.core.write_files({ + Path("src/agent_comms/moved.py"): "moved = True\n"})) + opened(github_cli, pair.core, pair.core.number, moved) + elif damage == "tree": + path = pair.checkout.path / "not-reviewed.py" + path.write_text("unreviewed = True\n") + pair.core.git(pair.checkout.path, "add", "not-reviewed.py") + pair.core.git(pair.checkout.path, "commit", "-m", f"Change the combined tree\n\n{inputs.trailer}") + carrier = pair.publish_carrier(github_cli) + elif damage == "carrier": + # The same Git payload under another PR number is not its captured carrier. + pair.core.git(pair.checkout.path, "push", "origin", f"{carrier.head}:refs/pull/102/head") + carrier = opened(github_cli, pair.core, 102, carrier.head) + else: + # Keep valid canonical encoding while attempting to omit an actual declared client. + incomplete = replace(inputs, sources=(inputs.source(pair.core.github),)) + pair.core.git(pair.checkout.path, "commit", "--allow-empty", "-m", incomplete.trailer) + carrier = pair.publish_carrier(github_cli) + with pytest.raises(ToolRefusedError): + promote(pair.context, carrier) + assert not PromotedHeadReview.accepted(carrier.repo, carrier.head) + assert not StoredHeadReview.head_reviews(comms.goals.todos, carrier) + assert not any(call[:2] == ("pr", "merge") for call in github_cli.calls()) + + def test_integration_reads_latest_review_in_the_declared_context(github_cli, tmp_path): source = SourceRepository.create(tmp_path / 'status-source') github_cli.configure(source.github) @@ -113,7 +647,7 @@ def test_orderless_head_family_persists_both_outcomes_and_posts_each_exact_head( current = ReadPullRequestHead(request.repo, request.pr, request.head, base="main") assert StoredHeadReview.head_reviews(comms.goals.todos, current) == StoredHeadReview.head_reviews(comms.goals.todos, request) assert tuple(todo.strike_count for todo in comms.goals.todos.list(request.repo)) == before - posts = tuple(call for call in github.calls() if f"statuses/{request.head}" in call[1]) + posts = review_posts(github, request.head) assert len(posts) == 2 assert "state=success" in posts[0] and "state=failure" in posts[1] @@ -134,43 +668,57 @@ def test_judge_ok_and_back_dispatch_orderless_prs_without_positional_edit_ids(he "--slop", "IMPL-5", "--fact", "the same decision is written twice", "--at", location, "--derive", "base-method", "--by", location], context) assert isinstance(StoredHeadReview.head_reviews(comms.goals.todos, request)[0].verdict, RejectedHeadVerdict) - assert len(tuple(call for call in github.calls() if f"statuses/{request.head}" in call[1])) == 2 - - -def test_judge_ok_requests_merge_at_the_judged_head_unless_draft_and_fails_loudly( - head_bus, reviewed_source, capsys): - """2026-10-26: #3454 printed ok and stayed unarmed; #3444 needed a hand bypass-merge.""" + assert len(review_posts(github, request.head)) == 2 + + +@pytest.mark.parametrize("ready,refused", ((False, False), (True, False), (True, True))) +def test_only_explicit_combined_promotion_arms_the_reviewed_head_and_reports_forge_failure( + mutual_sources: MutualSources, github_cli: GitHubFixture, capsys, + ready: bool, refused: bool) -> None: + pair = mutual_sources + comms = pair.context.comms + inputs = pair.capture(("P2", "P3"), "explicit combined promotion owns enqueue") + request = pair.publish_carrier(github_cli) + for source in inputs.sources: + for head in source.heads: + invoke(comms, head, ReviewedHeadVerdict(ACCEPTED)) + assert request.pr not in {head.pr for head in inputs.source(request.repo).heads} + assert not any(call[:2] == ("pr", "merge") for call in github_cli.calls()) + if ready: + stamp = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") + github_cli.put(request.repo, UnmergedListedPullRequest( + request.pr, "combined", request.head, stamp, "main", reviewable_times=(stamp,))) + if refused: + github_cli.refuse_arm(request.repo, request.pr, "Merge permission denied") + arguments = ["promote", "--repo", request.repo.to_wire(), "--pr", str(request.pr), + "--head", request.head, "--why", ACCEPTED.text] + context = order.Commander.read_root(comms.root) + if refused: + with pytest.raises(ToolRefusedError, match="Merge permission denied"): + judge.main(arguments, context) + else: + judge.main(arguments, context) + arms = [call for call in github_cli.calls() if call[:2] == ("pr", "merge")] + assert arms == ([("pr", "merge", str(request.pr), "--repo", request.repo.to_wire(), + "--auto", "--merge", "--match-head-commit", request.head)] if ready else []) + if not ready: + assert f"#{request.pr} is a draft, so not armed" in capsys.readouterr().out + assert StoredHeadReview.current_verdict(comms.goals.todos, request) is ReviewedHeadVerdict + assert PromotedHeadReview.accepted(request.repo, request.head) + ReviewedHeadVerdict.require_merge(request) + + +def test_source_acceptance_never_enqueues_and_promotion_refuses_an_individual_member(head_bus) -> None: comms, github, request = head_bus - for number in (8, 9): - reviewed_source.git(reviewed_source.checkout, "push", "-q", "origin", - f"{request.head}:refs/pull/{number}/head") context = order.Commander.read_root(comms.root) - stamp = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") - github.put(request.repo, DraftListedPullRequest( - 8, "fix/draft", request.head, stamp, "main", reviewable_times=(stamp,))) - github.put(request.repo, UnmergedListedPullRequest( - 9, "fix/refused", request.head, stamp, "main", reviewable_times=(stamp,))) - github.refuse_arm(request.repo, 9, "Merge permission denied") - - def ok(number): - judge.main(["ok", "--repo", request.repo.to_wire(), "--pr", str(number), - "--head", request.head, "--why", ACCEPTED.text], context) - - def arms(number): - return [call for call in github.calls() if call[:3] == ("pr", "merge", str(number))] - - ok(request.pr) - assert arms(request.pr) == [("pr", "merge", str(request.pr), "--repo", request.repo.to_wire(), - "--auto", "--merge", "--match-head-commit", request.head)] - assert f"#{request.pr} merge requested at {request.head}" in capsys.readouterr().out - ok(8) - assert not arms(8) - assert "#8 is a draft, so not armed" in capsys.readouterr().out - with pytest.raises(ToolRefusedError, match="Merge permission denied"): - ok(9) - assert "ok" not in capsys.readouterr().out - assert isinstance(StoredHeadReview.head_reviews( - comms.goals.todos, replace(request, pr=9))[0].verdict, ReviewedHeadVerdict) + arguments = ["--repo", request.repo.to_wire(), "--pr", str(request.pr), + "--head", request.head, "--why", ACCEPTED.text] + judge.main(["ok", *arguments], context) + assert HeadReview.accepted(request.repo, request.head) + with pytest.raises(ToolRefusedError, match="captured combined carrier"): + judge.main(["promote", *arguments], context) + assert not PromotedHeadReview.accepted(request.repo, request.head) + assert not any(call[:2] == ("pr", "merge") for call in github.calls()) def test_a_verdict_lists_every_argument_refusal_in_one_run_and_records_nothing(head_bus): @@ -204,16 +752,17 @@ def test_judge_reviewing_after_ok_withdraws_the_accepted_status_on_that_head( common = ["--pr", str(request.pr), "--head", request.head] judge.main(["ok", *common, "--why", ACCEPTED.text], context) judge.main(["reviewing", *common], context) - posts = tuple(call for call in github.calls() if f"statuses/{request.head}" in call[1]) + posts = review_posts(github, request.head) assert [next(part for part in post if part.startswith("state=")) for post in posts] == [ "state=success", "state=pending"] -def test_moved_head_posts_and_records_nothing(head_bus): +@pytest.mark.parametrize("verdict", (ReviewedHeadVerdict(ACCEPTED), QualifiedHeadVerdict(QUALIFIED))) +def test_moved_head_posts_and_records_nothing(head_bus, verdict): comms, github, request = head_bus moved = replace(request, head="a" * 40) with pytest.raises(ToolRefusedError): - invoke(comms, moved, ReviewedHeadVerdict(ACCEPTED)) + invoke(comms, moved, verdict) assert not StoredHeadReview.head_reviews(comms.goals.todos, moved) assert not any("statuses/" in call[1] for call in github.calls()) @@ -238,7 +787,7 @@ def test_review_ok_follows_an_unchanged_change(head_bus, reviewed_source): assert pulls.open(request.repo)[0].has_head_review assert StoredHeadReview.head_reviews(comms.goals.todos, moved)[0].verdict == CarriedHeadVerdict(ACCEPTED, request) assert not OrderlessUnreviewedHead.scan_reviews(comms.goals, "lead", pulls) - carried_posts = tuple(call for call in github.calls() if f"statuses/{merged}" in call[1]) + carried_posts = review_posts(github, merged) assert len(carried_posts) == 1 assert f"description=carried from {request.head[:9]}: same change" in carried_posts[0] @@ -391,7 +940,7 @@ def test_acceptance_over_the_growth_limit_needs_a_reason_naming_each_growing_fil assert not StoredHeadReview.head_reviews(comms.goals.todos, request) invoke(comms, request, GrownHeadVerdict(ACCEPTED, GrowthReason( "grown.py declares forty one values that the reviewed request needs as separate names"))) - posts = tuple(call for call in github.calls() if f"statuses/{head}" in call[1]) + posts = review_posts(github, head) assert len(posts) == 1 and any(f"(src +{grown})" in part for part in posts[0]) # The actual CLI must report both independent failures, then infer the unique @@ -453,7 +1002,7 @@ def commit(parent, lines, message): github.put(source.github, UnmergedListedPullRequest( 9, "fix/child", head, stamp, "fix/parent", reviewable_times=(stamp,))) invoke(comms, PullRequestHead(source.github, 9, head), ReviewedHeadVerdict(ACCEPTED)) - posts = tuple(call for call in github.calls() if f"statuses/{head}" in call[1]) + posts = review_posts(github, head) assert len(posts) == 1 and any("(src +1)" in part for part in posts[0]) diff --git a/tests/test_historical_notifications.py b/tests/test_historical_notifications.py index 6c18bb27d0..ca1a14e3fe 100644 --- a/tests/test_historical_notifications.py +++ b/tests/test_historical_notifications.py @@ -26,9 +26,10 @@ from agent_comms.coordination_database import CoordinationStore from agent_comms.coordination_tables.harness_deliveries import HarnessDelivery from agent_comms.coordinator import Coordination -from agent_comms.harness_delivery_states import DeliveredOutcome +from agent_comms.harness_delivery_states import DeliveredOutcome, ObservedMonitor from agent_comms.coordination_errors import DMScopeChanged, SchemaVersionError from agent_comms.field_codec import InputId, MessageId, TurnId +from transcript_record_fixture import transcript_record from agent_comms.read_basis import HistoricalDisplay, ReadCursor from agent_comms.read_ledger import ReadLedger from agent_comms.thread_identity import AgentThreadRole, UserThreadRole @@ -72,7 +73,6 @@ def test_recorded_display_is_sparse_and_exact_incarnation(archived_notification) assert unpainted.displayed_to == () # A human display is Read and nothing else: no context entry rides along with it. assert painted.state == unpainted.state and not painted.state.consumed - assert MessageNotification.summary((painted,)).endswith("Consumed 0/1 · Read 1/1") assert live.bus.reads.cursor("reader", live.registry.snapshot()) == ReadCursor() ledger = ReadLedger.at(source.root) document = ledger.read() @@ -108,7 +108,8 @@ def test_harness_consumption_survives_human_display(archived_notification): with Coordination(str(CoordinationStore.path_in(live.root))) as store, \ store.session.writing() as db: attempt = HarnessDelivery(ClaudeBridge, recipient.recipient_lookup, shown.message_id, shown.seq, - DeliveredOutcome) + DeliveredOutcome, ObservedMonitor( + transcript_record('complete-original'), shown.timestamp)) assert attempt.begin(store.assignments, db) attempt.record(store.assignments, db) @@ -122,7 +123,8 @@ def human_display(): shown, human.incarnation, live.registry.snapshot(), document=None) consumed = notification() - assert consumed.label == "Consumed" and consumed.state.consumed and not consumed.displayed_to + assert consumed.state.consumed and not consumed.displayed_to + assert consumed.label == consumed.timed_state('Consumed', round(shown.timestamp * 1000)) ledger = live.bus.reads.read() assert human_display() == () # The UI's "mark read" on the agent's thread, as the registered human viewer. @@ -131,7 +133,6 @@ def human_display(): assert human_display() == (human.incarnation,) displayed = notification() assert displayed.state == consumed.state and not displayed.displayed_to - assert MessageNotification.summary((displayed,)).endswith("Consumed 1/1 · Read 0/1") def test_native_context_commit_is_consumed(): @@ -258,7 +259,8 @@ def test_sent_recipient_progresses_through_durable_consumption_and_answer(archiv activity=queued_activity, updated_at_ms=1_000, ) - consumed_activity = replace(sent_activity, progress=ShownToModelProgress(2_000)) + context_entry = ShownToModelProgress(2_000) + consumed_activity = replace(sent_activity, progress=context_entry) consumed = engagement.notification( recipient, SENDER, @@ -323,8 +325,9 @@ def delivered_to(observation, activity): assert idle.label.startswith("Consumed ") and not idle.state.seen_working unconsumed = delivered_to(observed(ActivityState.WORKING), sent_activity) assert unconsumed.label.startswith("Sent ") and not unconsumed.state.seen_working - confirmed = replace(unconsumed, state=unconsumed.state.harness_consumed()) - assert confirmed.label == "Seen, working" and confirmed.state.seen_working + confirmed = replace(unconsumed, state=unconsumed.state.harness_consumed(context_entry.at_ms)) + assert confirmed.state.seen_working and confirmed.state.consumed + assert confirmed.label == confirmed.timed_state(WorkingTakenProgress.label, context_entry.at_ms) # An explicit ignore decision is the only Seen, not answering. ignored = IgnoredAssignment.build(FullWake(), None, None) declined = ignored.notification(recipient, SENDER, observation=RecordedRecipientActivity(), @@ -334,9 +337,8 @@ def delivered_to(observation, activity): activity=sent_activity, updated_at_ms=2_000) assert unconfirmed.label == "Not answering" and not unconfirmed.state.seen_not_answering assert unconfirmed.state == IgnoredProgress(SentProgress(1_000)) - assert MessageNotification.summary((working, declined, unconfirmed, idle)) == ( - "Seen, working 1 · Seen, not answering 1 · Delivered 4/4 · Consumed 3/4 · Read 0/4") - assert MessageNotification.summary((unconfirmed,)) == "Not answering · Delivered 1/1 · Consumed 0/1 · Read 0/1" + assert sum(row.state.consumed for row in (working, declined, unconfirmed, idle)) == 3 + assert not unconfirmed.state.consumed assert delivered_answer.label.startswith("Answered ") assert delivered_answer.answer == MessageReference(99, reply_id) assert sent.recipient_identity == consumed.recipient_identity == answered.recipient_identity @@ -478,16 +480,29 @@ def test_live_cli_ack_does_not_become_archived_ack(archived_notification): store.session.writing() as db: for message in unread: attempt = HarnessDelivery(ClaudeBridge, lookup, message.message_id, message.seq, - DeliveredOutcome) + DeliveredOutcome, ObservedMonitor( + transcript_record('complete-original'), message.timestamp)) assert attempt.begin(store.assignments, db) attempt.record(store.assignments, db) current, = old.views.message_notifications_for_references(AllOutcomes(), (messages[0].reference,)).outcomes[ messages[0].seq, messages[0].message_id ] - assert current.label == "Consumed" and current.state.consumed and not current.displayed_to + assert current.state.consumed and not current.displayed_to + assert current.label == current.timed_state('Consumed', round(messages[0].timestamp * 1000)) with live.bus.history.read_sources() as sources: archived, = source.notification_references(sources, (messages[0].reference,), AllOutcomes()).outcomes[ messages[0].seq, messages[0].message_id ] assert archived.label == "Sent" and not archived.displayed_to + # The root-free recorded operation reads the same archive, not today's live evidence, + # and a key that is no longer attached refuses instead of falling back to live reads. + from agent_comms.coordination_errors import HistoricalSourceChanged + from agent_comms.notification_reads import RecordedNotificationRead + + recorded, = RecordedNotificationRead((messages[0].reference,), source.key).read(live).outcomes[ + messages[0].seq, messages[0].message_id + ] + assert recorded == archived + with pytest.raises(HistoricalSourceChanged): + RecordedNotificationRead((messages[0].reference,), source.key + "-detached").read(live) assert FieldCodec.decode(RecipientActivity, FieldCodec.encode(RecordedRecipientActivity())) == RecordedRecipientActivity() diff --git a/tests/test_historical_views.py b/tests/test_historical_views.py index 84f2c3bc21..9840d8e064 100644 --- a/tests/test_historical_views.py +++ b/tests/test_historical_views.py @@ -270,6 +270,12 @@ def test_recorded_transcript_witness_roundtrip_keeps_original_namespace(tmp_path assert FieldCodec.value_schema(type[TranscriptReadIdentity])["enum"] == list(TranscriptReadIdentity.names()) with pytest.raises(ValueError): FieldCodec.decode(TranscriptReadIdentity, {**FieldCodec.encode(read.identity), "kind": "unregistered"}) + from agent_comms.order_tools import TranscriptPageRead + from test_transcript_pages import shared_page + + recorded = shared_page(live, TranscriptPageRead(thread="alice", historical_source=source.key)) + assert recorded.identity == read.identity + assert any(event.text == "original recorded answer" for event in recorded.page.events) rebound = live.transcripts.bind_page_read("recorded-alice", decoded) assert rebound.current_identity() == decoded assert any(event.text == "original recorded answer" for event in rebound.read().events) diff --git a/tests/test_holder_standing_rules.py b/tests/test_holder_standing_rules.py index a38f6af895..8a73e47752 100644 --- a/tests/test_holder_standing_rules.py +++ b/tests/test_holder_standing_rules.py @@ -153,14 +153,14 @@ def test_stop_rejects_a_human_before_any_registry_change(comms, tmp_path): @pytest.fixture def claim_order(held_bus): """Publish the real order and claim messages, using their retained identities.""" - def claim(problem): + def claim(problem, *, pull_request=1, head="a" * 40): todo = held_bus.goals.todos.get(problem) lead = held_bus.registry.require("lead").incarnation worker = held_bus.registry.require("worker").incarnation original = held_bus.messaging.send("lead", "worker", todo.order.instructions(todo)).message todo = held_bus.goals.todos.apply( todo.id, OrderMessage(todo.revision, lead, original.message_id, at=RecordedEventTime(time.time()))) - claimed = OrderClaim.for_order(todo, worker, 1, "a" * 40, + claimed = OrderClaim.for_order(todo, worker, pull_request, head, "Review the published repair", original.reference, store=held_bus.goals.todos) claimed = replace(claimed, message=replace( diff --git a/tests/test_input_delivery_history.py b/tests/test_input_delivery_history.py index 19c94ada35..e5172282e1 100644 --- a/tests/test_input_delivery_history.py +++ b/tests/test_input_delivery_history.py @@ -1,23 +1,35 @@ """Current owner queues scope notices without altering durable input evidence.""" from agent_comms.repo import Checkout -import io -import json -from contextlib import redirect_stdout +import asyncio from dataclasses import replace -from agent_comms.cli import main +import pytest +from acp.schema import TextContentBlock + +from agent_comms.acp_extension import ( + GoalSnapshotReply, QueuePromptRequest, TurnChangedUpdate, UpdateMetadata, encode_request, +) +from agent_comms.child_process import NoProcess +from agent_comms.errors import RelationViolationError from agent_comms.input_key import AcpInputKey, BusInputKey from agent_comms.comms import wire -from agent_comms.field_codec import GoalId, InputId, TurnId +from agent_comms.field_codec import FieldCodec, GoalId, InputId, TurnId from agent_comms.input_attempt import ( OwnerQueueInputDeliveryScope, UnobservedInputDeliveryScope, UnresolvedInput, ) from agent_comms.input_disposition import InputDispositions +from agent_comms.goal_actions import SetGoalAction +from agent_comms.goals import NoGoal +from agent_comms.order_tools import InputDeliveryRead, OwnerGoalRead from agent_comms.reported_turn import ReportedTurn +from agent_comms.thread_status import RunningThreadStatus, StoppedThreadStatus from agent_comms.threads import Thread +from agent_comms.tool_caller import ToolCaller +from agent_comms.ui_model.observation import ReadOwnerOperation +from delivery_owner_fixture import canonical_delivery_owner from stored_inputs import stored_inputs @@ -103,18 +115,80 @@ def test_history_clear_is_notice_only_and_survives_rename_and_reopen(tmp_path): assert stored_inputs(ledger.root).rows[BusInputKey(7)].native_id == "b" * 32 -def test_commander_reads_a_units_failed_inputs_through_the_tool(tmp_path): - """2026-10-24: the commander hand-wrote SQL against the bus to list pi-mi's lost inputs.""" - comms = wire(tmp_path) - comms.registry.declare(Thread('owner', frozenset(), Checkout(tmp_path))) - seed(comms, 'owner') - output = io.StringIO() - with redirect_stdout(output): - code = main(['--root', str(comms.root), 'tool', '--tool', 'comms_input_delivery', - '--arguments', json.dumps({'thread': 'owner'})]) - assert code == 0, output.getvalue() - overview = json.loads(output.getvalue()) - assert {row['sequence'] for row in overview['inputs']} >= {2, 8} +async def test_mcp_and_the_ui_read_input_delivery_from_the_live_owner_queue(tmp_path): + """2026-10-24: the commander hand-wrote SQL to list pi-mi's lost inputs; the tool then + answered from the connector's own process, where no owner queue exists. Both boundaries + now carry one InputDeliveryRead to the owner and receive the owner's own answer.""" + async with canonical_delivery_owner(tmp_path) as (comms, agent, _, _): + comms.registry.register(comms.registry.require("alpha"), RunningThreadStatus()) + comms.agents.begin_turn("alpha", TurnId.mint(), "Held native fixture") + agent.inputs.state("alpha").open_inbox() + await agent.prompt("alpha", [TextContentBlock(type="text", text="queued")], + field_meta=encode_request(QueuePromptRequest("queued"))) + InputDispositions(comms.root).record( + BusInputKey(91), seq=91, owner="alpha", claim_scope=1, target="alpha", text="Off queue") + owner = comms.registry.require("alpha") + scope = await agent.inputs.delivery_scope("alpha") + assert type(scope) is OwnerQueueInputDeliveryScope + unobserved = comms.goals.input_delivery("alpha") + assert BusInputKey(91) in [row.input_id for row in unobserved.inputs] + answers = {} + for include_history in (False, True): + owned = comms.goals.input_delivery("alpha", include_history=include_history, scope=scope) + async with asyncio.timeout(30): + mcp = await asyncio.to_thread( + InputDeliveryRead.invoke, comms, + {"thread": "alpha", "include_history": include_history}, + caller=ToolCaller.from_actor("sender")) + ui = await ReadOwnerOperation( + str(comms.root), owner.incarnation, InputDeliveryRead(include_history=include_history), + ).answer(comms) + assert mcp == FieldCodec.encode(owned) + assert ui.value == owned + answers[include_history] = owned + assert BusInputKey(91) not in [row.input_id for row in answers[False].inputs] + assert answers[False].historical_inputs == () + assert BusInputKey(91) in [row.input_id for row in answers[True].historical_inputs] + assert replace(answers[True], historical_inputs=()) == answers[False] + comms.registry.register(replace(owner, process_identity=NoProcess()), StoppedThreadStatus()) + with pytest.raises(RelationViolationError): + async with asyncio.timeout(30): + await asyncio.to_thread(InputDeliveryRead.invoke, comms, {"thread": "alpha"}, + caller=ToolCaller.from_actor("sender")) + + +async def test_mcp_and_the_ui_read_the_owner_goal_from_the_live_owner(tmp_path): + """The owner answers comms_owner_goal under its own registration and session: MCP and the + UI's owner envelope receive the owner's own projection, its turn metadata included, for an + empty goal and for a set one.""" + async with canonical_delivery_owner(tmp_path) as (comms, agent, _, _): + comms.registry.register(comms.registry.require("alpha"), RunningThreadStatus()) + comms.agents.begin_turn("alpha", TurnId.mint(), "Held native fixture") + owner = comms.registry.require("alpha") + replies = [] + for change in (lambda: None, + lambda: comms.goals.update_goal("alpha", SetGoalAction(text="Ship the read"))): + change() + owned = GoalSnapshotReply.from_snapshot( + comms.goals.goal_snapshot("alpha"), + UpdateMetadata.of(TurnChangedUpdate(agent.turns.turn_state("alpha")))) + async with asyncio.timeout(30): + mcp = await asyncio.to_thread( + OwnerGoalRead.invoke, comms, {"thread": "alpha"}, + caller=ToolCaller.from_actor("sender")) + ui = await ReadOwnerOperation( + str(comms.root), owner.incarnation, OwnerGoalRead()).answer(comms) + assert mcp == FieldCodec.encode(owned) + assert ui.value == owned + assert ui.value.meta.batch.updates == (TurnChangedUpdate(agent.turns.turn_state("alpha")),) + replies.append(owned) + assert replies[0].goal == NoGoal() + assert replies[1].goal.text == "Ship the read" + comms.registry.register(replace(owner, process_identity=NoProcess()), StoppedThreadStatus()) + with pytest.raises(RelationViolationError): + async with asyncio.timeout(30): + await asyncio.to_thread(OwnerGoalRead.invoke, comms, {"thread": "alpha"}, + caller=ToolCaller.from_actor("sender")) def test_no_owner_queue_observation_never_dismisses_inputs(tmp_path): diff --git a/tests/test_integration.py b/tests/test_integration.py index 2e91c7e33c..0b722dea5d 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -1,22 +1,38 @@ """Integration policy at real TOML/Git/gh boundaries; no patched Core methods.""" +from collections.abc import Callable +from dataclasses import replace +import json import os from pathlib import Path import subprocess +import sys +import tomllib import pytest +import tomli_w from agent_comms import integration -from agent_comms.head_review import PullRequestHead +from agent_comms.errors import ToolRefusedError +from agent_comms.head_review import BaseSide, HeadReview, HeadSide, IntegrationInput, PromotedHeadReview, PullRequestHead from agent_comms.repository_name import RepositoryName from agent_comms.repo import Repo from agent_comms.todos import ( - DraftListedPullRequest, ReviewedHeadVerdict, UnmergedListedPullRequest, VerdictReason, + DraftListedPullRequest, MergedListedPullRequest, PullRequestCommit, QualifiedHeadVerdict, + ReviewedHeadVerdict, UnmergedListedPullRequest, VerdictReason, ) +from agent_comms.field_codec import FieldCodec +from agent_comms.goal_actions import CreateProblemGoalAction +from agent_comms.recorded_time import RecordedEventTime +from agent_comms.installer.runtime_resolution import StackSources +from github_fixture import GitHubFixture +from integration_source_fixture import MutualSources from source_revision_fixture import SourceRepository from fixture_git import ENVIRONMENT, SETTINGS +from debt_ratchet import ClientTypeErrors, compare +from module_layers import MaterializedClients, Package -pytest_plugins = ("github_fixture",) +pytest_plugins = ("github_fixture", "test_holder_standing_rules", "integration_source_fixture") @pytest.fixture @@ -25,6 +41,277 @@ def heads() -> tuple[PullRequestHead, ...]: return (PullRequestHead(repo, 7, "a" * 40), PullRequestHead(repo, 8, "b" * 40)) +@pytest.mark.parametrize("event", ("pull_request", "merge_group")) +def test_actual_ci_cli_uses_verified_carrier_not_synthetic_message_and_revalidates( + mutual_sources: MutualSources, github_cli: GitHubFixture, + tmp_path: Path, event: str, +) -> None: + pair = mutual_sources + inputs = pair.capture(("P2", "P3"), "CI source graph") + carrier = pair.publish_carrier(github_cli) + local = inputs.source(pair.core.github) + # Actual synthetic merge: its own message deliberately has no source metadata. + tested = pair.core.git(pair.core.checkout, "commit-tree", local.candidate_tree, + "-p", local.baseline.commit, "-p", carrier.head, + "-m", "Synthetic PR/queue merge; no input metadata").strip() + checkout = integration.IntegrationCheckout.create(pair.core, tmp_path / "ci-checkout", tested) + assert IntegrationInput.from_message(Repo.git(checkout.path, "show", "--no-patch", "--format=%B", "HEAD")) == () + documents = { + "pull_request": {"number": carrier.pr, "pull_request": {"head": {"sha": carrier.head}}}, + "merge_group": {"merge_group": {"head_sha": tested, "base_sha": local.baseline.commit, + "head_ref": f"refs/heads/gh-readonly-queue/main/pr-{carrier.pr}-{carrier.head}"}}, + } + document = tmp_path / "event.json" + document.write_text(json.dumps(documents[event])) + output = tmp_path / "resolved" + environment = {**os.environ, **ENVIRONMENT, "GITHUB_EVENT_NAME": event, + "GITHUB_EVENT_PATH": str(document), "GITHUB_SHA": tested, + "GITHUB_REPOSITORY": pair.core.github.to_wire(), "RUNNER_TEMP": str(tmp_path), + "PYTHONPATH": str(Path(__file__).resolve().parents[1] / "src")} + command = (sys.executable, "-m", "agent_comms.integration") + result = subprocess.run((*command, "--resolve-ci-inputs", str(output)), cwd=checkout.path, + env=environment, capture_output=True, text=True, timeout=120) + assert result.returncode == 0, result.stdout + result.stderr + assert IntegrationInput.at(output) == inputs + assert inputs.carrier.pr == carrier.pr != pair.core.number + assert local.heads == (PullRequestHead(pair.core.github, pair.core.number, pair.core.branch_commit),) + assert pair.core.git(pair.checkout.path, "rev-parse", "HEAD^").strip() == inputs.carrier.head + for source in inputs.sources: + for side in (BaseSide, HeadSide): + side.require_materialized(output, source) + original = pair.core.git(pair.core.checkout, "rev-parse", "HEAD").strip() + result = subprocess.run((*command, "--revalidate-ci-inputs", str(output)), cwd=checkout.path, + env=environment, capture_output=True, text=True, timeout=120) + assert result.returncode == 0, result.stdout + result.stderr + pair.client.git(pair.client.checkout, "push", "origin", "HEAD:main") + stale = subprocess.run((*command, "--revalidate-ci-inputs", str(output)), cwd=checkout.path, + env=environment, capture_output=True, text=True, timeout=120) + assert stale.returncode != 0 and "main moved" in stale.stderr + assert pair.core.git(pair.core.checkout, "rev-parse", "HEAD").strip() == original + assert not any(call[:2] == ("pr", "merge") or "state=success" in call for call in github_cli.calls()) + + +def test_actual_ordinary_ci_cli_captures_main_once_and_never_steals_an_unlinked_overlay( + mutual_sources: MutualSources, github_cli: GitHubFixture, tmp_path: Path, + materialized_clients: Callable[[Path, IntegrationInput], MaterializedClients], +) -> None: + pair = mutual_sources + head = pair.core.branch_commit + baseline = pair.core.main_commit + tree = pair.core.git(pair.core.checkout, "rev-parse", f"{head}^{{tree}}").strip() + tested = pair.core.git(pair.core.checkout, "commit-tree", tree, "-p", baseline, + "-p", head, "-m", "Actual ordinary synthetic merge").strip() + checkout = integration.IntegrationCheckout.create(pair.core, tmp_path / "ordinary-ci", tested) + document = tmp_path / "ordinary-event.json" + document.write_text(json.dumps({"number": pair.core.number, "pull_request": {"head": {"sha": head}}})) + output = tmp_path / "ordinary-inputs" + environment = {**os.environ, **ENVIRONMENT, "GITHUB_EVENT_NAME": "pull_request", + "GITHUB_EVENT_PATH": str(document), "GITHUB_SHA": tested, + "GITHUB_REPOSITORY": pair.core.github.to_wire(), "RUNNER_TEMP": str(tmp_path), + "PYTHONPATH": str(Path(__file__).resolve().parents[1] / "src")} + result = subprocess.run((sys.executable, "-m", "agent_comms.integration", "--resolve-ci-inputs", str(output)), + cwd=checkout.path, env=environment, capture_output=True, text=True, timeout=120) + assert result.returncode == 0, result.stdout + result.stderr + inputs = IntegrationInput.at(output) + core = inputs.source(pair.core.github) + client = inputs.source(pair.client.github) + assert core.baseline.commit == baseline + assert core.heads == (PullRequestHead(pair.core.github, pair.core.number, head),) + assert client.heads == () and client.baseline.commit == pair.client.main_commit + resource = materialized_clients(output, inputs) + base = BaseSide.require_materialized(output, core) + candidate = HeadSide.require_materialized(output, core) + assert Package(base, resource.at(checkout.path, baseline, tested, BaseSide)).missing_names == () + assert {name for _, name in Package(candidate, resource.at(checkout.path, baseline, tested, HeadSide)).missing_names} == {"old_api"} + assert not any("state=success" in call or call[:2] == ("pr", "merge") for call in github_cli.calls()) + refused = subprocess.run((sys.executable, "-m", "agent_comms.integration", "--check-merge-inputs", + str(tmp_path / "refused-individual")), cwd=checkout.path, + env=environment, capture_output=True, text=True, timeout=120) + assert refused.returncode != 0 and "individual source" in refused.stderr + + +@pytest.mark.parametrize("event", ("pull_request", "merge_group")) +def test_required_hosted_merge_gate_distinguishes_source_review_from_combined_promotion( + mutual_sources: MutualSources, github_cli: GitHubFixture, tmp_path: Path, event: str, +) -> None: + pair = mutual_sources + inputs = pair.capture(("P2", "P3"), "hosted gate must refuse an individually armed source") + carrier = pair.publish_carrier(github_cli) + for source in inputs.sources: + for head in source.heads: + ReviewedHeadVerdict(VerdictReason("The exact source is accepted for this actual combined carrier")).post( + head.repo, head.head, "lead", "source") + # A source review on the combined head is still not a merge promotion. + ReviewedHeadVerdict(VerdictReason("The combined source is reviewed but not yet promoted for merge")).post( + carrier.repo, carrier.head, "lead", "combined source") + local = inputs.source(pair.core.github) + tested = pair.core.git(pair.core.checkout, "commit-tree", local.candidate_tree, + "-p", local.baseline.commit, "-p", carrier.head, + "-m", "Actual queue/PR merge").strip() + checkout = integration.IntegrationCheckout.create(pair.core, tmp_path / "merge-gate", tested) + documents = { + "pull_request": {"number": carrier.pr, "pull_request": {"head": {"sha": carrier.head}}}, + "merge_group": {"merge_group": {"head_sha": tested, "base_sha": local.baseline.commit, + "head_ref": f"refs/heads/gh-readonly-queue/main/pr-{carrier.pr}-{carrier.head}"}}, + } + document = tmp_path / "event.json" + document.write_text(json.dumps(documents[event])) + environment = {**os.environ, **ENVIRONMENT, "GITHUB_EVENT_NAME": event, + "GITHUB_EVENT_PATH": str(document), "GITHUB_SHA": tested, + "GITHUB_REPOSITORY": pair.core.github.to_wire(), "RUNNER_TEMP": str(tmp_path), + "PYTHONPATH": str(Path(__file__).resolve().parents[1] / "src")} + command = (sys.executable, "-m", "agent_comms.integration", "--check-merge-inputs") + result = subprocess.run((*command, str(tmp_path / "before-promotion")), cwd=checkout.path, + env=environment, capture_output=True, text=True, timeout=120) + assert (result.returncode == 0) == (event == "pull_request"), result.stdout + result.stderr + if event == "merge_group": + assert "no explicit reviewed merge promotion" in result.stderr + ReviewedHeadVerdict(VerdictReason("The actual combined carrier is explicitly reviewed for merge")).post( + carrier.repo, carrier.head, "lead", "source") + PromotedHeadReview().post(carrier.repo, carrier.head, "lead", "promotion") + promoted = subprocess.run((*command, str(tmp_path / "after-promotion")), cwd=checkout.path, + env=environment, capture_output=True, text=True, timeout=120) + assert promoted.returncode == 0, promoted.stdout + promoted.stderr + assert not any(call[:2] == ("pr", "merge") for call in github_cli.calls()) + + +def test_actual_main_cli_keeps_merged_local_members_exact_and_refuses_later_head_changes( + mutual_sources: MutualSources, github_cli: GitHubFixture, tmp_path: Path, +) -> None: + pair = mutual_sources + peer = SourceRepository(checkout=pair.core.checkout, github=pair.core.github, number=8) + peer.git(peer.checkout, "switch", "-c", "second-local-source", "main") + peer_head = peer.publish_change("Independent local source", *peer.write_files({ + Path("src/agent_comms/extra.py"): "extra = 1\n"})) + peer.git(peer.checkout, "switch", "feature") + assert peer.git(peer.checkout, "ls-remote", "origin", f"refs/pull/{peer.number}/head").split() == [ + peer_head, f"refs/pull/{peer.number}/head"] + stamp = "2026-10-24T00:00:00Z" + github_cli.put(peer.github, DraftListedPullRequest( + peer.number, "second-local-source", peer_head, stamp, "main", reviewable_times=(stamp,))) + primary = pair.context.comms.goals.todos.get("P2") + pair.context.comms.goals.update_goal("worker", CreateProblemGoalAction( + problem="P4", commander="lead", request=primary.request, + assigner=pair.context.comms.registry.require("lead").incarnation, + repo=peer.github, text="Include the existing local source", + order=replace(primary.order.checked(), pull_requests=(peer.link,), + check=replace(primary.order.check, directory=peer.checkout)), + review_at=RecordedEventTime(9999999999.0))) + inputs = pair.capture(("P2", "P3", "P4"), "actual main tuple") + carrier = pair.publish_carrier(github_cli) + pair.core.git(pair.core.checkout, "switch", "main") + pair.core.git(pair.core.checkout, "merge", "--no-ff", "--no-edit", carrier.head) + tested = pair.core.git(pair.core.checkout, "rev-parse", "HEAD").strip() + pair.core.git(pair.core.checkout, "push", "origin", "main") + for number, head in ((carrier.pr, carrier.head), (peer.number, peer_head)): + github_cli.put(pair.core.github, MergedListedPullRequest( + number, "feature", head, stamp, "main", reviewable_times=(stamp,), + merge=PullRequestCommit(tested), merged=stamp)) + document = tmp_path / "main-event.json" + document.write_text(json.dumps({"ref": "refs/heads/main", "after": tested})) + output = tmp_path / "main-inputs" + environment = {**os.environ, **ENVIRONMENT, "GITHUB_EVENT_NAME": "push", + "GITHUB_EVENT_PATH": str(document), "GITHUB_SHA": tested, + "GITHUB_REPOSITORY": pair.core.github.to_wire(), "RUNNER_TEMP": str(tmp_path), + "PYTHONPATH": str(Path(__file__).resolve().parents[1] / "src")} + command = (sys.executable, "-m", "agent_comms.integration") + result = subprocess.run((*command, "--resolve-ci-inputs", str(output)), cwd=pair.core.checkout, + env=environment, capture_output=True, text=True, timeout=120) + assert result.returncode == 0, result.stdout + result.stderr + assert IntegrationInput.at(output) == inputs + result = subprocess.run((*command, "--revalidate-ci-inputs", str(output)), cwd=pair.core.checkout, + env=environment, capture_output=True, text=True, timeout=120) + assert result.returncode == 0, result.stdout + result.stderr + github_cli.put(peer.github, MergedListedPullRequest( + peer.number, "second-local-source", "f" * 40, stamp, "main", reviewable_times=(stamp,), + merge=PullRequestCommit(tested), merged=stamp)) + stale = subprocess.run((*command, "--revalidate-ci-inputs", str(output)), cwd=pair.core.checkout, + env=environment, capture_output=True, text=True, timeout=120) + assert stale.returncode != 0 and "exact captured source" in stale.stderr + assert not any("state=success" in call or call[:2] == ("pr", "merge") for call in github_cli.calls()) + + +def test_real_client_checker_compares_the_matched_tuple_in_its_one_interpreter( + mutual_sources: MutualSources, github_cli: GitHubFixture, + materialized_clients: Callable[[Path, IntegrationInput], MaterializedClients], +) -> None: + pair = mutual_sources + plugin = ('from mypy.plugin import Plugin\nfrom agent_comms.api import old_api\n' + 'from performance.capture_live import capture\n' + 'def plugin(version: str) -> type[Plugin]:\n' + ' assert old_api() == capture() == "baseline"\n return Plugin\n') + capture = 'from agent_comms.api import old_api\n\ndef capture() -> str:\n return old_api()\n' + script = ('from agent_comms.api import old_api\nfrom performance.capture_live import capture\n' + 'assert old_api() == capture() == "baseline"\n') + # The native config and its plugin are real baseline source, then migrated on feature. + pair.client.git(pair.client.checkout, "switch", "main") + project = tomllib.loads((pair.client.checkout / "pyproject.toml").read_text()) + project["tool"]["mypy"] = {"plugins": ["mypy_source_plugin.py"], "mypy_path": ["src", "tools"]} + pair.client.commit_change("Declare baseline script roots and native mypy plugin", *pair.client.write_files({ + Path("pyproject.toml"): tomli_w.dumps(project), Path("mypy_source_plugin.py"): plugin, + Path("tools/performance/capture_live.py"): capture, Path("tools/acceptance.py"): script})) + pair.client.git(pair.client.checkout, "push", "origin", "main") + pair.client.git(pair.client.checkout, "switch", "feature") + pair.client.git(pair.client.checkout, "merge", "--no-edit", "main") + head = pair.client.publish_change("Migrate scripts and native plugin alongside the client", *pair.client.write_files({ + path: text.replace("old_api", "new_api").replace("baseline", "candidate") + for path, text in ((Path("mypy_source_plugin.py"), plugin), + (Path("tools/performance/capture_live.py"), capture), + (Path("tools/acceptance.py"), script))})) + stamp = "2026-10-24T00:00:00Z" + github_cli.put(pair.client.github, DraftListedPullRequest( + pair.client.number, "feature", head, stamp, "main", reviewable_times=(stamp,))) + inputs = pair.capture(("P2", "P3"), "real client checker tuple and native plugins") + source = inputs.source(pair.core.github) + head = pair.core.git(pair.checkout.path, "rev-parse", "HEAD").strip() + clients = materialized_clients(pair.output, inputs) + assert len(ClientTypeErrors.parts(pair.core.checkout, source.baseline.commit, head, + {"src/agent_comms/api.py"}, "src/", clients)) == 2 + for side in (BaseSide, HeadSide): + client, = clients.at(pair.core.checkout, source.baseline.commit, head, side) + assert client.import_paths == tuple(str(client.checkout / path) for path in ("src", "tools")) + assert client.files_importing("agent_comms") == [ + "mypy_source_plugin.py", "src/workflow.py", "tools/acceptance.py", "tools/performance/capture_live.py"] + # Python's script entrypoint and the plugin's import must see this same cut. + entry = subprocess.run((sys.executable, "tools/acceptance.py"), cwd=client.checkout, + env={**os.environ, "PYTHONPATH": os.pathsep.join(( + str(side.require_materialized(pair.output, source) / "src"), *client.import_paths))}, + capture_output=True, text=True, timeout=30) + assert entry.returncode == 0, entry.stdout + entry.stderr + # Both namespace files are selected under --explicit-package-bases: a duplicate + # performance/tools.performance identity aborts the real checker, not a zero count. + result = compare(pair.core.checkout, source.baseline.commit, head, "src", 1, + measured=(ClientTypeErrors,), clients=clients) + assert not result.failed + assert all(value == 0 for value in result.delta.values()) + + +def test_ci_event_refuses_unrelated_graph_and_carrier_without_materialization( + mutual_sources: MutualSources, github_cli: GitHubFixture, tmp_path: Path, +) -> None: + pair = mutual_sources + carrier = PullRequestHead(pair.core.github, pair.core.number, pair.core.branch_commit) + # main is not a two-parent PR test graph, regardless of plausible event text. + with pytest.raises(integration.IntegrationFailed, match="two-parent"): + integration.PullRequestSources.resolve(pair.core, pair.core.main_commit, + {"number": carrier.pr, "pull_request": {"head": {"sha": carrier.head}}}, + environment=integration.RunnerEnvironment(root=tmp_path)) + with pytest.raises(integration.IntegrationFailed, match="queued carrier"): + integration.MergeGroupSources.resolve(pair.core, pair.core.main_commit, + {"merge_group": {"head_sha": pair.core.main_commit, "base_sha": pair.core.main_commit, + "head_ref": "refs/heads/not-a-GitHub-queue-reference"}}, + environment=integration.RunnerEnvironment(root=tmp_path)) + assert not any(call[:2] == ("pr", "merge") for call in github_cli.calls()) + + +def test_stack_source_identity_normalizes_package_spelling_and_refuses_duplicate_aliases() -> None: + source = '{git = "https://github.com/example/client.git", branch = "main"}' + assert StackSources.declared_sources(f'[tool.uv.sources]\nSource_Client = {source}\n') == { + "source-client": ("https://github.com/example/client.git", "main")} + with pytest.raises(ValueError, match="declares package source-client twice"): + StackSources.declared_sources(f'[tool.uv.sources]\nSource_Client = {source}\nsource-client = {source}\n') + + def test_selection_uses_the_project_declaration_and_unions_existing_files(tmp_path: Path): (tmp_path / "pyproject.toml").write_text( '[tool.pytest.ini_options]\ntestpaths = ["checks"]\n' @@ -50,6 +337,20 @@ def test_only_accepted_ready_members_are_selected(github_cli, tmp_path: Path): github_cli.put(source.github, DraftListedPullRequest( 8, "draft", source.branch_commit, stamp, "main", reviewable_times=(stamp,), )) + (source.checkout / "stage.py").write_text("source_stage = True\n") + source.git(source.checkout, "add", "stage.py") + source.git(source.checkout, "commit", "-m", "Prepare an incomplete source stage") + qualified = source.git(source.checkout, "rev-parse", "HEAD").strip() + source.git(source.checkout, "push", "origin", "HEAD:refs/pull/9/head") + github_cli.put(source.github, UnmergedListedPullRequest( + 9, "qualified", qualified, stamp, "main", reviewable_times=(stamp,), + )) + github_cli.put(source.github, UnmergedListedPullRequest( + 10, "unaccepted", source.main_commit, stamp, "main", reviewable_times=(stamp,), + )) + QualifiedHeadVerdict(VerdictReason("The source stage is reviewed while paired implementation remains incomplete")).post( + source.github, qualified, "lead", "source stage", + ) run = integration.IntegrationRun(integration.IntegrationCheckout(source.checkout), tmp_path / "output", "ready-head selection") assert run.accepted_heads() == () @@ -59,6 +360,86 @@ def test_only_accepted_ready_members_are_selected(github_cli, tmp_path: Path): assert run.accepted_heads() == (PullRequestHead(source.github, 7, source.branch_commit),) +def test_qualification_captures_actual_orders_without_reviewing_or_arming_sources( + mutual_sources: MutualSources, github_cli: GitHubFixture) -> None: + pair = mutual_sources + run = integration.IntegrationRun(pair.checkout, pair.output, "mutually migrated API/import source") + inputs = pair.capture(("P2", "P3"), run.risk) + assert FieldCodec.decode(IntegrationInput, FieldCodec.parse((pair.output / "input.json").read_bytes())) == inputs + carrier = Repo.git(pair.checkout.path, "rev-parse", "HEAD").strip() + assert IntegrationInput.from_message(Repo.git(pair.checkout.path, "show", "--format=%B", "--no-patch", carrier)) == (inputs,) + for source in inputs.sources: + assert len(source.heads) == 1 + assert Repo.descends(HeadSide.checkout(pair.output, source.repository), source.heads[0].head, "HEAD") + assert Repo.git(BaseSide.checkout(pair.output, source.repository), "rev-parse", "HEAD").strip() == source.baseline.commit + assert Repo.git(HeadSide.checkout(pair.output, source.repository), "rev-parse", "HEAD^{tree}").strip() == source.candidate_tree + assert not HeadReview.accepted(source.repository, source.heads[0].head) + assert run.accepted_heads() == () + assert inputs.carrier.pr != pair.core.number + assert pair.core.git(pair.checkout.path, "rev-parse", "HEAD^").strip() == inputs.carrier.head + assert all(not head.same_pull_request(inputs.carrier) for source in inputs.sources for head in source.heads) + assert not any(call[:2] == ("pr", "merge") or "POST" in call for call in github_cli.calls()) + + +def test_real_mutual_imports_work_for_both_matched_cuts_and_fail_for_both_mixed_cuts( + mutual_sources: MutualSources) -> None: + import sys + + pair = mutual_sources + pair.capture(("P2", "P3"), "both mixed imports must fail") + results = {} + for core_side in (BaseSide, HeadSide): + for client_side in (BaseSide, HeadSide): + environment = {**os.environ, "PYTHONPATH": os.pathsep.join(( + str(core_side.checkout(pair.output, pair.core.github) / "src"), + str(client_side.checkout(pair.output, pair.client.github) / "src"), + ))} + result = subprocess.run( + (sys.executable, "-c", "from workflow import run; print(run())"), + cwd=pair.output, env=environment, text=True, capture_output=True, timeout=30, + ) + results[core_side, client_side] = result + assert results[BaseSide, BaseSide].returncode == 0 + assert results[BaseSide, BaseSide].stdout.strip() == "baseline" + assert results[HeadSide, HeadSide].returncode == 0 + assert results[HeadSide, HeadSide].stdout.strip() == "candidate" + assert results[BaseSide, HeadSide].returncode != 0 + assert results[HeadSide, BaseSide].returncode != 0 + + +def test_changed_main_during_source_review_refuses_the_frozen_input( + mutual_sources: MutualSources, github_cli: GitHubFixture) -> None: + pair = mutual_sources + run = integration.IntegrationRun(pair.checkout, pair.output, "stale baseline must refuse") + inputs = pair.capture(("P2", "P3"), run.risk) + carrier = pair.publish_carrier(github_cli) + pair.client.git(pair.client.checkout, "checkout", "main") + pair.client.commit_change("Advance client main", *pair.client.write_files({Path("main-only.py"): "advanced = True\n"})) + pair.client.git(pair.client.checkout, "push", "origin", "main") + with pytest.raises(ToolRefusedError, match="main moved"): + inputs.validate(pair.context.repos, carrier) + + +def test_common_runtime_contract_allows_local_source_and_metadata_changes() -> None: + baseline = ('[project]\nname="old-label"\nversion="1"\n' + 'dependencies=["agent-comms", "old-client", "packaging>=25"]\n[tool.uv]\npackage=false\n' + '[tool.uv.sources]\nagent-comms={path="..",editable=true}\n' + 'old-client={git="https://github.com/example/old.git",branch="main"}\n') + candidate = ('[project]\nname="new-label"\nversion="2"\n' + 'dependencies=["agent-comms", "new-client", "packaging>=25"]\n[tool.uv]\npackage=false\n' + '[tool.uv.sources]\nagent-comms={path="..",editable=true}\n' + 'new-client={git="https://github.com/example/new.git",branch="main"}\n') + StackSources.require_compatible(baseline, candidate) + StackSources.require_compatible(candidate, baseline) + + +def test_common_runtime_contract_refuses_unsupported_external_dependency_migration() -> None: + baseline = '[project]\ndependencies=["packaging>=25"]\n[tool.uv]\npackage=false\n' + candidate = '[project]\ndependencies=["packaging>=26"]\n[tool.uv]\npackage=false\n' + with pytest.raises(ValueError): + StackSources.require_compatible(baseline, candidate) + + def test_main_checkout_is_refused_without_changes(tmp_path: Path): source = SourceRepository.create(tmp_path / "main-source") before = source.main_commit diff --git a/tests/test_live_context_notifications.py b/tests/test_live_context_notifications.py new file mode 100644 index 0000000000..edbb443491 --- /dev/null +++ b/tests/test_live_context_notifications.py @@ -0,0 +1,171 @@ +"""Real reserved inputs, source files and journals; no native/provider substitute.""" +from collections.abc import Callable +from contextlib import closing +from dataclasses import replace +from pathlib import Path +import sqlite3 + +import pytest + +from agent_comms.coordination_errors import IdentityConflict +from agent_comms.coordinator import Coordination +from agent_comms.field_codec import InputId +from agent_comms.native_input_proof import NativeContextJournal, NativeContextProof +from agent_comms.native_runtime_input import NativeRuntimeInput +from agent_comms.presentation import AllOutcomes +from agent_comms.private_send_claim import PrivateSendClaim +from agent_comms.private_sidecar import native_request_digest +from agent_comms.selected_triage import AbsentTriageDecisionRecord, DecidedTriageRecord, IgnoreSelectedTriage +from native_proof_cases import append_input_entry, recorded_context, write_proof_rows +from test_coordinated_runtime import tmp_path # noqa: F401 +from test_native_failure_recovery import SentHistory, prepared_native_input, prepared_native_triage + + +async def recorded_input_context(claim: PrivateSendClaim) -> NativeContextProof: + """Join the isolated raw writer before returning its retained session/journal proof.""" + def write() -> NativeContextProof: + session, participant = claim.session, claim.selected + SentHistory.write(claim, session, participant) + identity = session.identity + entry = 'input-before-answer' + offset = append_input_entry(identity.path, { + 'type': 'message', 'id': entry, + 'timestamp': '2026-10-01T00:00:00.000Z', + 'message': {'role': 'user', 'inputId': claim.input_id, + 'inputDigest': native_request_digest(claim.prompt.text), + 'content': [{'type': 'text', 'text': claim.prompt.text}]}, + }) + record = { + 'schema': 2, 'type': 'context_committed', + 'inputId': claim.input_id, 'sessionId': identity.session_id, + 'sessionEntryId': entry, 'requestGeneration': 1, + 'llmContextDigest': 'f' * 64, 'committedAtMs': 1_791_964_800_001, + } + write_proof_rows(session.files(participant.comms.root), (record,)) + return recorded_context(identity.path, record, session_entry_start=offset) + + return await Coordination.run_worker(write) + + +async def acknowledge_context(claim: PrivateSendClaim, context: NativeContextProof) -> None: + async with claim.selected.cursor_delivery.reading() as source_reads: + await claim.context_committed(context, source_reads) + + +@pytest.mark.parametrize('source_count', (1, 3)) +async def test_reserved_context_notifies_before_any_assistant_reply(tmp_path: Path, source_count: int): + async with prepared_native_input( + tmp_path, source_bodies=tuple(f'Grouped source {index}' for index in range(source_count - 1)), + ) as claim: + context = await recorded_input_context(claim) + comms = claim.selected.comms + originals = tuple(row.source for row in claim.stage.assignments) + assert len(originals) == source_count + before = comms.views.message_notifications_for_references(AllOutcomes(), originals) + before_rows = tuple(row for rows in before.outcomes.values() for row in rows) + assert before_rows and all(not row.state.consumed for row in before_rows) + + await acknowledge_context(claim, context) + + after = comms.views.message_notifications_for_references(AllOutcomes(), originals) + after_rows = tuple(row for rows in after.outcomes.values() for row in rows) + assert after_rows and all(row.state.consumed for row in after_rows) + assert all(row.answer is None for row in after_rows) + assert all(not row.displayed_to for row in after_rows) + assert claim.stage.progress.current.starting + with Coordination(claim.store_path) as store: + original = NativeRuntimeInput.require(store.session._connection, input_id=claim.input_id) + assert original.require_context_proof() == context + claim.commit(store, context) + claim.commit(store, context) + retained = NativeRuntimeInput.require(store.session._connection, input_id=claim.input_id) + assert retained == original + + +async def test_triage_consumption_precedes_and_survives_later_verdict(tmp_path: Path): + async with prepared_native_triage(tmp_path) as claim: + context = await recorded_input_context(claim) + originals = tuple(row.source for row in claim.stage.assignments) + before = claim.selected.comms.views.message_notifications_for_references(AllOutcomes(), originals) + assert all(not row.state.consumed for rows in before.outcomes.values() for row in rows) + await acknowledge_context(claim, context) + active = claim.selected.comms.views.message_notifications_for_references(AllOutcomes(), originals) + assert sum(row.state.consumed for rows in active.outcomes.values() for row in rows) == 1 + assert all(row.answer is None and not row.displayed_to + for rows in active.outcomes.values() for row in rows) + with Coordination(claim.store_path) as store: + original = NativeRuntimeInput.require(store.session._connection, input_id=claim.input_id) + assert original.verdict == AbsentTriageDecisionRecord() + claim.commit(store, context) + claim.stage.commit(store, claim.participant, claim.input_id, claim.token_digest, + context, IgnoreSelectedTriage()) + retained = NativeRuntimeInput.require(store.session._connection, input_id=claim.input_id) + assert retained.require_context_proof() == context + assert retained.verdict == DecidedTriageRecord(IgnoreSelectedTriage) + with store.session.writing() as db: + with pytest.raises(IdentityConflict): + retained.commit_context(db, replace(context, llm_context_digest='e' * 64)) + with pytest.raises(sqlite3.IntegrityError): + NativeRuntimeInput.update(db, where='input_id=?', parameters=(claim.input_id,), + verdict=AbsentTriageDecisionRecord()) + + +async def test_triage_failure_keeps_previously_acknowledged_context_and_no_retry(tmp_path: Path): + async with prepared_native_triage(tmp_path) as claim: + context = await recorded_input_context(claim) + await acknowledge_context(claim, context) + with Coordination(claim.store_path) as store: + claim.stage.fail_decision(store, claim.participant, claim.input_id, claim.token_digest, context) + retained = NativeRuntimeInput.require(store.session._connection, input_id=claim.input_id) + assert retained.require_context_proof() == context + assert retained.verdict == AbsentTriageDecisionRecord() + with pytest.raises(IdentityConflict): + claim.stage.reserve(store, claim.participant, claim.token_digest) + assert len(NativeRuntimeInput.select(store.session._connection)) == 1 + + +async def test_later_corroborated_request_preserves_first_context_receipt(tmp_path: Path): + async with prepared_native_input(tmp_path) as claim: + context = await recorded_input_context(claim) + await acknowledge_context(claim, context) + files = claim.session.files(claim.selected.comms.root) + # Append through the declared journal owner, preserving its original file identity. + with closing(sqlite3.connect(files.input_proof)) as db, db: + first = NativeContextJournal.for_input(db, claim.input_id, context.request_generation) + assert first is not None + later = replace(first, request_generation=first.request_generation + 1, + llm_context_digest='d' * 64, committed_at_ms=first.committed_at_ms + 1) + later.insert(db) + later_context = later.at(context.session_file, session_entry_start=context.session_entry_start) + await acknowledge_context(claim, later_context) + with Coordination(claim.store_path) as store: + claim.commit(store, later_context) + retained = NativeRuntimeInput.require(store.session._connection, input_id=claim.input_id) + assert retained.require_context_proof() == context + + +async def test_same_request_contradiction_cannot_replace_context_receipt(tmp_path: Path): + async with prepared_native_input(tmp_path) as claim: + context = await recorded_input_context(claim) + await acknowledge_context(claim, context) + with Coordination(claim.store_path) as store: + with pytest.raises(IdentityConflict): + claim.commit(store, replace(context, llm_context_digest='e' * 64)) + retained = NativeRuntimeInput.require(store.session._connection, input_id=claim.input_id) + assert retained.require_context_proof() == context + + +@pytest.mark.parametrize('corrupt', ( + lambda context: replace(context, llm_context_digest='e' * 64), + lambda context: replace(context, input_id=InputId('00000000000000000000000000000001')), +)) +async def test_uncorroborated_live_context_never_grants_consumption( + tmp_path: Path, corrupt: Callable[[NativeContextProof], NativeContextProof], +): + async with prepared_native_input(tmp_path) as claim: + context = await recorded_input_context(claim) + with pytest.raises(IdentityConflict): + await acknowledge_context(claim, corrupt(context)) + with Coordination(claim.store_path) as store: + retained = NativeRuntimeInput.require(store.session._connection, input_id=claim.input_id) + assert not retained.reference.recorded diff --git a/tests/test_message_notifications.py b/tests/test_message_notifications.py index 23d86badf8..2ea2ed8c64 100644 --- a/tests/test_message_notifications.py +++ b/tests/test_message_notifications.py @@ -70,3 +70,21 @@ def test_individual_view_keeps_alias_goal_and_active_owner_semantics(tmp_path): assert state.active.phase.summary in comms.views.thread_presentations(('beta',)).require('beta').summary comms.agents.finish_turn(lease) assert not comms.views.thread_presentations(('beta',)).require('beta').status.working + + +def test_a_live_batch_beyond_one_window_chunk_keeps_every_message(tmp_path): # noqa: F811 + """The live operation reads every 120-reference chunk into one window; no batch cap.""" + from agent_comms.notification_evidence import NotificationWindow + from agent_comms.notification_reads import LiveNotificationRead + + _path, _root_id, comms, initial, _people = _root(tmp_path) + messages = [initial.message, *( + comms.messaging.send('sender', '#team', f'Batch {number}').message + for number in range(NotificationWindow.window_limit + 5) + )] + window = LiveNotificationRead(tuple(message.reference for message in messages)).read(comms) + assert set(window.outcomes) == {(message.seq, message.message_id) for message in messages} + for message in (messages[0], messages[-1]): + alone = LiveNotificationRead((message.reference,)).read(comms) + key = (message.seq, message.message_id) + assert window.outcomes[key] == alone.outcomes[key] and window.outcomes[key] diff --git a/tests/test_module_layers_clients.py b/tests/test_module_layers_clients.py index 64b113fb92..90aa7f89ed 100644 --- a/tests/test_module_layers_clients.py +++ b/tests/test_module_layers_clients.py @@ -1,13 +1,25 @@ """A stack client's imports root reachability and fail on modules Core lacks.""" import ast +from collections.abc import Callable import os import shutil import subprocess import sys +import tomllib from pathlib import Path -from module_layers import OutgoingWire, Package, StackClient +import pytest +import tomli_w + +from module_layers import MaterializedClients, OutgoingWire, Package, StackClient +from debt_ratchet import ClientTypeErrors +from agent_comms.head_review import BaseSide, HeadSide, IntegrationInput, Side +from agent_comms.todos import DraftListedPullRequest +from github_fixture import GitHubFixture +from integration_source_fixture import MutualSources + +pytest_plugins = ("github_fixture", "test_holder_standing_rules", "integration_source_fixture") def write(path: Path, text: str = "") -> None: @@ -15,6 +27,140 @@ def write(path: Path, text: str = "") -> None: path.write_text(text) +@pytest.mark.parametrize("declaration, roots", ( + ("", ("src", ".")), + ('mypy_path = ["src", "tools"]', ("src", "tools")), + ('mypy_path = "src, tools"', ("src", "tools")), + ("mypy_path = []", ()), + ('mypy_path = ["{external}", "src"]', ("{external}", "src")), +)) +def test_client_roots_follow_each_projects_native_declaration( + tmp_path: Path, declaration: str, roots: tuple[str, ...], +) -> None: + external = tmp_path / "external-library" + external.mkdir() + client = tmp_path / "client" + dependency = tmp_path / "dependency" + write(client / "pyproject.toml", '[tool.mypy]\n' + declaration.format(external=external)) + write(dependency / "pyproject.toml", '[tool.mypy]\nmypy_path = ["lib"]\n') + acquired = StackClient("client", client, dependencies=(dependency,)) + expected = tuple(str((client / path.format(external=external)).resolve()) for path in roots) + assert acquired.source_paths == expected + assert acquired.import_paths == (*expected, str(dependency / "lib")) + + +def test_materialized_clients_check_matched_sources_and_both_mixed_sources_fail( + mutual_sources: MutualSources, github_cli: GitHubFixture, + materialized_clients: Callable[[Path, IntegrationInput], MaterializedClients], +) -> None: + pair = mutual_sources + inputs = pair.capture(("P2", "P3"), "materialized module clients") + core = inputs.source(pair.core.github) + base = BaseSide.require_materialized(pair.output, core) + head = HeadSide.require_materialized(pair.output, core) + files = tuple(sorted(pair.output.rglob(".git"))) + resource = materialized_clients(pair.output, inputs) + base_clients = resource.cuts[BaseSide] + head_clients = resource.cuts[HeadSide] + assert tuple(sorted(pair.output.rglob(".git"))) == files # No later fresh main clone. + assert Package(base, base_clients).missing_names == () + assert Package(head, head_clients).missing_names == () + assert {name for entry, name in Package(base, head_clients).missing_names} == {"new_api"} + assert {name for entry, name in Package(head, base_clients).missing_names} == {"old_api"} + assert not any(call[:2] == ("pr", "merge") for call in github_cli.calls()) + + +@pytest.mark.parametrize("baseline_has_client,candidate_has_client", ((False, True), (True, False))) +def test_actual_client_membership_stays_absent_on_its_own_cut( + mutual_sources: MutualSources, github_cli: GitHubFixture, + baseline_has_client: bool, candidate_has_client: bool, + materialized_clients: Callable[[Path, IntegrationInput], MaterializedClients], +) -> None: + pair = mutual_sources + full = (pair.core.checkout / "stack/pyproject.toml").read_text() + absent = tomllib.loads(full) + absent["project"]["dependencies"].remove("source-client") + del absent["tool"]["uv"]["sources"]["source-client"] + absent_text = tomli_w.dumps(absent) + if not baseline_has_client: + pair.core.git(pair.core.checkout, "switch", "main") + pair.core.commit_change("Baseline has no client declaration", *pair.core.write_files({ + Path("stack/pyproject.toml"): absent_text})) + pair.core.git(pair.core.checkout, "push", "origin", "main") + pair.core.git(pair.core.checkout, "switch", "feature") + pair.core.git(pair.core.checkout, "merge", "--no-edit", "main") + head = pair.core.publish_change("Declare only the candidate's actual client membership", *pair.core.write_files({ + Path("stack/pyproject.toml"): full if candidate_has_client else absent_text})) + stamp = "2026-10-24T00:00:00Z" + github_cli.put(pair.core.github, DraftListedPullRequest( + pair.core.number, "feature", head, stamp, "main", reviewable_times=(stamp,))) + inputs = pair.capture(("P2", "P3"), "actual added and removed client") + resource = materialized_clients(pair.output, inputs) + core = inputs.source(pair.core.github) + tested = pair.core.git(pair.checkout.path, "rev-parse", "HEAD").strip() + for side, present in ((BaseSide, baseline_has_client), (HeadSide, candidate_has_client)): + clients = resource.at(pair.core.checkout, core.baseline.commit, tested, side) + assert len(clients) == int(present) + assert all(client.checkout == side.checkout(pair.output, pair.client.github) for client in clients) + assert len(ClientTypeErrors.parts(pair.core.checkout, core.baseline.commit, tested, + {"src/agent_comms/api.py"}, "src/", resource)) == 1 + + +def test_materialized_client_files_come_from_each_actual_cut( + mutual_sources: MutualSources, github_cli: GitHubFixture, + materialized_clients: Callable[[Path, IntegrationInput], MaterializedClients], +) -> None: + pair = mutual_sources + pair.client.git(pair.client.checkout, "mv", "src/workflow.py", "src/replacement.py") + commit = pair.client.publish_change("Move the candidate client file", "src") + stamp = "2026-10-24T00:00:00Z" + github_cli.put(pair.client.github, DraftListedPullRequest( + pair.client.number, "feature", commit, stamp, "main", reviewable_times=(stamp,))) + inputs = pair.capture(("P2", "P3"), "side-specific client files") + cuts: dict[type[Side], set[str]] = {} + resource = materialized_clients(pair.output, inputs) + for side in (BaseSide, HeadSide): + client, = resource.cuts[side] + cuts[side] = {entry.path for entry in client.imports} + assert cuts[BaseSide] == {"src/workflow.py"} + assert cuts[HeadSide] == {"src/replacement.py"} + + +@pytest.mark.parametrize("name", ("source-client", "source-renamed")) +def test_materialized_selection_includes_peer_imports_but_not_absent_paths( + mutual_sources: MutualSources, github_cli: GitHubFixture, name: str, + materialized_clients: Callable[[Path, IntegrationInput], MaterializedClients], +) -> None: + pair = mutual_sources + client_project = tomllib.loads((pair.client.checkout / "pyproject.toml").read_text()) + client_project["project"]["name"] = name + files = pair.client.write_files({ + Path("src/workflow.py"): 'def run() -> str:\n return "local"\n', + Path("pyproject.toml"): tomli_w.dumps(client_project)}) + commit = pair.client.publish_change("Keep the file while removing its candidate Core import", *files) + stack = tomllib.loads((pair.core.checkout / "stack/pyproject.toml").read_text()) + stack["project"]["dependencies"] = [name if item == "source-client" else item + for item in stack["project"]["dependencies"]] + stack["tool"]["uv"]["sources"][name] = stack["tool"]["uv"]["sources"].pop("source-client") + core_head = pair.core.publish_change("Declare the actual candidate client name", *pair.core.write_files({ + Path("stack/pyproject.toml"): tomli_w.dumps(stack)})) + stamp = "2026-10-24T00:00:00Z" + github_cli.put(pair.core.github, DraftListedPullRequest( + pair.core.number, "feature", core_head, stamp, "main", reviewable_times=(stamp,))) + github_cli.put(pair.client.github, DraftListedPullRequest( + pair.client.number, "feature", commit, stamp, "main", reviewable_times=(stamp,))) + inputs = pair.capture(("P2", "P3"), "selected file union") + resource = materialized_clients(pair.output, inputs) + core = inputs.source(pair.core.github) + head = pair.core.git(pair.checkout.path, "rev-parse", "HEAD").strip() + for side in (BaseSide, HeadSide): + client, = resource.at(pair.core.checkout, core.baseline.commit, head, side) + assert client.files_importing("agent_comms") == ["src/workflow.py"] + assert client.checkout == side.checkout(pair.output, pair.client.github) + candidate, = resource.at(pair.core.checkout, core.baseline.commit, head, HeadSide) + assert candidate.imports == () + + def test_client_imports_root_reachability_and_name_missing_modules(tmp_path): core = tmp_path / "core" write( diff --git a/tests/test_monitor_delivery.py b/tests/test_monitor_delivery.py index 12518c2c05..abc7a7549a 100644 --- a/tests/test_monitor_delivery.py +++ b/tests/test_monitor_delivery.py @@ -39,8 +39,9 @@ ) from agent_comms.harness_delivery_states import ( DeliveredOutcome, EmittedOutcome, MismatchedMonitor, ObservedMonitor, ReceivingMonitor, - UnconfirmedOutcome, UnrecordedMonitor, + UnconfirmedOutcome, UnrecordedMonitor, WakeOnlyMonitor, ) +from agent_comms.notification_progress import SentProgress from agent_comms.agent_inbox import TakenInputStart from agent_comms.errors import RelationViolationError from agent_comms.read_ledger import ReadLedger @@ -50,9 +51,11 @@ from agent_comms.coordinator import Coordination from agent_comms.field_codec import FieldCodec from agent_comms.message_reference import BridgeStart, MessageReference -from agent_comms.runtime_updates import HiddenItem -from agent_comms.turn_item import NativeTurnId -from agent_comms.messages import InfoKind, Message +from agent_comms.runtime_updates import ( + AllTurnsEnded, FinishedItem, FinishedTurn, HiddenItem, RuntimeUpdate, +) +from agent_comms.turn_item import NativeItemId, NativeTurnId +from agent_comms.messages import AlertKind, InfoKind, Message, NoticeRole from agent_comms.threads import Thread @@ -125,8 +128,8 @@ async def test_empty_journal_starts_at_binding_and_ignores_user_reads(comms, tmp @pytest.fixture def bridge(comms, tmp_path): - declare_test_bus(comms, threads=tuple( - Thread(name, frozenset(), Checkout(tmp_path)) + declare_test_bus(comms, channels=('monitor',), threads=tuple( + Thread(name, frozenset({'monitor'}), Checkout(tmp_path)) for name in ('claude-code', 'viewer', 'peer') )) practice = PracticeCopy(comms.root, tmp_path / 'copied-bus') @@ -536,6 +539,33 @@ async def test_a_body_that_fits_one_event_goes_inline_and_exports_nothing(bridge assert not bridge.exports.exists() +@pytest.mark.asyncio +@pytest.mark.parametrize('original_body', ('Original board packet', ClaudeBridge.board_wake)) +async def test_board_wake_observation_cannot_confirm_original_packet(bridge, original_body): + """Exercise the actual wake transport, including an accidental exact-body collision.""" + message = bridge.comms.messaging.send('peer', 'viewer', original_body).message + human_reads = ReadLedger.at(bridge.comms.root).read() + initial = bridge.attempt(message, UnconfirmedOutcome) + assert await bridge.coordinator_write(initial.begin) + async with attached_monitor(bridge) as reader: + outcome = await bridge.wake(message) + line = await only_event(reader) + assert ClaudeMonitorFrame.decode_line(line).body == ClaudeBridge.board_wake + await bridge.finish(bridge.attempt(message, outcome), partial(replace, outcome=outcome)) + # Re-reading the wake cannot manufacture a proof of the original body. + for _ in range(2): + await observe(bridge, notification(line).updates(ClaudeSession())) + attached = await bridge.reader.prepare_delivery(bridge) + assert attached.attachment.pending_parts == {} + retained, = await bridge.attempts(message.message_id) + assert retained.outcome is EmittedOutcome + assert retained.monitor == WakeOnlyMonitor() + assert not retained.consumed_progress(SentProgress(round(message.timestamp * 1000))).consumed + assert await bridge.coordinator_read(lambda db: HarnessObservedPart.of(db, retained)) == () + assert not await bridge.coordinator_write(initial.begin) + assert ReadLedger.at(bridge.comms.root).read() == human_reads + + @pytest.mark.asyncio @pytest.mark.parametrize('parts', (2, 8, 23, 25, 46, 26)) async def test_a_larger_message_is_one_notice_and_one_whole_read_confirms_it(bridge, parts): @@ -1017,6 +1047,98 @@ async def observed(): await asyncio.wait_for(pending, 5) assert type(claim()) is DeliveredAssignment +@pytest.fixture(params=(FinishedTurn(NativeTurnId('reply-turn')), AllTurnsEnded())) +def reply_end(request: pytest.FixtureRequest) -> RuntimeUpdate: + return request.param + + +async def confirm_and_finish_reply( + bridge: ClaudeBridge, message: Message, expected: tuple[str, ...], end: RuntimeUpdate, +) -> None: + """Real monitor consumption feeds the ordinary staged output and terminal update.""" + from agent_comms.transcript_events import AssistantTranscript + + assert bridge.binding.bus == 'viewer' + assert bridge.binding.thread == 'session' + output = bridge.output_type.for_comms( + bridge.comms.transcripts.external(bridge.binding.bus), bridge.comms) + async with attached_monitor(bridge) as reader: + await bridge.accept(message) + line = await only_event(reader) + updates = [await update.observed(bridge) + for update in notification(line).updates(ClaudeSession())] + # A long diagnostic path can make the harness request a Read instead of inline text. + exported = Path(MonitorReadFile.naming(bridge.exports, message.reference).path) + if exported.is_file(): + for update in read_records(exported, shown(exported.read_text()), 'reply-read'): + updates.append(await update.observed(bridge)) + delivered, = await bridge.attempts(message.message_id) + assert delivered.outcome is DeliveredOutcome + assert isinstance(delivered.monitor, ObservedMonitor) + assert message.reference not in { + row.reference for row in bridge.comms.bus.inbox( + bridge.binding.bus, after=TakenInputStart()).page.messages} + + def replies() -> tuple[tuple[str, str], ...]: + return tuple((row.target, row.body) for row in bridge.comms.bus.bus.message_history() + if row.sender == bridge.binding.bus) + + output.publish_batch(updates) + turn, item = NativeTurnId('reply-turn'), NativeItemId('answer') + output.publish_batch([FinishedItem(turn, item, (AssistantTranscript( + 'unrelated assistant final', timestamp=2., position=item),))]) + assert replies() == () + output.publish_batch([end]) + assert replies() == tuple((target, 'unrelated assistant final') for target in expected) + # Completion is idempotent, and another turn cannot inherit these routes. + later, later_item = NativeTurnId('later-turn'), NativeItemId('later-answer') + output.publish_batch([FinishedTurn(turn), AllTurnsEnded(), FinishedItem( + later, later_item, (AssistantTranscript( + 'a later final', timestamp=3., position=later_item),)), AllTurnsEnded()]) + assert replies() == tuple((target, 'unrelated assistant final') for target in expected) + + +@pytest.mark.asyncio +async def test_diagnostic_notice_is_consumed_without_replying_to_the_failed_sender(bridge, reply_end): + from agent_comms.diagnostics import record_terminal_failure + from agent_comms.field_codec import InputId + from agent_comms.turn_failure import BackendFailed + + diagnostic = record_terminal_failure( + bridge.comms.root, subject=InputId.mint(), thread='peer', + failure=BackendFailed, sequences=(), source_error=RuntimeError('native request failed')) + message = bridge.comms.messaging.send( + 'peer', 'viewer', f'Request failed. No automatic retry. [Open diagnostic]({diagnostic.as_uri()})', + AlertKind, role=NoticeRole()).message + await confirm_and_finish_reply(bridge, message, (), reply_end) + assert diagnostic.is_file() + assert json.loads(diagnostic.read_text())['failure'] == BackendFailed.declared_name + assert bridge.comms.bus.pending_count('peer') == 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize(('target', 'body', 'expected'), ( + ('viewer', 'ordinary direct request', ('claude-code',)), + ('#monitor', 'channel information', ()), + ('#monitor', '@viewer please answer this channel request', ('#monitor',)), +)) +async def test_confirmed_reply_routes_follow_direct_and_channel_policy( + bridge, reply_end, target, body, expected, +): + message = bridge.comms.messaging.send('claude-code', target, body).message + await confirm_and_finish_reply(bridge, message, expected, reply_end) + + +@pytest.mark.asyncio +async def test_confirmed_human_direct_request_receives_one_reply(bridge, reply_end): + from agent_comms.channel_targets import ThreadTarget + + message = bridge.comms.messaging.send_user_message( + ThreadTarget('viewer'), 'please answer once', + worktree=Checkout(bridge.options.state.parent)) + await confirm_and_finish_reply(bridge, message, (message.sender,), reply_end) + + def test_only_commanded_sessions_reply_to_their_delivered_senders(comms): """The commander answers its owner in its own transcript: a unit that messaged it mid-turn never receives that turn's answer as a direct message.""" diff --git a/tests/test_native_context_reference.py b/tests/test_native_context_reference.py index a45e53ac0d..5ad9bd5fa5 100644 --- a/tests/test_native_context_reference.py +++ b/tests/test_native_context_reference.py @@ -36,7 +36,7 @@ UnrecordedNativeColumn, UnrecordedNativeInputReference, ) -from agent_comms.native_pi import NativeContextProof +from agent_comms.native_input_proof import NativeContextProof from agent_comms.native_runtime_input import CurrentNativeCursor, NativeRuntimeInput from agent_comms.private_path import FileIdentity from agent_comms.proven_source_coverage import InjectedInput, NoInjectedInput, ProvenSourceCoverage, SourcePrefix diff --git a/tests/test_native_evidence_custody.py b/tests/test_native_evidence_custody.py index 1978f924fe..883ae20be6 100644 --- a/tests/test_native_evidence_custody.py +++ b/tests/test_native_evidence_custody.py @@ -13,8 +13,8 @@ from agent_comms.coordinator import Coordination from agent_comms.native_entries import NativeEntry, NativeEvidenceScope from agent_comms.native_failures import NativePiUnavailable -from agent_comms.native_input_record import NativeInputLocation -from agent_comms.native_pi import NativeContextJournal, NativeContextProof, PrivateEvidenceRead +from agent_comms.native_input_proof import NativeContextJournal, NativeContextProof, NativeInputLocation +from agent_comms.native_pi import PrivateEvidenceRead from agent_comms.native_transcript import NativeTranscript from agent_comms.store_files import ( ExclusiveLock, diff --git a/tests/test_native_failure_recovery.py b/tests/test_native_failure_recovery.py index 30386eccb9..a4055a2526 100644 --- a/tests/test_native_failure_recovery.py +++ b/tests/test_native_failure_recovery.py @@ -21,13 +21,17 @@ from agent_comms.launch_runtime import ThreadLaunch from agent_comms.native_runtime_input import NativeRuntimeInput from agent_comms.private_send_claim import PrivateSendClaim +from agent_comms.private_send_stage import TriageNativeSend +from agent_comms.owner_fence import prepare_fence_token +from agent_comms.selected_request import SelectedRequest from agent_comms.selected_actions import CodingSelectedAction from agent_comms.selected_participant import SelectedParticipant from agent_comms.selected_session import SelectedSession -from agent_comms.selected_turn import SelectedAttempt +from agent_comms.selected_turn import SelectedAttempt, SelectedPrompt from agent_comms.selected_write_authority import NoSelectedWritePlans from agent_comms.text_digest import TextDigest from native_proof_cases import append_input_entry, recorded_context, write_proof_rows +from conftest import accept_initial, publish_initial from test_coordinated_runtime import _root, tmp_path # noqa: F401 @@ -114,33 +118,63 @@ def record(cls, claim, session, participant): @asynccontextmanager -async def prepared_native_input( - directory: Path, *, max_attempts: int = 1, -) -> AsyncIterator[PrivateSendClaim]: - """Keep the real selected attempt and its bound input open for an owner operation.""" - root, root_id, comms, _initial, _people = _root(directory, direct=True) +async def prepared_native_participant( + directory: Path, *, direct: bool, source_bodies: tuple[str, ...] = (), +) -> AsyncIterator[SelectedParticipant]: + """One real publication/selection/store lifetime for full and triage controls.""" + root, root_id, comms, initial, _people = _root(directory, direct=direct) + for body in source_bodies: + message = publish_initial(comms, 'sender', initial.message.target, body) + with Coordination(root / 'coordination.sqlite3') as store: + accept_initial(comms.bus, root_id, message.seq, store) with closing(CursorDelivery()) as cursor_delivery: with Coordination(root / "coordination.sqlite3") as store: async with SelectedParticipant.select( comms, store, root_id, "beta", 0, cursor_delivery=cursor_delivery, ) as participant: assert participant is not None - created = store.create( - participant.batch.execution_id, WireOrigin, - participant.lookup, max_attempts, - sources=participant.batch.sources, - ).value - attempt = await SelectedAttempt.engage(participant, created) - directory = participant.owner.thread.native_session_directory(root) - identity = create_fresh_private_session(directory, worktree=Path(participant.owner.thread.worktree)) - participant.owner = comms.registry.attach_native_session(participant.owner, identity.session_file) - session = SelectedSession.for_launch( - directory, identity.path, root=root, - worktree=Path(participant.owner.thread.worktree), - owner=participant.owner.thread.incarnation.require_recorded(), - ) - request, _action = await attempt.prepare(session, CodingSelectedAction(), NoSelectedWritePlans()) - yield request.claim_scope + yield participant + + +def attach_native_session(participant: SelectedParticipant) -> SelectedSession: + root, comms = participant.comms.root, participant.comms + directory = participant.owner.thread.native_session_directory(root) + identity = create_fresh_private_session(directory, worktree=Path(participant.owner.thread.worktree)) + participant.owner = comms.registry.attach_native_session(participant.owner, identity.session_file) + return SelectedSession.for_launch( + directory, identity.path, root=root, + worktree=Path(participant.owner.thread.worktree), + owner=participant.owner.thread.incarnation.require_recorded(), + ) + + +@asynccontextmanager +async def prepared_native_input( + directory: Path, *, max_attempts: int = 1, source_bodies: tuple[str, ...] = (), +) -> AsyncIterator[PrivateSendClaim]: + """Keep the real selected attempt and its bound input open for an owner operation.""" + async with prepared_native_participant(directory, direct=True, source_bodies=source_bodies) as participant: + created = participant.store.create( + participant.batch.execution_id, WireOrigin, + participant.lookup, max_attempts, sources=participant.batch.sources, + ).value + attempt = await SelectedAttempt.engage(participant, created) + session = attach_native_session(participant) + request, _action = await attempt.prepare(session, CodingSelectedAction(), NoSelectedWritePlans()) + yield request.claim_scope + + +@asynccontextmanager +async def prepared_native_triage(directory: Path) -> AsyncIterator[PrivateSendClaim]: + """Reserve an actual triage batch; no native process or model is substituted.""" + async with prepared_native_participant(directory, direct=False) as participant: + assert participant.batch.requires_triage + session = attach_native_session(participant) + request = await SelectedRequest.reserve( + participant, session, TriageNativeSend(participant.batch.assignments), + prepare_fence_token(), SelectedPrompt(participant).triage(), + ) + yield request.claim_scope async def release_native_input(directory: Path, history: type[NativeSendHistory] = FailedHistory) -> ReleasedNativeInput: diff --git a/tests/test_native_input_bus_path.py b/tests/test_native_input_bus_path.py index d6c39b0356..31b2398020 100644 --- a/tests/test_native_input_bus_path.py +++ b/tests/test_native_input_bus_path.py @@ -20,7 +20,7 @@ from agent_comms.native_package import VerifiedNativePackage from agent_comms.turn_watchdog import ModelWait from agent_comms.native_arguments import NativeInputProofArgument -from agent_comms.native_pi import NativeContextJournal, NativeContextProof +from agent_comms.native_input_proof import NativeContextJournal, NativeContextProof from agent_comms.native_transcript import NativeTranscript from bus_history import read_history from delivery_owner_fixture import reply_turn_finished diff --git a/tests/test_native_input_evidence_index.py b/tests/test_native_input_evidence_index.py index 69b88a07d6..e572fd5a8d 100644 --- a/tests/test_native_input_evidence_index.py +++ b/tests/test_native_input_evidence_index.py @@ -4,7 +4,7 @@ import pytest from agent_comms.native_entries import NativeEvidenceScope -from agent_comms.native_pi import NativeContextProof +from agent_comms.native_input_proof import NativeContextProof from agent_comms.native_session_reopen import NativeSessionIdentity from native_proof_cases import write_proof_rows from test_native_pi import INPUT_ID, _evidence diff --git a/tests/test_native_pi.py b/tests/test_native_pi.py index 4b735a4679..aa2b1ba2f9 100644 --- a/tests/test_native_pi.py +++ b/tests/test_native_pi.py @@ -16,9 +16,8 @@ from agent_comms.native_arguments import SelectedModel from agent_comms.native_entries import NativeEntry from agent_comms.native_failures import NativePiUnavailable +from agent_comms.native_input_proof import NativeContextJournal, NativeContextProof from agent_comms.native_pi import ( - NativeContextJournal, - NativeContextProof, NativePiRpcLaunch, NativePiTerminalFailure, ) diff --git a/tests/test_notification_read_state.py b/tests/test_notification_read_state.py index f9f1aa477e..bb6b4a195a 100644 --- a/tests/test_notification_read_state.py +++ b/tests/test_notification_read_state.py @@ -1,26 +1,53 @@ +import pickle import sys from dataclasses import replace from pathlib import Path +import pytest + +from agent_comms.bus_read import BusRead, UiRead from agent_comms.comms import Comms +from agent_comms.coordination_errors import StaleRevision +from agent_comms.field_codec import FieldCodec, MessageId from agent_comms.history_views import HistoryViews +from agent_comms.message_reference import MessageReference +from agent_comms.notification_read import NotificationsRead, UnreadableNotificationsRead +from agent_comms.notification_reads import ( + LiveNotificationRead, NotificationRead, RecordedNotificationRead, +) +from agent_comms.order_tools import ( + CommanderUnitRead, InputDeliveryRead, OwnerGoalRead, TranscriptPageRead, +) from agent_comms.private_path import FileRevision +from agent_comms.registry_provenance import DeletedThread +from agent_comms.repo import Checkout from agent_comms.presentation import NoRevision, WireRevision from agent_comms.revision_views import ( ActivityView, NotificationEvidenceView, NotificationsView, RegistrationsView, SidebarView, StatusView, StoresView, ) +from agent_comms.thread_identity import AgentThreadRole, UserThreadRole +from agent_comms.threads import Thread +from agent_comms.tool_declaration import NativeRequestTools +from agent_comms.tool_request import ToolRequest +from agent_comms.tools import CommsMessageOutcomesTool +from agent_comms.acp_extension import GoalSnapshotReply, TranscriptSnapshotUpdate +from agent_comms.bus_read import OwnerLiveRead +from agent_comms.commander_hud import CommanderUnitDetail +from agent_comms.field_codec import OrderId +from agent_comms.thread_identity import RecordedThreadIncarnation +from agent_comms.transcripts import NativeTranscriptCursor from agent_comms.ui_model.observation import ( - NotificationsRead, ObserveNotifications, ReadNotifications, ReadRecordedNotifications, - UnreadableNotificationsRead, + ObserveNotifications, ReadNotifications, ReadOperation, ReadOwnerOperation, + ReadTranscript, ReadTranscriptUpdate, RefreshTranscript, SubscribeTranscript, ) def test_notification_read_states_use_the_request_revision_contract(comms): original = WireRevision.empty() later = replace(original, expiry_tick=original.expiry_tick + 60) - live = ReadNotifications("root", ()) - recorded = ReadRecordedNotifications("root", (), "archive") + live = LiveNotificationRead(()) + recorded = RecordedNotificationRead((), "archive") settled = NotificationsRead(original, {}) assert not live.changed_since(comms, original, settled) assert not live.changed_since(comms, later, settled) @@ -29,13 +56,71 @@ def test_notification_read_states_use_the_request_revision_contract(comms): assert not live.changed_since(comms, original, UnreadableNotificationsRead(original)) +TEAM = (("sender", AgentThreadRole), ("worker", AgentThreadRole), ("human", UserThreadRole), + ("archived", AgentThreadRole), ("gone", AgentThreadRole)) + + +def declare_team(comms: Comms, tmp_path: Path) -> MessageReference: + """One real #team message from an agent sender, seen by a human and by three native + agent recipients; one of those is archived or deleted afterwards. The window names only + native recipients, so the sender and the human are never watched.""" + comms.channels.create_tag("team") + for name, role in TEAM: + comms.registry.declare(Thread(name, frozenset({"team"}), Checkout(tmp_path), role=role)) + return comms.messaging.send("sender", "#team", "Who is still here?").message.reference + + +def settled_read(comms: Comms, reference: MessageReference) -> tuple[WireRevision, NotificationsRead]: + revision = comms.views.revision() + window = LiveNotificationRead((reference,)).read(comms) + return revision, NotificationsRead.of(revision, comms, {}, window) + + +def test_the_window_names_native_recipients_and_activity_holds_only_projected_ones(comms, tmp_path): + reference = declare_team(comms, tmp_path) + comms.registry.archive("archived") + comms.registry.remove("gone") + revision, settled = settled_read(comms, reference) + + names = settled.notification_window.activity_names() + assert names == {"worker", "archived", "gone"} + assert not names & {"sender", "human"} + assert settled.activity.keys() == names - {"archived"} + assert settled.activity["gone"] == DeletedThread("gone") + assert settled.activity["worker"] == comms.views.observed_activity(revision, ("worker",))["worker"] + assert not settled.live_changed(comms, revision) + + +def test_an_archived_participant_becoming_visible_changes_the_read(comms, tmp_path): + reference = declare_team(comms, tmp_path) + comms.registry.archive("archived") + revision, settled = settled_read(comms, reference) + assert "archived" not in settled.activity + + comms.registry.declare(Thread("archived", frozenset({"team"}), Checkout(tmp_path))) + later = comms.views.revision() + assert later.changed_since(ActivityView(), revision) + assert settled.live_changed(comms, later) + + +def test_a_participant_deleted_after_the_read_changes_it(comms, tmp_path): + reference = declare_team(comms, tmp_path) + revision, settled = settled_read(comms, reference) + assert "gone" in settled.activity and settled.activity["gone"] != DeletedThread("gone") + + comms.registry.remove("gone") + assert settled.live_changed(comms, comms.views.revision()) + + def test_notification_read_only_acquires_activity_when_its_source_changed(comms, tmp_path): + reference = declare_team(comms, tmp_path) + comms.registry.archive("archived") + original, settled = settled_read(comms, reference) + assert settled.notification_window.activity_names() - settled.activity.keys() == {"archived"} marker = tmp_path / 'changed-source' marker.write_text('changed') source = FileRevision.capture(marker).revisions() - original = WireRevision.empty() assert replace(original, history=source).changed_since(NotificationsView(), original) - settled = NotificationsRead(original, {'absent-participant': None}) calls = 0 def count_reads(frame, event, argument): @@ -48,7 +133,7 @@ def count_reads(frame, event, argument): try: assert not settled.live_changed(comms, replace(original, reads=source)) assert calls == 0 - assert settled.live_changed(comms, replace(original, activity=source)) + assert not settled.live_changed(comms, replace(original, activity=source)) assert calls == 1 finally: sys.setprofile(previous) @@ -66,9 +151,9 @@ def test_unrelated_registry_change_keeps_the_notification_window(comms: Comms, t def test_changed_notification_interest_keeps_only_retained_read_states(): - kept = ReadNotifications('kept', ()) - removed = ReadNotifications('removed', ()) - added = ReadNotifications('added', ()) + kept = ReadNotifications('kept', LiveNotificationRead(())) + removed = ReadNotifications('removed', LiveNotificationRead(())) + added = ReadNotifications('added', LiveNotificationRead(())) previous = ObserveNotifications(frozenset((kept, removed))).observation() state = NotificationsRead(WireRevision.empty()) previous.read_states = {kept: state, removed: state} @@ -80,6 +165,80 @@ def test_changed_notification_interest_keeps_only_retained_read_states(): assert requested.read_states == {kept: state} +REFERENCES = (MessageReference(1, MessageId.mint("original")),) + + +@pytest.mark.parametrize("operation", [ + LiveNotificationRead(REFERENCES), RecordedNotificationRead(REFERENCES, "archive"), +], ids=["live", "recorded"]) +def test_notification_operations_are_root_free_family_records(operation: NotificationRead) -> None: + assert operation.references == REFERENCES + encoded = FieldCodec.encode(operation) + assert "root" not in encoded and "request_id" not in encoded + decoded = FieldCodec.decode(NotificationRead, encoded) + assert decoded == operation and decoded.references == REFERENCES + assert FieldCodec.decode(UiRead, encoded) == operation + request = ReadNotifications("root", operation, request_id=4) + assert FieldCodec.decode(ReadNotifications, FieldCodec.encode(request)) == request + transported = pickle.loads(pickle.dumps(request)) + assert transported == request and hash(transported) == hash(request) + assert {transported: "held"}[request] == "held" and len({request, transported}) == 1 + + +def test_root_request_references_and_archive_each_distinguish_a_notification_request(): + other_reference = (MessageReference(2, MessageId.mint("other")),) + request = ReadNotifications("root", RecordedNotificationRead(REFERENCES, "archive"), request_id=4) + distinct = ( + replace(request, root="other-root"), + replace(request, request_id=5), + replace(request, operation=RecordedNotificationRead(other_reference, "archive")), + replace(request, operation=RecordedNotificationRead(REFERENCES, "other-archive")), + replace(request, operation=LiveNotificationRead(REFERENCES)), + ) + assert len({request, *distinct}) == 1 + len(distinct) + + +def test_equal_requests_keep_their_read_state_and_a_new_archive_drops_it(): + kept = ReadNotifications("root", RecordedNotificationRead(REFERENCES, "archive")) + replaced = ReadNotifications("root", RecordedNotificationRead(REFERENCES, "detached")) + reattached = ReadNotifications("root", RecordedNotificationRead(REFERENCES, "attached")) + previous = ObserveNotifications(frozenset((kept, replaced))).observation() + state = NotificationsRead(WireRevision.empty()) + previous.read_states = {kept: state, replaced: state} + requested = ObserveNotifications(frozenset(( + ReadNotifications("root", RecordedNotificationRead(REFERENCES, "archive")), reattached, + ))).observation() + + assert previous.for_interest(requested) is requested + assert requested.read_states == {kept: state} + assert requested.read_states[kept] is state + + +def test_one_operation_in_two_root_envelopes_keeps_root_validation(comms): + operation = LiveNotificationRead(REFERENCES) + ours = ReadNotifications(str(comms.root), operation) + foreign = ReadNotifications(str(comms.root.parent / "another-wire"), operation) + assert ours.operation is foreign.operation and ours != foreign + ours.require_root(comms) + with pytest.raises(StaleRevision): + foreign.require_root(comms) + + +def test_notification_operations_are_ui_reads_and_not_tools(): + assert BusRead.members_with(NotificationRead) == (LiveNotificationRead, RecordedNotificationRead) + assert set(BusRead.members_with(NotificationRead)) <= set(BusRead.members_with(UiRead)) + tools = (*ToolRequest.members_with(ToolRequest), *NativeRequestTools.members_with(NativeRequestTools)) + assert not any(issubclass(tool, NotificationRead) for tool in tools) + assert CommsMessageOutcomesTool in ToolRequest.members_with(ToolRequest) + assert CommsMessageOutcomesTool in NativeRequestTools.members_with(NativeRequestTools) + + +def test_a_notification_request_refuses_an_operation_outside_the_notification_family(): + encoded = FieldCodec.encode(ReadNotifications("root", LiveNotificationRead(REFERENCES))) + with pytest.raises(ValueError, match="is not a member of NotificationRead"): + FieldCodec.decode(ReadNotifications, {**encoded, "operation": FieldCodec.encode(InputDeliveryRead())}) + + def test_activity_views_receive_expiry_without_expiring_store_or_evidence_views(): original = WireRevision.empty() later = replace(original, expiry_tick=60) @@ -113,3 +272,78 @@ def test_notification_rank_overrides_follow_the_mro_without_changing_parent_rank assert ShownToModelProgress(100).rank() == DeliveryRank.position_of(SettledDelivery) assert SendingProgress(100).rank() == DeliveryRank.position_of(ActiveDelivery) assert SelectedConsumedProgress(100).rank() == DeliveryRank.position_of(ActiveDelivery) + + +LEAD, WORKER = RecordedThreadIncarnation("lead", 1.0), RecordedThreadIncarnation("worker", 2.0) +ORDERS = (OrderId.mint(),) +UI_ONLY_FIELDS = {"root", "request_id", "previous", "owner", "identity"} +SHARED_UI_READS = [ + CommanderUnitRead(commander=LEAD, unit=WORKER, orders=ORDERS), + TranscriptPageRead(thread="worker", before=NativeTranscriptCursor("session.jsonl", 7), + historical_source="archive"), +] + + +@pytest.mark.parametrize("operation", SHARED_UI_READS, ids=["commander_unit", "transcript_page"]) +def test_shared_ui_reads_are_root_free_family_records_in_one_generic_envelope(operation): + encoded = FieldCodec.encode(operation) + assert not UI_ONLY_FIELDS & set(encoded) + assert FieldCodec.decode(UiRead, encoded) == operation + request = ReadOperation("root", operation, request_id=4) + assert FieldCodec.decode(ReadOperation, FieldCodec.encode(request)) == request + transported = pickle.loads(pickle.dumps(request)) + assert transported == request and hash(transported) == hash(request) + assert {transported: "held"}[request] == "held" and len({request, transported}) == 1 + + +def test_the_owner_goal_is_a_root_free_owner_live_record_in_its_envelope(): + operation = OwnerGoalRead(thread="worker") + encoded = FieldCodec.encode(operation) + assert not UI_ONLY_FIELDS & set(encoded) + assert FieldCodec.decode(OwnerLiveRead, encoded) == operation + request = ReadOwnerOperation("root", WORKER, operation=operation, request_id=4) + transported = pickle.loads(pickle.dumps(request)) + assert transported == request and hash(transported) == hash(request) + + +def test_witnesses_cursors_and_archive_each_distinguish_a_shared_read_request(): + commander = ReadOperation("root", SHARED_UI_READS[0], request_id=4) + page = ReadOperation("root", SHARED_UI_READS[1], request_id=4) + distinct = ( + replace(commander, root="other-root"), + replace(commander, request_id=5), + replace(commander, operation=replace(SHARED_UI_READS[0], unit=RecordedThreadIncarnation("worker", 3.0))), + replace(commander, operation=replace(SHARED_UI_READS[0], orders=(OrderId.mint(),))), + page, + replace(page, operation=replace(SHARED_UI_READS[1], before=NativeTranscriptCursor("session.jsonl", 8))), + replace(page, operation=replace(SHARED_UI_READS[1], historical_source="other-archive")), + ) + assert len({commander, *distinct}) == 1 + len(distinct) + + +def test_one_shared_read_in_two_root_envelopes_keeps_root_validation(comms): + operation = SHARED_UI_READS[1] + ours = ReadOperation(str(comms.root), operation) + foreign = ReadOperation(str(comms.root.parent / "another-wire"), operation) + assert ours.operation is foreign.operation and ours != foreign + ours.require_root(comms) + with pytest.raises(StaleRevision): + foreign.require_root(comms) + + +def test_the_three_public_reads_are_cataloged_once_with_their_original_results(): + sys.path.insert(0, str(Path(__file__).parent)) + from test_tool_replies import result_type + + tools = ToolRequest.members_with(ToolRequest) + native = NativeRequestTools.members_with(NativeRequestTools) + results = {CommanderUnitRead: CommanderUnitDetail, OwnerGoalRead: GoalSnapshotReply, + TranscriptPageRead: TranscriptSnapshotUpdate} + for operation, result in results.items(): + assert tools.count(operation) == 1 and native.count(operation) == 1 + assert result_type(operation) is result + assert set(ToolRequest.members_with(UiRead)) == {CommanderUnitRead, TranscriptPageRead} + assert set(ToolRequest.members_with(OwnerLiveRead)) == {InputDeliveryRead, OwnerGoalRead} + lifecycle = (ReadOperation, ReadOwnerOperation, ReadNotifications, ReadTranscript, + SubscribeTranscript, ReadTranscriptUpdate, RefreshTranscript) + assert not set(lifecycle) & {*tools, *native} diff --git a/tests/test_notification_recipient_window.py b/tests/test_notification_recipient_window.py new file mode 100644 index 0000000000..82f44eac67 --- /dev/null +++ b/tests/test_notification_recipient_window.py @@ -0,0 +1,178 @@ +"""Bound real frozen recipient acquisition before assignment/harness/reply decoding. + +No model, provider, product monkeypatch or hand-edited coordination row is used. +""" +from __future__ import annotations + +from pathlib import Path +import pytest + +from agent_comms.bus_publication import Delivered +from agent_comms.cli_bridges.claude import ClaudeBridge +from agent_comms.comms import Comms +from agent_comms.coordination_tables.harness_deliveries import HarnessDelivery +from agent_comms.coordinator import Coordination +from agent_comms.harness_delivery_states import PersistedOutcome +from agent_comms.notification_assignment import AnsweredResponse, NotificationAssignment +from agent_comms.notification_reads import LiveNotificationRead +from agent_comms.tool_caller import ThreadCaller +from agent_comms.notification_evidence import RecipientSelection +from agent_comms.presentation import AllOutcomes, AfterRecipient, MessageNotification, OwnerOutcomes +from agent_comms.read_ledger import ReadLedger +from agent_comms.repo import Checkout +from agent_comms.threads import Thread +from agent_comms.ui_model.observation import ReadNotifications +from conftest import accept_initial, publish_initial +from test_coordinated_runtime import _root, tmp_path # noqa: F401 + + +@pytest.fixture +def frozen_window(tmp_path: Path) -> tuple[Comms, Delivered]: + root, root_id, comms, _initial, people = _root(tmp_path) + for index in range(14): + comms.registry.declare(Thread( + f"window-{index}", frozenset({"team"}), Checkout(tmp_path), + process_identity=people[1].process_identity, + )) + message = publish_initial(comms, "sender", "#team", "A frozen recipient window") + source = comms.bus.bus.delivery(message.seq) + with Coordination(root / "coordination.sqlite3") as store: + for recipient in source.recipients: + store.participants.register(recipient.recipient_lookup, committed=True) + accept_initial(comms.bus, root_id, message.seq, store) + return comms, source + + +def record_attempt(comms: Comms, source: Delivered, recipient_position: int) -> None: + recipient = source.recipients[recipient_position] + delivery = HarnessDelivery(ClaudeBridge, recipient.recipient_lookup, + source.message.message_id, source.message.seq, PersistedOutcome) + with Coordination(comms.root / "coordination.sqlite3") as store: + with store.session.writing() as connection: + assert delivery.begin(store.assignments, connection) + + +def test_acquired_rows_and_proofs_are_exact_bounded_subset_without_rewriting_original( + frozen_window: tuple[Comms, Delivered], +) -> None: + comms, source = frozen_window + frozen = source.recipients + before_reads = ReadLedger.at(comms.root).read() + before_wire = comms.bus.bus.revision() + viewer = AllOutcomes() + full = NotificationAssignment.read(comms.root, viewer.selection((source,))) + selection = viewer.selection((source,), limit=3) + bounded = NotificationAssignment.read(comms.root, selection) + selected = {recipient.recipient_lookup for recipient in selection.pages[0].recipients} + assert len(full.rows) == len(frozen) > len(bounded.rows) == 3 + assert {row.recipient_lookup for row, _activity in bounded.rows} == selected + assert bounded.sources[0] is source + assert source.recipients is frozen + full_window = MessageNotification.delivery_window(comms.root, comms.registry, full) + bounded_window = MessageNotification.delivery_window(comms.root, comms.registry, bounded) + key = source.message.seq, source.message.message_id + assert bounded_window.outcomes[key] == tuple( + row for row in full_window.outcomes[key] if row.recipient_identity.recipient_lookup in selected) + assert bounded_window.remaining(source.message.reference) == len(frozen) - 3 + assert ReadLedger.at(comms.root).read() == before_reads + assert comms.bus.bus.revision() == before_wire + + +def test_retained_cut_ignores_other_recipient_change_but_observes_its_own_proof( + frozen_window: tuple[Comms, Delivered], +) -> None: + comms, source = frozen_window + selection = AllOutcomes().selection((source,), limit=3) + before = NotificationAssignment.read(comms.root, selection) + record_attempt(comms, source, len(source.recipients) - 1) + assert NotificationAssignment.changed((before,), comms.root) == () + record_attempt(comms, source, 0) + assert NotificationAssignment.changed((before,), comms.root) == (before,) + changed = NotificationAssignment.read(comms.root, selection) + assert len(changed.attempts) == 1 + assert changed.attempts[0].recipient_lookup == source.recipients[0].recipient_lookup + + +def test_cursor_and_owner_scope_are_applied_by_actual_read_request_before_acquisition( + frozen_window: tuple[Comms, Delivered], +) -> None: + comms, source = frozen_window + reference = source.message.reference + first = ReadNotifications(str(comms.root), LiveNotificationRead((reference,), limit=3)).read(comms) + key = reference.seq, reference.message_id + cursor = AfterRecipient(first.outcomes[key][-1].recipient_identity) + second = ReadNotifications(str(comms.root), LiveNotificationRead((reference,), after=cursor, limit=3)).read(comms) + assert len(first.outcomes[key]) == len(second.outcomes[key]) == 3 + assert {row.recipient_identity for row in first.outcomes[key]}.isdisjoint( + row.recipient_identity for row in second.outcomes[key]) + assert second.remaining(reference) == len(source.recipients) - 6 + own = source.recipients[-1] + recipient = LiveNotificationRead((reference,), limit=1).read_as(comms, ThreadCaller(own.canonical_thread)) + assert tuple(row.recipient_identity for row in recipient.outcomes[key]) == (own,) + with pytest.raises(ValueError): + AllOutcomes().selection((source,), limit=RecipientSelection.max_limit + 1) + + +def test_reply_acquisition_returns_only_first_reply_of_selected_frozen_author( + frozen_window: tuple[Comms, Delivered], +) -> None: + comms, source = frozen_window + selected, other = source.recipients[0], source.recipients[-1] + first = comms.messaging.send(selected.canonical_thread, source.message.sender, "first answer", + answers=(source.message.reference,)).message + comms.messaging.send(selected.canonical_thread, source.message.sender, "later answer", + answers=(source.message.reference,)) + comms.messaging.send(other.canonical_thread, source.message.sender, "other author's answer", + answers=(source.message.reference,)) + replies = comms.bus.bus.recipient_replies(((selected.recipient_lookup, source.message.reference),)) + assert tuple(reply.message.reference for reply in replies) == (first.reference,) + + +def test_independent_reply_changes_retained_evidence_without_claiming_context( + frozen_window: tuple[Comms, Delivered], +) -> None: + comms, source = frozen_window + selection = AllOutcomes().selection((source,), limit=1) + recipient = selection.pages[0].recipients[0] + record_attempt(comms, source, 0) + before = NotificationAssignment.read(comms.root, selection) + reads = ReadLedger.at(comms.root).read() + response = comms.messaging.send(recipient.canonical_thread, source.message.sender, + "An independent answer", answers=(source.message.reference,)).message + assert NotificationAssignment.changed((before,), comms.root) == (before,) + after = NotificationAssignment.read(comms.root, selection) + window = MessageNotification.delivery_window(comms.root, comms.registry, after) + row, = window.outcomes[source.message.seq, source.message.message_id] + assert row.answer == response.reference + assert not row.state.consumed + assert after.rows == before.rows and after.attempts == before.attempts + assert ReadLedger.at(comms.root).read() == reads + later = comms.messaging.send(recipient.canonical_thread, source.message.sender, + "A later independent answer", answers=(source.message.reference,)).message + assert NotificationAssignment.changed((after,), comms.root) == () + assert AnsweredResponse(later.reference, int(later.timestamp * 1000)).answered(row) is row + + +def test_independent_reply_without_any_claim_is_retained_without_context_or_read_mutation( + frozen_window: tuple[Comms, Delivered], +) -> None: + comms, _accepted_source = frozen_window + message = publish_initial(comms, "sender", "#team", "An original without a handling claim") + source = comms.bus.bus.delivery(message.seq) + selection = AllOutcomes().selection((source,), limit=1) + recipient = selection.pages[0].recipients[0] + reads = ReadLedger.at(comms.root).read() + before = NotificationAssignment.read(comms.root, selection) + assert before.rows == before.attempts == () + response = comms.messaging.send(recipient.canonical_thread, source.message.sender, + "An answer without an input claim", answers=(message.reference,)).message + assert NotificationAssignment.changed((before,), comms.root) == (before,) + after = NotificationAssignment.read(comms.root, selection) + assert after.rows == after.attempts == () + assert after.responses[recipient.recipient_lookup, message.reference] == AnsweredResponse( + response.reference, int(response.timestamp * 1000)) + window = MessageNotification.delivery_window(comms.root, comms.registry, after) + row, = window.outcomes[message.seq, message.message_id] + assert row.answer == response.reference and row.changed + assert not row.state.consumed and not row.displayed_to + assert ReadLedger.at(comms.root).read() == reads diff --git a/tests/test_observer_death.py b/tests/test_observer_death.py index 78379f87c3..2d90709887 100644 --- a/tests/test_observer_death.py +++ b/tests/test_observer_death.py @@ -4,6 +4,7 @@ from agent_comms.repo import Checkout import asyncio import os +from functools import partial import signal import pytest @@ -15,62 +16,68 @@ from agent_comms.thread_identity import RecordedThreadIncarnation from agent_comms.owner_connection import OwnerIdentityChangedError from agent_comms.message_reference import MessageReference +from agent_comms.bus_read import BusRead, OwnerLiveRead +from agent_comms.notification_reads import LiveNotificationRead from agent_comms.ui_model.observation import ( - ObservationProcess, ObserverDied, OwnerRead, ReadNotifications, StopSidebar, + ObservationProcess, ObserverDied, ReadNotifications, ReadOwnerOperation, StopSidebar, ) +from agent_comms.thread_execution import ExternalThreadExecution from agent_comms.threads import Thread +from owner_socket_fixture import declare_owner -async def readable(process: ObservationProcess) -> None: - """Wait until the observer's pipe has something to read, as the UI's event loop does.""" - ready = asyncio.Event() - loop = asyncio.get_running_loop() - loop.add_reader(process.fileno(), ready.set) - try: - async with asyncio.timeout(30): - await ready.wait() - finally: - loop.remove_reader(process.fileno()) +def owner_reads(): + """Every owner read the UI can send: each owner-live operation in its one envelope.""" + for operation in BusRead.members_with(OwnerLiveRead): + yield pytest.param(partial(ReadOwnerOperation, operation=operation()), id=operation.declared_name) + + +def empty_window(root) -> ReadNotifications: + return ReadNotifications(str(root), LiveNotificationRead(())) async def test_a_killed_observer_is_reported_once_with_its_signal_and_fails_awaited_reads(comms, durable_home): - process = ObservationProcess(comms.route) - try: - await readable(process) - assert process.results() # the first observation: the service is running - os.kill(process.process.pid, signal.SIGSTOP) # the read below is still unanswered when it dies - waiting = asyncio.ensure_future(process.read(ReadNotifications(str(comms.root), ()))) - await asyncio.sleep(0) - os.kill(process.process.pid, signal.SIGKILL) - with pytest.raises(ObserverDied) as died: - while True: - await readable(process) - process.results() - assert died.value.pid == process.process.pid and died.value.exit_status == -signal.SIGKILL - assert f"pid {process.process.pid}" in str(died.value) and "signal 9" in str(died.value) - with pytest.raises(ObserverDied) as awaited: - await waiting - assert awaited.value is died.value + async with ObservationProcess.awaited(comms.route) as process: + async with asyncio.timeout(30): + assert (await process.read(empty_window(comms.root))).outcomes == {} # the service runs + child = process.process.pid + os.kill(child, signal.SIGSTOP) # the read below stays unanswered until the child dies + waiting = asyncio.ensure_future(process.read(empty_window(comms.root))) + try: + async with asyncio.timeout(30): + _, stopped = await asyncio.to_thread(os.waitpid, child, os.WUNTRACED) + assert os.WIFSTOPPED(stopped) + await asyncio.sleep(0) + assert len(process.reads) == 1 and not waiting.done() + os.kill(child, signal.SIGKILL) + with pytest.raises(ObserverDied) as died: + async with asyncio.timeout(30): + await waiting + finally: + if process.process.exitcode is None: + os.kill(child, signal.SIGKILL) + waiting.cancel() + await asyncio.gather(waiting, return_exceptions=True) + assert died.value.pid == child and died.value.exit_status == -signal.SIGKILL + assert f"pid {child}" in str(died.value) and "signal 9" in str(died.value) with pytest.raises(ObserverDied) as stopping: process.request(StopSidebar()) # what a closing view sends assert stopping.value is died.value with pytest.raises(ObserverDied) as later: - await process.read(ReadNotifications(str(comms.root), ())) + await process.read(empty_window(comms.root)) assert later.value is died.value - finally: - process.close() async def test_an_invalid_notification_reference_does_not_kill_the_observer(comms, durable_home): - with ObservationProcess.awaited(comms.route) as process: + async with ObservationProcess.awaited(comms.route) as process: with pytest.raises(RelationViolationError): - await process.read(ReadNotifications( - str(comms.root), (MessageReference(1, MessageId.mint("not-the-original")),) - )) - assert await process.read(ReadNotifications(str(comms.root), ())) == {} + await process.read(ReadNotifications(str(comms.root), LiveNotificationRead( + (MessageReference(1, MessageId.mint("not-the-original")),) + ))) + assert (await process.read(empty_window(comms.root))).outcomes == {} -@pytest.mark.parametrize("read_type", OwnerRead.__subclasses__()) +@pytest.mark.parametrize("read_type", owner_reads()) @pytest.mark.parametrize("declared, refusal", [ (True, RelationViolationError), (False, UnregisteredThreadError), ]) @@ -80,25 +87,41 @@ async def test_unavailable_owner_reads_preserve_the_observer( owner = Thread("read-subject", frozenset(), Checkout(comms.root.parent)) if declared: comms.registry.declare(owner) - with ObservationProcess.awaited(comms.route) as process: + async with ObservationProcess.awaited(comms.route) as process: with pytest.raises(refusal): await process.read(read_type(str(comms.root), owner.incarnation)) - assert await process.read(ReadNotifications(str(comms.root), ())) == {} + assert (await process.read(empty_window(comms.root))).outcomes == {} assert process.process.is_alive() process.status.raise_if_failed() -@pytest.mark.parametrize("read_type", OwnerRead.__subclasses__()) +@pytest.mark.parametrize("read_type", owner_reads()) @pytest.mark.parametrize("other_root", [False, True]) async def test_stale_owner_reads_preserve_the_observer(comms, durable_home, read_type, other_root): owner = Thread("read-subject", frozenset(), Checkout(comms.root.parent)) comms.registry.declare(owner) root = str(comms.root.parent / "another-wire") if other_root else str(comms.root) identity = owner.incarnation if other_root else RecordedThreadIncarnation(owner.name, 1.0) - with ObservationProcess.awaited(comms.route) as process: + async with ObservationProcess.awaited(comms.route) as process: with pytest.raises(StaleRevision if other_root else OwnerIdentityChangedError): await process.read(read_type(root, identity)) - assert await process.read(ReadNotifications(str(comms.root), ())) == {} + assert (await process.read(empty_window(comms.root))).outcomes == {} + assert process.process.is_alive() + process.status.raise_if_failed() + + +@pytest.mark.parametrize("read_type", owner_reads()) +async def test_external_owner_reads_refuse_before_waiting_for_a_socket( + comms, durable_home, runtime_processes, read_type, +): + """An external bridge has a live process and no runtime socket: the read refuses with the + original error instead of waiting for a socket that will never be served.""" + owner = declare_owner(comms, "external-owner", runtime_processes[0].identity, ExternalThreadExecution) + async with ObservationProcess.awaited(comms.route) as process: + with pytest.raises(RelationViolationError): + async with asyncio.timeout(30): + await process.read(read_type(str(comms.root), owner.incarnation)) + assert (await process.read(empty_window(comms.root))).outcomes == {} assert process.process.is_alive() process.status.raise_if_failed() @@ -106,14 +129,10 @@ async def test_stale_owner_reads_preserve_the_observer(comms, durable_home, read async def test_an_observer_that_failed_is_reported_with_its_own_traceback_and_exit_status(tmp_path, durable_home): not_a_wire = tmp_path / "not-a-wire" not_a_wire.write_text("") - process = ObservationProcess(LocalRoute(not_a_wire)) - try: + async with ObservationProcess.awaited(LocalRoute(not_a_wire)) as process: with pytest.raises(ObserverDied) as died: - while True: - await readable(process) - process.results() - assert died.value.exit_status == 1 - assert "Traceback" in died.value.error and "serve" in died.value.error - assert "exited with status 1" in str(died.value) and died.value.error in str(died.value) - finally: - process.close() + async with asyncio.timeout(30): + await process.read(empty_window(not_a_wire)) + assert died.value.exit_status == 1 + assert "Traceback" in died.value.error and "serve" in died.value.error + assert "exited with status 1" in str(died.value) and died.value.error in str(died.value) diff --git a/tests/test_outbound_transcript_source.py b/tests/test_outbound_transcript_source.py index 83dec2c377..e2649113a5 100644 --- a/tests/test_outbound_transcript_source.py +++ b/tests/test_outbound_transcript_source.py @@ -1,10 +1,11 @@ """Whole conversations derive their original frozen wire sources, in both directions.""" -from agent_comms.presentation import AllOutcomes +from agent_comms.presentation import AllOutcomes, OwnerOutcomes from agent_comms.repo import Checkout from agent_comms.channel_targets import CommandTarget import sqlite3 +import sys from dataclasses import replace import pytest @@ -33,28 +34,103 @@ def test_original_read_contention_is_unavailable_then_same_receipts_return(tmp_p root, _root_id, comms, initial, _people = _root(tmp_path) database = root / "coordination.sqlite3" - captured = NotificationAssignment.read(root, (initial,)) - original = NotificationAssignment.resolve(root, captured) + selection = AllOutcomes().selection((initial,)) + captured = NotificationAssignment.read(root, selection) + original = NotificationAssignment.resolve(captured) wire_before = comms.bus.bus.revision() assert original with Coordination(str(database)) as writer: with writer.session.writing(): with pytest.raises(CoordinationReadUnavailable) as unavailable: - NotificationAssignment.read(root, (initial,)) + NotificationAssignment.read(root, selection) assert unavailable.value.__cause__.sqlite_errorcode & 0xFF in ( sqlite3.SQLITE_BUSY, sqlite3.SQLITE_LOCKED, ) with pytest.raises(CoordinationReadUnavailable): with TranscriptRead.publication_source(root): pytest.fail("Busy acquisition reached the native schema decoder") - reopened = NotificationAssignment.read(root, (initial,)) + reopened = NotificationAssignment.read(root, selection) assert reopened == captured - assert NotificationAssignment.resolve(root, reopened) == original + assert NotificationAssignment.resolve(reopened) == original with TranscriptRead.publication_source(root) as native_read: assert native_read is not None assert comms.bus.bus.revision() == wire_before +def test_a_read_selects_only_its_viewers_claims_and_attempts_before_decoding(tmp_path): # noqa: F811 + from agent_comms.notification_assignment import AssignmentActivityEvidence + + root, _root_id, comms, initial, people = _root(tmp_path) + lookups = {person.name: person.incarnation.lookup for person in people} + reference = initial.message.reference + everyone = NotificationAssignment.read(root, AllOutcomes().selection((initial,))) + recipients = {recipient.canonical_thread: recipient.recipient_lookup + for recipient in initial.recipients} + assert len(recipients) >= 2 and "sender" not in recipients + assert {claim.recipient for claim, _evidence in everyone.rows} == recipients.keys() + decoded = 0 + decode = AssignmentActivityEvidence._decode_row.__func__.__code__ + + def count_decodes(frame, event, argument): + nonlocal decoded + if event == "call" and frame.f_code is decode: + decoded += 1 + + for name, lookup in recipients.items(): + viewer = OwnerOutcomes(lookup) + decoded = 0 + profiling = sys.getprofile() + sys.setprofile(count_decodes) + try: + own = NotificationAssignment.read(root, viewer.selection((initial,))) + finally: + sys.setprofile(profiling) + assert decoded == 1 + assert own.selection.viewer == viewer and own.sources == (initial,) + assert own.rows == tuple(row for row in everyone.rows if row[0].recipient == name) + assert own.evidence[reference].claims == own.rows + assert own.evidence[reference].attempts == { + recipient: attempts for recipient, attempts in everyone.evidence[reference].attempts.items() + if recipient == lookup} + sender = NotificationAssignment.read(root, OwnerOutcomes(lookups["sender"]).selection((initial,))) + assert sender.rows == everyone.rows + assert sender.evidence == everyone.evidence + + +def test_invisible_recipients_allocate_no_placeholder_source(tmp_path): # noqa: F811 + root, _root_id, _comms, initial, people = _root(tmp_path) + lookups = {person.name: person.incarnation.lookup for person in people} + recipient = initial.recipients[0] + (source, outcomes), = NotificationAssignment.for_deliveries(OwnerOutcomes(recipient.recipient_lookup).selection((initial,)), ()) + assert source is initial + assert [outcome.recipient() for outcome in outcomes] == [recipient] + (_source, all_outcomes), = NotificationAssignment.for_deliveries(OwnerOutcomes(lookups["sender"]).selection((initial,)), ()) + assert [outcome.recipient() for outcome in all_outcomes] == list(initial.recipients) + + +def test_retained_reads_refresh_only_for_changes_their_viewer_sees(tmp_path): # noqa: F811 + from agent_comms.assignment_states import DeferredAssignment + from agent_comms.coordination_cohort import sealed_cohort_assignments + + root, _root_id, _comms, initial, people = _root(tmp_path) + lookups = {person.name: person.incarnation.lookup for person in people} + changed, unchanged = initial.recipients[:2] + reads = { + "changed": NotificationAssignment.read(root, OwnerOutcomes(changed.recipient_lookup).selection((initial,))), + "foreign": NotificationAssignment.read(root, OwnerOutcomes(unchanged.recipient_lookup).selection((initial,))), + "sender": NotificationAssignment.read(root, OwnerOutcomes(lookups["sender"]).selection((initial,))), + "operator": NotificationAssignment.read(root, AllOutcomes().selection((initial,))), + } + assert NotificationAssignment.changed(reads.values(), root) == () + with Coordination(str(root / "coordination.sqlite3")) as store: + assignment, = sealed_cohort_assignments(store, changed.recipient_lookup) + store.assignments.transition_preengagement( + assignment.assignment_id, DeferredAssignment, expected_revision=assignment.revision) + refreshed = NotificationAssignment.changed(reads.values(), root) + assert {name for name, read in reads.items() if read in refreshed} == { + "changed", "sender", "operator"} + + @pytest.mark.parametrize('direct', [False, True]) def test_saved_notification_does_not_mix_owner_drain_readiness(tmp_path, direct): # noqa: F811 from agent_comms.activity import StoppedDrainDiagnostic @@ -128,15 +204,16 @@ def test_capture_page_window_names_zero_and_nonzero_sent_windows(tmp_path): # n def test_notification_window_preserves_exact_receipts_and_rejects_duplicates(tmp_path): # noqa: F811 root, _root_id, comms, initial, _people = _root(tmp_path) - records = NotificationAssignment.resolve(root, NotificationAssignment.read(root, (initial,))) + selection = AllOutcomes().selection((initial,)) + records = NotificationAssignment.resolve(NotificationAssignment.read(root, selection)) assert records - projected = tuple(NotificationAssignment.for_deliveries((initial,), records)) + projected = tuple(NotificationAssignment.for_deliveries(selection, records)) assert projected[0][0] is initial assert any(outcome is records[0] for outcome in projected[0][1]) with pytest.raises(ValueError): - tuple(NotificationAssignment.for_deliveries((initial,), (*records, records[0]))) + tuple(NotificationAssignment.for_deliveries(selection, (*records, records[0]))) # Duplicate evidences outside the acquired deliveries do not change their result. - assert tuple(NotificationAssignment.for_deliveries((), (*records, records[0]))) == () + assert tuple(NotificationAssignment.for_deliveries(AllOutcomes().selection(()), (*records, records[0]))) == () def test_publication_intent_joins_original_sender_and_target_only(tmp_path): # noqa: F811 @@ -257,3 +334,77 @@ def test_missing_bus_cannot_be_an_empty_presentation(tmp_path): # noqa: F811 with pytest.raises(sqlite3.OperationalError) as missing: comms.views.thread_presentations(("beta",)).require("beta") assert missing.value.sqlite_errorcode == sqlite3.SQLITE_CANTOPEN + + +def test_a_forced_key_batch_captures_the_same_evidence_through_the_recipient_index(tmp_path): # noqa: F811 + from conftest import accept_initial, publish_initial + from agent_comms.coordination_tables.assignments import WakeAssignment + from agent_comms.native_runtime_input import NativeRuntimeTable + from agent_comms.notification_evidence import NotificationWindow + from agent_comms.registry_store import RegistryStore + + root, root_id, comms, initial, _people = _root(tmp_path) + database = root / "coordination.sqlite3" + privacy = RegistryStore.at(root).privacy() + # The lowered limit still admits capture's own schema validation: each catalog read + # binds every declared object name and every table name of its schema. + limit = max(6, *( + len({name for row in schema.schema_rows() for name in row.schema_names}) + + len(schema.schema_rows()) + for schema in (privacy.coordinator_schema(), NativeRuntimeTable))) + key_fields = 3 + originals = limit // (key_fields * len(initial.recipients)) + 1 + assert originals + 1 <= NotificationWindow.window_limit + sources = [initial] + for index in range(originals - 1): + later = publish_initial(comms, "sender", "#team", f"Broadcast {index} for the window") + with Coordination(str(database)) as store: + accept_initial(comms.bus, root_id, later.seq, store) + sources.append(comms.bus.bus.delivery(later.seq)) + sources = (*sources, initial) + keys = tuple({"recipient_lookup": recipient.recipient_lookup, "wire_seq": source.message.seq, + "message_id": source.message.message_id} + for source in sources for recipient in source.recipients) + distinct = tuple(dict.fromkeys(tuple(key.values()) for key in keys)) + assert key_fields * len(distinct) > limit and len(keys) > len(distinct) + ordinary = NotificationAssignment.read(root, AllOutcomes().selection(sources)) + assert ordinary.rows + statements: list[str] = [] + with Coordination(str(database)) as store, store.session.read(): + connection = store.session._connection + unlimited, = WakeAssignment.requested_keys(connection, keys) + previous = connection.setlimit(sqlite3.SQLITE_LIMIT_VARIABLE_NUMBER, limit) + assert limit < previous + connection.set_trace_callback(statements.append) + try: + bound = connection.getlimit(sqlite3.SQLITE_LIMIT_VARIABLE_NUMBER) + batches = tuple(WakeAssignment.requested_keys(connection, keys)) + limited = NotificationAssignment.capture(connection, AllOutcomes().selection(sources), privacy) + finally: + connection.set_trace_callback(None) + connection.setlimit(sqlite3.SQLITE_LIMIT_VARIABLE_NUMBER, previous) + index, = (name for _seq, name, unique, *_rest in connection.execute( + f'PRAGMA index_list("{WakeAssignment.declared_name}")') + if unique and [column for _rank, _cid, column in connection.execute( + f'PRAGMA index_info("{name}")')] == ["recipient_lookup", "wire_seq"]) + claim_statements = [statement for statement in statements + if statement.startswith("WITH requested") and "w AS MATERIALIZED" in statement] + plans = [[row[3] for row in connection.execute(f"EXPLAIN QUERY PLAN {statement}")] + for statement in claim_statements] + + assert len(batches) > 1 + assert all(len(batch.bindings) <= bound for batch in batches) + assert sum(len(batch.bindings) for batch in batches) == key_fields * len(distinct) + assert tuple(value for batch in batches for value in batch.bindings) == unlimited.bindings + assert limited.selection.viewer == AllOutcomes() and limited.sources == sources + assert limited.rows == ordinary.rows + assert limited.attempts == ordinary.attempts + assert limited.evidence == ordinary.evidence + assert len({claim.assignment_id for claim, _evidence in limited.rows}) == len(limited.rows) + order = [(-claim.wire_seq, claim.recipient) for claim, _evidence in limited.rows] + assert order == sorted(order) + assert len(claim_statements) >= 2 + for plan in plans: + assert any(detail.startswith(f"SEARCH {WakeAssignment.declared_name} USING INDEX {index}") + for detail in plan), plan + assert not any(detail.startswith(f"SCAN {WakeAssignment.declared_name}") for detail in plan), plan diff --git a/tests/test_pi_payloads.py b/tests/test_pi_payloads.py index 2aec06eb6a..6d0def274b 100644 --- a/tests/test_pi_payloads.py +++ b/tests/test_pi_payloads.py @@ -198,7 +198,7 @@ def test_optional_native_observations_have_named_absence_and_one_serialization() def test_tracked_commit_observations_preserve_original_receipt_and_refuse_repetition(): from agent_comms.native_failures import NativePiUnavailable - from agent_comms.tracked_turn import ObservedNativeCommit, PendingNativeCommit + from agent_comms.backend import ObservedNativeCommit, PendingNativeCommit event = decode({"type": "input_committed", "inputId": "a" * 32, "sessionId": "session", "sessionEntryId": "entry", "sessionEntryStart": 128}) pending = PendingNativeCommit() diff --git a/tests/test_project_metadata.py b/tests/test_project_metadata.py new file mode 100644 index 0000000000..034a8a0600 --- /dev/null +++ b/tests/test_project_metadata.py @@ -0,0 +1,185 @@ +"""Actual standard backend output owns package facts; no patched metadata producers.""" + +from collections.abc import Callable +from pathlib import Path +import sys +import tomllib +from zipfile import ZipFile + +from packaging.requirements import Requirement +from packaging.specifiers import SpecifierSet +import pytest +import tomli_w + +from agent_comms.head_review import BaseSide, HeadSide, IntegrationInput +from agent_comms.installer.commands import InstallCommands +from agent_comms.installer.refusals import UpgradeRefused +from agent_comms.installer.runtime_resolution import ProjectMetadata, StackSources +from agent_comms.repository_name import RepositoryName +from github_fixture import GitHubFixture +from integration_source_fixture import MutualSources +from module_layers import MaterializedClients +from source_revision_fixture import SourceRepository + +pytest_plugins = ("github_fixture", "test_holder_standing_rules", "integration_source_fixture") + + +def package(root: Path, name: str, *, dependencies: tuple[str, ...] = (), + extras: dict[str, list[str]] | None = None, python: str = ">=3.10") -> SourceRepository: + source = SourceRepository.create(root, RepositoryName.from_wire(f"example/{name}"), baseline_files={ + Path("pyproject.toml"): SourceRepository.package_project( + name, dependencies=dependencies, extras=extras, python=python), + Path("src/module.py"): "value = 1\n", + }) + source.git(source.checkout, "switch", "feature") + return source + + +@pytest.mark.parametrize("backend", ("hatch", "poetry")) +def test_standard_backend_metadata_preserves_identity_python_and_all_optional_edges( + tmp_path: Path, test_uv: Path, backend: str, +) -> None: + source = package(tmp_path / "source", "source-package", dependencies=("packaging>=24",), + extras={"syntax": ["idna>=3.9; python_version >= '3.10'"]}) + if backend == "poetry": + source.publish_change("Use the declared Poetry backend, without a project table", *source.write_files({ + Path("pyproject.toml"): tomli_w.dumps({ + "build-system": {"requires": ["poetry-core>=1.2.0"], "build-backend": "poetry.core.masonry.api"}, + "tool": {"poetry": {"name": "source-package", "version": "0.0.1", + "description": "Real backend control", "authors": ["Source "], + "packages": [{"include": "module.py", "from": "src"}], + "dependencies": {"python": ">=3.10", "packaging": ">=24", + "idna": {"version": ">=3.9", "optional": True, "python": ">=3.10"}}, + "extras": {"syntax": ["idna"]}}}, + })})) + with InstallCommands() as commands: + metadata = ProjectMetadata.acquire(source.checkout, uv=test_uv, python=Path(sys.executable), + commands=commands, output=tmp_path / "metadata") + assert metadata.name == "source-package" and metadata.extras == frozenset({"syntax"}) + assert len(metadata.requirements) == 2 + optional, = (edge for edge in metadata.requirements if edge.name == "idna") + assert optional.marker is not None and "extra" in str(optional.marker) + assert not optional.marker.evaluate({"extra": ""}) and optional.marker.evaluate({"extra": "syntax"}) + StackSources.require_package(metadata, "source_package") + with pytest.raises(ValueError, match="not stack package"): + StackSources.require_package(metadata, "different-package") + with pytest.raises(ValueError, match="common interpreter"): + ProjectMetadata.require_python(SpecifierSet("<1")) + environment = StackSources.marker_environment(Path(sys.executable)) + with pytest.raises(ValueError, match="selected common interpreter"): + StackSources.marker_environment(tmp_path / "different-interpreter") + root = {"project": {"dependencies": ["source-package", "inactive; python_version < '1'"]}, + "tool": {"uv": {"package": False}}} + assert all(edge.name != "inactive" for edge in StackSources.active_requirements(root, (metadata,), environment)) + assert all(edge.name != "idna" for edge in StackSources.active_requirements(root, (metadata,), environment)) + root["project"]["dependencies"] = ["source-package[syntax]"] + assert optional in StackSources.active_requirements(root, (metadata,), environment) + assert len(metadata.requirements) == 2 # Inactive edges were excluded from selection, never erased. + + +def test_runtime_extra_closure_keeps_bootstrap_only_changes_outside_runtime( + tmp_path: Path, test_uv: Path, +) -> None: + front = package(tmp_path / "front", "front", extras={"image": ["peer[paint]"]}) + before = package(tmp_path / "before", "peer", extras={ + "paint": ["front[image]", "idna>=3.9"], "checking": ["packaging>=24"]}) + after = package(tmp_path / "after", "peer", extras={ + "paint": ["front[image]", "idna>=3.10"], "checking": ["packaging>=25"]}) + with InstallCommands() as commands: + front_metadata, before_metadata, after_metadata = ( + ProjectMetadata.acquire(source.checkout, uv=test_uv, python=Path(sys.executable), + commands=commands, output=tmp_path / "metadata") + for source in (front, before, after)) + environment = StackSources.marker_environment(Path(sys.executable)) + root = {"project": {"dependencies": ["front"]}, "tool": {"uv": {"package": False}}} + before_active = StackSources.active_requirements(root, (front_metadata, before_metadata), environment) + after_active = StackSources.active_requirements(root, (front_metadata, after_metadata), environment) + assert before_active == after_active == (Requirement("front"),) + root["project"]["dependencies"] = ["front[image]"] + before_active = StackSources.active_requirements(root, (front_metadata, before_metadata), environment) + after_active = StackSources.active_requirements(root, (front_metadata, after_metadata), environment) + assert Requirement('peer[paint]; extra == "image"') in before_active + assert any(edge.name == "idna" for edge in before_active) + assert not any(edge.name == "packaging" for edge in before_active) + assert before_active != after_active + root["project"]["dependencies"] = ["peer[checking]"] + assert StackSources.active_requirements(root, (before_metadata,), environment) != StackSources.active_requirements( + root, (after_metadata,), environment) + + +def test_materialized_metadata_is_built_once_and_revalidated_when_borrowed( + mutual_sources: MutualSources, github_cli: GitHubFixture, + materialized_clients: Callable[[Path, IntegrationInput], MaterializedClients], +) -> None: + pair = mutual_sources + inputs = pair.capture(("P2", "P3"), "backend metadata lifetime") + resource = materialized_clients(pair.output, inputs) + metadata = resource.metadata + built = len(resource.commands.clock.phases) + assert built == sum(len(projects) for projects in metadata.values()) + for side in (BaseSide, HeadSide): + assert resource.projects(side) is metadata[side] + resource.cuts[side] + assert resource.metadata is metadata and len(resource.commands.clock.phases) == built + checkout = HeadSide.checkout(pair.output, pair.client.github) + (checkout / "src/workflow.py").write_text("dirty = True\n") + with pytest.raises(ValueError, match="dirty materialized"): + resource.projects(HeadSide) + + +@pytest.mark.parametrize("mutation", ("dirty", "moved")) +def test_backend_cannot_mutate_the_captured_source(tmp_path: Path, test_uv: Path, mutation: str) -> None: + source = package(tmp_path / "source", "source-package") + document = tomllib.loads((source.checkout / "pyproject.toml").read_text()) + document["tool"]["hatch"]["build"]["hooks"] = {"custom": {"path": "hatch_build.py"}} + action = '(Path(self.root) / "src/module.py").write_text("changed = True\\n")' + if mutation == "moved": + action += ('\n subprocess.run(["git", "-C", self.root, "-c", "user.name=Source", ' + '"-c", "user.email=source@example.invalid", "commit", "-am", "Backend moved source"], check=True)') + hook = ('from pathlib import Path\nimport subprocess\n' + 'from hatchling.builders.hooks.plugin.interface import BuildHookInterface\n' + 'class Hook(BuildHookInterface):\n' + ' def initialize(self, version, build_data):\n ' + action + '\n') + source.publish_change("Declare a real mutating backend hook", *source.write_files({ + Path("pyproject.toml"): tomli_w.dumps(document), Path("hatch_build.py"): hook})) + with InstallCommands() as commands, pytest.raises(ValueError, match="moved or dirtied"): + ProjectMetadata.acquire(source.checkout, uv=test_uv, python=Path(sys.executable), + commands=commands, output=tmp_path / "metadata") + + +def test_backend_failure_is_not_empty_metadata(tmp_path: Path, test_uv: Path) -> None: + source = package(tmp_path / "source", "source-package") + document = tomllib.loads((source.checkout / "pyproject.toml").read_text()) + document["project"]["version"] = "not-a-version" + source.publish_change("Declare invalid backend metadata", *source.write_files({ + Path("pyproject.toml"): tomli_w.dumps(document)})) + with InstallCommands() as commands, pytest.raises(UpgradeRefused): + ProjectMetadata.acquire(source.checkout, uv=test_uv, python=Path(sys.executable), + commands=commands, output=tmp_path / "metadata") + + +@pytest.mark.parametrize("corruption", ("missing", "duplicate", "name", "header")) +def test_wheel_decoder_refuses_malformed_real_backend_output( + tmp_path: Path, test_uv: Path, corruption: str, +) -> None: + source = package(tmp_path / "source", "source-package") + output = tmp_path / "wheel" + with InstallCommands() as commands: + commands.run(test_uv, "build", "--wheel", "--out-dir", output, "--python", Path(sys.executable), source.checkout) + wheel, = output.glob("*.whl") + with ZipFile(wheel) as archive: + entries = [(entry, archive.read(entry)) for entry in archive.infolist()] + with ZipFile(wheel, "w") as archive: + for entry, data in entries: + if entry.filename.endswith(".dist-info/METADATA"): + if corruption == "missing": + continue + if corruption == "name": + data = data.replace(b"Name: source-package", b"Name: different-package") + if corruption == "header": + data = data.replace(b"Name: source-package", b"Name: source-package\nName: source-package") + if corruption == "duplicate": + archive.writestr(entry, data) + archive.writestr(entry, data) + with pytest.raises(ValueError): + ProjectMetadata.from_wheel(wheel) diff --git a/tests/test_queue_bypass_hook.py b/tests/test_queue_bypass_hook.py index 0bb049f910..9f4cd2f823 100644 --- a/tests/test_queue_bypass_hook.py +++ b/tests/test_queue_bypass_hook.py @@ -7,11 +7,11 @@ import pytest -from agent_comms.todos import ReviewedHeadVerdict, RejectedHeadVerdict +from agent_comms.todos import QualifiedHeadVerdict, ReviewedHeadVerdict, RejectedHeadVerdict from agent_comms.shell_commands import ShellCommand from agent_comms_hooks.queue_bypass import MergeTarget, MergeTargetUnreadable, QueueBypass from agent_comms_hooks.watch import WatchClaim, WatchRecord -from test_head_review import ACCEPTED, REASON, invoke, opened +from test_head_review import ACCEPTED, QUALIFIED, REASON, invoke, opened pytest_plugins = ("test_head_review",) @@ -137,3 +137,5 @@ def refused(head): opened(github, reviewed_source, request.pr, request.head) invoke(comms, request, RejectedHeadVerdict(REASON)) refused(request.head) + invoke(comms, request, QualifiedHeadVerdict(QUALIFIED)) + refused(request.head) diff --git a/tests/test_recipient_activity_index.py b/tests/test_recipient_activity_index.py index 6d35c16edb..fa2208114e 100644 --- a/tests/test_recipient_activity_index.py +++ b/tests/test_recipient_activity_index.py @@ -86,8 +86,62 @@ def test_viewer_selects_the_outcomes_of_a_delivery_window(tmp_path, viewer_of, s original = comms.bus.bus.delivery(sent.seq) lookups = {name: thread.incarnation.lookup for name, thread in comms.registry.snapshot().threads.items()} + viewer = viewer_of(lookups) window = MessageNotification.delivery_window( - comms.root, comms.registry, (original,), - NotificationAssignment.read(comms.root, (original,)), viewer=viewer_of(lookups), + comms.root, comms.registry, + NotificationAssignment.read(comms.root, viewer.selection((original,))), ) assert {item.recipient for item in window.outcomes[(sent.seq, sent.message_id)]} == seen + + +def test_a_viewer_follows_frozen_lookups_through_rename_and_never_into_a_rebirth(tmp_path): + comms = canonical_goal_wire(tmp_path / "bus") + for name in ("sender", "peer-a", "peer-b"): + comms.registry.declare(Thread(name, frozenset({"ci"}), Checkout(tmp_path))) + sent = comms.messaging.send("sender", "#ci", "channel outcome").message + original = comms.bus.bus.delivery(sent.seq) + key = (sent.seq, sent.message_id) + + def seen_by(name: str) -> set[str]: + viewer = OwnerOutcomes.of(comms.registry.require(name)) + read = NotificationAssignment.read(comms.root, viewer.selection((original,))) + assert read.selection.viewer == viewer + window = MessageNotification.delivery_window(comms.root, comms.registry, read) + return {item.recipient for item in window.outcomes[key]} + + assert seen_by("peer-a") == {"peer-a"} + comms.registry.rename("peer-a", "renamed-a") + assert seen_by("renamed-a") == {"peer-a"} + comms.registry.remove("peer-b") + comms.registry.declare(Thread("peer-b", frozenset({"ci"}), Checkout(tmp_path))) + assert seen_by("peer-b") == set() + assert seen_by("sender") == {"peer-a", "peer-b"} + + +def test_a_public_root_without_a_coordinator_keeps_its_viewer_and_projects_visible_recipients(tmp_path): + from agent_comms.coordination_database import CoordinationStore + + comms = wire(tmp_path / "bus") + for name in ("sender", "peer-a", "peer-b"): + comms.registry.declare(Thread(name, frozenset({"ci"}), Checkout(tmp_path))) + sent = comms.messaging.send("sender", "#ci", "channel outcome").message + original = comms.bus.bus.delivery(sent.seq) + CoordinationStore.path_in(comms.root).unlink(missing_ok=True) + viewer = OwnerOutcomes.of(comms.registry.require("peer-a")) + read = NotificationAssignment.read(comms.root, viewer.selection((original,))) + assert (read.sources, read.rows, read.attempts, read.selection.viewer) == ((original,), (), (), viewer) + window = MessageNotification.delivery_window(comms.root, comms.registry, read) + assert {item.recipient for item in window.outcomes[(sent.seq, sent.message_id)]} == {"peer-a"} + + +def test_a_private_root_without_a_coordinator_still_refuses_notification_reads(private_comms): + from agent_comms.coordination_database import CoordinationStore + from agent_comms.coordination_errors import PublicationActivationBlocked + + for name in ("sender", "peer-a"): + private_comms.registry.declare(Thread(name, frozenset({"ci"}), Checkout(private_comms.root))) + sent = private_comms.messaging.send("sender", "#ci", "channel outcome").message + original = private_comms.bus.bus.delivery(sent.seq) + CoordinationStore.path_in(private_comms.root).unlink() + with pytest.raises(PublicationActivationBlocked): + NotificationAssignment.read(private_comms.root, AllOutcomes().selection((original,))) diff --git a/tests/test_runtime.py b/tests/test_runtime.py index 0a609af0cb..ec43cb2299 100644 --- a/tests/test_runtime.py +++ b/tests/test_runtime.py @@ -20,7 +20,16 @@ decode_updates, encode_updates, ) -from agent_comms.child_process import ProcessIdentity, SpawnedProcess +from agent_comms.child_process import ProcessIdentity +from agent_comms.errors import RelationViolationError +from agent_comms.field_codec import FieldCodec +from agent_comms.input_disposition import InputDispositions, InputDocument +from agent_comms.input_key import BusInputKey +from agent_comms.bus_read import OwnerReadRuntimeRequest +from agent_comms.order_tools import InputDeliveryRead +from agent_comms.owner_connection import OwnerIdentityChangedError, OwnerReplyConnection, RuntimeRequest +from agent_comms.thread_execution import ExternalThreadExecution +from agent_comms.tool_caller import ToolCaller from agent_comms.comms import wire from agent_comms.field_codec import RootId from agent_comms.owner_protocol import OwnerAnnouncement, OwnerReady, OwnerResult, OwnerUpdate @@ -32,42 +41,16 @@ from agent_comms.session_file import RegistrySessionLocation from agent_comms.native_arguments import GivenBinary from delivery_owner_fixture import canonical_agent +from owner_socket_fixture import declare_owner, silent_socket +from stored_inputs import stored_inputs from agent_comms.threads import Thread -@pytest.fixture -def runtime_processes(): - import sys - - children = [] - try: - for _ in range(2): - children.append( - SpawnedProcess.launch((sys.executable, "-c", "import time; time.sleep(60)")) - ) - yield tuple(child.identity for child in children) - finally: - for child in children: - child.stop_sync() - - async def test_dispatched_request_without_a_reply_remains_unknown_and_is_not_replayed(comms): process = ProcessIdentity.capture(os.getpid()) - comms.registry.declare(Thread( - "reply-loss", frozenset(), Checkout(comms.root.parent), process_identity=process, - )) - path = socket_path(comms.root, process.pid) - path.parent.mkdir(parents=True, exist_ok=True) - received = [] - - async def disconnect_after_request(reader, writer): - received.append(json.loads(await reader.readline())) - writer.close() - await writer.wait_closed() - - with PrivateSocketRole.address(path) as address: - server = await asyncio.start_unix_server(disconnect_after_request, path=address) - connection = RuntimeConnection(comms, "reply-loss", path) + declare_owner(comms, "reply-loss", process) + async with silent_socket(comms, process) as received: + connection = RuntimeConnection(comms, "reply-loss", socket_path(comms.root, process.pid)) try: with pytest.raises(OwnerReplyLostError) as lost: await connection.request("cancel") @@ -76,9 +59,113 @@ async def disconnect_after_request(reader, writer): assert [request["action"] for request in received] == ["cancel"] finally: await connection.close() - server.close() - await server.wait_closed() - path.unlink(missing_ok=True) + + +def recorded_input(comms) -> InputDocument: + """One durable input of the read's subject; a refused or lost read leaves it unchanged.""" + ledger = InputDispositions(comms.root) + ledger.record(BusInputKey(5), seq=5, owner="held", claim_scope=1, target="held", text="Kept") + return stored_inputs(ledger.root) + + +async def mcp_input_delivery(comms, include_history: bool = True): + async with asyncio.timeout(30): + return await asyncio.to_thread( + InputDeliveryRead.invoke, comms, {"thread": "held", "include_history": include_history}, + caller=ToolCaller.from_actor(None), + ) + + +@pytest.mark.parametrize("include_history", [False, True]) +async def test_an_input_delivery_read_without_a_reply_is_lost_after_one_dispatch(comms, include_history): + """The owner received exactly the declared operation once, then hung up: the caller + gets the original reply loss, and no durable input moved. The peer runs no domain body.""" + process = ProcessIdentity.capture(os.getpid()) + declare_owner(comms, "held", process) + before = recorded_input(comms) + async with silent_socket(comms, process) as received: + with pytest.raises(OwnerReplyLostError): + await mcp_input_delivery(comms, include_history) + assert [FieldCodec.decode(RuntimeRequest, request) for request in received] == [ + OwnerReadRuntimeRequest( + thread="held", operation=InputDeliveryRead(thread="held", include_history=include_history), + ), + ] + assert stored_inputs(comms.root) == before + + +async def test_a_held_owner_read_refuses_a_replaced_root_before_dispatch(comms): + process = ProcessIdentity.capture(os.getpid()) + owner = declare_owner(comms, "held", process) + reading = OwnerReadRuntimeRequest(thread="held", operation=InputDeliveryRead(thread="held")) + async with silent_socket(comms, process) as received: + connection = OwnerReplyConnection.held(comms, owner.incarnation, reading) + moved = comms.root.with_name(comms.root.name + "-replaced") + comms.root.rename(moved) + comms.root.mkdir() + try: + with pytest.raises(OwnerIdentityChangedError): + async with asyncio.timeout(30): + await reading.read(connection) + finally: + comms.root.rmdir() + moved.rename(comms.root) + await connection.close() + assert received == [] + + +async def test_a_same_name_recreation_refuses_the_held_read_and_the_old_witness(comms): + process = ProcessIdentity.capture(os.getpid()) + old = declare_owner(comms, "held", process) + reading = OwnerReadRuntimeRequest(thread="held", operation=InputDeliveryRead(thread="held")) + async with silent_socket(comms, process) as received: + connection = OwnerReplyConnection.held(comms, old.incarnation, reading) + try: + comms.registry.unregister("held") + comms.registry.begin_delete("held") + with comms.registry.store.editing() as edit: + edit.document.remove_originals((old,)) + edit.commit() + new = declare_owner(comms, "held", process) + assert new.name == old.name and new.incarnation != old.incarnation + assert comms.registry.require("held").incarnation == new.incarnation + with pytest.raises(OwnerIdentityChangedError): + async with asyncio.timeout(30): + await reading.read(connection) + with pytest.raises(OwnerIdentityChangedError): + OwnerReplyConnection.held(comms, old.incarnation, reading) + finally: + await connection.close() + assert received == [] + + +async def test_a_held_owner_read_refuses_after_its_process_dies(comms, runtime_processes): + child = runtime_processes[0] + owner = declare_owner(comms, "held", child.identity) + reading = OwnerReadRuntimeRequest(thread="held", operation=InputDeliveryRead(thread="held")) + before = recorded_input(comms) + async with silent_socket(comms, child.identity) as received: + connection = OwnerReplyConnection.held(comms, owner.incarnation, reading) + try: + child.stop_sync() + assert not child.identity.alive() + with pytest.raises(ConnectionError): + async with asyncio.timeout(30): + await reading.read(connection) + finally: + await connection.close() + assert received == [] + assert stored_inputs(comms.root) == before + + +async def test_an_external_owner_refuses_the_input_delivery_read_before_any_socket_wait( + comms, runtime_processes, +): + declare_owner(comms, "held", runtime_processes[0].identity, ExternalThreadExecution) + before = recorded_input(comms) + with pytest.raises(RelationViolationError): + await mcp_input_delivery(comms) + assert stored_inputs(comms.root) == before def test_owner_cursor_scope_rebases_only_attachment_session_alias(): @@ -185,7 +272,7 @@ async def test_existing_proxy_follows_renamed_owner_restart_and_resubscribes( tmp_path, runtime_processes ): comms = wire(tmp_path / "wire") - old, new = runtime_processes + old, new = (child.identity for child in runtime_processes) old_pid, new_pid = old.pid, new.pid comms.registry.declare(Thread('worker', frozenset(), Checkout(tmp_path), process_identity=old)) client = canonical_agent(comms) diff --git a/tests/test_runtime_resolution.py b/tests/test_runtime_resolution.py index 17c50d3cfb..6083cc5e88 100644 --- a/tests/test_runtime_resolution.py +++ b/tests/test_runtime_resolution.py @@ -1,4 +1,5 @@ import os +import tomllib from pathlib import Path import pytest @@ -21,7 +22,11 @@ def test_package_probe_entry_uses_one_declaration_before_refusing_build_identity ) -def manifest(core="core", toad="toad", wheel="old-wheel"): +TOAD_COMMIT = "a" * 40 +TEXTUAL_COMMIT = "b" * 40 + + +def manifest(core="core", toad=TOAD_COMMIT, wheel="old-wheel"): return RuntimeResolution.model_validate( { "core_commit": core, @@ -35,7 +40,7 @@ def manifest(core="core", toad="toad", wheel="old-wheel"): "version": "3", "source": { "url": "https://example/textual", - "vcs_info": {"commit_id": "textual"}, + "vcs_info": {"commit_id": TEXTUAL_COMMIT}, }, }, "rich": {"version": "4", "source": None}, @@ -62,12 +67,12 @@ def lock(): { "name": "batrachian-toad", "version": "2", - "source": {"git": "https://example/toad#toad"}, + "source": {"git": f"https://example/toad#{TOAD_COMMIT}"}, }, { "name": "textual", "version": "3", - "source": {"git": "https://example/textual#textual"}, + "source": {"git": f"https://example/textual#{TEXTUAL_COMMIT}"}, }, {"name": "rich", "version": "4", "source": {"registry": "https://pypi.org/simple"}}, # The universal lock contains entries not selected by this synced environment. @@ -88,7 +93,7 @@ def test_recorded_sources_match_selected_lock_not_wheel_locations(): assert built.pins() == pins assert manifest(wheel="another-wheel").pins() == pins assert manifest(core="changed").pins() != pins - assert manifest(toad="changed").pins() != pins + assert manifest(toad="d" * 40).pins() != pins assert lock().pins({**selected, "unselected": "5"}, "core") != built.pins() @@ -110,7 +115,8 @@ def test_the_stack_relocks_every_branch_tracking_package_and_refuses_a_unit_bran tracked = lock.tracked() assert {"batrachian-toad", "metaclass-registry", "textual"} <= set(tracked) assert "objectstate" not in tracked - toad = next(entry for entry in lock.package if entry.name == "batrachian-toad") - toad.source.git = toad.source.git.replace("branch=main", "branch=work/unit-toad--P694") + document = tomllib.loads(path.read_text()) + toad = next(entry for entry in document["package"] if entry["name"] == "batrachian-toad") + toad["source"]["git"] = toad["source"]["git"].replace("branch=main", "branch=work/unit-toad--P694") with pytest.raises(ValueError, match="batrachian-toad"): - lock.tracked() + ResolvedLock.model_validate(document).tracked() diff --git a/tests/test_source_revision.py b/tests/test_source_revision.py index a47f64e270..1f94bce4ea 100644 --- a/tests/test_source_revision.py +++ b/tests/test_source_revision.py @@ -1,5 +1,6 @@ """Citations use fetched Git objects, not whichever source is on disk.""" from dataclasses import replace +from pathlib import Path import shutil import pytest @@ -28,6 +29,43 @@ def test_source_creation_does_not_reuse_a_retired_directory(tmp_path): assert replacement.git(replacement.checkout, 'rev-parse', 'HEAD').strip() == replacement.main_commit +def test_seeded_source_publishes_descendants_and_merges_main_without_replacing_a_pr(tmp_path: Path) -> None: + files = { + Path("pyproject.toml"): SourceRepository.package_project("source-package"), + Path("src/api.py"): 'def old_api() -> str:\n return "baseline"\n', + } + source = SourceRepository.create(tmp_path, baseline_files=files) + main, feature = source.main_commit, source.branch_commit + assert source.git(source.checkout, "rev-parse", "HEAD").strip() == main + assert source.git(source.checkout, "show", f"{main}:owner.py") == source.main_source + assert source.git(source.checkout, "show", f"{feature}:owner.py") == ( + source.main_source + 'class Member(Parent):\n pass\n') + assert source.git(source.checkout, "show", f"{feature}:branch.py") == '@decorated\nclass BranchOnly:\n pass\n' + for revision in (main, feature): + for path, text in files.items(): + assert source.git(source.checkout, "show", f"{revision}:{path}") == text + assert source.descends(source.checkout, main, feature) + source.git(source.checkout, "switch", "feature") + candidate = source.publish_change("Replace the seeded API", *source.write_files({ + Path("src/api.py"): 'def new_api() -> str:\n return "candidate"\n'})) + assert source.descends(source.checkout, feature, candidate) + assert source.git(source.remote, "rev-parse", "refs/pull/7/head").strip() == candidate + source.git(source.checkout, "switch", "main") + advanced = source.commit_change("Advance main independently", *source.write_files({ + Path("main_only.py"): "advanced = True\n"})) + source.git(source.checkout, "push", "origin", "main") + assert source.git(source.remote, "rev-parse", "main").strip() == advanced + assert source.git(source.remote, "rev-parse", "refs/pull/7/head").strip() == candidate + source.git(source.checkout, "switch", "feature") + source.git(source.checkout, "merge", "--no-edit", "main") + final = source.publish_change("Publish the merged descendant", *source.write_files({ + Path("src/next.py"): "next_value = 2\n"})) + for ancestor in (candidate, advanced): + assert source.descends(source.checkout, ancestor, final) + assert source.git(source.remote, "rev-parse", "refs/pull/7/head").strip() == final + assert source.git(source.checkout, "show", f"{final}:main_only.py") == "advanced = True\n" + + def test_main_revision_refuses_branch_lines_and_ignores_dirty_disk(tmp_path): source = SourceRepository.create(tmp_path) (source.checkout / 'owner.py').write_text('disk-only\n' * 100) diff --git a/tests/test_stack_native_compaction.py b/tests/test_stack_native_compaction.py index 33400ba2d3..647460a16c 100644 --- a/tests/test_stack_native_compaction.py +++ b/tests/test_stack_native_compaction.py @@ -87,7 +87,7 @@ def _is_summary(request: dict) -> bool: "summary_failure", "post_compaction_tool_rounds", ]) async def test_pi_engine_compacts_inside_a_core_turn( - case: str, monkeypatch, native_runtime_package, private_directory) -> None: + case: str, monkeypatch, native_runtime_package, private_directory, cursor_delivery) -> None: native_bin = os.environ.get("AC_NATIVE_STACK_BIN") if not native_bin: pytest.skip("Set AC_NATIVE_STACK_BIN to the prepared pinned Pi launcher") @@ -230,6 +230,7 @@ def log_message(self, *_args): root=project, owner=NativeBackendFixture.participant_at(project, project), env_extra=child_env, session_file=str(session), input_start=AnnouncedInputStart(), + cursor_delivery=cursor_delivery, persistent_session=pi_session, package=native_runtime_package, tool_mode=NoSelectedTools(), diff --git a/tests/test_thread_presentation.py b/tests/test_thread_presentation.py index 5679155b19..402e1a9b9c 100644 --- a/tests/test_thread_presentation.py +++ b/tests/test_thread_presentation.py @@ -43,7 +43,8 @@ def test_missing_presentation_owns_source_absence_and_retired_status(comms): presentations = comms.views.thread_presentations(("missing",)) absent = presentations.read("missing") assert absent == presentations.view("missing") == NoThreadPresentation() - assert not absent.notification_window.activity_names() + assert absent.published() is absent + assert absent.sent_notifications() == () assert absent.label == "" painted = ThreadStatusModel.known("missing", absent) retired = ThreadStatusModel.retired("missing") diff --git a/tests/test_thread_read_results.py b/tests/test_thread_read_results.py index 14356cfcbd..04c44f17b4 100644 --- a/tests/test_thread_read_results.py +++ b/tests/test_thread_read_results.py @@ -77,7 +77,8 @@ def test_assignment_row_changes_attribute_only_their_view_without_projection( from agent_comms.coordinator import Coordination from agent_comms.presentation import MessageNotification from agent_comms.threads import Thread - from agent_comms.ui_model.observation import NotificationsRead, ReadNotifications + from agent_comms.notification_read import NotificationsRead + from agent_comms.notification_reads import LiveNotificationRead root, root_id, comms, initial, _people = _root(private_directory, direct=True) for name in ('gamma', 'outside'): @@ -100,7 +101,7 @@ def test_assignment_row_changes_attribute_only_their_view_without_projection( revision = comms.views.revision() activity = comms.views.observed_activity(revision, names) previous = ThreadsObserved(revision, results, names, activity) - notification_window = ReadNotifications(root_id, (notification_source.reference,)).read(comms) + notification_window = LiveNotificationRead((notification_source.reference,)).read(comms) notification_read = NotificationsRead.of(revision, comms, activity, notification_window) if changed_name in ('read', 'unrelated_read'): ledger = comms.bus.reads @@ -145,3 +146,94 @@ def count_projection(frame, event, argument): assert calls == 0 if changed_name == 'expiry': assert evidence_calls == 0 + + +def reachable_types(root: object) -> set[str]: + """Class names of every instance reachable from root, not following classes, + modules or functions.""" + import gc + import types + + seen, names, frontier = set(), set(), [root] + skipped = (type, types.ModuleType, types.FunctionType, types.BuiltinFunctionType, + types.MethodType) + while frontier: + item = frontier.pop() + if id(item) in seen or isinstance(item, skipped): + continue + seen.add(id(item)) + names.add(type(item).__name__) + frontier.extend(gc.get_referents(item)) + return names + + +def pickled_types(value: object) -> set[str]: + """Class names of every object the transport pickles for value.""" + import io + import pickle + + names: set[str] = set() + + class Recording(pickle.Pickler): + def reducer_override(self, obj): + names.add(type(obj).__name__) + return NotImplemented + + Recording(io.BytesIO(), protocol=pickle.HIGHEST_PROTOCOL).dump(value) + return names + + +ACQUIRED_EVIDENCE = { + "AcquiredThreadPresentation", "CapturedNotificationWindow", "NotificationAssignmentRead", + "SourceEvidence", "WakeAssignment", "AssignmentActivityEvidence", "HarnessDelivery", + "Delivered", "NotificationReadMark", +} + + +def test_thread_reading_keeps_acquired_evidence_and_publishes_its_projection(private_directory): + import pickle + + from conftest import publish_initial + from test_coordinated_runtime import _root + from agent_comms.thread_presentation import AcquiredThreadPresentation, ThreadPresentation + + _path, _root_id, comms, initial, _people = _root(private_directory, direct=True) + reading = ObserveThreads(frozenset({'beta'})).observation() + status = OpenThreadStatus(lambda flush: None) + status.open('beta') + + def observe() -> ThreadsObserved: + published = [step for step in reading.read(comms, comms.views.revision()) + if type(step) is ThreadsObserved] + assert len(published) == 1 + status.observed(published[0]) + status.rows.flush() + return published[0] + + first = observe() + retained = reading.observed.results['beta'].presentation + assert type(retained) is AcquiredThreadPresentation + assert retained.incoming_references + outgoing = first.results['beta'].presentation + assert type(outgoing) is ThreadPresentation + assert outgoing == retained.published() + assert outgoing.status_display() == retained.status_display() + assert outgoing.read_identity is retained.read_identity + assert outgoing.notifications == retained.notifications + incoming = [item for item in outgoing.notifications if item.message.seq == initial.message.seq] + assert incoming and all(item.message == initial.envelope for item in incoming) + assert not ACQUIRED_EVIDENCE & pickled_types(first) + with pytest.raises(TypeError, match="published"): + pickle.dumps(replace(first, results={'beta': ThreadPresented(retained)})) + assert (ThreadStatusModel.known('beta', outgoing).lines + == ThreadStatusModel.known('beta', retained).lines) + + publish_initial(comms, 'sender', 'beta', 'A later message for beta') + second = observe() + assert second.results['beta'].presentation != outgoing + assert reading.observed.results['beta'].presentation is not retained + # The UI's current and last-published rows hold projections only; the old acquired + # value is reachable from neither. + assert status.rows._published == status.rows.rows + assert not ACQUIRED_EVIDENCE & reachable_types(status.rows) + assert type(status.rows.rows['beta'].presentation) is ThreadPresentation diff --git a/tests/test_tool_replies.py b/tests/test_tool_replies.py index 7e8f779ab1..0a21f3cdf1 100644 --- a/tests/test_tool_replies.py +++ b/tests/test_tool_replies.py @@ -1,18 +1,62 @@ """Every declared tool's reply is the encoding of its declared result record.""" -from typing import get_type_hints +from types import get_original_bases +from typing import TypeVar, get_args, get_origin, get_type_hints +import pytest + +from agent_comms.bus_read import BusRead from agent_comms.comms import Comms from agent_comms.tool_result import ToolResult from agent_comms.tool_request import ToolRequest -def result_type(tool: type[ToolRequest]) -> type: +def bus_read_result(tool: type[BusRead]) -> type: + """The result a bus read declares once: the first concrete specialisation of a BusRead + capability along its MRO. A family header still carrying its type parameter declares + nothing; two different concrete answers contradict each other.""" + declared = tuple(dict.fromkeys( + argument + for declaration in tool.__mro__ + for base in get_original_bases(declaration) + if isinstance(get_origin(base), type) and issubclass(get_origin(base), BusRead) + for argument in get_args(base) + if not isinstance(argument, TypeVar) + )) + if len(declared) != 1: + raise TypeError(f"{tool.__qualname__} declares results {declared!r}, not exactly one") + return declared[0] + + +def apply_result(tool: type[ToolRequest]) -> type: return get_type_hints(tool.apply, localns={ "Comms": Comms, **{param.__name__: param for param in ToolRequest.__type_params__}, })["return"] +def result_type(tool: type[ToolRequest]) -> type: + if issubclass(tool, BusRead): + return bus_read_result(tool) + return apply_result(tool) + + +def test_bus_read_result_follows_an_inherited_specialisation_and_refuses_an_unspecialised_family(): + from agent_comms.bus_read import OwnerLiveRead, UiRead + from agent_comms.notification_evidence import NotificationWindow + from agent_comms.notification_reads import RecordedNotificationRead + + # RecordedNotificationRead restates no header: it inherits NotificationRead's UiRead[...]. + assert bus_read_result(RecordedNotificationRead) is NotificationWindow + for family in (BusRead, UiRead, OwnerLiveRead): + with pytest.raises(TypeError, match="not exactly one"): + bus_read_result(family) + + +def test_an_ordinary_tool_keeps_its_apply_annotation(): + ordinary = next(tool for tool in ToolRequest.members_with(ToolRequest) if not issubclass(tool, BusRead)) + assert result_type(ordinary) is apply_result(ordinary) + + def stores(comms: Comms) -> dict: """registry.json, the todo store and the goal store, byte for byte.""" paths = (comms.root / "registry.json", comms.goals.todos.path, @@ -22,6 +66,5 @@ def stores(comms: Comms) -> dict: def test_every_tool_declares_a_tool_result(): for tool in ToolRequest.members_with(ToolRequest): - from typing import get_args result = result_type(tool) assert all(issubclass(member, ToolResult) for member in (get_args(result) or (result,))), tool.declared_name diff --git a/tests/test_tools.py b/tests/test_tools.py index aed648ac23..d752df53d1 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -139,7 +139,8 @@ def test_threads_tool_summarizes_each_recent_sent_message_once(self, comms, monk presentation = comms.views.thread_presentations(("sender",)).require("sender") assert all(row.message.reference != sent.reference for row in presentation.notifications) outcomes = presentation.notification_window.outcomes[sent.seq, sent.message_id] - assert [summary.reference for summary in SentMessageSummary.recent(presentation)] == [sent.reference] + assert [summary.reference for summary in SentMessageSummary.for_sources( + presentation.sent_notifications())] == [sent.reference] result = invoke_tool(comms, "comms_threads", {}, caller=ThreadCaller("sender")) # One row per sent message, never one per recipient, and never the message body. diff --git a/tests/test_transcript_observation_lifetime.py b/tests/test_transcript_observation_lifetime.py index 3d04b673f7..e5a37f215c 100644 --- a/tests/test_transcript_observation_lifetime.py +++ b/tests/test_transcript_observation_lifetime.py @@ -12,6 +12,7 @@ from agent_comms.coordinator import Coordination from agent_comms.native_arguments import GivenBinary from agent_comms.runtime import RuntimeProxy, socket_path +from agent_comms.notification_reads import LiveNotificationRead from agent_comms.ui_model.observation import ( ObservationProcess, ObserverDied, @@ -90,7 +91,7 @@ async def test_subscriptions_share_process_and_last_close_joins_it(headless_owne async def test_broken_pipe_ends_subscription_with_observer_died(headless_owner): process = ObservationProcess(headless_owner._comms.route) pending = asyncio.create_task(process.read( - ReadNotifications(str(headless_owner._comms.root), ()))) + ReadNotifications(str(headless_owner._comms.root), LiveNotificationRead(())))) try: await asyncio.sleep(0) process.process.terminate() diff --git a/tests/test_transcript_pages.py b/tests/test_transcript_pages.py index f9e86fbf87..38fa13ed2d 100644 --- a/tests/test_transcript_pages.py +++ b/tests/test_transcript_pages.py @@ -12,6 +12,12 @@ from agent_comms.thread_identity import RecordedThreadIncarnation from agent_comms.transcripts import NativeTranscriptCursor, WireTranscriptCursor from agent_comms.thread_text import ForkedParent, GivenThreadText +from agent_comms.acp_extension import TranscriptSnapshotUpdate +from agent_comms.field_codec import FieldCodec +from agent_comms.order_tools import TranscriptPageRead +from agent_comms.tool_caller import ThreadCaller +from agent_comms.tools import invoke_tool +from agent_comms.ui_model.observation import ReadTranscript def transcript(path, count): @@ -57,6 +63,44 @@ def test_pages_reach_beginning_and_return_to_tail_without_duplicates(tmp_path): assert seen == [f"Record {i}" for i in range(105)] +def shared_page(comms, operation: TranscriptPageRead) -> TranscriptSnapshotUpdate: + """One page operation served directly, inside the UI's root-bound envelope and through MCP: + all three carry the same page, draft and witness as the domain capture.""" + expected = TranscriptSnapshotUpdate.from_read(comms.transcripts.capture_page_read( + operation.thread, before=operation.before, after=operation.after, + through=operation.through, historical_source=operation.historical_source)) + direct = operation.read(comms) + assert ReadTranscript(str(comms.root), operation).read(comms) == direct == expected + assert invoke_tool(comms, operation.declared_name, FieldCodec.encode_fields(operation), + caller=ThreadCaller(operation.thread)) == FieldCodec.encode(direct) + return direct + + +def test_one_page_operation_pages_alike_for_the_ui_and_mcp(tmp_path): + path = tmp_path / "session.jsonl" + transcript(path, 45) + comms = installed_comms(tmp_path / "wire") + comms.registry.declare(Thread('worker', frozenset(), Checkout(tmp_path), session_file=RegistrySessionLocation.from_path(str(path), comms.root))) + snapshot = shared_page(comms, TranscriptPageRead(thread="worker")) + through = snapshot.page.after + seen = [event.text for event in snapshot.page.events] + pages = 1 + while snapshot.page.has_older: + snapshot = shared_page(comms, TranscriptPageRead(thread="worker", before=snapshot.page.before)) + seen[:0] = [event.text for event in snapshot.page.events] + pages += 1 + assert pages > 1 and seen == [f"Record {i}" for i in range(45)] + seen = [event.text for event in snapshot.page.events] + while snapshot.page.has_newer: + snapshot = shared_page(comms, TranscriptPageRead( + thread="worker", after=snapshot.page.after, through=through)) + seen.extend(event.text for event in snapshot.page.events) + assert seen == [f"Record {i}" for i in range(45)] + comms.registry.declare(Thread('quiet', frozenset(), Checkout(tmp_path))) + empty = shared_page(comms, TranscriptPageRead(thread="quiet")) + assert not empty.page.events and not empty.page.has_older and not empty.page.has_newer + + def test_oversized_message_and_file_identity(tmp_path): path = tmp_path / "session.jsonl" text = "x" * 200000 diff --git a/tests/test_unreadable_history_observation.py b/tests/test_unreadable_history_observation.py index 3791deea7a..2320995c3d 100644 --- a/tests/test_unreadable_history_observation.py +++ b/tests/test_unreadable_history_observation.py @@ -26,6 +26,7 @@ from agent_comms.owner_connection import OwnerIdentityChangedError, OwnerReplyLostError from agent_comms.context_store import ContextRequest from agent_comms.presentation import WireRevision +from agent_comms.notification_reads import LiveNotificationRead from agent_comms.ui_model.observation import ( ObservationProcess, ObserveSidebar, @@ -45,7 +46,7 @@ ) from agent_comms.ui_model.status import OpenThreadStatus from agent_comms.turn_context import RecordedContextManifest -from test_observer_death import readable +from observed_queue import ObservedQueue from test_turn_context_wire import corrupt_context_kind, manifest from agent_comms.threads import Thread @@ -222,56 +223,23 @@ async def test_unknown_saved_member_does_not_kill_the_real_observation_process(c corrupt_context_kind(comms, "saved", "unavailable_future_segment") original = comms.contexts.path.read_bytes() original_bus = comms.bus.bus.path.read_bytes() - process = ObservationProcess(comms.route) - try: - await readable(process) - process.results() + delivered = ObservedQueue() + async with ObservationProcess(comms.route) as process: + process.start_receiving(delivered.observed, delivered.failed) process.request(ObserveSidebar(Checkout(comms.root.parent), True, True)) - async with asyncio.timeout(30): - while True: - await readable(process) - results = process.results() - sidebar = next( - (result for result in results if isinstance(result, SidebarObserved)), None - ) - if sidebar is not None: - break + await delivered.until(lambda result: isinstance(result, SidebarObserved)) assert process.process.is_alive() process.request(ObserveThreads(frozenset({"saved"}))) - async with asyncio.timeout(30): - while True: - await readable(process) - results = process.results() - threads = next( - (result for result in results if isinstance(result, ThreadsObserved)), None - ) - if threads is not None: - break + threads = await delivered.until(lambda result: isinstance(result, ThreadsObserved)) assert not isinstance(threads.results["saved"], ThreadUnavailable) # Only the request needing the damaged payload is refused, over the # same actual child/pipe that served the sidebar and thread reads. - damaged = asyncio.create_task(process.read(ReadRecordedContext(str(comms.root), "saved"))) - await asyncio.sleep(0) - async with asyncio.timeout(30): - while not damaged.done(): - await readable(process) - process.results() - await asyncio.sleep(0) with pytest.raises(StoredHistoryUnreadable): - await damaged + async with asyncio.timeout(30): + await process.read(ReadRecordedContext(str(comms.root), "saved")) # Serve a separate, readable request using the same actual pipe/child. - waiting = asyncio.create_task(process.read(ReadNotifications(str(comms.root), ()))) - await asyncio.sleep(0) async with asyncio.timeout(30): - while not waiting.done(): - await readable(process) - process.results() - # The pipe resolved the Future; let its awaiting Task resume. - # An unchanged error no longer emits an unrelated extra wake. - await asyncio.sleep(0) - assert await waiting == {} + assert (await process.read(ReadNotifications(str(comms.root), LiveNotificationRead(())))).outcomes == {} assert process.process.is_alive() assert comms.contexts.path.read_bytes() == original assert comms.bus.bus.path.read_bytes() == original_bus - finally: - process.close() diff --git a/tools/checker_stack.py b/tools/checker_stack.py new file mode 100644 index 0000000000..b1f7c35bc2 --- /dev/null +++ b/tools/checker_stack.py @@ -0,0 +1,205 @@ +"""Build the one checker environment from materialized CI sources, never resolve sources here.""" + +from __future__ import annotations + +import argparse +import hashlib +import os +import shutil +import sys +import tomllib +from dataclasses import dataclass +from functools import cached_property +from pathlib import Path + +from packaging.requirements import Requirement +from packaging.utils import canonicalize_name +import tomli_w + +from agent_comms.head_review import BaseSide, HeadSide, IntegrationInput, Side +from agent_comms.installer.commands import InstallCommands +from agent_comms.installer.declared_uv import DeclaredUv +from agent_comms.installer.runtime_resolution import ( + CorePackage, PrivateUvProject, ResolvedLock, RuntimeResolution, StackSources, VersionedPackage, +) +from agent_comms.installer.upgrade import Stack +from agent_comms.integration import RunnerEnvironment +from agent_comms.launch_runtime import LaunchRuntime +from agent_comms.repo import Repo +from module_layers import MaterializedClients + + +@dataclass(frozen=True, kw_only=True) +class CheckerStack(Stack, StackSources): + """Stack's command/probe/runtime lifetime, with a private unchanged-settings CI project. + + This consumes IntegrationInput. It cannot choose a new source, read the live bus, + post a verdict, or replace a production project/lock/interpreter. + """ + + source_directory: Path + inputs: IntegrationInput + uv_program: Path + + @property + def uv(self) -> Path: + return self.uv_program + + @cached_property + def core(self) -> Path: + return HeadSide.require_materialized(self.source_directory, self.inputs.source(CorePackage.repository)) + + @cached_property + def local_names(self) -> frozenset[str]: + source = self.inputs.source(CorePackage.repository) + return frozenset({CorePackage.distribution, *(name for side in Side.sides() + for name in self.sources(side.require_materialized(self.source_directory, source) / "stack"))}) + + def require_uv(self) -> None: + declared = DeclaredUv.of(self.core / "pyproject.toml", self.uv) + if not declared.accepts(self.commands.run(self.uv, "--version")): + raise ValueError("The bootstrap checker must use Core's declared uv") + + @cached_property + def clients(self) -> MaterializedClients: + return MaterializedClients(self.source_directory, self.inputs, self.uv, self.python, self.commands) + + def requirements(self, side: type[Side]) -> tuple[Requirement, ...]: + core = side.require_materialized(self.source_directory, self.inputs.source(CorePackage.repository)) + # Identity validation and runtime selection borrow the SAME backend metadata objects. + projects = self.clients.projects(side) + return self.active_requirements(tomllib.loads((core / "stack/pyproject.toml").read_text()), + tuple(projects.values()), self.clients.environment) + + def require_runtime_compatibility(self) -> None: + # Local package presence differs legitimately; only the real external contract agrees. + before, after = (frozenset(str(requirement) for requirement in self.requirements(side) + if canonicalize_name(requirement.name) not in self.local_names) + for side in (BaseSide, HeadSide)) + if before != after: + raise ValueError("External runtime dependency migration requires an explicit compatible environment") + + @cached_property + def bootstrap_lock(self) -> ResolvedLock: + return ResolvedLock.read(self.core / "uv.lock") + + @cached_property + def runtime_locks(self) -> dict[type[Side], ResolvedLock]: + source = self.inputs.source(CorePackage.repository) + return {side: ResolvedLock.read( + side.require_materialized(self.source_directory, source) / "stack/uv.lock") for side in Side.sides()} + + @cached_property + def shared_names(self) -> frozenset[str]: + runtime_names = {name for lock in self.runtime_locks.values() for name in lock.external(self.local_names)} + return frozenset(runtime_names & self.bootstrap_lock.external(self.local_names).keys()) + + @cached_property + def retained_packages(self) -> dict[str, VersionedPackage]: + """Derived disjoint lock evidence, never another editable selection ledger.""" + excluded = self.local_names | self.shared_names + retained = self.bootstrap_lock.external(excluded) + for lock in self.runtime_locks.values(): + for name, package in lock.external(excluded).items(): + if name in retained: + retained[name].require_retained(package) + retained[name] = package + return retained + + def private_requirements(self) -> tuple[Requirement, ...]: + return self.active_requirements( + tomllib.loads((self.checkout / "stack/pyproject.toml").read_text()), + tuple(self.clients.projects(HeadSide).values()), self.clients.environment) + + def write_project(self) -> None: + """Declare both complete root graphs and retained identities before the one solve.""" + self.require_runtime_compatibility() + retained = self.retained_packages # Conflicting runtime-only locks refuse before prefix mutation. + original = self.core / "stack" + project = self.checkout / "stack" + project.mkdir(parents=True, exist_ok=False) + document = PrivateUvProject.read(original / "pyproject.toml") + projects = self.clients.projects(HeadSide) + document.enrich(projects[CorePackage.repository]) + materialized = {identity: ( + HeadSide.require_materialized(self.source_directory, self.inputs.source(identity)), metadata) + for identity, metadata in projects.items()} + document.materialize(original, self.core, materialized) + for package in retained.values(): + package.constrain(document.tool.uv) + (project / "pyproject.toml").write_text(tomli_w.dumps(document.document())) + shutil.copyfile(original / "uv.lock", self.lock) + + def resolve(self) -> None: + # No --upgrade: unrelated release and revision pins survive the local-path projection. + self.commands.run(self.uv, "lock", "--project", self.checkout / "stack", "--python", self.python) + + def require_resolution(self, resolution: RuntimeResolution) -> None: + locked = ResolvedLock.read(self.lock) + external = locked.external(self.local_names) + selected = {name: record.version for name, record in resolution.packages.items() + if name not in self.local_names} + wanted = locked.pins(selected, resolution.core_commit) + actual = {name: resolution.packages[name].pin() for name in selected} + if actual != wanted: + raise ValueError("The one actual checker environment differs from its private lock") + for name, package in self.retained_packages.items(): + # A universal lock can include an unselected optional dependency. Constraints + # preserve its identity without inventing a request to install that extra. + if name in external: + package.require_retained(external[name]) + for requirements in (*(self.requirements(side) for side in Side.sides()), self.private_requirements()): + for requirement in requirements: + record = resolution.packages[canonicalize_name(requirement.name)] + if not requirement.specifier.contains(record.version, prereleases=True): + raise ValueError(f"The common environment does not satisfy {requirement}") + for name in sorted(self.shared_names & selected.keys()): + preferences = [package.version for lock in (*self.runtime_locks.values(), self.bootstrap_lock) + for key, package in lock.external(self.local_names).items() if key == name] + print(f"CHECKER SHARED {name}: original={preferences} solved={selected[name]}", flush=True) + + def prepare(self) -> RuntimeResolution: + prefix = self.runtime.prefix.resolve() + runner = RunnerEnvironment.read().root.resolve() + for path in (self.source_directory.resolve(), prefix): + if not path.is_relative_to(runner): + raise ValueError("The common checker requires its isolated runner bootstrap prefix and input directory") + if not self.checkout.resolve().is_relative_to(self.source_directory.resolve()): + raise ValueError("The private checker project must be under its captured source directory") + if not (prefix / "pyvenv.cfg").is_file(): + raise ValueError("The common checker requires its isolated runner bootstrap prefix and input directory") + self.require_uv() + self.write_project() + self.resolve() + # Both graphs belong to this solve; retaining stray files cannot substitute for roots. + self.commands.run(self.uv, "sync", "--project", self.checkout / "stack", "--locked", + "--no-editable", "--python", self.python, + env={**os.environ, "UV_PROJECT_ENVIRONMENT": str(self.runtime.prefix)}) + self.commands.run(self.uv, "pip", "check", "--python", self.python) + commit = Repo.git(self.core, "rev-parse", "HEAD").strip() + resolution = RuntimeResolution.capture(self, commit) + self.require_resolution(resolution) + resolution.save(self.runtime.prefix) + digest = hashlib.sha256(self.lock.read_bytes()).hexdigest() + bootstrap_digest = hashlib.sha256((self.core / "uv.lock").read_bytes()).hexdigest() + print(f"CHECKER INTERPRETER {self.python}\nCHECKER LOCK {digest}\n" + f"CHECKER BOOTSTRAP LOCK {bootstrap_digest}\n{resolution.model_dump_json(indent=2)}", flush=True) + return resolution + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--ci-inputs", type=Path, required=True) + args = parser.parse_args() + directory = args.ci_inputs.resolve() + executable = StackSources.frontend() + inputs = IntegrationInput.at(directory) + with InstallCommands() as commands: + CheckerStack(checkout=directory / "checker", commands=commands, home=directory, + runtime=LaunchRuntime(Path(sys.executable)), source_directory=directory, + inputs=inputs, uv_program=executable).prepare() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/debt_ratchet.py b/tools/debt_ratchet.py index de0196c55b..74f55cf6b6 100644 --- a/tools/debt_ratchet.py +++ b/tools/debt_ratchet.py @@ -5,6 +5,8 @@ import argparse import ast +import builtins +import copy import atexit import dataclasses import enum @@ -21,7 +23,6 @@ import os import pickle import re -import shutil import subprocess import sys import tempfile @@ -56,10 +57,13 @@ MroDispatch, handles, ) -from module_layers import PACKAGE, StackClient +from module_layers import PACKAGE, ClientSources, MaterializedClients, StackClient +from agent_comms.installer.commands import InstallCommands +from agent_comms.launch_runtime import LaunchRuntime from stored_formats import StoredForm from typing_names import TypingImports, TypingReference from agent_comms.field_codec import FieldCodec, ViewExclusion, boundary_decoder, projected +from agent_comms.head_review import BaseSide, HeadSide, Side from agent_comms.phase_evidence import MainTree, PlanReading from agent_comms.plan_phases import PlanPhase from agent_comms.repo import Repo @@ -142,31 +146,6 @@ def memo[Value](cls, compute: Callable[[], Value], *parts: str) -> Value: -class Side(DeclaredFamily, affix="Side"): - """One of the two revisions a comparison measures; its declared name names a checker's cache.""" - - @staticmethod - @abstractmethod - def ref(base: str, head: str) -> str: - """This side's revision of the compared pair.""" - - @classmethod - def sides(cls) -> tuple[type[Side], ...]: - return cls.members_with(cls) - - -class BaseSide(Side): - @staticmethod - def ref(base: str, head: str) -> str: - return base - - -class HeadSide(Side): - @staticmethod - def ref(base: str, head: str) -> str: - return head - - class Measure(DeclaredFamily, affix="Measure"): @classmethod def owns(cls, key: MeasureCount) -> bool: @@ -182,9 +161,20 @@ def at(cls, path: str) -> FileCount: return FileCount(cls, path) @classmethod - def blocks(cls, delta: Counts) -> bool: + def grown(cls, delta: Counts) -> bool: + """Whether any of this measure's counts grew.""" return any(value > 0 for key, value in delta.items() if cls.owns(key)) + @classmethod + def blocks(cls, delta: Counts) -> bool: + return cls.grown(delta) + + @classmethod + def missing(cls, head: Counts, introduced: dict[FileCount, list[int]]) -> list[MeasureCount]: + """This measure's counts the comparison has no required evidence for; a measure that + needs none beyond its counts misses nothing.""" + return [] + @classmethod def grew(cls, key: MeasureCount, grown: int) -> str: """The line reporting one count's growth; the count words it.""" @@ -219,7 +209,7 @@ def source_paths(root: str, tree: Path) -> tuple[str, ...]: @classmethod @abstractmethod def parts(cls, repo: Path, base: str, head: str, changed: set[str], - root: str) -> tuple[Callable[[], object], ...]: + root: str, clients: ClientSources = ClientSources()) -> tuple[Callable[[], object], ...]: """Independent pieces of the comparison; compare() runs each in a separate process.""" @classmethod @@ -233,9 +223,10 @@ def align(cls, base: Counts, head: Counts) -> tuple[Counts, Counts]: return base, head @classmethod - def compare(cls, repo: Path, base: str, head: str, changed: set[str], root: str): + def compare(cls, repo: Path, base: str, head: str, changed: set[str], root: str, + clients: ClientSources = ClientSources()) -> tuple[Counts, Counts]: """Return independent before/after measurements for this declaration.""" - parts = cls.parts(repo, base, head, changed, root) + parts = cls.parts(repo, base, head, changed, root, clients) return cls.combine([part() for part in parts], repo, base, head, changed, root) @@ -335,7 +326,7 @@ def counted(cls, repo: Path, base: str, head: str, changed: set[str], @classmethod def parts(cls, repo: Path, base: str, head: str, changed: set[str], - root: str) -> tuple[Callable[[], object], ...]: + root: str, clients: ClientSources = ClientSources()) -> tuple[Callable[[], object], ...]: return (partial(cls.counted, repo, base, head, changed, root),) @classmethod @@ -390,7 +381,7 @@ def read_fresh(cls, repo: Path, ref: str, paths: list[str]) -> dict[str, Reading @classmethod def parts(cls, repo: Path, base: str, head: str, changed: set[str], - root: str) -> tuple[Callable[[], object], ...]: + root: str, clients: ClientSources = ClientSources()) -> tuple[Callable[[], object], ...]: before, after = cls.inventory(repo, base, root), cls.inventory(repo, head, root) shared = sorted((before & after) - ChangedPaths.of(repo, base, head).touched) shards = os.process_cpu_count() or 1 @@ -483,8 +474,8 @@ def run_by(cls) -> type[Measure]: @classmethod def parts(cls, repo: Path, base: str, head: str, changed: set[str], - root: str) -> tuple[Callable[[], object], ...]: - return SyntaxPasses.parts(repo, base, head, changed, root) + root: str, clients: ClientSources = ClientSources()) -> tuple[Callable[[], object], ...]: + return SyntaxPasses.parts(repo, base, head, changed, root, clients) @classmethod def combine(cls, results: list, repo: Path, base: str, head: str, changed: set[str], @@ -635,7 +626,7 @@ def side(cls, side: type[Side], repo: Path, ref: str, @classmethod def parts(cls, repo: Path, base: str, head: str, changed: set[str], - root: str) -> tuple[Callable[[], object], ...]: + root: str, clients: ClientSources = ClientSources()) -> tuple[Callable[[], object], ...]: shards = os.process_cpu_count() or 1 return tuple(partial(cls.side, side, repo, ref, paths[shard::shards]) for side in Side.sides() for ref in (side.ref(base, head),) @@ -789,9 +780,10 @@ def inputs(cls, repo: Path, ref: str, head: str, root: str, selected: list[str]) @classmethod def findings(cls, repo: Path, ref: str, head: str, root: str, selected: list[str], side: type[Side]) -> tuple[type[Side], Counter[str]]: - return side, ContentCache.memo( - partial(cls.checked, repo, ref, head, root, selected, side), - cls.__name__, root, cls.inputs(repo, ref, head, root, selected), *selected) + """Each comparison runs the checker. A checker that reads installed packages (mypy) + depends on more than the committed tree, so its count is never stored by content; + the checker's own cache tracks those inputs.""" + return side, cls.checked(repo, ref, head, root, selected, side) @classmethod def checked(cls, repo: Path, ref: str, head: str, root: str, selected: list[str], @@ -818,7 +810,7 @@ def checked(cls, repo: Path, ref: str, head: str, root: str, selected: list[str] @classmethod def parts(cls, repo: Path, base: str, head: str, changed: set[str], - root: str) -> tuple[Callable[[], object], ...]: + root: str, clients: ClientSources = ClientSources()) -> tuple[Callable[[], object], ...]: return tuple(partial(cls.findings, repo, ref, head, root, batch, side) for side in Side.sides() for ref in (side.ref(base, head),) for batch in cls.batches(sorted(changed & python_paths(repo, ref, root))) @@ -891,6 +883,15 @@ class LintFindings(Advisory, CheckedRevision): def command(tree: Path, selected: list[str], side: type[Side]) -> list[str]: return ["ruff", "check", "--no-cache", "--output-format", "concise", *selected] + @classmethod + def findings(cls, repo: Path, ref: str, head: str, root: str, selected: list[str], + side: type[Side]) -> tuple[type[Side], Counter[str]]: + """Ruff reads only the revision's tree and the head's configuration, so its count is + stored under them.""" + return side, ContentCache.memo( + partial(cls.checked, repo, ref, head, root, selected, side), + cls.__name__, root, cls.inputs(repo, ref, head, root, selected), *selected) + class TypeErrors(CheckedRevision): """Mypy errors per changed file. Mypy checks the changed files and every module they import, @@ -921,53 +922,38 @@ def environment(root: str, tree: Path) -> dict[str, str]: class ClientTypeErrors(SideCounts): - """Mypy errors in each stack client file that imports agent_comms, checked against each - revision's package. Both sides read the same client commit, so the client's own errors cancel - and an error the head adds is a name or signature the client uses that the change took away: - #3110 deleted TurnState.settles as unused while Toad's conversation_turn.py:262 called it, and - only the running app found it.""" - - finding = TypeErrors.finding + """Mypy growth across coherent Core/client source tuples in one checker environment. - @staticmethod - @cache - def clients(repo: Path, head: str) -> tuple[StackClient, ...]: - """The head's stack clients, cloned once per run and read by both sides; a revision - declaring no stack has none.""" - if not git(repo, "ls-tree", "--name-only", head, "--", "stack/pyproject.toml"): - return () - into = Path(tempfile.mkdtemp(prefix="debt-ratchet-clients-")) - atexit.register(shutil.rmtree, into, True) - (into / "stack").mkdir() - (into / "stack/pyproject.toml").write_bytes(git(repo, "show", f"{head}:stack/pyproject.toml")) - return StackClient.fetched(into / "stack", into) + Matched migrations intentionally compare different client code. Selection is the union + of both real cuts, with absent files absent and each cut's own configuration preserved. + Ordinary local comparisons retain the existing one-main client acquisition. + """ - @staticmethod - def selected(client: StackClient) -> list[str]: - return sorted({entry.path for entry in client.imports - if entry.module.partition(".")[0] == PACKAGE}) + finding = TypeErrors.finding @classmethod def findings(cls, repo: Path, ref: str, client: StackClient, side: type[Side]) -> tuple[type[Side], Counter[str]]: - commit = git(client.checkout, "rev-parse", "HEAD").decode().strip() - tree = git(repo, "rev-parse", f"{ref}:src").decode().strip() - return side, ContentCache.memo(partial(cls.checked, repo, ref, client, side), - cls.__name__, client.name, commit, tree) + """Mypy resolves the client's other imports from the installed environment, which no + content key names, so each comparison checks afresh under mypy's own cache.""" + return side, cls.checked(repo, ref, client, side) @classmethod def checked(cls, repo: Path, ref: str, client: StackClient, side: type[Side]) -> Counter[str]: - selected = cls.selected(client) + selected = client.files_importing(PACKAGE) cache = ContentCache.directory.with_name("debt-ratchet-mypy-clients") / client.name / side.declared_name with tempfile.TemporaryDirectory(prefix="debt-ratchet-client-") as scratch: subprocess.run(["tar", "-x", "-C", scratch], input=git(repo, "archive", ref, "src"), check=True, timeout=60) - # The client checks under its own configuration, from its checkout. + # Native config plugins import Python too: both readers need the same cut, + # not a baseline static path alongside candidate runtime imports. + imports = (str(Path(scratch, "src")), *client.import_paths) + runtime = LaunchRuntime.from_interpreter(sys.executable) result = SpawnedProcess.run( - (sys.executable, "-m", *TypeErrors.mypy( + (str(runtime.interpreter), "-m", *TypeErrors.mypy( cache, "--follow-imports=silent", "--ignore-missing-imports", *selected)), - cwd=client.checkout, timeout=600, - env={**os.environ, "MYPYPATH": os.pathsep.join( - (str(Path(scratch, "src")), str(client.checkout / "src")))}, + cwd=client.checkout, timeout=600, runtime=runtime, + env={**os.environ, "MYPYPATH": os.pathsep.join(imports), + "PYTHONPATH": os.pathsep.join(imports)}, ) output = cls.checker_output(result, f"{client.name} at {ref}") return Counter(f"{client.name}/{match[1]}" for match in cls.finding.finditer(output) @@ -975,14 +961,14 @@ def checked(cls, repo: Path, ref: str, client: StackClient, side: type[Side]) -> @classmethod def parts(cls, repo: Path, base: str, head: str, changed: set[str], - root: str) -> tuple[Callable[[], object], ...]: - # Only a change to the package the clients import can change what they see. - if not any(path.startswith(f"src/{PACKAGE}/") for path in changed): + root: str, clients: ClientSources = ClientSources()) -> tuple[Callable[[], object], ...]: + # A matched resource also owns client-only changes, even with an unchanged Core patch. + if not any(path.startswith(f"src/{PACKAGE}/") for path in changed) and not clients.requires_clients(): return () return tuple(partial(cls.findings, repo, side.ref(base, head), client, side) - for client in cls.clients(repo, head) - if cls.selected(client) - for side in Side.sides()) + for side in Side.sides() + for client in clients.at(repo, base, head, side) + if client.files_importing(PACKAGE)) @classmethod def measured(cls, found: dict[type[Side], Counter[str]], changed: set[str]) -> list[str]: @@ -1042,11 +1028,30 @@ def departed(cls, repo: Path, base: str, head: str, path: str) -> ChangedSource: class ScopeNames(ast.NodeVisitor): - """Each node's enclosing class and function names, dotted, through the AST's own dispatch.""" + """Each node's enclosing class and function names, dotted, through the AST's own dispatch, + with the names each node binds where it stands and the nodes that declare them: a class or + function, an import alias, an annotated name. A star import binds ``*``.""" def __init__(self) -> None: self.found: dict[ast.AST, str] = {} self.scope = "" + self.binds: dict[ast.AST, tuple[str, ...]] = {} + self.declares: set[ast.AST] = set() + + @classmethod + def of(cls, tree: ast.AST) -> ScopeNames: + names = cls() + names.visit(tree) + return names + + def bindings(self, scope: str, before: tuple[ast.AST, ...] = ()) -> dict[str, list[ast.AST]]: + """Bindings in one scope, excluding declarations whose evaluation has not assigned them.""" + found: dict[str, list[ast.AST]] = {} + for node, names in self.binds.items(): + if node not in before and self.found[node] == scope: + for name in names: + found.setdefault(name, []).append(node) + return found def generic_visit(self, node: ast.AST) -> None: self.found[node] = self.scope @@ -1054,10 +1059,69 @@ def generic_visit(self, node: ast.AST) -> None: def visit_named(self, node: ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef) -> None: self.found[node] = outer = self.scope + self.binds[node] = (node.name,) + self.declares.add(node) self.scope = f"{outer}.{node.name}" super().generic_visit(node) self.scope = outer + def visit_alias(self, node: ast.alias) -> None: + self.binds[node] = (node.asname or node.name.split(".")[0],) + self.declares.add(node) + self.generic_visit(node) + + def visit_arg(self, node: ast.arg) -> None: + self.binds[node] = (node.arg,) + self.generic_visit(node) + + def visit_Name(self, node: ast.Name) -> None: + match node.ctx: + case ast.Store() | ast.Del(): + self.binds[node] = (node.id,) + self.generic_visit(node) + + def visit_AnnAssign(self, node: ast.AnnAssign) -> None: + match node.target: + case ast.Name(): + self.declares.add(node.target) + self.generic_visit(node) + + def visit_ExceptHandler(self, node: ast.ExceptHandler) -> None: + self.binds[node] = (node.name,) if node.name else () + self.generic_visit(node) + + def visit_MatchAs(self, node: ast.MatchAs) -> None: + self.binds[node] = (node.name,) if node.name else () + self.generic_visit(node) + + def visit_MatchStar(self, node: ast.MatchStar) -> None: + self.binds[node] = (node.name,) if node.name else () + self.generic_visit(node) + + def visit_MatchMapping(self, node: ast.MatchMapping) -> None: + self.binds[node] = (node.rest,) if node.rest else () + self.generic_visit(node) + + def visit_TypeVar(self, node: ast.TypeVar) -> None: + self.binds[node] = (node.name,) + self.generic_visit(node) + + def visit_ParamSpec(self, node: ast.ParamSpec) -> None: + self.binds[node] = (node.name,) + self.generic_visit(node) + + def visit_TypeVarTuple(self, node: ast.TypeVarTuple) -> None: + self.binds[node] = (node.name,) + self.generic_visit(node) + + def visit_Global(self, node: ast.Global) -> None: + self.binds[node] = tuple(node.names) + self.generic_visit(node) + + def visit_Nonlocal(self, node: ast.Nonlocal) -> None: + self.binds[node] = tuple(node.names) + self.generic_visit(node) + def visit_FunctionDef(self, node: ast.FunctionDef) -> None: self.visit_named(node) @@ -1068,6 +1132,154 @@ def visit_ClassDef(self, node: ast.ClassDef) -> None: self.visit_named(node) +class NominalProof: + """What an expression's resolved owners prove it is; only two proved identities with + the same owners and structure are the same.""" + + def same(self, other: NominalProof) -> bool: + return False + + def matches(self, key: str) -> bool: + return False + + +class UnprovedIdentity(NominalProof): + """A reference the binding facts could not resolve: equal to nothing, itself included.""" + + +@dataclass(frozen=True) +class ProvedIdentity(NominalProof): + key: str + + def same(self, other: NominalProof) -> bool: + return other.matches(self.key) + + def matches(self, key: str) -> bool: + return key == self.key + + +@dataclass(frozen=True) +class DeclaredClass: + """One top-level class of one changed module at one revision, read for its fields.""" + + path: str + tree: ast.Module + node: ast.ClassDef + + @classmethod + def top(cls, path: str, tree: ast.Module) -> dict[str, DeclaredClass]: + """The module's classes whose module-level name is still that one declaration.""" + return {name: cls(path, tree, statement) + for name, (_, statement) in DecoratorBinding.declarations(tree, "", tree.body).items() + if isinstance(statement, ast.ClassDef)} + + @property + def scope(self) -> str: + return f".{self.node.name}" + + @cached_property + def fields(self) -> dict[str, ast.AnnAssign]: + """The class body's annotated names bound by that annotation alone.""" + return {name: statement for name, (_, statement) in DecoratorBinding.declarations( + self.tree, self.scope, self.node.body).items() + if isinstance(statement, ast.AnnAssign)} + + def binds(self, name: str) -> bool: + """Whether the class scope still binds ``name`` in any form.""" + return name in DecoratorBinding.bindings(self.tree, self.scope) + + @cached_property + def annotated(self) -> DecoratorBinding: + return DecoratorBinding.named("typing", "Annotated", self.tree, self.path, self.scope) + + def compared(self, field: str) -> list[ast.expr]: + """The field's name, its type without typing.Annotated metadata, and its default.""" + statement = self.fields[field] + match statement.annotation: + case ast.Subscript(value=value, slice=ast.Tuple(elts=[declared, *_])) if self.annotated.marks(value): + annotation: ast.expr = declared + case annotation: + pass + return [statement.target, annotation, *([statement.value] if statement.value else [])] + + def declared(self, field: str) -> NominalProof: + """The field's type and default, each reference resolved to its owner in this class's + scope; the name is the field's own and needs no owner.""" + _, *written = self.compared(field) + return DecoratorBinding.identity(self.tree, self.path, self.scope, ast.Tuple(elts=written)) + + def holding(self, held: DeclaredClass, base: DeclaredClass) -> tuple[str, ...]: + """The fields this class newly declares whose type is the module-level binding of + ``held``.""" + binding = DecoratorBinding.named( + ".".join(DecoratorBinding.module_of(held.path)), held.node.name, self.tree, self.path, + self.scope) + return tuple(name for name, statement in self.fields.items() + if name not in base.fields and binding.marks(statement.annotation)) + + +class FieldExtraction: + """Fields a class gave up to the one new class it now holds in a typed field. + + The held field's annotation is the provenance edge: an old class present at both + revisions newly declares one field typed as one class absent from the base, and no other + old class holds that class. Each field the old class lost, declared once in the new class + under the same name with the same type (typing.Annotated metadata aside) and the same + default, carries its occurrences there. Only occurrences inside that one proved field + declaration are credited, each spending one occurrence that left the old class, so a + retained field, an extra field, a method or a second copy is still written. + """ + + @classmethod + def credit(cls, measure: type[CleanChangedLines], sources: list[ChangedSource], + left: list[ChangedSource], departed: Counter[tuple[str, str]]) -> Counter[ast.AST]: + """Each proved destination occurrence's credit; ``departed`` spends what it pays.""" + existing = {name for source in [*sources, *left] for name in DeclaredClass.top( + source.path, source.base_tree)} + new = [held for source in sources + for name, held in DeclaredClass.top(source.path, source.tree).items() + if name not in existing] + edges: list[tuple[DeclaredClass, DeclaredClass, DeclaredClass]] = [] + for source in sources: + before = DeclaredClass.top(source.path, source.base_tree) + for name, owner in DeclaredClass.top(source.path, source.tree).items(): + if name in before: + for held in new: + edges.extend((before[name], owner, held) for _ in owner.holding(held, before[name])) + owners = Counter((base.path, base.node.name) for base, _, _ in edges) + holders = Counter((held.path, held.node.name) for _, _, held in edges) + credited: Counter[ast.AST] = Counter() + for base, owner, held in edges: + if owners[base.path, base.node.name] == holders[held.path, held.node.name] == 1: + for field in sorted(base.fields): + if (not owner.binds(field) and field in held.fields + and base.declared(field).same(held.declared(field))): + cls.transfer(measure, base, held, field, departed, credited) + return credited + + @staticmethod + def transfer(measure: type[CleanChangedLines], base: DeclaredClass, held: DeclaredClass, + field: str, departed: Counter[tuple[str, str]], credited: Counter[ast.AST]) -> None: + """One field's occurrences, paid one for one by those that left the old declaration. + The declarations are proved the same, so their parts pair node for node in walk order; + parts whose raw shapes differ pair nothing.""" + scope = f".{base.node.name}" + counted = {declared: ({id(node) for node in measure.sites(declared.tree, declared.path)}, + measure.scopes(declared.tree)) for declared in (base, held)} + for gave, took in zip(base.compared(field), held.compared(field), strict=True): + left, arrived = list(ast.walk(gave)), list(ast.walk(took)) + if [type(node) for node in left] != [type(node) for node in arrived]: + continue + for old, new in zip(left, arrived, strict=True): + if ((carried := min(measure.occurrences(old), measure.occurrences(new))) + and id(old) in counted[base][0] and id(new) in counted[held][0]): + identity = counted[base][1][old], ast.unparse(old) + if identity[0] == scope: + paid = min(carried, departed[identity]) + departed[identity] -= paid + credited[new] += paid + + class CleanChangedLines(OccurrenceMeasure): """No occurrence may be written, even when another occurrence is deleted. @@ -1078,9 +1290,10 @@ class CleanChangedLines(OccurrenceMeasure): class moved between files, is the same occurrence moved, not a new one written. """ - @staticmethod + @classmethod def introduced( - repo: Path, base: str, head: str, paths: tuple[str, ...], departed_from: tuple[str, ...], + cls: type[typing.Self], repo: Path, base: str, head: str, paths: tuple[str, ...], + departed_from: tuple[str, ...], ) -> dict[FileCount, list[int]]: """Each strict measure's sites on each path's changed lines. Occurrences that left the changed modules in ``departed_from`` (those present at the base) pay for the same @@ -1088,14 +1301,15 @@ def introduced( sources = [ChangedSource.read(repo, base, head, path) for path in paths] left = [ChangedSource.departed(repo, base, head, path) for path in departed_from] found: dict[FileCount, list[int]] = {} - for measure in Measure.members_with(CleanChangedLines): + for measure in cls.members_with(cls): departed: Counter[tuple[str, str]] = Counter() for source in left: gone = measure.identities(source.base_tree, source.path) gone.subtract(measure.identities(source.tree, source.path)) departed.update(+gone) + extracted = FieldExtraction.credit(measure, sources, left, departed) for source in sources: - found[measure.at(source.path)] = measure.changed_sites(source, departed) + found[measure.at(source.path)] = measure.changed_sites(source, departed, extracted) return found @staticmethod @@ -1116,9 +1330,26 @@ def identities(cls, tree: ast.AST, filename: str) -> Counter[tuple[str, str]]: return found @classmethod - def changed_sites(cls, source: ChangedSource, departed: Counter[tuple[str, str]]) -> list[int]: + def blocks(cls, delta: Counts) -> bool: + """Never on raw growth: Comparison.failed fails every introduced occurrence on its own, + and a moved or extracted occurrence grows its destination's raw count while writing + nothing. A new occurrence still fails through ``introduced`` even when an unrelated + deletion offsets the totals.""" + return False + + @classmethod + def missing(cls, head: Counts, introduced: dict[FileCount, list[int]]) -> list[MeasureCount]: + """A file with occurrences at the head but no introduced-site report is unproved: only + an explicit report, even an empty one, says no occurrence there is new.""" + return [key for key, value in head.items() + if cls.owns(key) and value > 0 and key not in introduced] + + @classmethod + def changed_sites(cls, source: ChangedSource, departed: Counter[tuple[str, str]], + extracted: Counter[ast.AST]) -> list[int]: """The first changed line of each occurrence written there that the base lacks and no - changed module gave up; a moved occurrence spends the departure it came from.""" + changed module gave up; a moved occurrence spends the departure it came from, and an + occurrence in a proved field extraction spends its credit.""" scopes = cls.scopes(source.tree) unwritten = cls.identities(source.tree, source.path) unwritten.subtract(cls.identities(source.base_tree, source.path)) @@ -1131,7 +1362,9 @@ def changed_sites(cls, source: ChangedSource, departed: Counter[tuple[str, str]] unwritten[identity] -= arrived moved = min(arrived, departed[identity]) departed[identity] -= moved - sites.extend([min(touched)] * (arrived - moved)) + carried = min(arrived - moved, extracted[node]) + extracted[node] -= carried + sites.extend([min(touched)] * (arrived - moved - carried)) return sites @@ -1232,42 +1465,139 @@ def lines(cls, source: bytes, filename: str) -> list[int]: return sorted([*super().lines(source, filename), *cls.prose_lines(module)]) @classmethod - def changed_sites(cls, source: ChangedSource, departed: Counter[tuple[str, str]]) -> list[int]: + def changed_sites(cls, source: ChangedSource, departed: Counter[tuple[str, str]], + extracted: Counter[ast.AST]) -> list[int]: prose = cls.prose_lines(source) & source.lines - return [*super().changed_sites(source, departed), *prose] + return [*super().changed_sites(source, departed, extracted), *prose] @dataclass(frozen=True) class DecoratorBinding: - """The spellings one module binds to a declared decorator, read from its imports alone. + """The spellings one module binds to a declared name (a decorator, a field factory, a + class), read from its module-level imports and declarations. - A decorator of the same name declared or imported from anywhere else marks nothing. + A name of the same spelling declared or imported from anywhere else marks nothing, and + neither does a spelling the module binds twice, conditionally or by star import. """ names: frozenset[str] @classmethod def of(cls, decorator: typing.Callable[..., object], tree: ast.AST, - filename: str) -> DecoratorBinding: - owner, name = decorator.__module__, decorator.__name__ + filename: str, scope: str = "", before: tuple[ast.AST, ...] = ()) -> DecoratorBinding: + return cls.named(decorator.__module__, decorator.__name__, tree, filename, scope, before) + + @classmethod + def sites(cls, decorator: typing.Callable[..., object], tree: ast.AST, + filename: str) -> Iterator[ast.AST]: + """Walk outside functions whose enclosing scope proves the declared decorator.""" + scopes = ScopeNames.of(tree) + pending = [tree] + while pending: + node = pending.pop() + match node: + case ast.FunctionDef(decorator_list=[_, *_] as decorators) | ast.AsyncFunctionDef( + decorator_list=[_, *_] as decorators): + marker = cls.of(decorator, tree, filename, scopes.found[node], (node,)) + if any(map(marker.marks, decorators)): + continue + yield node + pending.extend(ast.iter_child_nodes(node)) + + @staticmethod + def module_of(filename: str) -> tuple[str, ...]: + """The dotted module a source path imports as; ``src`` is the import root.""" parts = Path(filename).with_suffix("").parts - module = parts[1:] if parts[:1] == ("src",) else parts + return parts[1:] if parts[:1] == ("src",) else parts + + @classmethod + def named(cls, owner: str, name: str, tree: ast.AST, filename: str, + scope: str = "", before: tuple[ast.AST, ...] = ()) -> DecoratorBinding: + """The spellings binding ``owner.name`` in the module at ``filename``, as read from + ``scope``: a spelling whose root that scope rebinds names something else there. + Decorators supply their own not-yet-bound definition; every other binding remains.""" + module = cls.module_of(filename) package = module[:-1] - names = {name} if ".".join(module) == owner else set() - for node in ast.walk(tree): - match node: - case ast.Import(names=aliases): - names.update(f"{alias.asname or alias.name}.{name}" - for alias in aliases if alias.name == owner) - case ast.ImportFrom(module=source, names=aliases, level=level): + names: set[str] = set() + body = typing.cast(ast.Module, tree).body + for bound, (declaration, statement) in cls.declarations(tree, "", body, before).items(): + match statement, declaration: + case ast.ClassDef() | ast.FunctionDef() | ast.AsyncFunctionDef(), _ if ( + ".".join(module) == owner and bound == name): + names.add(name) + case ast.Import(), ast.alias(name=imported, asname=alias) if imported == owner: + names.add(f"{alias or imported}.{name}") + case ast.ImportFrom(module=source, level=level), ast.alias(name=imported): base = package[:len(package) - level + 1] if level else () resolved = ".".join((*base, source) if source else base) - for alias in aliases: - if f"{resolved}.{alias.name}" == owner: - names.add(f"{alias.asname or alias.name}.{name}") - if resolved == owner and alias.name == name: - names.add(alias.asname or name) - return cls(frozenset(names)) + if f"{resolved}.{imported}" == owner: + names.add(f"{bound}.{name}") + if resolved == owner and imported == name: + names.add(bound) + hidden = cls.bindings(tree, scope, before).keys() if scope else frozenset() + return cls(frozenset(spelling for spelling in names if spelling.split(".")[0] not in hidden)) + + @staticmethod + def bindings(tree: ast.AST, scope: str, before: tuple[ast.AST, ...] = ()) -> dict[str, list[ast.AST]]: + """Every binding of each name in one lexical module or class scope, as ScopeNames + decodes it; bindings inside nested functions and classes are theirs, not this scope's.""" + return ScopeNames.of(tree).bindings(scope, before) + + @staticmethod + def declarations(tree: ast.AST, scope: str, + body: list[ast.stmt], before: tuple[ast.AST, ...] = ()) -> dict[str, tuple[ast.AST, ast.stmt]]: + """Each name the scope binds exactly once, by a declaring node that is one of its body's + statements or such a statement's direct child, with that statement. A star import + leaves no name proved.""" + names = ScopeNames.of(tree) + bindings = names.bindings(scope, before) + if "*" in bindings: + return {} + owners = {child: statement for statement in body + for child in (statement, *ast.iter_child_nodes(statement))} + return {name: (node, owners[node]) for name, (node, *others) in bindings.items() + if not others and node in names.declares and node in owners} + + @classmethod + def identity(cls, tree: ast.Module, filename: str, scope: str, expression: ast.expr) -> NominalProof: + """The expression with every name and attribute reference replaced by the module and + name that own it, as evaluated in ``scope``: an imported name is its source module's + export, a declared class its own module's, a builtin one the module never binds. A name + the scope itself binds, or any reference these declarations cannot resolve, proves + nothing.""" + module = cls.module_of(filename) + package = module[:-1] + declared = cls.declarations(tree, "", tree.body) + bound = cls.bindings(tree, "") + enclosing = cls.bindings(tree, scope) + unresolved: list[str] = [] + + def owner(name: str) -> str: + match declared.get(name) if name not in enclosing else (): + case ast.alias(name=imported, asname=alias), ast.Import(): + return imported if alias else imported.split(".")[0] + case ast.alias(name=imported), ast.ImportFrom(module=source, level=level): + base = package[:len(package) - level + 1] if level else () + return ".".join((*base, *(source.split(".") if source else ()), imported)) + case _, ast.ClassDef(): + return ".".join((*module, name)) + if name not in bound and name not in enclosing and name in vars(builtins): + return f"builtins.{name}" + unresolved.append(name) + return name + + class Owned(ast.NodeTransformer): + def visit_Name(self, node: ast.Name) -> ast.Name: + return ast.Name(id=owner(node.id)) + + def visit_Attribute(self, node: ast.Attribute) -> ast.AST: + match ast.unparse(node).split("."): + case [root, *attributes] if root.isidentifier() and all(map(str.isidentifier, attributes)): + return ast.Name(id=".".join((owner(root), *attributes))) + return self.generic_visit(node) + + owned = Owned().visit(copy.deepcopy(expression)) + return UnprovedIdentity() if unresolved else ProvedIdentity(ast.dump(owned)) def marks(self, decorator: ast.expr) -> bool: return ast.unparse(decorator) in self.names @@ -1325,16 +1655,7 @@ class NoneAbsence(CleanChangedLines, PerFileOccurrenceMeasure): @classmethod def sites(cls, tree: ast.AST, filename: str) -> Iterator[ast.AST]: - marker = DecoratorBinding.of(boundary_decoder, tree, filename) - pending = [tree] - while pending: - node = pending.pop() - match node: - case ast.FunctionDef(decorator_list=decorators) | ast.AsyncFunctionDef( - decorator_list=decorators) if any(map(marker.marks, decorators)): - continue - yield node - pending.extend(ast.iter_child_nodes(node)) + return DecoratorBinding.sites(boundary_decoder, tree, filename) @staticmethod def occurrences(node: ast.AST) -> int: @@ -2324,6 +2645,10 @@ class ForeignAbsenceProbe(PerFileOccurrenceMeasure): occurrences = AuditForeignAbsenceProbe.count + @classmethod + def sites(cls, tree: ast.AST, filename: str) -> Iterator[ast.AST]: + return DecoratorBinding.sites(boundary_decoder, tree, filename) + class CodecSubclass(PerFileOccurrenceMeasure): """TIME-9 census measure; the sealed FieldCodec guard also resolves aliases.""" @@ -2678,7 +3003,7 @@ class DonePhaseBroken(Measure): @classmethod def parts(cls, repo: Path, base: str, head: str, changed: set[str], - root: str) -> tuple[Callable[[], object], ...]: + root: str, clients: ClientSources = ClientSources()) -> tuple[Callable[[], object], ...]: declared = Path(inspect.getfile(PlanPhase)).resolve() if root != f"{declared.relative_to(REPOSITORY).parts[0]}/": return () @@ -2814,14 +3139,25 @@ def found_report(self) -> dict[str, list[Site]]: def delta_report(self) -> dict[str, int]: return self.keyed(self.delta) + @property + def missing(self) -> list[MeasureCount]: + """Counts whose measure's required evidence the comparison lacks.""" + return [key for measure in Measure.members_with(Measure) + for key in measure.missing(self.head, self.introduced)] + + @projected(view="report", name="missing") + def missing_report(self) -> list[str]: + return sorted(key.text for key in self.missing) + @property def failed(self) -> bool: return (any(measure.blocks(self.delta) for measure in Measure.members_with(Measure)) - or any(self.introduced.values())) + or any(self.introduced.values()) or bool(self.missing)) def compare(repo: Path, base: str, head: str, root: str, workers: int, - measured: tuple[type[Measure], ...] = Measure.members_with(Measure)) -> Comparison: + measured: tuple[type[Measure], ...] = Measure.members_with(Measure), + clients: ClientSources = ClientSources()) -> Comparison: source_root = Path(root) if source_root.is_absolute() or ".." in source_root.parts or not source_root.parts: raise ValueError("--root must be a repository-relative source directory") @@ -2833,9 +3169,9 @@ def compare(repo: Path, base: str, head: str, root: str, workers: int, before: Counts = {} after: Counts = {} introduced: dict[FileCount, list[int]] = {} - # Every measure reads production Python paths only, so with none touched each would - # report no change. - if not touched & (production_paths(repo, base) | production_paths(repo, head)): + # Ordinary comparisons need a touched production path. Matched inputs can change + # an external client while leaving the local Core patch unchanged. + if not touched & (production_paths(repo, base) | production_paths(repo, head)) and not clients.requires_clients(): return Comparison(root, base, head, (), before, after, introduced, {}) # A pass of a shared runner is measured once, by that runner. @@ -2843,7 +3179,7 @@ def compare(repo: Path, base: str, head: str, root: str, workers: int, # Forked workers start from this already-imported module instead of importing it again. with ForkedProcessPool(workers) as pool: parts = {measure: [pool.submit(part) - for part in measure.parts(repo, base, head, paths, root)] + for part in measure.parts(repo, base, head, paths, root, clients)] for measure in measures} sites = [pool.submit( CleanChangedLines.introduced, repo, base, head, @@ -2873,7 +3209,7 @@ def compare(repo: Path, base: str, head: str, root: str, workers: int, changed.paired(after), introduced, {}) # Only a grown measure is read for its sites, so a clean change pays for none. grown = [measure for measure in LocatedSource.members_with(LocatedSource) - if measure in measured and measure.blocks(comparison.delta)] + if measure in measured and measure.grown(comparison.delta)] if not grown: return comparison sources = GitMeasure.sources(repo, head, paths & python_paths(repo, head, root)) @@ -2890,8 +3226,13 @@ def main() -> int: parser.add_argument("--head", required=True) parser.add_argument("--workers", type=int, default=os.process_cpu_count() or 1, help="Pool width; a caller running ratchets in parallel narrows it") + parser.add_argument("--ci-inputs", type=Path, + help="the integration entrypoint's exact materialized Core/client cuts") args = parser.parse_args() - result = compare(Path.cwd(), args.base, args.head, args.root, args.workers) + with InstallCommands() as commands: + clients = MaterializedClients.read(args.ci_inputs, uv=MaterializedClients.frontend(), + python=Path(sys.executable), commands=commands) if args.ci_inputs else ClientSources() + result = compare(Path.cwd(), args.base, args.head, args.root, args.workers, clients=clients) # Flushed first: stdout is block-buffered into a pipe, and the growth lines must end the log. print(json.dumps(FieldCodec.project(result, "report"), indent=2), flush=True) # The failing step's last lines become check annotations (.github/run-step.sh merges @@ -2905,6 +3246,8 @@ def main() -> int: for identity, lines in sorted(result.introduced_report.items()): if lines: print(f"introduced {identity} at lines {lines}", file=sys.stderr) + for identity in result.missing_report: + print(f"missing introduced-site report for {identity}", file=sys.stderr) return int(result.failed) diff --git a/tools/module_layers.py b/tools/module_layers.py index d64dbc316b..bfb5cd98ba 100644 --- a/tools/module_layers.py +++ b/tools/module_layers.py @@ -25,7 +25,9 @@ import argparse import ast +import atexit import inspect +import shutil import subprocess import sys import tempfile @@ -33,10 +35,18 @@ import typing from abc import ABC, abstractmethod from collections.abc import Iterable, Sequence -from dataclasses import dataclass -from functools import cached_property +from dataclasses import dataclass, replace +from functools import cache, cached_property from pathlib import Path +from mypy.config_parser import toml_config_types + +from agent_comms.field_codec import FieldCodec, boundary_decoder +from agent_comms.head_review import BaseSide, HeadSide, IntegrationInput, Side +from agent_comms.installer.commands import InstallCommands +from agent_comms.installer.runtime_resolution import CorePackage, ProjectMetadata, StackSources +from agent_comms.repository_name import RepositoryName +from agent_comms.repo import Repo from agent_comms.installer.runtime_wire import ClientImport, CrossesRuntimes from typing_names import TypingImports, TypingReference @@ -298,27 +308,57 @@ def visit_ImportFrom(self, node: ast.ImportFrom) -> None: @dataclass(frozen=True) -class StackClient: +class StackClient(StackSources): """A package the stack installs from a git source, read from a checkout of it.""" name: str checkout: Path + compared: tuple[Path, ...] = () + dependencies: tuple[Path, ...] = () - @staticmethod - def sources(stack: Path) -> dict[str, tuple[str, str]]: - """Each git source stack/pyproject.toml declares: name to (url, branch).""" - match tomllib.loads((stack / "pyproject.toml").read_text()): - case {"tool": {"uv": {"sources": dict() as sources}}}: - found = {} - for name, source in sources.items(): - match source: - case {"git": str(url), "branch": str(branch)}: - found[name] = (url, branch) - return found - raise ValueError(f"{stack / 'pyproject.toml'} declares no [tool.uv.sources] table") + @cached_property + @boundary_decoder + def source_paths(self) -> tuple[str, ...]: + """The acquired project's declaration owns development import bases. + + Use mypy's native TOML conversion (array or delimited string), not another + path dialect. An explicit empty override stays empty. Projects without an + override retain the existing installed-package/src convention. + """ + project = tomllib.loads((self.checkout / "pyproject.toml").read_text()) + declared = project.get("tool", {}).get("mypy", {}).get("mypy_path", ["src", "."]) + roots = FieldCodec.decode(list[str], toml_config_types["mypy_path"](declared)) + return tuple(str((self.checkout / root).resolve()) for root in roots) + + @property + def import_paths(self) -> tuple[str, ...]: + return tuple(dict.fromkeys((*self.source_paths, + *(path for checkout in self.dependencies + for path in StackClient(self.name, checkout).source_paths)))) + + def files_importing(self, module: str) -> list[str]: + """A selected path comes from either cut, but absence on this cut stays absent.""" + clients = (self, *(StackClient(self.name, path) for path in self.compared)) + return sorted({entry.path for client in clients for entry in client.imports + if entry.module.partition(".")[0] == module + and (self.checkout / entry.path).is_file()}) @classmethod - def fetched(cls, stack: Path, into: Path) -> tuple[StackClient, ...]: + @cache + def revision(cls, repo: Path, head: str, uv: Path, python: Path) -> tuple[StackClient, ...]: + """Ordinary local comparisons retain one head-declared main capture for both sides.""" + if not Repo.git(repo, "ls-tree", "--name-only", head, "--", "stack/pyproject.toml"): + return () + into = Path(tempfile.mkdtemp(prefix="debt-ratchet-clients-")) + atexit.register(shutil.rmtree, into, True) + (into / "stack").mkdir() + (into / "stack/pyproject.toml").write_text(Repo.git(repo, "show", f"{head}:stack/pyproject.toml")) + with InstallCommands() as commands: + return cls.fetched(into / "stack", into, uv=uv, python=python, commands=commands) + + @classmethod + def fetched(cls, stack: Path, into: Path, *, uv: Path, python: Path, + commands: InstallCommands) -> tuple[StackClient, ...]: """Shallow clones of every git source at its declared branch head.""" clients = [] for name, (url, branch) in StackClient.sources(stack).items(): @@ -327,6 +367,26 @@ def fetched(cls, stack: Path, into: Path) -> tuple[StackClient, ...]: ["git", "clone", "--quiet", "--depth", "1", "--branch", branch, url, str(checkout)], check=True, ) + metadata = ProjectMetadata.acquire(checkout, uv=uv, python=python, + commands=commands, output=into / "metadata") + cls.require_package(metadata, name) + clients.append(cls(name, checkout)) + return tuple(clients) + + @classmethod + def materialized(cls, stack: Path, directory: Path, side: type[Side], + inputs: IntegrationInput, + metadata: dict[RepositoryName, ProjectMetadata]) -> tuple[StackClient, ...]: + """Only this actual cut's declarations, from the already-frozen Git input. + + A package absent on this side is absent here; the comparison owner forms the + selected-file union across the two returned sets, not a phantom main clone. + """ + clients = [] + for name, (url, branch) in cls.sources(stack).items(): + source = inputs.source(Repo.repository_url(url)) + checkout = side.require_materialized(directory, source) + cls.require_package(metadata[source.repository], name) clients.append(cls(name, checkout)) return tuple(clients) @@ -341,6 +401,96 @@ def imports(self) -> tuple[ClientImport, ...]: return tuple(found) +@dataclass(frozen=True) +class ClientSources(StackSources): + """The comparison's acquisition policy, passed to every measure before workers fork.""" + + def requires_clients(self) -> bool: + return False + + def at(self, repo: Path, base: str, head: str, side: type[Side]) -> tuple[StackClient, ...]: + return StackClient.revision(repo, head, self.frontend(), Path(sys.executable)) + + +@dataclass(frozen=True) +class MaterializedClients(ClientSources): + directory: Path + inputs: IntegrationInput + uv: Path + python: Path + commands: InstallCommands + + @classmethod + def read(cls, directory: Path, *, uv: Path, python: Path, + commands: InstallCommands) -> MaterializedClients: + return cls(directory, IntegrationInput.at(directory), uv, python, commands) + + @cached_property + def environment(self) -> dict[str, str]: + return self.marker_environment(self.python) + + @cached_property + def metadata(self) -> dict[type[Side], dict[RepositoryName, ProjectMetadata]]: + """One acquisition per actual immutable project, shared by all consumers here.""" + _ = self.environment # Prove the selected interpreter once before using its Python version. + result: dict[type[Side], dict[RepositoryName, ProjectMetadata]] = {} + core_source = self.inputs.source(CorePackage.repository) + for side in Side.sides(): + core = side.require_materialized(self.directory, core_source) + packages = {core_source.repository: CorePackage.distribution, + **{Repo.repository_url(url): name + for name, (url, branch) in self.sources(core / "stack").items()}} + projects: dict[RepositoryName, ProjectMetadata] = {} + for identity in sorted(packages, key=lambda identity: identity.key): + source = self.inputs.source(identity) + checkout = side.require_materialized(self.directory, source) + try: + metadata = ProjectMetadata.acquire(checkout, uv=self.uv, python=self.python, + commands=self.commands, output=self.directory / "metadata") + self.require_package(metadata, packages[identity]) + ProjectMetadata.require_python(metadata.python) + projects[identity] = metadata + finally: + side.require_materialized(self.directory, source) + result[side] = projects + return result + + def projects(self, side: type[Side]) -> dict[RepositoryName, ProjectMetadata]: + projects = self.metadata[side] + for identity in projects: + side.require_materialized(self.directory, self.inputs.source(identity)) + return projects + + @cached_property + def cuts(self) -> dict[type[Side], tuple[StackClient, ...]]: + core_source = self.inputs.source(CorePackage.repository) + return {side: StackClient.materialized( + side.require_materialized(self.directory, core_source) / "stack", + self.directory, side, self.inputs, self.projects(side)) for side in Side.sides()} + + def requires_clients(self) -> bool: + # A companion can change while the local Core patch is unchanged. + return True + + def at(self, repo: Path, base: str, head: str, side: type[Side]) -> tuple[StackClient, ...]: + source = self.inputs.source(Repo.of(repo).github) + if Repo.git(repo, "rev-parse", base).strip() != source.baseline.commit or ( + Repo.git(repo, "rev-parse", f"{head}^{{tree}}").strip() != source.candidate_tree): + raise ValueError("Checker Core cuts do not match the materialized source input") + for cut in Side.sides(): + self.projects(cut) + cuts = self.cuts + clients = tuple(client for found in cuts.values() for client in found) + origins = {member.checkout(self.directory, source.repository): source.repository + for member in Side.sides() for source in self.inputs.sources} + return tuple(replace(client, + compared=tuple(peer.checkout for peer in clients + if origins[peer.checkout] == origins[client.checkout] + and peer.checkout != client.checkout), + dependencies=tuple(peer.checkout for peer in cuts[side] if peer.checkout != client.checkout)) + for client in cuts[side]) + + class OutgoingWire(StackClient): """Every declaration carrying the cross-runtime capability, from its owning registry.""" @@ -694,9 +844,24 @@ def main() -> int: parser.add_argument( "--report", type=Path, help="with --check, also write the layer report to this file" ) + parser.add_argument("--ci-inputs", type=Path, + help="use the integration entrypoint's already-materialized exact source input") arguments = parser.parse_args() - with tempfile.TemporaryDirectory() as clones: - package = Package(ROOT, StackClient.fetched(ROOT / "stack", Path(clones))) + with tempfile.TemporaryDirectory() as clones, InstallCommands() as commands: + if arguments.ci_inputs: + inputs = IntegrationInput.at(arguments.ci_inputs) + local = inputs.source(Repo.of(ROOT).github) + core = HeadSide.require_materialized(arguments.ci_inputs, local) + resource = MaterializedClients(arguments.ci_inputs, inputs, StackSources.frontend(), + Path(sys.executable), commands) + package = Package(core, resource.cuts[HeadSide]) + base = BaseSide.require_materialized(arguments.ci_inputs, local) + if arguments.base and Repo.git(ROOT, "rev-parse", arguments.base).strip() != local.baseline.commit: + raise ValueError("The outgoing wire must check this input's actual captured base") + else: + package = Package(ROOT, StackClient.fetched(ROOT / "stack", Path(clones), + uv=StackSources.frontend(), python=Path(sys.executable), commands=commands)) + base = Path(clones) / "base" if not arguments.check: sys.stdout.write(package.markdown()) return 0 @@ -704,7 +869,8 @@ def main() -> int: arguments.report.write_text(package.markdown()) lacking = report(package) if arguments.base: - base = base_tree(arguments.base, Path(clones) / "base") + if not arguments.ci_inputs: + base = base_tree(arguments.base, base) print(f"outgoing wire against Core at {arguments.base}:") lacking = report(Package(base, (OutgoingWire.at(),))) or lacking return 1 if lacking else 0 diff --git a/tools/render_native_proof_schema.py b/tools/render_native_proof_schema.py index 32663c505f..d79e57cea0 100644 --- a/tools/render_native_proof_schema.py +++ b/tools/render_native_proof_schema.py @@ -4,7 +4,7 @@ import sys from pathlib import Path -from agent_comms.native_pi import NativeContextJournal +from agent_comms.native_input_proof import NativeContextJournal def main() -> None: diff --git a/uv.lock b/uv.lock index 14240be34f..faa005c9f7 100644 --- a/uv.lock +++ b/uv.lock @@ -36,6 +36,7 @@ dev = [ { name = "pytest-timeout" }, { name = "pytest-xdist" }, { name = "ruff" }, + { name = "tomli-w" }, { name = "uv" }, ] @@ -55,6 +56,7 @@ requires-dist = [ { name = "pytest-timeout", marker = "extra == 'dev'", specifier = ">=2.3" }, { name = "pytest-xdist", marker = "extra == 'dev'", specifier = ">=3.6" }, { name = "ruff", marker = "extra == 'dev'", specifier = "==0.16.0" }, + { name = "tomli-w", marker = "extra == 'dev'", specifier = "==1.2.0" }, { name = "uv", marker = "extra == 'dev'", specifier = "==0.13.1" }, ] provides-extras = ["dev"] @@ -773,6 +775,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/9a/8415f2657cbe200f41a4531ccededf135505a92d4a012229121f885b26f9/ruff-0.16.0-py3-none-win_arm64.whl", hash = "sha256:14296fedcd2705c77ab8235439278bbb38f285cf7da5528b00b3e330c3d4872d", size = 11273407, upload-time = "2026-07-23T19:11:28.705Z" }, ] +[[package]] +name = "tomli-w" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/19/75/241269d1da26b624c0d5e110e8149093c759b7a286138f4efd61a60e75fe/tomli_w-1.2.0.tar.gz", hash = "sha256:2dd14fac5a47c27be9cd4c976af5a12d87fb1f0b4512f81d69cce3b35ae25021", size = 7184, upload-time = "2025-01-15T12:07:24.262Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/18/c86eb8e0202e32dd3df50d43d7ff9854f8e0603945ff398974c1d91ac1ef/tomli_w-1.2.0-py3-none-any.whl", hash = "sha256:188306098d013b691fcadc011abd66727d3c414c571bb01b1a174ba8c983cf90", size = 6675, upload-time = "2025-01-15T12:07:22.074Z" }, +] + [[package]] name = "tree-sitter" version = "0.25.2"