diff --git a/src/agent_comms/active_route.py b/src/agent_comms/active_route.py index dca5e83a8d..b95a5b2b4a 100644 --- a/src/agent_comms/active_route.py +++ b/src/agent_comms/active_route.py @@ -19,6 +19,7 @@ from pathlib import Path from typing import Annotated, ClassVar +from .child_process import Platform from .coordination_errors import CoordinationError from .declared_family import DeclaredFamily from .errors import RelationViolationError @@ -193,15 +194,13 @@ def read_active_route(path: Path | None = None) -> ActiveRoute | None: parent = path.parent.lstat() if ( not stat.S_ISDIR(parent.st_mode) - or parent.st_uid != os.geteuid() - or stat.S_IMODE(parent.st_mode) != 0o700 + or not Platform.current().is_private(parent, 0o700) ): raise ValueError("active comms route directory must be owner-only") info = os.fstat(fd) if ( not stat.S_ISREG(info.st_mode) - or info.st_uid != os.geteuid() - or stat.S_IMODE(info.st_mode) != 0o600 + or not Platform.current().is_private(info, 0o600) or info.st_nlink != 1 or not 0 < info.st_size <= 4096 ): @@ -244,12 +243,11 @@ def guard_default_route_write(expected_root: Path, *, blocking: bool = True) -> parent = path.parent.lstat() if ( not stat.S_ISDIR(info.st_mode) - or info.st_uid != os.geteuid() + or not Platform.current().is_owned(info) or (parent.st_dev, parent.st_ino) != (info.st_dev, info.st_ino) ): raise ValueError("active comms route directory changed or is not owned") - if stat.S_IMODE(info.st_mode) != 0o700: - os.fchmod(directory, 0o700) + Platform.current().make_private_descriptor(directory, 0o700) route = CommsRoute.current(path) # Local roots may be created under this guard. ActiveRoute owns the # stronger observation of an already published root and its identity. @@ -451,10 +449,10 @@ def route_directory(path: Path) -> Iterator[int]: acquire_exclusive_route_lock(directory) info = os.fstat(directory) parent = path.parent.lstat() - if (not stat.S_ISDIR(info.st_mode) or info.st_uid != os.geteuid() + if (not stat.S_ISDIR(info.st_mode) or not Platform.current().is_owned(info) or (parent.st_dev, parent.st_ino) != (info.st_dev, info.st_ino)): raise ValueError("active comms route directory changed or is not owned") - os.fchmod(directory, 0o700) + Platform.current().make_private_descriptor(directory, 0o700) yield directory finally: os.close(directory) @@ -475,12 +473,12 @@ def _publish_active_route_locked( parent = path.parent.lstat() if ( not stat.S_ISDIR(info.st_mode) - or info.st_uid != os.geteuid() + or not Platform.current().is_owned(info) or parent.st_dev != info.st_dev or parent.st_ino != info.st_ino ): raise ValueError("active comms route directory changed or is not owned") - os.fchmod(directory, 0o700) + Platform.current().make_private_descriptor(directory, 0o700) if backup is None and read_active_route(path) is not None: raise ValueError("active comms route is already installed") payload = (json.dumps(route.to_record(), sort_keys=True) + "\n").encode() diff --git a/src/agent_comms/agent_events.py b/src/agent_comms/agent_events.py index 0cafba4a34..909bf63e48 100644 --- a/src/agent_comms/agent_events.py +++ b/src/agent_comms/agent_events.py @@ -7,7 +7,7 @@ from __future__ import annotations from abc import ABC, abstractmethod -from dataclasses import asdict, dataclass, field, replace +from dataclasses import dataclass, field, replace from functools import partial from typing import TYPE_CHECKING, Any, ClassVar @@ -22,7 +22,7 @@ ) from .turn_terminal import FailedGoalTurn from .turn_goal_account import OriginGoalSettlement, VerifiedGoalSettlement -from .diagnostics import FailureReason, terminal_failure_reason +from .turn_failure import BackendFailed, TurnFailure from .child_process import ProcessIdentity from .declared_family import DeclaredFamily from .field_codec import NullMember @@ -258,7 +258,7 @@ def phase(self) -> str: class Done(AgentEvent, FailedGoalTurn): text: str ok: bool - reason_code: str | None = None + failure: TurnFailure | None = None diagnostic: dict[str, Any] | None = None outcome: NativeTurnOutcome = field(default_factory=UnresolvedNativeTurnOutcome) # Immutable completion snapshot of TurnOutput's already decoded result. @@ -272,8 +272,12 @@ def resolve( return replace(self, outcome=outcome) @property - def failure_reason(self) -> FailureReason: - return terminal_failure_reason(asdict(self)) + def failure_kind(self) -> type[TurnFailure]: + return BackendFailed if self.failure is None else type(self.failure) + + @property + def reason_code(self) -> str | None: + return None if self.ok else self.failure_kind.declared_name def failure_observation(self, account, permit, captured): return None if self.ok else super().failure_observation(account, permit, captured) diff --git a/src/agent_comms/attempt_store.py b/src/agent_comms/attempt_store.py index 07a948c19b..8745af9289 100644 --- a/src/agent_comms/attempt_store.py +++ b/src/agent_comms/attempt_store.py @@ -19,12 +19,7 @@ from agent_comms.coordination_snapshot import RecoverySnapshot from agent_comms.coordination_tables.assignments import WakeAssignment from agent_comms.coordination_tables.attempts import AttemptRecord, ReplayFact -from agent_comms.coordination_tables.recovery import ( - ACPClientConnectivity, - ConnectivityFacet, - OwnerConnectivity, - RecoveryAudit, -) +from agent_comms.coordination_tables.recovery import RecoveryAudit from agent_comms.owner_fence import OwnerFence from agent_comms.turn_failure import TurnFailure from agent_comms.recovery_reader import RecoveryReader @@ -208,34 +203,6 @@ def fail_unknown( ) return Applied(self.get(fence.execution_id)) - def observe_connectivity( - self, - fence: OwnerFence, - *, - expected_pointer_revision: int, - expected_revision: int | None, - owner: OwnerConnectivity, - acp_client: ACPClientConnectivity, - ) -> Applied[RecoverySnapshot]: - owner, acp_client = OwnerConnectivity(owner), ACPClientConnectivity(acp_client) - with self.session.writing() as db: - snapshot, _ = self.require_fence(fence) - before = snapshot.connectivity - if ( - snapshot.pointer_revision != expected_pointer_revision - or (before.revision if before else None) != expected_revision - ): - raise StaleRevision("connectivity or pointer revision changed") - now = self.session.now(before.observed_at_ms if before else 0) - ConnectivityFacet( - fence.execution_id, - owner, - acp_client, - before.revision + 1 if before else 1, - now, - ).upsert(db) - return Applied(self.get(fence.execution_id)) - def append_recovery_audit( self, fence: OwnerFence, diff --git a/src/agent_comms/backend.py b/src/agent_comms/backend.py index e4d932e2f0..e09c4fe0f2 100644 --- a/src/agent_comms/backend.py +++ b/src/agent_comms/backend.py @@ -47,7 +47,6 @@ from .acp_failure import UnrecordedFailureSource from .child_process import TimedOutOutcome from .coordinator import Coordination -from .diagnostics import FailureReason from .extension_ui import ExtensionUiSession from .field_codec import Id128Text, MalformedRecord, boundary_decoder from .image_inputs import ImageInput @@ -158,8 +157,8 @@ async def stream_agent_events( session_file=session_file, )) except NativePiUnavailable as error: - yield events.Error(text=str(error), reason_code="native_launch_invalid") - yield events.Done(ok=False, reason_code="native_launch_invalid", text=str(error)) + yield events.Error(text=str(error), reason_code=failures.NativeLaunchInvalid.declared_name) + yield events.Done(ok=False, failure=failures.NativeLaunchInvalid(str(error)), text=str(error)) return from .session_fence import session_writer_fence @@ -539,7 +538,6 @@ async def receive_record(self) -> AsyncIterator[events.AgentEvent]: return if not self.line: if self.awaiting_native_attestation: - self.output.preflight_failure = FailureReason.PREFLIGHT_EXIT self.output.diagnostic = { "elapsed_ms": round( (self.loop.time() - self.watchdog.launch_started_at) * 1000 @@ -548,7 +546,7 @@ async def receive_record(self) -> AsyncIterator[events.AgentEvent]: } self.output.diagnostic.update(self.watchdog.session_load.diagnostic) self.output.record_failure( - failures.InputIdUnavailable( + failures.NativePreflightExit( "Pi native input-ID capability preflight ended before attestation." ) ) @@ -586,7 +584,7 @@ async def invalidate_identity(self) -> tuple[events.AgentEvent, ...]: self.output.discard_text() updates = (self.context_info(), events.TurnState( state="failed", - reason_code="session_identity_uncertain", + reason_code=failures.IdentityUncertain.declared_name, elapsed_ms=0, phase=ShutdownPhase(), )) diff --git a/src/agent_comms/bus_tables.py b/src/agent_comms/bus_tables.py index 29dce0497c..71210c8602 100644 --- a/src/agent_comms/bus_tables.py +++ b/src/agent_comms/bus_tables.py @@ -26,6 +26,7 @@ from agent_comms.field_codec import FieldCodec, WireName, WireOrder from agent_comms.message_reference import MessageReference from agent_comms.typed_table import ( + TableReference, AppendOnlyTables, Column, ComposedSchemas, @@ -303,7 +304,7 @@ class ClaimOperations(TypedTable, BusProjectionTables): @classmethod def references(cls) -> tuple[ForeignKey, ...]: - return (ForeignKey(("seq",), messages.Claimed.table, ("seq",)),) + return (ForeignKey(columns=("seq",), target=messages.Claimed.table, target_columns=("seq",)),) @classmethod def used(cls, db: sqlite3.Connection, operation_id: str) -> bool: @@ -319,7 +320,7 @@ class HeldResources(TypedTable, BusProjectionTables): @classmethod def references(cls) -> tuple[ForeignKey, ...]: - return (ForeignKey(("claim_seq",), messages.Claimed.table, ("seq",)),) + return (ForeignKey(columns=("claim_seq",), target=messages.Claimed.table, target_columns=("seq",)),) @classmethod def owner_for(cls, db: sqlite3.Connection, resource: str) -> ResourceClaim: @@ -429,7 +430,7 @@ def advance(self, removed: tuple[str, ...], projected: ClaimProjection) -> None: class CapabilityRow(TypedTable, BusTables, declared_name=None): """Every generated capability row belongs to one original bus sequence.""" - seq: Annotated[int, Column(primary_key=True, references=(BusRow, "seq"))] + seq: Annotated[int, Column(primary_key=True, references=TableReference(BusRow, ("seq",)))] class _CapabilityTable: diff --git a/src/agent_comms/child_process.py b/src/agent_comms/child_process.py index ee9ba6e83a..b8de4b85ad 100644 --- a/src/agent_comms/child_process.py +++ b/src/agent_comms/child_process.py @@ -37,6 +37,7 @@ from dataclasses import dataclass from functools import partial from pathlib import Path +from tempfile import TemporaryDirectory from typing import TYPE_CHECKING, Any, ClassVar, NoReturn from .declared_family import DeclaredFamily @@ -428,9 +429,24 @@ def sync_directory(self, path: Path) -> None: ... @abstractmethod def is_private(self, info: os.stat_result, permissions: int) -> bool: ... + @abstractmethod + def is_owned(self, info: os.stat_result) -> bool: ... + @abstractmethod def make_private(self, path: Path, permissions: int) -> None: ... + @abstractmethod + def make_private_descriptor(self, descriptor: int, permissions: int) -> None: ... + + @contextmanager + def socket_address(self, path: Path, descriptor: int) -> Generator[Path, None, None]: + with TemporaryDirectory(prefix="ac-socket-", dir="/tmp") as directory: + parent = Path(directory) / "d" + parent.symlink_to(path.parent, target_is_directory=True) + if not os.path.samestat(os.stat(parent), os.fstat(descriptor)): + raise ValueError("Socket directory changed during address acquisition") + yield parent / path.name + @abstractmethod def try_store_lock(self, descriptor: int, *, mode: LockMode) -> None: """One nonblocking native attempt; busy is BlockingIOError.""" @@ -507,13 +523,20 @@ def sync_directory(self, path: Path) -> None: os.close(descriptor) def is_private(self, info: os.stat_result, permissions: int) -> bool: - return stat.S_IMODE(info.st_mode) == permissions and info.st_uid == os.geteuid() + return stat.S_IMODE(info.st_mode) == permissions and self.is_owned(info) + + def is_owned(self, info: os.stat_result) -> bool: + return info.st_uid == os.geteuid() def make_private(self, path: Path, permissions: int) -> None: with suppress(FileNotFoundError): if stat.S_IMODE(path.lstat().st_mode) != permissions: os.chmod(path, permissions, follow_symlinks=False) + def make_private_descriptor(self, descriptor: int, permissions: int) -> None: + if stat.S_IMODE(os.fstat(descriptor).st_mode) != permissions: + os.fchmod(descriptor, permissions) + def acquire_store_lock(self, descriptor: int, *, mode: LockMode, blocking: bool) -> None: import fcntl @@ -559,6 +582,10 @@ class LinuxProcessRecord: class LinuxPlatform(NamespaceContainment, PidfdHandles, PosixPlatform): + @contextmanager + def socket_address(self, path: Path, descriptor: int) -> Generator[Path, None, None]: + yield Path(f"/proc/{os.getpid()}/fd/{descriptor}") / path.name + def parent_death_binding(self) -> int: """The exec gate arms PR_SET_PDEATHSIG and binds to this launcher's PID.""" return os.getpid() @@ -728,9 +755,15 @@ def is_private(self, info: os.stat_result, permissions: int) -> bool: # NTFS ACLs, not POSIX mode bits, determine privacy. return True + def is_owned(self, info: os.stat_result) -> bool: + return True + def make_private(self, path: Path, permissions: int) -> None: pass + def make_private_descriptor(self, descriptor: int, permissions: int) -> None: + pass + def try_store_lock(self, descriptor: int, *, mode: LockMode) -> None: import errno import msvcrt @@ -1858,24 +1891,24 @@ def __init__(self, process: subprocess.Popen[bytes], identity: ProcessIdentity, self.process = process @classmethod - def launch( + @contextmanager + def launching( cls, command: tuple[str, ...], *, cwd: str | Path | None = None, env: Mapping[str, str] = INHERITED_ENVIRONMENT, output: Any = subprocess.DEVNULL, - before_start: Callable[[ProcessIdentity], None] | None = None, runtime: LaunchRuntime = LaunchRuntime.running(), lifetime: type[ChildLifetime] = DiesWithLauncherLifetime, - ) -> SpawnedProcess: + ) -> Generator[SpawnedProcess]: + """Yield the child before it executes; leaving the body releases it, raising reaps it.""" if not command: raise ValueError("A child command is required") with Platform.current().launch(command, (), runtime, lifetime) as launch: child = launch.spawn(cwd=cwd, env=env, stdout=output, stderr=output) try: - if before_start is not None: - before_start(child.identity) + yield child except BaseException: launch.cancel_before_release(child.identity) child.reap() @@ -1887,7 +1920,12 @@ def launch( except BaseException: child.close() raise - return child + + @classmethod + def launch(cls, command: tuple[str, ...], **options: Any) -> SpawnedProcess: + """Launch with nothing to reserve before the child executes.""" + with cls.launching(command, **options) as child: + return child def collect(self, *, timeout: float) -> subprocess.CompletedProcess[bytes]: """Collect within the caller's owned context; its exit retires descendants.""" diff --git a/src/agent_comms/cohort_schema.py b/src/agent_comms/cohort_schema.py index e4954330f9..73609f473c 100644 --- a/src/agent_comms/cohort_schema.py +++ b/src/agent_comms/cohort_schema.py @@ -203,10 +203,10 @@ class ClaimBatchMembers(GuardedInserts, ImmutableRows, CohortTable, TypedTable): @classmethod def references(cls): return ( - ForeignKey(("claim_id",), WakeAssignment, ("assignment_id",)), - ForeignKey(("recipient_lookup",), Participants, ("participant_lookup",)), + ForeignKey(columns=("claim_id",), target=WakeAssignment, target_columns=("assignment_id",)), + ForeignKey(columns=("recipient_lookup",), target=Participants, target_columns=("participant_lookup",)), ForeignKey( - ("wire_root_id", "wire_seq"), ClaimBatchAcknowledgements, ("wire_root_id", "wire_seq") + columns=("wire_root_id", "wire_seq"), target=ClaimBatchAcknowledgements, target_columns=("wire_root_id", "wire_seq") ), ) @@ -282,10 +282,10 @@ def sealed_sequences_after( @classmethod def references(cls): return ( - ForeignKey(("recipient_lookup",), Participants, ("participant_lookup",)), - ForeignKey(("claim_id",), WakeAssignment, ("assignment_id",)), + ForeignKey(columns=("recipient_lookup",), target=Participants, target_columns=("participant_lookup",)), + ForeignKey(columns=("claim_id",), target=WakeAssignment, target_columns=("assignment_id",)), ForeignKey( - ("wire_root_id", "wire_seq"), ClaimBatchAcknowledgements, ("wire_root_id", "wire_seq") + columns=("wire_root_id", "wire_seq"), target=ClaimBatchAcknowledgements, target_columns=("wire_root_id", "wire_seq") ), ) @@ -359,12 +359,12 @@ def require_obligation(self, execution, link, assignment, obligation) -> None: @classmethod def references(cls): return ( - ForeignKey(("claim_id",), WakeAssignment, ("assignment_id",)), - ForeignKey(("owner_lookup",), Participants, ("participant_lookup",)), + ForeignKey(columns=("claim_id",), target=WakeAssignment, target_columns=("assignment_id",)), + ForeignKey(columns=("owner_lookup",), target=Participants, target_columns=("participant_lookup",)), ForeignKey( - ("wire_root_id", "wire_seq", "claim_id"), - ClaimBatchMembers, - ("wire_root_id", "wire_seq", "claim_id"), + columns=("wire_root_id", "wire_seq", "claim_id"), + target=ClaimBatchMembers, + target_columns=("wire_root_id", "wire_seq", "claim_id"), ), ) diff --git a/src/agent_comms/commander/order.py b/src/agent_comms/commander/order.py index 8f545458b9..76ef41ddb3 100644 --- a/src/agent_comms/commander/order.py +++ b/src/agent_comms/commander/order.py @@ -50,7 +50,7 @@ boundary_decoder, ) from agent_comms.goal_actions import CreateProblemGoalAction, OrderGoalAction -from agent_comms.goals import OrderCheck, ProblemGoalOrder +from agent_comms.goals import OrderCheck from agent_comms.message_reference import MessageReference from agent_comms.messages import Message from agent_comms.native_tool_call import SelectedToolDenied @@ -71,12 +71,10 @@ DeliverySource, DirectiveText, LocatedProblemFinding, - MeasuredReview, NoSuccessor, OperationDelivery, OrderDelivery, OrderedHunt, - OrderMessage, OrderProgress, OrderText, OrderWords, @@ -319,133 +317,6 @@ def resolved(self, context: CommanderContext, finding: ProblemFinding) -> OrderF check = replace(self.order.check, directory=str(context.repo_directory(self.order.check.directory))) return replace(self, order=replace(self.order, finding=finding, check=check)) - - def assignment_refusals(self, context: CommanderContext, unit: str, - group: Sequence[str], listed: Mapping[str, Todo], - target: OrderedProblems, - traced: FileLine | NoSourceReference) -> list[str]: - """The document's selected problems, phase and existing owners must agree.""" - reading = context.plan_reading(context.comms.goals.todos.history_for_commander(context.incarnation)) - problems = list(self.order.phase.order_refusals(reading, self.unplanned_reason)) - held = tuple(todo for todo in listed.values() if todo.holder) - busy = tuple(todo for todo in held if todo.holder == unit and todo.id not in group) - if busy: - snapshot = context.comms.registry.snapshot() - occupied = {todo.holder for todo in held} | {context.commander} | context.parked - idle = sorted(thread.name for thread in snapshot.active_threads(snapshot.threads.values()) - if thread.name not in occupied and thread.role.executor(thread)) - problems.append(f'{unit} already holds orders ' - f'{", ".join(todo.id for todo in busy)}; ' - f'running units holding no order: {", ".join(idle) or "none"}') - if not group or any(not ProblemNumber.accepts(pid) for pid in group): - problems.append('problems: fix: supply an array of P IDs, or use order problem') - problems.extend(target.refusals(self, group, listed)) - for pid in group: - try: - with MalformedRecord.refusing(pid): - if pid in listed: - listed[pid].holder_state.require_trace(unit, traced) - except MalformedRecord as error: - problems.append(str(error)) - return problems - - def check_refusals(self, context: CommanderContext) -> tuple[str, ...]: - """Send and correction use the check owner's clean-main acceptance rule.""" - try: - with MalformedRecord.refusing('order.check'): - self.order.require_writable() - check = self.order.check - if Path(check.directory).is_dir(): - Repo.git(Path(check.directory), 'fetch', '-q', 'origin', 'main') - base = self.order.check_base(Path(check.directory)) - check.require_acceptance(base, context.worktrees) - except MalformedRecord as error: - return (f'order.check: {error}; fix: supply a focused measurable command',) - return () - - def source_refusals(self, context: CommanderContext, - traced: FileLine | NoSourceReference) -> list[str]: - """Scope, prose and pattern citations share the order's source revision.""" - order, check = self.order, self.order.check - problems = [] - for label, validate in (('order.source', order.require_references), - ('order.check.directory', check.require_directory)): - try: - with MalformedRecord.refusing(label): - validate() - except MalformedRecord as error: - problems.append(f'{label}: {error}; fix: name an existing source location or directory') - references = tuple(dict.fromkeys(( - *FileLine.scan('\n'.join((self.prose, *order.edits))), - *(reference for slop in self.slop for reference in (slop.at, slop.by)), - *traced.located(), - ))) - deleted = {Path(module).stem for module in order.deletes.modules} - if deleted and (unread := context.importers(deleted, check)): - problems.append(f'the order deletes {sorted(deleted)}, which {unread} import; ' - 'the check must also count their `from agent_comms` imports') - for reference in references: - try: - with MalformedRecord.refusing(reference.to_wire()): - reference.require_exists(Path(check.directory), order.cited_revision) - except MalformedRecord as error: - problems.append(f'{error}; fix: cite an existing source line') - body = '\n'.join((self.prose, *order.edits, order.report)) - if any((Path(check.directory) / reference.file).resolve().is_relative_to( - Path(check.directory) / 'tools/') for reference in references) and not all( - said in body for said in ('--root tools', '--root src')): - problems.append('the order changes tools/: say "ratchet with --root tools and with ' - '--root src, each as its own command"') - return problems - - def require_send(self, context: CommanderContext, unit: str, - group: Sequence[str], listed: Mapping[str, Todo], target: OrderedProblems, - reassign: Reassignment, holders: Sequence[str], clean: bool, - traced: FileLine | NoSourceReference) -> None: - """Collect field refusals before recording any part of this document.""" - problems = self.problems_found(clean) - try: - context.comms.registry.storage_rule.require_quota(context.comms.registry.require(unit)) - except SelectedToolDenied as quota: - problems.append(f'{unit}: {quota}: it cleans up its own checkouts and scratch ' - 'before it gets new work') - problems.extend(self.assignment_refusals(context, unit, group, listed, target, traced)) - problems.extend(self.source_refusals(context, traced)) - problems.extend(self.check_refusals(context)) - problems.extend(reassign.refusals(holders)) - if problems: - sys.exit('Refused order:\n ' + '\n '.join(problems)) - fired = tuple(listed[pid] for pid in group if pid in listed and listed[pid].struck_out(unit)) - self.order.run.require_assignment(context.comms.registry.require(unit).incarnation, - fired, ProblemGoalOrder.StrikeLimit) - - def message(self, context: CommanderContext, unit: str, group: Sequence[str], - replacing: bool, reassign: Reassignment) -> str: - """Render owned fields; never parse this message back into an order.""" - instructions = '\n'.join((self.prose, *(slop.render() for slop in self.slop), - f'Problems: {", ".join(group)}', *self.order.work_instructions())) - replacement = '\nThis replaces my earlier order on it; drop that one.' if replacing else '' - storage = context.comms.registry.storage_rule.order_text( - context.comms.registry.require(unit), - [f'{repo.checkout}/.venv/bin/python' for repo in context.repos]) - return (instructions + replacement + reassign.note() + '\n' + storage - + '\nIf two lines of this order conflict, take the narrower reading, ' - 'say which one you took, and proceed; do not stop to ask.' - + '\nStart now and push as you go.\n' - + '\n'.join(line for pid in group for line in self.order.scope.instructions(pid))) - - def record(self, context: CommanderContext, unit: str, group: Sequence[str], - target: OrderedProblems, traced: FileLine | NoSourceReference, - review_at: RecordedEventTime, request: OwnerRequestId) -> None: - """The native tools record the typed order; the document supplies no second store.""" - target.record(context, self.order, unit, group, traced, request, review_at) - - def send_back(self, context: Commander, unit: str, group: Sequence[str], - replacing: bool, clean: bool, reassign: Reassignment, struck_already: bool) -> None: - if replacing and not clean: - reassign.send_back(context, self, unit, group[0], struck_already) - - STAGES = ( "tool call", "review", @@ -543,23 +414,6 @@ def last_activity(self, unit): thread = self.comms.registry.require(unit) return thread.runtime.history_source(self.comms.root, thread.incarnation).last_tool_call() - def board_text(self) -> str: - """Render the live Todo view without saving a competing board file.""" - lines = ['# Fleet board', '', - "Live view of Core's Todo store; no board file is saved.", '', - '| Problem | Owner file | Owner | Order | Due | Check | Progress | Order state ' - '| Strikes | Owner last acted |', - '|---|---|---|---|---|---|---|---|---|---|'] - listed, read = self.todos(), self.progress() - activity = {unit: self.last_activity(unit) for unit in {t.holder for t in listed if t.holder}} - for t in sorted(listed, key=lambda t: ProblemNumber.parse(t.id).number): - lines.append(f'| {t.id}: {t.text.replace("|", "|")} | {t.order.board_cells(t)} | ' - f'{read[t.id].line if t.id in read else "-"} | {t.order.line} | ' - f'{t.strike_count} | {activity[t.holder].clock() if t.holder else "-"} |') - history = self.comms.goals.todos.history_for_commander(self.incarnation) - lines += ['', '## Withdrawn orders', '', - *(f'- {line}' for t in history for line in t.order.withdrawals(t))] - return '\n'.join(lines) + '\n' class OrderedProblems(ABC): @@ -738,29 +592,7 @@ def send(self, unit: str, document: OrderFile, clean: bool = False, answers: tuple[MessageReference, ...] = (), *, request: RequestSelection) -> None: """Record the decoded order, or report every failed prerequisite before any write.""" document = document.resolved(self, finding) - group = target.group(document) - listed = {todo.id: todo for todo in self.todos()} - holders = sorted({listed[pid].holder for pid in group if pid in listed} - {unit, None}) - replacing = target.replacing(unit, group, listed) - document.require_send(self, unit, group, listed, target, reassign, holders, clean, traced) - struck_already = replacing and listed[group[0]].struck_on_order(unit) - measured = MeasuredReview.read(self.comms.goals.todos) - review_at = measured.review_at(document.order) - message = document.message(self, unit, group, replacing, reassign) - request_id = request.for_order(self.comms.goals.todos, self.incarnation) - document.record(self, unit, group, target, traced, review_at, request_id) - sent = self.post(VerdictAudience((unit,)), message, answers) - for pid in group: - self.call(CommsChangeProblemTool(problem=pid, operation=OrderMessage( - self.comms.goals.todos.get(pid).revision, self.actor_identity, sent))) - for other in holders: - self.post(VerdictAudience((other,)), - f'Taken off you: {", ".join(group)} ("{listed[group[0]].text[:120]}") ' - f'now belongs to {unit} because: {reassign.because()}. ' - 'Stop that work; do not touch its files.') - print(f'{measured.summary}; review timer {review_at.clock()}') - print(document.order.id) - document.send_back(self, unit, group, replacing, clean, reassign, struck_already) + document.order.send(self, document, unit, clean, reassign, target, traced, answers, request) def correct_check(self, pid: str, line: str, why: str) -> None: """Correct the current check against the source preceding this order's delivery.""" @@ -769,9 +601,7 @@ def correct_check(self, pid: str, line: str, why: str) -> None: check = self.read_check(line) if len(why.split()) < 8: sys.exit('Refused: --why says what was wrong with the recorded check (8+ words)') - repo = self.repo_directory(check.directory) - Repo.git(repo, 'fetch', '-q', 'origin', 'main') - check.require_acceptance(order.check_base(repo), self.worktrees) + order.require_check_acceptance(check, self.worktrees) self.call(CommsOrderCheckTool(problem=pid, check=check, why=why)) print(f'{pid}: order {order.id} now checks `{check.command}`') @@ -1415,11 +1245,6 @@ def run(self, ctx: Commander) -> None: print(thread.name) -class BoardVerb(OrderVerb): - help = "show Core's current Todo store without saving a board file" - - def run(self, ctx: Commander) -> None: - print(ctx.board_text(), end='') def main(a, ctx: Commander): diff --git a/src/agent_comms/commander/rules/tests-and-commands.md b/src/agent_comms/commander/rules/tests-and-commands.md index 6258c64fbe..72dde39bea 100644 --- a/src/agent_comms/commander/rules/tests-and-commands.md +++ b/src/agent_comms/commander/rules/tests-and-commands.md @@ -12,5 +12,5 @@ type: feedback - Refactor tests with the code: per-member copies become one family test, hand-built setup one fixture owner. Tristan, 2026-10-18: "no stupid test theater, refactor the slop out of tests as you go, they're fucking sloppy". - A red test or failure you meet is fixed at its owner in the change at hand, or handed the same tick to the agent whose open work touches that code, named with its PR. "Not my change" without a named owner is a hidden failure, and every later run pays for it by asking whether a red is real. Tristan, 2026-10-22: "imagine if it doesn't get fixed and in the future you always end up checking oh is this a real failure?"; "not my change is valid if another agent is touching it, but they must be named". - An order check counts over the whole source tree (`grep -r ... src`), never one file. -- Run verb gate loops under `bash -c`: zsh passes a quoted "order board" as one argument. +- Run verb gate loops under `bash -c`: zsh passes a quoted "order idle" as one argument. - A wait or kill by process pattern brackets its first letter (`grep -q "[v]erify"`) or waits on a file or PID, or it matches itself. diff --git a/src/agent_comms/commander/vibe_check.py b/src/agent_comms/commander/vibe_check.py index 9c45c4ff0d..9096489391 100644 --- a/src/agent_comms/commander/vibe_check.py +++ b/src/agent_comms/commander/vibe_check.py @@ -189,16 +189,25 @@ class OverloadDegradation(Degradation, BlocksRelease, ToolResult): @classmethod def of(cls, comms: Comms, commander: str) -> Self: - return cls( - len(StandingRule.pending_todos(comms, commander)), + return cls.from_orders( + StandingRule.todos(comms, commander), len(ReviewTimerUp.violations(comms, commander)), OrderEditCountRead.count(comms, commander), ) + @classmethod + def from_orders(cls, orders: Sequence[Todo], timers_up: int, unjudged: int) -> Self: + """The same load projection for a captured HUD read and an independent rule read.""" + return cls(sum(todo.pending for todo in orders), timers_up, unjudged) + @projected(view="wire") def total(self) -> float: return self.owed + 2 * self.timers_up + self.unjudged / 10 + @projected(view="wire") + def overloaded(self) -> bool: + return self.total >= self.ceiling + def __str__(self) -> str: return ( f"load {self.total:.0f} ({self.owed} open orders, " @@ -208,7 +217,7 @@ def __str__(self) -> str: @classmethod def found(cls, comms: Comms, commander: str) -> tuple[Sign, ...]: load = cls.of(comms, commander) - if load.total < cls.ceiling: + if not load.overloaded: return () return (Sign(cls, (int(time.time() // cls.round_seconds),), f"{load}, over {cls.ceiling:.0f}"),) diff --git a/src/agent_comms/commander_hud.py b/src/agent_comms/commander_hud.py new file mode 100644 index 0000000000..467cd6f074 --- /dev/null +++ b/src/agent_comms/commander_hud.py @@ -0,0 +1,143 @@ +"""Commander display derived from the native record owners in one read.""" + +from __future__ import annotations + +import time +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from agent_comms.commander.config import CommanderConfig +from agent_comms.commander.vibe_check import OverloadDegradation +from agent_comms.field_codec import projected +from agent_comms.plan_phases import PlanPhase +from agent_comms.recorded_time import RecordedEventTime +from agent_comms.thread_identity import ThreadIncarnation +from agent_comms.todos import OrderProgress, ProgressRead, PullRequest, Todo, UnmergedListedPullRequest +from agent_comms.tool_result import ToolResult +from agent_comms.turn_digests import OrderEditCountRead + +if TYPE_CHECKING: + from agent_comms.comms import Comms + + +@dataclass(frozen=True) +class CommanderOrder: + """Small order card; identifiers/text come from Todo, review rounds from TodoStore.""" + + problem: str + text: str + order: str + state: str + progress: OrderProgress + review_timers_up: tuple[RecordedEventTime, ...] + strikes: int + unjudged_edits: int + + +@dataclass(frozen=True) +class CommanderUnit: + """One holder's orders, live strikes and GitHub's open ready/draft requests.""" + + unit: ThreadIncarnation + orders: tuple[CommanderOrder, ...] + strikes: int + ready_prs: tuple[PullRequest, ...] + draft_prs: tuple[PullRequest, ...] + + def line(self) -> str: + orders = "; ".join( + f"{order.problem}: {order.state}; {order.progress.line}" for order in self.orders + ) + return ( + f"{self.unit.name}: orders={orders} " + f"review timers={sum(len(order.review_timers_up) for order in self.orders)} " + f"strikes={self.strikes} edits={sum(order.unjudged_edits for order in self.orders)} " + f"ready={','.join(str(pr.number) for pr in self.ready_prs)} " + f"draft={','.join(str(pr.number) for pr in self.draft_prs)}" + ) + + +@dataclass(frozen=True) +class CommanderPhase: + """PlanPhase supplies every answer; no HUD status taxonomy is maintained beside it.""" + + phase: type[PlanPhase] + done: bool + ready: bool + holders: tuple[ThreadIncarnation, ...] + + def line(self) -> str: + return ( + f"{self.phase.plan.declared_name}/{self.phase.declared_name} " + f"done={self.done} ready={self.ready} " + f"holders={','.join(unit.name for unit in self.holders)}" + ) + + +@dataclass(frozen=True) +class CommanderHud(ToolResult): + """One disposable view of native records, shared by comms_hud and the commander board. + + TodoStore owns orders, review timers and strikes; OrderEditCountRead owns edit counts; + PullRequests owns ready/draft facts; PlanPhase owns phase state; OverloadDegradation + owns the load formula and decision. Nothing here writes or retains a board. + """ + + commander: ThreadIncarnation + units: tuple[CommanderUnit, ...] + phases: tuple[CommanderPhase, ...] + load: OverloadDegradation + + @classmethod + def read(cls, comms: Comms, commander: str) -> CommanderHud: + snapshot, now = comms.registry.snapshot(), time.time() + owner = snapshot.require(commander).incarnation + store = comms.goals.todos + history = store.history_for_commander(owner) + held = tuple(todo for todo in history if todo.holder) + pulls = ProgressRead(held) + progress = {todo.id: OrderProgress.read(todo, comms.registry, pulls) for todo in held} + timers = store.timers_up_in(held, now, progress) + edits = { + read.todo.id: read.read() + for read in OrderEditCountRead.for_orders(comms.root, snapshot, held) + } + grouped: dict[ThreadIncarnation, list[Todo]] = {} + for todo in held: + grouped.setdefault(todo.held().owner_identity, []).append(todo) + units = [] + for unit, orders in grouped.items(): + requests = { + (todo.repo, request.number): request + for todo in orders + for request in pulls.listed(todo.repo, UnmergedListedPullRequest, todo) + } + units.append(CommanderUnit( + unit, + tuple(CommanderOrder( + todo.id, todo.text, todo.order.checked().id, todo.order.line, progress[todo.id], + tuple(at for due, at in timers if due.id == todo.id), + todo.strike_count, edits[todo.id], + ) for todo in orders), + sum(len(todo.live_strikes(unit.name)) for todo in history), + tuple(request for request in requests.values() if not request.draft), + tuple(request for request in requests.values() if request.draft), + )) + reading = CommanderConfig.read_root(comms.root).plan_reading(history) + phases = tuple(CommanderPhase( + phase, phase.done(reading), phase.ready(reading), + tuple(dict.fromkeys(todo.held().owner_identity for todo in phase.holding(reading))), + ) for phase in PlanPhase.members_with(PlanPhase)) + return cls(owner, tuple(units), phases, + OverloadDegradation.from_orders(held, len(timers), sum(edits.values()))) + + @projected(view="wire") + def frame(self) -> tuple[str, ...]: + """One line per declared phase or holder, with their original order and PR identities.""" + return ( + str(self.load), + *(row.line() for row in self.phases), + *(row.line() for row in self.units), + ) + + diff --git a/src/agent_comms/commander_parity/__init__.py b/src/agent_comms/commander_parity/__init__.py index 610372fdef..528ebf57e5 100644 --- a/src/agent_comms/commander_parity/__init__.py +++ b/src/agent_comms/commander_parity/__init__.py @@ -1,12 +1,12 @@ -"""Parity declarations own matrix generation and the cutover decision. +"""The historical capability matrix and optional native workflow diagnostics. -Delete this transition package when both machines run the native commander. +The declared native phases own completion; this transition package never grants cutover. """ from __future__ import annotations from abc import ABC, abstractmethod -from collections.abc import Callable, Iterable +from collections.abc import Callable from dataclasses import dataclass from pathlib import Path from typing import TYPE_CHECKING, ClassVar @@ -15,7 +15,6 @@ if TYPE_CHECKING: from ..plan_phases import PlanPhase - from ..todos import Todo class ParityComparison(ABC): @@ -48,10 +47,6 @@ class ParityRow( phases: ClassVar[tuple[type[PlanPhase], ...]] comparison: ClassVar[Callable[[], ParityComparison]] = UnverifiedParity - @classmethod - def compare(cls, todos: Iterable[Todo]) -> ParityComparison: - return UnverifiedParity() - @classmethod def verify(cls) -> ParityComparison: """Run this declared workflow now; a run is not durable cutover authority.""" @@ -65,11 +60,6 @@ def rows(cls) -> tuple[type[ParityRow], ...]: raise ValueError("Parity declarations require unique positive row numbers") return rows - @classmethod - def done(cls, todos: Iterable[Todo]) -> bool: - history = tuple(todos) - return all(row.compare(history).passed for row in cls.rows()) - @classmethod def matrix(cls) -> str: """Render declared facts only; no mutable completion column lives here.""" diff --git a/src/agent_comms/commander_parity/measurements.py b/src/agent_comms/commander_parity/measurements.py deleted file mode 100644 index dc2910238b..0000000000 --- a/src/agent_comms/commander_parity/measurements.py +++ /dev/null @@ -1,103 +0,0 @@ -"""Observe measurement owners through the standing-rule delivery they supply.""" - -from __future__ import annotations - -from dataclasses import dataclass -from pathlib import Path -from sys import monitoring -from types import CodeType, MethodType - -from agent_comms.commander.config import CommanderConfig -from agent_comms.commander_parity import ParityComparison, UnverifiedParity -from agent_comms.comms import Comms -from agent_comms.private_path import FileRevision -from agent_comms.standing_rules import DiskLow, StandingRule -from agent_comms.storage_rules import StorageViolation -from agent_comms.todos import ProgressRead -from agent_comms.ui_model.unit_state import UnitState - - -@dataclass(frozen=True) -class RuleComparison(ParityComparison): - owner: str - current: tuple[str, ...] - native: tuple[str, ...] - calls: int - private_state_unchanged: bool - - @staticmethod - def state_files(directory: Path) -> tuple[tuple[str, FileRevision], ...]: - return tuple( - (str(path.relative_to(directory)), FileRevision.from_stat(path.stat())) - for path in sorted(directory.rglob("*")) - if path.is_file() - ) - - @property - def passed(self) -> bool: - return self.calls == 1 and self.current == self.native and self.private_state_unchanged - - def describe(self) -> str: - return ( - f"owner={self.owner}; native_calls={self.calls}; " - f"current_count={len(self.current)}; native_count={len(self.native)}; " - f"equal={self.current == self.native}; " - f"private_state_unchanged={self.private_state_unchanged}" - ) - - @classmethod - def read(cls, owner: type[StandingRule]) -> ParityComparison: - method = owner.scan_observed - if not isinstance(method, MethodType): - return UnverifiedParity( - f"{owner.__qualname__}.scan_observed is not a bound classmethod" - ) - code = method.__func__.__code__ - config = CommanderConfig.read(CommanderConfig.path()) - comms = Comms(config.root) - states = UnitState.current(comms, config.commander, ProgressRead()) - before = cls.state_files(config.state) - observed: list[tuple[StandingRule, ...]] = [] - - def observe(_code: CodeType, _offset: int, value: tuple[StandingRule, ...]) -> None: - observed.append(value) - - # Subscribe to the owner's real return event, not string-dispatched profile traffic. - monitoring.use_tool_id(monitoring.PROFILER_ID, __name__) - try: - monitoring.register_callback( - monitoring.PROFILER_ID, monitoring.events.PY_RETURN, observe - ) - monitoring.set_local_events(monitoring.PROFILER_ID, code, monitoring.events.PY_RETURN) - current = owner.broken(comms, config.commander, observed=states) - finally: - monitoring.free_tool_id(monitoring.PROFILER_ID) - if len(observed) != 1: - return UnverifiedParity( - f"Expected one {owner.__qualname__}.scan_observed call: {len(observed)}" - ) - native = tuple(rule for rule in observed[0] if rule.broke_after(0.0)) - return cls( - f"{owner.__module__}.{owner.__qualname__}", - tuple(rule.line() for rule in current), - tuple(rule.line() for rule in native), - len(observed), - before == cls.state_files(config.state), - ) - - -@dataclass(frozen=True) -class StorageComparison(ParityComparison): - units: ParityComparison - disk: ParityComparison - - @property - def passed(self) -> bool: - return self.units.passed and self.disk.passed - - def describe(self) -> str: - return f"units: {self.units.describe()}; disk: {self.disk.describe()}" - - @classmethod - def read(cls) -> StorageComparison: - return cls(RuleComparison.read(StorageViolation.scan_owner()), RuleComparison.read(DiskLow)) diff --git a/src/agent_comms/commander_parity/rows.py b/src/agent_comms/commander_parity/rows.py index bb7d9b6448..80c9c4e443 100644 --- a/src/agent_comms/commander_parity/rows.py +++ b/src/agent_comms/commander_parity/rows.py @@ -11,14 +11,12 @@ StrikeLimitCase, verify_order_progress, ) -from agent_comms.commander_parity.measurements import StorageComparison from agent_comms.commander_parity.story import StoryCase from agent_comms.commander_parity.verdicts import VerdictComparison from agent_comms.commander_parity.wording import ( ApplicationRunComparison, DecisionAlertsComparison, OrderWordingComparison, - PlainWordsComparison, UnitWordingComparison, ) from agent_comms.plan_phases import ( @@ -112,10 +110,6 @@ class PlainWordsParityRow(ParityRow): native = "`PlainText`, `StrikeReason`" phases = () - @classmethod - def verify(cls) -> ParityComparison: - return PlainWordsComparison.read() - class OrderWordingParityRow(ParityRow): number = 10 @@ -199,10 +193,6 @@ class StorageHygieneParityRow(ParityRow): native = "`StorageViolation.scan_observed`, `DiskLow.violations`; live `StorageRule` thresholds" phases = (ControlPlanePhase,) - @classmethod - def verify(cls) -> ParityComparison: - return StorageComparison.read() - class OrderOnlyContactParityRow(ParityRow): number = 20 diff --git a/src/agent_comms/commander_parity/wording.py b/src/agent_comms/commander_parity/wording.py index dec85ab7ad..22d020605b 100644 --- a/src/agent_comms/commander_parity/wording.py +++ b/src/agent_comms/commander_parity/wording.py @@ -5,7 +5,7 @@ import io import json import os -from collections.abc import Callable, Iterable +from collections.abc import Callable from contextlib import redirect_stderr, redirect_stdout from dataclasses import dataclass, replace from functools import partial @@ -15,7 +15,7 @@ from agent_comms.child_process import ProcessIdentity from agent_comms.commander.config import CommanderConfig -from agent_comms.commander.order import CommanderContext, OrderFile, refuse +from agent_comms.commander.order import CommanderContext, OrderFile from agent_comms.commander_parity import ParityComparison, UnverifiedParity from agent_comms.commander_parity.lifecycle import ( InstalledEvidenceCase, @@ -75,62 +75,23 @@ def messages(context: CommanderContext) -> tuple[Message, ...]: @dataclass(frozen=True) -class WordingComparison(ParityComparison): - observations: tuple[tuple[WordingOutcome, WordingOutcome], ...] - native_owner: ClassVar[type[OrderWords]] +class OrderWordingComparison(ParityComparison): + """Compare distinct CLI and native request boundaries, never one constructor with itself.""" - @staticmethod - def agrees(current: WordingOutcome, native: WordingOutcome) -> bool: - return current == native + observations: tuple[tuple[WordingOutcome, WordingOutcome], ...] + native_owner: ClassVar[type[OrderWords]] = OrderText @property def passed(self) -> bool: - return bool(self.observations) and all(self.agrees(a, b) for a, b in self.observations) + return bool(self.observations) and all( + current.accepted == native.accepted for current, native in self.observations + ) def describe(self) -> str: return ( f"observed={len(self.observations)}; " - f"mismatches={sum(not self.agrees(a, b) for a, b in self.observations)}; " - f"current_refusals={sum(not a.accepted for a, _ in self.observations)}; " - f"native_refusals={sum(not b.accepted for _, b in self.observations)}; " - f"native={self.native_owner.__module__}.{self.native_owner.__qualname__}; " - "compared actual refusal and warning results; no publication" - ) - - @classmethod - def compare_texts(cls, texts: Iterable[str]) -> ParityComparison: - observed = tuple( - ( - WordingOutcome.observe(partial(refuse, text, cls.native_owner)), - WordingOutcome.observe(partial(cls.native_owner, text)), - ) - for text in texts - ) - return cls(observed) if observed else UnverifiedParity("No actual wording input exists.") - - -class PlainWordsComparison(WordingComparison): - native_owner = PlainText - - @classmethod - def read(cls) -> ParityComparison: - return cls.compare_texts( - strike.reason for todo in history(selected_context()) for strike in todo.strikes - ) - - -class OrderWordingComparison(WordingComparison): - native_owner = OrderText - - @staticmethod - def agrees(current: WordingOutcome, native: WordingOutcome) -> bool: - # This row asks whether soft orders are refused, not whether surrounding - # CLI prose emits the same unrelated wording notes as the native request. - return current.accepted == native.accepted - - def describe(self) -> str: - return super().describe() + ( - f"; current={OrderFile.problems_found.__qualname__}; " + f"mismatches={sum(a.accepted != b.accepted for a, b in self.observations)}; " + f"current={OrderFile.problems_found.__qualname__}; " f"native_entry={OrderGoalAction.require_ready.__qualname__}" ) diff --git a/src/agent_comms/connector.py b/src/agent_comms/connector.py index 36039f635b..ab60ea614f 100644 --- a/src/agent_comms/connector.py +++ b/src/agent_comms/connector.py @@ -60,6 +60,7 @@ AuthorizationRequest, ClaudeCallback, ConnectorGrants, ConnectorStore, IssuedTokens, RegisteredClient, TrustedBrowser, digest, ) +from .locked_store import DocumentChange from .messages import QuestionKind from .commander.standing import UnownedOrderWatch from .commander_heartbeat import CommanderHeartbeat @@ -208,7 +209,7 @@ def received(cls, form: Mapping[str, object]) -> Self: def redeemed(self, grants: ConnectorGrants) -> ConnectorGrants: """The grants with this credential spent; refuses a credential it does not hold.""" - def exchange(self, grants: ConnectorGrants) -> tuple[ConnectorGrants, IssuedTokens]: + def exchange(self, grants: ConnectorGrants) -> DocumentChange[ConnectorGrants, IssuedTokens]: return self.redeemed(grants).issued(self.client_id) @@ -666,10 +667,10 @@ async def register(self, request: web.Request) -> web.Response: client_id = secrets.token_urlsafe(16) await asyncio.to_thread( self.store.update, - lambda grants: replace( + lambda grants: DocumentChange.to(replace( grants, clients=grants.clients | {client_id: RegisteredClient(registration.client_name)}, - ), + )), ) return web.json_response(FieldCodec.encode(ClientRegistrationReply( client_id=client_id, client_id_issued_at=int(time.time()), @@ -688,17 +689,17 @@ async def authorize(self, request: web.Request) -> web.Response: browser = request.cookies.get(TrustedBrowser.cookie_name, "") notices: list[AuthorizationRequest] = [] - def decided(grants: ConnectorGrants) -> ConnectorGrants: + def decided(grants: ConnectorGrants) -> DocumentChange[ConnectorGrants, ConnectorGrants]: now = time.time() grants = grants.unexpired(now) if query.client_id not in grants.clients: raise web.HTTPBadRequest(text="Unknown client.") if grants.trusts(browser): - return grants.code_issued(query, code, now) + return DocumentChange.to(grants.code_issued(query, code, now)) if query.request_id not in grants.requests: grants = grants.waiting(query, digest(approval), now) notices.append(grants.requests[query.request_id]) - return grants + return DocumentChange.to(grants) try: grants = await asyncio.to_thread(self.store.update, decided) @@ -763,23 +764,21 @@ async def confirm(self, request: web.Request) -> web.Response: """Consume the owner's POST and establish browser trust in the same update.""" link = FieldCodec.decode(ApprovalLink, dict(request.match_info)) code, browser = secrets.token_urlsafe(32), secrets.token_urlsafe(32) - approved: list[AuthorizationQuery] = [] - def consumed(grants: ConnectorGrants) -> ConnectorGrants: + def consumed(grants: ConnectorGrants) -> DocumentChange[ConnectorGrants, AuthorizationQuery]: now = time.time() held = grants.require_approval(link.request_id, link.secret, now) - approved.append(held.query) - return replace( + return DocumentChange(replace( grants.unexpired(now).code_issued(held.query, code, now), trusted_browsers=grants.trusted_browsers | {digest(browser): TrustedBrowser(now)}, - ) + ), held.query) try: - await asyncio.to_thread(self.store.update, consumed) + approved = await asyncio.to_thread(self.store.update, consumed) except ApprovalRefused as refusal: raise web.HTTPBadRequest(text=str(refusal)) from refusal response = web.HTTPFound( - approved[0].redirect(code), + approved.redirect(code), headers={"Cache-Control": "no-store", "Referrer-Policy": "no-referrer"}, ) response.set_cookie( @@ -793,19 +792,12 @@ async def token(self, request: web.Request) -> web.Response: grant = TokenGrant.received(await request.post()) except MalformedRecord as error: return OAuthError("invalid_request", str(error)).response() - exchanged: list[IssuedTokens] = [] - - def spent(grants: ConnectorGrants) -> ConnectorGrants: - grants, response = grant.exchange(grants) - exchanged.append(response) - return grants - try: - await asyncio.to_thread(self.store.update, spent) + exchanged = await asyncio.to_thread(self.store.update, grant.exchange) except OAuthError as refusal: return refusal.response() return web.json_response( - FieldCodec.encode(exchanged[0]), headers={"Cache-Control": "no-store"} + FieldCodec.encode(exchanged), headers={"Cache-Control": "no-store"} ) @staticmethod @@ -918,9 +910,9 @@ class RevokeVerb(ConnectorVerb): help = "Revoke every issued token, code and trusted browser" def apply(self, ctx: ConnectorStore) -> int | str: - ctx.update(lambda grants: replace( + ctx.update(lambda grants: DocumentChange.to(replace( grants, requests={}, codes={}, access={}, refresh={}, trusted_browsers={} - )) + ))) return 0 diff --git a/src/agent_comms/connector_store.py b/src/agent_comms/connector_store.py index c447e29e6f..676874a2ac 100644 --- a/src/agent_comms/connector_store.py +++ b/src/agent_comms/connector_store.py @@ -13,6 +13,7 @@ from urllib.parse import urlencode from .field_codec import FieldCodec +from .locked_store import DocumentChange from .maintenance_barrier import CarriedDocument ACCESS_SECONDS = 3600 @@ -162,7 +163,7 @@ def code_issued(self, query: AuthorizationQuery, code: str, now: float) -> Conne }, ) - def issued(self, client_id: str) -> tuple[ConnectorGrants, IssuedTokens]: + def issued(self, client_id: str) -> DocumentChange[ConnectorGrants, IssuedTokens]: """A new access and refresh token pair, dropping every expired grant and wait.""" now = time.time() access, refresh = secrets.token_urlsafe(32), secrets.token_urlsafe(32) @@ -173,7 +174,7 @@ def issued(self, client_id: str) -> tuple[ConnectorGrants, IssuedTokens]: | {digest(access): IssuedAccess(client_id, expires_at=now + ACCESS_SECONDS)}, refresh=current.refresh | {digest(refresh): client_id}, ) - return grants, IssuedTokens(access, refresh) + return DocumentChange(grants, IssuedTokens(access, refresh)) def holds(self, token: str) -> bool: key = digest(token) @@ -201,12 +202,12 @@ def granted(self, client_id: str) -> Iterator[str]: token = secrets.token_urlsafe(32) held = IssuedAccess(client_id, expires_at=time.time() + CODE_SECONDS) key = digest(token) - self.update(lambda grants: replace(grants, access=grants.access | {key: held})) + self.update(lambda grants: DocumentChange.to(replace(grants, access=grants.access | {key: held}))) try: yield token finally: self.update( - lambda grants: replace( + lambda grants: DocumentChange.to(replace( grants, access={k: v for k, v in grants.access.items() if k != key} - ) + )) ) diff --git a/src/agent_comms/coordination_response.py b/src/agent_comms/coordination_response.py index 2da5b96167..dbc9d73be4 100644 --- a/src/agent_comms/coordination_response.py +++ b/src/agent_comms/coordination_response.py @@ -80,8 +80,8 @@ class PublicationAppendDispatches(ResponseTable, TypedTable): @classmethod def references(cls): - return (ForeignKey(("execution_id", "exact_target"), PublicationIntents, - ("execution_id", "exact_target")),) + return (ForeignKey(columns=("execution_id", "exact_target"), target=PublicationIntents, + target_columns=("execution_id", "exact_target")),) def matches(self, wire_root_id: str, fence: OwnerFence, exact_target: str) -> bool: return self.matches_attempt( diff --git a/src/agent_comms/coordination_snapshot.py b/src/agent_comms/coordination_snapshot.py index a9a8ca174e..f8ac5189f7 100644 --- a/src/agent_comms/coordination_snapshot.py +++ b/src/agent_comms/coordination_snapshot.py @@ -33,7 +33,7 @@ ReplayAssessments, ) from agent_comms.coordination_tables.publications import PublicationEvidence, PublicationIntents -from agent_comms.coordination_tables.recovery import ConnectivityFacet, RecoveryAudit +from agent_comms.coordination_tables.recovery import RecoveryAudit from agent_comms.coordination_tables.responses import ResponseObligation from agent_comms.field_codec import boundary_decoder, projected, SnapshotName, WireName from agent_comms.reason_codes import ReasonCode @@ -59,7 +59,6 @@ class RecoverySnapshot: publication_evidence: Annotated[ tuple[PublicationEvidence, ...], WireName("publication_receipts") ] - connectivity: ConnectivityFacet | None last_recovery: RecoveryAudit | None current_execution_id: PointerExecution current_attempt_ordinal: PointerAttempt @@ -150,7 +149,6 @@ def validate_membership(self) -> None: *self.obligations, *self.publication_intents, *self.publication_evidence, - self.connectivity, self.last_recovery, ) if any( diff --git a/src/agent_comms/coordination_tables/assignments.py b/src/agent_comms/coordination_tables/assignments.py index ef2de34189..cc25d84dda 100644 --- a/src/agent_comms/coordination_tables/assignments.py +++ b/src/agent_comms/coordination_tables/assignments.py @@ -251,22 +251,22 @@ def references(cls): from agent_comms.coordination_tables.responses import ResponseObligation return ( - ForeignKey(("recipient_lookup",), Participants, ("participant_lookup",)), - ForeignKey(("execution_id",), ExecutionRecord, ("execution_id",)), + ForeignKey(columns=("recipient_lookup",), target=Participants, target_columns=("participant_lookup",)), + ForeignKey(columns=("execution_id",), target=ExecutionRecord, target_columns=("execution_id",)), ForeignKey( - ("execution_id", "exact_target"), ResponseObligation, - ("execution_id", "exact_target"), deferred=True, + columns=("execution_id", "exact_target"), target=ResponseObligation, + target_columns=("execution_id", "exact_target"), deferred=True, ), ForeignKey( - ("execution_id", "assignment_id"), - ExecutionAssignmentLink, - ("execution_id", "assignment_id"), + columns=("execution_id", "assignment_id"), + target=ExecutionAssignmentLink, + target_columns=("execution_id", "assignment_id"), deferred=True, ), ForeignKey( - ("execution_id", "claim_status_kind"), - ExecutionRecord, - ("execution_id", "claim_status_kind"), + columns=("execution_id", "claim_status_kind"), + target=ExecutionRecord, + target_columns=("execution_id", "claim_status_kind"), deferred=True, ), ) @@ -301,8 +301,8 @@ def references(cls): from agent_comms.coordination_tables.executions import ExecutionRecord return ( - ForeignKey(("execution_id",), ExecutionRecord, ("execution_id",), on_delete="RESTRICT"), + ForeignKey(columns=("execution_id",), target=ExecutionRecord, target_columns=("execution_id",), on_delete="RESTRICT"), ForeignKey( - ("assignment_id", "execution_id"), WakeAssignment, ("assignment_id", "execution_id") + columns=("assignment_id", "execution_id"), target=WakeAssignment, target_columns=("assignment_id", "execution_id") ), ) diff --git a/src/agent_comms/coordination_tables/attempts.py b/src/agent_comms/coordination_tables/attempts.py index b2830e39a5..ea9d128548 100644 --- a/src/agent_comms/coordination_tables/attempts.py +++ b/src/agent_comms/coordination_tables/attempts.py @@ -97,7 +97,7 @@ class AttemptRecord( declared_name="attempts", ): counter_reference = ForeignKey( - ("owner_lookup", "owner_generation"), OwnerGenerations, ("owner_lookup", "generation") + columns=("owner_lookup", "owner_generation"), target=OwnerGenerations, target_columns=("owner_lookup", "generation") ) column_orders = ( NonDecreasing(("updated_at_ms", "last_progress_at_ms", "backend_done", "process_dead")), @@ -289,12 +289,12 @@ def references(cls): from agent_comms.coordination_tables.participants import Participants return ( - ForeignKey(("execution_id",), ExecutionRecord, ("execution_id",)), - ForeignKey(("owner_lookup",), Participants, ("participant_lookup",)), + ForeignKey(columns=("execution_id",), target=ExecutionRecord, target_columns=("execution_id",)), + ForeignKey(columns=("owner_lookup",), target=Participants, target_columns=("participant_lookup",)), ForeignKey( - ("execution_id", "attempt_ordinal", "owner_lookup", "active_required_status"), - ExecutionRecord, - ("execution_id", "current_attempt_ordinal", "owner_lookup", "status"), + columns=("execution_id", "attempt_ordinal", "owner_lookup", "active_required_status"), + target=ExecutionRecord, + target_columns=("execution_id", "current_attempt_ordinal", "owner_lookup", "status"), deferred=True, ), ) diff --git a/src/agent_comms/coordination_tables/executions.py b/src/agent_comms/coordination_tables/executions.py index 6b51f79a58..d5278b2e67 100644 --- a/src/agent_comms/coordination_tables/executions.py +++ b/src/agent_comms/coordination_tables/executions.py @@ -159,17 +159,17 @@ def references(cls): from agent_comms.coordination_tables.participants import Participants return ( - ForeignKey(("owner_lookup",), Participants, ("participant_lookup",)), + ForeignKey(columns=("owner_lookup",), target=Participants, target_columns=("participant_lookup",)), ForeignKey( - ("execution_id", "attempt_ordinal", "owner_lookup", "required_active"), - ExecutionRecord, - ("execution_id", "current_attempt_ordinal", "owner_lookup", "status"), + columns=("execution_id", "attempt_ordinal", "owner_lookup", "required_active"), + target=ExecutionRecord, + target_columns=("execution_id", "current_attempt_ordinal", "owner_lookup", "status"), deferred=True, ), ForeignKey( - ("execution_id", "attempt_ordinal", "owner_lookup", "required_active"), - AttemptRecord, - ("execution_id", "attempt_ordinal", "owner_lookup", "phase_kind"), + columns=("execution_id", "attempt_ordinal", "owner_lookup", "required_active"), + target=AttemptRecord, + target_columns=("execution_id", "attempt_ordinal", "owner_lookup", "phase_kind"), deferred=True, ), ) @@ -398,34 +398,34 @@ def references(cls): from agent_comms.coordination_tables.participants import Participants return ( - ForeignKey(("owner_lookup",), Participants, ("participant_lookup",)), + ForeignKey(columns=("owner_lookup",), target=Participants, target_columns=("participant_lookup",)), ForeignKey( - ( + columns=( "execution_id", "current_attempt_ordinal", "owner_lookup", "required_attempt_kind", ), - AttemptRecord, - ("execution_id", "attempt_ordinal", "owner_lookup", "phase_kind"), + target=AttemptRecord, + target_columns=("execution_id", "attempt_ordinal", "owner_lookup", "phase_kind"), deferred=True, ), ForeignKey( - ("owner_lookup", "active_execution_id", "active_attempt_ordinal"), - CurrentExecutions, - ("owner_lookup", "execution_id", "attempt_ordinal"), + columns=("owner_lookup", "active_execution_id", "active_attempt_ordinal"), + target=CurrentExecutions, + target_columns=("owner_lookup", "execution_id", "attempt_ordinal"), deferred=True, ), ForeignKey( - ("wire_execution_id", "wire_claim_ordinal"), - ExecutionAssignmentLink, - ("execution_id", "ordinal"), + columns=("wire_execution_id", "wire_claim_ordinal"), + target=ExecutionAssignmentLink, + target_columns=("execution_id", "ordinal"), deferred=True, ), ForeignKey( - ("deferred_replay_id", "deferred_replay_required"), - ReplayAssessments, - ("execution_id", "retry_authorized"), + columns=("deferred_replay_id", "deferred_replay_required"), + target=ReplayAssessments, + target_columns=("execution_id", "retry_authorized"), deferred=True, ), ) @@ -437,7 +437,7 @@ class ExecutionReference: @classmethod def references(cls) -> tuple[ForeignKey, ...]: return ( - ForeignKey(("execution_id",), ExecutionRecord, ("execution_id",), on_delete="RESTRICT"), + ForeignKey(columns=("execution_id",), target=ExecutionRecord, target_columns=("execution_id",), on_delete="RESTRICT"), ) diff --git a/src/agent_comms/coordination_tables/participants.py b/src/agent_comms/coordination_tables/participants.py index cf8ebe55a6..fec59f03ef 100644 --- a/src/agent_comms/coordination_tables/participants.py +++ b/src/agent_comms/coordination_tables/participants.py @@ -64,7 +64,7 @@ class ParticipantAliases(ImmutableRows, CoordinatorRow): @classmethod def references(cls): - return (ForeignKey(("participant_lookup",), Participants, ("participant_lookup",)),) + return (ForeignKey(columns=("participant_lookup",), target=Participants, target_columns=("participant_lookup",)),) @dataclass(frozen=True, kw_only=True) class OwnerGenerations( @@ -86,7 +86,7 @@ def require_recorded_owner(self, thread): @classmethod def references(cls): - return (ForeignKey(("owner_lookup",), Participants, ("participant_lookup",)),) + return (ForeignKey(columns=("owner_lookup",), target=Participants, target_columns=("participant_lookup",)),) @property def lookup(self) -> str: diff --git a/src/agent_comms/coordination_tables/publications.py b/src/agent_comms/coordination_tables/publications.py index abad411e2b..c54dbd01c2 100644 --- a/src/agent_comms/coordination_tables/publications.py +++ b/src/agent_comms/coordination_tables/publications.py @@ -137,11 +137,11 @@ def references(cls): from agent_comms.coordination_tables.responses import ResponseObligation return ( - ForeignKey(("execution_id",), ExecutionRecord, ("execution_id",), on_delete="RESTRICT"), + ForeignKey(columns=("execution_id",), target=ExecutionRecord, target_columns=("execution_id",), on_delete="RESTRICT"), ForeignKey( - ("execution_id", "exact_target", "obligation_intent_required"), - ResponseObligation, - ("execution_id", "exact_target", "intent_settled"), + columns=("execution_id", "exact_target", "obligation_intent_required"), + target=ResponseObligation, + target_columns=("execution_id", "exact_target", "intent_settled"), deferred=True, ), ) @@ -218,14 +218,14 @@ def references(cls): return ( ForeignKey( - ("execution_id", "exact_target"), - PublicationIntents, - ("execution_id", "exact_target"), + columns=("execution_id", "exact_target"), + target=PublicationIntents, + target_columns=("execution_id", "exact_target"), ), ForeignKey( - ("execution_id", "exact_target", "obligation_publication_required"), - ResponseObligation, - ("execution_id", "exact_target", "publication_settled"), + columns=("execution_id", "exact_target", "obligation_publication_required"), + target=ResponseObligation, + target_columns=("execution_id", "exact_target", "publication_settled"), deferred=True, ), ) diff --git a/src/agent_comms/coordination_tables/recovery.py b/src/agent_comms/coordination_tables/recovery.py index 10f2746c08..fb26ea0405 100644 --- a/src/agent_comms/coordination_tables/recovery.py +++ b/src/agent_comms/coordination_tables/recovery.py @@ -4,7 +4,6 @@ from dataclasses import dataclass from dataclasses import field as dataclass_field -from enum import StrEnum from typing import Annotated from agent_comms import field_codec @@ -19,55 +18,14 @@ from agent_comms.typed_table import ( Column, ForeignKey, - FrozenColumn, ImmutableRows, - IncrementingColumn, Index, - MonotoneColumns, - NoDelete, - NonDecreasing, RowId, TextStorage, UnallocatedRowId, ) -class OwnerConnectivity(StrEnum): - CONNECTED = "connected" - RECONNECTING = "reconnecting" - OFFLINE = "offline" - - -class ACPClientConnectivity(StrEnum): - CONNECTED = "connected" - DISCONNECTED = "disconnected" - - -@dataclass(frozen=True, slots=True) -class ConnectivityFacet( - IncrementingColumn, - MonotoneColumns, - ExecutionReference, - NoDelete, - CoordinatorRow, - declared_name="connectivity", -): - - column_orders = (NonDecreasing(('observed_at_ms',)),) - - execution_id: Annotated[str, field_codec.SnapshotExclude(), FrozenColumn(primary_key=True)] - owner: OwnerConnectivity - acp_client: ACPClientConnectivity - revision: Annotated[int, Column(check="revision>0")] - observed_at_ms: Annotated[int, Column(check="observed_at_ms>=0")] - - def __post_init__(self) -> None: - validate_execution_id(self.execution_id) - object.__setattr__(self, "owner", OwnerConnectivity(self.owner)) - object.__setattr__(self, "acp_client", ACPClientConnectivity(self.acp_client)) - if self.revision <= 0 or self.observed_at_ms < 0: - raise ValueError("revision must be positive and observed_at_ms non-negative") - @dataclass(frozen=True, slots=True) class RecoveryAudit(ImmutableRows, ExecutionReference, CoordinatorRow): # Recovery condition is a stored fact; row identity is the table family tag. @@ -102,7 +60,7 @@ def references(cls) -> tuple[ForeignKey, ...]: return super().references() + ( ForeignKey( - ("execution_id", "attempt"), AttemptRecord, ("execution_id", "attempt_ordinal") + columns=("execution_id", "attempt"), target=AttemptRecord, target_columns=("execution_id", "attempt_ordinal") ), ) diff --git a/src/agent_comms/diagnostics.py b/src/agent_comms/diagnostics.py index 4ac6633ffa..6aff801f1a 100644 --- a/src/agent_comms/diagnostics.py +++ b/src/agent_comms/diagnostics.py @@ -7,18 +7,19 @@ import os import time from contextlib import contextmanager +from collections.abc import Mapping from dataclasses import dataclass, field, replace -from enum import StrEnum from pathlib import Path from traceback import TracebackException -from typing import TYPE_CHECKING, Literal +from typing import TYPE_CHECKING from uuid import uuid4 from .child_process import Platform -from .field_codec import FieldCodec, Id128Text, MalformedRecord +from .field_codec import FieldCodec, Id128Text from .locked_store import DurableFile, PatternFile from .store_files import AtomicFile +from .turn_failure import TurnFailure if TYPE_CHECKING: from .activity import DrainDiagnostic @@ -44,20 +45,6 @@ def owner_process_output(root: Path, thread: Thread): yield output -class FailureReason(StrEnum): - BACKEND_FAILED = "backend_failed" - PREFLIGHT_TIMEOUT = "native_preflight_timeout" - PREFLIGHT_EXIT = "native_preflight_exit" - INPUT_ID_UNAVAILABLE = "pi_input_id_unavailable" - IDENTITY_UNCERTAIN = "session_identity_uncertain" - AUTHORITY_CHANGED = "input_authority_changed" - FOLLOWUP_UNRECOGNIZED = "unrecognized_followup_input" - INPUT_MISSING = "current_prompt_input_missing" - MODEL_REQUEST_FAILED = "model_request_failed" - FINAL_STOP_MISSING = "assistant_final_stop_missing" - QUEUED_INPUT_MISSING = "queued_input_start_missing" - - @dataclass class PublicationMeasurements: """Bounded transport timing counters, not a message or phase authority.""" @@ -134,34 +121,20 @@ def record(cls, root, lease, record): output.write(json.dumps(record) + "\n") -def terminal_failure_reason(event: dict) -> FailureReason: - """Allowlisted backend terminal classification; never diagnostic prose.""" - measurements = event.get("diagnostic", {}) - if not isinstance(measurements, dict): - measurements = {} - try: - reason = measurements.get("reason") or event.get("reason_code") - return FieldCodec.decode(FailureReason, reason) - except MalformedRecord: - return FailureReason.BACKEND_FAILED - - def record_terminal_failure( root: Path, *, turn_id: str, thread: str, - event: dict, + failure: type[TurnFailure], sequences: tuple[int, ...], + measurements: Mapping[str, object] | None = None, native_response: Response | None = None, source_error: BaseException | None = None, ) -> Path: """Persist before publishing the failure notice; this record grants no retry authority.""" Id128Text.decode(turn_id) - measurements = event.get("diagnostic", {}) - if not isinstance(measurements, dict): - measurements = {} - reason = terminal_failure_reason(event) + measurements = measurements or {} safe = { key: value for key in ( @@ -177,7 +150,7 @@ def record_terminal_failure( "version": 1, "turn_id": turn_id, "thread": thread, - "reason": reason.value, + "failure": failure.declared_name, "sequences": list(sequences), "measurements": safe, "outcome": "failed; inputs must not be replayed automatically", @@ -188,40 +161,6 @@ def record_terminal_failure( return DiagnosticDocument.write(root, turn_id, document) -@dataclass(frozen=True) -class NativeTerminalFailureRecord: - """The document ``record_terminal_failure`` writes for a native terminal failure. - - Core writes it only after verifying the failed input's live context. - """ - - version: Literal[1] - turn_id: str - thread: str - reason: FailureReason - sequences: tuple[int, ...] - measurements: dict[str, int] - outcome: str - source_error: str - native: dict[str, str] - - @classmethod - def read(cls, root: Path, input_id: str) -> NativeTerminalFailureRecord: - path = DiagnosticDocument.path_in(root, name=input_id) - record = FieldCodec.decode(cls, json.loads(path.read_text())) - if record.turn_id != input_id or not record.raised.startswith( - "agent_comms.native_pi.NativePiTerminalFailure: " - ): - raise ValueError("Diagnostic does not record a native terminal failure for this input") - return record - - @property - def raised(self) -> str: - """The first unindented line of the last traceback is the raised exception.""" - final = self.source_error.rpartition("Traceback (most recent call last):\n")[2] - return next((line for line in final.splitlines() if not line.startswith(" ")), "") - - def record_drain_failure( root: Path, *, thread: str, diagnostic: DrainDiagnostic, source_error: BaseException, ) -> Path: diff --git a/src/agent_comms/external_transcript.py b/src/agent_comms/external_transcript.py index fba350ed2d..4821789403 100644 --- a/src/agent_comms/external_transcript.py +++ b/src/agent_comms/external_transcript.py @@ -223,7 +223,7 @@ class ExternalEventRow(TypedTable): ) foreign_keys = ( - ForeignKey(("thread", "seq"), ExternalItemRow, ("thread", "seq"), on_delete="CASCADE"), + ForeignKey(columns=("thread", "seq"), target=ExternalItemRow, target_columns=("thread", "seq"), on_delete="CASCADE"), ) indexes = ( Index(("thread", "kind", "tool", "timestamp", "call_id")), diff --git a/src/agent_comms/goal_attempts.py b/src/agent_comms/goal_attempts.py index aff5467abc..9341f10c50 100644 --- a/src/agent_comms/goal_attempts.py +++ b/src/agent_comms/goal_attempts.py @@ -63,7 +63,7 @@ from .pi_counts import PiCount, UnreportedCount from .pi_payloads import PiUsage from .store_files import AtomicFile -from .typed_table import Column, Generated, SQLiteJournalMode, TypedTable +from .typed_table import Column, Generated, SQLiteJournalMode, TableReference, TypedTable if TYPE_CHECKING: from .goal_management import Goals @@ -281,7 +281,7 @@ class AttemptRecord(GoalLedgerTable, TypedTable): str, Column( generated="json_extract(reservation,'$.goal_id')", - references=(Generation, "goal_id"), + references=TableReference(Generation, ("goal_id",)), nullable=True, ), ) @@ -438,7 +438,7 @@ def retire( @dataclass(frozen=True) class GoalWorkDecision(GoalLedgerTable, TypedTable): """Explicit new-work authority, named by this declaration.""" - goal_id: Annotated[str, Column(primary_key=True, references=(Generation, "goal_id"))] + goal_id: Annotated[str, Column(primary_key=True, references=TableReference(Generation, ("goal_id",)))] decision_id: Annotated[str, Column(primary_key=True)] generation: int @@ -452,7 +452,7 @@ def claim(self, conn: sqlite3.Connection) -> None: @dataclass(frozen=True) class GoalProviderUsage(GoalLedgerTable, TypedTable): attempt_id: Annotated[ - str, Column(primary_key=True, references=(AttemptRecord, "attempt_id")) + str, Column(primary_key=True, references=TableReference(AttemptRecord, ("attempt_id",))) ] response_id: Annotated[str, Column(primary_key=True)] # The canonical opaque provider object remains exact; no second native usage codec. diff --git a/src/agent_comms/goal_failure_observation.py b/src/agent_comms/goal_failure_observation.py index 035bd0bdcc..03f26f02d2 100644 --- a/src/agent_comms/goal_failure_observation.py +++ b/src/agent_comms/goal_failure_observation.py @@ -12,7 +12,7 @@ from dataclasses import dataclass, field from typing import TYPE_CHECKING, Annotated -from .diagnostics import FailureReason +from .turn_failure import TurnFailure from .field_codec import Id128Text, WireName from .goal_attempt_identity import GoalAttemptIdentity from .thread_identity import ( @@ -62,7 +62,7 @@ class FailedTurnEvidence(GoalLedgerTable, TypedTable, GoalFailureEvidence): turn_generation: Annotated[int, Column(check="turn_generation>0")] goal_revision: Annotated[int, Column(check="goal_revision>=0")] turn_id: Annotated[str, Column(unique=True, check="length(turn_id)=32")] - reason: FailureReason + failure: type[TurnFailure] def __post_init__(self) -> None: # These are the durable observation's constraints, independent of any @@ -74,8 +74,6 @@ def __post_init__(self) -> None: if not math.isfinite(self.owner_created_at): raise ValueError("Observation incarnation must be finite") Id128Text.from_text(self.turn_id) - if not isinstance(self.reason, FailureReason): - raise ValueError("Observation reason must be decoded") def execution(self, goal_id: str) -> GoalExecution: from .goal_presentation import AttemptFailedExecution @@ -112,7 +110,7 @@ def matches_owner(self, owner: Thread, claim_scope: int | None) -> bool: def references(cls): from .goal_attempts import AttemptRecord - return (ForeignKey(("attempt_id",), AttemptRecord, ("attempt_id",)),) + return (ForeignKey(columns=("attempt_id",), target=AttemptRecord, target_columns=("attempt_id",)),) @dataclass(frozen=True) @@ -132,7 +130,7 @@ def from_terminal( claim_scope: int, current_owner: Thread | None, current_claim: int | None, - reason: FailureReason, + failure: type[TurnFailure], ) -> FailedTurnObservation | None: # Observation is passive: a changed owner, goal or lease yields no # observation. The evidence is built from already-trusted values, so a @@ -140,7 +138,7 @@ def from_terminal( evidence = FailedTurnEvidence( reservation.attempt_id, reservation.goal_id, reservation.generation, owner.name, owner.created_at, owner.worktree, claim_scope, - lease.identity.generation, goal.revision, turn_id, reason, + lease.identity.generation, goal.revision, turn_id, failure, ) if owner.goal != goal or reservation.goal_id != goal.id: return None diff --git a/src/agent_comms/goal_presentation.py b/src/agent_comms/goal_presentation.py index 9dbb269f5f..13426e100d 100644 --- a/src/agent_comms/goal_presentation.py +++ b/src/agent_comms/goal_presentation.py @@ -269,7 +269,7 @@ class AttemptFailedExecution(BlockedExecution, RetryableExecution): @property def reason(self) -> str: - return self.evidence.reason.value + return self.evidence.failure.declared_name class GoalPublicationSnapshot(ABC): diff --git a/src/agent_comms/goal_waits.py b/src/agent_comms/goal_waits.py index 8f131f70c0..7ff14d348c 100644 --- a/src/agent_comms/goal_waits.py +++ b/src/agent_comms/goal_waits.py @@ -382,7 +382,7 @@ class GoalWaitTargetRow(TypedTable): @classmethod def references(cls) -> tuple[ForeignKey, ...]: - return (ForeignKey(("goal_id",), StoredGoalWaitRow, ("goal_id",), on_delete="CASCADE"),) + return (ForeignKey(columns=("goal_id",), target=StoredGoalWaitRow, target_columns=("goal_id",), on_delete="CASCADE"),) @dataclass(frozen=True, slots=True) diff --git a/src/agent_comms/input_disposition.py b/src/agent_comms/input_disposition.py index 432603f1da..75e54bd03d 100644 --- a/src/agent_comms/input_disposition.py +++ b/src/agent_comms/input_disposition.py @@ -3,6 +3,7 @@ from __future__ import annotations import sqlite3 +from collections.abc import Callable from contextlib import ExitStack from dataclasses import dataclass, field, replace from pathlib import Path @@ -26,6 +27,7 @@ ) from .input_origin import InputOrigin, UnattributedInputOrigin from .lifecycle_capability import Unresolved +from .locked_store import DocumentChange from .message_reference import MessageReference from .messages import Message from .threads import Thread @@ -235,15 +237,18 @@ def _select(self, where: str, parameters: tuple = ()) -> InputDocument: with self.snapshot() as connection: return self._document(connection, where, parameters) - def _update(self, where: str, parameters: tuple, change) -> InputDocument: + def _update[R]( + self, where: str, parameters: tuple, + change: Callable[[InputDocument], DocumentChange[InputDocument, R]], + ) -> R: """Change only the selected rows; the callback returns the original to skip writing.""" with self.writing() as connection: original = self._document(connection, where, parameters) changed = change(original) - for key, row in changed.rows.items(): + for key, row in changed.document.rows.items(): if original.rows.get(key) is not row: InputDispositionRow(key, row).upsert(connection) - return changed + return changed.result def read(self, keys: tuple[str, ...]) -> InputDocument: return self._select(*_members("key", keys)) @@ -277,14 +282,14 @@ def record_originals( self, *originals: ReservedInput, ownership: ExitStack | None = None, ) -> InputDocument: """Publish one original cohort; return its exact captured document.""" - def reserve(original: InputDocument) -> InputDocument: + def reserve(original: InputDocument) -> DocumentChange[InputDocument, InputDocument]: changed = original.record(*originals) if changed is not original and ownership is not None: ownership.callback( self.settle_unbound, tuple(key for key in changed.rows if key not in original.rows), ) - return changed + return DocumentChange.to(changed) return self._update(*_members("key", (row.key for row in originals)), reserve) @@ -309,21 +314,18 @@ def reserve_originals( return document def _transition(self, key: str, change) -> bool: - changed = False - - def update(document: InputDocument) -> InputDocument: - nonlocal changed + def update(document: InputDocument) -> DocumentChange[InputDocument, bool]: successor = document.transition((key,), change) - changed = successor is not document - return successor + return DocumentChange(successor, successor is not document) - self._update(*_members("key", (key,)), update) - return changed + return self._update(*_members("key", (key,)), update) def clear_unbound(self, keys: tuple[str, ...]) -> InputDocument: return self._update( *_members("key", keys), - lambda document: document.transition(keys, lambda row: row.clear_unbound()), + lambda document: DocumentChange.to( + document.transition(keys, lambda row: row.clear_unbound()) + ), ) def clear_native(self, key: str, native_id: str, text: str) -> None: @@ -361,22 +363,17 @@ def bind_originals( self, keys: tuple[str, ...], *, claim_scope: int, turn_id: str, native_id: str, text: str ) -> bool: """Bind the complete captured original atomically; no partial batch grant.""" - bound = False - - def change(document: InputDocument) -> InputDocument: - nonlocal bound + def change(document: InputDocument) -> DocumentChange[InputDocument, bool]: if not keys or len(set(keys)) != len(keys): - return document + return DocumentChange(document, False) successors = {name: row for key in keys for name, row in document.lookup(key).bind( claim_scope=claim_scope, turn_id=turn_id, native_id=native_id, text=text ).successors(key).items()} if successors.keys() != set(keys): - return document - bound = True - return replace(document, rows={**document.rows, **successors}) + return DocumentChange(document, False) + return DocumentChange(replace(document, rows={**document.rows, **successors}), True) - self._update(*_members("key", keys), change) - return bound + return self._update(*_members("key", keys), change) def started(self, key: str, *, turn_id: str, native_id: str, text: str) -> bool: return self._transition( @@ -385,7 +382,9 @@ def started(self, key: str, *, turn_id: str, native_id: str, text: str) -> bool: def settle_unbound(self, keys: tuple[str, ...]) -> InputDocument: """Publish one complete retirement and return that exact document cut.""" - return self._update(*_members("key", keys), lambda document: document.settle_unbound(keys)) + return self._update( + *_members("key", keys), lambda document: DocumentChange.to(document.settle_unbound(keys)) + ) def review_for_goal( self, @@ -399,7 +398,7 @@ def review_for_goal( ) -> None: decision = GoalInputDecision(goal_revision, turn_id) - def review(document: InputDocument) -> InputDocument: + def review(document: InputDocument) -> DocumentChange[InputDocument, InputDocument]: rows = dict(document.rows) for row in observed: if row.key not in keys or (row.key in rows and rows[row.key] != row): @@ -411,11 +410,11 @@ def review(document: InputDocument) -> InputDocument: ): raise ValueError("Reviewed inputs changed; inspect them again.") if not keys: - return document - return replace( + return DocumentChange.to(document) + return DocumentChange.to(replace( document, rows={**rows, **{key: rows[key].review(goal_id, decision) for key in keys}}, - ) + )) self._update(*_members("key", keys), review) @@ -425,13 +424,15 @@ def dismiss_historical( *, scope: InputDeliveryScope = UNOBSERVED_DELIVERY_SCOPE, ) -> DeliveryOverview: - def dismiss(document: InputDocument) -> InputDocument: + def dismiss(document: InputDocument) -> DocumentChange[InputDocument, InputDocument]: changed = { row.key: replace(row, notice_dismissed=True) for row in document.unknown(owners) if scope.historical_notice(row) and not row.notice_dismissed } - return replace(document, rows={**document.rows, **changed}) if changed else document + return DocumentChange.to( + replace(document, rows={**document.rows, **changed}) if changed else document + ) where, parameters = _members("owner", sorted(owners)) return self._update(f"unresolved = 1 AND {where}", parameters, dismiss).delivery_overview( diff --git a/src/agent_comms/locked_store.py b/src/agent_comms/locked_store.py index 769463c31a..3306b7d333 100644 --- a/src/agent_comms/locked_store.py +++ b/src/agent_comms/locked_store.py @@ -36,6 +36,19 @@ T = TypeVar("T") +@dataclass(frozen=True, slots=True) +class DocumentChange[D, R]: + """A document's successor and the answer its change gives the caller.""" + + document: D + result: R + + @classmethod + def to(cls, document: D) -> DocumentChange[D, D]: + """A change whose answer is the successor itself.""" + return cls(document, document) + + class JsonDocument(ABC, Generic[T]): """Declared JSON codec and atomic file I/O; LockedStore owns transactions.""" @@ -568,13 +581,13 @@ def read(self) -> T: with self.reading() as value: return value - def update(self, change: Callable[[T], T]) -> T: + def update[R](self, change: Callable[[T], DocumentChange[T, R]]) -> R: with self.locked(): original = self._read_unlocked() changed = change(original) - if changed is not original: - self._publish_unlocked(changed) - return changed + if changed.document is not original: + self._publish_unlocked(changed.document) + return changed.result def replace(self, value: T) -> None: """Publish a complete owned snapshot without reading the discarded one. diff --git a/src/agent_comms/native_child.py b/src/agent_comms/native_child.py index b04f6c88e5..6d9757a8ed 100644 --- a/src/agent_comms/native_child.py +++ b/src/agent_comms/native_child.py @@ -123,11 +123,21 @@ async def request(self, command): return response.data.require_payload() @property - def retired(self) -> bool: + def retirement_result(self) -> ChildOutcome | BaseException | None: + """The original completed shutdown result; pending work has no result.""" job = self._close_job - if job is None or not job.done(): - return False - return not job.cancelled() and job.exception() is None and isinstance(job.result(), ChildOutcome) + if job is None or not job.done() or job.cancelled(): + return None + return job.exception() or job.result() + + @property + def retired(self) -> bool: + return isinstance(self.retirement_result, ChildOutcome) + + @property + def release_failure(self) -> BaseException | None: + result = self.retirement_result + return result if isinstance(result, BaseException) else None def forward_input(self, operation: Callable[[], Coroutine[None, None, None]]) -> None: self.forwarding = self.forwarding.start(operation) diff --git a/src/agent_comms/native_runtime_input.py b/src/agent_comms/native_runtime_input.py index 33d696970e..cc915b13f4 100644 --- a/src/agent_comms/native_runtime_input.py +++ b/src/agent_comms/native_runtime_input.py @@ -39,6 +39,7 @@ TriageDecisionRecord, ) from .typed_table import ( + TableReference, Column, DurableTables, FinalizedColumn, @@ -254,11 +255,11 @@ def published_replies(self, db, user, owner_lookup): execution_id: Annotated[ NativeColumn, FrozenColumn( - references=(ExecutionRecord, "execution_id"), storage=TextStorage, nullable=True + references=TableReference(ExecutionRecord, ("execution_id",)), storage=TextStorage, nullable=True ), ] attempt_ordinal: Annotated[NativeColumn, FrozenColumn(storage=IntegerStorage, nullable=True)] - owner_lookup: Annotated[str, FrozenColumn(references=(Participants, "participant_lookup"))] + owner_lookup: Annotated[str, FrozenColumn(references=TableReference(Participants, ("participant_lookup",)))] owner_thread: Annotated[str, FrozenColumn()] owner_generation: Annotated[int, FrozenColumn(check="owner_generation>0")] owner_token_digest: Annotated[str, FrozenColumn(check="length(owner_token_digest)=64")] @@ -360,7 +361,7 @@ def context_anchor(self) -> NativeColumn: ), ] recipient_lookup: Annotated[ - str, FrozenColumn(primary_key=True, references=(Participants, "participant_lookup")) + str, FrozenColumn(primary_key=True, references=TableReference(Participants, ("participant_lookup",))) ] owner_thread: Annotated[str, FrozenColumn(check="owner_thread<>''")] owner_generation: Annotated[int, FrozenColumn(primary_key=True, check="owner_generation>0")] @@ -382,7 +383,7 @@ def owner_identity(self): ] input_id: Annotated[ NativeColumn, - Column(references=(NativeRuntimeInput, "input_id"), storage=TextStorage, nullable=True), + Column(references=TableReference(NativeRuntimeInput, ("input_id",)), storage=TextStorage, nullable=True), NativeContext(str), ] stage: Annotated[ diff --git a/src/agent_comms/owner_lifecycle.py b/src/agent_comms/owner_lifecycle.py index 4a02d8b060..26c1a19546 100644 --- a/src/agent_comms/owner_lifecycle.py +++ b/src/agent_comms/owner_lifecycle.py @@ -494,25 +494,16 @@ def _launch_owner( env = initial_input.launch(thread.launch()).apply(env) env[NativeArguments.binary_key] = agent_bin agent_args.apply(env) - owned: Thread | None = None - - def reserve(identity: ProcessIdentity) -> None: - nonlocal owned - owned = self._register_owner(thread, identity, initial_input) - - with owner_process_output(self.root, thread) as output: - SpawnedProcess.launch( - (str(launch_runtime.interpreter), "-m", "agent_comms.worker", - *LaunchRoot.from_path(self.launch_authority.child_root).arguments()), - env=env, - cwd=thread.worktree, - output=output, - before_start=reserve, - runtime=launch_runtime, - lifetime=OutlivesLauncherLifetime, - ) - assert owned is not None - return owned + with owner_process_output(self.root, thread) as output, SpawnedProcess.launching( + (str(launch_runtime.interpreter), "-m", "agent_comms.worker", + *LaunchRoot.from_path(self.launch_authority.child_root).arguments()), + env=env, + cwd=thread.worktree, + output=output, + runtime=launch_runtime, + lifetime=OutlivesLauncherLifetime, + ) as child: + return self._register_owner(thread, child.identity, initial_input) def _require_same_stop_owner(self, thread: Thread, claimed_generation: int) -> None: snapshot = self.registry.snapshot() diff --git a/src/agent_comms/phase_evidence.py b/src/agent_comms/phase_evidence.py index 7e58b6d4d7..51d3e8d67f 100644 --- a/src/agent_comms/phase_evidence.py +++ b/src/agent_comms/phase_evidence.py @@ -19,7 +19,6 @@ from pathlib import Path from typing import TYPE_CHECKING, ClassVar -from agent_comms.commander_parity import ParityRow from agent_comms.field_codec import FieldCodec from agent_comms.head_review import HeadReview from agent_comms.pinned_audit import ForeignAbsenceProbe, GodFunction, ParsedModule @@ -351,14 +350,8 @@ def describe(self) -> str: @dataclass(frozen=True) -class ParityEvidence(PhaseEvidence): - """The native/current comparison rows, which own the cutover decision.""" - - def holds(self, reading: PlanReading) -> bool: - return ParityRow.done(reading.todos) - - def describe(self) -> str: - return "every parity row passes" +class ParityEvidence(AllEvidence): + """Native owners supply the facts; the historical comparison matrix cannot overrule them.""" @dataclass(frozen=True) diff --git a/src/agent_comms/plan_phases.py b/src/agent_comms/plan_phases.py index 77ae3efa9d..e8bd49d81d 100644 --- a/src/agent_comms/plan_phases.py +++ b/src/agent_comms/plan_phases.py @@ -193,18 +193,6 @@ class ControlPlanePhase(CommanderNativePhase): ) -class ParityPhase(CommanderNativePhase): - after = (RecordPlanePhase, ReviewQueuePhase, ControlPlanePhase) - evidence = ParityEvidence() - - -class CutoverPhase(CommanderNativePhase): - """The parity gate passes and the machine-wide commander configuration is gone.""" - - after = (RecordPlanePhase, ReviewQueuePhase, ControlPlanePhase, ParityPhase) - evidence = AllEvidence((ParityEvidence(), PatternAbsent(r"\bCommanderConfig\b"))) - - OUTSIDE_PARITY = ("src", ":!src/agent_comms/commander_parity") @@ -234,6 +222,20 @@ class CommanderPagePhase(CommanderNativePhase): ) +class ParityPhase(CommanderNativePhase): + """The native capability owners, not a second matrix of demonstrations, decide completion.""" + + after = (RecordPlanePhase, ReviewQueuePhase, ControlPlanePhase, CommanderPagePhase) + evidence = ParityEvidence(tuple(phase.evidence for phase in after)) + + +class CutoverPhase(CommanderNativePhase): + """The native evidence passes and the machine-wide commander configuration is gone.""" + + after = (ParityPhase,) + evidence = AllEvidence((ParityPhase.evidence, PatternAbsent(r"\bCommanderConfig\b"))) + + class SkillPhase(CommanderNativePhase): """The skill names no script.""" @@ -683,7 +685,7 @@ class RecordsPhase(RemainingSlopPhase): evidence = AllEvidence( ( - PatternAbsent(r"ConnectivityFacet|observe_connectivity"), + PatternAbsent(r"[C]onnectivityFacet|[o]bserve_connectivity"), PatternAbsent( r"^\s*nonlocal\b", ( diff --git a/src/agent_comms/private_path.py b/src/agent_comms/private_path.py index e14127e3f0..83acd733fb 100644 --- a/src/agent_comms/private_path.py +++ b/src/agent_comms/private_path.py @@ -8,15 +8,14 @@ import os import stat -import sys from abc import ABC, abstractmethod from collections.abc import Callable, Iterator from contextlib import contextmanager from dataclasses import dataclass from pathlib import Path -from tempfile import TemporaryDirectory from typing import BinaryIO, ClassVar +from .child_process import Platform from .declared_family import DeclaredFamily from .errors import RelationViolationError from .field_codec import NullMember, RecordMember, SubclassUnion @@ -285,15 +284,8 @@ def address(cls, path: Path) -> Iterator[Path]: try: if not os.path.samestat(observed, os.fstat(fd)): raise ValueError("Socket directory changed during acquisition") - if sys.platform.startswith("linux"): - yield Path(f"/proc/{os.getpid()}/fd/{fd}") / path.name - else: - with TemporaryDirectory(prefix="ac-socket-", dir="/tmp") as directory: - parent = Path(directory) / "d" - parent.symlink_to(path.parent, target_is_directory=True) - if not os.path.samestat(os.stat(parent), os.fstat(fd)): - raise ValueError("Socket directory changed during address acquisition") - yield parent / path.name + with Platform.current().socket_address(path, fd) as address: + yield address finally: os.close(fd) diff --git a/src/agent_comms/private_registry_guard.py b/src/agent_comms/private_registry_guard.py index 151e20e6b2..f99525879f 100644 --- a/src/agent_comms/private_registry_guard.py +++ b/src/agent_comms/private_registry_guard.py @@ -183,6 +183,10 @@ def missing_notification_assignments( def coordinator_schema(self) -> type[TableSchema]: """The complete schema this root's authority requires during reconciliation.""" + @abstractmethod + def publication_connection(self, connection: sqlite3.Connection) -> sqlite3.Connection | None: + """This root's recorded native publications, or no private publication source.""" + class PublicRegistry(RegistryPrivacy): @@ -191,6 +195,9 @@ def rename( ) -> tuple[str, str]: return threads.registry.rename(before.name, new_name) + def publication_connection(self, connection: sqlite3.Connection) -> None: + """Public attached history has no private native publication relations.""" + def coordinator_schema(self) -> type[TableSchema]: from .coordination_schema import CoordinatorTable @@ -306,6 +313,13 @@ def rename_applied(self, root: Path) -> None: filename = ".registry-owner-guard" + def publication_connection(self, connection: sqlite3.Connection) -> sqlite3.Connection: + from .coordination_response import ResponseTable + + NativeRuntimeTable.require_installed(connection) + ResponseTable.require_installed(connection) + return connection + def coordinator_schema(self) -> type[TableSchema]: from .private_runtime_schema import PrivateCoordinatorSchema, PrivateCoordinatorTable diff --git a/src/agent_comms/read_ledger.py b/src/agent_comms/read_ledger.py index c0b0b3c68f..8bb2994b5b 100644 --- a/src/agent_comms/read_ledger.py +++ b/src/agent_comms/read_ledger.py @@ -18,6 +18,7 @@ from .channel_targets import CommandTarget from .coordination_errors import DMScopeChanged from .field_codec import FieldCodec, NullMember, SubclassUnion +from .locked_store import DocumentChange from .maintenance_barrier import CarriedDocument from .private_path import ObservedPath from .read_basis import ( @@ -79,17 +80,12 @@ def schema(cls) -> dict[str, object]: return FieldCodec.value_schema(str) def take(self, ledger: ReadLedger) -> ReadNotice: - taken: ReadNotice = NoReadNotice() - - def clear(document: ReadDocument) -> ReadDocument: - nonlocal taken + def clear(document: ReadDocument) -> DocumentChange[ReadDocument, ReadNotice]: if document.notice != self: - return document - taken = self - return replace(document, notice=NoReadNotice()) + return DocumentChange(document, NoReadNotice()) + return DocumentChange(replace(document, notice=NoReadNotice()), self) - ledger.update(clear) - return taken + return ledger.update(clear) def notify_if_new(self, previous: ReadNotice, notify: Callable[[str], object]) -> None: if self != previous: @@ -187,7 +183,7 @@ def mark_displayed(self, viewer: str, displayed: DisplayBasis) -> None: if viewer != displayed.viewer: raise DMScopeChanged("Display belongs to a different viewer.") - def advance(document: ReadDocument) -> ReadDocument: + def advance(document: ReadDocument) -> DocumentChange[ReadDocument, ReadDocument]: messages = ( dict(document.messages) if document.bus_identity.matches(displayed.bus_identity) else {} @@ -195,7 +191,7 @@ def advance(document: ReadDocument) -> ReadDocument: messages[displayed.viewer_identity] = messages.get( displayed.viewer_identity, ReadCursor(), ).add(displayed.conversations) - return ( + return DocumentChange.to( replace(document, messages=messages, bus_identity=displayed.bus_identity) if messages != document.messages or not document.bus_identity.matches(displayed.bus_identity) @@ -260,12 +256,12 @@ def remove_threads(self, threads: Sequence[Thread]) -> None: for thread in threads: sources.update(thread.runtime.read_keys()) - def remove(document: ReadDocument) -> ReadDocument: + def remove(document: ReadDocument) -> DocumentChange[ReadDocument, ReadDocument]: messages = {k: v for k, v in document.messages.items() if k not in viewers} transcripts = {k: v for k, v in document.transcripts.items() if k[1] not in sources and k[0] not in viewer_names} if messages == document.messages and transcripts == document.transcripts: - return document - return replace(document, messages=messages, transcripts=transcripts) + return DocumentChange.to(document) + return DocumentChange.to(replace(document, messages=messages, transcripts=transcripts)) self.update(remove) @@ -284,10 +280,10 @@ def mark_transcript( ) -> None: key = self._transcript_key(viewer, source, inode) - def advance(document: ReadDocument) -> ReadDocument: + def advance(document: ReadDocument) -> DocumentChange[ReadDocument, ReadDocument]: previous = document.transcripts.get(key, 0) if previous <= size and previous >= through: - return document - return replace(document, transcripts={**document.transcripts, key: through}) + return DocumentChange.to(document) + return DocumentChange.to(replace(document, transcripts={**document.transcripts, key: through})) self.update(advance) diff --git a/src/agent_comms/recovery_reader.py b/src/agent_comms/recovery_reader.py index f0b771cce7..c8206ce654 100644 --- a/src/agent_comms/recovery_reader.py +++ b/src/agent_comms/recovery_reader.py @@ -18,10 +18,7 @@ PublicationIntents, PublicationEvidence, ) -from agent_comms.coordination_tables.recovery import ( - ConnectivityFacet, - RecoveryAudit, -) +from agent_comms.coordination_tables.recovery import RecoveryAudit from agent_comms.coordination_tables.responses import ResponseObligation from agent_comms.participant_store import ParticipantStore @@ -57,7 +54,6 @@ def get(self, execution_id: str) -> RecoverySnapshot: parameters=(execution_id,), order_by=("exact_target",))) evidence = tuple(PublicationEvidence.select(db, where="execution_id=?", parameters=(execution_id,), order_by=("exact_target",))) - connectivity = ConnectivityFacet.one(db, execution_id=execution_id) audit = next( iter( RecoveryAudit.read( @@ -80,7 +76,6 @@ def get(self, execution_id: str) -> RecoverySnapshot: obligations, intents, evidence, - connectivity, audit, pointer.execution_id, pointer.attempt_ordinal, diff --git a/src/agent_comms/relationships.py b/src/agent_comms/relationships.py index 035deb8a15..94b32a2a0e 100644 --- a/src/agent_comms/relationships.py +++ b/src/agent_comms/relationships.py @@ -24,6 +24,7 @@ from .display_order import CreatedThreadSort, LastActivityThreadSort, ThreadSort from .errors import RelationViolationError, UnregisteredThreadError from .goals import Goal, GoalMentionBinding +from .locked_store import DocumentChange from .maintenance_barrier import CarriedDocument from .messages import Message from .peer_relevance import ( @@ -664,15 +665,17 @@ def remove_threads(self, threads: Sequence[Thread]) -> None: """A collaboration or order ends with either deleted incarnation.""" deleted = {thread.incarnation for thread in threads} - def remove(document: RelationshipDocument) -> RelationshipDocument: + def remove( + document: RelationshipDocument, + ) -> DocumentChange[RelationshipDocument, RelationshipDocument]: collaborations = tuple( edge for edge in document.collaborations if not {edge.owner_incarnation, edge.peer_incarnation} & deleted ) orders = tuple(order for order in document.orders if order.incarnation not in deleted) if (collaborations, orders) == (document.collaborations, document.orders): - return document - return replace(document, collaborations=collaborations, orders=orders) + return DocumentChange.to(document) + return DocumentChange.to(replace(document, collaborations=collaborations, orders=orders)) self.update(remove) @@ -827,25 +830,24 @@ def edit( first.role.require_executable(ValueError("Collaborations relate agent threads")) registry = self.registry.snapshot() peer = registry.canonical_name(peer) - result = None - def change(document: RelationshipDocument) -> RelationshipDocument: - nonlocal result + def change( + document: RelationshipDocument, + ) -> DocumentChange[RelationshipDocument, Collaboration | None]: resolved = document.resolved(registry) changed, result = command.apply( RelationshipEditContext(resolved, first, peer, registry.threads.get(peer)) ) - return document if changed == document else changed + return DocumentChange(document if changed == document else changed, result) - self.store.update(change) - return result + return self.store.update(change) def set_order(self, owner: str, group: type[SortableRelationshipGroup], order: type[ThreadSort]) -> type[ThreadSort]: thread = self.registry.require(owner) registry = self.registry.snapshot() self.store.update( - lambda document: group.ordered(document.resolved(registry), thread, order) + lambda document: DocumentChange.to(group.ordered(document.resolved(registry), thread, order)) ) return order diff --git a/src/agent_comms/runtime_children.py b/src/agent_comms/runtime_children.py index 6ac2bd193c..933f4854ac 100644 --- a/src/agent_comms/runtime_children.py +++ b/src/agent_comms/runtime_children.py @@ -23,6 +23,7 @@ from .declared_family import DeclaredFamily from .field_codec import PathText from .launch_runtime import CallingThreadUnavailable, LaunchRuntime, ThreadLaunch +from .locked_store import DocumentChange from .maintenance_barrier import CarriedDocument @@ -167,12 +168,12 @@ async def launching_async( yield RuntimeChildReservation.capture(store) def finished(self, identity: ProcessIdentity) -> None: - self.update(lambda children: children.finished(identity)) + self.update(lambda children: DocumentChange.to(children.finished(identity))) @classmethod def retire(cls, runtime_prefix: Path) -> None: store = cls.at(runtime_prefix.resolve()) - children = store.update(lambda current: RetiredRuntimeChildren(current.records)) + children = store.update(lambda current: DocumentChange.to(RetiredRuntimeChildren(current.records))) deadline = time.monotonic() + 10.0 failures = [] for child in children.records: @@ -187,4 +188,4 @@ def retire(cls, runtime_prefix: Path) -> None: failures.append(str(error)) if failures: raise ChildRetirementFailed("\n".join(failures)) - store.update(lambda current: RetiredRuntimeChildren()) + store.update(lambda current: DocumentChange.to(RetiredRuntimeChildren())) diff --git a/src/agent_comms/runtime_info.py b/src/agent_comms/runtime_info.py index a7db0a17a8..604d97a64a 100644 --- a/src/agent_comms/runtime_info.py +++ b/src/agent_comms/runtime_info.py @@ -12,6 +12,7 @@ from .declared_family import DeclaredFamily from .errors import RelationViolationError from .field_codec import FieldCodec, SingleFieldValue, WireName, WireRequired +from .locked_store import DocumentChange from .maintenance_barrier import CarriedDocument from .pi_counts import PiCount, UnreportedCount from .thread_model import ThreadModel, UndeclaredModel @@ -107,20 +108,22 @@ def for_threads(self, names: Iterable[str]) -> dict[str, RuntimeInfo]: def observe(self, thread: str, timestamp: float, observation: RuntimeInfoObservation) -> None: """Apply only this observation's fields in the original store transaction.""" - def apply(document: RuntimeInfoDocument) -> RuntimeInfoDocument: + def apply( + document: RuntimeInfoDocument, + ) -> DocumentChange[RuntimeInfoDocument, RuntimeInfoDocument]: values = document.observations current = values.get(thread) if current is None: current = AgentRuntimeInfo(thread=thread, timestamp=timestamp) observed = observation.project_runtime_info(replace(current, timestamp=timestamp)) - return RuntimeInfoDocument({**values, thread: observed}) + return DocumentChange.to(RuntimeInfoDocument({**values, thread: observed})) self.update(apply) def remove_threads(self, threads: Sequence[Thread]) -> None: names = {thread.name for thread in threads} self.update( - lambda document: ( + lambda document: DocumentChange.to( RuntimeInfoDocument({name: value for name, value in document.observations.items() if name not in names}) if names & document.observations.keys() else document @@ -128,13 +131,15 @@ def remove_threads(self, threads: Sequence[Thread]) -> None: ) def rename_thread(self, old_name: str, new_name: str) -> None: - def rename(document: RuntimeInfoDocument) -> RuntimeInfoDocument: + def rename( + document: RuntimeInfoDocument, + ) -> DocumentChange[RuntimeInfoDocument, RuntimeInfoDocument]: values = document.observations if old_name not in values or old_name == new_name: - return document - return RuntimeInfoDocument({ + return DocumentChange.to(document) + return DocumentChange.to(RuntimeInfoDocument({ **{name: value for name, value in values.items() if name != old_name}, new_name: replace(values[old_name], thread=new_name), - }) + })) self.update(rename) diff --git a/src/agent_comms/selected_result.py b/src/agent_comms/selected_result.py index c9aef68e53..ff91b72f82 100644 --- a/src/agent_comms/selected_result.py +++ b/src/agent_comms/selected_result.py @@ -12,6 +12,7 @@ from .diagnostics import record_terminal_failure from .messages import AlertKind, NoticeRole from .send_result import SendResult +from .turn_failure import BackendFailed @dataclass(frozen=True, slots=True) @@ -54,7 +55,7 @@ def publish_native_failure( participant.comms.root, turn_id=input_id, thread=participant.owner.thread.name, - event={}, + failure=BackendFailed, sequences=tuple(source.delivery.message.seq for source in participant.batch.sources), native_response=native_response, source_error=source_error, diff --git a/src/agent_comms/service_passes.py b/src/agent_comms/service_passes.py index 0ee378c81c..d5b63fb7fb 100644 --- a/src/agent_comms/service_passes.py +++ b/src/agent_comms/service_passes.py @@ -6,7 +6,7 @@ from dataclasses import dataclass, field, replace from typing import ClassVar -from .locked_store import LockedStore, RuntimeFile +from .locked_store import DocumentChange, LockedStore, RuntimeFile @dataclass(frozen=True) @@ -33,10 +33,10 @@ def empty(self) -> ServicePasses: return ServicePasses() def failed(self, watch: str, error: BaseException) -> None: - def change(passes: ServicePasses) -> ServicePasses: + def change(passes: ServicePasses) -> DocumentChange[ServicePasses, ServicePasses]: since = passes.failing.get(watch, FailingPass(time.time(), "")).since failing = FailingPass(since, f"{type(error).__name__}: {error}") - return replace(passes, failing=passes.failing | {watch: failing}) + return DocumentChange.to(replace(passes, failing=passes.failing | {watch: failing})) self.update(change) @@ -44,10 +44,10 @@ def passed(self, watch: str) -> None: """A pass that succeeds ends its failure; an unfailed watch writes nothing.""" if watch in self.read().failing: self.update( - lambda passes: replace( + lambda passes: DocumentChange.to(replace( passes, failing={ name: failing for name, failing in passes.failing.items() if name != watch }, - ) + )) ) diff --git a/src/agent_comms/shared_ledger.py b/src/agent_comms/shared_ledger.py index 3ce3dd6c1f..0b0268d902 100644 --- a/src/agent_comms/shared_ledger.py +++ b/src/agent_comms/shared_ledger.py @@ -9,6 +9,7 @@ from agent_comms.cli_result import CliResult from agent_comms.declared_family import DeclaredFamily from agent_comms.field_codec import FieldCodec, JsonShapeFamily, JsonShapeMember +from agent_comms.locked_store import DocumentChange from agent_comms.maintenance_barrier import CarriedDocument from agent_comms.thread_owned_state import ThreadOwnedState @@ -115,22 +116,18 @@ def empty(self) -> LedgerDocument: def merge(self, updates: Mapping[str, object], author: str) -> None: changes = FieldCodec.decode(LedgerDocument, dict(updates)) - self.update(lambda document: LedgerDocument( - {**document.value, **changes.value, "last_updated_by": TextLedgerValue(author)})) + self.update(lambda document: DocumentChange.to(LedgerDocument( + {**document.value, **changes.value, "last_updated_by": TextLedgerValue(author)}))) def remove_threads(self, threads: Sequence[Thread]) -> None: names = frozenset(thread.name for thread in threads) - self.update(lambda document: document.without_threads(names)) + self.update(lambda document: DocumentChange.to(document.without_threads(names))) def rename_thread(self, old_name: str, new_name: str) -> int: """Replace exact structural references without touching free text.""" - changed = 0 - def change(document: LedgerDocument) -> LedgerDocument: - nonlocal changed + def change(document: LedgerDocument) -> DocumentChange[LedgerDocument, int]: renamed = document.renamed_thread(old_name, new_name) - changed = renamed.count - return renamed.value if changed else document + return DocumentChange(renamed.value if renamed.count else document, renamed.count) - self.update(change) - return changed + return self.update(change) diff --git a/src/agent_comms/skills/comms-commander/SKILL.md b/src/agent_comms/skills/comms-commander/SKILL.md index 9ba0a225c7..a0d07b626d 100644 --- a/src/agent_comms/skills/comms-commander/SKILL.md +++ b/src/agent_comms/skills/comms-commander/SKILL.md @@ -72,7 +72,7 @@ The declaration supplies the ID and first-call times when omitted. A mechanical ## The board -Core's Todo store is the only record of fixes and their history; `order board` renders it. Orders, timers, owners, delivery and installed evidence change only through Core's Todo operations. A defect with no owner gets one the same tick. +Core's Todo store is the only record of fixes and their history; `comms_hud` reads it beside plan phases, PRs and overload. Orders, timers, owners, delivery and installed evidence change only through Core's Todo operations. A defect with no owner gets one the same tick. Priority: the owner's latest message; a broken install, lost or replayed input, wrong messages, or a unit blocked on you; then the owner's plans, which are the default work. Slop found outside a plan goes into the plan's surface or a plan phase, never ahead of it. diff --git a/src/agent_comms/todos.py b/src/agent_comms/todos.py index 6a4d8cb7a9..adaa6fbd82 100644 --- a/src/agent_comms/todos.py +++ b/src/agent_comms/todos.py @@ -75,6 +75,7 @@ from agent_comms.lifecycle_capability import LifecycleCapability, Pending from agent_comms.message_reference import MessageReference from agent_comms.messages import InfoKind, Message +from agent_comms.native_tool_call import SelectedToolDenied from agent_comms.order_runs import OrderRun from agent_comms.order_story import FleetEvent, ProblemStory, StoryEvent, StoryLine, StorySource from agent_comms.owner_requests import ( @@ -83,6 +84,7 @@ RequestAtRevision, RequestDirective, RequestLink, + RequestSelection, RequestState, ) from agent_comms.plan_phases import OrderPhase, UnplannedPhase @@ -103,6 +105,7 @@ ) from agent_comms.threads import Thread from agent_comms.typed_table import ( + TableReference, Column, DurableStore, DurableTables, @@ -116,6 +119,9 @@ from agent_comms.work_branches import WorkBranchError, WorkBranches if TYPE_CHECKING: + from agent_comms.commander.order import ( + Commander, CommanderContext, OrderedProblems, OrderFile, Reassignment, + ) from agent_comms.bus import BusReader from agent_comms.comms import Comms from agent_comms.goal_management import Goals @@ -511,6 +517,7 @@ class PullRequest(DeclaredFamily, affix="PullRequest"): orders: tuple[str, ...] = () base: str = field(default=WorkBranches.main, kw_only=True) problems: tuple[str, ...] = field(default=(), kw_only=True) + draft: bool = field(default=False, kw_only=True) def belongs_to(self, todo: Todo) -> bool: return (todo.id in self.problems @@ -563,10 +570,11 @@ class ListedPullRequest(DeclaredFamily, affix="ListedPullRequest"): head: str updated: str base: str + draft: bool = field(default=False, kw_only=True) def parsed(self, problems: tuple[str, ...], orders: tuple[str, ...]) -> PullRequest: return UnmergedPullRequest(self.number, self.branch, self.head, _github_time(self.updated), - orders, base=self.base, problems=problems) + orders, base=self.base, problems=problems, draft=self.draft) class UnmergedListedPullRequest(ListedPullRequest): @@ -586,7 +594,7 @@ def parsed(self, problems: tuple[str, ...], orders: tuple[str, ...]) -> MergedPu return MergedPullRequest(self.number, self.branch, self.head, _github_time(self.updated), orders, base=self.base, merge=self.merge.oid, merged_at=_github_time(self.merged), - problems=problems) + problems=problems, draft=self.draft) class PullRequests: @@ -625,7 +633,7 @@ def read(self, repo: RepositoryName, kind: type[ListedPullRequest], return () selections = [ f'p{number}: pullRequest(number: {number}) {{ number branch: headRefName ' - 'head: headRefOid base: baseRefName closed merge: mergeCommit { oid } ' + 'head: headRefOid base: baseRefName draft: isDraft closed merge: mergeCommit { oid } ' 'merged: mergedAt updated: updatedAt }' for number in linked] query = (f'query {{ repository(owner: {json.dumps(repo.owner)}, ' f'name: {json.dumps(repo.name)}) {{ {" ".join(selections)} }} }}') @@ -640,16 +648,18 @@ def read(self, repo: RepositoryName, kind: type[ListedPullRequest], raise WorkBranchError( listed.stderr.strip() or f"GitHub order lookup in {repo.to_wire()} failed.") rows = FieldCodec.decode(dict[str, ListedPullRequest], FieldCodec.parse(listed.stdout)) - requests = tuple(row.parsed( + requests = tuple((row, row.parsed( tuple(candidate.id for candidate in linked[row.number]), tuple(dict.fromkeys(order.id for candidate in linked[row.number] for order in candidate.order_history - if PullRequestLink(repo, row.number) in order.pull_requests))) - for row in rows.values() if isinstance(row, kind)) - for candidate in problems.values(): - self._read[repo, kind, candidate.id] = tuple(sorted( - (request for request in requests if request.belongs_to(candidate)), - key=lambda request: request.updated_at, reverse=True)) + if PullRequestLink(repo, row.number) in order.pull_requests)))) + for row in rows.values()) + for selection in dict.fromkeys((kind, *ListedPullRequest.members_with(ListedPullRequest))): + for candidate in problems.values(): + self._read[repo, selection, candidate.id] = tuple(sorted( + (request for row, request in requests + if isinstance(row, selection) and request.belongs_to(candidate)), + key=lambda request: request.updated_at, reverse=True)) return self._read[repo, kind, todo.id] @@ -1556,9 +1566,6 @@ def reissued(self, at: float) -> PendingProblemOrder: """This order sent again at ``at`` to a new owner, its first-call window kept.""" raise ValueError("Only an undelivered order can change owner.") - def board_cells(self, todo: Todo) -> str: - return f"{self.finding.place()} | unassigned | - | - | -" - def checked(self) -> CheckedProblemOrder: raise ValueError("This problem has no checked order.") @@ -1806,6 +1813,57 @@ def require_references(self) -> None: self.finding.require_references(root, revision) self.scope.require_references(root, revision) + def check_refusals(self, worktrees: Path) -> tuple[str, ...]: + """Send and correction use the check owner's clean-main acceptance rule.""" + try: + with MalformedRecord.refusing('order.check'): + self.require_writable() + if Path(self.check.directory).is_dir(): + self.require_check_acceptance(self.check, worktrees) + except MalformedRecord as error: + return (f'order.check: {error}; fix: supply a focused measurable command',) + return () + + def require_check_acceptance(self, check: OrderCheck, worktrees: Path) -> None: + repo = Path(check.directory) + Repo.git(repo, 'fetch', '-q', 'origin', 'main') + check.require_acceptance(self.check_base(repo), worktrees) + + def source_refusals(self, context: CommanderContext, document: OrderFile, + traced: FileLine | NoSourceReference) -> list[str]: + """Scope, prose and pattern citations share the order's source revision.""" + check = self.check + problems = [] + for label, validate in (('order.source', self.require_references), + ('order.check.directory', check.require_directory)): + try: + with MalformedRecord.refusing(label): + validate() + except MalformedRecord as error: + problems.append(f'{label}: {error}; fix: name an existing source location or directory') + references = tuple(dict.fromkeys(( + *FileLine.scan('\n'.join((document.prose, *self.edits))), + *(reference for slop in document.slop for reference in (slop.at, slop.by)), + *traced.located(), + ))) + deleted = {Path(module).stem for module in self.deletes.modules} + if deleted and (unread := context.importers(deleted, check)): + problems.append(f'the order deletes {sorted(deleted)}, which {unread} import; ' + 'the check must also count their `from agent_comms` imports') + for reference in references: + try: + with MalformedRecord.refusing(reference.to_wire()): + reference.require_exists(Path(check.directory), self.cited_revision) + except MalformedRecord as error: + problems.append(f'{error}; fix: cite an existing source line') + body = '\n'.join((document.prose, *self.edits, self.report)) + if any((Path(check.directory) / reference.file).resolve().is_relative_to( + Path(check.directory) / 'tools/') for reference in references) and not all( + said in body for said in ('--root tools', '--root src')): + problems.append('the order changes tools/: say "ratchet with --root tools and with ' + '--root src, each as its own command"') + return problems + def require_wording(self) -> None: """Check fresh order prose, never reinterpret stored history or shell commands.""" for text in (*self.edits, *self.banned, self.report, *self.run.instructions()): @@ -1837,13 +1895,6 @@ def declarations(self) -> tuple[CheckedProblemOrder, ...]: ) serves: Annotated[type[OrderPhase], WireOmitDefault()] = UnplannedPhase - def board_cells(self, todo: Todo) -> str: - last = todo.released[-1:] - owner = todo.holder or (f"unassigned: released from {last[0].previous.owner}: " - f"{last[0].reason.text}") - check = json.dumps(FieldCodec.encode(self.check)).replace("|", "|") - return f"{self.owner.to_wire()} | {owner} | {self.id} | {todo.review_at.clock()} | {check}" - @property def phase(self) -> type[OrderPhase]: return self.serves @@ -2039,6 +2090,123 @@ class PendingProblemOrder(Pending, DeliverableProblemOrder): first_call_deadline: EventTime = field( default_factory=lambda: RecordedEventTime(time.time() + 120)) + def assignment_refusals(self, context: CommanderContext, document: OrderFile, unit: str, + group: Sequence[str], listed: Mapping[str, Todo], + target: OrderedProblems, + traced: FileLine | NoSourceReference) -> list[str]: + """The document's selected problems, phase and existing owners must agree.""" + reading = context.plan_reading( + context.comms.goals.todos.history_for_commander(context.incarnation)) + problems = list(self.phase.order_refusals(reading, document.unplanned_reason)) + held = tuple(todo for todo in listed.values() if todo.holder_state.current_assignments()) + busy = tuple(todo for todo in held if todo.held().owner == unit and todo.id not in group) + if busy: + snapshot = context.comms.registry.snapshot() + occupied = {todo.held().owner for todo in held} | {context.commander} | context.parked + idle = sorted(thread.name for thread in snapshot.active_threads(snapshot.threads.values()) + if thread.name not in occupied and thread.role.executor(thread)) + problems.append(f'{unit} already holds orders ' + f'{", ".join(todo.id for todo in busy)}; ' + f'running units holding no order: {", ".join(idle) or "none"}') + if not group or any(not ProblemNumber.accepts(pid) for pid in group): + problems.append('problems: fix: supply an array of P IDs, or use order problem') + problems.extend(target.refusals(document, group, listed)) + for pid in group: + try: + with MalformedRecord.refusing(pid): + if pid in listed: + listed[pid].holder_state.require_trace(unit, traced) + except MalformedRecord as error: + problems.append(str(error)) + return problems + + def require_send(self, context: CommanderContext, document: OrderFile, unit: str, + group: Sequence[str], listed: Mapping[str, Todo], target: OrderedProblems, + reassign: Reassignment, holders: Sequence[str], clean: bool, + traced: FileLine | NoSourceReference) -> None: + """Collect field refusals before recording any part of this document.""" + problems = document.problems_found(clean) + try: + context.comms.registry.storage_rule.require_quota(context.comms.registry.require(unit)) + except SelectedToolDenied as quota: + problems.append(f'{unit}: {quota}: it cleans up its own checkouts and scratch ' + 'before it gets new work') + problems.extend(self.assignment_refusals( + context, document, unit, group, listed, target, traced)) + problems.extend(self.source_refusals(context, document, traced)) + problems.extend(self.check_refusals(context.worktrees)) + problems.extend(reassign.refusals(holders)) + if problems: + sys.exit('Refused order:\n ' + '\n '.join(problems)) + fired = tuple(listed[pid] for pid in group if pid in listed and listed[pid].struck_out(unit)) + self.run.require_assignment(context.comms.registry.require(unit).incarnation, + fired, ProblemGoalOrder.StrikeLimit) + + def message(self, context: CommanderContext, document: OrderFile, unit: str, group: Sequence[str], + replacing: bool, reassign: Reassignment) -> str: + """Render owned fields; never parse this message back into an order.""" + instructions = '\n'.join((document.prose, *(slop.render() for slop in document.slop), + f'Problems: {", ".join(group)}', *self.work_instructions())) + replacement = '\nThis replaces my earlier order on it; drop that one.' if replacing else '' + storage = context.comms.registry.storage_rule.order_text( + context.comms.registry.require(unit), + [f'{repo.checkout}/.venv/bin/python' for repo in context.repos]) + return (instructions + replacement + reassign.note() + '\n' + storage + + '\nIf two lines of this order conflict, take the narrower reading, ' + 'say which one you took, and proceed; do not stop to ask.' + + '\nStart now and push as you go.\n' + + '\n'.join(line for pid in group for line in self.scope.instructions(pid))) + + def send( + self, context: Commander, document: OrderFile, unit: str, clean: bool, + reassign: Reassignment, target: OrderedProblems, + traced: FileLine | NoSourceReference, answers: tuple[MessageReference, ...], + request: RequestSelection, + ) -> None: + """Validate the complete issuance before recording, then publish its owned effects.""" + group = target.group(document) + listed = {todo.id: todo for todo in context.todos()} + holders = sorted({assignment.owner for pid in group if pid in listed + for assignment in listed[pid].holder_state.current_assignments() + if assignment.owner != unit}) + replacing = target.replacing(unit, group, listed) + self.require_send(context, document, unit, group, listed, target, + reassign, holders, clean, traced) + struck_already = replacing and listed[group[0]].struck_on_order(unit) + measured = MeasuredReview.read(context.comms.goals.todos) + review_at = measured.review_at(self) + message = self.message(context, document, unit, group, replacing, reassign) + request_id = request.for_order(context.comms.goals.todos, context.incarnation) + target.record(context, self, unit, group, traced, request_id, review_at) + self.publish(context, unit, group, message, answers) + self.notify_replaced(context, unit, group, holders, listed, reassign) + print(f'{measured.summary}; review timer {review_at.clock()}') + print(self.id) + if replacing and not clean: + reassign.send_back(context, document, unit, group[0], struck_already) + + def publish( + self, context: Commander, unit: str, group: Sequence[str], message: str, + answers: tuple[MessageReference, ...], + ) -> None: + """Bind the one published message to every problem through its native command.""" + sent = context.post(VerdictAudience((unit,)), message, answers) + for pid in group: + context.comms.goals.change_problem(pid, OrderMessage( + context.comms.goals.todos.get(pid).revision, context.actor_identity, sent, + ), context.commander) + + def notify_replaced( + self, context: Commander, unit: str, group: Sequence[str], holders: Sequence[str], + listed: Mapping[str, Todo], reassign: Reassignment, + ) -> None: + """The order that takes over the work tells each former holder why.""" + for previous in holders: + context.post(VerdictAudience((previous,)), + f'Taken off you: {", ".join(group)} ("{listed[group[0]].text[:120]}") ' + f'now belongs to {unit} because: {reassign.because()}. ' + 'Stop that work; do not touch its files.') + def reissued(self, at: float) -> PendingProblemOrder: return self.fresh_order(at, self.edits) @@ -3264,7 +3432,7 @@ class Todo(TypedTable): state: type[TodoState] revision: Annotated[int, Column(check="revision>0")] request: Annotated[ - RequestLink, Column(storage=TextStorage, nullable=True, references=(OwnerRequest, "id")), + RequestLink, Column(storage=TextStorage, nullable=True, references=TableReference(OwnerRequest, ("id",))), ] assignment: Assignment | None order: ProblemOrder @@ -3856,9 +4024,15 @@ def timers_up_for_commander(self, commander: ThreadIncarnation, registry: Regist """Held work whose review timer is up, with its round; merged work waits on install.""" held = self.held_for_commander(commander) pulls = ProgressRead(held) + progress = {todo.id: OrderProgress.read(todo, registry, pulls) for todo in held} + return self.timers_up_in(held, now, progress) + + @staticmethod + def timers_up_in(held: Sequence[Todo], now: float, + progress: Mapping[str, OrderProgress]) -> tuple[tuple[Todo, RecordedEventTime], ...]: + """Review rounds from this read's orders and PR facts, excluding installation waits.""" return tuple((todo, up) for todo in held for up in todo.timer_up(now) - if not todo.order.waiting_on_install( - todo, {todo.id: OrderProgress.read(todo, registry, pulls)})) + if not todo.order.waiting_on_install(todo, progress)) def held_for_commander(self, commander: ThreadIncarnation) -> tuple[Todo, ...]: return self._select("creator=? AND creator_created=? AND assignment IS NOT NULL", diff --git a/src/agent_comms/tools.py b/src/agent_comms/tools.py index 9afc1c0ad9..77347c8114 100644 --- a/src/agent_comms/tools.py +++ b/src/agent_comms/tools.py @@ -57,6 +57,7 @@ ) from .cli_result import ToolCatalogResult from .commander.vibe_check import OverloadDegradation +from .commander_hud import CommanderHud from .coordination_tables.refusals import SendRefusal from .declared_family import DeclaredFamily from .display_order import ChannelSort, ThreadSort @@ -980,6 +981,18 @@ def apply(self, comms: Comms, caller: ToolCaller) -> OverloadDegradation: return OverloadDegradation.of(comms, caller.subject(self.thread)) +@dataclass(frozen=True, kw_only=True) +class CommsHudTool(ThreadReadTool): + label = "Commander HUD" + description = ( + "comms_hud reads orders by holder, review timers, strikes, unjudged edit counts, " + "ready/draft PRs, declared plan phases and the overload decision from their owners." + ) + + def apply(self, comms: Comms, caller: ToolCaller) -> CommanderHud: + return CommanderHud.read(comms, caller.subject(self.thread)) + + @dataclass(frozen=True, kw_only=True) class CommsFireTool(ThreadToolRequest): label = "Fire unit" diff --git a/src/agent_comms/transcripts.py b/src/agent_comms/transcripts.py index 5ca82a8a05..3faec6cc24 100644 --- a/src/agent_comms/transcripts.py +++ b/src/agent_comms/transcripts.py @@ -14,10 +14,8 @@ from .agent_backend import RuntimeLocation from .bus_publication import Delivered -from agent_comms.native_runtime_input import NativeRuntimeTable from .coordination_database import CoordinationStore from .coordination_errors import CoordinationReadUnavailable, StaleRevision, TranscriptChanged -from .coordination_response import ResponseTable from .declared_family import DeclaredFamily from .errors import RelationViolationError from .external_transcript import ExternalTranscript @@ -30,6 +28,7 @@ from .native_transcript import NativeTranscript, RecordsCut from .presentation import MessageNotification from .private_path import FileObservation, FileRevision +from .private_registry_guard import RegistryPrivacy from .recorded_bus import RecordedBusIdentity from .registration import Registration from .routing import TurnRouting @@ -394,21 +393,20 @@ class TranscriptRead: @classmethod @contextmanager def publication_source(cls, root): - with CoordinationStore.observed(CoordinationStore.path_in(root)).observing(cls) as db: + privacy = RegistryPrivacy.capture(root) + with CoordinationStore.observed(CoordinationStore.path_in(root)).observing(cls, privacy) as db: yield db @classmethod - def capture(cls, db): - NativeRuntimeTable.require_installed(db) - ResponseTable.require_installed(db) - return db + def capture(cls, db, privacy: RegistryPrivacy): + return privacy.publication_connection(db) @classmethod - def missing_store(cls, store): + def missing_store(cls, store, privacy: RegistryPrivacy): return None @classmethod - def invalid_store(cls, store): + def invalid_store(cls, store, privacy: RegistryPrivacy): raise RelationViolationError("Native reply source has an invalid coordinator") def current(self) -> bool: diff --git a/src/agent_comms/triage_native_sources.py b/src/agent_comms/triage_native_sources.py index 4732af6952..9b4ab4aa87 100644 --- a/src/agent_comms/triage_native_sources.py +++ b/src/agent_comms/triage_native_sources.py @@ -10,6 +10,7 @@ from .coordination_errors import IdentityConflict from .native_runtime_input import NativeRuntimeInput, NativeRuntimeTable from .typed_table import ( + TableReference, Column, GuardedInserts, ImmutableRows, @@ -34,7 +35,7 @@ class TriageNativeSources( JOIN json_each(NEW.assignment_ids) member ON member.value=previous.value))""" input_id: Annotated[ - str, Column(primary_key=True, references=(NativeRuntimeInput, "input_id")) + str, Column(primary_key=True, references=TableReference(NativeRuntimeInput, ("input_id",))) ] assignment_ids: tuple[str, ...] diff --git a/src/agent_comms/turn_failure.py b/src/agent_comms/turn_failure.py index d25ecf679e..fdeafae6a8 100644 --- a/src/agent_comms/turn_failure.py +++ b/src/agent_comms/turn_failure.py @@ -7,7 +7,6 @@ from typing import TYPE_CHECKING, ClassVar from .declared_family import DeclaredFamily -from .diagnostics import FailureReason from .native_arguments import SelectedModel if TYPE_CHECKING: @@ -18,7 +17,6 @@ @dataclass(frozen=True) class TurnFailure(DeclaredFamily): text: str - code: ClassVar[str | None] = None # A failure of the model request itself; Pi re-sending that request ends it. ends_with_request_retry: ClassVar[bool] = False @@ -54,19 +52,25 @@ class InputIdUnavailable(TurnFailure): def with_startup_diagnostics(self, stderr: str) -> TurnFailure: return self.with_text(self.text + "\nBackend startup reported:\n" + stderr) - code = FailureReason.INPUT_ID_UNAVAILABLE + +class NativePreflightTimeout(InputIdUnavailable): + """Input-ID preflight expired before the native child attested.""" + + +class NativePreflightExit(InputIdUnavailable): + """The native child exited before input-ID preflight attested.""" class IdentityUncertain(TurnFailure): - code = FailureReason.IDENTITY_UNCERTAIN + pass class AuthorityChanged(TurnFailure): - code = FailureReason.AUTHORITY_CHANGED + pass class FollowupUnrecognized(TurnFailure): - code = FailureReason.FOLLOWUP_UNRECOGNIZED + pass class TerminalFailure(TurnFailure): @@ -84,7 +88,6 @@ def explanation(cls, output: TurnOutput) -> str: class InputMissing(TerminalFailure): - code = FailureReason.INPUT_MISSING default_text = "Pi RPC run ended without this prompt's user message start." @classmethod @@ -112,7 +115,6 @@ def raise_native(self, proof, model: SelectedModel) -> None: raise NativePiTerminalFailure(self.text, proof, model, self) - code = FailureReason.MODEL_REQUEST_FAILED ends_with_request_retry = True @@ -141,7 +143,6 @@ class ProviderTransportFailed(RetryableProviderFailure, ModelRequestFailed): class FinalStopMissing(TerminalFailure): - code = FailureReason.FINAL_STOP_MISSING default_text = "Pi RPC run ended without an authoritative final assistant stop." @classmethod @@ -154,7 +155,6 @@ def explanation(cls, output: TurnOutput) -> str: class QueuedInputMissing(TerminalFailure): - code = FailureReason.QUEUED_INPUT_MISSING default_text = "Pi RPC run ended with an unstarted queued input; delivery is uncertain." @classmethod @@ -176,3 +176,11 @@ class ExtensionUiFailed(TurnFailure): class BackendDidNotExit(TurnFailure): pass + + +class BackendFailed(TurnFailure): + """A failed terminal without a more specific native failure declaration.""" + + +class NativeLaunchInvalid(BackendFailed): + """The configured native launch could not be acquired.""" diff --git a/src/agent_comms/turn_output.py b/src/agent_comms/turn_output.py index 24cbd3ba06..d79a5c0c14 100644 --- a/src/agent_comms/turn_output.py +++ b/src/agent_comms/turn_output.py @@ -30,7 +30,6 @@ class TurnOutput: default_factory=UnrecordedFailureSource, init=False, repr=False) final_assistant_stop: bool = False diagnostic: dict[str, int] = field(default_factory=dict) - preflight_failure: str | None = None # The live turn owns the one privacy-safe decoded retry result. Completion # and failure presentation retain this immutable value; they do not decode it. retry_result: ProviderRetryResult | None = None @@ -123,11 +122,10 @@ def done(self, session: TurnSession, stderr: str) -> Iterator[events.AgentEvent] terminal = events.Done( text=self.terminal_text(success, stderr, session.native.proc.returncode), ok=success and session.native.attestation.trustworthy, - reason_code=self.failure.code if self.failure else None, + failure=self.declared_failure if self.failure else None, retry_result=self.retry_result, diagnostic={ **self.diagnostic, - **({"reason": self.preflight_failure} if self.preflight_failure else {}), **( {"exit_code": session.native.proc.returncode} if session.native.proc.returncode is not None diff --git a/src/agent_comms/turn_progress.py b/src/agent_comms/turn_progress.py index 2489c74684..65667e5d16 100644 --- a/src/agent_comms/turn_progress.py +++ b/src/agent_comms/turn_progress.py @@ -322,7 +322,8 @@ def publish_checkpoint(self, terminal: events.Done) -> tuple[SendResult, ...]: self.comms.root, turn_id=self.turn_lease.turn_id, thread=self.thread_name, - event=vars(terminal), + failure=terminal.failure_kind, + measurements=terminal.diagnostic, sequences=(), ) results = [] diff --git a/src/agent_comms/turn_terminal.py b/src/agent_comms/turn_terminal.py index 22406c8b81..9f5d2923f2 100644 --- a/src/agent_comms/turn_terminal.py +++ b/src/agent_comms/turn_terminal.py @@ -7,7 +7,7 @@ from contextlib import suppress from typing import TYPE_CHECKING -from .diagnostics import FailureReason, terminal_failure_reason +from .turn_failure import BackendFailed, TurnFailure from .goal_failure_observation import FailedTurnObservation from .goal_attempts import StaleAttemptError @@ -77,12 +77,12 @@ def failure_observation(self, account, permit, captured): claim_scope=account.lease.claimed_generation, current_owner=snapshot.threads.get(account.thread_name), current_claim=snapshot.claim_generation(account.thread_name), - reason=self.failure_reason, + failure=self.failure_kind, ) @property @abstractmethod - def failure_reason(self) -> FailureReason: ... + def failure_kind(self) -> type[TurnFailure]: ... def settle_launched(self, account, permit, store, captured, current) -> None: if permit.has_verified_progress(store): @@ -90,15 +90,15 @@ def settle_launched(self, account, permit, store, captured, current) -> None: observation = self.failure_observation(account, permit, captured) with suppress(StaleAttemptError): permit.reservation.fail( - store, self.failure_reason.value, + store, self.failure_kind.declared_name, observation=observation, ) class EndedWithoutResult(FailedGoalTurn): @property - def failure_reason(self) -> FailureReason: - return terminal_failure_reason({}) + def failure_kind(self) -> type[TurnFailure]: + return BackendFailed class OperatorStopped(TurnTerminal): diff --git a/src/agent_comms/turn_watchdog.py b/src/agent_comms/turn_watchdog.py index 80c0c03498..a49d75960a 100644 --- a/src/agent_comms/turn_watchdog.py +++ b/src/agent_comms/turn_watchdog.py @@ -12,7 +12,6 @@ from . import turn_failure as failures from . import turn_phase as phases from .declared_family import DeclaredFamily -from .diagnostics import FailureReason from .mro_dispatch import MroDispatch, handles from .native_startup import SessionLoad @@ -76,7 +75,7 @@ async def expire(self, session: TurnSession) -> AsyncIterator[events.AgentEvent] class PreflightWait(TurnWait): - reason = "native_preflight_timeout" + reason = failures.NativePreflightTimeout.declared_name phase = "await_get_state" async def expire(self, session: TurnSession) -> AsyncIterator[events.AgentEvent]: @@ -84,7 +83,6 @@ async def expire(self, session: TurnSession) -> AsyncIterator[events.AgentEvent] elapsed_ms = round((watchdog.clock() - watchdog.launch_started_at) * 1000) wait_ms = round(self.elapsed * 1000) budget_ms = round(self.budget * 1000) - session.output.preflight_failure = FailureReason.PREFLIGHT_TIMEOUT session.output.diagnostic = { "elapsed_ms": elapsed_ms, "wait_ms": wait_ms, @@ -94,7 +92,7 @@ async def expire(self, session: TurnSession) -> AsyncIterator[events.AgentEvent] session.output.diagnostic.update(watchdog.session_load.diagnostic) session_size = watchdog.session_load.summary session.output.record_failure( - failures.InputIdUnavailable( + failures.NativePreflightTimeout( "Pi native input-ID capability preflight timed out " f"(phase={self.phase}, elapsed_ms={elapsed_ms}, " f"wait_ms={wait_ms}, budget_ms={budget_ms}, " diff --git a/src/agent_comms/typed_table.py b/src/agent_comms/typed_table.py index 10454db5f1..04c4777692 100644 --- a/src/agent_comms/typed_table.py +++ b/src/agent_comms/typed_table.py @@ -281,6 +281,31 @@ def from_sql(cls, value: object) -> object: return FieldCodec.parse(value) +class Reference(ABC): + """The reference clause carried by a column or a composite foreign key.""" + + @abstractmethod + def reference_sql(self) -> str: ... + + +@dataclass(frozen=True) +class NoReference(Reference): + def reference_sql(self) -> str: + return "" + + +@dataclass(frozen=True) +class TableReference(Reference): + target: type[TypedTable] + target_columns: tuple[str, ...] + + def reference_sql(self) -> str: + return ( + f" REFERENCES {_identifier(self.target.declared_name)} " + f"({self.target._column_list(self.target_columns)})" + ) + + @dataclass(frozen=True) class Column(FieldDirective): """Constraints on a field; its name and type remain on the dataclass field.""" @@ -289,7 +314,7 @@ class Column(FieldDirective): record_key: bool = False auto_increment: bool = False unique: bool = False - references: tuple[type[TypedTable], str] | None = None + references: Reference = field(default_factory=NoReference) check: str | None = None index: bool = False generated: str | None = None @@ -402,12 +427,10 @@ def term_sql(self, owner: type[TypedTable]) -> str: @dataclass(frozen=True) -class ForeignKey: +class ForeignKey(TableReference): """Composite references declared beside their row, including cyclic owners.""" columns: tuple[str, ...] - target: type[TypedTable] - target_columns: tuple[str, ...] deferred: bool = False on_delete: str | None = None @@ -415,16 +438,23 @@ def sql(self, owner: type[TypedTable]) -> str: if len(self.columns) != len(self.target_columns): raise ValueError("Foreign key column counts differ") return ( - f"FOREIGN KEY ({owner._column_list(self.columns)}) " - f"REFERENCES {_identifier(self.target.declared_name)} " - f"({self.target._column_list(self.target_columns)})" + f"FOREIGN KEY ({owner._column_list(self.columns)})" + + self.reference_sql() + (f" ON DELETE {self.on_delete}" if self.on_delete else "") + (" DEFERRABLE INITIALLY DEFERRED" if self.deferred else "") ) +class SqlProjection: + """Scalar and record projections read SQLite's result columns at one boundary.""" + + @staticmethod + def result_columns(cursor: sqlite3.Cursor) -> tuple[str, ...]: + return tuple(column[0] for column in (cursor.description or ())) + + @dataclass(frozen=True) -class SqlValue: +class SqlValue(SqlProjection): """A compiled SQLite value; named record fields add their column declaration.""" annotation: object @@ -438,7 +468,7 @@ def for_annotation(cls, annotation: object) -> SqlValue: def read(self, cursor: sqlite3.Cursor) -> list: """Read a single-column projection without losing empty or nullable results.""" try: - if cursor.description is None or len(cursor.description) != 1: + if len(self.result_columns(cursor)) != 1: raise MalformedRecord("Scalar projection requires exactly one SQLite column") return [self.decode(row[0]) for row in cursor] finally: @@ -497,7 +527,7 @@ def sql_literal(value: str | None | type[DeclaredFamily]) -> str: return "NULL" if text is None else "'" + text.replace("'", "''") + "'" -class TypedRow: +class TypedRow(SqlProjection): """Typed query projection; also the shared decoder for stored row types.""" @classmethod @@ -538,9 +568,9 @@ def _computed_columns(cls) -> frozenset[str]: @classmethod def _positions(cls, cursor: sqlite3.Cursor, extra: Mapping[str, str] = types.MappingProxyType({})) -> dict[str, int]: - if cursor.description is None: + names = cls.result_columns(cursor) + if not names: raise ValueError("Typed read requires a result set") - names = tuple(item[0] for item in cursor.description) projection = {**cls.column_projection(), **extra} carried = set(names).difference(cls._computed_columns()) if len(set(names)) != len(names) or carried != set(projection.values()): @@ -1516,9 +1546,7 @@ def _table_sql(cls, *, if_not_exists: bool = False, columns: tuple[str, ...] = ( sql += " NOT NULL" if column.unique: sql += " UNIQUE" - if column.references is not None: - target, name = column.references - sql += f" REFERENCES {_identifier(target.declared_name)} ({target._column_list((name,))})" + sql += column.references.reference_sql() for constraint in item.storage.constraints(item.wire_name, item.annotation): sql += f" CHECK ({constraint})" if column.check: @@ -2071,7 +2099,7 @@ def writer_connection(self, *, check_same_thread: bool = True) -> Iterator[sqlit def _retire_committed(self, lifetime: ExitStack) -> None: try: lifetime.close() - except Exception as error: + except (sqlite3.Error, OSError) as error: raise StoreCommittedError( f"Committed {self.path}, but connection retirement failed: {error}" ) from error diff --git a/tests/conftest.py b/tests/conftest.py index 3af4e5ad23..9c4229f80a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,7 @@ """Shared fixtures: an isolated Comms wire per test.""" import asyncio +import fcntl import os import sqlite3 import subprocess @@ -136,21 +137,13 @@ def publish_initial(comms, sender, target, body, type=InfoKind, *, role=Conversa ) def pytest_sessionstart(session): - """The controller prepares this checkout's pinned inputs before any worker runs. - - Its pinned submodules are populated as every tree of ours is (Checkout), and - prepare-pi-native keys its build by the pin and reuses a verified one, so an unchanged - pin costs nothing and a changed pin rebuilds without anyone remembering. - Collection loads declarations only; it neither starts workers nor needs either. - """ + """The controller populates this checkout's pinned submodules before any worker runs, + as every tree of ours is (Checkout). Collection loads declarations only.""" if session.config.option.collectonly or is_xdist_worker(session): return from agent_comms.repo import Checkout - root = Path(__file__).resolve().parents[1] - Checkout(root).prepare() - if not pinned_location().is_dir(): - subprocess.run([str(root / "stack/bin/prepare-pi-native"), "--python", sys.executable], check=True) + Checkout(Path(__file__).resolve().parents[1]).prepare() @pytest.fixture @@ -191,12 +184,31 @@ def cursor_delivery(source_reads): @pytest.fixture(scope="session") -def native_runtime_package(): +def pinned_native() -> Path: + """The pinned Pi build, prepared the first time a test asks for it. + + prepare-pi-native keys its build by the pin and reuses a verified one, so an unchanged + pin costs nothing and a changed pin rebuilds. A run whose tests never ask never builds; + workers asking together wait on the stack directory's lock and the first one builds. + """ + stack = Path(__file__).resolve().parents[1] / "stack" + held = os.open(stack, os.O_RDONLY) + try: + fcntl.flock(held, fcntl.LOCK_EX) + if not pinned_location().is_dir(): + subprocess.run([str(stack / "bin/prepare-pi-native"), "--python", sys.executable], check=True) + finally: + os.close(held) + return pinned_native_package() + + +@pytest.fixture(scope="session") +def native_runtime_package(pinned_native): """The running interpreter's artifact, qualified by the real installer before tests.""" from agent_comms.native_package import VerifiedNativePackage from agent_comms.unit_instructions import UnitInstructions - package = VerifiedNativePackage.qualify(pinned_native_package()) + package = VerifiedNativePackage.qualify(pinned_native) UnitInstructions.prepare() return package diff --git a/tests/native_backend_fixture.py b/tests/native_backend_fixture.py index d796e10452..cc371f9aaf 100644 --- a/tests/native_backend_fixture.py +++ b/tests/native_backend_fixture.py @@ -281,27 +281,25 @@ async def close_server(): await asyncio.gather(*connections, return_exceptions=True) async def retire_all(): - try: - async with AsyncExitStack() as retirement: - retirement.push_async_callback(close_server) - for child in owner.children: - retirement.push_async_callback(child.shutdown) - retirement.push_async_callback(owner.persistent.close) - finally: - assert all(child.retired for child in owner.children) - print(json.dumps({ - "saved_native_fixture": str(root), - "localhost_posts": provider.posts, - "native_input_proofs": sum( - len(read_proof_rows(files)) - for files in { - child.key[0].session.files(child.key[0].root) - for child in owner.children - } - if files.input_proof.is_file() - ), - "children": [FieldCodec.encode(child.proc.identity) for child in owner.children], - "children_retired": True, - })) + async with AsyncExitStack() as retirement: + retirement.push_async_callback(close_server) + for child in owner.children: + retirement.push_async_callback(child.shutdown) + retirement.push_async_callback(owner.persistent.close) + assert all(child.retired or child.release_failure is not None for child in owner.children) + print(json.dumps({ + "saved_native_fixture": str(root), + "localhost_posts": provider.posts, + "native_input_proofs": sum( + len(read_proof_rows(files)) + for files in { + child.key[0].session.files(child.key[0].root) + for child in owner.children + } + if files.input_proof.is_file() + ), + "children": [FieldCodec.encode(child.proc.identity) for child in owner.children], + "children_retired": all(child.retired for child in owner.children), + })) await join_retirement(asyncio.create_task(retire_all())) diff --git a/tests/pi_session_turn.py b/tests/pi_session_turn.py index a607819701..38f987a94e 100644 --- a/tests/pi_session_turn.py +++ b/tests/pi_session_turn.py @@ -8,12 +8,12 @@ import agent_comms.backend as backend from agent_comms.comms import Comms from agent_comms.pi_native_backend import PersistentPiSession -from agent_comms.native_package import VerifiedNativePackage, pinned_native_package +from agent_comms.native_package import VerifiedNativePackage from agent_comms.selected_actions import NoSelectedTools async def one_turn_events(agent_bin, agent_args, task, cwd, env_extra=None, - *, tool_mode=NoSelectedTools(), **kwargs): + *, package: Path, tool_mode=NoSelectedTools(), **kwargs): """Run ``backend.stream_agent_events`` on a fresh session and close its child after.""" session = PersistentPiSession() comms = Comms(Path(cwd)) @@ -24,7 +24,7 @@ async def one_turn_events(agent_bin, agent_args, task, cwd, env_extra=None, async with aclosing( backend.stream_agent_events(agent_bin, agent_args, task, cwd, env_extra, persistent_session=session, root=comms.root, owner=owner, tool_mode=tool_mode, - package=VerifiedNativePackage.acquire(pinned_native_package()), + package=VerifiedNativePackage.acquire(package), session_directory=Path(cwd), **kwargs) ) as stream: async for event in stream: diff --git a/tests/round2/test_typed_table.py b/tests/round2/test_typed_table.py index 0ccec59c00..cab820e21c 100644 --- a/tests/round2/test_typed_table.py +++ b/tests/round2/test_typed_table.py @@ -44,7 +44,7 @@ class TableChildRow(TypedTable): @classmethod def references(cls): - return (ForeignKey(("parent", "parent_count"), TableParentRow, ("key", "count")),) + return (ForeignKey(columns=("parent", "parent_count"), target=TableParentRow, target_columns=("key", "count")),) def test_required_row_distinguishes_absence_from_duplicate_results(): diff --git a/tests/test_agent_events.py b/tests/test_agent_events.py index ecb0535a0e..823008c17c 100644 --- a/tests/test_agent_events.py +++ b/tests/test_agent_events.py @@ -172,8 +172,10 @@ def test_payload_fields_and_frozen_multiple_inheritance(): for value in (start, events.OriginalInputStarted(), events.ModelChanged("id", True)): with pytest.raises(FrozenInstanceError): value.extra = "not a declared field" - done = events.Done("failed", False, "input_authority_changed", {"elapsed_ms": 3}) - assert done.reason_code == "input_authority_changed" + from agent_comms.turn_failure import AuthorityChanged + + done = events.Done("failed", False, AuthorityChanged("failed"), {"elapsed_ms": 3}) + assert done.reason_code == "authority_changed" assert done.diagnostic == {"elapsed_ms": 3} diff --git a/tests/test_backend.py b/tests/test_backend.py index c295094e8f..985fd0fc8a 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -11,13 +11,7 @@ import agent_comms.agent_events as ae import agent_comms.backend as backend from agent_comms.agent_backend import InputContent, InputId, InputRequest, WhenBusy -from agent_comms.native_pi import CAPABILITY from agent_comms.pi_rpc import PiRpcChannel -from agent_comms.turn_phase import ( - ModelWaitPhase, - SettlingStatsPhase, - ToolRunningPhase, -) from agent_comms.turn_watchdog import ModelWait from pi_session_turn import one_turn_events @@ -45,171 +39,12 @@ def _stub(tmp_path: Path, body: str, name: str = "pi-stub") -> str: return str(stub) -def _replay_stub(tmp_path: Path, lines: str, *, startup: str = "") -> str: - """Replay recorded RPC lines, answering each ``agent-comms-prompt`` reply with - the id of the request it names, as Pi does for an owned turn's prompt.""" - return _stub( - tmp_path, - f"#!{sys.executable}\n" - + f"""\ -import json, sys -from agent_comms.native_pi import CAPABILITY -from agent_comms.pi_payloads import PiMessage -initial_state = True -prompt_request = None -# One Pi session answers every state query from the same session state. -state_data = {{"nativeInputProofCapability": CAPABILITY}} -{startup}for line in {lines.splitlines()!r}: - record = json.loads(line) - message = record.get("message", {{}}) - if message.get("role") == "assistant": - message.setdefault("content", []) - command = record.get("command") - correlate = record.get("type") == "response" and ( - record.get("id") == "agent-comms-prompt" - or (command == "get_state" and initial_state) - or command == "get_session_stats" - ) - if correlate: - for request_line in sys.stdin: - request = json.loads(request_line) - if request.get("type") == command: - record["id"] = request["id"] - if command == "prompt": - prompt_request = request - break - print(json.dumps({{ - "id": request.get("id"), "type": "response", - "command": request.get("type"), "success": True, - "data": state_data if request.get("type") == "get_state" else {{}}, - }}), flush=True) - if command == "get_state": - record.setdefault("data", {{}})["nativeInputProofCapability"] = CAPABILITY - state_data = record["data"] - initial_state = False - if ( - prompt_request is not None - and record.get("type") == "message_start" - and message.get("role") == "user" - and PiMessage.from_wire(message).text == prompt_request.get("message") - ): - message.setdefault("inputId", prompt_request.get("inputId")) - print(json.dumps(record), flush=True) -""", - ) -_NATIVE_PROMPT_START = f"""\ -import sys -sys.path.insert(0, {str(Path(__file__).parent)!r}) -from pi_stub_protocol import answer_state, answer_stats, next_prompt, read_request -_raw_emit = emit -def emit(value): - message = value.get("message", {{}}) - if message.get("role") == "assistant": - message.setdefault("content", []) - _raw_emit(value) -state_data = {{"nativeInputProofCapability": {json.dumps(CAPABILITY)}}} -answer_state(state_data) -prompt = next_prompt() -emit({{"id": prompt["id"], "type": "response", "command": "prompt", "success": True}}) -emit({{"type": "message_start", "message": {{"role": "user", - "content": prompt["message"], "inputId": prompt["inputId"]}}}}) -# A real provider cannot complete in the same write as Pi's input start. Keep -# dynamic watchdog fixtures from collapsing those two protocol phases. -import time as _fixture_time -_fixture_time.sleep(0.01) -""" - - -async def _rpc_events( - tmp_path: Path, - records: list[dict], - *, - current_input: bool = True, -) -> list[ae.AgentEvent]: - # Pi's assistant wire type always carries its content array. Parser cases - # specify only the fields they vary; the fake child supplies the owned base. - records = [ - record | {"message": {"content": [], **record["message"]}} - if record.get("message", {}).get("role") == "assistant" - else record - for record in records - ] - if current_input: - # Most parser fixtures model an ordinary accepted prompt. They need a - # matching user start, not a fabricated assistant final. Tests of the - # missing/foreign input boundary explicitly opt out. - records = records.copy() - prompt_ack = next( - ( - index - for index, record in enumerate(records) - if record.get("type") == "response" - and record.get("command") == "prompt" - and record.get("success") is True - ), - -1, - ) - if prompt_ack == -1: - # The backend sends get_state before prompt. Preserve that order - # when the fixture includes the state response. - prompt_ack = ( - 1 - if records - and records[0].get("type") == "response" - and records[0].get("command") == "get_state" - else 0 - ) - records.insert( - prompt_ack, - { - "type": "response", - "command": "prompt", - "id": "agent-comms-prompt", - "success": True, - }, - ) - records.insert( - prompt_ack + 1, - {"type": "message_start", "message": {"role": "user", "content": "t"}}, - ) - lines = "\n".join(json.dumps(record) for record in records) - stub = _replay_stub(tmp_path, lines) - return [ - event - async for event in one_turn_events( - stub, [], "t", str(tmp_path) - ) - ] - - -def _assert_full_rpc_stream(events: list[ae.AgentEvent]) -> None: - stream_events = [e for e in events if not isinstance(e, ae.NativePhaseChanged)] - assert [type(e) for e in stream_events] == [ - ae.Chunk, - ae.Chunk, - ae.Thinking, - ae.ToolStart, - ae.ToolProgress, - ae.ToolEnd, - ae.Chunk, - ae.Done, - ] - assert [type(e.phase) for e in events if isinstance(e, ae.NativePhaseChanged)] == [ - ModelWaitPhase, - ToolRunningPhase, - ModelWaitPhase, - SettlingStatsPhase, - ] - assert stream_events[2].text == "Checking the workspace" - tool_start = stream_events[3] - assert tool_start.name == "bash" and tool_start.id == "t1" - assert "echo hi" in tool_start.title - assert stream_events[4].output == "running" - tool_end = stream_events[5] - assert tool_end.ok is True and "hi" in tool_end.output - assert stream_events[-1].text == "hello done" and stream_events[-1].ok is True + + + + class TestRpcParsing: @@ -239,7 +74,7 @@ async def test_partial_large_record_survives_cancelled_read(self): - async def test_foreign_tool_use_before_exact_input_start_is_not_progress(self, tmp_path): + async def test_foreign_tool_use_before_exact_input_start_is_not_progress(self, tmp_path, pinned_native): # A typed prompt ACK is not a current-turn start. A previous Pi # assistant message must not escape as human-visible progress. stub = _stub( @@ -262,7 +97,7 @@ async def test_foreign_tool_use_before_exact_input_start_is_not_progress(self, t events = [ event async for event in one_turn_events( - stub, [], "current prompt", str(tmp_path) + stub, [], "current prompt", str(tmp_path), package=pinned_native ) ] assert not [event for event in events if isinstance(event, ae.CommittedProgress)] diff --git a/tests/test_carry_drop_connectivity.py b/tests/test_carry_drop_connectivity.py new file mode 100644 index 0000000000..992ad09b8e --- /dev/null +++ b/tests/test_carry_drop_connectivity.py @@ -0,0 +1,31 @@ +"""A coordination database written before the connectivity facet was deleted loses it once.""" + +import sqlite3 + +from agent_comms.coordination_database import CoordinationStore +from conftest import maintenance_held +from cutover.carry_drop_connectivity import DropConnectivity + + +def test_carry_drops_the_table_and_its_triggers_once(tmp_path): + path = tmp_path / CoordinationStore.filename + with CoordinationStore(path): + pass + with sqlite3.connect(path) as db: + db.execute("CREATE TABLE connectivity (execution_id TEXT PRIMARY KEY, revision INTEGER)") + db.execute( + "CREATE TRIGGER connectivity_delete_before_abort BEFORE DELETE ON connectivity " + "BEGIN SELECT RAISE(ABORT, 'no delete'); END" + ) + step = DropConnectivity(tmp_path, tmp_path, tmp_path / "spill") + + with maintenance_held(tmp_path): + assert step.carry(apply=False).changed == 1 + assert step.carry(apply=True).rewritten == (CoordinationStore.filename,) + assert step.carry(apply=False).changed == 0 + with sqlite3.connect(path) as db: + assert not db.execute( + "SELECT name FROM sqlite_master WHERE tbl_name = 'connectivity'" + ).fetchall() + with CoordinationStore(path): + pass diff --git a/tests/test_child_process.py b/tests/test_child_process.py index bfc4803eb0..19c42a65ba 100644 --- a/tests/test_child_process.py +++ b/tests/test_child_process.py @@ -397,24 +397,17 @@ async def test_killed_launcher_takes_its_child_and_leaves_its_resident(tmp_path: def test_detached_reserves_identity_before_child_can_execute(tmp_path: Path) -> None: acknowledgement = tmp_path / "started" - recorded = [] - - def reserve(identity: ProcessIdentity) -> None: - assert not acknowledgement.exists() - assert identity.alive() - recorded.append(identity) - - child = SpawnedProcess.launch( + with SpawnedProcess.launching( ( sys.executable, "-c", 'import sys,time; open(sys.argv[1],"w").write("started"); time.sleep(60)', str(acknowledgement), ), - before_start=reserve, - ) + ) as child: + assert not acknowledgement.exists() + assert child.identity.alive() try: - assert recorded == [child.identity] stale = ObservedProcess(replace(child.identity, start_time=child.identity.start_time + 1)) with pytest.raises(IdentityMismatchError): stale.stop_sync() @@ -428,20 +421,16 @@ def test_reservation_failure_never_executes_and_reaps_child(tmp_path: Path) -> N acknowledgement = tmp_path / "should-not-exist" captured = [] - def reject(identity: ProcessIdentity) -> None: - captured.append(identity) + with pytest.raises(ValueError), SpawnedProcess.launching( + ( + sys.executable, + "-c", + "import sys; open(sys.argv[1], 'w').write('wrong')", + str(acknowledgement), + ), + ) as child: + captured.append(child.identity) raise ValueError("reservation refused") - - with pytest.raises(ValueError): - SpawnedProcess.launch( - ( - sys.executable, - "-c", - "import sys; open(sys.argv[1], 'w').write('wrong')", - str(acknowledgement), - ), - before_start=reject, - ) assert len(captured) == 1 assert not captured[0].alive() assert not acknowledgement.exists() diff --git a/tests/test_commander_forks.py b/tests/test_commander_forks.py index 4bd444d1a1..e90b4c2f8a 100644 --- a/tests/test_commander_forks.py +++ b/tests/test_commander_forks.py @@ -35,7 +35,7 @@ from agent_comms.problem_number import ProblemNumber from agent_comms.thread_execution import ExternalThreadExecution from agent_comms.thread_identity import RecordedThreadIncarnation -from agent_comms.tools import CommsCommanderLoadTool, CommsTodoListTool +from agent_comms.tools import CommsCommanderLoadTool, CommsHudTool, CommsTodoListTool from agent_comms.thread_model import DeclaredModel from agent_comms.todos import ( DeclarationDerivation, PendingProblemOrder, ProblemAllocation, ReleaseTodo, @@ -145,7 +145,10 @@ def test_registered_fork_sends_parent_problem_with_fork_actor(fork_bus, tmp_path store.require_unit_in(db, saved.held().owner_identity, reissuer) print(f'Core send actor: {saved.held().parent}; problem commander: {saved.creator}') ctx = CommanderContext.read_root(comms.root).model_copy(update={'actor': CommanderForkCaller(fork.name)}) - assert ctx.board_text() == Commander.read_root(comms.root).board_text() + hud = ctx.call(CommsHudTool(thread=ctx.commander)) + assert hud.commander == parent.incarnation + assert [(unit.unit, tuple(order.problem for order in unit.orders)) + for unit in hud.units] == [(saved.held().owner_identity, ('P40',))] assert set(ctx.model_copy(deep=True).progress()) == {'P40'} assert ctx.held_orders() == (saved,) assert ctx.call(CommsTodoListTool(thread=ctx.commander)).todos == (saved,) diff --git a/tests/test_commander_parity.py b/tests/test_commander_parity.py index 538aed0f9c..c86fb28773 100644 --- a/tests/test_commander_parity.py +++ b/tests/test_commander_parity.py @@ -1,16 +1,10 @@ from pathlib import Path -from agent_comms.commander_parity import ParityComparison, ParityRow +from agent_comms.commander_parity import ParityRow PLAN = Path(__file__).resolve().parents[1] / "plans/commander-native-plan.md" -class MatchingComparison(ParityComparison): - passed = True - - - - def test_generated_prose_is_not_input_to_the_gate(tmp_path): plan = tmp_path / "plan.md" plan.write_text( @@ -21,4 +15,3 @@ def test_generated_prose_is_not_input_to_the_gate(tmp_path): assert rendered.startswith("Before\n## Parity matrix\n\n") assert rendered.endswith("**Dropped deliberately:** historical text\nAfter\n") assert "Hand edited" not in rendered - assert not ParityRow.done(()) diff --git a/tests/test_commander_parity_measurements.py b/tests/test_commander_parity_measurements.py deleted file mode 100644 index ecc8fa48ad..0000000000 --- a/tests/test_commander_parity_measurements.py +++ /dev/null @@ -1,42 +0,0 @@ -"""Comparison invariants; configured live-owner proof is the --verify application run.""" - -from dataclasses import replace - -from agent_comms.commander_parity.measurements import RuleComparison, StorageComparison -from agent_comms.commander_parity.rows import ( - LoadForkingParityRow, - MeasuredLoadParityRow, - UnitTimeParityRow, -) - - -def test_measurement_family_requires_native_calls_exact_delivery_and_no_private_write(): - matched = RuleComparison("owner", ("delivered",), ("delivered",), 1, True) - assert matched.passed - for changed in ( - replace(matched, calls=0), - replace(matched, calls=2), - replace(matched, current=("different",)), - replace(matched, native=()), - replace(matched, private_state_unchanged=False), - ): - assert not changed.passed - assert not StorageComparison(changed, matched).passed - assert not StorageComparison(matched, changed).passed - assert StorageComparison(matched, matched).passed - missing = UnitTimeParityRow.verify() - assert not missing.passed - assert UnitTimeParityRow.native in missing.describe() - - -def test_a_load_report_does_not_prove_automatic_forking_or_self_report_comparison(): - for row in (LoadForkingParityRow, MeasuredLoadParityRow): - assert not row.verify().passed - - -def test_storage_comparison_notices_private_state_changes(tmp_path): - path = tmp_path / "private-state" - path.write_text("before") - before = RuleComparison.state_files(tmp_path) - path.write_text("after") - assert RuleComparison.state_files(tmp_path) != before diff --git a/tests/test_connector.py b/tests/test_connector.py index 5c02732325..dd5bab7f6f 100644 --- a/tests/test_connector.py +++ b/tests/test_connector.py @@ -39,6 +39,7 @@ IssuedCode, TrustedBrowser, digest, ) from agent_comms.coordinator import Coordination +from agent_comms.locked_store import DocumentChange from agent_comms.message_bus import WatchPass from agent_comms.service_passes import ServicePassStore from agent_comms.standing_rules import ServicePassFailing @@ -612,9 +613,9 @@ async def test_pending_authorizations_are_bounded(client, store, comms): )).status == 200 assert len(comms.bus.inbox(owner.name)) == before held = store.read().requests[repeated.request_id] - store.update(lambda grants: replace(grants, requests=grants.requests | { + store.update(lambda grants: DocumentChange.to(replace(grants, requests=grants.requests | { repeated.request_id: replace(held, expires_at=time.time() - 1) - })) + }))) assert (await client.get( "/authorize", params=FieldCodec.encode(blocked), allow_redirects=False )).status == 200 @@ -669,22 +670,22 @@ async def test_owner_browser_approval_lifecycle(client, store, comms): link = urlsplit(notices[-1].body.splitlines()[-1]) assert (await client.get(link.path + "wrong", allow_redirects=False)).status == 400 expired = replace(store.read().requests[query.request_id], expires_at=time.time() - 1) - store.update(lambda grants: replace( + store.update(lambda grants: DocumentChange.to(replace( grants, requests={query.request_id: expired}, codes=grants.codes | {"expired-code": IssuedCode( query.client_id, query.code_challenge, expires_at=expired.expires_at )}, - )) + ))) assert (await client.get(link.path, allow_redirects=False)).status == 400 - store.update(lambda grants: grants.issued(query.client_id)[0]) + store.update(lambda grants: grants.issued(query.client_id)) assert not store.read().requests and "expired-code" not in store.read().codes waiting = await client.get("/authorize", params=FieldCodec.encode(query)) assert waiting.status == 200 held = store.read().requests[query.request_id] - store.update(lambda grants: replace(grants, requests={ + store.update(lambda grants: DocumentChange.to(replace(grants, requests={ query.request_id: replace(held, expires_at=time.time() - 1) - })) + }))) fresh = replace(query, state="fresh-wait") assert (await client.get("/authorize", params=FieldCodec.encode(fresh))).status == 200 assert query.request_id not in store.read().requests diff --git a/tests/test_coordination.py b/tests/test_coordination.py index 9aa5cece55..f6499794ac 100644 --- a/tests/test_coordination.py +++ b/tests/test_coordination.py @@ -168,7 +168,6 @@ def snapshot( obligations=(response,) if record.origin is ExecutionOrigin.WIRE and response else (), publication_intents=(), publication_evidence=(), - connectivity=None, last_recovery=None, current_execution_id=PointedExecution(record.execution_id) if current else NoPointedExecution(), current_attempt_ordinal=(PointedAttempt(record.lifecycle.current_attempt_ordinal) if current @@ -954,7 +953,6 @@ def test_missing_replay_cannot_later_reauthorize_failed(db): obligations=(), publication_intents=(), publication_evidence=(), - connectivity=None, last_recovery=None, current_execution_id=NoPointedExecution(), current_attempt_ordinal=NoPointedAttempt(), @@ -1258,7 +1256,6 @@ def test_populated_durable_authorities_reject_delete_and_survive_reopen(db): activate(c) c.execute("INSERT INTO participant_aliases VALUES ('old-worker','p',100)") c.execute("INSERT INTO replay_assessments VALUES ('e',0,1,0,1)") - c.execute("INSERT INTO connectivity VALUES ('e','connected','connected',1,100)") c.execute( "INSERT INTO recovery_audit (execution_id,kind,reason_code," "sanitized_detail,attempt,elapsed_ms,observed_at_ms) " @@ -1275,7 +1272,6 @@ def test_populated_durable_authorities_reject_delete_and_survive_reopen(db): "execution_claims", "replay_assessments", "obligations", - "connectivity", "recovery_audit", ) for table in tables: @@ -1366,19 +1362,10 @@ def test_obligation_same_state_rewrite_is_rejected_including_terminal(db): -def test_recovery_and_connectivity_monotonic_reopen(db): +def test_recovery_audit_immutable_reopen(db): c, path = db sql_execution(c) activate(c) - c.execute("INSERT INTO connectivity VALUES ('e','connected','connected',1,100)") - for sql in ( - "UPDATE connectivity SET revision=0 WHERE execution_id='e'", - "UPDATE connectivity SET revision=3 WHERE execution_id='e'", - "UPDATE connectivity SET revision=2,observed_at_ms=99 WHERE execution_id='e'", - ): - with pytest.raises(sqlite3.IntegrityError): - c.execute(sql) - c.execute("UPDATE connectivity SET revision=2,observed_at_ms=101 WHERE execution_id='e'") c.execute( "INSERT INTO recovery_audit (execution_id,kind,reason_code," "sanitized_detail,attempt,elapsed_ms,observed_at_ms) " @@ -1392,9 +1379,6 @@ def test_recovery_and_connectivity_monotonic_reopen(db): c.execute(sql) with CoordinationStore(path) as reopened: reopened._connection.row_factory = None - assert reopened._connection.execute( - "SELECT revision,observed_at_ms FROM connectivity" - ).fetchone() == (2, 101) assert ( reopened._connection.execute("SELECT kind FROM recovery_audit").fetchone()[0] == "model_stalled" @@ -1415,7 +1399,6 @@ def test_recovery_and_connectivity_monotonic_reopen(db): "obligations", "publication_intents", "publication_receipts", - "connectivity", "recovery_audit", "participant_aliases", ], diff --git a/tests/test_coordination_store.py b/tests/test_coordination_store.py index a94891ccbe..ca14665040 100644 --- a/tests/test_coordination_store.py +++ b/tests/test_coordination_store.py @@ -55,7 +55,6 @@ from agent_comms.coordination_tables.publications import ( PublicationIntents, ) -from agent_comms.coordination_tables.recovery import ACPClientConnectivity, OwnerConnectivity from agent_comms.coordinator import StandaloneCoordination from agent_comms.response_acknowledgement import canonical_publication_key from agent_comms.execution_states import ( @@ -886,26 +885,9 @@ def test_ambiguity_forces_failed_partition(db_path: Path) -> None: assert settled.value.replay.require_assessment().facts == ReplayFact.TOOL_EXECUTED -def test_fenced_connectivity_audit_and_monotone_replay(db_path: Path) -> None: +def test_fenced_recovery_audit_and_monotone_replay(db_path: Path) -> None: with ready(db_path) as db: _, fence = started(db) - observed = db.observe_connectivity( - fence, - expected_pointer_revision=1, - expected_revision=None, - owner=OwnerConnectivity.RECONNECTING, - acp_client=ACPClientConnectivity.DISCONNECTED, - ) - assert observed.value.connectivity is not None - assert observed.value.connectivity.revision == 1 - with pytest.raises(StaleRevision): - db.observe_connectivity( - fence, - expected_pointer_revision=1, - expected_revision=None, - owner=OwnerConnectivity.OFFLINE, - acp_client=ACPClientConnectivity.DISCONNECTED, - ) with pytest.raises(IdentityConflict): db.append_recovery_audit( fence, expected_pointer_revision=1, kind=FailedRecovery, reason_code=GivenReasonCode("invented") diff --git a/tests/test_goal_failure_observation.py b/tests/test_goal_failure_observation.py index 5b9e2c271d..0d0a8793cc 100644 --- a/tests/test_goal_failure_observation.py +++ b/tests/test_goal_failure_observation.py @@ -8,7 +8,7 @@ import pytest from agent_comms.child_process import ProcessIdentity -from agent_comms.diagnostics import FailureReason +from agent_comms.turn_failure import BackendFailed, FinalStopMissing from agent_comms.goal_attempts import ( AttemptRecord, GoalAttemptStore, @@ -43,7 +43,7 @@ def bound(tmp_path, *, thread_registration): observation = FailedTurnObservation.from_terminal( permit.reservation, owner=owner, goal=goal, lease=lease, turn_id=lease.turn_id, claim_scope=3, current_owner=owner, current_claim=3, - reason=FailureReason.FINAL_STOP_MISSING, + failure=FinalStopMissing, ) assert observation is not None return store, owner, lease, observation @@ -81,7 +81,7 @@ def test_mismatched_turn_lease_is_not_bound(bound, mutation): rejected = FailedTurnObservation.from_terminal( observation.reservation, owner=owner, goal=owner.goal, lease=replace(lease, **mutation), turn_id=lease.turn_id, claim_scope=3, - current_owner=owner, current_claim=3, reason=FailureReason.BACKEND_FAILED, + current_owner=owner, current_claim=3, failure=BackendFailed, ) assert rejected is None observation.reservation.fail(store, "failed anyway", observation=rejected) diff --git a/tests/test_headless_diagnostics.py b/tests/test_headless_diagnostics.py index 68428f81db..466eb3463d 100644 --- a/tests/test_headless_diagnostics.py +++ b/tests/test_headless_diagnostics.py @@ -4,6 +4,7 @@ import os from agent_comms.diagnostics import record_terminal_failure +from agent_comms.turn_failure import NativePreflightTimeout def test_terminal_diagnostic_excludes_untrusted_payloads(tmp_path): @@ -12,22 +13,18 @@ def test_terminal_diagnostic_excludes_untrusted_payloads(tmp_path): turn_id="a" * 32, thread="worker", sequences=(6943,), - event={ - "reason_code": "native_preflight_timeout", - "text": "SECRET stderr data:image/png;base64,ABC123", - "prompt": "private user text", - "diagnostic": { - "wait_ms": 5000, - "session_bytes": 88255627, - "exit_code": -15, - "token": "SECRET", - "spawn_ms": "SECRET", - }, + failure=NativePreflightTimeout, + measurements={ + "wait_ms": 5000, + "session_bytes": 88255627, + "exit_code": -15, + "token": "SECRET", + "spawn_ms": "SECRET", }, ) raw = path.read_text() value = json.loads(raw) - assert value["reason"] == "native_preflight_timeout" + assert value["failure"] == NativePreflightTimeout.declared_name assert value["measurements"] == {"wait_ms": 5000, "session_bytes": 88255627, "exit_code": -15} assert value["sequences"] == [6943] assert all(fragment not in raw for fragment in ("SECRET", "private user", "base64")) diff --git a/tests/test_host_services.py b/tests/test_host_services.py index a94ced916c..9e1d552a1d 100644 --- a/tests/test_host_services.py +++ b/tests/test_host_services.py @@ -17,6 +17,7 @@ from agent_comms.installer.backup import StoreBackup from agent_comms.launch_root import LaunchRoot from agent_comms.launch_runtime import LaunchRuntime +from agent_comms.locked_store import DocumentChange def test_the_connector_is_a_declared_service(): @@ -40,7 +41,7 @@ def connector_service(comms, tmp_path, monkeypatch): home.mkdir() monkeypatch.setenv("HOME", str(home)) store = ConnectorStore.owned() - store.update(lambda grants: grants) + store.update(DocumentChange.to) with socket.socket() as reserved: reserved.bind(("127.0.0.1", 0)) port = reserved.getsockname()[1] diff --git a/tests/test_locked_store.py b/tests/test_locked_store.py index 4d51069fb5..62dbfe7a35 100644 --- a/tests/test_locked_store.py +++ b/tests/test_locked_store.py @@ -10,7 +10,7 @@ import pytest -from agent_comms.locked_store import LockedStore +from agent_comms.locked_store import DocumentChange, LockedStore from agent_comms.pi_counts import PiCount from agent_comms.runtime_info import AgentRuntimeInfo, RuntimeInfoDocument, RuntimeInfoStore from agent_comms.store_files import ExclusiveLock, SharedLock, _store_lock @@ -21,22 +21,21 @@ def observation(thread="goal"): def record(store, value): - store.update(lambda document: RuntimeInfoDocument({**document.observations, value.thread: value})) + store.update(lambda document: DocumentChange.to( + RuntimeInfoDocument({**document.observations, value.thread: value}) + )) def clear(store, thread, *, timestamp=None): - removed = False - def change(document): - nonlocal removed current = document.observations.get(thread) if current is None or (timestamp is not None and current.timestamp != timestamp): - return document - removed = True - return RuntimeInfoDocument({key: row for key, row in document.observations.items() if key != thread}) + return DocumentChange(document, False) + return DocumentChange(RuntimeInfoDocument( + {key: row for key, row in document.observations.items() if key != thread} + ), True) - store.update(change) - return removed + return store.update(change) def _hold_lock(path, mode, ready, release): diff --git a/tests/test_mcp_acceptance.py b/tests/test_mcp_acceptance.py index 9521ebf5b0..6da5b3fa0c 100644 --- a/tests/test_mcp_acceptance.py +++ b/tests/test_mcp_acceptance.py @@ -7,7 +7,6 @@ """ from __future__ import annotations -from agent_comms.native_package import pinned_native_package import asyncio import importlib.util @@ -44,8 +43,7 @@ TIMEOUT = 40 -def _node_run(node, env, script, cwd, artifact): - native_package = pinned_native_package() +def _node_run(native_package, node, env, script, cwd, artifact): command = [ node, "--no-global-search-paths", "--import", str(native_package / "dist/agent-comms-import-fence.mjs"), @@ -124,6 +122,7 @@ def _prepare(root, native_package): } api = native_package / "dist" / "index.js" _node_run( + native_package, node, env, f"import {{ProjectTrustStore}} from {json.dumps(api.as_uri())};" @@ -132,6 +131,7 @@ def _prepare(root, native_package): root / "project-trust", ) digest = _node_run( + native_package, node, env, "import {declarationDigest,parseNativeConfig} from " @@ -142,6 +142,7 @@ def _prepare(root, native_package): ) # Initial fixture setup uses the real package writer, never a Toad ledger. _node_run( + native_package, node, env, "import {recordProjectDecision} from " @@ -342,12 +343,12 @@ async def _observer(case, root): @pytest.mark.parametrize("case", ["allow", "no_controller", "revoke_midturn", "disconnect"]) -async def test_real_pi_mcp_acp_link(case, tmp_path, monkeypatch): +async def test_real_pi_mcp_acp_link(case, tmp_path, monkeypatch, request): binary = os.environ.get("AC_MCP_NATIVE_BIN") if not binary: pytest.skip("Set AC_MCP_NATIVE_BIN to an explicitly prepared native Pi launcher") assert binary == "pi" or (Path(binary).is_absolute() and os.access(binary, os.X_OK)) - native_package = pinned_native_package() + native_package = request.getfixturevalue("pinned_native") package, node, agent, project, digest, starts, calls, env = _prepare( tmp_path, native_package ) @@ -363,7 +364,7 @@ async def test_real_pi_mcp_acp_link(case, tmp_path, monkeypatch): json.dumps( { "launcher": binary, - "nativePackage": str(pinned_native_package()), + "nativePackage": str(native_package), "package": str(package), "case": case, } diff --git a/tests/test_mcp_relay.py b/tests/test_mcp_relay.py index 7c9fd0ddff..618b4fbe8f 100644 --- a/tests/test_mcp_relay.py +++ b/tests/test_mcp_relay.py @@ -3,7 +3,6 @@ import asyncio import json import sys -from pathlib import Path import pytest @@ -11,29 +10,16 @@ from agent_comms.pi_payloads import McpLiveAcknowledgement from agent_comms.owner_protocol import OwnerLine, OwnerPermissionRequest from agent_comms.runtime import SocketClient -from pi_session_turn import one_turn_events pytestmark = [ pytest.mark.skipif(sys.platform == "win32", reason="POSIX executable stub"), ] -def stub(tmp_path: Path, body: str) -> str: - path = tmp_path / "pi-rpc-stub" - path.write_text(f"#!{sys.executable}\n" + body) - path.chmod(0o755) - return str(path) -async def collect_backend(program, cwd, controller): - return [ - event - async for event in one_turn_events( - program, [], "fixture", str(cwd), ui_request=controller - ) - ] def test_live_receipt_rejects_stale_input_malformed_and_ambiguous_claims(): diff --git a/tests/test_native_only_execution.py b/tests/test_native_only_execution.py index a869219f2a..14e1310512 100644 --- a/tests/test_native_only_execution.py +++ b/tests/test_native_only_execution.py @@ -32,10 +32,11 @@ def test_headless_discovery_cannot_admit_an_unnamed_owner(tmp_path, arguments, c assert not root.exists() -async def test_unvalidated_configuration_refuses_before_child(tmp_path): +async def test_unvalidated_configuration_refuses_before_child(tmp_path, pinned_native): missing = tmp_path / "missing-native" with pytest.raises(FileNotFoundError) as caught: - [item async for item in one_turn_events(str(missing), (), "must not send", str(tmp_path))] + [item async for item in one_turn_events( + str(missing), (), "must not send", str(tmp_path), package=pinned_native)] assert caught.value.filename == str(missing) diff --git a/tests/test_native_owner_launcher_resolution.py b/tests/test_native_owner_launcher_resolution.py index 44ef5aa6ba..c6678a5e1d 100644 --- a/tests/test_native_owner_launcher_resolution.py +++ b/tests/test_native_owner_launcher_resolution.py @@ -7,18 +7,17 @@ from agent_comms.session_file import SavedSessionFile -from agent_comms import native_package from agent_comms.launch_root import LaunchRoot from agent_comms.launch_runtime import LaunchRuntime from agent_comms.native_package import VerifiedNativePackage -async def test_fork_runs_selected_python_before_copying_saved_history(tmp_path, monkeypatch, *, thread_registration): +async def test_fork_runs_selected_python_before_copying_saved_history(tmp_path, monkeypatch, pinned_native, *, thread_registration): from agent_comms.comms import Comms from agent_comms.fresh_private_session import create_fresh_private_session from agent_comms.pi_native_backend import PiNativeBackend - package = VerifiedNativePackage.acquire(native_package.pinned_native_package()) + package = VerifiedNativePackage.acquire(pinned_native) comms = Comms(tmp_path / "wire") Bus.install(Bus.path_in(comms.root)) LaunchRoot.from_path(comms.root).install() diff --git a/tests/test_order_claims.py b/tests/test_order_claims.py index 4e41fc9200..91860fbda1 100644 --- a/tests/test_order_claims.py +++ b/tests/test_order_claims.py @@ -28,7 +28,7 @@ ) from agent_comms.tool_caller import ThreadCaller from agent_comms.tools import ( - CommsHistoryTool, CommsOrderDeliverTool, CommsVerdictTool, TodoChanged, + CommsHistoryTool, CommsHudTool, CommsOrderDeliverTool, CommsVerdictTool, TodoChanged, ) from test_held_problem_orders import ( # noqa: F401 - shared copied-bus fixture family held_commander, @@ -68,7 +68,7 @@ def test_claim_verdict_family_on_retained_copy(held_commander): if message.message_id == claimed.order.claim.message.message_id) assert sent.sender == "worker" and sent.target == ctx.commander assert sent.answers and sent.answers == claimed.order.claim.message.answers - assert "Claimed by the unit" in ctx.board_text() + assert "Claimed by the unit" in "\n".join(ctx.call(CommsHudTool()).frame) with pytest.raises(TodoError, match="Only a pending order"): ThreadCaller("worker").invoke(request, comms) assert comms.goals.todos.get("P1") == claimed diff --git a/tests/test_owner_release_restart.py b/tests/test_owner_release_restart.py index 14ac6aefb6..2594824f98 100644 --- a/tests/test_owner_release_restart.py +++ b/tests/test_owner_release_restart.py @@ -1,6 +1,5 @@ """Real release-before-exit: retirement keeps checking the exact owner fence.""" from agent_comms.native_arguments import GivenArguments, GivenBinary -from agent_comms.native_package import pinned_native_package from agent_comms.restart_scope import OwnersNamed from agent_comms.turn_lease import NoTurnLease @@ -89,7 +88,7 @@ async def retire_owned_worker(): result = service.owners.start(name, agent_bin=str(Path(sys.executable).with_name('pi-comms-native')), agent_args=arguments) print(json.dumps(FieldCodec.encode(result)), flush=True) -''', name, str(native.project), str(native.session), str(pinned_native_package())), timeout=20, +''', name, str(native.project), str(native.session), str(native_runtime_package.path)), timeout=20, cwd=native.project, env=environment) assert setup.outcome.successful, setup.stderr.decode() InputDispositions(native.root).replace(original_document) diff --git a/tests/test_pi_rpc_nominal.py b/tests/test_pi_rpc_nominal.py index a91127be5d..d634653209 100644 --- a/tests/test_pi_rpc_nominal.py +++ b/tests/test_pi_rpc_nominal.py @@ -119,7 +119,7 @@ def test_failure_precedence_and_text_have_one_owner(left, right): expected = min((left, right), key=lambda cls: cls.declared_position()) assert type(session.output.failure) is expected assert session.output.failure_text == expected.__name__ - assert session.output.failure.code == expected.code + assert session.output.failure.declared_name == expected.declared_name def test_external_formats_are_derived_and_pin_wire_spellings(): diff --git a/tests/test_preflight_feedback.py b/tests/test_preflight_feedback.py index 0593970bda..8091e32927 100644 --- a/tests/test_preflight_feedback.py +++ b/tests/test_preflight_feedback.py @@ -2,22 +2,21 @@ import pytest from agent_comms.input_attempt import ReservedInput, NotSentInput, MissingInput from agent_comms.input_disposition import InputDocument -from agent_comms.turn_failure import InputIdUnavailable, PromptSendFailed +from agent_comms.turn_failure import InputIdUnavailable, NativePreflightExit, NativePreflightTimeout, PromptSendFailed from agent_comms.turn_output import TurnOutput -from agent_comms.diagnostics import FailureReason from stored_inputs import stored_inputs -@pytest.mark.parametrize('reason', [FailureReason.PREFLIGHT_EXIT, FailureReason.PREFLIGHT_TIMEOUT]) -def test_preflight_diagnostics_keep_original_phase(reason): - output = TurnOutput(preflight_failure=reason) - output.record_failure(InputIdUnavailable('Original preflight phase and timing')) +@pytest.mark.parametrize('kind', [NativePreflightExit, NativePreflightTimeout]) +def test_preflight_diagnostics_keep_original_phase(kind): + output = TurnOutput() + output.record_failure(kind('Original preflight phase and timing')) output.startup_error('Native startup cause\n') assert output.failure_text == 'Original preflight phase and timing\nBackend startup reported:\nNative startup cause' - output = TurnOutput(sensitive=True, preflight_failure=reason) - output.record_failure(InputIdUnavailable('Private phase')) + output = TurnOutput(sensitive=True) + output.record_failure(kind('Private phase')) output.startup_error('private image bytes') assert output.failure_text == 'Image prompt failed; backend diagnostics withheld.' - assert output.preflight_failure is reason + assert type(output.failure) is kind assert 'Private phase' not in repr(output) assert 'private image bytes' not in repr(output) ordinary = TurnOutput(failure=PromptSendFailed('Transport cause')) diff --git a/tests/test_registration_components.py b/tests/test_registration_components.py index 7c7fe61a43..26c957ba07 100644 --- a/tests/test_registration_components.py +++ b/tests/test_registration_components.py @@ -12,7 +12,7 @@ from agent_comms.child_process import ProcessIdentity from agent_comms.errors import RelationViolationError, UnregisteredThreadError from agent_comms.field_codec import FieldCodec -from agent_comms.locked_store import LockedStore +from agent_comms.locked_store import DocumentChange, LockedStore from agent_comms.registration import Registration from agent_comms.registry_document import RegistryDocument, ThreadSlice from agent_comms.registry_store import RegistryStore, UncachedRegistry @@ -94,7 +94,7 @@ def test_registry_update_persists_across_reopen(tmp_path): def change(original): modified = original.copy() modified.threads["owner"] = replace(modified.threads["owner"], title=GivenThreadText("a8 update")) - return modified + return DocumentChange.to(modified) registry.store.update(change) assert Registration(registry.store.path).require("owner").title == GivenThreadText("a8 update") diff --git a/tests/test_runtime_collaboration_documents.py b/tests/test_runtime_collaboration_documents.py index e7a4f10109..df37ba5880 100644 --- a/tests/test_runtime_collaboration_documents.py +++ b/tests/test_runtime_collaboration_documents.py @@ -6,6 +6,7 @@ import pytest from agent_comms.field_codec import FieldCodec +from agent_comms.locked_store import DocumentChange from agent_comms.runtime_info import RuntimeInfoStore from agent_comms.shared_ledger import SharedLedger from test_locked_store import assert_writer_excluded @@ -17,7 +18,7 @@ def test_document_readers_share_lock_and_exclude_updates(tmp_path, store_type): with store.reading() as first: assert first == store_type(store.path).read() == store.empty() assert_writer_excluded(store.path) - store.update(lambda document: document) + store.update(DocumentChange.to) assert store.read() == store.empty() diff --git a/tests/test_selected_tool_native_package.py b/tests/test_selected_tool_native_package.py index 0620ffea0a..5c37da3f1a 100644 --- a/tests/test_selected_tool_native_package.py +++ b/tests/test_selected_tool_native_package.py @@ -1,5 +1,4 @@ """The deployed Pi loader must admit the same tool selected by the owner.""" -from agent_comms.native_package import pinned_native_package import json import os @@ -11,8 +10,8 @@ @pytest.mark.parametrize("packaged", [True, False]) -def test_actual_pi_loader_admits_packaged_channel_tool_only(tmp_path, packaged): - package = pinned_native_package() +def test_actual_pi_loader_admits_packaged_channel_tool_only(tmp_path, packaged, pinned_native): + package = pinned_native extension = selected_extension(package) if not packaged: copied = tmp_path / "selected_claimed_write.mjs" diff --git a/tests/test_todos.py b/tests/test_todos.py index 4374a3d6d6..8dda605deb 100644 --- a/tests/test_todos.py +++ b/tests/test_todos.py @@ -42,6 +42,7 @@ from agent_comms.store_files import store_lock_path from agent_comms.thread_identity import CoreWatcherIncarnation from agent_comms.threads import Thread +from agent_comms.tools import CommsHudTool from agent_comms.todos import ( Assignment, ChangeTodoState, @@ -260,9 +261,7 @@ def test_merged_order_keeps_its_checked_fields(tmp_path: Path, source_repository "P1", RepositoryName.from_wire("OpenHCSDev/agent-comms"), "Fix the sidebar", creator=lead, assigner=lead.incarnation, request=request, owner=lead, generation="initial", review_at=RecordedEventTime(1000.0), order=order(source_repository)) merged = replace(todo, order=todo.order.checked().merged("2b9c1f1ab")) - cells = merged.order.board_cells(merged) - assert cells.startswith(f"{todo.order.checked().owner.to_wire()} | lead | checked-order | {RecordedEventTime(1000.0).clock()} | ") - assert json.loads(cells.split(" | ", 4)[-1]) == FieldCodec.encode(merged.order.checked().check) + assert merged.order.checked().check == todo.order.checked().check assert merged.order.checked().copy_as(PendingProblemOrder) == todo.order assert merged.order.line == "Merged, waiting on install" assert merged.order.waiting_on_install(merged, {}) == (merged,) @@ -529,7 +528,7 @@ def test_installed_merge_closes_only_the_run_that_has_been_proven(pr_commander): assert application.order.sha == mechanical.order.sha assert not application.order.accepted assert application.order.line == application.order.checked().run.merged_line - board = checked.board_text() + board = "\n".join(checked.call(CommsHudTool()).frame) assert application.order.line in board assert worker.name in board store.record_merges(store.merged_orders(checked.incarnation, checked.comms.registry)) diff --git a/tests/test_turn_failure_privacy.py b/tests/test_turn_failure_privacy.py index cd2c95b4bc..8cae43e5c6 100644 --- a/tests/test_turn_failure_privacy.py +++ b/tests/test_turn_failure_privacy.py @@ -22,7 +22,7 @@ def test_recorded_failure_has_one_safe_text_and_keeps_its_behavior(kind, sensiti assert original.text == 'private provider response body' assert stored.text == output.failure_text == expected assert output.terminal_text(False, 'private stderr', 1) == expected - assert stored.code == original.code + assert stored.declared_name == original.declared_name assert stored.declared_position() == original.declared_position() assert stored.ends_with_request_retry == original.ends_with_request_retry if sensitive: diff --git a/tools/cutover/carry_declared_failures.py b/tools/cutover/carry_declared_failures.py new file mode 100644 index 0000000000..1e36f683bc --- /dev/null +++ b/tools/cutover/carry_declared_failures.py @@ -0,0 +1,124 @@ +"""Carry obsolete reason labels into declared failure classes and the failure key. + +Only installation reads the retired spellings. Delete after every retained root +has been carried; running readers have no compatibility decoder. +""" + +from __future__ import annotations + +import json +from contextlib import closing +from pathlib import Path +from typing import ClassVar + +from agent_comms import turn_failure as failures +from agent_comms.diagnostics import DiagnosticDocument +from agent_comms.errors import RelationViolationError +from agent_comms.goal_attempts import AttemptRecord, GoalAttemptStore +from agent_comms.goal_failure_observation import FailedTurnEvidence, GoalLedgerTable +from agent_comms.installer.carry import Carried, Carry +from agent_comms.locked_store import SQLiteFile +from agent_comms.store_files import AtomicFile +from agent_comms.typed_table import HeldBusFile, sql_literal + +from .carry_bus_history import CarryCatalogSource + + +class DeclaredFailures(Carry): + old_names: ClassVar[dict[str, type[failures.TurnFailure]]] = { + "backend_failed": failures.BackendFailed, + "native_preflight_timeout": failures.NativePreflightTimeout, + "native_preflight_exit": failures.NativePreflightExit, + "pi_input_id_unavailable": failures.InputIdUnavailable, + "session_identity_uncertain": failures.IdentityUncertain, + "input_authority_changed": failures.AuthorityChanged, + "unrecognized_followup_input": failures.FollowupUnrecognized, + "current_prompt_input_missing": failures.InputMissing, + "model_request_failed": failures.ModelRequestFailed, + "assistant_final_stop_missing": failures.FinalStopMissing, + "queued_input_start_missing": failures.QueuedInputMissing, + } + + def roots(self) -> tuple[Path, ...]: + return tuple(dict.fromkeys(( + self.root, + *(source.mapped_root(self.root, self.original_root) + for source in CarryCatalogSource.read(self.root, self.original_root)), + ))) + + def backup_files(self) -> tuple[Path, ...]: + paths = tuple( + path for root in self.roots() + for path in ( + *GoalAttemptStore.files(GoalAttemptStore.path_in(root)), + *DiagnosticDocument.files_in(root), + *GoalAttemptStore.source_paths(root), + ) if path.exists() + ) + for path in paths: + self.require_confined(self.root, path) + return paths + + def carry(self, *, apply: bool) -> Carried: + self.backup_files() + if apply: + # The installer's root hold also covers its retained, inactive snapshots. + HeldBusFile(GoalAttemptStore, GoalAttemptStore.path_in(self.root)).require() + results = tuple(self.carry_root(root, apply=apply) for root in self.roots()) + return Carried( + sum(result.changed for result in results), + rewritten=tuple(name for result in results for name in result.rewritten), + ) + + def carry_root(self, root: Path, *, apply: bool) -> Carried: + changed = 0 + rewritten: list[str] = [] + path = GoalAttemptStore.path_in(root) + if path.exists(): + with closing(SQLiteFile.writer(":memory:")) as prepared: + with closing(SQLiteFile.read_snapshot(path)) as original: + original.backup(prepared) + columns = FailedTurnEvidence.stored_columns(prepared) + if "reason" in columns: + for (reason,) in prepared.execute( + f'SELECT DISTINCT reason FROM "{FailedTurnEvidence.declared_name}"' + ): + if reason not in self.old_names: + raise RelationViolationError(f"Unknown stored failure at {path}: {reason!r}") + cases = " ".join( + f"WHEN {sql_literal(old)} THEN {sql_literal(kind)}" + for old, kind in self.old_names.items() + ) + FailedTurnEvidence.rebuild(prepared, {"failure": f"CASE reason {cases} END"}) + changed += 1 + if AttemptRecord.stored_columns(prepared): + for old, kind in self.old_names.items(): + if old != kind.declared_name: + changed += prepared.execute( + f'UPDATE "{AttemptRecord.declared_name}" SET resolution=? ' + "WHERE resolution=? AND json_extract(phase,'$.kind')='failed'", + (kind.declared_name, old), + ).rowcount + if changed: + GoalLedgerTable.require_schema(prepared) + if apply: + prepared.commit() + with closing(SQLiteFile.writer(path)) as destination: + prepared.backup(destination) + GoalAttemptStore.publish_change(path) + rewritten.extend(str(file.relative_to(self.root)) for file in ( + path, *GoalAttemptStore.source_paths(root), + )) + for diagnostic in DiagnosticDocument.matching_in(root): + document = json.loads(diagnostic.read_text()) + if "turn_id" not in document or "reason" not in document: + continue + if "failure" in document: + raise RelationViolationError(f"Both failure formats are present at {diagnostic}") + old = document.pop("reason") + document["failure"] = self.old_names[old].declared_name + if apply: + AtomicFile(diagnostic).write_text(json.dumps(document, indent=2) + "\n") + rewritten.append(str(diagnostic.relative_to(self.root))) + changed += 1 + return Carried(changed, rewritten=tuple(rewritten)) diff --git a/tools/cutover/carry_drop_connectivity.py b/tools/cutover/carry_drop_connectivity.py new file mode 100644 index 0000000000..33cdebac74 --- /dev/null +++ b/tools/cutover/carry_drop_connectivity.py @@ -0,0 +1,48 @@ +"""Drop the connectivity table the coordinator no longer declares. + +Schema installation creates declared tables and never drops undeclared ones, so +every coordination database written before the facet was deleted still holds the +table and its four triggers. Dropping the table drops its triggers and index. + +Delete after every bus has run this step. +""" + +from __future__ import annotations + +from contextlib import closing +from pathlib import Path + +from agent_comms.coordination_database import CoordinationStore +from agent_comms.installer.carry import Carried, Carry +from agent_comms.locked_store import SQLiteFile +from agent_comms.store_files import _store_lock + +TABLE = "connectivity" + + +class DropConnectivity(Carry): + @property + def path(self) -> Path: + return self.root / CoordinationStore.filename + + def backup_files(self) -> tuple[Path, ...]: + self.require_confined(self.root, self.path) + return (self.path,) if self.path.exists() else () + + def carry(self, *, apply: bool) -> Carried: + if not self.path.exists(): + return Carried(0) + with _store_lock(self.path), closing( + CoordinationStore.writer(self.path, isolation_level=None) + ) as db: + if not db.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (TABLE,) + ).fetchone(): + return Carried(0) + (rows,) = db.execute(f'SELECT count(*) FROM "{TABLE}"').fetchone() + if not apply: + return Carried(1, {TABLE: rows}) + CoordinationStore.schema_change(self.path).require() + with SQLiteFile.write_transaction(db): + db.execute(f'DROP TABLE "{TABLE}"') + return Carried(1, {TABLE: rows}, (CoordinationStore.filename,))