From 8049393acf85def700acff5d14954f7c25947ae8 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sun, 1 Nov 2026 01:53:01 -0500 Subject: [PATCH 01/28] Retain selected original journal context entries --- src/agent_comms/context_store.py | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/src/agent_comms/context_store.py b/src/agent_comms/context_store.py index 18246929c8..3fd1815795 100644 --- a/src/agent_comms/context_store.py +++ b/src/agent_comms/context_store.py @@ -8,7 +8,7 @@ import sqlite3 from abc import ABC, abstractmethod from collections.abc import Iterable, Iterator -from contextlib import contextmanager +from contextlib import closing, contextmanager from dataclasses import dataclass, field from pathlib import Path from typing import Annotated, ClassVar, Self @@ -484,6 +484,33 @@ def entries( session_file=identity.session_file, entry_id=entry_id)} + def retain_entries( + self, + destination: ContextStore, + identities: Iterable[NativeSessionIdentity], + ) -> int: + """Retain original journal-entry manifests in a fresh context store. + + Session identity is data here: neither its path nor its entry keys are + relocated. Request/predecessor state and producer identity do not travel + with these complete entry manifests. The primary-key prefix selects only + the requested journals; streaming never decodes unrelated context history. + """ + if destination.path.exists() or destination.path.is_symlink(): + raise FileExistsError(destination.path) + count = 0 + with self.snapshot() as source, destination.writing() as retained: + for identity in dict.fromkeys(identities): + with closing(ContextEntry.stream( + source, + session_id=identity.session_id, + session_file=identity.session_file, + )) as entries: + for entry in entries: + entry.insert(retained) + count += 1 + return count + def prior(self, owner: ThreadIncarnation) -> RecordedContextState: """Read the thread's previous request from a snapshot, outside any write lock.""" with self.snapshot() as db: From e7368e03bf4cca8c0e18212601685d2829fa9612 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sun, 1 Nov 2026 04:58:03 -0500 Subject: [PATCH 02/28] Expose column and table foreign-key declarations for retained selection --- docs/retained-practice-inputs.md | 105 +++++++++++++++++++++++++++++++ src/agent_comms/typed_table.py | 23 +++++++ 2 files changed, 128 insertions(+) create mode 100644 docs/retained-practice-inputs.md diff --git a/docs/retained-practice-inputs.md b/docs/retained-practice-inputs.md new file mode 100644 index 0000000000..02f06bf2a0 --- /dev/null +++ b/docs/retained-practice-inputs.md @@ -0,0 +1,105 @@ +# Original-coordinate practice inputs — implementation checkpoint + +Core #3829 remains unfinished. Entry-manifest retention exists; bounded coordinator +retention, native-journal reading and the prepared-root application run do not. +This document records the source decisions, not an executable selection roster. + +## Existing owners and implemented entry selection + +`ContextStore.retain_entries(destination, identities)` streams `ContextEntry` rows +through `TypedTable` under the source snapshot and destination write lifetime. +Original session/file/item keys and manifest values remain unchanged. It refuses +an existing destination. It does not copy `ContextRequest`, segment versions or +store identity and therefore does not qualify the saved-request browser. + +The earlier owner-path run retained 6,274 entries in 13,877,248 materialized bytes; +source-before, retained and source-after row digests matched. No native or physical +acceptance follows from that result. + +## Foreign-key inspection prerequisite + +Previously `TypedTable.references()` exposed only table-level constraints. Column +references, including inherited/generated columns, live on `_columns` through +`Column.references`. Reading only the former silently misses edges such as +`NativeRuntimeInput.execution_id` and `owner_lookup`. + +The new `TypedTable.foreign_keys()` derives both forms. `Reference.foreign_keys` +requires an answer from the reference kind: `NoReference` contributes no edge, +while `TableReference` binds its declared target to the source column. Composite +`ForeignKey` declarations retain their original objects and constraint options. +There is no table-name roster, SQL parser, alternate codec or stored graph. + +SQL generation remains unchanged: column clauses and table constraints keep their +existing placement, so this inspection API does not change a stored schema or +require a carry. This is a prerequisite API, not a wired retention selector, and +has no application acceptance yet. + +## Required preservation that SQL foreign keys do not establish + +* **Uncertain inputs:** `InputDispositions.unresolved(entries)` joins original + ledger state with `harness_attempts(entries)`. `stored_unresolved()` is not an + equivalent seed. Stored rows in every state suppress a synthesized attempted + key; copying only unresolved ledger rows could resurrect a completed attempt. + No synthesized `ReservedInput` is written back merely to construct a copy. +* **Harness claims:** `HarnessDelivery.settle_claims` and + `AssignmentStore.settle_harness_inputs` relate the original recipient and + message reference to claims. `HarnessDelivery.references()` declares only its + participant dependency. The natural-key claim relation is not an FK edge. +* **Sealed acknowledgements:** `ClaimBatchAcknowledgements.column_guards` requires + complete member/delivery sets and their original counts. Forward FK closure + does not supply those children. Child insert guards reject an already-sealed + parent: loading must not temporarily rewrite the retained sealed state. + `TypedTable.rebuild` already distinguishes loading writable data from installing + guards; the retained-copy implementation must use that owning mechanism. +* **Native source membership:** `NativeInputExecution.source_membership_sql` and + `source_assignment_ids` own full/triage membership. Selection must consume that + behavior rather than maintain another list of stages or link tables. +* **Producer/publication evidence:** executions, attempts, obligations, immutable + intents, receipts and append dispatches carry original state and cyclic or + generated-column references. A lifecycle is not reset to make insertion easy. + The complete child/serialized-reference closure is still OPEN. +* **Serialized references:** `AnnotationRequestsRow.request` contains the original + `DisclosureRequest.grant` message reference and disclosure state. Its submitted + outcome means never resend automatically. `TranscriptRoute`/`InputDisplay` + routing, native context proofs, entry references and saved-request predecessor + links also require their existing value/store owners; absence from an FK list + is not evidence that these relations do not exist. + +## Copy and backup are different contracts + +`PracticeCopy.include` currently invokes each `RootFile` owner's whole-file +`copy_group`. At the checkpoint baseline, `PracticeBus.select` returns the same +input union for rehearsal and live backup. A bounded subset must not be labelled +as a complete original-store backup. Keep ordinary copy/backup behavior intact; +represent a bounded selection explicitly through the existing request/copy +owners. Do not add a parallel importer, ledger or reminted session identities. + +Measure the complete selected stores, WAL/copy groups and free space before any +copy. The requested production-copy proof is bounded to 3 GiB; zero freelist pages +in the oversized stores means compaction is not the repair. + +## Source evidence and remaining work + +The retained NRA source map at baseline `095c4cdfe85632f990be3b6d67411d9685e6dee9` +covers 632 Core `src` modules, 6,152 original class declarations/projections, +105 table-family source declarations, 7,263 annotations and 48,319 call sites, +with no parse omissions. The composition-derived projection contains 35 +declarations, including family/schema bases, not 35 concrete tables. It follows +`PrivateCoordinatorSchema`'s actual schema parents, including `AwarenessTable`: +a narrower coordinator/private-only query missed awareness's two declarations. +Exact hashes and source spans are retained in the owning run's +`p867-coordination-owner-map.json` and +`p867-coordination-composed-declarations.json`. The earlier narrower query is +retained as counterevidence, not used as complete schema coverage. + +Call spellings and nominal projections do not prove runtime binding or semantic +closure. External/dynamic bindings, serialized links and required reverse children +remain OPEN. The earlier d4/Toad corpus and its unresolved cases are separate; +these counts do not resolve them. New metadata methods require targeted reproof. + +Next: implement the declared-reference traversal and owned semantic groups, +consume them in the bounded copy request without weakening full backups, then +qualify the coherent implementation through the combined backend/carry and actual +prepared-root saved-history journey. No pytest, provider, donor mutation or new +native/physical run was performed for this source checkpoint. Reader promotion +still requires the separate default-storage and decorator creation-cut proofs. diff --git a/src/agent_comms/typed_table.py b/src/agent_comms/typed_table.py index bbd0948c23..fd19e44506 100644 --- a/src/agent_comms/typed_table.py +++ b/src/agent_comms/typed_table.py @@ -299,9 +299,17 @@ class Reference(ABC): @abstractmethod def reference_sql(self) -> str: ... + @abstractmethod + def foreign_keys(self, columns: tuple[str, ...]) -> tuple[ForeignKey, ...]: + """References made by these source columns; every reference declares its answer.""" + ... + @dataclass(frozen=True) class NoReference(Reference): + def foreign_keys(self, columns: tuple[str, ...]) -> tuple[ForeignKey, ...]: + return () + def reference_sql(self) -> str: return "" @@ -311,6 +319,9 @@ class TableReference(Reference): target: type[TypedTable] target_columns: tuple[str, ...] + def foreign_keys(self, columns: tuple[str, ...]) -> tuple[ForeignKey, ...]: + return (ForeignKey(target=self.target, target_columns=self.target_columns, columns=columns),) + def reference_sql(self) -> str: return ( f" REFERENCES {_identifier(self.target.declared_name)} " @@ -2230,9 +2241,21 @@ def _member_declarations(cls) -> tuple[type[DeclaredFamily], ...]: @classmethod def references(cls) -> tuple[ForeignKey, ...]: + """Table-level constraints; column references remain on their fields.""" # A method permits mutually referring row classes without string names. return () + @classmethod + def foreign_keys(cls) -> tuple[ForeignKey, ...]: + """All declared SQL edges, including inherited and generated columns. + + This describes referential integrity, not required dependent children or + references encoded inside a field's value. SQL generation keeps its + existing column/table placement; inspection never reparses that SQL. + """ + return (*cls.references(), *(reference for item in cls._columns + for reference in item.column.references.foreign_keys((item.name,)))) + @classmethod def _computed_columns(cls) -> frozenset[str]: return frozenset(item.wire_name for item in cls._columns if not item.column.writable) From 966c5a3fe7f93252cf5b59eeead752d8e66754d1 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sun, 1 Nov 2026 06:15:04 -0500 Subject: [PATCH 03/28] Add attached-source row retention primitives --- docs/retained-practice-inputs.md | 21 ++++++++++ src/agent_comms/typed_table.py | 66 +++++++++++++++++++++++++++++++- 2 files changed, 85 insertions(+), 2 deletions(-) diff --git a/docs/retained-practice-inputs.md b/docs/retained-practice-inputs.md index 02f06bf2a0..137d7e71ce 100644 --- a/docs/retained-practice-inputs.md +++ b/docs/retained-practice-inputs.md @@ -34,6 +34,27 @@ existing placement, so this inspection API does not change a stored schema or require a carry. This is a prerequisite API, not a wired retention selector, and has no application acceptance yet. +## Attached-source raw-row primitive — source only + +`AttachedPracticeRebuild.retain_rows` selects original writable SQL columns and +inserts them unchanged into its bound destination connection. The existing +`PracticeRebuild.attached` context supplies that connection with the active source; +a mere path descriptor cannot select another attachment through a caller-supplied +database. A declared primary key controls repeat selection; a missing key refuses. +Other constraint conflicts are not hidden by `INSERT OR IGNORE`. + +`ForeignKey.retain_parents` puts the original parent columns on the left of the +comparison, preserving SQLite parent affinity/collation and NULL-key behavior. +Internal aliases derive from the target declaration and cannot shadow its outer +table name. `TableSchema.retain_referenced_rows` expands one pass through declared +edges, including column references. It is not a fixed-point or semantic closure. + +This new primitive is not yet consumed by CoordinationStore/PracticeCopy. Guard-safe +loading, required children, source snapshot consistency, producer metadata, +sequences, schema compatibility, materialized bounds and copy behavior remain +unqualified. No original lifecycle is rewritten and no full-copy/backup behavior +is changed. These methods are implementation prerequisites, not a runnable root. + ## Required preservation that SQL foreign keys do not establish * **Uncertain inputs:** `InputDispositions.unresolved(entries)` joins original diff --git a/src/agent_comms/typed_table.py b/src/agent_comms/typed_table.py index fd19e44506..60648dc544 100644 --- a/src/agent_comms/typed_table.py +++ b/src/agent_comms/typed_table.py @@ -483,6 +483,27 @@ def sql(self, owner: type[TypedTable]) -> str: + (" DEFERRABLE INITIALLY DEFERRED" if self.deferred else "") ) + def retain_parents( + self, owner: type[TypedTable], original: AttachedPracticeRebuild, + ) -> int: + """Original parents of selected rows, using SQLite's FK value comparison. + + The parent column is on the left so its affinity/collation governs the + comparison, as it does for the actual constraint. A NULL source key + therefore contributes no parent; encoded SQL values are not decoded as + one domain type and re-encoded as a different target type. + """ + referencing = _identifier(self.target.declared_name + "_referencing") + related = " AND ".join( + f"{self.target.qualified(parent)} = {referencing}.{owner._column_list((child,))}" + for child, parent in zip(self.columns, self.target_columns, strict=True) + ) + return original.retain_rows( + self.target, + where=f"EXISTS (SELECT 1 FROM main.{_identifier(owner.declared_name)} " + f"AS {referencing} WHERE {related})", + ) + class SqlProjection: """Scalar and record projections read SQLite's result columns at one boundary.""" @@ -1009,12 +1030,12 @@ class PracticeRebuild(SchemaRebuild): database: ClassVar[str] = __qualname__ @contextmanager - def attached(self, db: sqlite3.Connection) -> Iterator[SchemaRebuild]: + def attached(self, db: sqlite3.Connection) -> Iterator[AttachedPracticeRebuild]: with closing(SQLiteFile.read_snapshot(self.path)) as source: db.execute(f'ATTACH DATABASE ? AS "{self.database}"', (self.path.resolve().as_uri() + "?mode=ro",)) try: - yield AttachedPracticeRebuild(source) + yield AttachedPracticeRebuild(source, db) finally: db.execute(f'DETACH DATABASE "{self.database}"') @@ -1022,6 +1043,35 @@ def attached(self, db: sqlite3.Connection) -> Iterator[SchemaRebuild]: @dataclass(frozen=True) class AttachedPracticeRebuild(SchemaRebuild): source: sqlite3.Connection + destination: sqlite3.Connection + + def retain_rows( + self, row: type[TypedTable], *, + where: str = "1", parameters: tuple[object, ...] = (), + ) -> int: + """Copy original writable SQL values through this active source attachment. + + This is a row primitive, not a complete-store preservation certificate. + The caller owns destination freshness, guard/loading lifetime, semantic + dependencies and producer metadata. No stored JSON is reconstructed. + """ + keys = tuple(item.name for item in row._columns if item.column.primary_key) + if not keys: + raise RelationViolationError(f"{row.declared_name} has no declared retention key") + fields = tuple(item.name for item in row._writable_fields()) + retained_alias = _identifier(row.declared_name + "_retained") + retained = " AND ".join( + f"{retained_alias}.{row._column_list((key,))} IS {row.qualified(key)}" for key in keys + ) + table = _identifier(row.declared_name) + return self.destination.execute( + f"INSERT INTO main.{table} ({row._column_list(fields)}) " + f"SELECT {row.qualified(*fields)} " + f"FROM {_identifier(PracticeRebuild.database)}.{table} AS {table} " + f"WHERE ({row._sql(where)}) AND NOT EXISTS " + f"(SELECT 1 FROM main.{table} AS {retained_alias} WHERE {retained})", + parameters, + ).rowcount def rebuild( self, schema: type[DerivedTables], db: sqlite3.Connection, @@ -1175,6 +1225,18 @@ class TableSchema(ABC): def schema_rows(cls) -> tuple[type[TypedTable], ...]: return tuple(TypedTable.members_with(cls)) + @classmethod + def retain_referenced_rows( + cls, original: AttachedPracticeRebuild, + ) -> int: + """One forward-FK expansion from selected rows; not semantic closure. + + Required dependent children and references inside serialized values must + be supplied by their owning declarations, never inferred absent here. + """ + return sum(reference.retain_parents(row, original) + for row in cls.schema_rows() for reference in row.foreign_keys()) + @classmethod def declares_schema(cls) -> bool: """Whether this class names a file's tables, not only a parent of such classes.""" From 4d72c9a2ad3f591e9c825d4e13bacb934f860286 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sun, 1 Nov 2026 06:43:37 -0500 Subject: [PATCH 04/28] Share row loading and retain declared acknowledgement groups --- docs/retained-practice-inputs.md | 30 ++++++++- src/agent_comms/cohort_schema.py | 9 +++ src/agent_comms/typed_table.py | 106 +++++++++++++++++++++++++------ 3 files changed, 124 insertions(+), 21 deletions(-) diff --git a/docs/retained-practice-inputs.md b/docs/retained-practice-inputs.md index 137d7e71ce..17b188580c 100644 --- a/docs/retained-practice-inputs.md +++ b/docs/retained-practice-inputs.md @@ -47,7 +47,8 @@ Other constraint conflicts are not hidden by `INSERT OR IGNORE`. comparison, preserving SQLite parent affinity/collation and NULL-key behavior. Internal aliases derive from the target declaration and cannot shadow its outer table name. `TableSchema.retain_referenced_rows` expands one pass through declared -edges, including column references. It is not a fixed-point or semantic closure. +edges, including column references and required groups supplied by row overrides. +It is not a fixed-point or complete semantic closure. This new primitive is not yet consumed by CoordinationStore/PracticeCopy. Guard-safe loading, required children, source snapshot consistency, producer metadata, @@ -55,6 +56,33 @@ sequences, schema compatibility, materialized bounds and copy behavior remain unqualified. No original lifecycle is rewritten and no full-copy/backup behavior is changed. These methods are implementation prerequisites, not a runnable root. +## Shared loading lifetime and acknowledgement groups — source only + +`TypedTable.loading` owns table creation, the load interval, then installation of +that row's indexes/guards. Existing `rebuild` consumes the same sequence rather +than keeping a separate loading procedure. NRA's ModuleSyntaxIndex, StatementSource +and exact SourceTextPatch selected the original statements and preserved their +source geometry. This is an authored context composition, not a native equivalence +certificate or a class-member promotion. + +`TableSchema.loading_rows` opens one transaction with FK enforcement suspended, +creates every table before yielding, installs every row's guards after loading, +then checks references before committing. Exceptions roll back through the existing +SQLiteFile transaction owner. The same `require_related_rows` check is used by +ordinary schema reconciliation. No ordinary runtime insertion guard is weakened. + +`TypedTable.retain_required_rows` supplies the default empty contribution; +`ClaimBatchAcknowledgements` overrides it because its sealed counts describe the +complete original child group. The schema follows inverse **declared FK edges** +only for that requested parent, rather than expanding every participant's history. +`ForeignKey` owns both directions and their one parent-affinity comparison. +Original sealed state is copied unchanged; no seal transition or replay is issued. + +These are source implementations, not executed copy/rollback/guard controls. +CoordinationStore/PracticeCopy has no selection consumer yet. Natural-key harness, +serialized/cross-store and producer dependencies remain OPEN; complete FK validity +would not establish their preservation or the original group's validity. + ## Required preservation that SQL foreign keys do not establish * **Uncertain inputs:** `InputDispositions.unresolved(entries)` joins original diff --git a/src/agent_comms/cohort_schema.py b/src/agent_comms/cohort_schema.py index 194933cbfa..208db7c9cf 100644 --- a/src/agent_comms/cohort_schema.py +++ b/src/agent_comms/cohort_schema.py @@ -19,6 +19,7 @@ from .thread_text import AbsentText, GivenText, TextChoice from .response_acknowledgement import _canonical from .typed_table import ( + AttachedPracticeRebuild, Column, ColumnGuard, ComposedSchemas, @@ -35,6 +36,7 @@ SharedFileTables, SQLiteJournalMode, TextStorage, + TableSchema, TypedTable, WithoutRowid, ) @@ -118,6 +120,13 @@ class ClaimBatchAcknowledgements( accepted_at_ms: Annotated[int, FrozenColumn(check="accepted_at_ms >= 0")] sealed: bool = False + @classmethod + def retain_required_rows( + cls, original: AttachedPracticeRebuild, schema: type[TableSchema], + ) -> int: + """Retain all original children: sealed counts describe the complete group.""" + return schema.retain_referencing_rows(original, cls) + @classmethod def decision_hash(cls, delivery: Delivered) -> str: encoded = _canonical([decision.to_wire() for decision in delivery.decisions]) diff --git a/src/agent_comms/typed_table.py b/src/agent_comms/typed_table.py index 60648dc544..64b224ce8d 100644 --- a/src/agent_comms/typed_table.py +++ b/src/agent_comms/typed_table.py @@ -494,16 +494,31 @@ def retain_parents( one domain type and re-encoded as a different target type. """ referencing = _identifier(self.target.declared_name + "_referencing") - related = " AND ".join( - f"{self.target.qualified(parent)} = {referencing}.{owner._column_list((child,))}" - for child, parent in zip(self.columns, self.target_columns, strict=True) - ) + related = self._related_columns(owner, referencing, _identifier(self.target.declared_name)) return original.retain_rows( self.target, where=f"EXISTS (SELECT 1 FROM main.{_identifier(owner.declared_name)} " f"AS {referencing} WHERE {related})", ) + def _related_columns(self, owner: type[TypedTable], referencing: str, referenced: str) -> str: + """The parent side always supplies the declared FK comparison semantics.""" + return " AND ".join( + f"{referenced}.{self.target._column_list((parent,))} = " + f"{referencing}.{owner._column_list((child,))}" + for child, parent in zip(self.columns, self.target_columns, strict=True) + ) + + def retain_children(self, owner: type[TypedTable], original: AttachedPracticeRebuild) -> int: + """Original children of selected parents, only when a parent requires that group.""" + referenced = _identifier(owner.declared_name + "_referenced") + related = self._related_columns(owner, _identifier(owner.declared_name), referenced) + return original.retain_rows( + owner, + where=f"EXISTS (SELECT 1 FROM main.{_identifier(self.target.declared_name)} " + f"AS {referenced} WHERE {related})", + ) + class SqlProjection: """Scalar and record projections read SQLite's result columns at one boundary.""" @@ -1229,13 +1244,25 @@ def schema_rows(cls) -> tuple[type[TypedTable], ...]: def retain_referenced_rows( cls, original: AttachedPracticeRebuild, ) -> int: - """One forward-FK expansion from selected rows; not semantic closure. + """One declared dependency expansion, including owners' required groups. - Required dependent children and references inside serialized values must - be supplied by their owning declarations, never inferred absent here. + Undeclared natural-key, serialized and cross-store dependencies remain + open; a zero contribution is not a certificate that they do not exist. """ + rows = cls.schema_rows() return sum(reference.retain_parents(row, original) - for row in cls.schema_rows() for reference in row.foreign_keys()) + for row in rows for reference in row.foreign_keys()) + sum( + row.retain_required_rows(original, cls) for row in rows + ) + + @classmethod + def retain_referencing_rows( + cls, original: AttachedPracticeRebuild, parent: type[TypedTable], + ) -> int: + """Inverse declared edges for an owner that requires its complete child group.""" + return sum(reference.retain_children(row, original) + for row in cls.schema_rows() for reference in row.foreign_keys() + if reference.target is parent) @classmethod def declares_schema(cls) -> bool: @@ -1344,6 +1371,34 @@ def require_readable(cls, db: sqlite3.Connection) -> None: for row in cls.schema_rows(): row.require_decodable_rows(db) + @classmethod + def require_related_rows(cls, db: sqlite3.Connection) -> None: + """Reject stored references without changing their original rows.""" + with closing(SQLiteForeignKeyViolation.iterate( + db.execute('SELECT "table", parent, fkid FROM pragma_foreign_key_check()'), + )) as violations: + for violation in violations: + raise cls.schema_error( + f"{cls.__name__} foreign-key violation: table={violation.table}, " + f"parent={violation.parent}, fkid={violation.fkid}" + ) + + @classmethod + @contextmanager + def loading_rows(cls, db: sqlite3.Connection) -> Iterator[None]: + """Load a fresh declared schema in one transaction, preserving sealed values. + + Every table exists before loading starts; guards are installed only after + all selected rows have loaded. Required children and producer metadata + remain the selector's obligation, not something FK validity proves. + """ + with SQLiteForeignKeys.relations_suspended(db), SQLiteFile.write_transaction(db): + with ExitStack() as loading: + for row in cls.schema_rows(): + loading.enter_context(row.loading(db)) + yield + cls.require_related_rows(db) + @classmethod def create_schema(cls, db: sqlite3.Connection) -> None: cls.create_tables(db, cls.schema_rows()) @@ -1384,14 +1439,7 @@ def reconcile_schema( with SQLiteForeignKeys.relations_suspended(db): with rebuild.attached(db) as attached, SQLiteFile.write_transaction(db): cls.carry_schema(db, rebuild=attached) - with closing(SQLiteForeignKeyViolation.iterate( - db.execute('SELECT "table", parent, fkid FROM pragma_foreign_key_check()'), - )) as violations: - for violation in violations: - raise cls.schema_error( - f"{cls.__name__} foreign-key violation: table={violation.table}, " - f"parent={violation.parent}, fkid={violation.fkid}" - ) + cls.require_related_rows(db) return True @classmethod @@ -2401,6 +2449,25 @@ def drop(cls, db: sqlite3.Connection) -> None: """Drop the declared table; SQLite drops its indexes with it.""" db.execute(f"DROP TABLE IF EXISTS {_identifier(cls.declared_name)}") + @classmethod + def retain_required_rows( + cls, original: AttachedPracticeRebuild, schema: type[TableSchema], + ) -> int: + """Additional groups this row's declaration requires beyond its FK parents.""" + return 0 + + @classmethod + @contextmanager + def loading(cls, db: sqlite3.Connection) -> Iterator[None]: + """Create the table, load its rows, then install its declared guards. + + The caller owns rollback. No retained row state is rewritten to pass + insertion guards that describe ordinary runtime writes. + """ + db.execute(cls._table_sql()) + yield + cls.create(db, if_not_exists=True) + @classmethod def rebuild(cls, db: sqlite3.Connection, changed: Mapping[str, str] = types.MappingProxyType({})) -> None: @@ -2421,10 +2488,9 @@ def rebuild(cls, db: sqlite3.Connection, db.execute(f"CREATE TEMP TABLE {temporary} AS SELECT {selected} FROM {table}") SQLiteCatalogObject.release_guards(db, cls) cls.drop(db) - db.execute(cls._table_sql()) - columns = cls._column_list(tuple(item.name for item in fields)) - db.execute(f"INSERT INTO {table} ({columns}) SELECT {columns} FROM {temporary}") - cls.create(db, if_not_exists=True) + with cls.loading(db): + columns = cls._column_list(tuple(item.name for item in fields)) + db.execute(f"INSERT INTO {table} ({columns}) SELECT {columns} FROM {temporary}") db.execute(f"DROP TABLE {temporary}") @classmethod From 6a9c89ff35dd8019f8b5167c633976ecdd3fee23 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sun, 1 Nov 2026 07:58:53 -0500 Subject: [PATCH 05/28] Read practice metadata through the attached database --- docs/retained-practice-inputs.md | 29 +++++++++ src/agent_comms/bus_tables.py | 7 ++- src/agent_comms/messages.py | 7 ++- src/agent_comms/typed_table.py | 105 ++++++++++++++++++++----------- 4 files changed, 107 insertions(+), 41 deletions(-) diff --git a/docs/retained-practice-inputs.md b/docs/retained-practice-inputs.md index 17b188580c..12c1a4a899 100644 --- a/docs/retained-practice-inputs.md +++ b/docs/retained-practice-inputs.md @@ -56,6 +56,35 @@ sequences, schema compatibility, materialized bounds and copy behavior remain unqualified. No original lifecycle is rewritten and no full-copy/backup behavior is changed. These methods are implementation prerequisites, not a runnable root. +## One attached source for metadata and rows — source only + +`PracticeRebuild.attached` previously opened a separate read-snapshot connection +for producer/last-sequence checks while raw copying read the destination's SQLite +attachment. Under WAL those connections could observe different source revisions. +It now obtains the actual `SQLiteDatabase` through SQLite's database-list reader; +metadata checks and copies address that attachment on the same connection. The +existing reconciliation owner holds their write-transaction lifetime. + +`SQLiteDatabase` composes `TableSource` and owns qualified table names. The default +source preserves ordinary SQLite name search, including legitimate temporary +relations; it is not silently changed to an explicit `main` prefix. The existing +TypedTable query owner supplies this source to select/stream, indexed key batches, +counts, existence and catalog checks. Producer checks and the sole bus identity +prefix consumer use it. No second connection, SQL parser or query codec remains. +The source argument qualifies the selected table, not arbitrary SQL subqueries: +joined relations still need their declaration-owned source qualification. + +The FK predicates deliberately retain explicit `main` references: those are the +selected destination children/parents, not another original-source alias. The +original row still comes through `AttachedPracticeRebuild.retain_rows`. Replacing +those destination references with the original source would change the selection. + +The existing rebuild API forwards the same transaction connection through schema +reconciliation; that parameter is not an independently maintained source snapshot. +No additional rebuild hierarchy or unchanged-whole-file gate was added. Actual +WAL interleaving, source/target isolation and producer-prefix controls are still +unrun, and no CoordinationStore/PracticeCopy consumer is claimed. + ## Shared loading lifetime and acknowledgement groups — source only `TypedTable.loading` owns table creation, the load interval, then installation of diff --git a/src/agent_comms/bus_tables.py b/src/agent_comms/bus_tables.py index c9d8b69ab4..a56b291134 100644 --- a/src/agent_comms/bus_tables.py +++ b/src/agent_comms/bus_tables.py @@ -20,6 +20,7 @@ ProjectionProducerTables, RowNotFoundError, TableReference, + TableSource, TypedRow, ) @@ -112,10 +113,10 @@ class MessageIdentitySource(MessageReference, messages.BusProjectionTables): """A record's existing reference, searchable without recomputing its content hash.""" @classmethod - def practice_sequence(cls, db: sqlite3.Connection, source: sqlite3.Connection) -> int: - through = messages.BusRow.last_seq(source) + def practice_sequence(cls, db: sqlite3.Connection, source: TableSource) -> int: + through = messages.BusRow.last_seq(db, table_source=source) if through: - expected = cls.require(source, seq=through) + expected = cls.require(db, table_source=source, seq=through) if cls.require(db, seq=through) != expected: raise RelationViolationError("Practice bus identity differs at its last sequence") return through diff --git a/src/agent_comms/messages.py b/src/agent_comms/messages.py index ef3e84b3e3..2aedc5bfb4 100644 --- a/src/agent_comms/messages.py +++ b/src/agent_comms/messages.py @@ -30,7 +30,7 @@ ) from agent_comms.typed_table import ( AppendOnlyTables, Column, DerivedTables, DurableTables, ForeignKey, Index, - ProducedTable, SharedFileTables, TableReference, TypedRow, TypedTable, + LOCAL_TABLE_SOURCE, ProducedTable, SharedFileTables, TableReference, TableSource, TypedRow, TypedTable, ) from .change_report import NoChangeReport, ReportEvidence, SourceRoots @@ -109,9 +109,10 @@ class BusRow(BusTables): at: float @classmethod - def last_seq(cls, db: sqlite3.Connection) -> int: + def last_seq(cls, db: sqlite3.Connection, *, + table_source: TableSource = LOCAL_TABLE_SOURCE) -> int: return FieldCodec.decode(int, db.execute( - cls._sql(f'SELECT COALESCE(MAX({{seq}}), 0) FROM "{cls.declared_name}"') + cls._sql(f'SELECT COALESCE(MAX({{seq}}), 0) FROM {table_source.qualified(cls.declared_name)}') ).fetchone()[0]) @classmethod diff --git a/src/agent_comms/typed_table.py b/src/agent_comms/typed_table.py index 64b224ce8d..4fe487c280 100644 --- a/src/agent_comms/typed_table.py +++ b/src/agent_comms/typed_table.py @@ -810,12 +810,36 @@ def __str__(self) -> str: return f"{self.name}: stored {self.stored or 'nothing'}; declared {self.declared or 'nothing'}" +class TableSource: + """A query's table location; the default retains SQLite's normal name search.""" + + __slots__ = () + + def qualified(self, name: str) -> str: + return _identifier(name) + + +LOCAL_TABLE_SOURCE = TableSource() + + @dataclass(frozen=True) -class SQLiteDatabase(TypedRow): +class SQLiteDatabase(TypedRow, TableSource): + """An actual attached database owns qualification through its recorded name.""" + seq: int name: str file: str + @classmethod + def named(cls, db: sqlite3.Connection, name: str) -> Self: + database, = cls.read(db.execute( + "SELECT seq,name,file FROM pragma_database_list WHERE name=?", (name,), + )) + return database + + def qualified(self, name: str) -> str: + return f"{_identifier(self.name)}.{_identifier(name)}" + @dataclass(frozen=True) class SQLiteSchemaObject(TypedRow): @@ -1046,18 +1070,17 @@ class PracticeRebuild(SchemaRebuild): @contextmanager def attached(self, db: sqlite3.Connection) -> Iterator[AttachedPracticeRebuild]: - with closing(SQLiteFile.read_snapshot(self.path)) as source: - db.execute(f'ATTACH DATABASE ? AS "{self.database}"', - (self.path.resolve().as_uri() + "?mode=ro",)) - try: - yield AttachedPracticeRebuild(source, db) - finally: - db.execute(f'DETACH DATABASE "{self.database}"') + db.execute(f'ATTACH DATABASE ? AS "{self.database}"', + (self.path.resolve().as_uri() + "?mode=ro",)) + try: + yield AttachedPracticeRebuild(SQLiteDatabase.named(db, self.database), db) + finally: + db.execute(f'DETACH DATABASE "{self.database}"') @dataclass(frozen=True) class AttachedPracticeRebuild(SchemaRebuild): - source: sqlite3.Connection + source: SQLiteDatabase destination: sqlite3.Connection def retain_rows( @@ -1082,7 +1105,7 @@ def retain_rows( return self.destination.execute( f"INSERT INTO main.{table} ({row._column_list(fields)}) " f"SELECT {row.qualified(*fields)} " - f"FROM {_identifier(PracticeRebuild.database)}.{table} AS {table} " + f"FROM {self.source.qualified(row.declared_name)} AS {table} " f"WHERE ({row._sql(where)}) AND NOT EXISTS " f"(SELECT 1 FROM main.{table} AS {retained_alias} WHERE {retained})", parameters, @@ -1093,7 +1116,7 @@ def rebuild( rows: tuple[type[TypedTable], ...], ) -> None: projections = tuple(row for row in schema.projection_rows() if row in rows) - if not ProjectionProducer.matches(self.source, projections): + if not ProjectionProducer.matches(db, projections, table_source=self.source): raise RelationViolationError("Practice projection producers differ from this runtime") through = max((row.practice_sequence(db, self.source) for row in schema.projection_rows()), default=0) @@ -1101,7 +1124,7 @@ def rebuild( schema.create_tables(db, rows) for row in projections: db.execute(f'INSERT INTO "{row.declared_name}" ' - f'SELECT * FROM "{PracticeRebuild.database}"."{row.declared_name}"') + f'SELECT * FROM {self.source.qualified(row.declared_name)}') schema.replay(db, rows, after=through) @@ -2545,18 +2568,21 @@ def _condition( @classmethod def select_keys(cls, db: sqlite3.Connection, - keys: Sequence[Mapping[str, object]]) -> list[Self]: + keys: Sequence[Mapping[str, object]], *, + table_source: TableSource = LOCAL_TABLE_SOURCE) -> list[Self]: """Rows equal to any of these named keys under the same null-safe IS equality select() uses: the requested keys drive an indexed join, one batch per the connection's own variable limit, each distinct key probed once.""" - return [row for requested in cls.requested_keys(db, keys) for row in cls.iterate(db.execute( + return [row for requested in cls.requested_keys(db, keys, table_source=table_source) + for row in cls.iterate(db.execute( f"WITH {requested.values} " f"SELECT {cls.qualified(*cls.column_projection())} {requested.join}", requested.bindings))] @classmethod def requested_keys(cls, db: sqlite3.Connection, - keys: Sequence[Mapping[str, object]]) -> Iterator[RequestedKeys]: + keys: Sequence[Mapping[str, object]], *, + table_source: TableSource = LOCAL_TABLE_SOURCE) -> Iterator[RequestedKeys]: """The distinct named keys, encoded by their declared columns, in batches that fit the connection's own variable limit; each batch joins this table as ``FROM requested JOIN ON ...`` under select()'s null-safe IS equality.""" @@ -2576,7 +2602,7 @@ def requested_keys(cls, db: sqlite3.Connection, for chunk in batched(bound, batch): yield RequestedKeys( f"requested({','.join(aliases)}) AS (VALUES {','.join(probe for _ in chunk)})", - f"FROM requested JOIN {table} ON {joined}", + f"FROM requested JOIN {table_source.qualified(cls.declared_name)} ON {joined}", tuple(value for _, values in chunk for value in values)) @classmethod @@ -2584,11 +2610,12 @@ def _query( cls, db: sqlite3.Connection, where: str, parameters: tuple, order_by: tuple[str, ...], descending: bool, key: dict[str, object], row_type: type[RelatedRow] | None, limit: int | None, + table_source: TableSource, ) -> Generator[Self | RelatedRow]: condition, parameters = cls._condition(where, parameters, key) direction, selected = " DESC" if descending else "", row_type or cls return selected.iterate(db.execute( - f"SELECT {selected._select_list(cls)} FROM {_identifier(cls.declared_name)} " + f"SELECT {selected._select_list(cls)} FROM {table_source.qualified(cls.declared_name)} " f"WHERE {condition}" + (" ORDER BY " + ", ".join(cls._column_list((name,)) + direction for name in order_by) if order_by else "") + (" LIMIT ?" if limit is not None else ""), @@ -2600,7 +2627,7 @@ def _query( def select( cls, db: sqlite3.Connection, *, row_type: type[RelatedRow], where: str = "1", parameters: tuple = (), order_by: tuple[str, ...] = (), descending: bool = False, - limit: int | None = None, **key: object, + limit: int | None = None, table_source: TableSource = LOCAL_TABLE_SOURCE, **key: object, ) -> list[RelatedRow]: ... @overload @@ -2608,7 +2635,7 @@ def select( def select( cls, db: sqlite3.Connection, *, row_type: None = None, where: str = "1", parameters: tuple = (), order_by: tuple[str, ...] = (), descending: bool = False, - limit: int | None = None, **key: object, + limit: int | None = None, table_source: TableSource = LOCAL_TABLE_SOURCE, **key: object, ) -> list[Self]: ... @classmethod @@ -2621,16 +2648,17 @@ def select( order_by: tuple[str, ...] = (), descending: bool = False, row_type: type[RelatedRow] | None = None, limit: int | None = None, - **key: object, + table_source: TableSource = LOCAL_TABLE_SOURCE, **key: object, ) -> Sequence[Self | RelatedRow]: - return list(cls._query(db, where, parameters, order_by, descending, key, row_type, limit)) + return list(cls._query(db, where, parameters, order_by, descending, key, row_type, limit, + table_source)) @overload @classmethod def stream( cls, db: sqlite3.Connection, *, row_type: type[RelatedRow], where: str = "1", parameters: tuple = (), order_by: tuple[str, ...] = (), descending: bool = False, - limit: int | None = None, **key: object, + limit: int | None = None, table_source: TableSource = LOCAL_TABLE_SOURCE, **key: object, ) -> Generator[RelatedRow]: ... @overload @@ -2638,7 +2666,7 @@ def stream( def stream( cls, db: sqlite3.Connection, *, row_type: None = None, where: str = "1", parameters: tuple = (), order_by: tuple[str, ...] = (), descending: bool = False, - limit: int | None = None, **key: object, + limit: int | None = None, table_source: TableSource = LOCAL_TABLE_SOURCE, **key: object, ) -> Generator[Self]: ... @classmethod @@ -2651,10 +2679,11 @@ def stream( order_by: tuple[str, ...] = (), descending: bool = False, row_type: type[RelatedRow] | None = None, limit: int | None = None, - **key: object, + table_source: TableSource = LOCAL_TABLE_SOURCE, **key: object, ) -> Generator[Self | RelatedRow]: """Rows decoded as the caller consumes them; close it to release the query.""" - return cls._query(db, where, parameters, order_by, descending, key, row_type, limit) + return cls._query(db, where, parameters, order_by, descending, key, row_type, limit, + table_source) @classmethod def wire_record(cls, db: sqlite3.Connection, plan: FieldPlan) -> str: @@ -2731,23 +2760,27 @@ def objects_of( return [item for item in stored if item.name in declared or item.name not in claimed] @classmethod - def present(cls, db: sqlite3.Connection) -> bool: + def present(cls, db: sqlite3.Connection, *, + table_source: TableSource = LOCAL_TABLE_SOURCE) -> bool: return SqlValue.for_annotation(bool).read(db.execute( - "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name=?)", + f"SELECT EXISTS(SELECT 1 FROM {table_source.qualified('sqlite_master')} " + "WHERE type='table' AND name=?)", (cls.declared_name,)))[0] @classmethod - def exists(cls, db: sqlite3.Connection, *, where: str = "1", parameters: tuple = (), **key: object) -> bool: + def exists(cls, db: sqlite3.Connection, *, where: str = "1", parameters: tuple = (), + table_source: TableSource = LOCAL_TABLE_SOURCE, **key: object) -> bool: condition, parameters = cls._condition(where, parameters, key) return SqlValue.for_annotation(bool).read(db.execute( - f"SELECT EXISTS(SELECT 1 FROM {_identifier(cls.declared_name)} WHERE {condition})", + f"SELECT EXISTS(SELECT 1 FROM {table_source.qualified(cls.declared_name)} WHERE {condition})", parameters))[0] @classmethod - def count(cls, db: sqlite3.Connection, *, where: str = "1", parameters: tuple = (), **key: object) -> int: + def count(cls, db: sqlite3.Connection, *, where: str = "1", parameters: tuple = (), + table_source: TableSource = LOCAL_TABLE_SOURCE, **key: object) -> int: condition, parameters = cls._condition(where, parameters, key) return SqlValue.for_annotation(int).read(db.execute( - f"SELECT COUNT(*) FROM {_identifier(cls.declared_name)} WHERE {condition}", parameters))[0] + f"SELECT COUNT(*) FROM {table_source.qualified(cls.declared_name)} WHERE {condition}", parameters))[0] @classmethod @cache @@ -2856,7 +2889,7 @@ def writer_declarations(cls) -> tuple[type[ProjectionOutputs], ...]: if owner.shares_outputs_with(cls)) @classmethod - def practice_sequence(cls, db: sqlite3.Connection, source: sqlite3.Connection) -> int: + def practice_sequence(cls, db: sqlite3.Connection, source: TableSource) -> int: """Only the identity-owning row contributes a verified history prefix.""" return 0 @@ -2976,12 +3009,14 @@ def records_for(cls, rows: tuple[type[ProducedTable], ...]) -> tuple[Self, ...]: return tuple(cls(name, digest) for name, digest in digests.items()) @classmethod - def matches(cls, db: sqlite3.Connection, rows: tuple[type[ProducedTable], ...]) -> bool: + def matches(cls, db: sqlite3.Connection, rows: tuple[type[ProducedTable], ...], *, + table_source: TableSource = LOCAL_TABLE_SOURCE) -> bool: if not rows: return True - if not cls.present(db): + if not cls.present(db, table_source=table_source): return False - stored = {record.projection: record.producer_digest for record in cls.select(db)} + stored = {record.projection: record.producer_digest + for record in cls.select(db, table_source=table_source)} digests = cls.digests_for(rows) return all(stored.get(name) == digest for name, digest in digests.items()) From 8ef5f1ea09ff512a4c28f42c3dccd7018a1e663c Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sun, 1 Nov 2026 10:26:23 -0500 Subject: [PATCH 06/28] Keep unresolved seed projection on InputDispositions --- docs/retained-practice-inputs.md | 20 +++++++++++ .../coordination_tables/harness_deliveries.py | 26 ++++++++++++-- src/agent_comms/input_disposition.py | 35 +++++++++++++------ 3 files changed, 68 insertions(+), 13 deletions(-) diff --git a/docs/retained-practice-inputs.md b/docs/retained-practice-inputs.md index 12c1a4a899..8d7ee30a47 100644 --- a/docs/retained-practice-inputs.md +++ b/docs/retained-practice-inputs.md @@ -143,6 +143,26 @@ would not establish their preservation or the original group's validity. links also require their existing value/store owners; absence from an FK list is not evidence that these relations do not exist. +## Unresolved-input seed projection + +`HarnessDelivery.unconfirmed_for` now owns the incarnation/outcome query. +Membership derives from `Outcome.members_with(UnconfirmedOutcome)`; requested +keys are encoded by the declared columns through `TypedTable.select_keys`, not +hand-written outcome names. Normal input reads use that same query. + +`SnapshotInputDispositions` inherits the unchanged `unresolved(entries)` and +all-state suppression algorithm. Its sole override reads harness rows through an +active `AttachedPracticeRebuild` source on that attachment's connection. It has +no new file/schema declaration. Its `root` must contain the retained bus and +complete ledger, captured consistently with the coordinator source; the caller +owns that capture and attachment lifetime. This is not a second ledger or an +input replay operation. + +The view is not yet consumed by a coordinator materializer or `PracticeCopy`. +Runtime class creation/MRO/registration, the actual attached query, cross-store +capture and complete preservation remain unqualified. No seed-only database is +published as a runnable root or complete backup. + ## Copy and backup are different contracts `PracticeCopy.include` currently invokes each `RootFile` owner's whole-file diff --git a/src/agent_comms/coordination_tables/harness_deliveries.py b/src/agent_comms/coordination_tables/harness_deliveries.py index c1726c4d84..a68f8d883b 100644 --- a/src/agent_comms/coordination_tables/harness_deliveries.py +++ b/src/agent_comms/coordination_tables/harness_deliveries.py @@ -16,10 +16,14 @@ from agent_comms.coordination_schema import CoordinatorRow from agent_comms.coordination_tables.participants import Participants from agent_comms.nominal.field_codec import MessageId, ThreadLookup, TranscriptRecordId -from agent_comms.harness_delivery_states import MonitorEvidence, Outcome, UnrecordedMonitor +from agent_comms.harness_delivery_states import ( + MonitorEvidence, Outcome, UnconfirmedOutcome, UnrecordedMonitor, +) from agent_comms.harness_install import HarnessSession from agent_comms.message_reference import MessageReference -from agent_comms.typed_table import Column, ForeignKey, FrozenColumn, ImmutableRows, Index, NoDelete +from agent_comms.typed_table import ( + LOCAL_TABLE_SOURCE, Column, ForeignKey, FrozenColumn, ImmutableRows, Index, NoDelete, TableSource, +) if TYPE_CHECKING: from agent_comms.assignment_store import AssignmentStore @@ -110,6 +114,24 @@ def for_recipients(cls, db: sqlite3.Connection, key=lambda row: (row.seq, row.recipient_lookup, row.bridge.declared_name), )) + @classmethod + def unconfirmed_for( + cls, db: sqlite3.Connection, owners: tuple[ThreadLookup, ...], *, + table_source: TableSource = LOCAL_TABLE_SOURCE, + ) -> tuple[Self, ...]: + """Original uncertain attempts for these incarnations, in message order. + + The outcome family owns membership; declared columns own query encoding. + A source attachment changes location, not the stored outcome or identity. + """ + outcomes = Outcome.members_with(UnconfirmedOutcome) + keys = tuple( + {"recipient_lookup": owner, "outcome": outcome} + for owner in owners for outcome in outcomes + ) + return tuple(sorted(cls.select_keys(db, keys, table_source=table_source), + key=lambda row: row.seq)) + @classmethod def latest(cls, db: sqlite3.Connection, bridge: type[HarnessSession], recipient_lookup: ThreadLookup) -> tuple[MessageReference, ...]: diff --git a/src/agent_comms/input_disposition.py b/src/agent_comms/input_disposition.py index dd41a7c38d..b1587f4163 100644 --- a/src/agent_comms/input_disposition.py +++ b/src/agent_comms/input_disposition.py @@ -17,8 +17,7 @@ from .coordination_database import CoordinationStore, ReadableStore from .coordination_tables.harness_deliveries import HarnessDelivery from .errors import RelationViolationError -from .nominal.field_codec import GoalId, InputId, RootId, TurnId -from .harness_delivery_states import Outcome, UnconfirmedOutcome +from .nominal.field_codec import GoalId, InputId, RootId, ThreadLookup, TurnId from .input_attempt import ( UNOBSERVED_DELIVERY_SCOPE, GoalInputDecision, @@ -41,6 +40,7 @@ from .reported_turn import ReportedTurnState from .tool_result import ToolResult from .typed_table import ( + AttachedPracticeRebuild, Column, DurableStore, Generated, @@ -287,19 +287,15 @@ def lookup(self, key: InputKey) -> InputAttempt: def coordinator_reader(self) -> ReadableStore: return ReadableStore(CoordinationStore.path_in(self.root)) + def _harness_deliveries(self, owners: tuple[ThreadLookup, ...]) -> tuple[HarnessDelivery, ...]: + with self.coordinator_reader.reader_snapshot() as db: + return HarnessDelivery.unconfirmed_for(db, owners) + def harness_attempts(self, entries: tuple[RegistryEntry, ...]) -> tuple[ReservedInput, ...]: """Uncertain harness attempts for these entries' frozen incarnations, as the inputs they are. Derived on every read and never stored by reading.""" owners = {entry.thread.incarnation.require_recorded().lookup: entry for entry in entries} - outcomes = Outcome.members_with(UnconfirmedOutcome) - with self.coordinator_reader.reader_snapshot() as db: - attempts = HarnessDelivery.select( - db, - where=(f'"recipient_lookup" IN ({", ".join("?" for _ in owners)}) ' - f'AND "outcome" IN ({", ".join("?" for _ in outcomes)})'), - parameters=(*owners, *(outcome.declared_name for outcome in outcomes)), - order_by=("seq",), - ) + attempts = self._harness_deliveries(tuple(owners)) messages = Bus(Bus.path_in(self.root)).input_messages_for_references( attempt.reference for attempt in attempts) return tuple( @@ -565,3 +561,20 @@ def dismiss(document: InputDocument) -> DocumentChange[InputDocument, InputDocum return self._update(*self._owned_or_attempted(owners, attempted), dismiss).record( *attempted).delivery_overview(owners, scope=scope) + + +@dataclass(frozen=True, slots=True) +class SnapshotInputDispositions(InputDispositions, declared_name=None): + """The same unresolved projection, with harness rows from an active source cut. + + ``root`` names the retained bus and complete suppression ledger. The caller + owns their consistent capture and this attachment's lifetime. This transient + view declares no file/schema, writes no synthetic attempt and grants no replay. + """ + + original: AttachedPracticeRebuild + + def _harness_deliveries(self, owners: tuple[ThreadLookup, ...]) -> tuple[HarnessDelivery, ...]: + return HarnessDelivery.unconfirmed_for( + self.original.destination, owners, table_source=self.original.source, + ) From 75c753d08dda01004a7961f7e98cb767721602d0 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sun, 1 Nov 2026 11:12:47 -0500 Subject: [PATCH 07/28] Retain selected saved contexts through PracticeCopy --- docs/retained-practice-inputs.md | 61 +++++++++---- src/agent_comms/context_store.py | 86 +++++++++++++------ src/agent_comms/coordination_database.py | 14 +-- src/agent_comms/coordination_session.py | 3 +- src/agent_comms/installer/practice.py | 25 +++++- src/agent_comms/typed_table.py | 104 +++++++++++++++++++++-- tests/test_context_manifest_deltas.py | 94 +++++++++++++++++++- 7 files changed, 327 insertions(+), 60 deletions(-) diff --git a/docs/retained-practice-inputs.md b/docs/retained-practice-inputs.md index 8d7ee30a47..26376920a1 100644 --- a/docs/retained-practice-inputs.md +++ b/docs/retained-practice-inputs.md @@ -1,20 +1,36 @@ # Original-coordinate practice inputs — implementation checkpoint -Core #3829 remains unfinished. Entry-manifest retention exists; bounded coordinator -retention, native-journal reading and the prepared-root application run do not. +Core #3829 remains unfinished. Entry and selected saved-context retention now have +a PracticeCopy consumer in source; bounded coordinator selection, native-journal +reading and the prepared-root application run remain unfinished. This document records the source decisions, not an executable selection roster. ## Existing owners and implemented entry selection -`ContextStore.retain_entries(destination, identities)` streams `ContextEntry` rows -through `TypedTable` under the source snapshot and destination write lifetime. -Original session/file/item keys and manifest values remain unchanged. It refuses -an existing destination. It does not copy `ContextRequest`, segment versions or -store identity and therefore does not qualify the saved-request browser. - -The earlier owner-path run retained 6,274 entries in 13,877,248 materialized bytes; -source-before, retained and source-after row digests matched. No native or physical -acceptance follows from that result. +`ContextStore.retain_contexts(destination, identities, requests=...)` replaces the +entry-only API. `PracticeCopy.include_contexts` consumes it explicitly; normal +whole-file include and live-root backup selection are unchanged. Selected writable +SQL values travel through the existing attached source, without decode/re-encode, +coordinate relocation, source-identity minting or a compatibility entry point. + +`ReconciledFile.retaining_rows` owns fresh staging, exact source-schema validation, +the existing load-before-guards transaction, repeated declared-dependency expansion, +and exclusive atomic publication. A refusal or failed closure never overwrites an +existing destination or publishes the partially loaded staged database. + +`ContextRequest.retain_required_rows` owns its serialized version IDs, segment +entry references and earlier request IDs. Missing originals and foreign-thread +predecessors refuse. Whole retained predecessor records receive their own closure, +not an incomplete predecessor that only one selected segment can read. The original +ContextStoreInfo is required when requests are selected; entry-only selection does +not invent one. `RowidSequenced` retains SQLite's original sequence high-water mark +so an omitted original ID cannot become the next newly allocated ID. + +The earlier entry-only API's owner-path run retained 6,274 entries in 13,877,248 +materialized bytes with matching source-before/retained/source-after row digests. +It was not rerun on this implementation. Source compilation is not qualification +of the selected saved-request browser, atomic publication, rollback, sequence +metadata, resource bounds or native/physical behavior. ## Foreign-key inspection prerequisite @@ -50,10 +66,15 @@ table name. `TableSchema.retain_referenced_rows` expands one pass through declar edges, including column references and required groups supplied by row overrides. It is not a fixed-point or complete semantic closure. -This new primitive is not yet consumed by CoordinationStore/PracticeCopy. Guard-safe -loading, required children, source snapshot consistency, producer metadata, -sequences, schema compatibility, materialized bounds and copy behavior remain -unqualified. No original lifecycle is rewritten and no full-copy/backup behavior +`retain_keys` reuses TypedTable's indexed key-batch compiler. It shares raw insertion +and primary-key deduplication with `retain_rows`; no second key encoder or raw-copy +procedure is added. Completed changes come from the connection, since SQLite's +WITH-prefixed INSERT reports Cursor.rowcount=-1. + +ContextStore and PracticeCopy now consume the primitive; the coordinator selector +still does not. Guard-safe loading, required children, source snapshot consistency, +producer metadata, sequences, schema compatibility, materialized bounds and copy +behavior remain unqualified. No original lifecycle is rewritten and no full-copy/backup behavior is changed. These methods are implementation prerequisites, not a runnable root. ## One attached source for metadata and rows — source only @@ -170,7 +191,11 @@ published as a runnable root or complete backup. input union for rehearsal and live backup. A bounded subset must not be labelled as a complete original-store backup. Keep ordinary copy/backup behavior intact; represent a bounded selection explicitly through the existing request/copy -owners. Do not add a parallel importer, ledger or reminted session identities. +owners. `PracticeCopy.include_contexts` is an explicit selection API; its request +continues to describe original-to-retained path mapping, not a complete-backup +certificate. BackedUpBus still constructs LiveFilesRequest for the original live +root, rather than backing up the selected practice bytes. Do not add a parallel +importer, ledger or reminted session identities. Measure the complete selected stores, WAL/copy groups and free space before any copy. The requested production-copy proof is bounded to 3 GiB; zero freelist pages @@ -195,8 +220,8 @@ closure. External/dynamic bindings, serialized links and required reverse childr remain OPEN. The earlier d4/Toad corpus and its unresolved cases are separate; these counts do not resolve them. New metadata methods require targeted reproof. -Next: implement the declared-reference traversal and owned semantic groups, -consume them in the bounded copy request without weakening full backups, then +Next: finish coordinator seeds and the remaining semantic groups, wire that +selection through PracticeCopy and enforce the aggregate copy budget, then qualify the coherent implementation through the combined backend/carry and actual prepared-root saved-history journey. No pytest, provider, donor mutation or new native/physical run was performed for this source checkpoint. Reader promotion diff --git a/src/agent_comms/context_store.py b/src/agent_comms/context_store.py index 3fd1815795..666ec378cb 100644 --- a/src/agent_comms/context_store.py +++ b/src/agent_comms/context_store.py @@ -8,7 +8,7 @@ import sqlite3 from abc import ABC, abstractmethod from collections.abc import Iterable, Iterator -from contextlib import closing, contextmanager +from contextlib import contextmanager from dataclasses import dataclass, field from pathlib import Path from typing import Annotated, ClassVar, Self @@ -31,7 +31,10 @@ UnrecordedSegment, ) -from .typed_table import StoreRows, Column, DurableStore, Generated, Index, RowidSequenced, SingletonRow, TypedTable +from .typed_table import ( + AttachedPracticeRebuild, StoreRows, Column, DurableStore, Generated, Index, + RowidSequenced, SingletonRow, TableSchema, TypedTable, +) from .turn_item import NativeItemId from .native_session_reopen import PiSessionId @@ -242,6 +245,35 @@ def recorded(self, db: sqlite3.Connection) -> RecordedContextManifest: parts.append(ContextSegmentVersion.require(db, id=version).segment) return self.identity.with_segments(tuple(parts)) + @classmethod + def retain_required_rows( + cls, original: AttachedPracticeRebuild, schema: type[TableSchema], + ) -> int: + """The stored request owns its serialized version, entry and predecessor edges.""" + changed = super().retain_required_rows(original, schema) + db = original.destination + for request in cls.select(db): + changed += original.retain_rows(ContextStoreInfo) + ContextStoreInfo.require(db, singleton=1) + changed += original.retain_keys( + ContextSegmentVersion, tuple({"id": version} for version in request.versions), + ) + record = request.recorded(db) # Missing original versions refuse, never disappear. + for part in record.segments: + predecessors = part.predecessors(request.id) + changed += original.retain_keys(cls, tuple({"id": key} for key in predecessors)) + for key in predecessors: + if cls.require(db, id=key).identity.thread != request.identity.thread: + raise RelationViolationError("Context predecessor is missing or foreign") + changed += original.retain_keys(ContextEntry, tuple( + {"session_id": ref.identity.session_id, + "session_file": ref.identity.session_file, "entry_id": ref.entry_id} + for ref in part.entries + )) + for reference in part.entries: + ContextEntry.original(db, reference) + return changed + class ContextPriorSupersededError(RelationViolationError): """Another writer recorded this thread's context after the prior this writer holds.""" @@ -484,32 +516,34 @@ def entries( session_file=identity.session_file, entry_id=entry_id)} - def retain_entries( - self, - destination: ContextStore, - identities: Iterable[NativeSessionIdentity], - ) -> int: - """Retain original journal-entry manifests in a fresh context store. + def retain_contexts( + self, destination: ContextStore, identities: Iterable[NativeSessionIdentity], *, + requests: Iterable[ContextManifestReference] = (), + ) -> None: + """Retain original entry manifests and explicitly selected saved requests. - Session identity is data here: neither its path nor its entry keys are - relocated. Request/predecessor state and producer identity do not travel - with these complete entry manifests. The primary-key prefix selects only - the requested journals; streaming never decodes unrelated context history. + Raw writable values, source identity, predecessor/version/entry closure + and sequence high-water marks travel unchanged. Session paths are keys, + not relocated storage or permission to launch or replay a native session. + Entry-only selection retains no request or fabricated store identity. """ - if destination.path.exists() or destination.path.is_symlink(): - raise FileExistsError(destination.path) - count = 0 - with self.snapshot() as source, destination.writing() as retained: - for identity in dict.fromkeys(identities): - with closing(ContextEntry.stream( - source, - session_id=identity.session_id, - session_file=identity.session_file, - )) as entries: - for entry in entries: - entry.insert(retained) - count += 1 - return count + with type(self).retaining_rows(self.path, destination.path) as original: + original.retain_keys(ContextEntry, tuple( + {"session_id": identity.session_id, "session_file": identity.session_file} + for identity in dict.fromkeys(identities) + )) + selected = tuple(dict.fromkeys(requests)) + if selected: + original.retain_rows(ContextStoreInfo) + source = ContextStoreInfo.require(original.destination, singleton=1).source + for reference in selected: + if reference.source != source: + raise RelationViolationError("Selected context belongs to another store") + original.retain_keys(ContextRequest, tuple( + {"id": reference.record_id} for reference in selected + )) + for reference in selected: + ContextRequest.require(original.destination, id=reference.record_id) def prior(self, owner: ThreadIncarnation) -> RecordedContextState: """Read the thread's previous request from a snapshot, outside any write lock.""" diff --git a/src/agent_comms/coordination_database.py b/src/agent_comms/coordination_database.py index 4bccb0aa98..9aebd67ca2 100644 --- a/src/agent_comms/coordination_database.py +++ b/src/agent_comms/coordination_database.py @@ -119,7 +119,7 @@ def __init__( with CoordinationReadUnavailable.checking(): self._connection = self._open_connection(contention) try: - self._configure_connection() + self.configure_connection(self._connection) self._initialize_schema() self._enforce_private_modes() except BaseException: @@ -133,13 +133,15 @@ def _initialize_schema(self) -> None: """The root's own composition of this file decides its schema and its strays.""" self.schema_at(self.path).install_schema(self._connection, self.schema_change(self.path)) - def _configure_connection(self) -> None: + @classmethod + def configure_connection(cls, connection: sqlite3.Connection) -> None: + """Ordinary opening and selected loading use the same row-owned SQL behavior.""" for row_type in CoordinatorTable.declarations(): - row_type.configure_connection(self._connection) - self._connection.row_factory = sqlite3.Row - self._connection.execute("PRAGMA foreign_keys = ON") + row_type.configure_connection(connection) + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") # Rollback journal avoids a post-schema WAL-mode race among fresh openers. - self._connection.execute("PRAGMA synchronous = FULL") + connection.execute("PRAGMA synchronous = FULL") def _prepare_private_file(self) -> None: self.path.parent.mkdir(parents=True, exist_ok=True) diff --git a/src/agent_comms/coordination_session.py b/src/agent_comms/coordination_session.py index 1a621299a1..8269bea2bb 100644 --- a/src/agent_comms/coordination_session.py +++ b/src/agent_comms/coordination_session.py @@ -206,7 +206,8 @@ def _prepare_private_file(self) -> None: def _open_connection(self, contention: StoreContention) -> sqlite3.Connection: return self._open_reader(self.path, contention) - def _configure_connection(self) -> None: + @classmethod + def configure_connection(cls, connection: sqlite3.Connection) -> None: pass # _open_reader configured query-only behavior before publication. def _initialize_schema(self) -> None: diff --git a/src/agent_comms/installer/practice.py b/src/agent_comms/installer/practice.py index 51137d8fb1..d14d8a496c 100644 --- a/src/agent_comms/installer/practice.py +++ b/src/agent_comms/installer/practice.py @@ -7,7 +7,10 @@ from abc import ABC, abstractmethod from dataclasses import dataclass, field from pathlib import Path +from collections.abc import Iterable +from agent_comms.context_store import ContextManifestReference, ContextStore +from agent_comms.native_session_reopen import NativeSessionIdentity from agent_comms.nominal.field_codec import boundary_decoder from agent_comms.installer.refusals import UpgradeRefused from agent_comms.installer.declared_uv import DeclaredUv @@ -133,11 +136,31 @@ def copied_names(self) -> tuple[Path, ...]: resource.live(self.root), resource.path, source, ) for source, resource in self.copied.items())) - def include(self, paths: tuple[Path, ...]) -> None: + def prepare(self) -> None: + """One space and private-directory owner for whole and selected materialization.""" floor = StorageRule.load(self.root).floor_gib * 2**30 PrivateDirectoryRole.make_parents(self.path) if shutil.disk_usage(self.path.parent).free < floor: raise UpgradeRefused("Insufficient free space before building the practice copy") + + def include_contexts( + self, identities: Iterable[NativeSessionIdentity], *, + requests: Iterable[ContextManifestReference] = (), + ) -> None: + """Explicit selected-context inputs, never a full-file backup. + + Normal include() and the installer's live-root backup remain whole-file + operations. This selection must precede including the context resource; + an existing destination is refused rather than overwritten or reimported. + """ + self.prepare() + source = ContextStore(self.root) + resource, = RootFile.owned_files(LiveFilesRequest(self.root), (source.path,)) + source.retain_contexts(ContextStore(self.path), identities, requests=requests) + self.copied[source.path] = resource + + def include(self, paths: tuple[Path, ...]) -> None: + self.prepare() witnesses = set(MaintenanceBarrier(RegistryFile.path_in(self.root)).witness_paths) for resource in RootFile.owned_files(LiveFilesRequest(self.root), paths): source = resource.live(self.root) diff --git a/src/agent_comms/typed_table.py b/src/agent_comms/typed_table.py index 4fe487c280..ecb37930fa 100644 --- a/src/agent_comms/typed_table.py +++ b/src/agent_comms/typed_table.py @@ -1093,6 +1093,37 @@ def retain_rows( The caller owns destination freshness, guard/loading lifetime, semantic dependencies and producer metadata. No stored JSON is reconstructed. """ + return self._retain_selection( + row, f"FROM {self.source.qualified(row.declared_name)} AS {_identifier(row.declared_name)}", + where=where, parameters=parameters, + ) + + def retain_keys( + self, row: type[TypedTable], keys: Sequence[Mapping[str, object]], + ) -> int: + """The declared indexed-key query supplies raw selected rows, not re-encoded records.""" + return sum(self._retain_selection( + row, requested.join, prefix=f"WITH {requested.values}", + parameters=requested.bindings, + ) for requested in row.requested_keys( + self.destination, keys, table_source=self.source, + )) + + def require_schema(self, schema: type[TableSchema]) -> None: + """An unknown source column or trigger is not permission to silently discard it.""" + stored = SQLiteSchemaObject.read(self.destination.execute( + f"SELECT name,sql FROM {self.source.qualified('sqlite_master')} " + "WHERE sql IS NOT NULL AND name NOT LIKE 'sqlite_%'", + )) + differences = SQLiteSchemaObject.differences(stored, schema.declared_schema()) + if differences: + raise schema.schema_error("Retained source schema differs: " + " | ".join( + str(difference) for difference in differences)) + + def _retain_selection( + self, row: type[TypedTable], source: str, *, where: str = "1", + parameters: tuple[object, ...] = (), prefix: str = "", + ) -> int: keys = tuple(item.name for item in row._columns if item.column.primary_key) if not keys: raise RelationViolationError(f"{row.declared_name} has no declared retention key") @@ -1102,14 +1133,17 @@ def retain_rows( f"{retained_alias}.{row._column_list((key,))} IS {row.qualified(key)}" for key in keys ) table = _identifier(row.declared_name) - return self.destination.execute( - f"INSERT INTO main.{table} ({row._column_list(fields)}) " - f"SELECT {row.qualified(*fields)} " - f"FROM {self.source.qualified(row.declared_name)} AS {table} " + before = self.destination.total_changes + self.destination.execute( + f"{prefix} INSERT INTO main.{table} ({row._column_list(fields)}) " + f"SELECT {row.qualified(*fields)} {source} " f"WHERE ({row._sql(where)}) AND NOT EXISTS " f"(SELECT 1 FROM main.{table} AS {retained_alias} WHERE {retained})", parameters, - ).rowcount + ) + # Cursor.rowcount is -1 for WITH-prefixed INSERTs. The connection owns + # completed changes regardless of which selection grammar supplied them. + return self.destination.total_changes - before def rebuild( self, schema: type[DerivedTables], db: sqlite3.Connection, @@ -3078,6 +3112,39 @@ def schema_at(cls, path: Path) -> type[TableSchema]: def schema_change(cls, path: Path) -> SchemaChange: return HeldBusFile(cls, path) + @classmethod + def configure_connection(cls, connection: sqlite3.Connection) -> None: + """Connection behavior needed by this file's declared loading guards.""" + + @classmethod + @contextmanager + def retaining_rows(cls, source: Path, destination: Path) -> Iterator[AttachedPracticeRebuild]: + """Publish a fresh selected store only after loading and declared closure succeed. + + The caller supplies seeds and owns the wider cross-store cut. The source + attachment stays read-only in the loading transaction; no runtime writer, + producer regeneration, lifecycle reset or original coordinate is created. + """ + if destination.exists() or destination.is_symlink(): + raise FileExistsError(destination) + schema = cls.schema_at(source) + PrivateDirectoryRole.make_parents(destination.parent) + descriptor, staged = cls.stage_file(destination) + os.close(descriptor) + try: + with closing(cls.writer(staged, isolation_level=None)) as db: + cls.configure_connection(db) + with PracticeRebuild(source).attached(db) as original: + with schema.loading_rows(db): + original.require_schema(schema) + yield original + while schema.retain_referenced_rows(original): + pass + ExclusiveAtomicFile(destination).replace(staged) + finally: + for member in cls.files(staged): + member.unlink(missing_ok=True) + @classmethod def practice_rebuild(cls, path: Path, practice_root: Path) -> SchemaRebuild: """Files without a declared practice replay retain their normal rebuild.""" @@ -3407,6 +3474,33 @@ def __init_subclass__(cls, **kwargs: object) -> None: class RowidSequenced(TypedTable, declared_name=None): """The table's rows are numbered by SQLite's rowid, and an insert answers its number.""" + @classmethod + def retain_required_rows( + cls, original: AttachedPracticeRebuild, schema: type[TableSchema], + ) -> int: + """AUTOINCREMENT's original high-water mark survives a selected row set.""" + changed = super().retain_required_rows(original, schema) + # sqlite_sequence is SQLite-owned external metadata, absent when the + # source has no AUTOINCREMENT tables. Its presence is decoded here once. + if not SQLiteSchemaObject.read(original.destination.execute( + f"SELECT name,sql FROM {original.source.qualified('sqlite_master')} " + "WHERE name='sqlite_sequence'", + )): + return changed + source = original.source.qualified("sqlite_sequence") + changed += original.destination.execute( + "INSERT INTO main.sqlite_sequence(name,seq) " + f"SELECT name,seq FROM {source} AS original_sequence WHERE name=? " + "AND NOT EXISTS(SELECT 1 FROM main.sqlite_sequence AS retained_sequence " + "WHERE retained_sequence.name=original_sequence.name)", (cls.declared_name,), + ).rowcount + changed += original.destination.execute( + f"UPDATE main.sqlite_sequence SET seq=(SELECT seq FROM {source} WHERE name=?) " + f"WHERE name=? AND seq<(SELECT seq FROM {source} WHERE name=?)", + (cls.declared_name, cls.declared_name, cls.declared_name), + ).rowcount + return changed + @classmethod def insert_sequence(cls, db: sqlite3.Connection, **values: object) -> int: """Insert one row and answer the rowid SQLite assigned it.""" diff --git a/tests/test_context_manifest_deltas.py b/tests/test_context_manifest_deltas.py index b78358376d..842a80b731 100644 --- a/tests/test_context_manifest_deltas.py +++ b/tests/test_context_manifest_deltas.py @@ -3,19 +3,27 @@ from dataclasses import replace import hashlib import sys +from pathlib import Path +from collections.abc import Callable import pytest -from agent_comms.context_store import ContextRequest +from agent_comms.context_store import ( + CapturedContext, ContextEntry, ContextManifestReference, ContextRequest, ContextSegmentVersion, + ContextStore, ContextStoreIdentity, ContextStoreInfo, +) +from agent_comms.errors import RelationViolationError +from agent_comms.installer.practice import PracticeCopy +from agent_comms.typed_table import RowNotFoundError import json -from agent_comms.nominal.field_codec import FieldCodec, TurnId +from agent_comms.nominal.field_codec import ContextStoreId, FieldCodec, TurnId from agent_comms.native_session_reopen import NativeSessionIdentity from agent_comms.thread_identity import TurnIdentity from agent_comms.turn_context import ObservedDispatchRequest from agent_comms.turn_context import ( ContextManifest, JournalProvenance, NativeProvenance, RecordedContextPredecessor, - RecordedContextTurn, + RecordedContextTurn, RecordedSegmentManifest, SegmentManifest, SystemLayerSegment, ToolCatalogSegment, TranscriptSegment, ) from bus_record_fixture import context_comms @@ -210,3 +218,83 @@ def test_threads_record_concurrently_and_a_superseded_prior_is_refused(tmp_path) assert comms.contexts.page('Bob', comms.registry).resources[-1].record_id == recorded[1].record_id with pytest.raises(ContextPriorSupersededError): comms.contexts.record_after(alice, context(owners[0], tmp_path, 5, 3)) + + +def test_selected_context_copy_preserves_closure_and_original_sequences(tmp_path: Path) -> None: + comms = context_comms(tmp_path / 'source') + owner = comms.registry.require('Alice') + originals = tuple(comms.contexts.record_after( + UNREAD_CONTEXT, context(owner, tmp_path, count, turn), + ) for turn, count in enumerate((1, 2, 4), start=1)) + selected = originals[1] + reference = ContextManifestReference(selected.source, selected.record_id) + before = ContextStore.digest(comms.contexts.path) + copy = PracticeCopy(comms.root, tmp_path / 'retained') + copy.include_contexts((), requests=(reference, reference)) + retained = ContextStore(copy.path) + + with retained.snapshot() as db: + info = ContextStoreInfo.require(db, singleton=1) + assert info.source == selected.source + rows = ContextRequest.select(db, order_by=('id',)) + assert tuple(row.id for row in rows) == tuple(row.record_id for row in originals[:2]) + assert retained.resolve(db, info.source, rows[-1]).value == selected.value + assert ContextEntry.count(db) == 2 + retained_version = max(row.id for row in ContextSegmentVersion.select(db)) + with comms.contexts.snapshot() as db: + source_version = max(row.id for row in ContextSegmentVersion.select(db)) + assert retained_version < source_version + with retained.writing() as db: + # Real row owners allocate after the full original high-water mark, + # not after the greatest ID that happened to be in this selection. + assert ContextRequest.append(db, selected.recorded) == originals[-1].record_id + 1 + fresh = RecordedSegmentManifest(context(owner, tmp_path, 6, 4).segments[2]) + assert ContextSegmentVersion.append(db, fresh) == source_version + 1 + assert ContextStore.digest(comms.contexts.path) == before + assert ContextStore.path_in(comms.root) in copy.copied + assert copy.request.source_root() == comms.root + assert copy.request.root == copy.path + + +def foreign_context_reference( + store: ContextStore, previous: CapturedContext, reference: ContextManifestReference, +) -> ContextManifestReference: + return replace(reference, source=ContextStoreIdentity(ContextStoreId.mint())) + + +def missing_context_reference( + store: ContextStore, previous: CapturedContext, reference: ContextManifestReference, +) -> ContextManifestReference: + return replace(reference, record_id=reference.record_id + 100) + + +def missing_context_predecessor( + store: ContextStore, previous: CapturedContext, reference: ContextManifestReference, +) -> ContextManifestReference: + with store.writing() as db: + ContextRequest.delete(db, id=previous.record_id) + return reference + + +@pytest.mark.parametrize(('change', 'expected'), ( + pytest.param(foreign_context_reference, RelationViolationError, id='foreign-source'), + pytest.param(missing_context_reference, RowNotFoundError, id='missing-record'), + pytest.param(missing_context_predecessor, RowNotFoundError, id='missing-predecessor'), +)) +def test_selected_context_copy_refuses_without_publishing( + tmp_path: Path, + change: Callable[[ContextStore, CapturedContext, ContextManifestReference], ContextManifestReference], + expected: type[Exception], +) -> None: + comms = context_comms(tmp_path / 'source') + owner = comms.registry.require('Alice') + first = comms.contexts.record_after(UNREAD_CONTEXT, context(owner, tmp_path, 1, 1)) + second = comms.contexts.record_after(first, context(owner, tmp_path, 2, 2)) + reference = change(comms.contexts, first, ContextManifestReference(second.source, second.record_id)) + before = ContextStore.digest(comms.contexts.path) + destination = ContextStore(tmp_path / 'retained') + with pytest.raises(expected): + comms.contexts.retain_contexts(destination, (), requests=(reference,)) + assert not destination.path.exists() + assert not tuple(destination.root.iterdir()) + assert ContextStore.digest(comms.contexts.path) == before From aade79dfde278ca97a8932a1a65cebe59af63213 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sun, 1 Nov 2026 12:05:59 -0500 Subject: [PATCH 08/28] Bound selected context loading by its remaining copy allowance --- docs/retained-practice-inputs.md | 26 ++++++++++++- src/agent_comms/context_store.py | 6 ++- src/agent_comms/installer/practice.py | 53 ++++++++++++++++++++++----- src/agent_comms/typed_table.py | 35 +++++++++++++++++- tests/test_context_manifest_deltas.py | 49 +++++++++++++++++++++++-- 5 files changed, 152 insertions(+), 17 deletions(-) diff --git a/docs/retained-practice-inputs.md b/docs/retained-practice-inputs.md index 26376920a1..2df129a52a 100644 --- a/docs/retained-practice-inputs.md +++ b/docs/retained-practice-inputs.md @@ -7,7 +7,7 @@ This document records the source decisions, not an executable selection roster. ## Existing owners and implemented entry selection -`ContextStore.retain_contexts(destination, identities, requests=...)` replaces the +`ContextStore.retain_contexts(destination, identities, maximum_bytes=..., requests=...)` replaces the entry-only API. `PracticeCopy.include_contexts` consumes it explicitly; normal whole-file include and live-root backup selection are unchanged. Selected writable SQL values travel through the existing attached source, without decode/re-encode, @@ -201,6 +201,30 @@ Measure the complete selected stores, WAL/copy groups and free space before any copy. The requested production-copy proof is bounded to 3 GiB; zero freelist pages in the oversized stores means compaction is not the repair. +Selected context materialization now requires an explicit byte allowance. Through +PracticeCopy this is a **total private-tree allowance**, not a new default for full +backups. `PracticeMeasurement.capture` is the one tree-byte calculation: already +retained files, WAL/companions and staging files count whether or not the copied +resource index lists them. `PracticeCopy.remaining_bytes` subtracts those bytes and +also limits new allocation to actual disk space above the existing StorageRule +floor. Its caller owns serial materialization of the private root and the coherent +source cut; this is not a reservation against other processes consuming disk. + +`SQLiteDatabase.limit_bytes` consumes SQLite's actual page size and accepted +max_page_count through typed pragma records. It caps the staged database before +schema/rows/indexes are allocated; a zero-page allowance refuses, and SQLite can +raise SQLITE_FULL on later growth. `ReconciledFile.retaining_rows` additionally +counts the complete closed staged file group before exclusive publication. An +over-budget group refuses and the same staging cleanup runs. No cap, page size, +sequence, schema or lifecycle is written into the original store. + +This is authored admission/allocation code, **not a measured peak-footprint or +free-space race proof**. Journal/temp-spill peaks during loading, concurrent disk +consumption, rollback/cleanup under SQLITE_FULL, and the complete cross-store +selection/root budget still need qualification. Ordinary include()/PracticeBus +and live-root full backups remain unchanged; a caller cannot label their +unmeasured union bounded merely by passing an allowance to one selected store. + ## Source evidence and remaining work The retained NRA source map at baseline `095c4cdfe85632f990be3b6d67411d9685e6dee9` diff --git a/src/agent_comms/context_store.py b/src/agent_comms/context_store.py index 666ec378cb..47b14baab5 100644 --- a/src/agent_comms/context_store.py +++ b/src/agent_comms/context_store.py @@ -518,7 +518,7 @@ def entries( def retain_contexts( self, destination: ContextStore, identities: Iterable[NativeSessionIdentity], *, - requests: Iterable[ContextManifestReference] = (), + maximum_bytes: int, requests: Iterable[ContextManifestReference] = (), ) -> None: """Retain original entry manifests and explicitly selected saved requests. @@ -527,7 +527,9 @@ def retain_contexts( not relocated storage or permission to launch or replay a native session. Entry-only selection retains no request or fabricated store identity. """ - with type(self).retaining_rows(self.path, destination.path) as original: + with type(self).retaining_rows( + self.path, destination.path, maximum_bytes=maximum_bytes, + ) as original: original.retain_keys(ContextEntry, tuple( {"session_id": identity.session_id, "session_file": identity.session_file} for identity in dict.fromkeys(identities) diff --git a/src/agent_comms/installer/practice.py b/src/agent_comms/installer/practice.py index d14d8a496c..9633532427 100644 --- a/src/agent_comms/installer/practice.py +++ b/src/agent_comms/installer/practice.py @@ -7,6 +7,7 @@ from abc import ABC, abstractmethod from dataclasses import dataclass, field from pathlib import Path +from math import ceil from collections.abc import Iterable from agent_comms.context_store import ContextManifestReference, ContextStore @@ -27,15 +28,28 @@ class PracticeMeasurement: bytes: int + @classmethod + def capture(cls, copy: Path) -> PracticeMeasurement: + """Count the complete materialized tree, including SQLite companions and stages.""" + return cls(sum(path.stat().st_size for path in copy.rglob("*") if path.is_file())) + + def remaining(self, maximum_bytes: int) -> int: + """An explicit selected-input allowance, not a limit on complete backups.""" + if self.bytes > maximum_bytes: + raise UpgradeRefused( + f"Practice copy has {self.bytes} bytes, exceeding {maximum_bytes} allowed bytes") + return maximum_bytes - self.bytes + @classmethod def of(cls, copy: Path, floor: float) -> PracticeMeasurement: """Refuse a copy without room for the rehearsal's second copy above the disk floor. The free space is what a copy can cost; a fixed byte cap beside it refused a 3.01 GiB bus with 29 GiB free and stopped install 45.""" - size = sum(path.stat().st_size for path in copy.rglob("*") if path.is_file()) - if shutil.disk_usage(copy.parent).free < 2 * size + floor: - raise UpgradeRefused(f"Insufficient free space for rehearsal of {size} copied bytes") - return cls(size) + measured = cls.capture(copy) + if shutil.disk_usage(copy.parent).free < 2 * measured.bytes + floor: + raise UpgradeRefused( + f"Insufficient free space for rehearsal of {measured.bytes} copied bytes") + return measured @dataclass(frozen=True) @@ -136,15 +150,32 @@ def copied_names(self) -> tuple[Path, ...]: resource.live(self.root), resource.path, source, ) for source, resource in self.copied.items())) + @property + def floor_bytes(self) -> int: + return ceil(StorageRule.load(self.root).floor_gib * 2**30) + + @property + def available_bytes(self) -> int: + return shutil.disk_usage(self.path.parent).free - self.floor_bytes + def prepare(self) -> None: """One space and private-directory owner for whole and selected materialization.""" - floor = StorageRule.load(self.root).floor_gib * 2**30 PrivateDirectoryRole.make_parents(self.path) - if shutil.disk_usage(self.path.parent).free < floor: + if self.available_bytes < 0: raise UpgradeRefused("Insufficient free space before building the practice copy") + def remaining_bytes(self, maximum_bytes: int) -> int: + """Share one explicit total allowance across incrementally selected stores. + + The current tree, not the copied-resource index, owns used bytes. This + includes retained WAL/groups and data placed there by another copy step. + Caller owns this private copy's serial materialization and wider cut. + """ + return min(PracticeMeasurement.capture(self.path).remaining(maximum_bytes), + self.available_bytes) + def include_contexts( - self, identities: Iterable[NativeSessionIdentity], *, + self, identities: Iterable[NativeSessionIdentity], *, maximum_bytes: int, requests: Iterable[ContextManifestReference] = (), ) -> None: """Explicit selected-context inputs, never a full-file backup. @@ -156,7 +187,10 @@ def include_contexts( self.prepare() source = ContextStore(self.root) resource, = RootFile.owned_files(LiveFilesRequest(self.root), (source.path,)) - source.retain_contexts(ContextStore(self.path), identities, requests=requests) + source.retain_contexts( + ContextStore(self.path), identities, requests=requests, + maximum_bytes=self.remaining_bytes(maximum_bytes), + ) self.copied[source.path] = resource def include(self, paths: tuple[Path, ...]) -> None: @@ -182,5 +216,4 @@ def retain(self, paths: tuple[Path, ...]) -> None: @property def measurement(self) -> PracticeMeasurement: - return PracticeMeasurement.of( - self.path, StorageRule.load(self.root).floor_gib * 2**30) + return PracticeMeasurement.of(self.path, self.floor_bytes) diff --git a/src/agent_comms/typed_table.py b/src/agent_comms/typed_table.py index ecb37930fa..40b22016e3 100644 --- a/src/agent_comms/typed_table.py +++ b/src/agent_comms/typed_table.py @@ -840,6 +840,32 @@ def named(cls, db: sqlite3.Connection, name: str) -> Self: def qualified(self, name: str) -> str: return f"{_identifier(self.name)}.{_identifier(name)}" + def limit_bytes(self, db: sqlite3.Connection, maximum_bytes: int) -> None: + """Let SQLite refuse page growth before a selected database exceeds its allowance.""" + size, = SQLitePageSize.read(db.execute(f"PRAGMA {_identifier(self.name)}.page_size")) + pages = maximum_bytes // size.page_size + if pages < 1: + raise RelationViolationError("Selected store has no room for a SQLite page") + limit, = SQLiteMaximumPageCount.read(db.execute( + f"PRAGMA {_identifier(self.name)}.max_page_count={pages}", + )) + if limit.max_page_count > pages: + raise RelationViolationError("Selected database already exceeds its byte allowance") + + +@dataclass(frozen=True) +class SQLitePageSize(TypedRow): + """The page size returned by SQLite, not a copied platform default.""" + + page_size: int + + +@dataclass(frozen=True) +class SQLiteMaximumPageCount(TypedRow): + """SQLite's actual accepted allocation limit; an existing file can force it higher.""" + + max_page_count: int + @dataclass(frozen=True) class SQLiteSchemaObject(TypedRow): @@ -3118,7 +3144,9 @@ def configure_connection(cls, connection: sqlite3.Connection) -> None: @classmethod @contextmanager - def retaining_rows(cls, source: Path, destination: Path) -> Iterator[AttachedPracticeRebuild]: + def retaining_rows( + cls, source: Path, destination: Path, *, maximum_bytes: int, + ) -> Iterator[AttachedPracticeRebuild]: """Publish a fresh selected store only after loading and declared closure succeed. The caller supplies seeds and owns the wider cross-store cut. The source @@ -3134,12 +3162,17 @@ def retaining_rows(cls, source: Path, destination: Path) -> Iterator[AttachedPra try: with closing(cls.writer(staged, isolation_level=None)) as db: cls.configure_connection(db) + SQLiteDatabase.named(db, "main").limit_bytes(db, maximum_bytes) with PracticeRebuild(source).attached(db) as original: with schema.loading_rows(db): original.require_schema(schema) yield original while schema.retain_referenced_rows(original): pass + retained_bytes = sum(member.stat().st_size for member in cls.files(staged) + if member.is_file()) + if retained_bytes > maximum_bytes: + raise RelationViolationError("Selected SQLite file group exceeds its byte allowance") ExclusiveAtomicFile(destination).replace(staged) finally: for member in cls.files(staged): diff --git a/tests/test_context_manifest_deltas.py b/tests/test_context_manifest_deltas.py index 842a80b731..d0f2bf53ae 100644 --- a/tests/test_context_manifest_deltas.py +++ b/tests/test_context_manifest_deltas.py @@ -14,8 +14,9 @@ ) from agent_comms.errors import RelationViolationError from agent_comms.installer.practice import PracticeCopy -from agent_comms.typed_table import RowNotFoundError +from agent_comms.typed_table import RowNotFoundError, SQLitePageSize import json +import sqlite3 from agent_comms.nominal.field_codec import ContextStoreId, FieldCodec, TurnId from agent_comms.native_session_reopen import NativeSessionIdentity @@ -230,7 +231,7 @@ def test_selected_context_copy_preserves_closure_and_original_sequences(tmp_path reference = ContextManifestReference(selected.source, selected.record_id) before = ContextStore.digest(comms.contexts.path) copy = PracticeCopy(comms.root, tmp_path / 'retained') - copy.include_contexts((), requests=(reference, reference)) + copy.include_contexts((), requests=(reference, reference), maximum_bytes=2**20) retained = ContextStore(copy.path) with retained.snapshot() as db: @@ -294,7 +295,49 @@ def test_selected_context_copy_refuses_without_publishing( before = ContextStore.digest(comms.contexts.path) destination = ContextStore(tmp_path / 'retained') with pytest.raises(expected): - comms.contexts.retain_contexts(destination, (), requests=(reference,)) + comms.contexts.retain_contexts( + destination, (), requests=(reference,), maximum_bytes=2**20, + ) assert not destination.path.exists() assert not tuple(destination.root.iterdir()) assert ContextStore.digest(comms.contexts.path) == before + + +def test_selected_context_allowance_counts_existing_companions(tmp_path: Path) -> None: + comms = context_comms(tmp_path / 'source') + owner = comms.registry.require('Alice') + selected = comms.contexts.record_after(UNREAD_CONTEXT, context(owner, tmp_path, 1, 1)) + copy = PracticeCopy(comms.root, tmp_path / 'retained') + copy.prepare() + companion = copy.path / 'earlier-store.sqlite3-wal' + original_bytes = b'original retained companion' + companion.write_bytes(original_bytes) + before = ContextStore.digest(comms.contexts.path) + with pytest.raises(RelationViolationError): + copy.include_contexts((), maximum_bytes=len(original_bytes), requests=( + ContextManifestReference(selected.source, selected.record_id), + )) + assert not ContextStore(copy.path).path.exists() + assert tuple(copy.path.iterdir()) == (companion,) + assert companion.read_bytes() == original_bytes + assert not copy.copied + assert ContextStore.digest(comms.contexts.path) == before + + +def test_selected_context_sqlite_allocation_limit_leaves_no_publication(tmp_path: Path) -> None: + comms = context_comms(tmp_path / 'source') + owner = comms.registry.require('Alice') + selected = comms.contexts.record_after(UNREAD_CONTEXT, context(owner, tmp_path, 1, 1)) + copy = PracticeCopy(comms.root, tmp_path / 'retained') + before = ContextStore.digest(comms.contexts.path) + with comms.contexts.snapshot() as db: + page, = SQLitePageSize.read(db.execute('PRAGMA page_size')) + with pytest.raises(sqlite3.OperationalError) as refused: + copy.include_contexts((), maximum_bytes=page.page_size, requests=( + ContextManifestReference(selected.source, selected.record_id), + )) + assert refused.value.sqlite_errorcode == sqlite3.SQLITE_FULL + assert not ContextStore(copy.path).path.exists() + assert not tuple(copy.path.iterdir()) + assert not copy.copied + assert ContextStore.digest(comms.contexts.path) == before From 3e0a0356997098a4b7fa813e1fd6e01be79fef4f Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sun, 1 Nov 2026 12:48:22 -0500 Subject: [PATCH 09/28] Retain harness observation groups and original claim relations --- docs/retained-practice-inputs.md | 33 +++-- src/agent_comms/assignment_store.py | 8 +- src/agent_comms/cohort_schema.py | 11 +- .../coordination_tables/assignments.py | 26 +++- .../coordination_tables/harness_deliveries.py | 22 ++- src/agent_comms/typed_table.py | 14 ++ tests/test_retained_coordinator_groups.py | 137 ++++++++++++++++++ 7 files changed, 223 insertions(+), 28 deletions(-) create mode 100644 tests/test_retained_coordinator_groups.py diff --git a/docs/retained-practice-inputs.md b/docs/retained-practice-inputs.md index 2df129a52a..5b345ec648 100644 --- a/docs/retained-practice-inputs.md +++ b/docs/retained-practice-inputs.md @@ -121,15 +121,22 @@ then checks references before committing. Exceptions roll back through the exist SQLiteFile transaction owner. The same `require_related_rows` check is used by ordinary schema reconciliation. No ordinary runtime insertion guard is weakened. -`TypedTable.retain_required_rows` supplies the default empty contribution; -`ClaimBatchAcknowledgements` overrides it because its sealed counts describe the -complete original child group. The schema follows inverse **declared FK edges** -only for that requested parent, rather than expanding every participant's history. +`TypedTable.retain_required_rows` supplies the default empty contribution. +`RetainsChildren` composes that contribution with inverse **declared FK edges**. +ClaimBatchAcknowledgements inherits this capability because its sealed counts +describe the complete original child group; its forwarding override is deleted. +HarnessDelivery inherits the same behavior for its original observation parts. +Only a requested parent expands its children, not every participant's history. `ForeignKey` owns both directions and their one parent-affinity comparison. Original sealed state is copied unchanged; no seal transition or replay is issued. These are source implementations, not executed copy/rollback/guard controls. -CoordinationStore/PracticeCopy has no selection consumer yet. Natural-key harness, +The acknowledgement and harness rows share the same retained-group capability; +actual class creation/MRO, schema stability and group-copy behavior still require +qualification. Four new harness-group scenarios are authored, UNRUN: selected +observations with an existing post-handoff claim, observations with no claim, +full-reference conflict, and copy conflict with no partial publication. +CoordinationStore/PracticeCopy has no coordinator-seed selection consumer yet. Coordinator seed capture and the remaining serialized/cross-store and producer dependencies remain OPEN; complete FK validity would not establish their preservation or the original group's validity. @@ -140,10 +147,18 @@ would not establish their preservation or the original group's validity. equivalent seed. Stored rows in every state suppress a synthesized attempted key; copying only unresolved ledger rows could resurrect a completed attempt. No synthesized `ReservedInput` is written back merely to construct a copy. -* **Harness claims:** `HarnessDelivery.settle_claims` and - `AssignmentStore.settle_harness_inputs` relate the original recipient and - message reference to claims. `HarnessDelivery.references()` declares only its - participant dependency. The natural-key claim relation is not an FK edge. +* **Harness claims and observations:** `WakeAssignment.for_harness_sources` now + owns recipient/sequence candidate selection and full MessageReference conflict + checking. Normal settlement consumes it rather than maintaining its own SQL + predicate; selected HarnessDelivery retention reads through the same active + original attachment and copies every existing matching claim, including its + unchanged lifecycle/revision. Missing claims stay absent: an attempt can precede + claim acceptance. A same-sequence/different-message conflict refuses instead of + silently disappearing. This natural relation is not fabricated as a mandatory + FK. HarnessObservedPart's declared FK supplies its parent, not the reverse + observation set; RetainsChildren supplies that original child group through the + shared schema traversal. Original sender/body/record-ID/timestamp and partial + ReceivingMonitor state are not reconstructed, completed or replayed. * **Sealed acknowledgements:** `ClaimBatchAcknowledgements.column_guards` requires complete member/delivery sets and their original counts. Forward FK closure does not supply those children. Child insert guards reject an already-sealed diff --git a/src/agent_comms/assignment_store.py b/src/agent_comms/assignment_store.py index 38c5df6f14..a95cc6fba1 100644 --- a/src/agent_comms/assignment_store.py +++ b/src/agent_comms/assignment_store.py @@ -102,14 +102,8 @@ def settle_harness_inputs( Every step runs in the caller's coordinator write, so the harness attempt and its claim consequence commit together or not at all. """ - claims = WakeAssignment.select( - db, - where=f"recipient_lookup=? AND wire_seq IN ({','.join('?' * len(sources))})", - parameters=(recipient_lookup, *(source.seq for source in sources)), - ) + claims = WakeAssignment.for_harness_sources(db, recipient_lookup, sources) for claim in claims: - if claim.source not in sources: - raise IdentityConflict("harness delivery names another message at this sequence") for disposition in steps(claim.lifecycle): # The revision fence refuses a step if the native owner moved the claim meanwhile. claim = self.transition_in( diff --git a/src/agent_comms/cohort_schema.py b/src/agent_comms/cohort_schema.py index 208db7c9cf..322906cdf0 100644 --- a/src/agent_comms/cohort_schema.py +++ b/src/agent_comms/cohort_schema.py @@ -19,7 +19,6 @@ from .thread_text import AbsentText, GivenText, TextChoice from .response_acknowledgement import _canonical from .typed_table import ( - AttachedPracticeRebuild, Column, ColumnGuard, ComposedSchemas, @@ -33,10 +32,10 @@ Index, InitialOneWayColumn, NoDelete, + RetainsChildren, SharedFileTables, SQLiteJournalMode, TextStorage, - TableSchema, TypedTable, WithoutRowid, ) @@ -62,6 +61,7 @@ class AwarenessTable(SharedFileTables, DurableTables): @dataclass(frozen=True, kw_only=True) class ClaimBatchAcknowledgements( + RetainsChildren, InitialOneWayColumn, GuardedColumns, NoDelete, @@ -120,13 +120,6 @@ class ClaimBatchAcknowledgements( accepted_at_ms: Annotated[int, FrozenColumn(check="accepted_at_ms >= 0")] sealed: bool = False - @classmethod - def retain_required_rows( - cls, original: AttachedPracticeRebuild, schema: type[TableSchema], - ) -> int: - """Retain all original children: sealed counts describe the complete group.""" - return schema.retain_referencing_rows(original, cls) - @classmethod def decision_hash(cls, delivery: Delivered) -> str: encoded = _canonical([decision.to_wire() for decision in delivery.decisions]) diff --git a/src/agent_comms/coordination_tables/assignments.py b/src/agent_comms/coordination_tables/assignments.py index 25b8225162..aeadf074b9 100644 --- a/src/agent_comms/coordination_tables/assignments.py +++ b/src/agent_comms/coordination_tables/assignments.py @@ -2,8 +2,10 @@ from __future__ import annotations +import sqlite3 +from collections.abc import Iterator, Sequence from dataclasses import dataclass, replace -from typing import Annotated, ClassVar +from typing import Annotated, ClassVar, Self from agent_comms.assignment_states import AssignmentState, EngagedAssignment from agent_comms.frozen_recipient import FrozenRecipient @@ -25,6 +27,7 @@ from agent_comms.nominal.field_codec import AssignmentId, MessageId, SnapshotExclude, ThreadLookup, WireName, projected from agent_comms.message_reference import MessageReference from agent_comms.typed_table import ( + LOCAL_TABLE_SOURCE, Column, FirstValueState, ForeignKey, @@ -40,6 +43,7 @@ NoDelete, NonDecreasing, RequiredColumn, + TableSource, WriteOnceColumns, ) from agent_comms.wake_policy import ( @@ -126,6 +130,26 @@ def require_same_acceptance(self, candidate: WakeAssignment) -> None: def source(self) -> MessageReference: return MessageReference(self.wire_seq, self.message_id) + @classmethod + def for_harness_sources( + cls, db: sqlite3.Connection, recipient_lookup: ThreadLookup, + sources: Sequence[MessageReference], *, + table_source: TableSource = LOCAL_TABLE_SOURCE, + ) -> Iterator[Self]: + """Existing claims for these original messages, including settled claims. + + Sequence finds candidates through the declared unique key; the complete + reference must still agree. A conflicting message refuses rather than + disappearing from selection. An attempt need not already have a claim. + """ + claims = cls.select_keys(db, tuple( + {"recipient_lookup": recipient_lookup, "wire_seq": source.seq} for source in sources + ), table_source=table_source) + for claim in claims: + if claim.source not in sources: + raise IdentityConflict("harness delivery names another message at this sequence") + yield claim + def require_engaged_binding(self) -> AssignmentBinding: engagement = self.lifecycle.require_engagement() return AssignmentBinding( diff --git a/src/agent_comms/coordination_tables/harness_deliveries.py b/src/agent_comms/coordination_tables/harness_deliveries.py index a68f8d883b..f125c4c953 100644 --- a/src/agent_comms/coordination_tables/harness_deliveries.py +++ b/src/agent_comms/coordination_tables/harness_deliveries.py @@ -14,6 +14,7 @@ from agent_comms.coordination_errors import IdentityConflict from agent_comms.coordination_schema import CoordinatorRow +from agent_comms.coordination_tables.assignments import WakeAssignment from agent_comms.coordination_tables.participants import Participants from agent_comms.nominal.field_codec import MessageId, ThreadLookup, TranscriptRecordId from agent_comms.harness_delivery_states import ( @@ -22,7 +23,8 @@ from agent_comms.harness_install import HarnessSession from agent_comms.message_reference import MessageReference from agent_comms.typed_table import ( - LOCAL_TABLE_SOURCE, Column, ForeignKey, FrozenColumn, ImmutableRows, Index, NoDelete, TableSource, + LOCAL_TABLE_SOURCE, AttachedPracticeRebuild, Column, ForeignKey, FrozenColumn, + ImmutableRows, Index, NoDelete, RetainsChildren, TableSchema, TableSource, ) if TYPE_CHECKING: @@ -30,7 +32,7 @@ @dataclass(frozen=True, slots=True) -class HarnessDelivery(NoDelete, CoordinatorRow, declared_name="harness_deliveries"): +class HarnessDelivery(RetainsChildren, NoDelete, CoordinatorRow, declared_name="harness_deliveries"): """One original attempt to put a bus message into a recipient's harness session.""" bridge: Annotated[type[HarnessSession], FrozenColumn(primary_key=True)] @@ -49,6 +51,22 @@ def references(cls) -> tuple[ForeignKey, ...]: return (ForeignKey(columns=("recipient_lookup",), target=Participants, target_columns=("participant_lookup",)),) + @classmethod + def retain_required_rows( + cls, original: AttachedPracticeRebuild, schema: type[TableSchema], + ) -> int: + """Keep original observation parts and every existing claim on these attempts.""" + changed = super().retain_required_rows(original, schema) + for delivery in cls.select(original.destination): + claims = WakeAssignment.for_harness_sources( + original.destination, delivery.recipient_lookup, (delivery.reference,), + table_source=original.source, + ) + changed += original.retain_keys(WakeAssignment, tuple( + {"assignment_id": claim.assignment_id} for claim in claims + )) + return changed + @property def reference(self) -> MessageReference: return MessageReference(self.seq, self.message_id) diff --git a/src/agent_comms/typed_table.py b/src/agent_comms/typed_table.py index 40b22016e3..b114d84dc8 100644 --- a/src/agent_comms/typed_table.py +++ b/src/agent_comms/typed_table.py @@ -3504,6 +3504,20 @@ def __init_subclass__(cls, **kwargs: object) -> None: ) +class RetainsChildren(TypedTable, declared_name=None): + """A retained row requires its whole original child group through declared FK edges.""" + + __slots__ = () + + @classmethod + def retain_required_rows( + cls, original: AttachedPracticeRebuild, schema: type[TableSchema], + ) -> int: + return super().retain_required_rows(original, schema) + schema.retain_referencing_rows( + original, cls, + ) + + class RowidSequenced(TypedTable, declared_name=None): """The table's rows are numbered by SQLite's rowid, and an insert answers its number.""" diff --git a/tests/test_retained_coordinator_groups.py b/tests/test_retained_coordinator_groups.py new file mode 100644 index 0000000000..847f889eba --- /dev/null +++ b/tests/test_retained_coordinator_groups.py @@ -0,0 +1,137 @@ +"""Original harness groups survive selected loading without replaying their lifecycle.""" +from collections.abc import Iterator +from dataclasses import dataclass, replace +from pathlib import Path + +import pytest + +from agent_comms.assignment_states import FullPendingAssignment +from agent_comms.cli_bridges.claude import ClaudeBridge +from agent_comms.cli_bridges.codex import CodexBridge +from agent_comms.coordination_database import CoordinationStore +from agent_comms.coordination_errors import IdentityConflict +from agent_comms.coordination_tables.assignments import WakeAssignment +from agent_comms.coordination_tables.harness_deliveries import HarnessDelivery, HarnessObservedPart +from agent_comms.coordinator import StandaloneCoordination +from agent_comms.harness_delivery_states import EmittedOutcome, ReceivingMonitor +from agent_comms.message_reference import MessageReference +from agent_comms.nominal.field_codec import AssignmentId, MessageId, TranscriptRecordId +from agent_comms.wake_policy import FullWake, MessageAudience +from bus_record_fixture import RECIPIENT_LOOKUP + + +@dataclass(frozen=True) +class HarnessSource: + store: StandaloneCoordination + deliveries: tuple[HarnessDelivery, ...] + parts: tuple[HarnessObservedPart, ...] + claims: tuple[WakeAssignment, ...] + + @property + def path(self) -> Path: + return self.store.session.path + + +@pytest.fixture +def harness_source(tmp_path: Path) -> Iterator[HarnessSource]: + path = tmp_path / 'source' / 'coordination.sqlite3' + with StandaloneCoordination(str(path), clock_ms=lambda: 1000) as store: + store.participants.register(RECIPIENT_LOOKUP, committed=True) + claim = WakeAssignment( + assignment_id=AssignmentId.cohort(b'original-retained-claim'), + recipient='Alice', recipient_lookup=RECIPIENT_LOOKUP, + wire_seq=1, message_id=MessageId.mint('original-first-message'), + audience=MessageAudience.DIRECT, + lifecycle=FullPendingAssignment.load(FullWake(), None, None, None), + accepted_at_ms=1000, updated_at_ms=1000, + ) + store.assignments.accept(claim) + monitors = (ReceivingMonitor(2), ReceivingMonitor(1)) + deliveries = ( + HarnessDelivery(ClaudeBridge, RECIPIENT_LOOKUP, claim.message_id, 1, + EmittedOutcome, monitors[0]), + HarnessDelivery(ClaudeBridge, RECIPIENT_LOOKUP, + MessageId.mint('original-second-message'), 2, + EmittedOutcome, monitors[1]), + ) + parts = tuple( + HarnessObservedPart(delivery.bridge, delivery.recipient_lookup, delivery.message_id, + number, 'Sender', f'original body {delivery.seq}/{number}', + TranscriptRecordId(f'original-record-{delivery.seq}-{number}'), + 10.0 + delivery.seq + number / 10) + for delivery, monitor in zip(deliveries, monitors, strict=True) + for number in range(1, monitor.total + 1) + ) + with store.session.writing() as db: + for delivery in deliveries: + delivery.begin(store.assignments, db) + for part in parts: + part.insert(db) + # Keep the original post-handoff state and revision, not a reconstructed + # initial claim or a transition performed while materializing the copy. + yield HarnessSource(store, deliveries, parts, + (store.assignments.get(claim.assignment_id),)) + + +@pytest.mark.parametrize('selected_index', (0, 1), ids=('existing-claim', 'no-claim')) +def test_retained_harness_group_keeps_exact_parts_and_existing_claims( + harness_source: HarnessSource, tmp_path: Path, selected_index: int, +) -> None: + selected = harness_source.deliveries[selected_index] + destination = tmp_path / 'retained' / 'coordination.sqlite3' + before = CoordinationStore.digest(harness_source.path) + with CoordinationStore.retaining_rows( + harness_source.path, destination, maximum_bytes=8 * 2**20, + ) as original: + original.retain_keys(HarnessDelivery, ({ + 'bridge': selected.bridge, 'recipient_lookup': selected.recipient_lookup, + 'message_id': selected.message_id, + },)) + with StandaloneCoordination(str(destination)) as retained: + with retained.session.read(): + db = retained.session._connection + assert tuple(HarnessDelivery.select(db)) == (selected,) + assert tuple(HarnessObservedPart.select(db, order_by=('number',))) == tuple( + part for part in harness_source.parts if part.message_id == selected.message_id + ) + assert tuple(WakeAssignment.select(db)) == tuple( + claim for claim in harness_source.claims if claim.source == selected.reference + ) + assert CoordinationStore.digest(harness_source.path) == before + + +def test_harness_claim_query_refuses_conflicting_full_reference(harness_source: HarnessSource) -> None: + selected = harness_source.deliveries[0] + wrong = MessageReference(selected.seq, MessageId.mint('foreign-message-at-same-sequence')) + with harness_source.store.session.read(): + db = harness_source.store.session._connection + assert tuple(WakeAssignment.for_harness_sources( + db, selected.recipient_lookup, (selected.reference, selected.reference), + )) == harness_source.claims + assert tuple(WakeAssignment.for_harness_sources(db, selected.recipient_lookup, ())) == () + with pytest.raises(IdentityConflict): + tuple(WakeAssignment.for_harness_sources(db, selected.recipient_lookup, (wrong,))) + + +def test_retained_harness_conflict_refuses_without_publication( + harness_source: HarnessSource, tmp_path: Path, +) -> None: + # A foreign producer can leave a SQL-valid attempt whose natural-key claim + # names another message. Retention must refuse it, not filter the claim away. + conflicting = replace(harness_source.deliveries[0], bridge=CodexBridge, + message_id=MessageId.mint('foreign-source-message')) + with harness_source.store.session.writing() as db: + conflicting.insert(db) + before = CoordinationStore.digest(harness_source.path) + destination = tmp_path / 'retained' / 'coordination.sqlite3' + with pytest.raises(IdentityConflict): + with CoordinationStore.retaining_rows( + harness_source.path, destination, maximum_bytes=8 * 2**20, + ) as original: + original.retain_keys(HarnessDelivery, ({ + 'bridge': conflicting.bridge, 'recipient_lookup': conflicting.recipient_lookup, + 'message_id': conflicting.message_id, + },)) + assert not destination.exists() + assert not tuple(destination.parent.iterdir()) + assert CoordinationStore.digest(harness_source.path) == before From 8c8755981e05caebe0a6ca8bcde2e812e48d7d09 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sun, 1 Nov 2026 14:34:40 -0500 Subject: [PATCH 10/28] Retain native source membership and original execution children --- docs/retained-practice-inputs.md | 43 ++++++++ .../coordination_tables/executions.py | 2 + src/agent_comms/native_input_record.py | 40 ++++--- src/agent_comms/native_runtime_input.py | 36 ++++++- tests/test_retained_coordinator_groups.py | 101 +++++++++++++++++- 5 files changed, 206 insertions(+), 16 deletions(-) diff --git a/docs/retained-practice-inputs.md b/docs/retained-practice-inputs.md index 5b345ec648..ba0f396b5a 100644 --- a/docs/retained-practice-inputs.md +++ b/docs/retained-practice-inputs.md @@ -240,6 +240,49 @@ selection/root budget still need qualification. Ordinary include()/PracticeBus and live-root full backups remain unchanged; a caller cannot label their unmeasured union bounded merely by passing an allowance to one selected store. +## Native membership and execution groups + +A triage member's `assignment_id` is unique but has no declared assignment FK. +Forward FK closure alone therefore misses its original source claim. The +existing `NativeInputExecution.source_membership_sql` and +`source_assignment_ids` own this relationship for both TRIAGE and FULL. Their +queries now take the existing `TableSource` through every nested join; ordinary +calls still use normal local-table lookup. The membership query leads a left +join into the existing required `WakeAssignment` decoder, rather than silently +filtering a missing original claim through an inner join. That failure path is +source-reviewed, not yet executed. + +`NativeRuntimeInput` composes `RetainsChildren` for original triage/cursor +children and overrides the existing cooperative retention hook to consume its +held execution's original-source assignment query. It uses the shared raw-key +retainer for the claims; it does not write a second membership query, stage +roster or new native coordinate. `ExecutionRecord` composes the same capability +for its original declared child group, including native inputs, attempts and +links. No participant-wide reverse walk, lifecycle transition, dispatch or +claim acceptance is added. No field/DDL/carry declaration changed; actual schema, +MRO and load behavior remain unqualified. + +Three new scenarios are authored **UNRUN** in +`tests/test_retained_coordinator_groups.py`: selected triage keeps only its own +membership/claim; an execution seed keeps its original claimless FULL input; +explicit original-source lookup differs from empty destination-local lookup. +They use existing native-root/schema preparation and typed store owners, not a +handwritten table layout or fake native provider. These are SQL data-group +cases, not actual native admission or runnable-root acceptance. FULL wire +membership, malformed-source refusal, cursor/attempt/publication groups and +all-state suppression still need the necessary combined behavioral selection. +The PR now has thirteen added retention scenarios, all unrun in this source pass. + +The focused NRA before-map covers nineteen production modules and219 raw/static +class declarations with no parse omissions. The final delta additionally covers +the changed test module; counts are not evidence of runtime MRO, complete +cross-store closure or a delivered root. Complete paginated open-PR paths were +checked, including712/249-file requests. Published3820's superseded +`model_contexts_for` hunk is independent; its deletion stays, and the retention +hook has no `BusReader`/`MessageReference` dependency.3614's relevant changes are +nominal-import moves already present here. Source recipes/results are retained +under `p867-native-group-*`. + ## Source evidence and remaining work The retained NRA source map at baseline `095c4cdfe85632f990be3b6d67411d9685e6dee9` diff --git a/src/agent_comms/coordination_tables/executions.py b/src/agent_comms/coordination_tables/executions.py index 5472b8ef7a..027c6fffd2 100644 --- a/src/agent_comms/coordination_tables/executions.py +++ b/src/agent_comms/coordination_tables/executions.py @@ -55,6 +55,7 @@ NonDecreasing, NonIncreasing, OrdinalTransitions, + RetainsChildren, TextStorage, ) from agent_comms.wake_candidate_index import ProjectionUnavailableError @@ -275,6 +276,7 @@ def references(cls): @dataclass(frozen=True, slots=True) class ExecutionRecord( + RetainsChildren, LifecycleTransitions, OrdinalTransitions, IncrementingColumn, diff --git a/src/agent_comms/native_input_record.py b/src/agent_comms/native_input_record.py index c13507daf1..bb6aa1a8ed 100644 --- a/src/agent_comms/native_input_record.py +++ b/src/agent_comms/native_input_record.py @@ -36,7 +36,7 @@ from .native_integer import PositiveNativeIntegerBounds from .native_session_reopen import PiSessionId from .transcript_routes import InputDisplay, NoDisplayText -from .typed_table import SqlStorage, TextStorage +from .typed_table import LOCAL_TABLE_SOURCE, SqlStorage, TableSource, TextStorage if TYPE_CHECKING: from sqlite3 import Connection @@ -181,14 +181,25 @@ def proves_full_source(self, proof: HistoricalNativeInput) -> bool: @classmethod @abstractmethod - def source_membership_sql(cls) -> str: - """Select the assignments included in this stage's input.""" - - def source_assignment_ids(self, db: Connection, input_id: InputId) -> tuple[AssignmentId, ...]: + def source_membership_sql( + cls, *, table_source: TableSource = LOCAL_TABLE_SOURCE, + ) -> str: + """Select original membership from this query's actual table source.""" + + def source_assignment_ids( + self, db: Connection, input_id: InputId, *, + table_source: TableSource = LOCAL_TABLE_SOURCE, + ) -> tuple[AssignmentId, ...]: + """Membership leads: a missing claim reaches the required-row decoder. + + An inner join would silently erase a declared source whose claim was + absent, making corrupted membership look like a smaller valid group. + """ columns = ",".join(f"c.{column}" for column in WakeAssignment.columns()) originals = WakeAssignment.read(db.execute( - f"SELECT {columns} FROM {WakeAssignment.declared_name} c " - f"JOIN ({self.source_membership_sql()}) m ON m.assignment_id=c.assignment_id " + f"SELECT {columns} FROM ({self.source_membership_sql(table_source=table_source)}) m " + f"LEFT JOIN {table_source.qualified(WakeAssignment.declared_name)} c " + "ON m.assignment_id=c.assignment_id " "WHERE m.input_id=? ORDER BY m.ordinal", (input_id,), )) return tuple(original.assignment_id for original in originals) @@ -228,11 +239,13 @@ def transcript_events( super().transcript_events(entry, projection, user)] @classmethod - def source_membership_sql(cls) -> str: + def source_membership_sql( + cls, *, table_source: TableSource = LOCAL_TABLE_SOURCE, + ) -> str: from .triage_native_sources import TriageNativeSource return (f"SELECT s.input_id, s.assignment_id, s.ordinal " - f"FROM {TriageNativeSource.declared_name} s") + f"FROM {table_source.qualified(TriageNativeSource.declared_name)} s") def record_sources( self, db: Connection, input_id: InputId, assignments: tuple[WakeAssignment, ...], @@ -272,12 +285,15 @@ class FullNativeExecution(NativeInputExecution): attempt_ordinal: int @classmethod - def source_membership_sql(cls) -> str: + def source_membership_sql( + cls, *, table_source: TableSource = LOCAL_TABLE_SOURCE, + ) -> str: from .native_runtime_input import NativeRuntimeInput return (f"SELECT n.input_id, c.assignment_id, c.ordinal " - f"FROM {NativeRuntimeInput.declared_name} n " - f"JOIN {ExecutionAssignmentLink.declared_name} c ON c.execution_id=n.execution_id " + f"FROM {table_source.qualified(NativeRuntimeInput.declared_name)} n " + f"JOIN {table_source.qualified(ExecutionAssignmentLink.declared_name)} c " + "ON c.execution_id=n.execution_id " f"WHERE n.stage='{cls.declared_name}'") def record_sources( diff --git a/src/agent_comms/native_runtime_input.py b/src/agent_comms/native_runtime_input.py index f98a2ed4da..9149e79c71 100644 --- a/src/agent_comms/native_runtime_input.py +++ b/src/agent_comms/native_runtime_input.py @@ -58,6 +58,8 @@ from .tool_denial import SelectedToolDenied from .turn_item import FreshNativeId from .typed_table import ( + LOCAL_TABLE_SOURCE, + AttachedPracticeRebuild, Column, FrozenColumn, IntegerStorage, @@ -65,8 +67,11 @@ NoDelete, NonDecreasing, RequiredUpdateColumns, + RetainsChildren, StableReferenceColumns, TableReference, + TableSchema, + TableSource, TextStorage, TypedRow, WithoutRowid, @@ -96,6 +101,7 @@ class PublishedReplyRevision(TypedRow): @dataclass(frozen=True) class NativeRuntimeInput( + RetainsChildren, WriteOnceColumns, RequiredUpdateColumns, NoDelete, @@ -150,11 +156,37 @@ def write_once_columns(cls: type[NativeRuntimeInput]) -> tuple[str, ...]: required_update_columns = ('sent_owner_claimed_generation',) @classmethod - def source_membership_sql(cls) -> str: + def source_membership_sql( + cls, *, table_source: TableSource = LOCAL_TABLE_SOURCE, + ) -> str: return " UNION ALL ".join( - member.source_membership_sql() for member in NativeInputExecution.members_with(NativeInputExecution) + member.source_membership_sql(table_source=table_source) + for member in NativeInputExecution.members_with(NativeInputExecution) ) + @classmethod + def retain_required_rows( + cls, original: AttachedPracticeRebuild, schema: type[TableSchema], + ) -> int: + """Keep original triage/cursor children and the stage-owned source claims. + + Triage membership has no declared assignment FK. Its existing stage + query, not a parallel retention join or roster, owns that relationship. + Membership comes from the original attachment; only selected native ids + come from the destination. + """ + from .coordination_tables.assignments import WakeAssignment + + changed = super().retain_required_rows(original, schema) + for native in cls.stream(original.destination): + identifiers = native.execution.source_assignment_ids( + original.destination, native.input_id, table_source=original.source, + ) + changed += original.retain_keys(WakeAssignment, tuple( + {"assignment_id": identifier} for identifier in identifiers + )) + return changed + def __post_init__(self): self.execution # The selected identity and the later context acknowledgement are separate diff --git a/tests/test_retained_coordinator_groups.py b/tests/test_retained_coordinator_groups.py index 847f889eba..da032d044c 100644 --- a/tests/test_retained_coordinator_groups.py +++ b/tests/test_retained_coordinator_groups.py @@ -1,23 +1,32 @@ """Original harness groups survive selected loading without replaying their lifecycle.""" from collections.abc import Iterator +from contextlib import closing from dataclasses import dataclass, replace from pathlib import Path import pytest -from agent_comms.assignment_states import FullPendingAssignment +from agent_comms.assignment_states import FullPendingAssignment, TriagePendingAssignment +from agent_comms.comms import wire from agent_comms.cli_bridges.claude import ClaudeBridge from agent_comms.cli_bridges.codex import CodexBridge from agent_comms.coordination_database import CoordinationStore from agent_comms.coordination_errors import IdentityConflict from agent_comms.coordination_tables.assignments import WakeAssignment from agent_comms.coordination_tables.harness_deliveries import HarnessDelivery, HarnessObservedPart +from agent_comms.coordination_tables.executions import AcpOrigin, ExecutionRecord from agent_comms.coordinator import StandaloneCoordination from agent_comms.harness_delivery_states import EmittedOutcome, ReceivingMonitor from agent_comms.message_reference import MessageReference +from agent_comms.native_input_record import FullNativeExecution, TriageNativeExecution +from agent_comms.native_runtime_input import NativeRuntimeInput from agent_comms.nominal.field_codec import AssignmentId, MessageId, TranscriptRecordId +from agent_comms.triage_native_sources import TriageNativeSource +from agent_comms.typed_table import TypedTable from agent_comms.wake_policy import FullWake, MessageAudience -from bus_record_fixture import RECIPIENT_LOOKUP +from bus_record_fixture import ( + FIRST_INPUT_ID, SECOND_INPUT_ID, THIRD_INPUT_ID, RECIPIENT_LOOKUP, prepare_native_comms, +) @dataclass(frozen=True) @@ -135,3 +144,91 @@ def test_retained_harness_conflict_refuses_without_publication( assert not destination.exists() assert not tuple(destination.parent.iterdir()) assert CoordinationStore.digest(harness_source.path) == before + + +@dataclass(frozen=True) +class NativeGroupSource: + store: StandaloneCoordination + native: tuple[NativeRuntimeInput, ...] + claims: tuple[WakeAssignment, ...] + + @property + def path(self) -> Path: + return self.store.session.path + + +@pytest.fixture +def native_group_source(tmp_path: Path) -> Iterator[NativeGroupSource]: + comms = prepare_native_comms(wire(tmp_path / 'native-source')) + with StandaloneCoordination(str(comms.root / 'coordination.sqlite3'), + clock_ms=lambda: 1000) as store: + store.participants.register(RECIPIENT_LOOKUP, committed=True) + claims = tuple(WakeAssignment( + assignment_id=AssignmentId.cohort(f'original-triage-{number}'.encode()), + recipient='Alice', recipient_lookup=RECIPIENT_LOOKUP, wire_seq=number, + message_id=MessageId.mint(f'original-native-message-{number}'), + audience=MessageAudience.DIRECT, lifecycle=TriagePendingAssignment(), + accepted_at_ms=1000, updated_at_ms=1000, + ) for number in (1, 2)) + for claim in claims: + store.assignments.accept(claim) + store.create('original-claimless-full', AcpOrigin, RECIPIENT_LOOKUP, 1) + stages = (TriageNativeExecution(), TriageNativeExecution(), + FullNativeExecution('original-claimless-full', 1)) + native = tuple(NativeRuntimeInput( + input_id=input_id, stage=type(stage), **stage.binding_fields(), + owner_lookup=RECIPIENT_LOOKUP, owner_generation=1, + owner_token_digest='e' * 64, + ) for input_id, stage in zip( + (FIRST_INPUT_ID, SECOND_INPUT_ID, THIRD_INPUT_ID), stages, strict=True, + )) + with store.session.writing() as db: + for row, members in zip(native, ((claims[0],), (claims[1],), ()), strict=True): + row.insert(db) + row.execution.record_sources(db, row.input_id, members) + # These are original reserved SQL groups, not actual native admission or + # a running practice root. No provider or native process is started. + yield NativeGroupSource(store, native, claims) + + +@pytest.mark.parametrize('seed,keys,selected_index,claim_count,execution_ids', ( + (NativeRuntimeInput, {'input_id': FIRST_INPUT_ID}, 0, 1, ()), + (ExecutionRecord, {'execution_id': 'original-claimless-full'}, 2, 0, ('original-claimless-full',)), +), ids=('triage-input', 'full-execution')) +def test_retained_native_group_uses_original_membership_and_children( + native_group_source: NativeGroupSource, tmp_path: Path, seed: type[TypedTable], + keys: dict[str, object], selected_index: int, claim_count: int, execution_ids: tuple[str, ...], +) -> None: + source = native_group_source + selected = source.native[selected_index] + expected_claims = source.claims[:claim_count] + before = CoordinationStore.digest(source.path) + destination = tmp_path / 'native-retained' / 'coordination.sqlite3' + with CoordinationStore.retaining_rows(source.path, destination, maximum_bytes=8 * 2**20) as original: + original.retain_keys(seed, (keys,)) + with closing(CoordinationStore.read_snapshot(destination)) as db: + assert tuple(NativeRuntimeInput.select(db)) == (selected,) + assert tuple(WakeAssignment.select(db)) == expected_claims + assert tuple(TriageNativeSource.select(db)) == tuple( + TriageNativeSource(selected.input_id, ordinal, claim.assignment_id) + for ordinal, claim in enumerate(expected_claims) + ) + assert tuple(row.execution_id for row in ExecutionRecord.select(db)) == execution_ids + assert CoordinationStore.digest(source.path) == before + + +def test_native_membership_query_uses_the_supplied_attachment( + native_group_source: NativeGroupSource, tmp_path: Path, +) -> None: + source = native_group_source + selected = source.native[0] + before = CoordinationStore.digest(source.path) + destination = tmp_path / 'membership-query' / 'coordination.sqlite3' + with CoordinationStore.retaining_rows(source.path, destination, maximum_bytes=8 * 2**20) as original: + # Both table sets exist. Ordinary lookup sees the empty destination; + # qualification must keep every join on the original attachment. + assert selected.execution.source_assignment_ids(original.destination, selected.input_id) == () + assert selected.execution.source_assignment_ids( + original.destination, selected.input_id, table_source=original.source, + ) == (source.claims[0].assignment_id,) + assert CoordinationStore.digest(source.path) == before From d58a80c704a24365e970e5a821a6ccec34409218 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sun, 1 Nov 2026 15:23:34 -0500 Subject: [PATCH 11/28] Keep original input recipient birth in owner matching --- docs/retained-practice-inputs.md | 16 ++++++++++++++++ src/agent_comms/input_attempt.py | 7 ++++--- src/agent_comms/input_key.py | 11 +++++++++++ tests/test_input_documents.py | 16 ++++++++++++++++ 4 files changed, 47 insertions(+), 3 deletions(-) diff --git a/docs/retained-practice-inputs.md b/docs/retained-practice-inputs.md index ba0f396b5a..3013de2da9 100644 --- a/docs/retained-practice-inputs.md +++ b/docs/retained-practice-inputs.md @@ -5,6 +5,22 @@ a PracticeCopy consumer in source; bounded coordinator selection, native-journal reading and the prepared-root application run remain unfinished. This document records the source decisions, not an executable selection roster. +## Original recipient birth participates in owner matching + +Source tracing found `StoredInput.matches_owner` consulted only its origin, +ignoring the recipient birth already present in `OwnedBusInputKey`. For a neutral +origin, a recreated same-name owner could therefore satisfy that predicate even +when the input key named another original birth. + +`InputKey.matches_owner` now owns the key's contribution. Birthless keys add no +birth constraint; the owned bus member checks its original birth through the +incarnation's existing `recorded_or` behavior. `StoredInput` combines that answer +with the unchanged origin-owned author/name check. No key is re-encoded, missing +birth inferred, generation substituted, lifecycle changed or native permission +created. This tightens an existing consumer, not a second retention-only identity +resolver. The added ordinary owner/recreation/birthless-owner and queued-text +refusal scenario is authored **UNRUN**; the existing birthless-key case remains. + ## Existing owners and implemented entry selection `ContextStore.retain_contexts(destination, identities, maximum_bytes=..., requests=...)` replaces the diff --git a/src/agent_comms/input_attempt.py b/src/agent_comms/input_attempt.py index cf29663c95..dc61819881 100644 --- a/src/agent_comms/input_attempt.py +++ b/src/agent_comms/input_attempt.py @@ -339,9 +339,10 @@ def digest(self) -> TextDigest: return TextDigest.of(self.source_text) def matches_owner(self, source_owner: ThreadIncarnation) -> bool: - # Input with a recorded author uses that author's creation time. Older - # rows with only a name do not get an inferred thread identity. - return self.origin.matches_owner(self.owner, source_owner) + # The origin owns author evidence; an owned bus key additionally + # retains its original recipient birth. Neither invents a missing birth. + return (self.origin.matches_owner(self.owner, source_owner) + and self.key.matches_owner(source_owner)) def matches_claim(self, claim_scope: int) -> bool: return self.claim_generation == claim_scope diff --git a/src/agent_comms/input_key.py b/src/agent_comms/input_key.py index cd38a58dfd..cc3cb0ecd6 100644 --- a/src/agent_comms/input_key.py +++ b/src/agent_comms/input_key.py @@ -14,6 +14,7 @@ if TYPE_CHECKING: from .acp_extension import InputStartedUpdate + from .thread_identity import ThreadIncarnation class InputKey(DeclaredFamily): @@ -47,6 +48,10 @@ def public_text(self) -> str: def identify(self, update: InputStartedUpdate) -> InputStartedUpdate: return update + def matches_owner(self, incarnation: ThreadIncarnation) -> bool: + """A birthless key adds no birth claim to its origin's owner check.""" + return True + @dataclass(frozen=True) class AcpInputKey(InputKey): @@ -90,6 +95,12 @@ class OwnedBusInputKey(BusInputKey): def text(self) -> str: return super().text() + ":owner:" + self.owner_created.hex() + def matches_owner(self, incarnation: ThreadIncarnation) -> bool: + """The original recipient's recorded birth must still match.""" + return incarnation.recorded_or( + lambda recorded: recorded.created_at == self.owner_created, lambda: False, + ) + @dataclass(frozen=True) class TurnInputKey(InputKey): turn_id: TurnId diff --git a/tests/test_input_documents.py b/tests/test_input_documents.py index bd9eff783e..ec73e10676 100644 --- a/tests/test_input_documents.py +++ b/tests/test_input_documents.py @@ -281,6 +281,22 @@ def test_original_native_user_receipts_include_group_and_corrected_final_only(tm ) +def test_owned_bus_input_keeps_its_original_recipient_birth() -> None: + from agent_comms.input_key import OwnedBusInputKey + from agent_comms.thread_identity import UnrecordedThreadIncarnation + + row = ReservedInput( + OwnedBusInputKey(7, 10.0), 7, "owner", 4, "owner", "original input", + ) + assert row.matches_owner(RecordedThreadIncarnation("owner", 10.0)) + assert not row.matches_owner(RecordedThreadIncarnation("owner", 20.0)) + assert not row.matches_owner(RecordedThreadIncarnation("other", 10.0)) + assert not row.matches_owner(UnrecordedThreadIncarnation("owner")) + assert row.queued_text(RecordedThreadIncarnation("owner", 10.0), 4) == "original input" + with pytest.raises(RelationViolationError): + row.queued_text(RecordedThreadIncarnation("owner", 20.0), 4) + + def test_distinct_lifecycle_and_missing_state_never_supply_sent_evidence(tmp_path): store = InputDispositions(tmp_path) missing = stored_inputs(store.root).lookup(MISSING_KEY) From 6d61d8b206410ed25b8628bdc88156a08b4e11e3 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sun, 1 Nov 2026 16:24:00 -0500 Subject: [PATCH 12/28] Retain original publication receipts and append records --- docs/retained-practice-inputs.md | 33 +++++++++++ .../coordination_tables/publications.py | 3 +- tests/test_coordination_response.py | 57 +++++++++++++++++-- 3 files changed, 87 insertions(+), 6 deletions(-) diff --git a/docs/retained-practice-inputs.md b/docs/retained-practice-inputs.md index 3013de2da9..f73d4615e3 100644 --- a/docs/retained-practice-inputs.md +++ b/docs/retained-practice-inputs.md @@ -299,6 +299,39 @@ hook has no `BusReader`/`MessageReference` dependency.3614's relevant changes ar nominal-import moves already present here. Source recipes/results are retained under `p867-native-group-*`. +## Publication groups and the existing capture lifetime + +`PublicationIntents` now composes the same `RetainsChildren` capability as +`ExecutionRecord`. The schema's declared inverse relations retain the original +`PublicationEvidence` and `PublicationAppendDispatches` rows. The latter's foreign +key is declared in `coordination_response.py`, not in a retention-only roster. +`RecoveryReader.get` reads the original intents and receipts separately; +`PublicationAppendDispatches.resolve_retained` uses the original append record. +Copying the intent alone does not preserve those answers. The raw loader retains +absence as absence: no receipt, dispatch, acknowledgement or state transition is +created to fill a missing record. + +Four publication cases are authored **UNRUN** in +`tests/test_coordination_response.py`: intent/execution seeds crossed with frozen +intent/published response. They reuse the existing actual response preparation +and publication path, compare original intent/receipt/append rows, and check +that retention changes neither the source coordinator digest nor the bus +revision. The existing fixture now declares its actual `AssignmentId` and +`OwnerFence`, rather than `str` and `object`. These are stored SQL-group controls, +not runnable-root, native, recovery-pointer or physical acceptance. Together with +the earlier thirteen retention scenarios and one recipient-birth regression, +eighteen added scenarios remain unrun. + +The live installer already enters `mode.owners_stopped` before its practice +phase (`installer/upgrade.py`). `LiveMode` uses the existing maintenance hold, +stopped-owner and retargeted-runtime lifetime. `RehearsalMode` instead calls +`owners_kept` (`install_choices.py`). A plain rehearsal therefore does not supply +that quiet source boundary. The selected coordinator consumer must use the +existing qualified capture lifetime and preserve its registry/complete-ledger/ +bus/coordinator relationships; this source trace neither executes nor certifies +that aggregate capture. No new freeze state, installer or donor operation was +added. + ## Source evidence and remaining work The retained NRA source map at baseline `095c4cdfe85632f990be3b6d67411d9685e6dee9` diff --git a/src/agent_comms/coordination_tables/publications.py b/src/agent_comms/coordination_tables/publications.py index adf38fb3a8..69fad5de39 100644 --- a/src/agent_comms/coordination_tables/publications.py +++ b/src/agent_comms/coordination_tables/publications.py @@ -28,6 +28,7 @@ Generated, GuardedInserts, ImmutableRows, + RetainsChildren, ValidatedInserts, ) @@ -37,7 +38,7 @@ @dataclass(frozen=True, slots=True) class PublicationIntents( - ValidatedInserts, GuardedInserts, ImmutableRows, CoordinatorRow + RetainsChildren, ValidatedInserts, GuardedInserts, ImmutableRows, CoordinatorRow ): """One obligation route's frozen response: the exact Message it publishes.""" diff --git a/tests/test_coordination_response.py b/tests/test_coordination_response.py index 8febcd437e..8e4cc72bdb 100644 --- a/tests/test_coordination_response.py +++ b/tests/test_coordination_response.py @@ -4,6 +4,7 @@ from agent_comms.repo import Checkout import os +from contextlib import closing from dataclasses import dataclass, replace from pathlib import Path @@ -14,6 +15,7 @@ from agent_comms.child_process import ProcessIdentity from agent_comms.comms import Comms, wire from conftest import accept_initial, publish_initial, responder +from agent_comms.coordination_database import CoordinationStore from agent_comms.coordination_errors import ( IdentityConflict, StaleFence, @@ -21,24 +23,27 @@ ) from agent_comms.coordination_response import ( LiveResponseOwner, + PublicationAppendDispatches, prepare_fenced_response, publish_fenced_response, ) -from agent_comms.coordination_tables.executions import NoPointedExecution, WireOrigin +from agent_comms.coordination_tables.executions import ExecutionRecord, NoPointedExecution, WireOrigin +from agent_comms.coordination_tables.publications import PublicationEvidence, PublicationIntents from agent_comms.coordination_tables.responses import ResponseObligation from agent_comms.coordinator import Coordination -from agent_comms.nominal.field_codec import InputId, RootId, ThreadLookup, TurnId +from agent_comms.nominal.field_codec import AssignmentId, InputId, RootId, ThreadLookup, TurnId from agent_comms.lifecycle_capability import Active, Complete from agent_comms.message_bus import MessageBus from agent_comms.assignment_states import CompletedAssignment from agent_comms.coordination_results import AlreadyApplied from agent_comms.execution_states import CompletedExecution from agent_comms.obligation_states import PendingResponse, PublishedResponse, PublishingResponse -from agent_comms.owner_fence import prepare_fence_token +from agent_comms.owner_fence import OwnerFence, prepare_fence_token from agent_comms.publisher import ParticipantPublisher from agent_comms.registration import Registration from agent_comms.selected_source_batch import SelectedSource from agent_comms.threads import Thread +from agent_comms.typed_table import TypedTable from bus_record_fixture import prepare_native_comms # Private bus publication requires POSIX owner/mode ancestry; Windows stat @@ -55,8 +60,8 @@ class Fixture: origin_seq: int reply_target: str owner_lookup: ThreadLookup - assignment_id: str - fence: object + assignment_id: AssignmentId + fence: OwnerFence witness: LiveResponseOwner def close(self) -> None: @@ -367,3 +372,45 @@ def test_response_answers_the_messages_its_turn_took_in(tmp_path: Path) -> None: assert [reply.reference for reply in replies] == [response.reference] finally: case.close() + + +@pytest.mark.parametrize('seed', (PublicationIntents, ExecutionRecord), ids=('intent', 'execution')) +@pytest.mark.parametrize('published', (False, True), ids=('frozen-intent', 'published')) +def test_selected_publication_keeps_original_receipts_and_append_dispatches( + tmp_path: Path, seed: type[TypedTable], published: bool, +) -> None: + case = _ready(tmp_path) + try: + intent = prepare_fenced_response( + case.store, case.bus, case.fence, 'Original retained publication', + exact_target=case.reply_target, owner_witness=case.witness, + ).value + if published: + publish_fenced_response( + case.store, case.bus, case.fence, + exact_target=case.reply_target, owner_witness=case.witness, + ) + snapshot = case.store.get(intent.execution_id) + with case.store.session.read(): + dispatches = tuple(PublicationAppendDispatches.select( + case.store.session._connection, where='execution_id=?', + parameters=(intent.execution_id,), + )) + assert len(snapshot.publication_evidence) == int(published) + assert len(dispatches) == int(published) + source = case.store.session.path + before = CoordinationStore.digest(source) + bus_revision = case.bus.bus.revision() + destination = tmp_path / 'retained-publication' / 'coordination.sqlite3' + with CoordinationStore.retaining_rows( + source, destination, maximum_bytes=8 * 2**20, + ) as original: + original.retain_keys(seed, ({'execution_id': intent.execution_id},)) + with closing(CoordinationStore.read_snapshot(destination)) as db: + assert tuple(PublicationIntents.select(db)) == snapshot.publication_intents + assert tuple(PublicationEvidence.select(db)) == snapshot.publication_evidence + assert tuple(PublicationAppendDispatches.select(db)) == dispatches + assert CoordinationStore.digest(source) == before + assert case.bus.bus.revision() == bus_revision + finally: + case.close() From b01edf995ad21960842bd477f75a61f6ef6e5156 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sun, 1 Nov 2026 17:23:49 -0500 Subject: [PATCH 13/28] Select original coordinator inputs through the unresolved reader --- docs/retained-practice-inputs.md | 77 +++++++++++++++++++----- src/agent_comms/coordination_database.py | 29 ++++++++- src/agent_comms/input_disposition.py | 19 ++++++ src/agent_comms/input_origin.py | 21 ++++++- src/agent_comms/installer/practice.py | 31 ++++++++++ src/agent_comms/native_runtime_input.py | 15 +++++ tests/test_harness_receipt_context.py | 56 +++++++++++++++++ 7 files changed, 229 insertions(+), 19 deletions(-) diff --git a/docs/retained-practice-inputs.md b/docs/retained-practice-inputs.md index f73d4615e3..1c0af74c21 100644 --- a/docs/retained-practice-inputs.md +++ b/docs/retained-practice-inputs.md @@ -1,7 +1,8 @@ # Original-coordinate practice inputs — implementation checkpoint -Core #3829 remains unfinished. Entry and selected saved-context retention now have -a PracticeCopy consumer in source; bounded coordinator selection, native-journal +Core #3829 remains unfinished. Entry, selected saved-context and coordinator input +evidence now have explicit PracticeCopy consumers in source. Complete coordinator +relations, coherent aggregate capture, bounded peak materialization, native-journal reading and the prepared-root application run remain unfinished. This document records the source decisions, not an executable selection roster. @@ -87,8 +88,8 @@ and primary-key deduplication with `retain_rows`; no second key encoder or raw-c procedure is added. Completed changes come from the connection, since SQLite's WITH-prefixed INSERT reports Cursor.rowcount=-1. -ContextStore and PracticeCopy now consume the primitive; the coordinator selector -still does not. Guard-safe loading, required children, source snapshot consistency, +ContextStore and CoordinationStore now consume the primitive through their +explicit PracticeCopy selection methods. Guard-safe loading, required children, source snapshot consistency, producer metadata, sequences, schema compatibility, materialized bounds and copy behavior remain unqualified. No original lifecycle is rewritten and no full-copy/backup behavior is changed. These methods are implementation prerequisites, not a runnable root. @@ -120,7 +121,7 @@ The existing rebuild API forwards the same transaction connection through schema reconciliation; that parameter is not an independently maintained source snapshot. No additional rebuild hierarchy or unchanged-whole-file gate was added. Actual WAL interleaving, source/target isolation and producer-prefix controls are still -unrun, and no CoordinationStore/PracticeCopy consumer is claimed. +unrun; the coordinator consumer below is a source implementation, not qualification. ## Shared loading lifetime and acknowledgement groups — source only @@ -152,8 +153,9 @@ actual class creation/MRO, schema stability and group-copy behavior still requir qualification. Four new harness-group scenarios are authored, UNRUN: selected observations with an existing post-handoff claim, observations with no claim, full-reference conflict, and copy conflict with no partial publication. -CoordinationStore/PracticeCopy has no coordinator-seed selection consumer yet. Coordinator seed capture and the remaining -serialized/cross-store and producer dependencies remain OPEN; complete FK validity +The coordinator seed consumer below now selects through the existing input +projection. Aggregate capture and the remaining serialized/cross-store and producer +dependencies remain OPEN; complete FK validity would not establish their preservation or the original group's validity. ## Required preservation that SQL foreign keys do not establish @@ -210,10 +212,43 @@ complete ledger, captured consistently with the coordinator source; the caller owns that capture and attachment lifetime. This is not a second ledger or an input replay operation. -The view is not yet consumed by a coordinator materializer or `PracticeCopy`. -Runtime class creation/MRO/registration, the actual attached query, cross-store -capture and complete preservation remain unqualified. No seed-only database is -published as a runnable root or complete backup. +`SnapshotInputDispositions.retain_unresolved` now consumes that unchanged reader +and seeds original attempt groups at the selected keys. It asks the existing +harness producer a second time on the same attachment, not a copied key/outcome +or suppression algorithm. An unresolved stored ledger row can keep an additional +original attempt witness at its key; the complete ledger still wins the read. +The recipient lookup comes from those original source entries, not from a stored +name/generation or a fabricated birth. `HarnessAttemptOrigin.retain_harness_rows` +owns its bridge/reference selection and full-sequence check; origins without a +harness witness contribute no harness rows. No synthesized reservation is stored. + +`CoordinationStore.retain_inputs` runs these seeds through the existing +`ReconciledFile.retaining_rows` loading/dependency/publication lifetime. +`NativeRuntimeInput.retain_sessions` additionally selects original native groups +by both recorded session id and original session path. A bare ledger native id +is not treated as a coordinator FK. The existing row hooks retain the selected +original children, claims and forward references. + +`PracticeCopy.include_inputs` requires existing bus/ledger sources to have been +captured completely through `include`; a copied reader with no original source +refuses. The coordinator uses the remaining aggregate allowance, and its resource +is recorded only after successful publication. Ordinary whole copies/backups are +unchanged. The caller still owns the consistent source cut and serial copy; this +is not a capture certificate or native launch permission. + +Three new cases in `tests/test_harness_receipt_context.py` are authored **UNRUN**: +no ledger row, an original reservation, and an actually bound/started ledger row. +They use the existing native-root/message/claim producers, compare the ordinary +source and retained projections and original attempt rows, and check the source +coordinator bytes and bus revision. Together with the eighteen previous cases, +**twenty-one added scenarios remain unrun**. These do not cover every disposition, +native association or rollback/resource failure. + +Runtime class creation/MRO/registration, attached SQL, cross-store capture and +complete preservation remain unqualified. The selector has no participant-wide +reverse walk; current participant/pointer aggregates, inverse native membership, +serialized references and the remaining original recovery groups still require +semantic closure. No seed-only database is a runnable root or complete backup. ## Copy and backup are different contracts @@ -287,7 +322,7 @@ handwritten table layout or fake native provider. These are SQL data-group cases, not actual native admission or runnable-root acceptance. FULL wire membership, malformed-source refusal, cursor/attempt/publication groups and all-state suppression still need the necessary combined behavioral selection. -The PR now has thirteen added retention scenarios, all unrun in this source pass. +That checkpoint added thirteen retention scenarios, all unrun in the source pass. The focused NRA before-map covers nineteen production modules and219 raw/static class declarations with no parse omissions. The final delta additionally covers @@ -320,7 +355,7 @@ revision. The existing fixture now declares its actual `AssignmentId` and `OwnerFence`, rather than `str` and `object`. These are stored SQL-group controls, not runnable-root, native, recovery-pointer or physical acceptance. Together with the earlier thirteen retention scenarios and one recipient-birth regression, -eighteen added scenarios remain unrun. +those eighteen scenarios remain unrun; the later consumer adds three more above. The live installer already enters `mode.owners_stopped` before its practice phase (`installer/upgrade.py`). `LiveMode` uses the existing maintenance hold, @@ -351,9 +386,19 @@ closure. External/dynamic bindings, serialized links and required reverse childr remain OPEN. The earlier d4/Toad corpus and its unresolved cases are separate; these counts do not resolve them. New metadata methods require targeted reproof. -Next: finish coordinator seeds and the remaining semantic groups, wire that -selection through PracticeCopy and enforce the aggregate copy budget, then -qualify the coherent implementation through the combined backend/carry and actual +The current post-P859 selector slice covers 30 production/test/fixture modules +and 585 original/canonical class declarations, with no parse omissions. It extends +the 22-module/524-class before-slice through four fixture/test modules and four +moved-owner modules. The intervening 26-module/525-class result keeps its older +source epoch. Compilation and exact NRA-indexed AST comparisons preserve all 148 +existing functions in the six changed Python files. Five mapped dependencies +changed on rebase: registry resolution declarations, bus selection imports, the +FieldCodec compared-field/decoder owner, and two fixture import moves. Those +changes were read; they are not inherited runtime qualification. These are source +checks, not native transport or SQL execution. + +Next: finish the remaining coordinator semantic groups and qualify coherent +aggregate capture and peak budget, then the combined backend/carry and actual prepared-root saved-history journey. No pytest, provider, donor mutation or new native/physical run was performed for this source checkpoint. Reader promotion still requires the separate default-storage and decorator creation-cut proofs. diff --git a/src/agent_comms/coordination_database.py b/src/agent_comms/coordination_database.py index 9aebd67ca2..5c1941c80d 100644 --- a/src/agent_comms/coordination_database.py +++ b/src/agent_comms/coordination_database.py @@ -7,11 +7,11 @@ import stat import tempfile from abc import abstractmethod -from collections.abc import Iterator +from collections.abc import Iterable, Iterator from contextlib import closing, contextmanager, suppress from dataclasses import dataclass from pathlib import Path -from typing import Self +from typing import TYPE_CHECKING, Self from agent_comms.coordination_errors import CoordinationReadUnavailable, IntegrityViolationError from agent_comms.coordination_schema import CoordinatorTable @@ -27,6 +27,11 @@ from .nominal.declared_family import DeclaredFamily from .store_files import DEFAULT_STORE_CONTENTION, StoreContention +if TYPE_CHECKING: + from .input_disposition import InputDocument + from .native_session_reopen import NativeSessionIdentity + from .registry_document import RegistryEntry + class StoreFile(DeclaredFamily): """One decoded file state, with the consumer's result inside its read lifetime.""" @@ -82,6 +87,26 @@ def schema_at(cls, path: Path) -> type[TableSchema]: return RegistryStore.at(path.parent).privacy().coordinator_schema() + @classmethod + def retain_inputs( + cls, source: Path, destination: Path, entries: tuple[RegistryEntry, ...], *, + maximum_bytes: int, identities: Iterable[NativeSessionIdentity] = (), + ) -> InputDocument: + """Materialize original input witnesses under the existing fresh-file owner. + + The caller supplies a consistent retained bus/complete ledger beside the + destination and the original registry entries from that same cut. This + selects data only; capture consistency and runnable-root authority remain + the capture/preparation owners' separate obligations. + """ + from .input_disposition import SnapshotInputDispositions + from .native_runtime_input import NativeRuntimeInput + + with cls.retaining_rows(source, destination, maximum_bytes=maximum_bytes) as original: + selected = SnapshotInputDispositions(destination.parent, original).retain_unresolved(entries) + NativeRuntimeInput.retain_sessions(original, identities) + return selected + @classmethod def observed(cls, path: Path) -> StoreFile: """Classify through SQLite without dropping another connection's process locks. diff --git a/src/agent_comms/input_disposition.py b/src/agent_comms/input_disposition.py index b1587f4163..c25cd8b200 100644 --- a/src/agent_comms/input_disposition.py +++ b/src/agent_comms/input_disposition.py @@ -578,3 +578,22 @@ def _harness_deliveries(self, owners: tuple[ThreadLookup, ...]) -> tuple[Harness return HarnessDelivery.unconfirmed_for( self.original.destination, owners, table_source=self.original.source, ) + + def retain_unresolved(self, entries: tuple[RegistryEntry, ...]) -> InputDocument: + """Seed original attempt groups selected by the ordinary unresolved reader. + + The complete copied ledger owns suppression in every state. This offline + selection reuses the same attachment and producer a second time, rather + than restating its key/outcome/suppression decisions. An unresolved ledger + row may keep an additional original attempt witness at its key; it still + wins the projection. No native association is inferred from a scalar id. + """ + selected = self.unresolved(entries) + recipients = {entry.thread.name: entry.thread.incarnation.require_recorded().lookup + for entry in entries} + for attempted in self.harness_attempts(entries): + if attempted.key in selected.rows: + attempted.origin.retain_harness_rows( + self.original, recipients[attempted.owner], + ) + return selected diff --git a/src/agent_comms/input_origin.py b/src/agent_comms/input_origin.py index a9211b20a3..9d251aa733 100644 --- a/src/agent_comms/input_origin.py +++ b/src/agent_comms/input_origin.py @@ -10,7 +10,7 @@ from .bus_publication import HumanOrigin from .nominal.declared_family import DeclaredFamily from .errors import RelationViolationError -from .nominal.field_codec import RootId +from .nominal.field_codec import RootId, ThreadLookup from .goals import GoalCheckpoint from .harness_install import HarnessSession from .input_key import InputKey, InputKeyText @@ -24,6 +24,7 @@ from .input_attempt import StoredInput from .registry_document import RegistrySnapshot from .threads import Thread + from .typed_table import AttachedPracticeRebuild class InputDelivery: @@ -61,6 +62,12 @@ def retained_fact(self, source: StoredInput): def require_human(self) -> HumanInputOrigin: raise RelationViolationError("Input lacks an original human author witness") + def retain_harness_rows( + self, original: AttachedPracticeRebuild, recipient: ThreadLookup, + ) -> int: + """An origin without a harness witness contributes no harness rows.""" + return 0 + def matches_owner(self, recorded_name, incarnation): return recorded_name == incarnation.name @@ -124,6 +131,18 @@ class HarnessAttemptOrigin(InputOrigin): bridge: type[HarnessSession] reference: MessageReference + def retain_harness_rows(self, original, recipient): + from .coordination_errors import IdentityConflict + from .coordination_tables.harness_deliveries import HarnessDelivery + + key = {"bridge": self.bridge, "recipient_lookup": recipient, + "message_id": self.reference.message_id} + changed = original.retain_keys(HarnessDelivery, (key,)) + retained = HarnessDelivery.require(original.destination, **key) + if retained.reference != self.reference: + raise IdentityConflict("Retained harness attempt names another message sequence") + return changed + @dataclass(frozen=True) class HumanInputOrigin(InputOrigin, ClaimScoped): diff --git a/src/agent_comms/installer/practice.py b/src/agent_comms/installer/practice.py index 9633532427..a7f07aee71 100644 --- a/src/agent_comms/installer/practice.py +++ b/src/agent_comms/installer/practice.py @@ -10,7 +10,11 @@ from math import ceil from collections.abc import Iterable +from agent_comms.bus import Bus from agent_comms.context_store import ContextManifestReference, ContextStore +from agent_comms.coordination_database import CoordinationStore +from agent_comms.input_disposition import InputDispositions, InputDocument +from agent_comms.registry_document import RegistryEntry from agent_comms.native_session_reopen import NativeSessionIdentity from agent_comms.nominal.field_codec import boundary_decoder from agent_comms.installer.refusals import UpgradeRefused @@ -193,6 +197,33 @@ def include_contexts( ) self.copied[source.path] = resource + def include_inputs( + self, entries: tuple[RegistryEntry, ...], *, maximum_bytes: int, + identities: Iterable[NativeSessionIdentity] = (), + ) -> InputDocument: + """Selected coordinator evidence beside the complete captured input readers. + + include() owns capturing the original bus and ledger; this method never + substitutes a selected ledger or creates suppression evidence. Caller owns + one consistent cut and serial materialization, not native launch authority. + """ + self.prepare() + for owner in (Bus, InputDispositions): + source = owner.path_in(self.root) + if source.exists(): + if source not in self.copied: + raise UpgradeRefused(f"Capture complete {source.name} before selecting input evidence") + elif owner.path_in(self.path).exists(): + raise UpgradeRefused(f"Copied {source.name} has no original reader source") + source = CoordinationStore.path_in(self.root) + resource, = RootFile.owned_files(LiveFilesRequest(self.root), (source,)) + selected = CoordinationStore.retain_inputs( + source, CoordinationStore.path_in(self.path), entries, + identities=identities, maximum_bytes=self.remaining_bytes(maximum_bytes), + ) + self.copied[source] = resource + return selected + def include(self, paths: tuple[Path, ...]) -> None: self.prepare() witnesses = set(MaintenanceBarrier(RegistryFile.path_in(self.root)).witness_paths) diff --git a/src/agent_comms/native_runtime_input.py b/src/agent_comms/native_runtime_input.py index 9149e79c71..2478112841 100644 --- a/src/agent_comms/native_runtime_input.py +++ b/src/agent_comms/native_runtime_input.py @@ -7,6 +7,7 @@ from __future__ import annotations import sqlite3 +from collections.abc import Iterable from dataclasses import dataclass, field from functools import partial from typing import TYPE_CHECKING, Annotated @@ -164,6 +165,20 @@ def source_membership_sql( for member in NativeInputExecution.members_with(NativeInputExecution) ) + @classmethod + def retain_sessions( + cls, original: AttachedPracticeRebuild, identities: Iterable[NativeSessionIdentity], + ) -> int: + """Select the original SQL session group, never an input-id-only association. + + A selected journal and recorded session id must both agree. Raw retention + preserves absent context/claim fields; selection grants no native proof. + """ + return original.retain_keys(cls, tuple( + {"session_id": identity.session_id, "session_file": identity.session_file} + for identity in identities + )) + @classmethod def retain_required_rows( cls, original: AttachedPracticeRebuild, schema: type[TableSchema], diff --git a/tests/test_harness_receipt_context.py b/tests/test_harness_receipt_context.py index ddb6b14ec3..161e8259d1 100644 --- a/tests/test_harness_receipt_context.py +++ b/tests/test_harness_receipt_context.py @@ -1,10 +1,18 @@ """Interpret actual persisted receipt variants without starting a harness/model.""" +from collections.abc import Callable +from contextlib import closing from dataclasses import replace import pytest from agent_comms.cli_bridges.claude import ClaudeBridge +from agent_comms.bus import Bus +from agent_comms.coordination_database import CoordinationStore from agent_comms.coordination_tables.harness_deliveries import HarnessDelivery +from agent_comms.input_attempt import ReservedInput +from agent_comms.input_disposition import InputDispositions +from agent_comms.installer.practice import PracticeCopy +from agent_comms.nominal.field_codec import InputId, TurnId from agent_comms.coordinator import Coordination from agent_comms.harness_delivery_states import ( AcceptedOutcome, DeliveredOutcome, EmittedOutcome, FilteredOutcome, @@ -59,3 +67,51 @@ def test_only_retained_complete_body_proof_promotes_consumption( assert recorded.monitor == monitor if consumed: assert current.label == current.timed_state(HarnessConsumedProgress.label, 1791964800125) + + +def _record_attempt(ledger: InputDispositions, attempt: ReservedInput) -> None: + ledger.record_originals(attempt) + + +def _start_attempt(ledger: InputDispositions, attempt: ReservedInput) -> None: + _record_attempt(ledger, attempt) + turn, native = TurnId.mint(), InputId.mint() + assert ledger.bind(attempt.key, claim_scope=attempt.claim_generation, + turn_id=turn, native_id=native, text=attempt.source_text) + assert ledger.started(attempt.key, turn_id=turn, native_id=native, text=attempt.source_text) + + +@pytest.mark.parametrize(("record", "retained_attempts"), ( + (lambda ledger, attempt: None, 1), + (_record_attempt, 1), + (_start_attempt, 0), +)) +def test_selected_coordinator_materializer_uses_the_original_unresolved_projection( + tmp_path, record: Callable[[InputDispositions, ReservedInput], None], retained_attempts: int, +): + """Original absence stays visible; a stored completed input cannot resurrect.""" + path, _root_id, comms, initial, people = _root(tmp_path, direct=True) + recipient, = initial.recipients + original = HarnessDelivery(ClaudeBridge, recipient.recipient_lookup, + initial.message.message_id, initial.message.seq, EmittedOutcome) + with Coordination(CoordinationStore.path_in(path)) as store: + with store.session.writing() as db: + assert original.begin(store.assignments, db) + entry = comms.registry.store.snapshot_with_authority().snapshot.entry(people[2].name) + ledger = InputDispositions(path) + attempt, = ledger.harness_attempts((entry,)) + record(ledger, attempt) + expected = ledger.unresolved((entry,)) + coordinator_before = CoordinationStore.path_in(path).read_bytes() + bus_before = comms.bus.bus.revision + copy = PracticeCopy(path, tmp_path / "selected-inputs") + copy.include((Bus.path_in(path), ledger.path)) + selected = copy.include_inputs((entry,), maximum_bytes=3 * 2**30) + assert selected == expected + assert InputDispositions(copy.path).unresolved((entry,)) == expected + with closing(CoordinationStore.read_snapshot(CoordinationStore.path_in(copy.path))) as db: + retained = tuple(HarnessDelivery.select(db)) + assert len(retained) == retained_attempts + assert all(row == original for row in retained) + assert CoordinationStore.path_in(path).read_bytes() == coordinator_before + assert comms.bus.bus.revision == bus_before From 1ea0c60983adfd78da23d1b15172564364a68c94 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sun, 1 Nov 2026 18:12:25 -0500 Subject: [PATCH 14/28] Retain participant generation and current pointer through the existing aggregate --- docs/retained-practice-inputs.md | 44 ++++++++++-- src/agent_comms/coordination_database.py | 4 ++ .../coordination_tables/participants.py | 9 +++ src/agent_comms/goal_attempts.py | 7 +- src/agent_comms/participant_store.py | 54 +++++++++++--- src/agent_comms/typed_table.py | 36 +++++++--- tests/test_retained_coordinator_groups.py | 72 ++++++++++++++++++- 7 files changed, 196 insertions(+), 30 deletions(-) diff --git a/docs/retained-practice-inputs.md b/docs/retained-practice-inputs.md index 1c0af74c21..1e179f7466 100644 --- a/docs/retained-practice-inputs.md +++ b/docs/retained-practice-inputs.md @@ -241,14 +241,48 @@ no ledger row, an original reservation, and an actually bound/started ledger row They use the existing native-root/message/claim producers, compare the ordinary source and retained projections and original attempt rows, and check the source coordinator bytes and bus revision. Together with the eighteen previous cases, -**twenty-one added scenarios remain unrun**. These do not cover every disposition, -native association or rollback/resource failure. +that checkpoint's **twenty-one added scenarios remained unrun**. The participant +checkpoint below adds three more, for **twenty-four added scenarios UNRUN**. +These do not cover every disposition, native association or rollback/resource failure. Runtime class creation/MRO/registration, attached SQL, cross-store capture and complete preservation remain unqualified. The selector has no participant-wide -reverse walk; current participant/pointer aggregates, inverse native membership, -serialized references and the remaining original recovery groups still require -semantic closure. No seed-only database is a runnable root or complete backup. +reverse walk. Participant aggregates now contribute through the row's cooperative +retention hook below; inverse native membership, serialized references and the +remaining original recovery groups still require semantic closure. No seed-only database is a runnable root or complete backup. + +## Participant aggregate — source implementation, not acceptance + +`ParticipantStore.get` requires `Participants`, `OwnerGenerations` and +`CurrentExecutions`. A foreign-key-valid participant alone cannot answer that +reader. `ParticipantStore.read_at` now owns the same three reads and original +incomplete-aggregate refusal through the caller's already-held connection/source. +Ordinary `get` keeps its existing session read lifetime and consumes that method; +retention never constructs a borrowed `CoordinationSession` or another connection. + +`ParticipantStore.retain_owners` seeds selected original lookup keys, preserves an +absent participant as absent, and refuses a present incomplete aggregate. +`ParticipantSnapshot.retain_original` keeps its original generation/current pointer. +`Participants.retain_required_rows` contributes the same aggregate through the +existing cooperative hook, including participants discovered later by FK closure, +not just the initial registry entries. This is not a reverse traversal of every +participant execution. The current pointer's declared references retain its actual +pointed execution/attempt; no counter, revision, commitment or lifecycle is reset. + +`TypedTable.retention_key_fields` derives the primary key once from its declared +columns for raw-copy deduplication and held-row selection. `retain_original` gets +those values through existing `FieldPlan.current` and the shared indexed/raw-copy +compiler, not a field roster or decoded-row reinsertion. `one`, `require` and +`_keyed` now carry the same `TableSource` as `select`; the actual `Generation` +override carries it too while preserving its corruption exception. + +Three cases are authored **UNRUN**: an advanced generation and active current +pointer preserve the exact aggregate without unrelated/native history; missing +generation or missing pointer refuses publication. The existing two native-group +seed cases also require the retained aggregate. Source mapping covers 65 modules, +537 original class declarations, 519 canonical projections, no parse omissions; +18 function-local test declarations remain unprojected OPEN. That is source +coverage, not executed class/MRO/SQL behavior or complete closure. ## Copy and backup are different contracts diff --git a/src/agent_comms/coordination_database.py b/src/agent_comms/coordination_database.py index 5c1941c80d..ece03d2ff3 100644 --- a/src/agent_comms/coordination_database.py +++ b/src/agent_comms/coordination_database.py @@ -101,9 +101,13 @@ def retain_inputs( """ from .input_disposition import SnapshotInputDispositions from .native_runtime_input import NativeRuntimeInput + from .participant_store import ParticipantStore with cls.retaining_rows(source, destination, maximum_bytes=maximum_bytes) as original: selected = SnapshotInputDispositions(destination.parent, original).retain_unresolved(entries) + ParticipantStore.retain_owners(original, tuple( + entry.thread.incarnation.require_recorded().lookup for entry in entries + )) NativeRuntimeInput.retain_sessions(original, identities) return selected diff --git a/src/agent_comms/coordination_tables/participants.py b/src/agent_comms/coordination_tables/participants.py index 44c7bfaf7f..83cbc0f826 100644 --- a/src/agent_comms/coordination_tables/participants.py +++ b/src/agent_comms/coordination_tables/participants.py @@ -10,6 +10,7 @@ from agent_comms.nominal.field_codec import ThreadLookup from agent_comms.participant_identity import ParticipantIdentity from agent_comms.typed_table import ( + AttachedPracticeRebuild, Column, ColumnGuard, CounterAdvance, @@ -51,6 +52,14 @@ class Participants(MonotoneColumns, GuardedColumns, NoDelete, CoordinatorRow): ] committed: bool + @classmethod + def retain_required_rows(cls, original: AttachedPracticeRebuild) -> int: + from agent_comms.participant_store import ParticipantStore + + return super().retain_required_rows(original) + ParticipantStore.retain_owners( + original, tuple(person.participant_lookup for person in cls.select(original.destination)), + ) + @dataclass(frozen=True, kw_only=True) class OwnerGenerations( CounterAdvance, NoDelete, ParticipantIdentity, CoordinatorRow diff --git a/src/agent_comms/goal_attempts.py b/src/agent_comms/goal_attempts.py index 18f867a032..2491714837 100644 --- a/src/agent_comms/goal_attempts.py +++ b/src/agent_comms/goal_attempts.py @@ -76,6 +76,7 @@ WorkDecision, ) from .typed_table import ( + LOCAL_TABLE_SOURCE, Column, Generated, ReconciledFile, @@ -83,6 +84,7 @@ SQLiteJournalMode, StoreWriter, TableReference, + TableSource, TypedTable, ) @@ -212,9 +214,10 @@ def __post_init__(self) -> None: raise ValueError("Generation must be positive.") @classmethod - def _keyed(cls, db: sqlite3.Connection, key: dict[str, object]) -> list[Self]: + def _keyed(cls, db: sqlite3.Connection, key: dict[str, object], *, + table_source: TableSource = LOCAL_TABLE_SOURCE) -> list[Self]: try: - return super()._keyed(db, key) + return super()._keyed(db, key, table_source=table_source) except (sqlite3.Error, MalformedRecord) as error: raise StorageUncertainError("Invalid persisted goal generation.") from error diff --git a/src/agent_comms/participant_store.py b/src/agent_comms/participant_store.py index 8fc5243a74..5b22f1b456 100644 --- a/src/agent_comms/participant_store.py +++ b/src/agent_comms/participant_store.py @@ -2,6 +2,7 @@ from __future__ import annotations +import sqlite3 from dataclasses import dataclass from agent_comms.coordination_errors import ( @@ -21,6 +22,9 @@ Participants, ) from agent_comms.nominal.field_codec import ThreadLookup +from agent_comms.typed_table import ( + LOCAL_TABLE_SOURCE, AttachedPracticeRebuild, TableSource, +) @dataclass(frozen=True, slots=True) @@ -37,6 +41,11 @@ class ParticipantSnapshot: owner_identity: OwnerGenerations pointer: CurrentExecutions + def retain_original(self, original: AttachedPracticeRebuild) -> int: + """The generation and pointer are both required parts of this recorded answer.""" + return (self.owner_identity.retain_original(original) + + self.pointer.retain_original(original)) + class ParticipantStore: def __init__(self, session: CoordinationSession) -> None: @@ -44,17 +53,40 @@ def __init__(self, session: CoordinationSession) -> None: def get(self, lookup: ThreadLookup) -> ParticipantSnapshot: with self.session.read(): - person = Participants.one(self.session._connection, participant_lookup=lookup) - generation = OwnerGenerations.one(self.session._connection, owner_lookup=lookup) - pointer = CurrentExecutions.one(self.session._connection, owner_lookup=lookup) - if person is None or generation is None or pointer is None: - raise ParticipantNotRegistered(f"participant {lookup!r} aggregate is not registered") - return ParticipantSnapshot( - lookup, - person.committed, - generation, - pointer, - ) + return self.read_at(self.session._connection, lookup) + + @staticmethod + def read_at( + db: sqlite3.Connection, lookup: ThreadLookup, *, + table_source: TableSource = LOCAL_TABLE_SOURCE, + ) -> ParticipantSnapshot: + """Read the same aggregate through the caller's already-owned SQL lifetime.""" + person = Participants.one(db, participant_lookup=lookup, table_source=table_source) + generation = OwnerGenerations.one(db, owner_lookup=lookup, table_source=table_source) + pointer = CurrentExecutions.one(db, owner_lookup=lookup, table_source=table_source) + if person is None or generation is None or pointer is None: + raise ParticipantNotRegistered(f"participant {lookup!r} aggregate is not registered") + return ParticipantSnapshot(lookup, person.committed, generation, pointer) + + @classmethod + def retain_owners( + cls, original: AttachedPracticeRebuild, lookups: tuple[ThreadLookup, ...], + ) -> int: + """Keep selected original aggregates, not every participant's execution history. + + An absent participant remains absent. A present but incomplete aggregate + follows the ordinary reader's refusal before any file can be published. + """ + changed = 0 + for person in Participants.select_keys( + original.destination, tuple({"participant_lookup": lookup} for lookup in lookups), + table_source=original.source, + ): + snapshot = cls.read_at(original.destination, person.participant_lookup, + table_source=original.source) + changed += person.retain_original(original) + changed += snapshot.retain_original(original) + return changed def register( self, diff --git a/src/agent_comms/typed_table.py b/src/agent_comms/typed_table.py index b114d84dc8..d9c11e6611 100644 --- a/src/agent_comms/typed_table.py +++ b/src/agent_comms/typed_table.py @@ -1150,9 +1150,7 @@ def _retain_selection( self, row: type[TypedTable], source: str, *, where: str = "1", parameters: tuple[object, ...] = (), prefix: str = "", ) -> int: - keys = tuple(item.name for item in row._columns if item.column.primary_key) - if not keys: - raise RelationViolationError(f"{row.declared_name} has no declared retention key") + keys = tuple(item.name for item in row.retention_key_fields) fields = tuple(item.name for item in row._writable_fields()) retained_alias = _identifier(row.declared_name + "_retained") retained = " AND ".join( @@ -2772,23 +2770,43 @@ def decode_at(cls, row: tuple[object, ...], offset: int) -> Self: return cls._decode_row(row, {name: offset + index for index, name in enumerate(cls.column_projection())}) + @ClassFact + @staticmethod + def retention_key_fields(cls: type[TypedTable]) -> tuple[_Field, ...]: + """The declaration supplies both raw-copy deduplication and held-row selection.""" + keys = tuple(item for item in cls._columns if item.column.primary_key) + if not keys: + raise RelationViolationError(f"{cls.declared_name} has no declared retention key") + return keys + + def retain_original(self, original: AttachedPracticeRebuild) -> int: + """Keep this original row's raw SQL through its declared key, not reinsertion.""" + keys = self.retention_key_fields + plans = {plan.name: plan for plan in FieldCodec.field_plans(type(self))} + return original.retain_keys(type(self), ({ + item.name: plans[item.name].current(self) for item in keys + },)) + @classmethod - def one(cls, db: sqlite3.Connection, **key: object) -> Self | None: - rows = cls._keyed(db, key) + def one(cls, db: sqlite3.Connection, *, + table_source: TableSource = LOCAL_TABLE_SOURCE, **key: object) -> Self | None: + rows = cls._keyed(db, key, table_source=table_source) return next(iter(rows), None) @classmethod - def require(cls, db: sqlite3.Connection, **key: object) -> Self: + def require(cls, db: sqlite3.Connection, *, + table_source: TableSource = LOCAL_TABLE_SOURCE, **key: object) -> Self: """The one row a reference names; a missing row is corruption.""" - rows = cls._keyed(db, key) + rows = cls._keyed(db, key, table_source=table_source) if not rows: raise RowNotFoundError(f"Expected one {cls.declared_name} for {tuple(key)}") return rows[0] @classmethod - def _keyed(cls, db: sqlite3.Connection, key: dict[str, object]) -> list[Self]: + def _keyed(cls, db: sqlite3.Connection, key: dict[str, object], *, + table_source: TableSource = LOCAL_TABLE_SOURCE) -> list[Self]: """At most one row is stored for a key; a repeated key is corruption.""" - stored = cls.select(db, limit=2, **key) + stored = cls.select(db, limit=2, table_source=table_source, **key) if len(stored) > 1: raise MalformedRecord(f"Expected one {cls.declared_name} for {tuple(key)}") return stored diff --git a/tests/test_retained_coordinator_groups.py b/tests/test_retained_coordinator_groups.py index da032d044c..673aee2bc5 100644 --- a/tests/test_retained_coordinator_groups.py +++ b/tests/test_retained_coordinator_groups.py @@ -7,20 +7,26 @@ import pytest from agent_comms.assignment_states import FullPendingAssignment, TriagePendingAssignment +from agent_comms.attempt_start import AttemptStart from agent_comms.comms import wire from agent_comms.cli_bridges.claude import ClaudeBridge from agent_comms.cli_bridges.codex import CodexBridge from agent_comms.coordination_database import CoordinationStore -from agent_comms.coordination_errors import IdentityConflict +from agent_comms.coordination_errors import IdentityConflict, ParticipantNotRegistered from agent_comms.coordination_tables.assignments import WakeAssignment from agent_comms.coordination_tables.harness_deliveries import HarnessDelivery, HarnessObservedPart -from agent_comms.coordination_tables.executions import AcpOrigin, ExecutionRecord +from agent_comms.coordination_tables.executions import ( + AcpOrigin, CurrentExecutions, ExecutionRecord, NoPointedAttempt, NoPointedExecution, +) +from agent_comms.coordination_tables.participants import OwnerGenerations, Participants from agent_comms.coordinator import StandaloneCoordination from agent_comms.harness_delivery_states import EmittedOutcome, ReceivingMonitor from agent_comms.message_reference import MessageReference from agent_comms.native_input_record import FullNativeExecution, TriageNativeExecution from agent_comms.native_runtime_input import NativeRuntimeInput -from agent_comms.nominal.field_codec import AssignmentId, MessageId, TranscriptRecordId +from agent_comms.nominal.field_codec import AssignmentId, MessageId, ThreadLookup, TranscriptRecordId +from agent_comms.owner_fence import prepare_fence_token +from agent_comms.participant_store import ParticipantStore from agent_comms.triage_native_sources import TriageNativeSource from agent_comms.typed_table import TypedTable from agent_comms.wake_policy import FullWake, MessageAudience @@ -214,6 +220,66 @@ def test_retained_native_group_uses_original_membership_and_children( for ordinal, claim in enumerate(expected_claims) ) assert tuple(row.execution_id for row in ExecutionRecord.select(db)) == execution_ids + assert ParticipantStore.read_at(db, RECIPIENT_LOOKUP) == source.store.participants.get(RECIPIENT_LOOKUP) + assert CoordinationStore.digest(source.path) == before + + +def test_selected_participant_keeps_its_generation_and_current_pointer( + native_group_source: NativeGroupSource, tmp_path: Path, +) -> None: + source = native_group_source + source.store.participants.advance_generation(RECIPIENT_LOOKUP, expected_generation=1) + unrelated = ThreadLookup('9' * 32) + absent = ThreadLookup('8' * 32) + source.store.participants.register(unrelated, committed=True) + source.store.create('selected-current', AcpOrigin, RECIPIENT_LOOKUP, 1) + source.store.start(AttemptStart( + 'selected-current', 1, RECIPIENT_LOOKUP, 2, prepare_fence_token(), + expected_execution_revision=1, expected_pointer_revision=0, + )) + expected = source.store.participants.get(RECIPIENT_LOOKUP) + before = CoordinationStore.digest(source.path) + destination = tmp_path / 'participant-retained' / 'coordination.sqlite3' + with CoordinationStore.retaining_rows(source.path, destination, maximum_bytes=8 * 2**20) as original: + with pytest.raises(ParticipantNotRegistered): + ParticipantStore.read_at(original.destination, RECIPIENT_LOOKUP) + assert ParticipantStore.read_at( + original.destination, RECIPIENT_LOOKUP, table_source=original.source, + ) == expected + ParticipantStore.retain_owners(original, (RECIPIENT_LOOKUP, absent)) + with closing(CoordinationStore.read_snapshot(destination)) as db: + assert ParticipantStore.read_at(db, RECIPIENT_LOOKUP) == expected + assert tuple(Participants.select(db)) == (Participants( + participant_lookup=RECIPIENT_LOOKUP, committed=True, + ),) + assert tuple(row.execution_id for row in ExecutionRecord.select(db)) == ('selected-current',) + assert not NativeRuntimeInput.select(db) + assert not WakeAssignment.select(db) + assert expected.owner_identity.generation == 2 + assert expected.pointer.pointer_revision == 1 + assert CoordinationStore.digest(source.path) == before + + +@pytest.mark.parametrize('surviving', ( + OwnerGenerations(owner_lookup=ThreadLookup('7' * 32), generation=3), + CurrentExecutions(owner_lookup=ThreadLookup('7' * 32), + execution_id=NoPointedExecution(), attempt_ordinal=NoPointedAttempt(), + pointer_revision=5), +)) +def test_incomplete_original_participant_refuses_selected_publication( + native_group_source: NativeGroupSource, tmp_path: Path, surviving: TypedTable, +) -> None: + source = native_group_source + lookup = ThreadLookup('7' * 32) + with source.store.session.writing() as db: + Participants(participant_lookup=lookup, committed=True).insert(db) + surviving.insert(db) + before = CoordinationStore.digest(source.path) + destination = tmp_path / 'incomplete-participant' / 'coordination.sqlite3' + with pytest.raises(ParticipantNotRegistered): + with CoordinationStore.retaining_rows(source.path, destination, maximum_bytes=8 * 2**20) as original: + ParticipantStore.retain_owners(original, (lookup,)) + assert not destination.exists() assert CoordinationStore.digest(source.path) == before From c202df8a03b92971d5dc2c943146da439789b660 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sun, 1 Nov 2026 18:29:55 -0500 Subject: [PATCH 15/28] Preserve SQLite allocation metadata through the attached file copy --- docs/retained-practice-inputs.md | 42 ++++++++++++- .../coordination_tables/participants.py | 7 ++- src/agent_comms/typed_table.py | 59 ++++++++++--------- tests/test_retained_coordinator_groups.py | 58 +++++++++++++++++- 4 files changed, 132 insertions(+), 34 deletions(-) diff --git a/docs/retained-practice-inputs.md b/docs/retained-practice-inputs.md index 1e179f7466..9a30338c95 100644 --- a/docs/retained-practice-inputs.md +++ b/docs/retained-practice-inputs.md @@ -40,8 +40,9 @@ entry references and earlier request IDs. Missing originals and foreign-thread predecessors refuse. Whole retained predecessor records receive their own closure, not an incomplete predecessor that only one selected segment can read. The original ContextStoreInfo is required when requests are selected; entry-only selection does -not invent one. `RowidSequenced` retains SQLite's original sequence high-water mark -so an omitted original ID cannot become the next newly allocated ID. +not invent one. The attached file-copy owner retains SQLite's original sequence +high-water marks, so an omitted original ID cannot become the next newly allocated ID. +The allocation metadata is not restricted to RowidSequenced's insertion capability. The earlier entry-only API's owner-path run retained 6,274 entries in 13,877,248 materialized bytes with matching source-before/retained/source-after row digests. @@ -242,7 +243,8 @@ They use the existing native-root/message/claim producers, compare the ordinary source and retained projections and original attempt rows, and check the source coordinator bytes and bus revision. Together with the eighteen previous cases, that checkpoint's **twenty-one added scenarios remained unrun**. The participant -checkpoint below adds three more, for **twenty-four added scenarios UNRUN**. +checkpoint below adds three more; the subsequent sequence controls add two, +for **twenty-six added scenarios UNRUN**. These do not cover every disposition, native association or rollback/resource failure. Runtime class creation/MRO/registration, attached SQL, cross-store capture and @@ -284,6 +286,40 @@ seed cases also require the retained aggregate. Source mapping covers 65 modules 18 function-local test declarations remain unprojected OPEN. That is source coverage, not executed class/MRO/SQL behavior or complete closure. +## Hook correction and SQLite allocation metadata — source only + +Source follow-through caught an error in the participant checkpoint: its new +`Participants.retain_required_rows` omitted the `schema` argument that the base +and actual `TableSchema.retain_referenced_rows` caller supply. It now accepts and +passes that argument cooperatively. The whole declared hook family is checked at +source; this is not runtime dispatch/MRO qualification. The active-pointer case +also now calls the real `mark_pending` transition and expects revision 2 before +`start`; QueuedExecution does not admit direct start. No product fallback was added +to satisfy a faulty fixture. + +A separate preservation gap remained in the earlier rowid hook: `RecoveryAudit`, +`SpanAnnotationsRow` and `AnnotationRequestsRow` declare AUTOINCREMENT columns but +do not inherit `RowidSequenced`. Their SQLite high-water metadata could disappear. +Two sources with identical selected lower rows but different original high-water +marks require different next IDs; the selected rows alone cannot recover that. + +`AttachedPracticeRebuild.retain_sequences` now copies SQLite's own allocation +metadata once after dependency closure, inside the existing load transaction and +before guards/publication. The actual source/destination catalogs supply sequence +presence and table membership; there is no row-family/table/field roster. Existing +larger destination values are not lowered. Empty selection still preserves the +original allocation metadata. The competing per-RowidSequenced hook is deleted; +ordinary row insertion and complete copy/backup behavior are unchanged. + +Two additional cases are authored **UNRUN** using ordinary coordinator execution/ +Pending/start producers and typed audit-row insertion: retaining a lower audit, and +retaining no rows at all. Both ask the normal destination writer to allocate a +fresh audit ID after the omitted original high-water mark, and check source bytes. +These are SQL allocation controls, not a real model-stall/native or recovery proof. +The existing context-request/version allocation case remains part of the necessary +combined selection. **Twenty-six added scenarios remain unrun.** SQLite execution, +rollback, metadata/schema/guard behavior and peak resource bounds are unqualified. + ## Copy and backup are different contracts `PracticeCopy.include` currently invokes each `RootFile` owner's whole-file diff --git a/src/agent_comms/coordination_tables/participants.py b/src/agent_comms/coordination_tables/participants.py index 83cbc0f826..4f9346e0c0 100644 --- a/src/agent_comms/coordination_tables/participants.py +++ b/src/agent_comms/coordination_tables/participants.py @@ -20,6 +20,7 @@ MonotoneColumns, NoDelete, NonDecreasing, + TableSchema, ) @@ -53,10 +54,12 @@ class Participants(MonotoneColumns, GuardedColumns, NoDelete, CoordinatorRow): committed: bool @classmethod - def retain_required_rows(cls, original: AttachedPracticeRebuild) -> int: + def retain_required_rows( + cls, original: AttachedPracticeRebuild, schema: type[TableSchema], + ) -> int: from agent_comms.participant_store import ParticipantStore - return super().retain_required_rows(original) + ParticipantStore.retain_owners( + return super().retain_required_rows(original, schema) + ParticipantStore.retain_owners( original, tuple(person.participant_lookup for person in cls.select(original.destination)), ) diff --git a/src/agent_comms/typed_table.py b/src/agent_comms/typed_table.py index d9c11e6611..c454006456 100644 --- a/src/agent_comms/typed_table.py +++ b/src/agent_comms/typed_table.py @@ -1135,6 +1135,37 @@ def retain_keys( self.destination, keys, table_source=self.source, )) + def retain_sequences(self) -> None: + """Copy SQLite's original high-water marks once, including empty selections. + + Allocation belongs to SQLite metadata, not membership in a row-insertion + capability. Only tables present in the selected file's schema participate. + """ + if not SQLiteSchemaObject.read(self.destination.execute( + f"SELECT original.name,original.sql FROM {self.source.qualified('sqlite_master')} original " + "JOIN main.sqlite_master retained ON retained.name=original.name " + "WHERE original.name='sqlite_sequence' " + "AND original.type='table' AND retained.type='table'", + )): + return + source = self.source.qualified("sqlite_sequence") + self.destination.execute( + "INSERT INTO main.sqlite_sequence(name,seq) " + f"SELECT original_sequence.name,original_sequence.seq FROM {source} original_sequence " + "JOIN main.sqlite_master retained_table ON retained_table.name=original_sequence.name " + "AND retained_table.type='table' " + "WHERE NOT EXISTS(SELECT 1 FROM main.sqlite_sequence retained_sequence " + "WHERE retained_sequence.name=original_sequence.name)", + ) + self.destination.execute( + "UPDATE main.sqlite_sequence AS retained_sequence " + f"SET seq=(SELECT seq FROM {source} original_sequence " + "WHERE original_sequence.name=retained_sequence.name) " + f"WHERE EXISTS(SELECT 1 FROM {source} original_sequence " + "WHERE original_sequence.name=retained_sequence.name " + "AND retained_sequence.seq None: """An unknown source column or trigger is not permission to silently discard it.""" stored = SQLiteSchemaObject.read(self.destination.execute( @@ -3187,6 +3218,7 @@ def retaining_rows( yield original while schema.retain_referenced_rows(original): pass + original.retain_sequences() retained_bytes = sum(member.stat().st_size for member in cls.files(staged) if member.is_file()) if retained_bytes > maximum_bytes: @@ -3539,33 +3571,6 @@ def retain_required_rows( class RowidSequenced(TypedTable, declared_name=None): """The table's rows are numbered by SQLite's rowid, and an insert answers its number.""" - @classmethod - def retain_required_rows( - cls, original: AttachedPracticeRebuild, schema: type[TableSchema], - ) -> int: - """AUTOINCREMENT's original high-water mark survives a selected row set.""" - changed = super().retain_required_rows(original, schema) - # sqlite_sequence is SQLite-owned external metadata, absent when the - # source has no AUTOINCREMENT tables. Its presence is decoded here once. - if not SQLiteSchemaObject.read(original.destination.execute( - f"SELECT name,sql FROM {original.source.qualified('sqlite_master')} " - "WHERE name='sqlite_sequence'", - )): - return changed - source = original.source.qualified("sqlite_sequence") - changed += original.destination.execute( - "INSERT INTO main.sqlite_sequence(name,seq) " - f"SELECT name,seq FROM {source} AS original_sequence WHERE name=? " - "AND NOT EXISTS(SELECT 1 FROM main.sqlite_sequence AS retained_sequence " - "WHERE retained_sequence.name=original_sequence.name)", (cls.declared_name,), - ).rowcount - changed += original.destination.execute( - f"UPDATE main.sqlite_sequence SET seq=(SELECT seq FROM {source} WHERE name=?) " - f"WHERE name=? AND seq<(SELECT seq FROM {source} WHERE name=?)", - (cls.declared_name, cls.declared_name, cls.declared_name), - ).rowcount - return changed - @classmethod def insert_sequence(cls, db: sqlite3.Connection, **values: object) -> int: """Insert one row and answer the rowid SQLite assigned it.""" diff --git a/tests/test_retained_coordinator_groups.py b/tests/test_retained_coordinator_groups.py index 673aee2bc5..a82f97c60d 100644 --- a/tests/test_retained_coordinator_groups.py +++ b/tests/test_retained_coordinator_groups.py @@ -19,6 +19,7 @@ AcpOrigin, CurrentExecutions, ExecutionRecord, NoPointedAttempt, NoPointedExecution, ) from agent_comms.coordination_tables.participants import OwnerGenerations, Participants +from agent_comms.coordination_tables.recovery import RecoveryAudit from agent_comms.coordinator import StandaloneCoordination from agent_comms.harness_delivery_states import EmittedOutcome, ReceivingMonitor from agent_comms.message_reference import MessageReference @@ -27,8 +28,10 @@ from agent_comms.nominal.field_codec import AssignmentId, MessageId, ThreadLookup, TranscriptRecordId from agent_comms.owner_fence import prepare_fence_token from agent_comms.participant_store import ParticipantStore +from agent_comms.reason_codes import GivenReasonCode, NoDetail +from agent_comms.recovery_states import ModelStalledRecovery from agent_comms.triage_native_sources import TriageNativeSource -from agent_comms.typed_table import TypedTable +from agent_comms.typed_table import TypedTable, UnallocatedRowId from agent_comms.wake_policy import FullWake, MessageAudience from bus_record_fixture import ( FIRST_INPUT_ID, SECOND_INPUT_ID, THIRD_INPUT_ID, RECIPIENT_LOOKUP, prepare_native_comms, @@ -233,9 +236,10 @@ def test_selected_participant_keeps_its_generation_and_current_pointer( absent = ThreadLookup('8' * 32) source.store.participants.register(unrelated, committed=True) source.store.create('selected-current', AcpOrigin, RECIPIENT_LOOKUP, 1) + source.store.mark_pending('selected-current', expected_revision=1) source.store.start(AttemptStart( 'selected-current', 1, RECIPIENT_LOOKUP, 2, prepare_fence_token(), - expected_execution_revision=1, expected_pointer_revision=0, + expected_execution_revision=2, expected_pointer_revision=0, )) expected = source.store.participants.get(RECIPIENT_LOOKUP) before = CoordinationStore.digest(source.path) @@ -283,6 +287,56 @@ def test_incomplete_original_participant_refuses_selected_publication( assert CoordinationStore.digest(source.path) == before +@pytest.mark.parametrize('keys,selected_count', ( + (({'execution_id': 'sequence-selected'},), 1), + ((), 0), +), ids=('lower-audit', 'empty-selection')) +def test_selected_store_keeps_sqlite_high_water_without_rowid_mixin( + harness_source: HarnessSource, tmp_path: Path, + keys: tuple[dict[str, object], ...], selected_count: int, +) -> None: + source = harness_source + other = ThreadLookup('6' * 32) + source.store.participants.register(other, committed=True) + for execution, owner in (('sequence-selected', RECIPIENT_LOOKUP), ('sequence-omitted', other)): + source.store.create(execution, AcpOrigin, owner, 1) + source.store.mark_pending(execution, expected_revision=1) + source.store.start(AttemptStart( + execution, 1, owner, 1, prepare_fence_token(), + expected_execution_revision=2, expected_pointer_revision=0, + )) + # Actual stored audit rows exercise allocation, not model-stall or native proof. + with source.store.session.writing() as db: + cursor = RecoveryAudit( + execution_id=execution, kind=ModelStalledRecovery, + reason_code=GivenReasonCode('sequence_control'), sanitized_detail=NoDetail(), + attempt=1, elapsed_ms=0, observed_at_ms=1000, + ).insert(db) + source_high_water = cursor.lastrowid + assert source_high_water is not None + with source.store.session.read(): + audits = tuple(RecoveryAudit.select(source.store.session._connection, order_by=('audit_id',))) + before = CoordinationStore.digest(source.path) + destination = tmp_path / 'sequence-retained' / 'coordination.sqlite3' + with CoordinationStore.retaining_rows(source.path, destination, maximum_bytes=8 * 2**20) as original: + original.retain_keys(ExecutionRecord, keys) + with StandaloneCoordination(str(destination), clock_ms=lambda: 1000) as retained: + with retained.session.read(): + assert tuple(RecoveryAudit.select(retained.session._connection, order_by=('audit_id',))) == audits[:selected_count] + fresh_owner = ThreadLookup('5' * 32) + retained.participants.register(fresh_owner, committed=True) + retained.create('sequence-new', AcpOrigin, fresh_owner, 1) + retained.mark_pending('sequence-new', expected_revision=1) + retained.start(AttemptStart( + 'sequence-new', 1, fresh_owner, 1, prepare_fence_token(), + expected_execution_revision=2, expected_pointer_revision=0, + )) + with retained.session.writing() as db: + fresh = replace(audits[0], execution_id='sequence-new', audit_id=UnallocatedRowId()) + assert fresh.insert(db).lastrowid == source_high_water + 1 + assert CoordinationStore.digest(source.path) == before + + def test_native_membership_query_uses_the_supplied_attachment( native_group_source: NativeGroupSource, tmp_path: Path, ) -> None: From 9ffa104361501826ab4e51e29696bf23c0f7b9ea Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sun, 1 Nov 2026 19:22:19 -0500 Subject: [PATCH 16/28] Retain original annotations with selected context inputs --- src/agent_comms/context_store.py | 14 ++ src/agent_comms/coordination_database.py | 4 + .../coordination_tables/annotations.py | 85 +++++++- src/agent_comms/installer/practice.py | 9 + src/agent_comms/typed_table.py | 17 +- src/agent_comms/working_memory_annotations.py | 13 +- tests/test_retained_annotations.py | 200 ++++++++++++++++++ tests/test_working_memory_read.py | 10 +- tests/working_memory_fixture.py | 17 ++ 9 files changed, 336 insertions(+), 33 deletions(-) create mode 100644 tests/test_retained_annotations.py create mode 100644 tests/working_memory_fixture.py diff --git a/src/agent_comms/context_store.py b/src/agent_comms/context_store.py index 47b14baab5..6af54cd98e 100644 --- a/src/agent_comms/context_store.py +++ b/src/agent_comms/context_store.py @@ -516,6 +516,20 @@ def entries( session_file=identity.session_file, entry_id=entry_id)} + def original_segments(self) -> Iterator[SegmentManifest]: + """Original segments actually present in this selected entry/request store. + + Requests restore through the same original predecessor/entry reader as + the browser. Entry-only stores need no invented request/store identity. + """ + with self.snapshot() as db: + for entry in ContextEntry.stream(db): + yield from entry.manifest.original_values() + for request in ContextRequest.stream(db): + source = ContextStoreInfo.require(db, singleton=1).source + for root in self.resolve(db, source, request).value.segments: + yield from root.original_values() + def retain_contexts( self, destination: ContextStore, identities: Iterable[NativeSessionIdentity], *, maximum_bytes: int, requests: Iterable[ContextManifestReference] = (), diff --git a/src/agent_comms/coordination_database.py b/src/agent_comms/coordination_database.py index ece03d2ff3..ba5026dcff 100644 --- a/src/agent_comms/coordination_database.py +++ b/src/agent_comms/coordination_database.py @@ -31,6 +31,7 @@ from .input_disposition import InputDocument from .native_session_reopen import NativeSessionIdentity from .registry_document import RegistryEntry + from .turn_context import SegmentManifest class StoreFile(DeclaredFamily): @@ -91,6 +92,7 @@ def schema_at(cls, path: Path) -> type[TableSchema]: def retain_inputs( cls, source: Path, destination: Path, entries: tuple[RegistryEntry, ...], *, maximum_bytes: int, identities: Iterable[NativeSessionIdentity] = (), + segments: Iterable[SegmentManifest] = (), ) -> InputDocument: """Materialize original input witnesses under the existing fresh-file owner. @@ -102,6 +104,7 @@ def retain_inputs( from .input_disposition import SnapshotInputDispositions from .native_runtime_input import NativeRuntimeInput from .participant_store import ParticipantStore + from .coordination_tables.annotations import ContextSpanRows with cls.retaining_rows(source, destination, maximum_bytes=maximum_bytes) as original: selected = SnapshotInputDispositions(destination.parent, original).retain_unresolved(entries) @@ -109,6 +112,7 @@ def retain_inputs( entry.thread.incarnation.require_recorded().lookup for entry in entries )) NativeRuntimeInput.retain_sessions(original, identities) + ContextSpanRows.retain_segments(original, segments) return selected @classmethod diff --git a/src/agent_comms/coordination_tables/annotations.py b/src/agent_comms/coordination_tables/annotations.py index a582ecf341..e05b09f3b7 100644 --- a/src/agent_comms/coordination_tables/annotations.py +++ b/src/agent_comms/coordination_tables/annotations.py @@ -1,16 +1,25 @@ """Durable label/request declarations; the existing coordinator owns storage.""" from __future__ import annotations +import sqlite3 +from abc import abstractmethod +from collections.abc import Iterable, Iterator from dataclasses import dataclass, field -from typing import Annotated, ClassVar +from typing import Annotated, ClassVar, Self from agent_comms.coordination_schema import CoordinatorRow from agent_comms.message_reference import MessageReference +from agent_comms.turn_context import ContextSpan, SegmentManifest from agent_comms.typed_table import ( + LOCAL_TABLE_SOURCE, + AttachedPracticeRebuild, Column, Generated, Index, RowId, + TableSchema, + TableSource, + TypedTable, UnallocatedRowId, sql_literal, ) @@ -19,8 +28,45 @@ from agent_comms.working_memory_requests import AnnotationOutcome, DisclosureRequest +class ContextSpanRows(TypedTable, declared_name=None): + """Stored answers and request dispositions share original-span eligibility.""" + + __slots__ = () + + @property + @abstractmethod + def context_span(self) -> ContextSpan: ... + + @classmethod + def for_segments( + cls, db: sqlite3.Connection, segments: Iterable[SegmentManifest], *, + table_source: TableSource = LOCAL_TABLE_SOURCE, **key: object, + ) -> Iterator[Self]: + originals: dict[str, list[SegmentManifest]] = {} + for segment in segments: + originals.setdefault(segment.sha256, []).append(segment) + for row in cls.stream_keys(db, tuple( + dict(key, segment_digest=digest) for digest in originals + ), table_source=table_source): + span = row.context_span + if any(segment.contains_span(span) for segment in originals[span.segment_sha256]): + yield row + + @classmethod + def retain_segments( + cls, original: AttachedPracticeRebuild, segments: Iterable[SegmentManifest], + ) -> int: + # Both declared row members consume the same original set; their raw + # values and request outcomes survive, not merely an effective answer. + selected = tuple(segments) + return sum(row.retain_original(original) + for member in cls.members_with(cls) + for row in member.for_segments(original.destination, selected, + table_source=original.source)) + + @dataclass(frozen=True, kw_only=True) -class SpanAnnotationsRow(CoordinatorRow): +class SpanAnnotationsRow(ContextSpanRows, CoordinatorRow): label: ModelLabel created_at_ms: int id: Annotated[RowId, Column(primary_key=True, auto_increment=True)] = field( @@ -82,18 +128,13 @@ def effective(cls, rows: tuple[SpanAnnotationsRow, ...]): latest[type(row.label)] = row.label return SpanLabel.effective(tuple(latest.values())) - @classmethod - def for_digests(cls, db, digests, classifier): - if not digests: - return () - return cls.select(db, - where='segment_digest IN (' + ','.join('?' for _ in digests) + ')', - parameters=tuple(digests), order_by=("id",), - classifier=classifier.classifier, classifier_pin=classifier.pin) + @property + def context_span(self) -> ContextSpan: + return self.label.span @dataclass(frozen=True, kw_only=True) -class AnnotationRequestsRow(CoordinatorRow): +class AnnotationRequestsRow(ContextSpanRows, CoordinatorRow): request: DisclosureRequest outcome: AnnotationOutcome created_at_ms: int @@ -137,6 +178,28 @@ class AnnotationRequestsRow(CoordinatorRow): unique = (SpanAnnotationsRow.address,) indexes = (Index(("grant_reference", "created_at_ms")),) + @property + def context_span(self) -> ContextSpan: + return self.request.span + + @classmethod + def retain_required_rows( + cls, original: AttachedPracticeRebuild, schema: type[TableSchema], + ) -> int: + """Keep original grant usage and each request's complete answer/correction group.""" + from agent_comms.working_memory_annotations import WorkingMemoryAnnotations + + changes = super().retain_required_rows(original, schema) + changes += original.retain_keys(cls, tuple( + {"grant_reference": row.request.grant} for row in cls.stream(original.destination) + )) + changes += original.retain_keys(SpanAnnotationsRow, tuple( + WorkingMemoryAnnotations.address(row.request.span, row.request.question, + row.request.classifier) + for row in cls.stream(original.destination) + )) + return changes + @classmethod def requests_since(cls, db, reference: MessageReference, since: int) -> int: return cls.count(db, where='created_at_ms>?', parameters=(since,), grant_reference=reference) diff --git a/src/agent_comms/installer/practice.py b/src/agent_comms/installer/practice.py index a7f07aee71..135692839a 100644 --- a/src/agent_comms/installer/practice.py +++ b/src/agent_comms/installer/practice.py @@ -187,7 +187,11 @@ def include_contexts( Normal include() and the installer's live-root backup remain whole-file operations. This selection must precede including the context resource; an existing destination is refused rather than overwritten or reimported. + Select contexts before finalizing coordinator evidence, whose annotation + dependency selection consumes these exact retained segments. """ + if CoordinationStore.path_in(self.root) in self.copied: + raise UpgradeRefused("Select contexts before finalizing coordinator input evidence") self.prepare() source = ContextStore(self.root) resource, = RootFile.owned_files(LiveFilesRequest(self.root), (source.path,)) @@ -215,11 +219,16 @@ def include_inputs( raise UpgradeRefused(f"Capture complete {source.name} before selecting input evidence") elif owner.path_in(self.path).exists(): raise UpgradeRefused(f"Copied {source.name} has no original reader source") + contexts = ContextStore(self.path) + source_contexts = ContextStore.path_in(self.root) + if contexts.path.exists() and source_contexts not in self.copied: + raise UpgradeRefused("Copied contexts have no captured original reader source") source = CoordinationStore.path_in(self.root) resource, = RootFile.owned_files(LiveFilesRequest(self.root), (source,)) selected = CoordinationStore.retain_inputs( source, CoordinationStore.path_in(self.path), entries, identities=identities, maximum_bytes=self.remaining_bytes(maximum_bytes), + segments=contexts.original_segments() if source_contexts in self.copied else (), ) self.copied[source] = resource return selected diff --git a/src/agent_comms/typed_table.py b/src/agent_comms/typed_table.py index c454006456..2514db2424 100644 --- a/src/agent_comms/typed_table.py +++ b/src/agent_comms/typed_table.py @@ -2662,11 +2662,18 @@ def select_keys(cls, db: sqlite3.Connection, """Rows equal to any of these named keys under the same null-safe IS equality select() uses: the requested keys drive an indexed join, one batch per the connection's own variable limit, each distinct key probed once.""" - return [row for requested in cls.requested_keys(db, keys, table_source=table_source) - for row in cls.iterate(db.execute( - f"WITH {requested.values} " - f"SELECT {cls.qualified(*cls.column_projection())} {requested.join}", - requested.bindings))] + return list(cls.stream_keys(db, keys, table_source=table_source)) + + @classmethod + def stream_keys(cls, db: sqlite3.Connection, + keys: Sequence[Mapping[str, object]], *, + table_source: TableSource = LOCAL_TABLE_SOURCE) -> Generator[Self]: + """Consume the same indexed batches without materializing every matched row.""" + for requested in cls.requested_keys(db, keys, table_source=table_source): + yield from cls.iterate(db.execute( + f"WITH {requested.values} " + f"SELECT {cls.qualified(*cls.column_projection())} {requested.join}", + requested.bindings)) @classmethod def requested_keys(cls, db: sqlite3.Connection, diff --git a/src/agent_comms/working_memory_annotations.py b/src/agent_comms/working_memory_annotations.py index e85dca11b4..61874545b8 100644 --- a/src/agent_comms/working_memory_annotations.py +++ b/src/agent_comms/working_memory_annotations.py @@ -86,18 +86,15 @@ def for_context( @classmethod def for_segments(cls, source: ReadableStore, segments, classifier) -> tuple[ModelLabel, ...]: """One acquisition serves context and selected-segment readers alike.""" - originals = {} - for segment in segments: - originals.setdefault(segment.sha256, []).append(segment) with cls.reading(source) as db: - rows = SpanAnnotationsRow.for_digests(db, tuple(originals), classifier) + rows = sorted(SpanAnnotationsRow.for_segments( + db, segments, classifier=classifier.classifier, classifier_pin=classifier.pin, + ), key=lambda row: row.id) grouped = {} for row in rows: label = row.label - if any(segment.contains_span(label.span) - for segment in originals[label.span.segment_sha256]): - key = tuple(cls.address(label.span, label.question, label.classifier).values()) - grouped.setdefault(key, []).append(row) + key = tuple(cls.address(label.span, label.question, label.classifier).values()) + grouped.setdefault(key, []).append(row) return tuple(SpanAnnotationsRow.effective(tuple(rows)) for rows in grouped.values()) def reserve(self, request: DisclosureRequest, grant) -> AnnotationRequestsRow: diff --git a/tests/test_retained_annotations.py b/tests/test_retained_annotations.py new file mode 100644 index 0000000000..867c80eb7e --- /dev/null +++ b/tests/test_retained_annotations.py @@ -0,0 +1,200 @@ +"""Selected original annotation data/refusals, not external disclosure/native proof.""" +from collections.abc import Iterator +from dataclasses import dataclass, replace +from pathlib import Path + +import pytest + +from agent_comms.bus import Bus +from agent_comms.comms import wire +from agent_comms.context_store import ( + UNREAD_CONTEXT, CapturedContext, ContextManifestReference, ContextStore, +) +from agent_comms.coordination_database import CoordinationStore, ReadableStore +from agent_comms.coordination_tables.annotations import AnnotationRequestsRow, SpanAnnotationsRow +from agent_comms.coordinator import StandaloneCoordination +from agent_comms.input_disposition import InputDispositions +from agent_comms.installer.refusals import UpgradeRefused +from agent_comms.installer.practice import PracticeCopy +from agent_comms.message_reference import MessageReference +from agent_comms.nominal.field_codec import MessageId, TurnId +from agent_comms.repo import Checkout +from agent_comms.task_sources import AnnotationDisclosureGrant, ProjectTaskScope +from agent_comms.thread_identity import RecordedThreadIncarnation, TurnIdentity +from agent_comms.turn_context import ( + ContextManifest, ContextSpan, ContributionCoordinates, ObservedDispatchRequest, + RecordedContextTurn, SystemLayerSegment, ToolCatalogSegment, UnattributedProvenance, +) +from agent_comms.turn_item import PiRequestKey +from agent_comms.working_memory_annotations import PreviouslyRequestedAnnotation, WorkingMemoryAnnotations +from agent_comms.working_memory_disclosure import DisclosureState +from agent_comms.working_memory_labels import ( + HumanLabel, JevChoice, JevClassifier, JevResponse, JevUsage, ModelLabel, QuestionVersion, +) +from agent_comms.working_memory_questions import KindQuestion, OtherSpan, SpanAnswer +from agent_comms.working_memory_requests import ( + AnnotationBudgetExhausted, DisclosureRequest, FailedAnnotationOutcome, +) +from working_memory_fixture import local_system + + +@dataclass(frozen=True) +class AnnotationSource: + store: StandaloneCoordination + context: CapturedContext + grant: AnnotationDisclosureGrant + selected_requests: tuple[DisclosureRequest, ...] + + @property + def path(self) -> Path: + return self.store.session.path + + @property + def reference(self) -> ContextManifestReference: + return ContextManifestReference(self.context.source, self.context.record_id) + + +def completed_label( + store: StandaloneCoordination, request: DisclosureRequest, + grant: AnnotationDisclosureGrant, identifier: str, +) -> ModelLabel: + original = store.annotations.reserve(request, grant) + answers = request.question.question.answer_family.members_with(SpanAnswer) + choice = JevChoice('choice', OtherSpan.declared_name, 0.5, + {answer.declared_name: 1 / len(answers) for answer in answers}) + response = JevResponse(identifier, request.classifier.pin, 'fixture', + {request.question.question.declared_name: choice}, JevUsage(0, 3, 1)) + label = choice.label(request.span, request.question, request.classifier, response) + store.annotations.complete(original, response, label) + return label + + +def disclosed_request( + segment: SystemLayerSegment, span: ContextSpan, grant: MessageReference, +) -> DisclosureRequest: + return DisclosureRequest.capture( + span, JevClassifier.version(), KindQuestion, + DisclosureState.capture(segment, span, KindQuestion), grant, + ) + + +@pytest.fixture +def annotation_source(tmp_path: Path) -> Iterator[AnnotationSource]: + root = tmp_path / 'annotation-source' + comms = wire(root) + # The ordinary bus owner supplies the original empty bus, not a reminted + # destination source. Nothing here sends a message or contacts a provider. + with comms.bus.bus.reading(), InputDispositions(root).snapshot(): + pass + owner = RecordedThreadIncarnation('Fixture', 17001.0) + author = RecordedThreadIncarnation('FixtureHuman', 17002.0) + reference = MessageReference(1, MessageId.mint('stored-annotation-grant')) + grant = AnnotationDisclosureGrant( + scope=ProjectTaskScope(project=Checkout(tmp_path)), subject=reference, + source_user=author, recipient=owner, per_hour=8, segments=(SystemLayerSegment,), + ) + texts = ('Still submitted.\n', 'Recorded failure.\n', 'Original answer.') + positions = [] + offset = 0 + for text in texts: + positions.append(ContributionCoordinates.capture( + SystemLayerSegment, (UnattributedProvenance(),), offset, text, + )) + offset += len(text.encode()) + segment = local_system(''.join(texts), tuple(positions)) + spans = segment.public_spans() + manifest = ContextManifest( + owner, RecordedContextTurn(TurnId('1' * 32), TurnIdentity(owner, 1)), + (segment.measured_manifest(),), 'fixture-counter', + ObservedDispatchRequest(PiRequestKey('fixture-annotation-context')), + ) + captured = ContextStore(root).record_after(UNREAD_CONTEXT, manifest) + with StandaloneCoordination(str(CoordinationStore.path_in(root)), clock_ms=lambda: 1000) as store: + requests = tuple(disclosed_request(segment, span, reference) for span in spans) + store.annotations.reserve(requests[0], grant) + failed = store.annotations.reserve(requests[1], grant) + store.annotations.fail(failed, FailedAnnotationOutcome('recorded external failure')) + model = completed_label(store, requests[2], grant, 'fixture-answer') + # Stored correction data tests precedence, not human/publication authority. + correction = HumanLabel.correct(model, OtherSpan, author) + old_request = replace(requests[2], question=QuestionVersion(KindQuestion, 'b' * 64)) + completed_label(store, old_request, grant, 'fixture-old-question') + other_version = replace(model.classifier, pin=model.classifier.pin + '-fixture-other') + other_reference = MessageReference(2, MessageId.mint('stored-other-classifier-grant')) + other_grant = replace(grant, subject=other_reference, classifier=other_version) + other_request = replace(requests[2], classifier=other_version, grant=other_reference) + completed_label(store, other_request, other_grant, 'fixture-other-classifier') + other_segment = local_system('Not in the selected context.') + other_span, = other_segment.public_spans() + other_usage = disclosed_request(other_segment, other_span, reference) + completed_label(store, other_usage, grant, 'fixture-same-grant-other-context') + unused_reference = MessageReference(3, MessageId.mint('stored-unused-grant')) + unused_grant = replace(grant, subject=unused_reference) + store.annotations.reserve(replace(other_usage, + question=QuestionVersion(KindQuestion, 'c' * 64), grant=unused_reference), unused_grant) + foreign_span = ContextSpan(segment.sha256, ContributionCoordinates.capture( + ToolCatalogSegment, (UnattributedProvenance(),), 1, 't', + )) + assert not segment.measured_manifest().contains_span(foreign_span) + with store.session.writing() as db: + SpanAnnotationsRow(label=correction, created_at_ms=1000).insert(db) + SpanAnnotationsRow(label=replace(model, span=foreign_span), created_at_ms=1000).insert(db) + yield AnnotationSource(store, captured, grant, (*requests, old_request, other_request)) + + +def test_selected_context_copy_preserves_answers_versions_outcomes_and_grant_usage( + annotation_source: AnnotationSource, tmp_path: Path, +) -> None: + source = annotation_source + before = CoordinationStore.digest(source.path) + contexts_before = ContextStore(source.path.parent).path.read_bytes() + expected = WorkingMemoryAnnotations.for_context( + ReadableStore(source.path), (source.context.value,), JevClassifier.version(), + ) + assert len(expected) == 2 + copy = PracticeCopy(source.path.parent, tmp_path / 'selected-annotations') + copy.include((Bus.path_in(source.path.parent), InputDispositions.path_in(source.path.parent))) + copy.include_contexts((), requests=(source.reference,), maximum_bytes=8 * 2**20) + copy.include_inputs((), maximum_bytes=8 * 2**20) + retained_path = CoordinationStore.path_in(copy.path) + assert WorkingMemoryAnnotations.for_context( + ReadableStore(retained_path), (source.context.value,), JevClassifier.version(), + ) == expected + with source.store.session.read(): + db = source.store.session._connection + originals = tuple(AnnotationRequestsRow.select(db, order_by=('id',))) + labels = tuple(SpanAnnotationsRow.select(db, order_by=('id',))) + usage = AnnotationRequestsRow.requests_since(db, source.grant.subject, 0) + with StandaloneCoordination(str(retained_path)) as retained: + with retained.session.read(): + db = retained.session._connection + rows = tuple(AnnotationRequestsRow.select(db, order_by=('id',))) + assert rows == tuple(row for row in originals + if row.request.grant in {request.grant for request in source.selected_requests}) + assert tuple(SpanAnnotationsRow.select(db, order_by=('id',))) == tuple( + row for row in labels if row.label.span.coordinates.kind is SystemLayerSegment) + assert AnnotationRequestsRow.requests_since(db, source.grant.subject, 0) == usage == 5 + for request in source.selected_requests[:3]: + with pytest.raises(PreviouslyRequestedAnnotation): + retained.annotations.reserve(request, source.grant) + # The exact grant-wide count, including the unselected context, decides + # the same budget refusal on source and retained data; no request is sent. + exhausted = replace(source.grant, per_hour=usage) + with pytest.raises(AnnotationBudgetExhausted): + exhausted.require_budget(usage) + assert CoordinationStore.digest(source.path) == before + assert ContextStore(source.path.parent).path.read_bytes() == contexts_before + + +def test_context_selection_cannot_follow_finalized_coordinator_evidence( + annotation_source: AnnotationSource, tmp_path: Path, +) -> None: + source = annotation_source + before = CoordinationStore.digest(source.path) + copy = PracticeCopy(source.path.parent, tmp_path / 'wrong-selection-order') + copy.include((Bus.path_in(source.path.parent), InputDispositions.path_in(source.path.parent))) + copy.include_inputs((), maximum_bytes=8 * 2**20) + with pytest.raises(UpgradeRefused): + copy.include_contexts((), requests=(source.reference,), maximum_bytes=8 * 2**20) + assert not ContextStore(copy.path).path.exists() + assert CoordinationStore.digest(source.path) == before diff --git a/tests/test_working_memory_read.py b/tests/test_working_memory_read.py index a6b78d3ad7..8462f1e5f2 100644 --- a/tests/test_working_memory_read.py +++ b/tests/test_working_memory_read.py @@ -1,6 +1,5 @@ """Read/correct original annotation addresses without reader-side installation.""" from dataclasses import replace -import hashlib import pytest from agent_comms.coordination_database import ReadableStore @@ -13,14 +12,7 @@ from agent_comms.working_memory_labels import ( JevClassifier, ) - - -def local_system(text, sources=()): - """Unattested local content; no SDK/provider observation is claimed.""" - raw = text.encode() - return SystemLayerSegment(provenance=(UnattributedProvenance(),), content=text, - tokens=0, sha256=hashlib.sha256(raw).hexdigest(), utf8_bytes=len(raw), - source_spans=sources) +from working_memory_fixture import local_system def test_readers_do_not_create_a_missing_store(tmp_path): diff --git a/tests/working_memory_fixture.py b/tests/working_memory_fixture.py new file mode 100644 index 0000000000..0f6e18fcc5 --- /dev/null +++ b/tests/working_memory_fixture.py @@ -0,0 +1,17 @@ +"""Unattested local annotation data, never an SDK/provider observation.""" +import hashlib + +from agent_comms.turn_context import ( + ContributionCoordinates, SystemLayerSegment, UnattributedProvenance, +) + + +def local_system( + text: str, sources: tuple[ContributionCoordinates, ...] = (), +) -> SystemLayerSegment: + raw = text.encode() + return SystemLayerSegment( + provenance=(UnattributedProvenance(),), content=text, tokens=0, + sha256=hashlib.sha256(raw).hexdigest(), utf8_bytes=len(raw), + source_spans=sources, + ) From 857fbf0adaba51753594dae892fb5d87fccd6dfa Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sun, 1 Nov 2026 19:28:56 -0500 Subject: [PATCH 17/28] Stream original segment groups during annotation retention --- src/agent_comms/coordination_tables/annotations.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/agent_comms/coordination_tables/annotations.py b/src/agent_comms/coordination_tables/annotations.py index e05b09f3b7..eaff82e470 100644 --- a/src/agent_comms/coordination_tables/annotations.py +++ b/src/agent_comms/coordination_tables/annotations.py @@ -56,12 +56,13 @@ def for_segments( def retain_segments( cls, original: AttachedPracticeRebuild, segments: Iterable[SegmentManifest], ) -> int: - # Both declared row members consume the same original set; their raw - # values and request outcomes survive, not merely an effective answer. - selected = tuple(segments) + # Restored requests may repeat large original prefixes. Consume one + # segment at a time, not a second materialized context corpus. + members = cls.members_with(cls) return sum(row.retain_original(original) - for member in cls.members_with(cls) - for row in member.for_segments(original.destination, selected, + for segment in segments + for member in members + for row in member.for_segments(original.destination, (segment,), table_source=original.source)) From fa5ad0724de34630fe4c431ac16b08040a171da1 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sun, 1 Nov 2026 20:14:15 -0500 Subject: [PATCH 18/28] Require recorded IDs and required participant rows in retention readers Type the harness hook and remove its new low-level type-only/error-import detours. The unallocated SQLite ID owns numeric refusal; both annotation sorts use it. Participant reads use the existing required-row decoder and preserve domain missing-row refusal. One unrun original-ID case added. Source-only: NRA syntax/compilation of six files, zero omissions; no Core subjects/MRO/SQL/store/pytest/provider execution. Hosted debt-src at parent is red; remaining import/native projection findings and preservation/application qualification remain open on same #3829. --- .../coordination_tables/annotations.py | 2 +- src/agent_comms/input_origin.py | 8 +++++--- src/agent_comms/participant_store.py | 17 +++++++++++------ src/agent_comms/typed_table.py | 3 +++ src/agent_comms/working_memory_annotations.py | 2 +- tests/test_retained_annotations.py | 14 ++++++++++++++ 6 files changed, 35 insertions(+), 11 deletions(-) diff --git a/src/agent_comms/coordination_tables/annotations.py b/src/agent_comms/coordination_tables/annotations.py index eaff82e470..da7fae187b 100644 --- a/src/agent_comms/coordination_tables/annotations.py +++ b/src/agent_comms/coordination_tables/annotations.py @@ -125,7 +125,7 @@ def effective(cls, rows: tuple[SpanAnnotationsRow, ...]): # Row ids order successive corrections from the same authority. Nominal # inheritance, not sequence, decides human versus model authority. latest = {} - for row in sorted(rows, key=lambda value: value.id): + for row in sorted(rows, key=lambda value: int(value.id)): latest[type(row.label)] = row.label return SpanLabel.effective(tuple(latest.values())) diff --git a/src/agent_comms/input_origin.py b/src/agent_comms/input_origin.py index 9d251aa733..4cc56a041a 100644 --- a/src/agent_comms/input_origin.py +++ b/src/agent_comms/input_origin.py @@ -8,6 +8,7 @@ from .bus_publication import HumanOrigin +from .coordination_errors import IdentityConflict from .nominal.declared_family import DeclaredFamily from .errors import RelationViolationError from .nominal.field_codec import RootId, ThreadLookup @@ -18,13 +19,13 @@ from .repo import Checkout from .thread_identity import ClaimScoped, ClaimedIdentity from .turn_context import Provenance +from .typed_table import AttachedPracticeRebuild if TYPE_CHECKING: from .comms import Comms from .input_attempt import StoredInput from .registry_document import RegistrySnapshot from .threads import Thread - from .typed_table import AttachedPracticeRebuild class InputDelivery: @@ -131,8 +132,9 @@ class HarnessAttemptOrigin(InputOrigin): bridge: type[HarnessSession] reference: MessageReference - def retain_harness_rows(self, original, recipient): - from .coordination_errors import IdentityConflict + def retain_harness_rows( + self, original: AttachedPracticeRebuild, recipient: ThreadLookup, + ) -> int: from .coordination_tables.harness_deliveries import HarnessDelivery key = {"bridge": self.bridge, "recipient_lookup": recipient, diff --git a/src/agent_comms/participant_store.py b/src/agent_comms/participant_store.py index 5b22f1b456..53e8ddc101 100644 --- a/src/agent_comms/participant_store.py +++ b/src/agent_comms/participant_store.py @@ -23,7 +23,7 @@ ) from agent_comms.nominal.field_codec import ThreadLookup from agent_comms.typed_table import ( - LOCAL_TABLE_SOURCE, AttachedPracticeRebuild, TableSource, + LOCAL_TABLE_SOURCE, AttachedPracticeRebuild, RowNotFoundError, TableSource, ) @@ -61,11 +61,16 @@ def read_at( table_source: TableSource = LOCAL_TABLE_SOURCE, ) -> ParticipantSnapshot: """Read the same aggregate through the caller's already-owned SQL lifetime.""" - person = Participants.one(db, participant_lookup=lookup, table_source=table_source) - generation = OwnerGenerations.one(db, owner_lookup=lookup, table_source=table_source) - pointer = CurrentExecutions.one(db, owner_lookup=lookup, table_source=table_source) - if person is None or generation is None or pointer is None: - raise ParticipantNotRegistered(f"participant {lookup!r} aggregate is not registered") + try: + person = Participants.require(db, participant_lookup=lookup, table_source=table_source) + generation = OwnerGenerations.require( + db, owner_lookup=lookup, table_source=table_source, + ) + pointer = CurrentExecutions.require(db, owner_lookup=lookup, table_source=table_source) + except RowNotFoundError as error: + raise ParticipantNotRegistered( + f"participant {lookup!r} aggregate is not registered", + ) from error return ParticipantSnapshot(lookup, person.committed, generation, pointer) @classmethod diff --git a/src/agent_comms/typed_table.py b/src/agent_comms/typed_table.py index 2514db2424..a08965695a 100644 --- a/src/agent_comms/typed_table.py +++ b/src/agent_comms/typed_table.py @@ -423,6 +423,9 @@ def table_column(self, name: str) -> _Field: class UnallocatedRowId(NullMember): """SQLite allocates this row's AUTOINCREMENT id when the row is inserted.""" + def __int__(self) -> NoReturn: + raise TypeError("SQLite has not allocated this row id") + RowId = int | UnallocatedRowId diff --git a/src/agent_comms/working_memory_annotations.py b/src/agent_comms/working_memory_annotations.py index 61874545b8..4b47a48c7c 100644 --- a/src/agent_comms/working_memory_annotations.py +++ b/src/agent_comms/working_memory_annotations.py @@ -89,7 +89,7 @@ def for_segments(cls, source: ReadableStore, segments, classifier) -> tuple[Mode with cls.reading(source) as db: rows = sorted(SpanAnnotationsRow.for_segments( db, segments, classifier=classifier.classifier, classifier_pin=classifier.pin, - ), key=lambda row: row.id) + ), key=lambda row: int(row.id)) grouped = {} for row in rows: label = row.label diff --git a/tests/test_retained_annotations.py b/tests/test_retained_annotations.py index 867c80eb7e..9e8b433e12 100644 --- a/tests/test_retained_annotations.py +++ b/tests/test_retained_annotations.py @@ -186,6 +186,20 @@ def test_selected_context_copy_preserves_answers_versions_outcomes_and_grant_usa assert ContextStore(source.path.parent).path.read_bytes() == contexts_before +def test_effective_annotations_require_an_original_allocated_row_id( + annotation_source: AnnotationSource, +) -> None: + source = annotation_source + with source.store.session.read(): + stored = SpanAnnotationsRow.select( + source.store.session._connection, order_by=('id',), + )[0] + assert SpanAnnotationsRow.effective((stored,)) == stored.label + unstored = SpanAnnotationsRow(label=stored.label, created_at_ms=stored.created_at_ms) + with pytest.raises(TypeError): + SpanAnnotationsRow.effective((unstored,)) + + def test_context_selection_cannot_follow_finalized_coordinator_evidence( annotation_source: AnnotationSource, tmp_path: Path, ) -> None: From e94e5470e01f728e39aa6fd5f1681694dcbe7ff4 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sun, 1 Nov 2026 20:53:34 -0500 Subject: [PATCH 19/28] Keep selected input orchestration in PracticeCopy NRA source snapshots and compilation: five production modules plus one test, no omissions. Existing membership-query prefix is AST-identical; 85 existing annotations unchanged. Source checks only; no Core, MRO, SQL, store or test execution. Existing 29 authored scenarios still need the combined run. --- docs/retained-practice-inputs.md | 4 +- src/agent_comms/coordination_database.py | 38 +------------------ .../coordination_tables/annotations.py | 19 ++++++---- src/agent_comms/installer/practice.py | 21 +++++++--- src/agent_comms/native_input_record.py | 14 +++++-- src/agent_comms/native_runtime_input.py | 10 ++--- tests/test_retained_coordinator_groups.py | 4 ++ 7 files changed, 49 insertions(+), 61 deletions(-) diff --git a/docs/retained-practice-inputs.md b/docs/retained-practice-inputs.md index 9a30338c95..bec9b85ee4 100644 --- a/docs/retained-practice-inputs.md +++ b/docs/retained-practice-inputs.md @@ -223,8 +223,8 @@ name/generation or a fabricated birth. `HarnessAttemptOrigin.retain_harness_rows owns its bridge/reference selection and full-sequence check; origins without a harness witness contribute no harness rows. No synthesized reservation is stored. -`CoordinationStore.retain_inputs` runs these seeds through the existing -`ReconciledFile.retaining_rows` loading/dependency/publication lifetime. +`PracticeCopy.include_inputs` runs these seeds directly through the existing +`CoordinationStore.retaining_rows` loading/dependency/publication lifetime. `NativeRuntimeInput.retain_sessions` additionally selects original native groups by both recorded session id and original session path. A bare ledger native id is not treated as a coordinator FK. The existing row hooks retain the selected diff --git a/src/agent_comms/coordination_database.py b/src/agent_comms/coordination_database.py index ba5026dcff..fac2ad4701 100644 --- a/src/agent_comms/coordination_database.py +++ b/src/agent_comms/coordination_database.py @@ -7,11 +7,11 @@ import stat import tempfile from abc import abstractmethod -from collections.abc import Iterable, Iterator +from collections.abc import Iterator from contextlib import closing, contextmanager, suppress from dataclasses import dataclass from pathlib import Path -from typing import TYPE_CHECKING, Self +from typing import Self from agent_comms.coordination_errors import CoordinationReadUnavailable, IntegrityViolationError from agent_comms.coordination_schema import CoordinatorTable @@ -27,13 +27,6 @@ from .nominal.declared_family import DeclaredFamily from .store_files import DEFAULT_STORE_CONTENTION, StoreContention -if TYPE_CHECKING: - from .input_disposition import InputDocument - from .native_session_reopen import NativeSessionIdentity - from .registry_document import RegistryEntry - from .turn_context import SegmentManifest - - class StoreFile(DeclaredFamily): """One decoded file state, with the consumer's result inside its read lifetime.""" @@ -88,33 +81,6 @@ def schema_at(cls, path: Path) -> type[TableSchema]: return RegistryStore.at(path.parent).privacy().coordinator_schema() - @classmethod - def retain_inputs( - cls, source: Path, destination: Path, entries: tuple[RegistryEntry, ...], *, - maximum_bytes: int, identities: Iterable[NativeSessionIdentity] = (), - segments: Iterable[SegmentManifest] = (), - ) -> InputDocument: - """Materialize original input witnesses under the existing fresh-file owner. - - The caller supplies a consistent retained bus/complete ledger beside the - destination and the original registry entries from that same cut. This - selects data only; capture consistency and runnable-root authority remain - the capture/preparation owners' separate obligations. - """ - from .input_disposition import SnapshotInputDispositions - from .native_runtime_input import NativeRuntimeInput - from .participant_store import ParticipantStore - from .coordination_tables.annotations import ContextSpanRows - - with cls.retaining_rows(source, destination, maximum_bytes=maximum_bytes) as original: - selected = SnapshotInputDispositions(destination.parent, original).retain_unresolved(entries) - ParticipantStore.retain_owners(original, tuple( - entry.thread.incarnation.require_recorded().lookup for entry in entries - )) - NativeRuntimeInput.retain_sessions(original, identities) - ContextSpanRows.retain_segments(original, segments) - return selected - @classmethod def observed(cls, path: Path) -> StoreFile: """Classify through SQLite without dropping another connection's process locks. diff --git a/src/agent_comms/coordination_tables/annotations.py b/src/agent_comms/coordination_tables/annotations.py index da7fae187b..9828134538 100644 --- a/src/agent_comms/coordination_tables/annotations.py +++ b/src/agent_comms/coordination_tables/annotations.py @@ -188,17 +188,22 @@ def retain_required_rows( cls, original: AttachedPracticeRebuild, schema: type[TableSchema], ) -> int: """Keep original grant usage and each request's complete answer/correction group.""" - from agent_comms.working_memory_annotations import WorkingMemoryAnnotations - changes = super().retain_required_rows(original, schema) changes += original.retain_keys(cls, tuple( {"grant_reference": row.request.grant} for row in cls.stream(original.destination) )) - changes += original.retain_keys(SpanAnnotationsRow, tuple( - WorkingMemoryAnnotations.address(row.request.span, row.request.question, - row.request.classifier) - for row in cls.stream(original.destination) - )) + # One natural address can name multiple correction rows; it is not a + # foreign key. Compare the actual declared generated columns, without + # making storage import a reader to reconstruct their domain values. + related = " AND ".join( + f"{SpanAnnotationsRow.qualified(name)} IS {cls.qualified(name)}" + for name in SpanAnnotationsRow.address + ) + changes += original.retain_rows( + SpanAnnotationsRow, + where=f"EXISTS (SELECT 1 FROM main.{LOCAL_TABLE_SOURCE.qualified(cls.declared_name)} " + f"WHERE {related})", + ) return changes @classmethod diff --git a/src/agent_comms/installer/practice.py b/src/agent_comms/installer/practice.py index 135692839a..34351597ac 100644 --- a/src/agent_comms/installer/practice.py +++ b/src/agent_comms/installer/practice.py @@ -13,7 +13,10 @@ from agent_comms.bus import Bus from agent_comms.context_store import ContextManifestReference, ContextStore from agent_comms.coordination_database import CoordinationStore -from agent_comms.input_disposition import InputDispositions, InputDocument +from agent_comms.coordination_tables.annotations import ContextSpanRows +from agent_comms.input_disposition import InputDispositions, InputDocument, SnapshotInputDispositions +from agent_comms.native_runtime_input import NativeRuntimeInput +from agent_comms.participant_store import ParticipantStore from agent_comms.registry_document import RegistryEntry from agent_comms.native_session_reopen import NativeSessionIdentity from agent_comms.nominal.field_codec import boundary_decoder @@ -225,11 +228,17 @@ def include_inputs( raise UpgradeRefused("Copied contexts have no captured original reader source") source = CoordinationStore.path_in(self.root) resource, = RootFile.owned_files(LiveFilesRequest(self.root), (source,)) - selected = CoordinationStore.retain_inputs( - source, CoordinationStore.path_in(self.path), entries, - identities=identities, maximum_bytes=self.remaining_bytes(maximum_bytes), - segments=contexts.original_segments() if source_contexts in self.copied else (), - ) + segments = contexts.original_segments() if source_contexts in self.copied else () + with CoordinationStore.retaining_rows( + source, CoordinationStore.path_in(self.path), + maximum_bytes=self.remaining_bytes(maximum_bytes), + ) as original: + selected = SnapshotInputDispositions(self.path, original).retain_unresolved(entries) + ParticipantStore.retain_owners(original, tuple( + entry.thread.incarnation.require_recorded().lookup for entry in entries + )) + NativeRuntimeInput.retain_sessions(original, identities) + ContextSpanRows.retain_segments(original, segments) self.copied[source] = resource return selected diff --git a/src/agent_comms/native_input_record.py b/src/agent_comms/native_input_record.py index bb6aa1a8ed..f49fd393f3 100644 --- a/src/agent_comms/native_input_record.py +++ b/src/agent_comms/native_input_record.py @@ -36,7 +36,7 @@ from .native_integer import PositiveNativeIntegerBounds from .native_session_reopen import PiSessionId from .transcript_routes import InputDisplay, NoDisplayText -from .typed_table import LOCAL_TABLE_SOURCE, SqlStorage, TableSource, TextStorage +from .typed_table import LOCAL_TABLE_SOURCE, SqlStorage, TableSource, TextStorage, sql_literal if TYPE_CHECKING: from sqlite3 import Connection @@ -190,6 +190,14 @@ def source_assignment_ids( self, db: Connection, input_id: InputId, *, table_source: TableSource = LOCAL_TABLE_SOURCE, ) -> tuple[AssignmentId, ...]: + return tuple(row.assignment_id for row in self.source_assignments( + db, input_id, table_source=table_source, + )) + + def source_assignments( + self, db: Connection, input_id: InputId, *, + table_source: TableSource = LOCAL_TABLE_SOURCE, + ) -> tuple[WakeAssignment, ...]: """Membership leads: a missing claim reaches the required-row decoder. An inner join would silently erase a declared source whose claim was @@ -202,7 +210,7 @@ def source_assignment_ids( "ON m.assignment_id=c.assignment_id " "WHERE m.input_id=? ORDER BY m.ordinal", (input_id,), )) - return tuple(original.assignment_id for original in originals) + return tuple(originals) @abstractmethod def record_sources( @@ -294,7 +302,7 @@ def source_membership_sql( f"FROM {table_source.qualified(NativeRuntimeInput.declared_name)} n " f"JOIN {table_source.qualified(ExecutionAssignmentLink.declared_name)} c " "ON c.execution_id=n.execution_id " - f"WHERE n.stage='{cls.declared_name}'") + f"WHERE n.stage={sql_literal(cls)}") def record_sources( self, db: Connection, input_id: InputId, assignments: tuple[WakeAssignment, ...], diff --git a/src/agent_comms/native_runtime_input.py b/src/agent_comms/native_runtime_input.py index 2478112841..a222e4249d 100644 --- a/src/agent_comms/native_runtime_input.py +++ b/src/agent_comms/native_runtime_input.py @@ -190,16 +190,12 @@ def retain_required_rows( Membership comes from the original attachment; only selected native ids come from the destination. """ - from .coordination_tables.assignments import WakeAssignment - changed = super().retain_required_rows(original, schema) for native in cls.stream(original.destination): - identifiers = native.execution.source_assignment_ids( + for assignment in native.execution.source_assignments( original.destination, native.input_id, table_source=original.source, - ) - changed += original.retain_keys(WakeAssignment, tuple( - {"assignment_id": identifier} for identifier in identifiers - )) + ): + changed += assignment.retain_original(original) return changed def __post_init__(self): diff --git a/tests/test_retained_coordinator_groups.py b/tests/test_retained_coordinator_groups.py index a82f97c60d..b706674a1e 100644 --- a/tests/test_retained_coordinator_groups.py +++ b/tests/test_retained_coordinator_groups.py @@ -347,6 +347,10 @@ def test_native_membership_query_uses_the_supplied_attachment( with CoordinationStore.retaining_rows(source.path, destination, maximum_bytes=8 * 2**20) as original: # Both table sets exist. Ordinary lookup sees the empty destination; # qualification must keep every join on the original attachment. + assert selected.execution.source_assignments(original.destination, selected.input_id) == () + assert selected.execution.source_assignments( + original.destination, selected.input_id, table_source=original.source, + ) == (source.claims[0],) assert selected.execution.source_assignment_ids(original.destination, selected.input_id) == () assert selected.execution.source_assignment_ids( original.destination, selected.input_id, table_source=original.source, From 2b5de62abeee0fff8e411c502f6c3600967c46eb Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sun, 1 Nov 2026 22:58:45 -0500 Subject: [PATCH 20/28] Keep selected row groups on their retention owners --- docs/retained-practice-inputs.md | 41 ++++++++-------- .../coordination_tables/harness_deliveries.py | 15 ++++++ .../coordination_tables/participants.py | 12 ----- src/agent_comms/input_attempt.py | 5 +- src/agent_comms/input_disposition.py | 7 +-- src/agent_comms/input_key.py | 15 +++--- src/agent_comms/input_origin.py | 26 +++------- src/agent_comms/installer/practice.py | 3 +- src/agent_comms/native_input_record.py | 8 +-- src/agent_comms/participant_store.py | 12 ++++- src/agent_comms/typed_table.py | 15 +++++- tests/test_retained_coordinator_groups.py | 49 +++++++++++++------ 12 files changed, 122 insertions(+), 86 deletions(-) diff --git a/docs/retained-practice-inputs.md b/docs/retained-practice-inputs.md index bec9b85ee4..d18eb08eed 100644 --- a/docs/retained-practice-inputs.md +++ b/docs/retained-practice-inputs.md @@ -219,9 +219,10 @@ harness producer a second time on the same attachment, not a copied key/outcome or suppression algorithm. An unresolved stored ledger row can keep an additional original attempt witness at its key; the complete ledger still wins the read. The recipient lookup comes from those original source entries, not from a stored -name/generation or a fabricated birth. `HarnessAttemptOrigin.retain_harness_rows` -owns its bridge/reference selection and full-sequence check; origins without a -harness witness contribute no harness rows. No synthesized reservation is stored. +name/generation or a fabricated birth. `InputOrigin.harness_sources` exposes no +witness by default; `HarnessAttemptOrigin` exposes itself. `HarnessDelivery.retain_attempt` +owns keyed row retention and the full-reference refusal. No synthesized reservation +is stored, and the provenance family imports no coordinator row. `PracticeCopy.include_inputs` runs these seeds directly through the existing `CoordinationStore.retaining_rows` loading/dependency/publication lifetime. @@ -249,9 +250,10 @@ These do not cover every disposition, native association or rollback/resource fa Runtime class creation/MRO/registration, attached SQL, cross-store capture and complete preservation remain unqualified. The selector has no participant-wide -reverse walk. Participant aggregates now contribute through the row's cooperative -retention hook below; inverse native membership, serialized references and the -remaining original recovery groups still require semantic closure. No seed-only database is a runnable root or complete backup. +reverse walk. Participant aggregates contribute through the selected attachment's +expansion below; inverse native membership, serialized references and the remaining +original recovery groups still require semantic closure. No seed-only database is +a runnable root or complete backup. ## Participant aggregate — source implementation, not acceptance @@ -265,11 +267,14 @@ retention never constructs a borrowed `CoordinationSession` or another connectio `ParticipantStore.retain_owners` seeds selected original lookup keys, preserves an absent participant as absent, and refuses a present incomplete aggregate. `ParticipantSnapshot.retain_original` keeps its original generation/current pointer. -`Participants.retain_required_rows` contributes the same aggregate through the -existing cooperative hook, including participants discovered later by FK closure, -not just the initial registry entries. This is not a reverse traversal of every -participant execution. The current pointer's declared references retain its actual -pointed execution/attempt; no counter, revision, commitment or lifecycle is reset. +`ParticipantRows(AttachedPracticeRebuild)` adds that aggregate to each expansion of +the existing selected-file closure, including participants discovered later by FK, +not just the initial seeds. `PracticeCopy.include_inputs` selects this attachment +through `original_type`; generic foreign-key copying alone does not claim aggregate +completeness. The same read-only attachment and destination connection are borrowed; +there is no second closure loop or connection, and Participants imports no reader. +The current pointer's declared references retain its actual pointed execution/attempt; +no counter, revision, commitment or lifecycle is reset. `TypedTable.retention_key_fields` derives the primary key once from its declared columns for raw-copy deduplication and held-row selection. `retain_original` gets @@ -286,16 +291,12 @@ seed cases also require the retained aggregate. Source mapping covers 65 modules 18 function-local test declarations remain unprojected OPEN. That is source coverage, not executed class/MRO/SQL behavior or complete closure. -## Hook correction and SQLite allocation metadata — source only +## SQLite allocation metadata — source only -Source follow-through caught an error in the participant checkpoint: its new -`Participants.retain_required_rows` omitted the `schema` argument that the base -and actual `TableSchema.retain_referenced_rows` caller supply. It now accepts and -passes that argument cooperatively. The whole declared hook family is checked at -source; this is not runtime dispatch/MRO qualification. The active-pointer case -also now calls the real `mark_pending` transition and expects revision 2 before -`start`; QueuedExecution does not admit direct start. No product fallback was added -to satisfy a faulty fixture. +The coordinator cases share the real selected-file factory with `ParticipantRows`. +The active-pointer case uses the real `mark_pending` transition and expects revision +2 before `start`; QueuedExecution does not admit direct start. No product fallback +was added to satisfy a faulty fixture. These cases still require execution. A separate preservation gap remained in the earlier rowid hook: `RecoveryAudit`, `SpanAnnotationsRow` and `AnnotationRequestsRow` declare AUTOINCREMENT columns but diff --git a/src/agent_comms/coordination_tables/harness_deliveries.py b/src/agent_comms/coordination_tables/harness_deliveries.py index f125c4c953..92fbcaaff3 100644 --- a/src/agent_comms/coordination_tables/harness_deliveries.py +++ b/src/agent_comms/coordination_tables/harness_deliveries.py @@ -21,6 +21,7 @@ MonitorEvidence, Outcome, UnconfirmedOutcome, UnrecordedMonitor, ) from agent_comms.harness_install import HarnessSession +from agent_comms.input_origin import HarnessAttemptOrigin from agent_comms.message_reference import MessageReference from agent_comms.typed_table import ( LOCAL_TABLE_SOURCE, AttachedPracticeRebuild, Column, ForeignKey, FrozenColumn, @@ -51,6 +52,20 @@ def references(cls) -> tuple[ForeignKey, ...]: return (ForeignKey(columns=("recipient_lookup",), target=Participants, target_columns=("participant_lookup",)),) + @classmethod + def retain_attempt( + cls, original: AttachedPracticeRebuild, recipient: ThreadLookup, + attempt: HarnessAttemptOrigin, + ) -> int: + """Keep the original keyed attempt and reject a different full message reference.""" + key = {"bridge": attempt.bridge, "recipient_lookup": recipient, + "message_id": attempt.reference.message_id} + changed = original.retain_keys(cls, (key,)) + retained = cls.require(original.destination, table_source=LOCAL_TABLE_SOURCE, **key) + if retained.reference != attempt.reference: + raise IdentityConflict("Retained harness attempt names another message sequence") + return changed + @classmethod def retain_required_rows( cls, original: AttachedPracticeRebuild, schema: type[TableSchema], diff --git a/src/agent_comms/coordination_tables/participants.py b/src/agent_comms/coordination_tables/participants.py index 4f9346e0c0..44c7bfaf7f 100644 --- a/src/agent_comms/coordination_tables/participants.py +++ b/src/agent_comms/coordination_tables/participants.py @@ -10,7 +10,6 @@ from agent_comms.nominal.field_codec import ThreadLookup from agent_comms.participant_identity import ParticipantIdentity from agent_comms.typed_table import ( - AttachedPracticeRebuild, Column, ColumnGuard, CounterAdvance, @@ -20,7 +19,6 @@ MonotoneColumns, NoDelete, NonDecreasing, - TableSchema, ) @@ -53,16 +51,6 @@ class Participants(MonotoneColumns, GuardedColumns, NoDelete, CoordinatorRow): ] committed: bool - @classmethod - def retain_required_rows( - cls, original: AttachedPracticeRebuild, schema: type[TableSchema], - ) -> int: - from agent_comms.participant_store import ParticipantStore - - return super().retain_required_rows(original, schema) + ParticipantStore.retain_owners( - original, tuple(person.participant_lookup for person in cls.select(original.destination)), - ) - @dataclass(frozen=True, kw_only=True) class OwnerGenerations( CounterAdvance, NoDelete, ParticipantIdentity, CoordinatorRow diff --git a/src/agent_comms/input_attempt.py b/src/agent_comms/input_attempt.py index dc61819881..5b5c2fa7c5 100644 --- a/src/agent_comms/input_attempt.py +++ b/src/agent_comms/input_attempt.py @@ -342,7 +342,10 @@ def matches_owner(self, source_owner: ThreadIncarnation) -> bool: # The origin owns author evidence; an owned bus key additionally # retains its original recipient birth. Neither invents a missing birth. return (self.origin.matches_owner(self.owner, source_owner) - and self.key.matches_owner(source_owner)) + and source_owner.recorded_or( + lambda recorded: self.key.matches_recorded_owner(recorded.created_at), + self.key.matches_unrecorded_owner, + )) def matches_claim(self, claim_scope: int) -> bool: return self.claim_generation == claim_scope diff --git a/src/agent_comms/input_disposition.py b/src/agent_comms/input_disposition.py index c25cd8b200..984a936c8f 100644 --- a/src/agent_comms/input_disposition.py +++ b/src/agent_comms/input_disposition.py @@ -593,7 +593,8 @@ def retain_unresolved(self, entries: tuple[RegistryEntry, ...]) -> InputDocument for entry in entries} for attempted in self.harness_attempts(entries): if attempted.key in selected.rows: - attempted.origin.retain_harness_rows( - self.original, recipients[attempted.owner], - ) + for source in attempted.origin.harness_sources(): + HarnessDelivery.retain_attempt( + self.original, recipients[attempted.owner], source, + ) return selected diff --git a/src/agent_comms/input_key.py b/src/agent_comms/input_key.py index cc3cb0ecd6..9cf508ceb7 100644 --- a/src/agent_comms/input_key.py +++ b/src/agent_comms/input_key.py @@ -14,7 +14,6 @@ if TYPE_CHECKING: from .acp_extension import InputStartedUpdate - from .thread_identity import ThreadIncarnation class InputKey(DeclaredFamily): @@ -48,10 +47,13 @@ def public_text(self) -> str: def identify(self, update: InputStartedUpdate) -> InputStartedUpdate: return update - def matches_owner(self, incarnation: ThreadIncarnation) -> bool: + def matches_recorded_owner(self, created_at: float) -> bool: """A birthless key adds no birth claim to its origin's owner check.""" return True + def matches_unrecorded_owner(self) -> bool: + return True + @dataclass(frozen=True) class AcpInputKey(InputKey): @@ -95,11 +97,12 @@ class OwnedBusInputKey(BusInputKey): def text(self) -> str: return super().text() + ":owner:" + self.owner_created.hex() - def matches_owner(self, incarnation: ThreadIncarnation) -> bool: + def matches_recorded_owner(self, created_at: float) -> bool: """The original recipient's recorded birth must still match.""" - return incarnation.recorded_or( - lambda recorded: recorded.created_at == self.owner_created, lambda: False, - ) + return created_at == self.owner_created + + def matches_unrecorded_owner(self) -> bool: + return False @dataclass(frozen=True) class TurnInputKey(InputKey): diff --git a/src/agent_comms/input_origin.py b/src/agent_comms/input_origin.py index 4cc56a041a..ec2b8e7994 100644 --- a/src/agent_comms/input_origin.py +++ b/src/agent_comms/input_origin.py @@ -8,10 +8,9 @@ from .bus_publication import HumanOrigin -from .coordination_errors import IdentityConflict from .nominal.declared_family import DeclaredFamily from .errors import RelationViolationError -from .nominal.field_codec import RootId, ThreadLookup +from .nominal.field_codec import RootId from .goals import GoalCheckpoint from .harness_install import HarnessSession from .input_key import InputKey, InputKeyText @@ -19,7 +18,6 @@ from .repo import Checkout from .thread_identity import ClaimScoped, ClaimedIdentity from .turn_context import Provenance -from .typed_table import AttachedPracticeRebuild if TYPE_CHECKING: from .comms import Comms @@ -63,11 +61,9 @@ def retained_fact(self, source: StoredInput): def require_human(self) -> HumanInputOrigin: raise RelationViolationError("Input lacks an original human author witness") - def retain_harness_rows( - self, original: AttachedPracticeRebuild, recipient: ThreadLookup, - ) -> int: - """An origin without a harness witness contributes no harness rows.""" - return 0 + def harness_sources(self) -> tuple[HarnessAttemptOrigin, ...]: + """An origin without a harness witness contributes no harness sources.""" + return () def matches_owner(self, recorded_name, incarnation): return recorded_name == incarnation.name @@ -132,18 +128,8 @@ class HarnessAttemptOrigin(InputOrigin): bridge: type[HarnessSession] reference: MessageReference - def retain_harness_rows( - self, original: AttachedPracticeRebuild, recipient: ThreadLookup, - ) -> int: - from .coordination_tables.harness_deliveries import HarnessDelivery - - key = {"bridge": self.bridge, "recipient_lookup": recipient, - "message_id": self.reference.message_id} - changed = original.retain_keys(HarnessDelivery, (key,)) - retained = HarnessDelivery.require(original.destination, **key) - if retained.reference != self.reference: - raise IdentityConflict("Retained harness attempt names another message sequence") - return changed + def harness_sources(self) -> tuple[HarnessAttemptOrigin, ...]: + return (self,) @dataclass(frozen=True) diff --git a/src/agent_comms/installer/practice.py b/src/agent_comms/installer/practice.py index 34351597ac..5b7a067430 100644 --- a/src/agent_comms/installer/practice.py +++ b/src/agent_comms/installer/practice.py @@ -16,7 +16,7 @@ from agent_comms.coordination_tables.annotations import ContextSpanRows from agent_comms.input_disposition import InputDispositions, InputDocument, SnapshotInputDispositions from agent_comms.native_runtime_input import NativeRuntimeInput -from agent_comms.participant_store import ParticipantStore +from agent_comms.participant_store import ParticipantRows, ParticipantStore from agent_comms.registry_document import RegistryEntry from agent_comms.native_session_reopen import NativeSessionIdentity from agent_comms.nominal.field_codec import boundary_decoder @@ -232,6 +232,7 @@ def include_inputs( with CoordinationStore.retaining_rows( source, CoordinationStore.path_in(self.path), maximum_bytes=self.remaining_bytes(maximum_bytes), + original_type=ParticipantRows, ) as original: selected = SnapshotInputDispositions(self.path, original).retain_unresolved(entries) ParticipantStore.retain_owners(original, tuple( diff --git a/src/agent_comms/native_input_record.py b/src/agent_comms/native_input_record.py index f49fd393f3..93296f5f84 100644 --- a/src/agent_comms/native_input_record.py +++ b/src/agent_comms/native_input_record.py @@ -206,7 +206,7 @@ def source_assignments( columns = ",".join(f"c.{column}" for column in WakeAssignment.columns()) originals = WakeAssignment.read(db.execute( f"SELECT {columns} FROM ({self.source_membership_sql(table_source=table_source)}) m " - f"LEFT JOIN {table_source.qualified(WakeAssignment.declared_name)} c " + f"LEFT JOIN {WakeAssignment.qualified_table(table_source)} c " "ON m.assignment_id=c.assignment_id " "WHERE m.input_id=? ORDER BY m.ordinal", (input_id,), )) @@ -253,7 +253,7 @@ def source_membership_sql( from .triage_native_sources import TriageNativeSource return (f"SELECT s.input_id, s.assignment_id, s.ordinal " - f"FROM {table_source.qualified(TriageNativeSource.declared_name)} s") + f"FROM {TriageNativeSource.qualified_table(table_source)} s") def record_sources( self, db: Connection, input_id: InputId, assignments: tuple[WakeAssignment, ...], @@ -299,8 +299,8 @@ def source_membership_sql( from .native_runtime_input import NativeRuntimeInput return (f"SELECT n.input_id, c.assignment_id, c.ordinal " - f"FROM {table_source.qualified(NativeRuntimeInput.declared_name)} n " - f"JOIN {table_source.qualified(ExecutionAssignmentLink.declared_name)} c " + f"FROM {NativeRuntimeInput.qualified_table(table_source)} n " + f"JOIN {ExecutionAssignmentLink.qualified_table(table_source)} c " "ON c.execution_id=n.execution_id " f"WHERE n.stage={sql_literal(cls)}") diff --git a/src/agent_comms/participant_store.py b/src/agent_comms/participant_store.py index 53e8ddc101..f1b9e8daf0 100644 --- a/src/agent_comms/participant_store.py +++ b/src/agent_comms/participant_store.py @@ -23,7 +23,7 @@ ) from agent_comms.nominal.field_codec import ThreadLookup from agent_comms.typed_table import ( - LOCAL_TABLE_SOURCE, AttachedPracticeRebuild, RowNotFoundError, TableSource, + LOCAL_TABLE_SOURCE, AttachedPracticeRebuild, RowNotFoundError, TableSchema, TableSource, ) @@ -150,3 +150,13 @@ def advance_generation( generation=expected_generation + 1, ) return Applied(self.get(lookup)) + + +class ParticipantRows(AttachedPracticeRebuild): + """Keep whole participant answers inside the existing selected-file closure.""" + + def retain_referenced_rows(self, schema: type[TableSchema]) -> int: + changed = super().retain_referenced_rows(schema) + return changed + ParticipantStore.retain_owners(self, tuple( + person.participant_lookup for person in Participants.select(self.destination) + )) diff --git a/src/agent_comms/typed_table.py b/src/agent_comms/typed_table.py index a08965695a..8f6ead2774 100644 --- a/src/agent_comms/typed_table.py +++ b/src/agent_comms/typed_table.py @@ -1112,6 +1112,10 @@ class AttachedPracticeRebuild(SchemaRebuild): source: SQLiteDatabase destination: sqlite3.Connection + def retain_referenced_rows(self, schema: type[TableSchema]) -> int: + """Expand this attachment through its file's declared dependency groups.""" + return schema.retain_referenced_rows(self) + def retain_rows( self, row: type[TypedTable], *, where: str = "1", parameters: tuple[object, ...] = (), @@ -2790,6 +2794,11 @@ def wire_record(cls, db: sqlite3.Connection, plan: FieldPlan) -> str: item, = cls._fields_named((plan.name,)) return item.wire_record(db, plan, cls.declared_name) + @classmethod + def qualified_table(cls, table_source: TableSource = LOCAL_TABLE_SOURCE) -> str: + """Name this declared table at its SQL boundary, in the supplied database.""" + return table_source.qualified(cls.declared_name) + @classmethod def qualified(cls, *names: str) -> str: """Columns named through this table, for one select joining tables that share names.""" @@ -3205,6 +3214,7 @@ def configure_connection(cls, connection: sqlite3.Connection) -> None: @contextmanager def retaining_rows( cls, source: Path, destination: Path, *, maximum_bytes: int, + original_type: type[AttachedPracticeRebuild] = AttachedPracticeRebuild, ) -> Iterator[AttachedPracticeRebuild]: """Publish a fresh selected store only after loading and declared closure succeed. @@ -3222,11 +3232,12 @@ def retaining_rows( with closing(cls.writer(staged, isolation_level=None)) as db: cls.configure_connection(db) SQLiteDatabase.named(db, "main").limit_bytes(db, maximum_bytes) - with PracticeRebuild(source).attached(db) as original: + with PracticeRebuild(source).attached(db) as attached: + original = original_type(attached.source, attached.destination) with schema.loading_rows(db): original.require_schema(schema) yield original - while schema.retain_referenced_rows(original): + while original.retain_referenced_rows(schema): pass original.retain_sequences() retained_bytes = sum(member.stat().st_size for member in cls.files(staged) diff --git a/tests/test_retained_coordinator_groups.py b/tests/test_retained_coordinator_groups.py index b706674a1e..b8accb6fc5 100644 --- a/tests/test_retained_coordinator_groups.py +++ b/tests/test_retained_coordinator_groups.py @@ -1,6 +1,6 @@ """Original harness groups survive selected loading without replaying their lifecycle.""" -from collections.abc import Iterator -from contextlib import closing +from collections.abc import Callable, Iterator +from contextlib import AbstractContextManager, closing from dataclasses import dataclass, replace from pathlib import Path @@ -27,17 +27,26 @@ from agent_comms.native_runtime_input import NativeRuntimeInput from agent_comms.nominal.field_codec import AssignmentId, MessageId, ThreadLookup, TranscriptRecordId from agent_comms.owner_fence import prepare_fence_token -from agent_comms.participant_store import ParticipantStore +from agent_comms.participant_store import ParticipantRows, ParticipantStore from agent_comms.reason_codes import GivenReasonCode, NoDetail from agent_comms.recovery_states import ModelStalledRecovery from agent_comms.triage_native_sources import TriageNativeSource -from agent_comms.typed_table import TypedTable, UnallocatedRowId +from agent_comms.typed_table import AttachedPracticeRebuild, TypedTable, UnallocatedRowId from agent_comms.wake_policy import FullWake, MessageAudience from bus_record_fixture import ( FIRST_INPUT_ID, SECOND_INPUT_ID, THIRD_INPUT_ID, RECIPIENT_LOOKUP, prepare_native_comms, ) +def retaining_coordinator( + source: Path, destination: Path, +) -> AbstractContextManager[AttachedPracticeRebuild]: + """Use the real coordinator expansion with one shared test-copy allowance.""" + return CoordinationStore.retaining_rows( + source, destination, maximum_bytes=8 * 2**20, original_type=ParticipantRows, + ) + + @dataclass(frozen=True) class HarnessSource: store: StandaloneCoordination @@ -98,9 +107,7 @@ def test_retained_harness_group_keeps_exact_parts_and_existing_claims( selected = harness_source.deliveries[selected_index] destination = tmp_path / 'retained' / 'coordination.sqlite3' before = CoordinationStore.digest(harness_source.path) - with CoordinationStore.retaining_rows( - harness_source.path, destination, maximum_bytes=8 * 2**20, - ) as original: + with retaining_coordinator(harness_source.path, destination) as original: original.retain_keys(HarnessDelivery, ({ 'bridge': selected.bridge, 'recipient_lookup': selected.recipient_lookup, 'message_id': selected.message_id, @@ -143,9 +150,7 @@ def test_retained_harness_conflict_refuses_without_publication( before = CoordinationStore.digest(harness_source.path) destination = tmp_path / 'retained' / 'coordination.sqlite3' with pytest.raises(IdentityConflict): - with CoordinationStore.retaining_rows( - harness_source.path, destination, maximum_bytes=8 * 2**20, - ) as original: + with retaining_coordinator(harness_source.path, destination) as original: original.retain_keys(HarnessDelivery, ({ 'bridge': conflicting.bridge, 'recipient_lookup': conflicting.recipient_lookup, 'message_id': conflicting.message_id, @@ -213,7 +218,7 @@ def test_retained_native_group_uses_original_membership_and_children( expected_claims = source.claims[:claim_count] before = CoordinationStore.digest(source.path) destination = tmp_path / 'native-retained' / 'coordination.sqlite3' - with CoordinationStore.retaining_rows(source.path, destination, maximum_bytes=8 * 2**20) as original: + with retaining_coordinator(source.path, destination) as original: original.retain_keys(seed, (keys,)) with closing(CoordinationStore.read_snapshot(destination)) as db: assert tuple(NativeRuntimeInput.select(db)) == (selected,) @@ -244,7 +249,7 @@ def test_selected_participant_keeps_its_generation_and_current_pointer( expected = source.store.participants.get(RECIPIENT_LOOKUP) before = CoordinationStore.digest(source.path) destination = tmp_path / 'participant-retained' / 'coordination.sqlite3' - with CoordinationStore.retaining_rows(source.path, destination, maximum_bytes=8 * 2**20) as original: + with retaining_coordinator(source.path, destination) as original: with pytest.raises(ParticipantNotRegistered): ParticipantStore.read_at(original.destination, RECIPIENT_LOOKUP) assert ParticipantStore.read_at( @@ -270,20 +275,32 @@ def test_selected_participant_keeps_its_generation_and_current_pointer( execution_id=NoPointedExecution(), attempt_ordinal=NoPointedAttempt(), pointer_revision=5), )) +@pytest.mark.parametrize('retain_seed', ( + lambda original, delivery: ParticipantStore.retain_owners( + original, (delivery.recipient_lookup,), + ), + lambda original, delivery: delivery.retain_original(original), +), ids=('direct-owner', 'fk-discovered-owner')) def test_incomplete_original_participant_refuses_selected_publication( native_group_source: NativeGroupSource, tmp_path: Path, surviving: TypedTable, + retain_seed: Callable[[AttachedPracticeRebuild, HarnessDelivery], int], ) -> None: source = native_group_source lookup = ThreadLookup('7' * 32) + delivery = HarnessDelivery( + ClaudeBridge, lookup, MessageId.mint('incomplete-owner-message'), 7, EmittedOutcome, + ) with source.store.session.writing() as db: Participants(participant_lookup=lookup, committed=True).insert(db) surviving.insert(db) + delivery.insert(db) before = CoordinationStore.digest(source.path) destination = tmp_path / 'incomplete-participant' / 'coordination.sqlite3' with pytest.raises(ParticipantNotRegistered): - with CoordinationStore.retaining_rows(source.path, destination, maximum_bytes=8 * 2**20) as original: - ParticipantStore.retain_owners(original, (lookup,)) + with retaining_coordinator(source.path, destination) as original: + retain_seed(original, delivery) assert not destination.exists() + assert not tuple(destination.parent.iterdir()) assert CoordinationStore.digest(source.path) == before @@ -318,7 +335,7 @@ def test_selected_store_keeps_sqlite_high_water_without_rowid_mixin( audits = tuple(RecoveryAudit.select(source.store.session._connection, order_by=('audit_id',))) before = CoordinationStore.digest(source.path) destination = tmp_path / 'sequence-retained' / 'coordination.sqlite3' - with CoordinationStore.retaining_rows(source.path, destination, maximum_bytes=8 * 2**20) as original: + with retaining_coordinator(source.path, destination) as original: original.retain_keys(ExecutionRecord, keys) with StandaloneCoordination(str(destination), clock_ms=lambda: 1000) as retained: with retained.session.read(): @@ -344,7 +361,7 @@ def test_native_membership_query_uses_the_supplied_attachment( selected = source.native[0] before = CoordinationStore.digest(source.path) destination = tmp_path / 'membership-query' / 'coordination.sqlite3' - with CoordinationStore.retaining_rows(source.path, destination, maximum_bytes=8 * 2**20) as original: + with retaining_coordinator(source.path, destination) as original: # Both table sets exist. Ordinary lookup sees the empty destination; # qualification must keep every join on the original attachment. assert selected.execution.source_assignments(original.destination, selected.input_id) == () From 385885b20307cd14b80c1303a4868a7bd26f30ad Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sun, 1 Nov 2026 23:29:47 -0500 Subject: [PATCH 21/28] Enable read-only URI attachments and repair retention test originals The actual frozen90 integration run collected75:45pass/24fail/6setup errors. The selected-loading writer now enables the URI capability already used by reconcile; read-only ATTACH and rollback remain unchanged. Harness fixtures decode real UUID spellings; unresolved inputs receive actual registry entries. Incomplete-aggregate controls start with real owner registration and explicitly model foreign corruption by removing/restoring only the declaration-derived delete guard, keeping insertion membership intact and requiring schema before retention. NRA parsing and compilation of three changed modules:0 omissions,rc0. No author Core imports/MRO/SQL/pytest/provider execution. The original failed run is retained; corrected behavior and the wider original-data/native/physical journey are unverified. --- src/agent_comms/typed_table.py | 2 +- tests/test_input_documents.py | 11 +++++-- tests/test_retained_coordinator_groups.py | 35 ++++++++++++++--------- 3 files changed, 32 insertions(+), 16 deletions(-) diff --git a/src/agent_comms/typed_table.py b/src/agent_comms/typed_table.py index 8f6ead2774..bf0ca5e839 100644 --- a/src/agent_comms/typed_table.py +++ b/src/agent_comms/typed_table.py @@ -3229,7 +3229,7 @@ def retaining_rows( descriptor, staged = cls.stage_file(destination) os.close(descriptor) try: - with closing(cls.writer(staged, isolation_level=None)) as db: + with closing(cls.writer(staged, isolation_level=None, uri=True)) as db: cls.configure_connection(db) SQLiteDatabase.named(db, "main").limit_bytes(db, maximum_bytes) with PracticeRebuild(source).attached(db) as attached: diff --git a/tests/test_input_documents.py b/tests/test_input_documents.py index ec73e10676..bc6bbceb4c 100644 --- a/tests/test_input_documents.py +++ b/tests/test_input_documents.py @@ -1,6 +1,7 @@ """Typed saved boundaries and durable acceptance precede process-local scheduling.""" from agent_comms.repo import Checkout +from agent_comms.comms import wire import json import sqlite3 from contextlib import ExitStack, closing @@ -127,6 +128,9 @@ def _insert_raw(store: InputDispositions, key: InputKey, attempt: dict) -> None: def test_saved_discriminator_and_optional_notices_roundtrip_without_replay(tmp_path): goal_id = GoalId.mint() + comms = wire(tmp_path) + comms.registry.declare(Thread('worker', frozenset(), Checkout(tmp_path))) + entry = comms.registry.snapshot().entry('worker') store = InputDispositions(tmp_path) from agent_comms.text_digest import TextDigest @@ -156,7 +160,7 @@ def test_saved_discriminator_and_optional_notices_roundtrip_without_replay(tmp_p assert row.reviewed_for_goal(goal_id) assert FieldCodec.encode(row) == saved assert store.in_turn(TURN).rows == {OLD_KEY: row} - assert store.unresolved(frozenset({"worker"})).rows == {OLD_KEY: row} + assert store.unresolved((entry,)).rows == {OLD_KEY: row} assert not store.bind( row.key, claim_scope=1, @@ -446,6 +450,9 @@ def test_started_delivery_and_original_ingress_use_distinct_owned_relations( def test_reads_select_only_the_rows_their_index_names(tmp_path): + comms = wire(tmp_path) + comms.registry.declare(Thread('owner', frozenset(), Checkout(tmp_path))) + entry = comms.registry.snapshot().entry('owner') store = InputDispositions(tmp_path) for key, owner in ((A_KEY, "owner"), (B_KEY, "owner"), (C_KEY, "other")): assert store.record(key, seq=None, owner=owner, claim_scope=1, target=owner, text=str(key)) @@ -460,7 +467,7 @@ def test_reads_select_only_the_rows_their_index_names(tmp_path): A_KEY, turn_id=TURN, native_id=InputId("a" * 32), text="sent" ) assert store.in_turn(TURN).rows.keys() == {A_KEY} - assert store.unresolved(frozenset({"owner"})).rows.keys() == {B_KEY} + assert store.unresolved((entry,)).rows.keys() == {B_KEY} assert store.read((C_KEY, MISSING_KEY)).rows.keys() == {C_KEY} with closing(sqlite3.connect(store.path)) as connection: plans = [connection.execute("EXPLAIN QUERY PLAN SELECT * FROM input_disposition WHERE " + where, diff --git a/tests/test_retained_coordinator_groups.py b/tests/test_retained_coordinator_groups.py index b8accb6fc5..63d3eb65f9 100644 --- a/tests/test_retained_coordinator_groups.py +++ b/tests/test_retained_coordinator_groups.py @@ -3,6 +3,7 @@ from contextlib import AbstractContextManager, closing from dataclasses import dataclass, replace from pathlib import Path +from uuid import uuid4 import pytest @@ -16,7 +17,7 @@ from agent_comms.coordination_tables.assignments import WakeAssignment from agent_comms.coordination_tables.harness_deliveries import HarnessDelivery, HarnessObservedPart from agent_comms.coordination_tables.executions import ( - AcpOrigin, CurrentExecutions, ExecutionRecord, NoPointedAttempt, NoPointedExecution, + AcpOrigin, CurrentExecutions, ExecutionRecord, ) from agent_comms.coordination_tables.participants import OwnerGenerations, Participants from agent_comms.coordination_tables.recovery import RecoveryAudit @@ -25,13 +26,13 @@ from agent_comms.message_reference import MessageReference from agent_comms.native_input_record import FullNativeExecution, TriageNativeExecution from agent_comms.native_runtime_input import NativeRuntimeInput -from agent_comms.nominal.field_codec import AssignmentId, MessageId, ThreadLookup, TranscriptRecordId +from agent_comms.nominal.field_codec import AssignmentId, MessageId, ThreadLookup, TranscriptRecordIdText from agent_comms.owner_fence import prepare_fence_token from agent_comms.participant_store import ParticipantRows, ParticipantStore from agent_comms.reason_codes import GivenReasonCode, NoDetail from agent_comms.recovery_states import ModelStalledRecovery from agent_comms.triage_native_sources import TriageNativeSource -from agent_comms.typed_table import AttachedPracticeRebuild, TypedTable, UnallocatedRowId +from agent_comms.typed_table import AttachedPracticeRebuild, NoDelete, TypedTable, UnallocatedRowId from agent_comms.wake_policy import FullWake, MessageAudience from bus_record_fixture import ( FIRST_INPUT_ID, SECOND_INPUT_ID, THIRD_INPUT_ID, RECIPIENT_LOOKUP, prepare_native_comms, @@ -84,7 +85,7 @@ def harness_source(tmp_path: Path) -> Iterator[HarnessSource]: parts = tuple( HarnessObservedPart(delivery.bridge, delivery.recipient_lookup, delivery.message_id, number, 'Sender', f'original body {delivery.seq}/{number}', - TranscriptRecordId(f'original-record-{delivery.seq}-{number}'), + TranscriptRecordIdText.decode(str(uuid4())), 10.0 + delivery.seq + number / 10) for delivery, monitor in zip(deliveries, monitors, strict=True) for number in range(1, monitor.total + 1) @@ -269,12 +270,8 @@ def test_selected_participant_keeps_its_generation_and_current_pointer( assert CoordinationStore.digest(source.path) == before -@pytest.mark.parametrize('surviving', ( - OwnerGenerations(owner_lookup=ThreadLookup('7' * 32), generation=3), - CurrentExecutions(owner_lookup=ThreadLookup('7' * 32), - execution_id=NoPointedExecution(), attempt_ordinal=NoPointedAttempt(), - pointer_revision=5), -)) +@pytest.mark.parametrize('missing', (OwnerGenerations, CurrentExecutions), + ids=('missing-generation', 'missing-pointer')) @pytest.mark.parametrize('retain_seed', ( lambda original, delivery: ParticipantStore.retain_owners( original, (delivery.recipient_lookup,), @@ -282,7 +279,7 @@ def test_selected_participant_keeps_its_generation_and_current_pointer( lambda original, delivery: delivery.retain_original(original), ), ids=('direct-owner', 'fk-discovered-owner')) def test_incomplete_original_participant_refuses_selected_publication( - native_group_source: NativeGroupSource, tmp_path: Path, surviving: TypedTable, + native_group_source: NativeGroupSource, tmp_path: Path, missing: type[TypedTable], retain_seed: Callable[[AttachedPracticeRebuild, HarnessDelivery], int], ) -> None: source = native_group_source @@ -290,10 +287,22 @@ def test_incomplete_original_participant_refuses_selected_publication( delivery = HarnessDelivery( ClaudeBridge, lookup, MessageId.mint('incomplete-owner-message'), 7, EmittedOutcome, ) + source.store.participants.register(lookup, committed=True) + source.store.participants.get(lookup) # The original aggregate starts valid. with source.store.session.writing() as db: - Participants(participant_lookup=lookup, committed=True).insert(db) - surviving.insert(db) delivery.insert(db) + # Explicit foreign corruption, not an admitted coordinator transition: + # remove only this type's delete guard, leaving insertion membership intact, + # then restore the exact declared schema before the retained reader sees it. + delete_guards = NoDelete.mutation_guards(missing) + for guard in delete_guards: + db.execute(f'DROP TRIGGER "{guard.name}"') + missing.delete(db, owner_lookup=lookup) + for guard in delete_guards: + db.execute(guard.sql) + CoordinationStore.table_schema().require_schema(db) + with pytest.raises(ParticipantNotRegistered): + source.store.participants.get(lookup) before = CoordinationStore.digest(source.path) destination = tmp_path / 'incomplete-participant' / 'coordination.sqlite3' with pytest.raises(ParticipantNotRegistered): From 67b587c185848811ac88702a72b9f9c543876fbe Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sun, 1 Nov 2026 23:56:02 -0500 Subject: [PATCH 22/28] Compare selected copies through their original schema and read owners The75-case891bc2 integration result is70passed/5failed/rc1. Four corruption controls asked only the base coordinator schema to validate a private root; use schema_at(source.path). The context control compared a restored read with a writer object before contributor JournalProvenance restoration; compare the complete original-store resolve at the selected source/record with the complete retained resolve. All coordinates, guard restoration, refusals, digest and allocation checks remain. Two test modules parsed and compiled through NRA, zero omissions, rc0. No product edits or author Core/SQL/pytest/provider execution. Both failed epochs retained; corrected behavior and original-data/native/physical acceptance remain unverified. --- tests/test_context_manifest_deltas.py | 6 +++++- tests/test_retained_coordinator_groups.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_context_manifest_deltas.py b/tests/test_context_manifest_deltas.py index d0f2bf53ae..3ba487a366 100644 --- a/tests/test_context_manifest_deltas.py +++ b/tests/test_context_manifest_deltas.py @@ -229,6 +229,10 @@ def test_selected_context_copy_preserves_closure_and_original_sequences(tmp_path ) for turn, count in enumerate((1, 2, 4), start=1)) selected = originals[1] reference = ContextManifestReference(selected.source, selected.record_id) + with comms.contexts.snapshot() as db: + expected = comms.contexts.resolve( + db, reference.source, ContextRequest.require(db, id=reference.record_id), + ) before = ContextStore.digest(comms.contexts.path) copy = PracticeCopy(comms.root, tmp_path / 'retained') copy.include_contexts((), requests=(reference, reference), maximum_bytes=2**20) @@ -239,7 +243,7 @@ def test_selected_context_copy_preserves_closure_and_original_sequences(tmp_path assert info.source == selected.source rows = ContextRequest.select(db, order_by=('id',)) assert tuple(row.id for row in rows) == tuple(row.record_id for row in originals[:2]) - assert retained.resolve(db, info.source, rows[-1]).value == selected.value + assert retained.resolve(db, info.source, rows[-1]) == expected assert ContextEntry.count(db) == 2 retained_version = max(row.id for row in ContextSegmentVersion.select(db)) with comms.contexts.snapshot() as db: diff --git a/tests/test_retained_coordinator_groups.py b/tests/test_retained_coordinator_groups.py index 63d3eb65f9..a4f1f9d3b6 100644 --- a/tests/test_retained_coordinator_groups.py +++ b/tests/test_retained_coordinator_groups.py @@ -300,7 +300,7 @@ def test_incomplete_original_participant_refuses_selected_publication( missing.delete(db, owner_lookup=lookup) for guard in delete_guards: db.execute(guard.sql) - CoordinationStore.table_schema().require_schema(db) + CoordinationStore.schema_at(source.path).require_schema(db) with pytest.raises(ParticipantNotRegistered): source.store.participants.get(lookup) before = CoordinationStore.digest(source.path) From 4066a48a7e863110500b7cdc1c6e38dff09a605d Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Mon, 2 Nov 2026 01:45:47 -0500 Subject: [PATCH 23/28] Capture selected inputs inside the existing retargeted hold PracticeCopy pins selected source groups and drains each admitted writer separately before acquiring the original registry cut. No SQL lock is held while seeking another or the registry. Existing maintenance members own target/holder admission and unchanged-generation refusal. Source observations reject holder-side writes before accepting the materialization. Complete include and backup behavior is unchanged. Authored13 real-owner controls for admission, each of the four source writers, original registry association, holder side effects, hold generations and failure cleanup. Three changed modules parse and compile without execution. Controlled-copy tests, resource peak, original-data/native/physical acceptance remain OPEN; no replay or live/donor mutation. --- src/agent_comms/installer/practice.py | 39 ++++- src/agent_comms/maintenance_barrier.py | 23 +++ tests/test_retained_capture.py | 207 +++++++++++++++++++++++++ 3 files changed, 268 insertions(+), 1 deletion(-) create mode 100644 tests/test_retained_capture.py diff --git a/src/agent_comms/installer/practice.py b/src/agent_comms/installer/practice.py index 5b7a067430..fcf8ce54f2 100644 --- a/src/agent_comms/installer/practice.py +++ b/src/agent_comms/installer/practice.py @@ -8,7 +8,8 @@ from dataclasses import dataclass, field from pathlib import Path from math import ceil -from collections.abc import Iterable +from collections.abc import Iterable, Iterator +from contextlib import contextmanager from agent_comms.bus import Bus from agent_comms.context_store import ContextManifestReference, ContextStore @@ -21,6 +22,8 @@ from agent_comms.native_session_reopen import NativeSessionIdentity from agent_comms.nominal.field_codec import boundary_decoder from agent_comms.installer.refusals import UpgradeRefused +from agent_comms.installer.backup import pinned_bus_files, saved_bus_files +from agent_comms.registry_store import RegistryCut, RegistryStore from agent_comms.installer.declared_uv import DeclaredUv from agent_comms.launch_runtime import LaunchRuntime from agent_comms.locked_store import ( @@ -181,6 +184,40 @@ def remaining_bytes(self, maximum_bytes: int) -> int: return min(PracticeMeasurement.capture(self.path).remaining(maximum_bytes), self.available_bytes) + @contextmanager + def capturing_inputs(self) -> Iterator[RegistryCut]: + """Enclose the first and every subsequent selected-input materialization. + + The caller already owns the install's retargeted hold and does no source + writes here. Pins prevent last-close WAL checkpoints, not writers. Drain + admitted transactions separately, releasing each SQL lock before seeking + another or the registry. The hold excludes later non-holder transactions; + source observations reject holder-side writes during the materialization. + Ordinary whole-file include() and complete backups keep their own lifetime. + This supplies a recorded cut, not native launch or write authority. + """ + if self.copied or self.path.exists(): + raise UpgradeRefused("Selected input capture must start before materialization") + barrier = MaintenanceBarrier(RegistryFile.path_in(self.root)) + files = LiveFilesRequest(self.root) + resources = RootFile.owned_files(files, tuple( + owner.path_in(self.root) + for owner in (Bus, InputDispositions, ContextStore, CoordinationStore) + )) + present = tuple(resource for resource in resources if files.location(resource).exists()) + with (barrier.capturing(LaunchRuntime.running()), + pinned_bus_files(files, present)): + for resource in present: + with resource.owner.writers_drained(files.location(resource)): + pass + with RegistryStore.at(self.root).current_snapshot() as original: + before = saved_bus_files(files, resources) + yield original + if not original.matches_current(): + raise UpgradeRefused("Registry changed during selected input capture") + if saved_bus_files(files, resources) != before: + raise UpgradeRefused("Source stores changed during selected input capture") + def include_contexts( self, identities: Iterable[NativeSessionIdentity], *, maximum_bytes: int, requests: Iterable[ContextManifestReference] = (), diff --git a/src/agent_comms/maintenance_barrier.py b/src/agent_comms/maintenance_barrier.py index 4de9d1f58b..8361c25c64 100644 --- a/src/agent_comms/maintenance_barrier.py +++ b/src/agent_comms/maintenance_barrier.py @@ -106,6 +106,9 @@ def selected[T](held: T, granted: T) -> T: class MaintenanceTarget(SubclassUnion): """A hold without a target grants no additional runtime attachment.""" + def require_capture(self, runtime: LaunchRuntime) -> None: + raise MaintenanceNotHeldError("Selected input capture needs a retargeted maintenance hold") + @abstractmethod def allows_runtime(self, runtime: LaunchRuntime) -> bool: ... @@ -130,6 +133,10 @@ class TargetRuntime(LaunchRuntime, RecordMember, MaintenanceTarget): def allows_runtime(self, runtime: LaunchRuntime) -> bool: return self.prefix == runtime.prefix + def require_capture(self, runtime: LaunchRuntime) -> None: + if not self.allows_runtime(runtime): + raise MaintenanceNotHeldError("Selected input capture needs the held target runtime") + def require_drained_writes_open_to(self, state: MaintenanceState, holder: MaintenanceHolder) -> None: """Retargeted once the outgoing runtime has stopped its owners: from then on only @@ -375,6 +382,9 @@ def require_held_by(self, holder: MaintenanceHolder, change: str) -> None: never enabled maintenance has no holder.""" raise MaintenanceNotHeldError.unheld(change) + def require_capture(self, runtime: LaunchRuntime) -> MaintenanceState: + raise MaintenanceNotHeldError.unheld("Selected input capture") + @dataclass(frozen=True, eq=False) class MaintenanceOperator(ProcessIdentity, declared_name=None): @@ -512,6 +522,11 @@ def require_drained_writes_open_to(self, holder: MaintenanceHolder) -> None: def require_held_by(self, holder: MaintenanceHolder, change: str) -> None: self.phase.require_held_by(self, holder, change) + def require_capture(self, runtime: LaunchRuntime) -> MaintenanceState: + self.require_held_by(MaintenanceHolder.current(), "Selected input capture") + self.target.require_capture(runtime) + return self + def require_draining(self) -> None: if self.phase is not DrainingPhase: raise ValueError('Only an acquired draining phase can be cleared') @@ -634,6 +649,14 @@ def assert_drained_writes_open_unlocked(self) -> None: def read(self) -> MaintenanceStatus: return self.current_unlocked() + @contextmanager + def capturing(self, runtime: LaunchRuntime) -> Iterator[MaintenanceState]: + """Borrow a current retargeted hold; never acquire, refresh or rewrite it.""" + held = self.read().require_capture(runtime) + yield held + if self.read().require_capture(runtime) != held: + raise MaintenanceNotHeldError("Maintenance changed during selected input capture") + def held_by(self, operator: MaintenanceOperator) -> MaintenanceState: """``operator``'s hold as the root records it now, in whatever phase it is in.""" state = self.read() diff --git a/tests/test_retained_capture.py b/tests/test_retained_capture.py new file mode 100644 index 0000000000..6a1671b976 --- /dev/null +++ b/tests/test_retained_capture.py @@ -0,0 +1,207 @@ +"""Selected capture borrows a real hold and drains real store transactions.""" +from collections.abc import Callable +from concurrent.futures import ThreadPoolExecutor +from contextlib import AbstractContextManager, nullcontext +from pathlib import Path +from threading import Event + +import pytest + +from agent_comms.bus import Bus +from agent_comms.comms import Comms, wire +from agent_comms.context_store import ContextStore +from agent_comms.coordination_database import CoordinationStore +from agent_comms.errors import RelationViolationError +from agent_comms.input_disposition import InputDispositions +from agent_comms.input_key import TurnInputKey +from agent_comms.installer.practice import PracticeCopy +from agent_comms.installer.refusals import UpgradeRefused +from agent_comms.launch_runtime import LaunchRuntime +from agent_comms.maintenance_barrier import ( + MaintenanceBarrier, MaintenanceNotHeldError, MaintenanceOperator, NoMaintenanceHolder, +) +from agent_comms.nominal.field_codec import TurnId +from agent_comms.repo import Checkout +from agent_comms.threads import Thread +from agent_comms.typed_table import WalStore +from bus_record_fixture import prepare_native_comms + + +type StoreOpening = Callable[[Path], AbstractContextManager[WalStore | CoordinationStore]] + + +@pytest.fixture +def capture_source(tmp_path: Path) -> Comms: + comms = prepare_native_comms(wire(tmp_path / 'source')) + comms.registry.declare(Thread('capture-owner', frozenset(), Checkout(tmp_path), created_at=17001.0)) + with comms.bus.bus.reading(), InputDispositions(comms.root).writing(), ContextStore(comms.root).writing(): + pass + with CoordinationStore(CoordinationStore.path_in(comms.root)): + pass + return comms + + +stores = pytest.mark.parametrize('opening', ( + pytest.param(lambda root: nullcontext(Bus(Bus.path_in(root))), id='bus'), + pytest.param(lambda root: nullcontext(InputDispositions(root)), id='ledger'), + pytest.param(lambda root: nullcontext(ContextStore(root)), id='contexts'), + pytest.param(lambda root: CoordinationStore(CoordinationStore.path_in(root)), id='coordinator'), +)) + + +def test_selected_capture_refuses_an_open_or_only_draining_root( + capture_source: Comms, tmp_path: Path, +) -> None: + source = capture_source + copy = PracticeCopy(source.root, tmp_path / 'selected') + gate = MaintenanceBarrier(source.registry.store.path) + with pytest.raises(MaintenanceNotHeldError), copy.capturing_inputs(): + pytest.fail('open root supplied a capture cut') + held = gate.begin(MaintenanceOperator.current()) + try: + with held.holding(), pytest.raises(MaintenanceNotHeldError), copy.capturing_inputs(): + pytest.fail('unretargeted drain supplied a capture cut') + finally: + gate.clear(held) + assert not copy.path.exists() + + +def test_selected_capture_requires_holder_and_actual_target_runtime( + capture_source: Comms, tmp_path: Path, +) -> None: + source = capture_source + copy = PracticeCopy(source.root, tmp_path / 'selected') + gate = MaintenanceBarrier(source.registry.store.path) + held = gate.retarget(gate.begin(MaintenanceOperator.current()), + LaunchRuntime(tmp_path / 'other-runtime/bin/python')) + try: + with held.holding(), pytest.raises(MaintenanceNotHeldError), copy.capturing_inputs(): + pytest.fail('another runtime supplied a capture cut') + held = gate.retarget(held, LaunchRuntime.running()) + with NoMaintenanceHolder().selected(), pytest.raises(MaintenanceNotHeldError), copy.capturing_inputs(): + pytest.fail('non-holder supplied a capture cut') + finally: + gate.clear(held) + assert not copy.path.exists() + + +@stores +def test_capture_waits_for_each_already_admitted_transaction( + capture_source: Comms, tmp_path: Path, opening: StoreOpening, +) -> None: + source = capture_source + admitted, release, started, captured = Event(), Event(), Event(), Event() + copy = PracticeCopy(source.root, tmp_path / 'selected') + gate = MaintenanceBarrier(source.registry.store.path) + + def writer() -> None: + with NoMaintenanceHolder().selected(), opening(source.root) as store, store.writing() as db: + db.execute('SELECT 1').fetchone() + admitted.set() + assert release.wait(5), 'controller did not release the admitted writer' + + with ThreadPoolExecutor(max_workers=2) as workers: + pending = workers.submit(writer) + try: + assert admitted.wait(5), 'writer did not enter its ordinary transaction' + held = gate.retarget(gate.begin(MaintenanceOperator.current()), LaunchRuntime.running()) + + def capture() -> None: + with held.holding(): + started.set() + with copy.capturing_inputs(): + captured.set() + + taking = workers.submit(capture) + assert started.wait(5) + assert not captured.wait(0.05), 'capture passed an admitted SQL writer' + release.set() + pending.result(timeout=5) + taking.result(timeout=5) + assert captured.is_set() + gate.clear(held) + finally: + release.set() + + +@stores +def test_capture_refuses_later_non_holder_store_writes( + capture_source: Comms, tmp_path: Path, opening: StoreOpening, +) -> None: + source = capture_source + gate = MaintenanceBarrier(source.registry.store.path) + held = gate.retarget(gate.begin(MaintenanceOperator.current()), LaunchRuntime.running()) + copy = PracticeCopy(source.root, tmp_path / 'selected') + try: + with held.holding(), copy.capturing_inputs(): + with NoMaintenanceHolder().selected(), pytest.raises(RelationViolationError): + with opening(source.root) as store, store.writing(): + pytest.fail('later non-holder writer entered the captured store') + finally: + gate.clear(held) + + +def test_capture_keeps_original_registry_association_through_first_copy( + capture_source: Comms, tmp_path: Path, +) -> None: + source = capture_source + expected = source.registry.snapshot().entry('capture-owner') + before = source.bus.bus.digest(source.bus.bus.path) + gate = MaintenanceBarrier(source.registry.store.path) + held = gate.retarget(gate.begin(MaintenanceOperator.current()), LaunchRuntime.running()) + copy = PracticeCopy(source.root, tmp_path / 'selected') + try: + with held.holding(), copy.capturing_inputs() as original: + assert original.snapshot.entry('capture-owner') == expected + assert original.matches_current() + copy.include((Bus.path_in(source.root), InputDispositions.path_in(source.root))) + assert Bus.digest(Bus.path_in(copy.path)) == before + assert source.registry.snapshot().entry('capture-owner') == expected + assert source.bus.bus.digest(source.bus.bus.path) == before + with held.holding(), pytest.raises(UpgradeRefused), copy.capturing_inputs(): + pytest.fail('capture started after its first source copy') + finally: + gate.clear(held) + + +def test_capture_rejects_a_holder_side_effect_instead_of_accepting_mixed_sources( + capture_source: Comms, tmp_path: Path, +) -> None: + source = capture_source + gate = MaintenanceBarrier(source.registry.store.path) + held = gate.retarget(gate.begin(MaintenanceOperator.current()), LaunchRuntime.running()) + ledger = InputDispositions(source.root) + key = TurnInputKey(TurnId('a' * 32)) + copy = PracticeCopy(source.root, tmp_path / 'selected') + try: + with held.holding(), pytest.raises(UpgradeRefused, match='Source stores changed'): + with copy.capturing_inputs(): + copy.include((ledger.path,)) + ledger.record(key, seq=None, owner='capture-owner', claim_scope=1, + target='capture-owner', text='holder wrote after copying') + assert key in ledger.read((key,)).rows + assert key not in InputDispositions(copy.path).read((key,)).rows + finally: + gate.clear(held) + + +def test_capture_borrows_one_hold_generation_and_releases_resources_on_failure( + capture_source: Comms, tmp_path: Path, +) -> None: + source = capture_source + gate = MaintenanceBarrier(source.registry.store.path) + runtime = LaunchRuntime.running() + held = gate.retarget(gate.begin(MaintenanceOperator.current()), runtime) + copy = PracticeCopy(source.root, tmp_path / 'selected') + try: + with held.holding(), pytest.raises(MaintenanceNotHeldError): + with gate.capturing(runtime): + held = gate.retarget(held, runtime) + with held.holding(), pytest.raises(RuntimeError, match='stop materialization'): + with copy.capturing_inputs(): + raise RuntimeError('stop materialization') + finally: + gate.clear(held) + with InputDispositions(source.root).writing(): + pass + assert not copy.path.exists() From 8e89d46cf62edc7966017f596ff2c296df27410e Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Mon, 2 Nov 2026 03:53:33 -0500 Subject: [PATCH 24/28] Drop the source write claim during selected materialization Use existing NoMaintenanceHolder.acting inside the captured registry cut. Ordinary source writes now refuse before mutation, and the private target remains writable. Restore the original claim on every exit. Before/after source observations remain an additional fence. The declared target-only SelectedInputsProbe calls the complete selected operation before first include and through context/coordinator selection. Complete reader preflight refuses oversized groups; ordinary copies and backups remain unchanged. Exercise the decoded probe request and all four stores, donor-contained destination refusal and no-copy budget refusal. The preceding122-case execution is114passed/8failed/rc1. Seven cases met the real free-space guard on /tmp; exactly8affected cases passed in2.89s on fresh owned /home basetemp after fixing the stopped-thread oracle against unregister. Those are separate retained epochs, not122/122. Five correction modules compiled without executing subjects. New claim-scoped controls and original-data/native/physical acceptance remain OPEN. --- src/agent_comms/installer/practice.py | 55 ++++++++++++++++++---- src/agent_comms/installer/probes/inputs.py | 35 ++++++++++++++ tests/test_maintenance_barrier.py | 3 +- tests/test_retained_annotations.py | 42 ++++++++++++++++- tests/test_retained_capture.py | 30 +++++++++++- 5 files changed, 151 insertions(+), 14 deletions(-) create mode 100644 src/agent_comms/installer/probes/inputs.py diff --git a/src/agent_comms/installer/practice.py b/src/agent_comms/installer/practice.py index fcf8ce54f2..6eca546e1a 100644 --- a/src/agent_comms/installer/practice.py +++ b/src/agent_comms/installer/practice.py @@ -10,6 +10,7 @@ from math import ceil from collections.abc import Iterable, Iterator from contextlib import contextmanager +from typing import ClassVar from agent_comms.bus import Bus from agent_comms.context_store import ContextManifestReference, ContextStore @@ -20,7 +21,7 @@ from agent_comms.participant_store import ParticipantRows, ParticipantStore from agent_comms.registry_document import RegistryEntry from agent_comms.native_session_reopen import NativeSessionIdentity -from agent_comms.nominal.field_codec import boundary_decoder +from agent_comms.nominal.field_codec import ThreadLookup, boundary_decoder from agent_comms.installer.refusals import UpgradeRefused from agent_comms.installer.backup import pinned_bus_files, saved_bus_files from agent_comms.registry_store import RegistryCut, RegistryStore @@ -29,7 +30,7 @@ from agent_comms.locked_store import ( DeclaredFile, LiveFilesRequest, RegistryFile, RetainedCopyRequest, RootFile, UndeclaredFile, ) -from agent_comms.maintenance_barrier import MaintenanceBarrier +from agent_comms.maintenance_barrier import MaintenanceBarrier, NoMaintenanceHolder from agent_comms.storage_rules import StorageRule from agent_comms.private_path import PrivateDirectoryRole @@ -148,6 +149,7 @@ class PracticeCopy: root: Path path: Path copied: dict[Path, DeclaredFile | UndeclaredFile] = field(default_factory=dict) + input_readers: ClassVar[tuple[type[RootFile], ...]] = (Bus, InputDispositions) @property def request(self) -> RetainedCopyRequest: @@ -188,21 +190,26 @@ def remaining_bytes(self, maximum_bytes: int) -> int: def capturing_inputs(self) -> Iterator[RegistryCut]: """Enclose the first and every subsequent selected-input materialization. - The caller already owns the install's retargeted hold and does no source - writes here. Pins prevent last-close WAL checkpoints, not writers. Drain + The caller already owns the install's retargeted hold. Materialization + drops its claim, including the exported claim inherited by new children, + so ordinary source writers refuse while the private target stays writable. + Pins prevent last-close WAL checkpoints, not writers. Drain admitted transactions separately, releasing each SQL lock before seeking another or the registry. The hold excludes later non-holder transactions; - source observations reject holder-side writes during the materialization. + source observations additionally reject changes outside that writer path. Ordinary whole-file include() and complete backups keep their own lifetime. This supplies a recorded cut, not native launch or write authority. """ - if self.copied or self.path.exists(): + if self.copied or self.path.exists() or self.path.is_symlink(): raise UpgradeRefused("Selected input capture must start before materialization") + source, target = self.root.resolve(), self.path.resolve() + if target.is_relative_to(source) or source.is_relative_to(target): + raise UpgradeRefused("Selected input capture requires a separate destination") barrier = MaintenanceBarrier(RegistryFile.path_in(self.root)) files = LiveFilesRequest(self.root) resources = RootFile.owned_files(files, tuple( owner.path_in(self.root) - for owner in (Bus, InputDispositions, ContextStore, CoordinationStore) + for owner in (*self.input_readers, ContextStore, CoordinationStore) )) present = tuple(resource for resource in resources if files.location(resource).exists()) with (barrier.capturing(LaunchRuntime.running()), @@ -212,12 +219,42 @@ def capturing_inputs(self) -> Iterator[RegistryCut]: pass with RegistryStore.at(self.root).current_snapshot() as original: before = saved_bus_files(files, resources) - yield original + with NoMaintenanceHolder().acting(): + yield original if not original.matches_current(): raise UpgradeRefused("Registry changed during selected input capture") if saved_bus_files(files, resources) != before: raise UpgradeRefused("Source stores changed during selected input capture") + def capture_inputs( + self, owners: tuple[ThreadLookup, ...], *, maximum_bytes: int, + identities: tuple[NativeSessionIdentity, ...] = (), + requests: tuple[ContextManifestReference, ...] = (), + ) -> PracticeMeasurement: + """The selected operation, including its first complete reader copy. + + Resolve births from the original cut, not caller-supplied name/generation + records. No selection callback runs before or outside this lifetime. + The returned measurement describes data, never native launch authority. + """ + with self.capturing_inputs() as original: + entries = tuple(original.snapshot.entry(original.snapshot.born(owner).name) + for owner in owners) + self.prepare() + paths = tuple(owner.path_in(self.root) for owner in self.input_readers) + files = LiveFilesRequest(self.root) + readers = saved_bus_files(files, RootFile.owned_files(files, paths)) + required = sum(member.size for group in readers.values() for member in group.values()) + if required > self.remaining_bytes(maximum_bytes): + raise UpgradeRefused("Complete input readers exceed the selected copy allowance") + self.include(paths) + self.remaining_bytes(maximum_bytes) + self.include_contexts(identities, maximum_bytes=maximum_bytes, requests=requests) + self.include_inputs(entries, maximum_bytes=maximum_bytes, identities=identities) + measured = PracticeMeasurement.capture(self.path) + measured.remaining(maximum_bytes) + return measured + def include_contexts( self, identities: Iterable[NativeSessionIdentity], *, maximum_bytes: int, requests: Iterable[ContextManifestReference] = (), @@ -252,7 +289,7 @@ def include_inputs( one consistent cut and serial materialization, not native launch authority. """ self.prepare() - for owner in (Bus, InputDispositions): + for owner in self.input_readers: source = owner.path_in(self.root) if source.exists(): if source not in self.copied: diff --git a/src/agent_comms/installer/probes/inputs.py b/src/agent_comms/installer/probes/inputs.py new file mode 100644 index 0000000000..6b002a7387 --- /dev/null +++ b/src/agent_comms/installer/probes/inputs.py @@ -0,0 +1,35 @@ +"""Selected original inputs captured by the qualified install worker.""" +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Annotated + +from agent_comms.context_store import ContextManifestReference +from agent_comms.installer.interpreter_operation import RuntimeProbe, TargetProbe +from agent_comms.installer.practice import PracticeCopy, PracticeMeasurement +from agent_comms.native_session_reopen import NativeSessionIdentity +from agent_comms.nominal.field_codec import PathText, ThreadLookup + + +@dataclass(frozen=True) +class SelectedInputsProbe(TargetProbe, RuntimeProbe[PracticeMeasurement]): + """Explicit selected capture in the target worker's existing retargeted hold. + + Not the ordinary complete rehearsal, a new hold/stop operation, or native + launch authority. Original coordinates and the allowance belong to this + request; no supplied callback runs outside the captured source lifetime. + """ + + root: Annotated[Path, PathText] + destination: Annotated[Path, PathText] + maximum_bytes: int + owners: tuple[ThreadLookup, ...] = () + identities: tuple[NativeSessionIdentity, ...] = () + requests: tuple[ContextManifestReference, ...] = () + + def run(self) -> PracticeMeasurement: + return PracticeCopy(self.root, self.destination).capture_inputs( + self.owners, maximum_bytes=self.maximum_bytes, + identities=self.identities, requests=self.requests, + ) diff --git a/tests/test_maintenance_barrier.py b/tests/test_maintenance_barrier.py index 33f3992742..e997047b44 100644 --- a/tests/test_maintenance_barrier.py +++ b/tests/test_maintenance_barrier.py @@ -252,12 +252,13 @@ def test_rename_and_stopped_same_pid_cannot_reactivate_under_gate(tmp_path: Path comms.registry.declare(owner) comms.registry.rename("owner", "renamed") comms.registry.unregister("renamed") + stopped = comms.registry.snapshot().entry("renamed") comms.owners.maintenance.begin(MaintenanceOperator.current()) with pytest.raises(RelationViolationError): comms.owners.start_many(("owner",)) with pytest.raises(RelationViolationError): comms.registry.register(comms.registry.require("renamed"), new_owner=True) - assert comms.registry.require("renamed").same_local_process(owner.require_process()) + assert comms.registry.snapshot().entry("renamed") == stopped def test_cross_process_claim_races_pause_at_registry_lock(tmp_path: Path) -> None: diff --git a/tests/test_retained_annotations.py b/tests/test_retained_annotations.py index 9e8b433e12..d75d37fec0 100644 --- a/tests/test_retained_annotations.py +++ b/tests/test_retained_annotations.py @@ -15,9 +15,13 @@ from agent_comms.coordinator import StandaloneCoordination from agent_comms.input_disposition import InputDispositions from agent_comms.installer.refusals import UpgradeRefused -from agent_comms.installer.practice import PracticeCopy +from agent_comms.installer.practice import PracticeCopy, PracticeMeasurement +from agent_comms.installer.interpreter_operation import ProbeRequest +from agent_comms.installer.probes.inputs import SelectedInputsProbe +from agent_comms.launch_runtime import LaunchRuntime +from agent_comms.maintenance_barrier import MaintenanceBarrier, MaintenanceOperator from agent_comms.message_reference import MessageReference -from agent_comms.nominal.field_codec import MessageId, TurnId +from agent_comms.nominal.field_codec import FieldCodec, MessageId, TurnId from agent_comms.repo import Checkout from agent_comms.task_sources import AnnotationDisclosureGrant, ProjectTaskScope from agent_comms.thread_identity import RecordedThreadIncarnation, TurnIdentity @@ -212,3 +216,37 @@ def test_context_selection_cannot_follow_finalized_coordinator_evidence( copy.include_contexts((), requests=(source.reference,), maximum_bytes=8 * 2**20) assert not ContextStore(copy.path).path.exists() assert CoordinationStore.digest(source.path) == before + + +def test_selected_annotation_materialization_writes_only_the_private_target( + annotation_source: AnnotationSource, tmp_path: Path, +) -> None: + source = annotation_source + root = source.path.parent + before = CoordinationStore.digest(source.path) + contexts_before = ContextStore.digest(ContextStore.path_in(root)) + expected = WorkingMemoryAnnotations.for_context( + ReadableStore(source.path), (source.context.value,), JevClassifier.version(), + ) + comms = wire(root) + gate = MaintenanceBarrier(comms.registry.store.path) + held = gate.retarget(gate.begin(MaintenanceOperator.current()), LaunchRuntime.running()) + copy = PracticeCopy(root, tmp_path / 'captured-annotations') + try: + request = ProbeRequest(SelectedInputsProbe( + root, copy.path, 8 * 2**20, requests=(source.reference,), + ), str(held.nonce)) + decoded = FieldCodec.decode(ProbeRequest, FieldCodec.encode(request)) + with held.holding(): + reply = decoded.answer() + measured = FieldCodec.decode(PracticeMeasurement, reply.answer) + assert measured == PracticeMeasurement.capture(copy.path) + contexts = ContextStore(copy.path).original_segments() + retained = WorkingMemoryAnnotations.for_segments( + ReadableStore(CoordinationStore.path_in(copy.path)), contexts, JevClassifier.version(), + ) + assert retained == expected + assert CoordinationStore.digest(source.path) == before + assert ContextStore.digest(ContextStore.path_in(root)) == contexts_before + finally: + gate.clear(held) diff --git a/tests/test_retained_capture.py b/tests/test_retained_capture.py index 6a1671b976..50a13f902c 100644 --- a/tests/test_retained_capture.py +++ b/tests/test_retained_capture.py @@ -16,6 +16,7 @@ from agent_comms.input_key import TurnInputKey from agent_comms.installer.practice import PracticeCopy from agent_comms.installer.refusals import UpgradeRefused +from agent_comms.installer.probes.inputs import SelectedInputsProbe from agent_comms.launch_runtime import LaunchRuntime from agent_comms.maintenance_barrier import ( MaintenanceBarrier, MaintenanceNotHeldError, MaintenanceOperator, NoMaintenanceHolder, @@ -41,6 +42,31 @@ def capture_source(tmp_path: Path) -> Comms: return comms +def test_capture_refuses_a_destination_inside_its_source( + capture_source: Comms, +) -> None: + copy = PracticeCopy(capture_source.root, capture_source.root / 'nested-copy') + with pytest.raises(UpgradeRefused, match='separate destination'), copy.capturing_inputs(): + pytest.fail('capture accepted a destination inside its donor') + assert not copy.path.exists() + + +def test_selected_probe_refuses_oversized_complete_readers_before_copying( + capture_source: Comms, tmp_path: Path, +) -> None: + source = capture_source + gate = MaintenanceBarrier(source.registry.store.path) + held = gate.retarget(gate.begin(MaintenanceOperator.current()), LaunchRuntime.running()) + destination = tmp_path / 'too-small' + try: + with held.holding(), pytest.raises(UpgradeRefused, match='Complete input readers'): + SelectedInputsProbe(source.root, destination, 1).run() + assert not Bus.path_in(destination).exists() + assert not InputDispositions.path_in(destination).exists() + finally: + gate.clear(held) + + stores = pytest.mark.parametrize('opening', ( pytest.param(lambda root: nullcontext(Bus(Bus.path_in(root))), id='bus'), pytest.param(lambda root: nullcontext(InputDispositions(root)), id='ledger'), @@ -174,12 +200,12 @@ def test_capture_rejects_a_holder_side_effect_instead_of_accepting_mixed_sources key = TurnInputKey(TurnId('a' * 32)) copy = PracticeCopy(source.root, tmp_path / 'selected') try: - with held.holding(), pytest.raises(UpgradeRefused, match='Source stores changed'): + with held.holding(), pytest.raises(RelationViolationError): with copy.capturing_inputs(): copy.include((ledger.path,)) ledger.record(key, seq=None, owner='capture-owner', claim_scope=1, target='capture-owner', text='holder wrote after copying') - assert key in ledger.read((key,)).rows + assert key not in ledger.read((key,)).rows assert key not in InputDispositions(copy.path).read((key,)).rows finally: gate.clear(held) From f03454233836adfb38ef84fb43f0b583b0040c46 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Mon, 2 Nov 2026 04:25:03 -0500 Subject: [PATCH 25/28] Derive selected capture membership from file capabilities --- docs/retained-practice-inputs.md | 22 ++++++++++++++++++++++ src/agent_comms/bus.py | 9 +++++++-- src/agent_comms/context_store.py | 3 ++- src/agent_comms/coordination_database.py | 4 ++-- src/agent_comms/input_disposition.py | 4 ++-- src/agent_comms/installer/practice.py | 22 ++++++++++------------ src/agent_comms/locked_store.py | 12 ++++++++++++ 7 files changed, 57 insertions(+), 19 deletions(-) diff --git a/docs/retained-practice-inputs.md b/docs/retained-practice-inputs.md index d18eb08eed..74816687a9 100644 --- a/docs/retained-practice-inputs.md +++ b/docs/retained-practice-inputs.md @@ -473,3 +473,25 @@ aggregate capture and peak budget, then the combined backend/carry and actual prepared-root saved-history journey. No pytest, provider, donor mutation or new native/physical run was performed for this source checkpoint. Reader promotion still requires the separate default-storage and decorator creation-cut proofs. + +## Selected capture file capabilities + +The selected operation now derives its files through `RootFile.declarations_in`. +`InputCaptureFile` identifies each store participating in the common cut; +`CompleteInputReader(InputCaptureFile)` identifies stores copied whole before +context/coordinator selection. Bus and InputDispositions compose the latter; +ContextStore and CoordinationStore compose the former. Their existing storage, +writer, schema and copy behavior remains on their original parents. + +The `PracticeCopy.input_readers` roster is deleted. Capture pinning/draining, +complete-reader preflight/copy and missing-original-reader checks use the declared +capabilities. A new reader joins through its declaration rather than a second +list. This also removes the heterogeneous tuple that mypy inferred as `object`. + +The changed file-family bases and traversal were executed in the actual capture +and annotation tests: 19 collected, 19 passed, 4.97 seconds, exit0, using the host +TestLease and a fresh owned `/home/ts/wt` basetemp. This includes each original +store's in-flight/later writer controls and the decoded selected ProbeRequest. +Focused mypy on installer/practice.py passed. These results remain fixture/backend +qualification, not original-data, cross-runtime, resource-peak or native/physical +acceptance. The earlier 125-case execution belongs to the preceding source epoch. diff --git a/src/agent_comms/bus.py b/src/agent_comms/bus.py index 2fe9e49b3e..29144ffd70 100644 --- a/src/agent_comms/bus.py +++ b/src/agent_comms/bus.py @@ -50,7 +50,9 @@ ) from agent_comms.errors import CommittedMessageError, RelationViolationError, UnclaimedError from agent_comms.nominal.field_codec import ClaimId, FieldCodec, MessageId, ThreadLookup -from agent_comms.locked_store import BusFile, FileStore, SQLiteReader, StoreCommittedError +from agent_comms.locked_store import ( + BusFile, CompleteInputReader, FileStore, SQLiteReader, StoreCommittedError, +) from agent_comms.message_reference import MessageReference from agent_comms.private_path import FileIdentity, PresentPath from agent_comms.read_basis import CursorRow @@ -514,7 +516,10 @@ def observing(self) -> Iterator[sqlite3.Connection]: yield db -class Bus(FileStore, StoredBusReader, ReconciledFile, WalStore, CarriedWriter, BusFile, Watched): +class Bus( + FileStore, StoredBusReader, ReconciledFile, WalStore, CarriedWriter, BusFile, + CompleteInputReader, Watched, +): """One SQLite transaction owns the sequence and all of a record's fields.""" filename = "bus.sqlite3" diff --git a/src/agent_comms/context_store.py b/src/agent_comms/context_store.py index 6af54cd98e..8c7fb8574d 100644 --- a/src/agent_comms/context_store.py +++ b/src/agent_comms/context_store.py @@ -14,6 +14,7 @@ from typing import Annotated, ClassVar, Self from .errors import RelationViolationError, StoredHistoryUnreadable +from .locked_store import InputCaptureFile from .nominal.field_codec import ContextStoreId, FieldCodec, MalformedRecord, NullMember from .native_session_reopen import NativeSessionIdentity from .registration import Registration @@ -479,7 +480,7 @@ def reference(self, manifest: ContextManifest) -> ContextManifestReference: @dataclass(frozen=True) -class ContextStore(DurableStore): +class ContextStore(DurableStore, InputCaptureFile): root: Path filename: ClassVar[str] = "turn_context.sqlite3" tables = ContextStoreRows diff --git a/src/agent_comms/coordination_database.py b/src/agent_comms/coordination_database.py index fac2ad4701..f99b06e5b9 100644 --- a/src/agent_comms/coordination_database.py +++ b/src/agent_comms/coordination_database.py @@ -15,7 +15,7 @@ from agent_comms.coordination_errors import CoordinationReadUnavailable, IntegrityViolationError from agent_comms.coordination_schema import CoordinatorTable -from agent_comms.locked_store import FileStore, SQLiteReader +from agent_comms.locked_store import FileStore, InputCaptureFile, SQLiteReader from agent_comms.typed_table import ( CarriedWriter, ReconciledFile, @@ -66,7 +66,7 @@ def _capture(self, consumer, *arguments): yield consumer.capture(connection, *arguments) -class CoordinationStore(FileStore, CarriedWriter, ReconciledFile): +class CoordinationStore(FileStore, CarriedWriter, ReconciledFile, InputCaptureFile): """The private coordinator database and its declared schema.""" filename = "coordination.sqlite3" diff --git a/src/agent_comms/input_disposition.py b/src/agent_comms/input_disposition.py index 984a936c8f..0719a385c2 100644 --- a/src/agent_comms/input_disposition.py +++ b/src/agent_comms/input_disposition.py @@ -33,7 +33,7 @@ from .input_key import InputKey, InputKeyText, TurnInputKey from .input_origin import HarnessAttemptOrigin, InputOrigin, UnattributedInputOrigin from .lifecycle_capability import Unresolved -from .locked_store import DocumentChange, PublishedProperty +from .locked_store import CompleteInputReader, DocumentChange, PublishedProperty from .message_reference import MessageReference from .messages import MessageInput from .native_input_proof import NativeContextProof @@ -248,7 +248,7 @@ def _members(column: str, values) -> tuple[str, tuple]: @dataclass(frozen=True, slots=True) -class InputDispositions(DurableStore): +class InputDispositions(DurableStore, CompleteInputReader): """Indexed durable input rows; each operation reads only the rows it names.""" root: Path diff --git a/src/agent_comms/installer/practice.py b/src/agent_comms/installer/practice.py index 6eca546e1a..db36562689 100644 --- a/src/agent_comms/installer/practice.py +++ b/src/agent_comms/installer/practice.py @@ -10,13 +10,10 @@ from math import ceil from collections.abc import Iterable, Iterator from contextlib import contextmanager -from typing import ClassVar - -from agent_comms.bus import Bus from agent_comms.context_store import ContextManifestReference, ContextStore from agent_comms.coordination_database import CoordinationStore from agent_comms.coordination_tables.annotations import ContextSpanRows -from agent_comms.input_disposition import InputDispositions, InputDocument, SnapshotInputDispositions +from agent_comms.input_disposition import InputDocument, SnapshotInputDispositions from agent_comms.native_runtime_input import NativeRuntimeInput from agent_comms.participant_store import ParticipantRows, ParticipantStore from agent_comms.registry_document import RegistryEntry @@ -28,7 +25,8 @@ from agent_comms.installer.declared_uv import DeclaredUv from agent_comms.launch_runtime import LaunchRuntime from agent_comms.locked_store import ( - DeclaredFile, LiveFilesRequest, RegistryFile, RetainedCopyRequest, RootFile, UndeclaredFile, + CompleteInputReader, DeclaredFile, InputCaptureFile, LiveFilesRequest, RegistryFile, + RetainedCopyRequest, RootFile, UndeclaredFile, ) from agent_comms.maintenance_barrier import MaintenanceBarrier, NoMaintenanceHolder from agent_comms.storage_rules import StorageRule @@ -149,7 +147,6 @@ class PracticeCopy: root: Path path: Path copied: dict[Path, DeclaredFile | UndeclaredFile] = field(default_factory=dict) - input_readers: ClassVar[tuple[type[RootFile], ...]] = (Bus, InputDispositions) @property def request(self) -> RetainedCopyRequest: @@ -194,7 +191,7 @@ def capturing_inputs(self) -> Iterator[RegistryCut]: drops its claim, including the exported claim inherited by new children, so ordinary source writers refuse while the private target stays writable. Pins prevent last-close WAL checkpoints, not writers. Drain - admitted transactions separately, releasing each SQL lock before seeking + in-flight transactions separately, releasing each SQL lock before seeking another or the registry. The hold excludes later non-holder transactions; source observations additionally reject changes outside that writer path. Ordinary whole-file include() and complete backups keep their own lifetime. @@ -208,8 +205,8 @@ def capturing_inputs(self) -> Iterator[RegistryCut]: barrier = MaintenanceBarrier(RegistryFile.path_in(self.root)) files = LiveFilesRequest(self.root) resources = RootFile.owned_files(files, tuple( - owner.path_in(self.root) - for owner in (*self.input_readers, ContextStore, CoordinationStore) + owner.path_in(directory) + for owner, directory in InputCaptureFile.declarations_in(self.root) )) present = tuple(resource for resource in resources if files.location(resource).exists()) with (barrier.capturing(LaunchRuntime.running()), @@ -241,7 +238,8 @@ def capture_inputs( entries = tuple(original.snapshot.entry(original.snapshot.born(owner).name) for owner in owners) self.prepare() - paths = tuple(owner.path_in(self.root) for owner in self.input_readers) + paths = tuple(owner.path_in(directory) for owner, directory + in CompleteInputReader.declarations_in(self.root)) files = LiveFilesRequest(self.root) readers = saved_bus_files(files, RootFile.owned_files(files, paths)) required = sum(member.size for group in readers.values() for member in group.values()) @@ -289,8 +287,8 @@ def include_inputs( one consistent cut and serial materialization, not native launch authority. """ self.prepare() - for owner in self.input_readers: - source = owner.path_in(self.root) + for owner, directory in CompleteInputReader.declarations_in(self.root): + source = owner.path_in(directory) if source.exists(): if source not in self.copied: raise UpgradeRefused(f"Capture complete {source.name} before selecting input evidence") diff --git a/src/agent_comms/locked_store.py b/src/agent_comms/locked_store.py index 655566d983..6820c47fb0 100644 --- a/src/agent_comms/locked_store.py +++ b/src/agent_comms/locked_store.py @@ -693,6 +693,18 @@ def fresh(cls, root: Path) -> bool: ) +class InputCaptureFile(RootFile): + """A store participating in the original input capture's common source cut.""" + + __slots__ = () + + +class CompleteInputReader(InputCaptureFile): + """An input reader retained whole before selecting context and coordinator rows.""" + + __slots__ = () + + class WireWakeFile(RootFile): """Declared commit signals wake idle readers, unlike lock-file churn.""" From 7a2b1f5c791953f40127f7ff71804445e06d5b54 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Mon, 2 Nov 2026 04:47:47 -0500 Subject: [PATCH 26/28] Exercise selected capture through the real target interpreter worker --- docs/retained-practice-inputs.md | 12 ++++++++++++ stack/uv.lock | 4 ++-- tests/test_retained_annotations.py | 16 ++++++++-------- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/docs/retained-practice-inputs.md b/docs/retained-practice-inputs.md index 74816687a9..aa8e5cf3ce 100644 --- a/docs/retained-practice-inputs.md +++ b/docs/retained-practice-inputs.md @@ -495,3 +495,15 @@ store's in-flight/later writer controls and the decoded selected ProbeRequest. Focused mypy on installer/practice.py passed. These results remain fixture/backend qualification, not original-data, cross-runtime, resource-peak or native/physical acceptance. The earlier 125-case execution belongs to the preceding source epoch. + +A separate candidate build completed through `RuntimeStack.build/complete`, +including native preparation and client imports, without publishing a live route. +The selected annotation control now uses `InstallCommands.probe` and the real +interpreter worker pipes instead of calling `ProbeRequest.answer` in-process. +Executed through the existing `RuntimeStack.test` owner against that installed +candidate: one collected, one passed, 5.45 seconds, exit0. The outer command took +70.468 seconds; this is not a 5.45-second end-to-end timing. The declared dev extra +was installed into the existing candidate prefix by that test owner. Its package +path, resolved lock and build phases are retained in the owning build record. +This proves the worker transport on private fixture data, not the original-data +capture, aggregate resource peak or native-agent/physical journey. diff --git a/stack/uv.lock b/stack/uv.lock index 2f261b496e..86535f7dcc 100644 --- a/stack/uv.lock +++ b/stack/uv.lock @@ -210,7 +210,7 @@ wheels = [ [[package]] name = "batrachian-toad" version = "0.6.20" -source = { git = "https://github.com/OpenHCSDev/toad.git?branch=main#7805183bf36410b82c1c1b1e327d077c3458634c" } +source = { git = "https://github.com/OpenHCSDev/toad.git?branch=main#9e0452dca33f52f970da7740becc31706be997a2" } dependencies = [ { name = "agent-client-protocol" }, { name = "agent-comms" }, @@ -1029,7 +1029,7 @@ wheels = [ [[package]] name = "textual" version = "8.2.8" -source = { git = "https://github.com/OpenHCSDev/textual.git?branch=main#7c62867cce97d2b0c521c869750413132a315a77" } +source = { git = "https://github.com/OpenHCSDev/textual.git?branch=main#94d743f57df21d80141d0e6aad0be8818e330ac8" } dependencies = [ { name = "markdown-it-py", extra = ["linkify"] }, { name = "mdit-py-plugins" }, diff --git a/tests/test_retained_annotations.py b/tests/test_retained_annotations.py index d75d37fec0..7bba5df83b 100644 --- a/tests/test_retained_annotations.py +++ b/tests/test_retained_annotations.py @@ -16,12 +16,12 @@ from agent_comms.input_disposition import InputDispositions from agent_comms.installer.refusals import UpgradeRefused from agent_comms.installer.practice import PracticeCopy, PracticeMeasurement -from agent_comms.installer.interpreter_operation import ProbeRequest +from agent_comms.installer.commands import InstallCommands from agent_comms.installer.probes.inputs import SelectedInputsProbe from agent_comms.launch_runtime import LaunchRuntime from agent_comms.maintenance_barrier import MaintenanceBarrier, MaintenanceOperator from agent_comms.message_reference import MessageReference -from agent_comms.nominal.field_codec import FieldCodec, MessageId, TurnId +from agent_comms.nominal.field_codec import MessageId, TurnId from agent_comms.repo import Checkout from agent_comms.task_sources import AnnotationDisclosureGrant, ProjectTaskScope from agent_comms.thread_identity import RecordedThreadIncarnation, TurnIdentity @@ -233,13 +233,13 @@ def test_selected_annotation_materialization_writes_only_the_private_target( held = gate.retarget(gate.begin(MaintenanceOperator.current()), LaunchRuntime.running()) copy = PracticeCopy(root, tmp_path / 'captured-annotations') try: - request = ProbeRequest(SelectedInputsProbe( + probe = SelectedInputsProbe( root, copy.path, 8 * 2**20, requests=(source.reference,), - ), str(held.nonce)) - decoded = FieldCodec.decode(ProbeRequest, FieldCodec.encode(request)) - with held.holding(): - reply = decoded.answer() - measured = FieldCodec.decode(PracticeMeasurement, reply.answer) + ) + with held.holding(), InstallCommands() as commands: + measured = commands.probe( + probe, LaunchRuntime.running().interpreter, cwd=tmp_path, + ) assert measured == PracticeMeasurement.capture(copy.path) contexts = ContextStore(copy.path).original_segments() retained = WorkingMemoryAnnotations.for_segments( From c9c7aacf7161f38eecf2350ddfbed6a02f210bb1 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Mon, 2 Nov 2026 08:01:20 -0500 Subject: [PATCH 27/28] Prepare the actual source cut Agent-Comms-Unpublished: {"kind":"unpublished","predecessor":{"commit":"7da92df5e33ae1fa0cb1598cbf009c9a8824e327","repository":"openhcsdev/agent-comms"},"sources":[{"baseline":{"commit":"3872d91ae52792f2a7f61e17aa81b51c25557604","repository":"openhcsdev/agent-comms"},"candidate_tree":"f6e87cae9857c1f5454f9b9e67d3fe65472def93","heads":[{"head":"7a2b1f5c791953f40127f7ff71804445e06d5b54","pr":3829,"repo":"openhcsdev/agent-comms"}],"kind":"local_integration"},{"baseline":{"commit":"b67c97e5c3ad1eda5c06a464fdc578038d01e3e8","repository":"openhcsdev/formbridge"},"candidate_tree":"8632c131ec6c529cae78de44d86e191dbbbd844a","heads":[],"kind":"merged_integration"},{"baseline":{"commit":"60138d533ef0758c5f093d9110ba9f5656f0934a","repository":"openhcsdev/metaclass-registry"},"candidate_tree":"3c11508a7bf34cfbe79964127305e056227b6b56","heads":[],"kind":"merged_integration"},{"baseline":{"commit":"da87b77f22bd032bf7cd6d52630f684fa4a2b320","repository":"openhcsdev/textual"},"candidate_tree":"69923b8856c78a1053ca02d045f5b23e875a2c5b","heads":[],"kind":"merged_integration"},{"baseline":{"commit":"58c6ca55b09a250b872f8d9323cefad76b09aec9","repository":"openhcsdev/toad"},"candidate_tree":"aba05d441f89f32758f25638a46bf6c7815fde8b","heads":[],"kind":"merged_integration"},{"baseline":{"commit":"ba614bcb90b38c1f70789059a3c6aef8c017971b","repository":"openhcsdev/ui-theme"},"candidate_tree":"6a6cbe340cdd660e3d59cd086b15566d8836a0a2","heads":[],"kind":"merged_integration"}]} From 734247bebd763f328b837bd66cc439071350bc25 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Mon, 2 Nov 2026 08:01:29 -0500 Subject: [PATCH 28/28] Capture the resolved combined sources Agent-Comms-Integration: {"carrier":{"head":"c9c7aacf7161f38eecf2350ddfbed6a02f210bb1","pr":3871,"repo":"openhcsdev/agent-comms"},"kind":"combined","sources":[{"baseline":{"commit":"3872d91ae52792f2a7f61e17aa81b51c25557604","repository":"openhcsdev/agent-comms"},"candidate_tree":"f6e87cae9857c1f5454f9b9e67d3fe65472def93","heads":[{"head":"7a2b1f5c791953f40127f7ff71804445e06d5b54","pr":3829,"repo":"openhcsdev/agent-comms"}],"kind":"local_integration"},{"baseline":{"commit":"b67c97e5c3ad1eda5c06a464fdc578038d01e3e8","repository":"openhcsdev/formbridge"},"candidate_tree":"8632c131ec6c529cae78de44d86e191dbbbd844a","heads":[],"kind":"merged_integration"},{"baseline":{"commit":"60138d533ef0758c5f093d9110ba9f5656f0934a","repository":"openhcsdev/metaclass-registry"},"candidate_tree":"3c11508a7bf34cfbe79964127305e056227b6b56","heads":[],"kind":"merged_integration"},{"baseline":{"commit":"da87b77f22bd032bf7cd6d52630f684fa4a2b320","repository":"openhcsdev/textual"},"candidate_tree":"69923b8856c78a1053ca02d045f5b23e875a2c5b","heads":[],"kind":"merged_integration"},{"baseline":{"commit":"58c6ca55b09a250b872f8d9323cefad76b09aec9","repository":"openhcsdev/toad"},"candidate_tree":"aba05d441f89f32758f25638a46bf6c7815fde8b","heads":[],"kind":"merged_integration"},{"baseline":{"commit":"ba614bcb90b38c1f70789059a3c6aef8c017971b","repository":"openhcsdev/ui-theme"},"candidate_tree":"6a6cbe340cdd660e3d59cd086b15566d8836a0a2","heads":[],"kind":"merged_integration"}]}