Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
8049393
Retain selected original journal context entries
trissim Nov 1, 2026
e7368e0
Expose column and table foreign-key declarations for retained selection
trissim Nov 1, 2026
966c5a3
Add attached-source row retention primitives
trissim Nov 1, 2026
4d72c9a
Share row loading and retain declared acknowledgement groups
trissim Nov 1, 2026
6a9c89f
Read practice metadata through the attached database
trissim Nov 1, 2026
8ef5f1e
Keep unresolved seed projection on InputDispositions
trissim Nov 1, 2026
75c753d
Retain selected saved contexts through PracticeCopy
trissim Nov 1, 2026
aade79d
Bound selected context loading by its remaining copy allowance
trissim Nov 1, 2026
3e0a035
Retain harness observation groups and original claim relations
trissim Nov 1, 2026
8c87559
Retain native source membership and original execution children
trissim Nov 1, 2026
d58a80c
Keep original input recipient birth in owner matching
trissim Nov 1, 2026
6d61d8b
Retain original publication receipts and append records
trissim Nov 1, 2026
b01edf9
Select original coordinator inputs through the unresolved reader
trissim Nov 1, 2026
1ea0c60
Retain participant generation and current pointer through the existin…
trissim Nov 1, 2026
c202df8
Preserve SQLite allocation metadata through the attached file copy
trissim Nov 1, 2026
9ffa104
Retain original annotations with selected context inputs
trissim Nov 2, 2026
857fbf0
Stream original segment groups during annotation retention
trissim Nov 2, 2026
fa5ad07
Require recorded IDs and required participant rows in retention readers
trissim Nov 2, 2026
e94e547
Keep selected input orchestration in PracticeCopy
trissim Nov 2, 2026
2b5de62
Keep selected row groups on their retention owners
trissim Nov 2, 2026
385885b
Enable read-only URI attachments and repair retention test originals
trissim Nov 2, 2026
67b587c
Compare selected copies through their original schema and read owners
trissim Nov 2, 2026
4066a48
Capture selected inputs inside the existing retargeted hold
trissim Nov 2, 2026
8e89d46
Drop the source write claim during selected materialization
trissim Nov 2, 2026
f034542
Derive selected capture membership from file capabilities
trissim Nov 2, 2026
7a2b1f5
Exercise selected capture through the real target interpreter worker
trissim Nov 2, 2026
7da92df
Merge commit '7a2b1f5c791953f40127f7ff71804445e06d5b54' into work/pi-…
trissim Nov 2, 2026
c9c7aac
Prepare the actual source cut
trissim Nov 2, 2026
734247b
Capture the resolved combined sources
trissim Nov 2, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
509 changes: 509 additions & 0 deletions docs/retained-practice-inputs.md

Large diffs are not rendered by default.

8 changes: 1 addition & 7 deletions src/agent_comms/assignment_store.py
Original file line number Diff line number Diff line change
Expand Up @@ -102,14 +102,8 @@ def settle_harness_inputs(
Every step runs in the caller's coordinator write, so the harness attempt
and its claim consequence commit together or not at all.
"""
claims = WakeAssignment.select(
db,
where=f"recipient_lookup=? AND wire_seq IN ({','.join('?' * len(sources))})",
parameters=(recipient_lookup, *(source.seq for source in sources)),
)
claims = WakeAssignment.for_harness_sources(db, recipient_lookup, sources)
for claim in claims:
if claim.source not in sources:
raise IdentityConflict("harness delivery names another message at this sequence")
for disposition in steps(claim.lifecycle):
# The revision fence refuses a step if the native owner moved the claim meanwhile.
claim = self.transition_in(
Expand Down
9 changes: 7 additions & 2 deletions src/agent_comms/bus.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,9 @@
)
from agent_comms.errors import CommittedMessageError, RelationViolationError, UnclaimedError
from agent_comms.nominal.field_codec import ClaimId, FieldCodec, MessageId, ThreadLookup
from agent_comms.locked_store import BusFile, FileStore, SQLiteReader, StoreCommittedError
from agent_comms.locked_store import (
BusFile, CompleteInputReader, FileStore, SQLiteReader, StoreCommittedError,
)
from agent_comms.message_reference import MessageReference
from agent_comms.private_path import FileIdentity, PresentPath
from agent_comms.read_basis import CursorRow
Expand Down Expand Up @@ -523,7 +525,10 @@ def observing(self) -> Iterator[sqlite3.Connection]:
yield db


class Bus(FileStore, StoredBusReader, ReconciledFile, WalStore, CarriedWriter, BusFile, Watched):
class Bus(
FileStore, StoredBusReader, ReconciledFile, WalStore, CarriedWriter, BusFile,
CompleteInputReader, Watched,
):
"""One SQLite transaction owns the sequence and all of a record's fields."""

filename = "bus.sqlite3"
Expand Down
7 changes: 4 additions & 3 deletions src/agent_comms/bus_tables.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
ProjectionProducerTables,
RowNotFoundError,
TableReference,
TableSource,
TypedRow,
)

Expand Down Expand Up @@ -112,10 +113,10 @@ class MessageIdentitySource(MessageReference, messages.BusProjectionTables):
"""A record's existing reference, searchable without recomputing its content hash."""

@classmethod
def practice_sequence(cls, db: sqlite3.Connection, source: sqlite3.Connection) -> int:
through = messages.BusRow.last_seq(source)
def practice_sequence(cls, db: sqlite3.Connection, source: TableSource) -> int:
through = messages.BusRow.last_seq(db, table_source=source)
if through:
expected = cls.require(source, seq=through)
expected = cls.require(db, table_source=source, seq=through)
if cls.require(db, seq=through) != expected:
raise RelationViolationError("Practice bus identity differs at its last sequence")
return through
Expand Down
2 changes: 2 additions & 0 deletions src/agent_comms/cohort_schema.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
Index,
InitialOneWayColumn,
NoDelete,
RetainsChildren,
SharedFileTables,
SQLiteJournalMode,
TextStorage,
Expand Down Expand Up @@ -60,6 +61,7 @@ class AwarenessTable(SharedFileTables, DurableTables):

@dataclass(frozen=True, kw_only=True)
class ClaimBatchAcknowledgements(
RetainsChildren,
InitialOneWayColumn,
GuardedColumns,
NoDelete,
Expand Down
82 changes: 80 additions & 2 deletions src/agent_comms/context_store.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
from typing import Annotated, ClassVar, Self

from .errors import RelationViolationError, StoredHistoryUnreadable
from .locked_store import InputCaptureFile
from .nominal.field_codec import ContextStoreId, FieldCodec, MalformedRecord, NullMember
from .native_session_reopen import NativeSessionIdentity
from .registration import Registration
Expand All @@ -31,7 +32,10 @@
UnrecordedSegment,
)

from .typed_table import StoreRows, Column, DurableStore, Generated, Index, RowidSequenced, SingletonRow, TypedTable
from .typed_table import (
AttachedPracticeRebuild, StoreRows, Column, DurableStore, Generated, Index,
RowidSequenced, SingletonRow, TableSchema, TypedTable,
)
from .turn_item import NativeItemId
from .native_session_reopen import PiSessionId

Expand Down Expand Up @@ -242,6 +246,35 @@ def recorded(self, db: sqlite3.Connection) -> RecordedContextManifest:
parts.append(ContextSegmentVersion.require(db, id=version).segment)
return self.identity.with_segments(tuple(parts))

@classmethod
def retain_required_rows(
cls, original: AttachedPracticeRebuild, schema: type[TableSchema],
) -> int:
"""The stored request owns its serialized version, entry and predecessor edges."""
changed = super().retain_required_rows(original, schema)
db = original.destination
for request in cls.select(db):
changed += original.retain_rows(ContextStoreInfo)
ContextStoreInfo.require(db, singleton=1)
changed += original.retain_keys(
ContextSegmentVersion, tuple({"id": version} for version in request.versions),
)
record = request.recorded(db) # Missing original versions refuse, never disappear.
for part in record.segments:
predecessors = part.predecessors(request.id)
changed += original.retain_keys(cls, tuple({"id": key} for key in predecessors))
for key in predecessors:
if cls.require(db, id=key).identity.thread != request.identity.thread:
raise RelationViolationError("Context predecessor is missing or foreign")
changed += original.retain_keys(ContextEntry, tuple(
{"session_id": ref.identity.session_id,
"session_file": ref.identity.session_file, "entry_id": ref.entry_id}
for ref in part.entries
))
for reference in part.entries:
ContextEntry.original(db, reference)
return changed


class ContextPriorSupersededError(RelationViolationError):
"""Another writer recorded this thread's context after the prior this writer holds."""
Expand Down Expand Up @@ -447,7 +480,7 @@ def reference(self, manifest: ContextManifest) -> ContextManifestReference:


@dataclass(frozen=True)
class ContextStore(DurableStore):
class ContextStore(DurableStore, InputCaptureFile):
root: Path
filename: ClassVar[str] = "turn_context.sqlite3"
tables = ContextStoreRows
Expand Down Expand Up @@ -484,6 +517,51 @@ def entries(
session_file=identity.session_file,
entry_id=entry_id)}

def original_segments(self) -> Iterator[SegmentManifest]:
"""Original segments actually present in this selected entry/request store.

Requests restore through the same original predecessor/entry reader as
the browser. Entry-only stores need no invented request/store identity.
"""
with self.snapshot() as db:
for entry in ContextEntry.stream(db):
yield from entry.manifest.original_values()
for request in ContextRequest.stream(db):
source = ContextStoreInfo.require(db, singleton=1).source
for root in self.resolve(db, source, request).value.segments:
yield from root.original_values()

def retain_contexts(
self, destination: ContextStore, identities: Iterable[NativeSessionIdentity], *,
maximum_bytes: int, requests: Iterable[ContextManifestReference] = (),
) -> None:
"""Retain original entry manifests and explicitly selected saved requests.

Raw writable values, source identity, predecessor/version/entry closure
and sequence high-water marks travel unchanged. Session paths are keys,
not relocated storage or permission to launch or replay a native session.
Entry-only selection retains no request or fabricated store identity.
"""
with type(self).retaining_rows(
self.path, destination.path, maximum_bytes=maximum_bytes,
) as original:
original.retain_keys(ContextEntry, tuple(
{"session_id": identity.session_id, "session_file": identity.session_file}
for identity in dict.fromkeys(identities)
))
selected = tuple(dict.fromkeys(requests))
if selected:
original.retain_rows(ContextStoreInfo)
source = ContextStoreInfo.require(original.destination, singleton=1).source
for reference in selected:
if reference.source != source:
raise RelationViolationError("Selected context belongs to another store")
original.retain_keys(ContextRequest, tuple(
{"id": reference.record_id} for reference in selected
))
for reference in selected:
ContextRequest.require(original.destination, id=reference.record_id)

def prior(self, owner: ThreadIncarnation) -> RecordedContextState:
"""Read the thread's previous request from a snapshot, outside any write lock."""
with self.snapshot() as db:
Expand Down
19 changes: 10 additions & 9 deletions src/agent_comms/coordination_database.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
"""Private coordinator storage and declaration-owned schema installation."""

from __future__ import annotations
Expand All @@ -15,7 +15,7 @@

from agent_comms.coordination_errors import CoordinationReadUnavailable, IntegrityViolationError
from agent_comms.coordination_schema import CoordinatorTable
from agent_comms.locked_store import FileStore, SQLiteReader
from agent_comms.locked_store import FileStore, InputCaptureFile, SQLiteReader
from agent_comms.typed_table import (
CarriedWriter,
ReconciledFile,
Expand All @@ -27,7 +27,6 @@
from .nominal.declared_family import DeclaredFamily
from .store_files import DEFAULT_STORE_CONTENTION, StoreContention


class StoreFile(DeclaredFamily):
"""One decoded file state, with the consumer's result inside its read lifetime."""

Expand Down Expand Up @@ -67,7 +66,7 @@
yield consumer.capture(connection, *arguments)


class CoordinationStore(FileStore, CarriedWriter, ReconciledFile):
class CoordinationStore(FileStore, CarriedWriter, ReconciledFile, InputCaptureFile):
"""The private coordinator database and its declared schema."""

filename = "coordination.sqlite3"
Expand Down Expand Up @@ -119,7 +118,7 @@
with CoordinationReadUnavailable.checking():
self._connection = self._open_connection(contention)
try:
self._configure_connection()
self.configure_connection(self._connection)
self._initialize_schema()
self._enforce_private_modes()
except BaseException:
Expand All @@ -133,13 +132,15 @@
"""The root's own composition of this file decides its schema and its strays."""
self.schema_at(self.path).install_schema(self._connection, self.schema_change(self.path))

def _configure_connection(self) -> None:
@classmethod
def configure_connection(cls, connection: sqlite3.Connection) -> None:
"""Ordinary opening and selected loading use the same row-owned SQL behavior."""
for row_type in CoordinatorTable.declarations():
row_type.configure_connection(self._connection)
self._connection.row_factory = sqlite3.Row
self._connection.execute("PRAGMA foreign_keys = ON")
row_type.configure_connection(connection)
connection.row_factory = sqlite3.Row
connection.execute("PRAGMA foreign_keys = ON")
# Rollback journal avoids a post-schema WAL-mode race among fresh openers.
self._connection.execute("PRAGMA synchronous = FULL")
connection.execute("PRAGMA synchronous = FULL")

def _prepare_private_file(self) -> None:
self.path.parent.mkdir(parents=True, exist_ok=True)
Expand Down
3 changes: 2 additions & 1 deletion src/agent_comms/coordination_session.py
Original file line number Diff line number Diff line change
Expand Up @@ -206,7 +206,8 @@ def _prepare_private_file(self) -> None:
def _open_connection(self, contention: StoreContention) -> sqlite3.Connection:
return self._open_reader(self.path, contention)

def _configure_connection(self) -> None:
@classmethod
def configure_connection(cls, connection: sqlite3.Connection) -> None:
pass # _open_reader configured query-only behavior before publication.

def _initialize_schema(self) -> None:
Expand Down
93 changes: 81 additions & 12 deletions src/agent_comms/coordination_tables/annotations.py
Original file line number Diff line number Diff line change
@@ -1,16 +1,25 @@
"""Durable label/request declarations; the existing coordinator owns storage."""
from __future__ import annotations

import sqlite3
from abc import abstractmethod
from collections.abc import Iterable, Iterator
from dataclasses import dataclass, field
from typing import Annotated, ClassVar
from typing import Annotated, ClassVar, Self

from agent_comms.coordination_schema import CoordinatorRow
from agent_comms.message_reference import MessageReference
from agent_comms.turn_context import ContextSpan, SegmentManifest
from agent_comms.typed_table import (
LOCAL_TABLE_SOURCE,
AttachedPracticeRebuild,
Column,
Generated,
Index,
RowId,
TableSchema,
TableSource,
TypedTable,
UnallocatedRowId,
sql_literal,
)
Expand All @@ -19,8 +28,46 @@
from agent_comms.working_memory_requests import AnnotationOutcome, DisclosureRequest


class ContextSpanRows(TypedTable, declared_name=None):
"""Stored answers and request dispositions share original-span eligibility."""

__slots__ = ()

@property
@abstractmethod
def context_span(self) -> ContextSpan: ...

@classmethod
def for_segments(
cls, db: sqlite3.Connection, segments: Iterable[SegmentManifest], *,
table_source: TableSource = LOCAL_TABLE_SOURCE, **key: object,
) -> Iterator[Self]:
originals: dict[str, list[SegmentManifest]] = {}
for segment in segments:
originals.setdefault(segment.sha256, []).append(segment)
for row in cls.stream_keys(db, tuple(
dict(key, segment_digest=digest) for digest in originals
), table_source=table_source):
span = row.context_span
if any(segment.contains_span(span) for segment in originals[span.segment_sha256]):
yield row

@classmethod
def retain_segments(
cls, original: AttachedPracticeRebuild, segments: Iterable[SegmentManifest],
) -> int:
# Restored requests may repeat large original prefixes. Consume one
# segment at a time, not a second materialized context corpus.
members = cls.members_with(cls)
return sum(row.retain_original(original)
for segment in segments
for member in members
for row in member.for_segments(original.destination, (segment,),
table_source=original.source))


@dataclass(frozen=True, kw_only=True)
class SpanAnnotationsRow(CoordinatorRow):
class SpanAnnotationsRow(ContextSpanRows, CoordinatorRow):
label: ModelLabel
created_at_ms: int
id: Annotated[RowId, Column(primary_key=True, auto_increment=True)] = field(
Expand Down Expand Up @@ -78,22 +125,17 @@ def effective(cls, rows: tuple[SpanAnnotationsRow, ...]):
# Row ids order successive corrections from the same authority. Nominal
# inheritance, not sequence, decides human versus model authority.
latest = {}
for row in sorted(rows, key=lambda value: value.id):
for row in sorted(rows, key=lambda value: int(value.id)):
latest[type(row.label)] = row.label
return SpanLabel.effective(tuple(latest.values()))

@classmethod
def for_digests(cls, db, digests, classifier):
if not digests:
return ()
return cls.select(db,
where='segment_digest IN (' + ','.join('?' for _ in digests) + ')',
parameters=tuple(digests), order_by=("id",),
classifier=classifier.classifier, classifier_pin=classifier.pin)
@property
def context_span(self) -> ContextSpan:
return self.label.span


@dataclass(frozen=True, kw_only=True)
class AnnotationRequestsRow(CoordinatorRow):
class AnnotationRequestsRow(ContextSpanRows, CoordinatorRow):
request: DisclosureRequest
outcome: AnnotationOutcome
created_at_ms: int
Expand Down Expand Up @@ -137,6 +179,33 @@ class AnnotationRequestsRow(CoordinatorRow):
unique = (SpanAnnotationsRow.address,)
indexes = (Index(("grant_reference", "created_at_ms")),)

@property
def context_span(self) -> ContextSpan:
return self.request.span

@classmethod
def retain_required_rows(
cls, original: AttachedPracticeRebuild, schema: type[TableSchema],
) -> int:
"""Keep original grant usage and each request's complete answer/correction group."""
changes = super().retain_required_rows(original, schema)
changes += original.retain_keys(cls, tuple(
{"grant_reference": row.request.grant} for row in cls.stream(original.destination)
))
# One natural address can name multiple correction rows; it is not a
# foreign key. Compare the actual declared generated columns, without
# making storage import a reader to reconstruct their domain values.
related = " AND ".join(
f"{SpanAnnotationsRow.qualified(name)} IS {cls.qualified(name)}"
for name in SpanAnnotationsRow.address
)
changes += original.retain_rows(
SpanAnnotationsRow,
where=f"EXISTS (SELECT 1 FROM main.{LOCAL_TABLE_SOURCE.qualified(cls.declared_name)} "
f"WHERE {related})",
)
return changes

@classmethod
def requests_since(cls, db, reference: MessageReference, since: int) -> int:
return cls.count(db, where='created_at_ms>?', parameters=(since,), grant_reference=reference)
Loading
Loading