diff --git a/docs/retained-practice-inputs.md b/docs/retained-practice-inputs.md new file mode 100644 index 0000000000..aa8e5cf3ce --- /dev/null +++ b/docs/retained-practice-inputs.md @@ -0,0 +1,509 @@ +# Original-coordinate practice inputs — implementation checkpoint + +Core #3829 remains unfinished. Entry, selected saved-context and coordinator input +evidence now have explicit PracticeCopy consumers in source. Complete coordinator +relations, coherent aggregate capture, bounded peak materialization, native-journal +reading and the prepared-root application run remain unfinished. +This document records the source decisions, not an executable selection roster. + +## Original recipient birth participates in owner matching + +Source tracing found `StoredInput.matches_owner` consulted only its origin, +ignoring the recipient birth already present in `OwnedBusInputKey`. For a neutral +origin, a recreated same-name owner could therefore satisfy that predicate even +when the input key named another original birth. + +`InputKey.matches_owner` now owns the key's contribution. Birthless keys add no +birth constraint; the owned bus member checks its original birth through the +incarnation's existing `recorded_or` behavior. `StoredInput` combines that answer +with the unchanged origin-owned author/name check. No key is re-encoded, missing +birth inferred, generation substituted, lifecycle changed or native permission +created. This tightens an existing consumer, not a second retention-only identity +resolver. The added ordinary owner/recreation/birthless-owner and queued-text +refusal scenario is authored **UNRUN**; the existing birthless-key case remains. + +## Existing owners and implemented entry selection + +`ContextStore.retain_contexts(destination, identities, maximum_bytes=..., requests=...)` replaces the +entry-only API. `PracticeCopy.include_contexts` consumes it explicitly; normal +whole-file include and live-root backup selection are unchanged. Selected writable +SQL values travel through the existing attached source, without decode/re-encode, +coordinate relocation, source-identity minting or a compatibility entry point. + +`ReconciledFile.retaining_rows` owns fresh staging, exact source-schema validation, +the existing load-before-guards transaction, repeated declared-dependency expansion, +and exclusive atomic publication. A refusal or failed closure never overwrites an +existing destination or publishes the partially loaded staged database. + +`ContextRequest.retain_required_rows` owns its serialized version IDs, segment +entry references and earlier request IDs. Missing originals and foreign-thread +predecessors refuse. Whole retained predecessor records receive their own closure, +not an incomplete predecessor that only one selected segment can read. The original +ContextStoreInfo is required when requests are selected; entry-only selection does +not invent one. The attached file-copy owner retains SQLite's original sequence +high-water marks, so an omitted original ID cannot become the next newly allocated ID. +The allocation metadata is not restricted to RowidSequenced's insertion capability. + +The earlier entry-only API's owner-path run retained 6,274 entries in 13,877,248 +materialized bytes with matching source-before/retained/source-after row digests. +It was not rerun on this implementation. Source compilation is not qualification +of the selected saved-request browser, atomic publication, rollback, sequence +metadata, resource bounds or native/physical behavior. + +## Foreign-key inspection prerequisite + +Previously `TypedTable.references()` exposed only table-level constraints. Column +references, including inherited/generated columns, live on `_columns` through +`Column.references`. Reading only the former silently misses edges such as +`NativeRuntimeInput.execution_id` and `owner_lookup`. + +The new `TypedTable.foreign_keys()` derives both forms. `Reference.foreign_keys` +requires an answer from the reference kind: `NoReference` contributes no edge, +while `TableReference` binds its declared target to the source column. Composite +`ForeignKey` declarations retain their original objects and constraint options. +There is no table-name roster, SQL parser, alternate codec or stored graph. + +SQL generation remains unchanged: column clauses and table constraints keep their +existing placement, so this inspection API does not change a stored schema or +require a carry. This is a prerequisite API, not a wired retention selector, and +has no application acceptance yet. + +## Attached-source raw-row primitive — source only + +`AttachedPracticeRebuild.retain_rows` selects original writable SQL columns and +inserts them unchanged into its bound destination connection. The existing +`PracticeRebuild.attached` context supplies that connection with the active source; +a mere path descriptor cannot select another attachment through a caller-supplied +database. A declared primary key controls repeat selection; a missing key refuses. +Other constraint conflicts are not hidden by `INSERT OR IGNORE`. + +`ForeignKey.retain_parents` puts the original parent columns on the left of the +comparison, preserving SQLite parent affinity/collation and NULL-key behavior. +Internal aliases derive from the target declaration and cannot shadow its outer +table name. `TableSchema.retain_referenced_rows` expands one pass through declared +edges, including column references and required groups supplied by row overrides. +It is not a fixed-point or complete semantic closure. + +`retain_keys` reuses TypedTable's indexed key-batch compiler. It shares raw insertion +and primary-key deduplication with `retain_rows`; no second key encoder or raw-copy +procedure is added. Completed changes come from the connection, since SQLite's +WITH-prefixed INSERT reports Cursor.rowcount=-1. + +ContextStore and CoordinationStore now consume the primitive through their +explicit PracticeCopy selection methods. Guard-safe loading, required children, source snapshot consistency, +producer metadata, sequences, schema compatibility, materialized bounds and copy +behavior remain unqualified. No original lifecycle is rewritten and no full-copy/backup behavior +is changed. These methods are implementation prerequisites, not a runnable root. + +## One attached source for metadata and rows — source only + +`PracticeRebuild.attached` previously opened a separate read-snapshot connection +for producer/last-sequence checks while raw copying read the destination's SQLite +attachment. Under WAL those connections could observe different source revisions. +It now obtains the actual `SQLiteDatabase` through SQLite's database-list reader; +metadata checks and copies address that attachment on the same connection. The +existing reconciliation owner holds their write-transaction lifetime. + +`SQLiteDatabase` composes `TableSource` and owns qualified table names. The default +source preserves ordinary SQLite name search, including legitimate temporary +relations; it is not silently changed to an explicit `main` prefix. The existing +TypedTable query owner supplies this source to select/stream, indexed key batches, +counts, existence and catalog checks. Producer checks and the sole bus identity +prefix consumer use it. No second connection, SQL parser or query codec remains. +The source argument qualifies the selected table, not arbitrary SQL subqueries: +joined relations still need their declaration-owned source qualification. + +The FK predicates deliberately retain explicit `main` references: those are the +selected destination children/parents, not another original-source alias. The +original row still comes through `AttachedPracticeRebuild.retain_rows`. Replacing +those destination references with the original source would change the selection. + +The existing rebuild API forwards the same transaction connection through schema +reconciliation; that parameter is not an independently maintained source snapshot. +No additional rebuild hierarchy or unchanged-whole-file gate was added. Actual +WAL interleaving, source/target isolation and producer-prefix controls are still +unrun; the coordinator consumer below is a source implementation, not qualification. + +## Shared loading lifetime and acknowledgement groups — source only + +`TypedTable.loading` owns table creation, the load interval, then installation of +that row's indexes/guards. Existing `rebuild` consumes the same sequence rather +than keeping a separate loading procedure. NRA's ModuleSyntaxIndex, StatementSource +and exact SourceTextPatch selected the original statements and preserved their +source geometry. This is an authored context composition, not a native equivalence +certificate or a class-member promotion. + +`TableSchema.loading_rows` opens one transaction with FK enforcement suspended, +creates every table before yielding, installs every row's guards after loading, +then checks references before committing. Exceptions roll back through the existing +SQLiteFile transaction owner. The same `require_related_rows` check is used by +ordinary schema reconciliation. No ordinary runtime insertion guard is weakened. + +`TypedTable.retain_required_rows` supplies the default empty contribution. +`RetainsChildren` composes that contribution with inverse **declared FK edges**. +ClaimBatchAcknowledgements inherits this capability because its sealed counts +describe the complete original child group; its forwarding override is deleted. +HarnessDelivery inherits the same behavior for its original observation parts. +Only a requested parent expands its children, not every participant's history. +`ForeignKey` owns both directions and their one parent-affinity comparison. +Original sealed state is copied unchanged; no seal transition or replay is issued. + +These are source implementations, not executed copy/rollback/guard controls. +The acknowledgement and harness rows share the same retained-group capability; +actual class creation/MRO, schema stability and group-copy behavior still require +qualification. Four new harness-group scenarios are authored, UNRUN: selected +observations with an existing post-handoff claim, observations with no claim, +full-reference conflict, and copy conflict with no partial publication. +The coordinator seed consumer below now selects through the existing input +projection. Aggregate capture and the remaining serialized/cross-store and producer +dependencies remain OPEN; complete FK validity +would not establish their preservation or the original group's validity. + +## Required preservation that SQL foreign keys do not establish + +* **Uncertain inputs:** `InputDispositions.unresolved(entries)` joins original + ledger state with `harness_attempts(entries)`. `stored_unresolved()` is not an + equivalent seed. Stored rows in every state suppress a synthesized attempted + key; copying only unresolved ledger rows could resurrect a completed attempt. + No synthesized `ReservedInput` is written back merely to construct a copy. +* **Harness claims and observations:** `WakeAssignment.for_harness_sources` now + owns recipient/sequence candidate selection and full MessageReference conflict + checking. Normal settlement consumes it rather than maintaining its own SQL + predicate; selected HarnessDelivery retention reads through the same active + original attachment and copies every existing matching claim, including its + unchanged lifecycle/revision. Missing claims stay absent: an attempt can precede + claim acceptance. A same-sequence/different-message conflict refuses instead of + silently disappearing. This natural relation is not fabricated as a mandatory + FK. HarnessObservedPart's declared FK supplies its parent, not the reverse + observation set; RetainsChildren supplies that original child group through the + shared schema traversal. Original sender/body/record-ID/timestamp and partial + ReceivingMonitor state are not reconstructed, completed or replayed. +* **Sealed acknowledgements:** `ClaimBatchAcknowledgements.column_guards` requires + complete member/delivery sets and their original counts. Forward FK closure + does not supply those children. Child insert guards reject an already-sealed + parent: loading must not temporarily rewrite the retained sealed state. + `TypedTable.rebuild` already distinguishes loading writable data from installing + guards; the retained-copy implementation must use that owning mechanism. +* **Native source membership:** `NativeInputExecution.source_membership_sql` and + `source_assignment_ids` own full/triage membership. Selection must consume that + behavior rather than maintain another list of stages or link tables. +* **Producer/publication evidence:** executions, attempts, obligations, immutable + intents, receipts and append dispatches carry original state and cyclic or + generated-column references. A lifecycle is not reset to make insertion easy. + The complete child/serialized-reference closure is still OPEN. +* **Serialized references:** `AnnotationRequestsRow.request` contains the original + `DisclosureRequest.grant` message reference and disclosure state. Its submitted + outcome means never resend automatically. `TranscriptRoute`/`InputDisplay` + routing, native context proofs, entry references and saved-request predecessor + links also require their existing value/store owners; absence from an FK list + is not evidence that these relations do not exist. + +## Unresolved-input seed projection + +`HarnessDelivery.unconfirmed_for` now owns the incarnation/outcome query. +Membership derives from `Outcome.members_with(UnconfirmedOutcome)`; requested +keys are encoded by the declared columns through `TypedTable.select_keys`, not +hand-written outcome names. Normal input reads use that same query. + +`SnapshotInputDispositions` inherits the unchanged `unresolved(entries)` and +all-state suppression algorithm. Its sole override reads harness rows through an +active `AttachedPracticeRebuild` source on that attachment's connection. It has +no new file/schema declaration. Its `root` must contain the retained bus and +complete ledger, captured consistently with the coordinator source; the caller +owns that capture and attachment lifetime. This is not a second ledger or an +input replay operation. + +`SnapshotInputDispositions.retain_unresolved` now consumes that unchanged reader +and seeds original attempt groups at the selected keys. It asks the existing +harness producer a second time on the same attachment, not a copied key/outcome +or suppression algorithm. An unresolved stored ledger row can keep an additional +original attempt witness at its key; the complete ledger still wins the read. +The recipient lookup comes from those original source entries, not from a stored +name/generation or a fabricated birth. `InputOrigin.harness_sources` exposes no +witness by default; `HarnessAttemptOrigin` exposes itself. `HarnessDelivery.retain_attempt` +owns keyed row retention and the full-reference refusal. No synthesized reservation +is stored, and the provenance family imports no coordinator row. + +`PracticeCopy.include_inputs` runs these seeds directly through the existing +`CoordinationStore.retaining_rows` loading/dependency/publication lifetime. +`NativeRuntimeInput.retain_sessions` additionally selects original native groups +by both recorded session id and original session path. A bare ledger native id +is not treated as a coordinator FK. The existing row hooks retain the selected +original children, claims and forward references. + +`PracticeCopy.include_inputs` requires existing bus/ledger sources to have been +captured completely through `include`; a copied reader with no original source +refuses. The coordinator uses the remaining aggregate allowance, and its resource +is recorded only after successful publication. Ordinary whole copies/backups are +unchanged. The caller still owns the consistent source cut and serial copy; this +is not a capture certificate or native launch permission. + +Three new cases in `tests/test_harness_receipt_context.py` are authored **UNRUN**: +no ledger row, an original reservation, and an actually bound/started ledger row. +They use the existing native-root/message/claim producers, compare the ordinary +source and retained projections and original attempt rows, and check the source +coordinator bytes and bus revision. Together with the eighteen previous cases, +that checkpoint's **twenty-one added scenarios remained unrun**. The participant +checkpoint below adds three more; the subsequent sequence controls add two, +for **twenty-six added scenarios UNRUN**. +These do not cover every disposition, native association or rollback/resource failure. + +Runtime class creation/MRO/registration, attached SQL, cross-store capture and +complete preservation remain unqualified. The selector has no participant-wide +reverse walk. Participant aggregates contribute through the selected attachment's +expansion below; inverse native membership, serialized references and the remaining +original recovery groups still require semantic closure. No seed-only database is +a runnable root or complete backup. + +## Participant aggregate — source implementation, not acceptance + +`ParticipantStore.get` requires `Participants`, `OwnerGenerations` and +`CurrentExecutions`. A foreign-key-valid participant alone cannot answer that +reader. `ParticipantStore.read_at` now owns the same three reads and original +incomplete-aggregate refusal through the caller's already-held connection/source. +Ordinary `get` keeps its existing session read lifetime and consumes that method; +retention never constructs a borrowed `CoordinationSession` or another connection. + +`ParticipantStore.retain_owners` seeds selected original lookup keys, preserves an +absent participant as absent, and refuses a present incomplete aggregate. +`ParticipantSnapshot.retain_original` keeps its original generation/current pointer. +`ParticipantRows(AttachedPracticeRebuild)` adds that aggregate to each expansion of +the existing selected-file closure, including participants discovered later by FK, +not just the initial seeds. `PracticeCopy.include_inputs` selects this attachment +through `original_type`; generic foreign-key copying alone does not claim aggregate +completeness. The same read-only attachment and destination connection are borrowed; +there is no second closure loop or connection, and Participants imports no reader. +The current pointer's declared references retain its actual pointed execution/attempt; +no counter, revision, commitment or lifecycle is reset. + +`TypedTable.retention_key_fields` derives the primary key once from its declared +columns for raw-copy deduplication and held-row selection. `retain_original` gets +those values through existing `FieldPlan.current` and the shared indexed/raw-copy +compiler, not a field roster or decoded-row reinsertion. `one`, `require` and +`_keyed` now carry the same `TableSource` as `select`; the actual `Generation` +override carries it too while preserving its corruption exception. + +Three cases are authored **UNRUN**: an advanced generation and active current +pointer preserve the exact aggregate without unrelated/native history; missing +generation or missing pointer refuses publication. The existing two native-group +seed cases also require the retained aggregate. Source mapping covers 65 modules, +537 original class declarations, 519 canonical projections, no parse omissions; +18 function-local test declarations remain unprojected OPEN. That is source +coverage, not executed class/MRO/SQL behavior or complete closure. + +## SQLite allocation metadata — source only + +The coordinator cases share the real selected-file factory with `ParticipantRows`. +The active-pointer case uses the real `mark_pending` transition and expects revision +2 before `start`; QueuedExecution does not admit direct start. No product fallback +was added to satisfy a faulty fixture. These cases still require execution. + +A separate preservation gap remained in the earlier rowid hook: `RecoveryAudit`, +`SpanAnnotationsRow` and `AnnotationRequestsRow` declare AUTOINCREMENT columns but +do not inherit `RowidSequenced`. Their SQLite high-water metadata could disappear. +Two sources with identical selected lower rows but different original high-water +marks require different next IDs; the selected rows alone cannot recover that. + +`AttachedPracticeRebuild.retain_sequences` now copies SQLite's own allocation +metadata once after dependency closure, inside the existing load transaction and +before guards/publication. The actual source/destination catalogs supply sequence +presence and table membership; there is no row-family/table/field roster. Existing +larger destination values are not lowered. Empty selection still preserves the +original allocation metadata. The competing per-RowidSequenced hook is deleted; +ordinary row insertion and complete copy/backup behavior are unchanged. + +Two additional cases are authored **UNRUN** using ordinary coordinator execution/ +Pending/start producers and typed audit-row insertion: retaining a lower audit, and +retaining no rows at all. Both ask the normal destination writer to allocate a +fresh audit ID after the omitted original high-water mark, and check source bytes. +These are SQL allocation controls, not a real model-stall/native or recovery proof. +The existing context-request/version allocation case remains part of the necessary +combined selection. **Twenty-six added scenarios remain unrun.** SQLite execution, +rollback, metadata/schema/guard behavior and peak resource bounds are unqualified. + +## Copy and backup are different contracts + +`PracticeCopy.include` currently invokes each `RootFile` owner's whole-file +`copy_group`. At the checkpoint baseline, `PracticeBus.select` returns the same +input union for rehearsal and live backup. A bounded subset must not be labelled +as a complete original-store backup. Keep ordinary copy/backup behavior intact; +represent a bounded selection explicitly through the existing request/copy +owners. `PracticeCopy.include_contexts` is an explicit selection API; its request +continues to describe original-to-retained path mapping, not a complete-backup +certificate. BackedUpBus still constructs LiveFilesRequest for the original live +root, rather than backing up the selected practice bytes. Do not add a parallel +importer, ledger or reminted session identities. + +Measure the complete selected stores, WAL/copy groups and free space before any +copy. The requested production-copy proof is bounded to 3 GiB; zero freelist pages +in the oversized stores means compaction is not the repair. + +Selected context materialization now requires an explicit byte allowance. Through +PracticeCopy this is a **total private-tree allowance**, not a new default for full +backups. `PracticeMeasurement.capture` is the one tree-byte calculation: already +retained files, WAL/companions and staging files count whether or not the copied +resource index lists them. `PracticeCopy.remaining_bytes` subtracts those bytes and +also limits new allocation to actual disk space above the existing StorageRule +floor. Its caller owns serial materialization of the private root and the coherent +source cut; this is not a reservation against other processes consuming disk. + +`SQLiteDatabase.limit_bytes` consumes SQLite's actual page size and accepted +max_page_count through typed pragma records. It caps the staged database before +schema/rows/indexes are allocated; a zero-page allowance refuses, and SQLite can +raise SQLITE_FULL on later growth. `ReconciledFile.retaining_rows` additionally +counts the complete closed staged file group before exclusive publication. An +over-budget group refuses and the same staging cleanup runs. No cap, page size, +sequence, schema or lifecycle is written into the original store. + +This is authored admission/allocation code, **not a measured peak-footprint or +free-space race proof**. Journal/temp-spill peaks during loading, concurrent disk +consumption, rollback/cleanup under SQLITE_FULL, and the complete cross-store +selection/root budget still need qualification. Ordinary include()/PracticeBus +and live-root full backups remain unchanged; a caller cannot label their +unmeasured union bounded merely by passing an allowance to one selected store. + +## Native membership and execution groups + +A triage member's `assignment_id` is unique but has no declared assignment FK. +Forward FK closure alone therefore misses its original source claim. The +existing `NativeInputExecution.source_membership_sql` and +`source_assignment_ids` own this relationship for both TRIAGE and FULL. Their +queries now take the existing `TableSource` through every nested join; ordinary +calls still use normal local-table lookup. The membership query leads a left +join into the existing required `WakeAssignment` decoder, rather than silently +filtering a missing original claim through an inner join. That failure path is +source-reviewed, not yet executed. + +`NativeRuntimeInput` composes `RetainsChildren` for original triage/cursor +children and overrides the existing cooperative retention hook to consume its +held execution's original-source assignment query. It uses the shared raw-key +retainer for the claims; it does not write a second membership query, stage +roster or new native coordinate. `ExecutionRecord` composes the same capability +for its original declared child group, including native inputs, attempts and +links. No participant-wide reverse walk, lifecycle transition, dispatch or +claim acceptance is added. No field/DDL/carry declaration changed; actual schema, +MRO and load behavior remain unqualified. + +Three new scenarios are authored **UNRUN** in +`tests/test_retained_coordinator_groups.py`: selected triage keeps only its own +membership/claim; an execution seed keeps its original claimless FULL input; +explicit original-source lookup differs from empty destination-local lookup. +They use existing native-root/schema preparation and typed store owners, not a +handwritten table layout or fake native provider. These are SQL data-group +cases, not actual native admission or runnable-root acceptance. FULL wire +membership, malformed-source refusal, cursor/attempt/publication groups and +all-state suppression still need the necessary combined behavioral selection. +That checkpoint added thirteen retention scenarios, all unrun in the source pass. + +The focused NRA before-map covers nineteen production modules and219 raw/static +class declarations with no parse omissions. The final delta additionally covers +the changed test module; counts are not evidence of runtime MRO, complete +cross-store closure or a delivered root. Complete paginated open-PR paths were +checked, including712/249-file requests. Published3820's superseded +`model_contexts_for` hunk is independent; its deletion stays, and the retention +hook has no `BusReader`/`MessageReference` dependency.3614's relevant changes are +nominal-import moves already present here. Source recipes/results are retained +under `p867-native-group-*`. + +## Publication groups and the existing capture lifetime + +`PublicationIntents` now composes the same `RetainsChildren` capability as +`ExecutionRecord`. The schema's declared inverse relations retain the original +`PublicationEvidence` and `PublicationAppendDispatches` rows. The latter's foreign +key is declared in `coordination_response.py`, not in a retention-only roster. +`RecoveryReader.get` reads the original intents and receipts separately; +`PublicationAppendDispatches.resolve_retained` uses the original append record. +Copying the intent alone does not preserve those answers. The raw loader retains +absence as absence: no receipt, dispatch, acknowledgement or state transition is +created to fill a missing record. + +Four publication cases are authored **UNRUN** in +`tests/test_coordination_response.py`: intent/execution seeds crossed with frozen +intent/published response. They reuse the existing actual response preparation +and publication path, compare original intent/receipt/append rows, and check +that retention changes neither the source coordinator digest nor the bus +revision. The existing fixture now declares its actual `AssignmentId` and +`OwnerFence`, rather than `str` and `object`. These are stored SQL-group controls, +not runnable-root, native, recovery-pointer or physical acceptance. Together with +the earlier thirteen retention scenarios and one recipient-birth regression, +those eighteen scenarios remain unrun; the later consumer adds three more above. + +The live installer already enters `mode.owners_stopped` before its practice +phase (`installer/upgrade.py`). `LiveMode` uses the existing maintenance hold, +stopped-owner and retargeted-runtime lifetime. `RehearsalMode` instead calls +`owners_kept` (`install_choices.py`). A plain rehearsal therefore does not supply +that quiet source boundary. The selected coordinator consumer must use the +existing qualified capture lifetime and preserve its registry/complete-ledger/ +bus/coordinator relationships; this source trace neither executes nor certifies +that aggregate capture. No new freeze state, installer or donor operation was +added. + +## Source evidence and remaining work + +The retained NRA source map at baseline `095c4cdfe85632f990be3b6d67411d9685e6dee9` +covers 632 Core `src` modules, 6,152 original class declarations/projections, +105 table-family source declarations, 7,263 annotations and 48,319 call sites, +with no parse omissions. The composition-derived projection contains 35 +declarations, including family/schema bases, not 35 concrete tables. It follows +`PrivateCoordinatorSchema`'s actual schema parents, including `AwarenessTable`: +a narrower coordinator/private-only query missed awareness's two declarations. +Exact hashes and source spans are retained in the owning run's +`p867-coordination-owner-map.json` and +`p867-coordination-composed-declarations.json`. The earlier narrower query is +retained as counterevidence, not used as complete schema coverage. + +Call spellings and nominal projections do not prove runtime binding or semantic +closure. External/dynamic bindings, serialized links and required reverse children +remain OPEN. The earlier d4/Toad corpus and its unresolved cases are separate; +these counts do not resolve them. New metadata methods require targeted reproof. + +The current post-P859 selector slice covers 30 production/test/fixture modules +and 585 original/canonical class declarations, with no parse omissions. It extends +the 22-module/524-class before-slice through four fixture/test modules and four +moved-owner modules. The intervening 26-module/525-class result keeps its older +source epoch. Compilation and exact NRA-indexed AST comparisons preserve all 148 +existing functions in the six changed Python files. Five mapped dependencies +changed on rebase: registry resolution declarations, bus selection imports, the +FieldCodec compared-field/decoder owner, and two fixture import moves. Those +changes were read; they are not inherited runtime qualification. These are source +checks, not native transport or SQL execution. + +Next: finish the remaining coordinator semantic groups and qualify coherent +aggregate capture and peak budget, then the combined backend/carry and actual +prepared-root saved-history journey. No pytest, provider, donor mutation or new +native/physical run was performed for this source checkpoint. Reader promotion +still requires the separate default-storage and decorator creation-cut proofs. + +## Selected capture file capabilities + +The selected operation now derives its files through `RootFile.declarations_in`. +`InputCaptureFile` identifies each store participating in the common cut; +`CompleteInputReader(InputCaptureFile)` identifies stores copied whole before +context/coordinator selection. Bus and InputDispositions compose the latter; +ContextStore and CoordinationStore compose the former. Their existing storage, +writer, schema and copy behavior remains on their original parents. + +The `PracticeCopy.input_readers` roster is deleted. Capture pinning/draining, +complete-reader preflight/copy and missing-original-reader checks use the declared +capabilities. A new reader joins through its declaration rather than a second +list. This also removes the heterogeneous tuple that mypy inferred as `object`. + +The changed file-family bases and traversal were executed in the actual capture +and annotation tests: 19 collected, 19 passed, 4.97 seconds, exit0, using the host +TestLease and a fresh owned `/home/ts/wt` basetemp. This includes each original +store's in-flight/later writer controls and the decoded selected ProbeRequest. +Focused mypy on installer/practice.py passed. These results remain fixture/backend +qualification, not original-data, cross-runtime, resource-peak or native/physical +acceptance. The earlier 125-case execution belongs to the preceding source epoch. + +A separate candidate build completed through `RuntimeStack.build/complete`, +including native preparation and client imports, without publishing a live route. +The selected annotation control now uses `InstallCommands.probe` and the real +interpreter worker pipes instead of calling `ProbeRequest.answer` in-process. +Executed through the existing `RuntimeStack.test` owner against that installed +candidate: one collected, one passed, 5.45 seconds, exit0. The outer command took +70.468 seconds; this is not a 5.45-second end-to-end timing. The declared dev extra +was installed into the existing candidate prefix by that test owner. Its package +path, resolved lock and build phases are retained in the owning build record. +This proves the worker transport on private fixture data, not the original-data +capture, aggregate resource peak or native-agent/physical journey. diff --git a/src/agent_comms/assignment_store.py b/src/agent_comms/assignment_store.py index 38c5df6f14..a95cc6fba1 100644 --- a/src/agent_comms/assignment_store.py +++ b/src/agent_comms/assignment_store.py @@ -102,14 +102,8 @@ def settle_harness_inputs( Every step runs in the caller's coordinator write, so the harness attempt and its claim consequence commit together or not at all. """ - claims = WakeAssignment.select( - db, - where=f"recipient_lookup=? AND wire_seq IN ({','.join('?' * len(sources))})", - parameters=(recipient_lookup, *(source.seq for source in sources)), - ) + claims = WakeAssignment.for_harness_sources(db, recipient_lookup, sources) for claim in claims: - if claim.source not in sources: - raise IdentityConflict("harness delivery names another message at this sequence") for disposition in steps(claim.lifecycle): # The revision fence refuses a step if the native owner moved the claim meanwhile. claim = self.transition_in( diff --git a/src/agent_comms/bus.py b/src/agent_comms/bus.py index ff1bbee7e1..e7e5a6f49b 100644 --- a/src/agent_comms/bus.py +++ b/src/agent_comms/bus.py @@ -51,7 +51,9 @@ ) from agent_comms.errors import CommittedMessageError, RelationViolationError, UnclaimedError from agent_comms.nominal.field_codec import ClaimId, FieldCodec, MessageId, ThreadLookup -from agent_comms.locked_store import BusFile, FileStore, SQLiteReader, StoreCommittedError +from agent_comms.locked_store import ( + BusFile, CompleteInputReader, FileStore, SQLiteReader, StoreCommittedError, +) from agent_comms.message_reference import MessageReference from agent_comms.private_path import FileIdentity, PresentPath from agent_comms.read_basis import CursorRow @@ -523,7 +525,10 @@ def observing(self) -> Iterator[sqlite3.Connection]: yield db -class Bus(FileStore, StoredBusReader, ReconciledFile, WalStore, CarriedWriter, BusFile, Watched): +class Bus( + FileStore, StoredBusReader, ReconciledFile, WalStore, CarriedWriter, BusFile, + CompleteInputReader, Watched, +): """One SQLite transaction owns the sequence and all of a record's fields.""" filename = "bus.sqlite3" diff --git a/src/agent_comms/bus_tables.py b/src/agent_comms/bus_tables.py index 343189403a..f8abf75d70 100644 --- a/src/agent_comms/bus_tables.py +++ b/src/agent_comms/bus_tables.py @@ -20,6 +20,7 @@ ProjectionProducerTables, RowNotFoundError, TableReference, + TableSource, TypedRow, ) @@ -112,10 +113,10 @@ class MessageIdentitySource(MessageReference, messages.BusProjectionTables): """A record's existing reference, searchable without recomputing its content hash.""" @classmethod - def practice_sequence(cls, db: sqlite3.Connection, source: sqlite3.Connection) -> int: - through = messages.BusRow.last_seq(source) + def practice_sequence(cls, db: sqlite3.Connection, source: TableSource) -> int: + through = messages.BusRow.last_seq(db, table_source=source) if through: - expected = cls.require(source, seq=through) + expected = cls.require(db, table_source=source, seq=through) if cls.require(db, seq=through) != expected: raise RelationViolationError("Practice bus identity differs at its last sequence") return through diff --git a/src/agent_comms/cohort_schema.py b/src/agent_comms/cohort_schema.py index 194933cbfa..322906cdf0 100644 --- a/src/agent_comms/cohort_schema.py +++ b/src/agent_comms/cohort_schema.py @@ -32,6 +32,7 @@ Index, InitialOneWayColumn, NoDelete, + RetainsChildren, SharedFileTables, SQLiteJournalMode, TextStorage, @@ -60,6 +61,7 @@ class AwarenessTable(SharedFileTables, DurableTables): @dataclass(frozen=True, kw_only=True) class ClaimBatchAcknowledgements( + RetainsChildren, InitialOneWayColumn, GuardedColumns, NoDelete, diff --git a/src/agent_comms/context_store.py b/src/agent_comms/context_store.py index 18246929c8..8c7fb8574d 100644 --- a/src/agent_comms/context_store.py +++ b/src/agent_comms/context_store.py @@ -14,6 +14,7 @@ from typing import Annotated, ClassVar, Self from .errors import RelationViolationError, StoredHistoryUnreadable +from .locked_store import InputCaptureFile from .nominal.field_codec import ContextStoreId, FieldCodec, MalformedRecord, NullMember from .native_session_reopen import NativeSessionIdentity from .registration import Registration @@ -31,7 +32,10 @@ UnrecordedSegment, ) -from .typed_table import StoreRows, Column, DurableStore, Generated, Index, RowidSequenced, SingletonRow, TypedTable +from .typed_table import ( + AttachedPracticeRebuild, StoreRows, Column, DurableStore, Generated, Index, + RowidSequenced, SingletonRow, TableSchema, TypedTable, +) from .turn_item import NativeItemId from .native_session_reopen import PiSessionId @@ -242,6 +246,35 @@ def recorded(self, db: sqlite3.Connection) -> RecordedContextManifest: parts.append(ContextSegmentVersion.require(db, id=version).segment) return self.identity.with_segments(tuple(parts)) + @classmethod + def retain_required_rows( + cls, original: AttachedPracticeRebuild, schema: type[TableSchema], + ) -> int: + """The stored request owns its serialized version, entry and predecessor edges.""" + changed = super().retain_required_rows(original, schema) + db = original.destination + for request in cls.select(db): + changed += original.retain_rows(ContextStoreInfo) + ContextStoreInfo.require(db, singleton=1) + changed += original.retain_keys( + ContextSegmentVersion, tuple({"id": version} for version in request.versions), + ) + record = request.recorded(db) # Missing original versions refuse, never disappear. + for part in record.segments: + predecessors = part.predecessors(request.id) + changed += original.retain_keys(cls, tuple({"id": key} for key in predecessors)) + for key in predecessors: + if cls.require(db, id=key).identity.thread != request.identity.thread: + raise RelationViolationError("Context predecessor is missing or foreign") + changed += original.retain_keys(ContextEntry, tuple( + {"session_id": ref.identity.session_id, + "session_file": ref.identity.session_file, "entry_id": ref.entry_id} + for ref in part.entries + )) + for reference in part.entries: + ContextEntry.original(db, reference) + return changed + class ContextPriorSupersededError(RelationViolationError): """Another writer recorded this thread's context after the prior this writer holds.""" @@ -447,7 +480,7 @@ def reference(self, manifest: ContextManifest) -> ContextManifestReference: @dataclass(frozen=True) -class ContextStore(DurableStore): +class ContextStore(DurableStore, InputCaptureFile): root: Path filename: ClassVar[str] = "turn_context.sqlite3" tables = ContextStoreRows @@ -484,6 +517,51 @@ def entries( session_file=identity.session_file, entry_id=entry_id)} + def original_segments(self) -> Iterator[SegmentManifest]: + """Original segments actually present in this selected entry/request store. + + Requests restore through the same original predecessor/entry reader as + the browser. Entry-only stores need no invented request/store identity. + """ + with self.snapshot() as db: + for entry in ContextEntry.stream(db): + yield from entry.manifest.original_values() + for request in ContextRequest.stream(db): + source = ContextStoreInfo.require(db, singleton=1).source + for root in self.resolve(db, source, request).value.segments: + yield from root.original_values() + + def retain_contexts( + self, destination: ContextStore, identities: Iterable[NativeSessionIdentity], *, + maximum_bytes: int, requests: Iterable[ContextManifestReference] = (), + ) -> None: + """Retain original entry manifests and explicitly selected saved requests. + + Raw writable values, source identity, predecessor/version/entry closure + and sequence high-water marks travel unchanged. Session paths are keys, + not relocated storage or permission to launch or replay a native session. + Entry-only selection retains no request or fabricated store identity. + """ + with type(self).retaining_rows( + self.path, destination.path, maximum_bytes=maximum_bytes, + ) as original: + original.retain_keys(ContextEntry, tuple( + {"session_id": identity.session_id, "session_file": identity.session_file} + for identity in dict.fromkeys(identities) + )) + selected = tuple(dict.fromkeys(requests)) + if selected: + original.retain_rows(ContextStoreInfo) + source = ContextStoreInfo.require(original.destination, singleton=1).source + for reference in selected: + if reference.source != source: + raise RelationViolationError("Selected context belongs to another store") + original.retain_keys(ContextRequest, tuple( + {"id": reference.record_id} for reference in selected + )) + for reference in selected: + ContextRequest.require(original.destination, id=reference.record_id) + def prior(self, owner: ThreadIncarnation) -> RecordedContextState: """Read the thread's previous request from a snapshot, outside any write lock.""" with self.snapshot() as db: diff --git a/src/agent_comms/coordination_database.py b/src/agent_comms/coordination_database.py index 4bccb0aa98..f99b06e5b9 100644 --- a/src/agent_comms/coordination_database.py +++ b/src/agent_comms/coordination_database.py @@ -15,7 +15,7 @@ from agent_comms.coordination_errors import CoordinationReadUnavailable, IntegrityViolationError from agent_comms.coordination_schema import CoordinatorTable -from agent_comms.locked_store import FileStore, SQLiteReader +from agent_comms.locked_store import FileStore, InputCaptureFile, SQLiteReader from agent_comms.typed_table import ( CarriedWriter, ReconciledFile, @@ -27,7 +27,6 @@ from .nominal.declared_family import DeclaredFamily from .store_files import DEFAULT_STORE_CONTENTION, StoreContention - class StoreFile(DeclaredFamily): """One decoded file state, with the consumer's result inside its read lifetime.""" @@ -67,7 +66,7 @@ def _capture(self, consumer, *arguments): yield consumer.capture(connection, *arguments) -class CoordinationStore(FileStore, CarriedWriter, ReconciledFile): +class CoordinationStore(FileStore, CarriedWriter, ReconciledFile, InputCaptureFile): """The private coordinator database and its declared schema.""" filename = "coordination.sqlite3" @@ -119,7 +118,7 @@ def __init__( with CoordinationReadUnavailable.checking(): self._connection = self._open_connection(contention) try: - self._configure_connection() + self.configure_connection(self._connection) self._initialize_schema() self._enforce_private_modes() except BaseException: @@ -133,13 +132,15 @@ def _initialize_schema(self) -> None: """The root's own composition of this file decides its schema and its strays.""" self.schema_at(self.path).install_schema(self._connection, self.schema_change(self.path)) - def _configure_connection(self) -> None: + @classmethod + def configure_connection(cls, connection: sqlite3.Connection) -> None: + """Ordinary opening and selected loading use the same row-owned SQL behavior.""" for row_type in CoordinatorTable.declarations(): - row_type.configure_connection(self._connection) - self._connection.row_factory = sqlite3.Row - self._connection.execute("PRAGMA foreign_keys = ON") + row_type.configure_connection(connection) + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") # Rollback journal avoids a post-schema WAL-mode race among fresh openers. - self._connection.execute("PRAGMA synchronous = FULL") + connection.execute("PRAGMA synchronous = FULL") def _prepare_private_file(self) -> None: self.path.parent.mkdir(parents=True, exist_ok=True) diff --git a/src/agent_comms/coordination_session.py b/src/agent_comms/coordination_session.py index 1a621299a1..8269bea2bb 100644 --- a/src/agent_comms/coordination_session.py +++ b/src/agent_comms/coordination_session.py @@ -206,7 +206,8 @@ def _prepare_private_file(self) -> None: def _open_connection(self, contention: StoreContention) -> sqlite3.Connection: return self._open_reader(self.path, contention) - def _configure_connection(self) -> None: + @classmethod + def configure_connection(cls, connection: sqlite3.Connection) -> None: pass # _open_reader configured query-only behavior before publication. def _initialize_schema(self) -> None: diff --git a/src/agent_comms/coordination_tables/annotations.py b/src/agent_comms/coordination_tables/annotations.py index a582ecf341..9828134538 100644 --- a/src/agent_comms/coordination_tables/annotations.py +++ b/src/agent_comms/coordination_tables/annotations.py @@ -1,16 +1,25 @@ """Durable label/request declarations; the existing coordinator owns storage.""" from __future__ import annotations +import sqlite3 +from abc import abstractmethod +from collections.abc import Iterable, Iterator from dataclasses import dataclass, field -from typing import Annotated, ClassVar +from typing import Annotated, ClassVar, Self from agent_comms.coordination_schema import CoordinatorRow from agent_comms.message_reference import MessageReference +from agent_comms.turn_context import ContextSpan, SegmentManifest from agent_comms.typed_table import ( + LOCAL_TABLE_SOURCE, + AttachedPracticeRebuild, Column, Generated, Index, RowId, + TableSchema, + TableSource, + TypedTable, UnallocatedRowId, sql_literal, ) @@ -19,8 +28,46 @@ from agent_comms.working_memory_requests import AnnotationOutcome, DisclosureRequest +class ContextSpanRows(TypedTable, declared_name=None): + """Stored answers and request dispositions share original-span eligibility.""" + + __slots__ = () + + @property + @abstractmethod + def context_span(self) -> ContextSpan: ... + + @classmethod + def for_segments( + cls, db: sqlite3.Connection, segments: Iterable[SegmentManifest], *, + table_source: TableSource = LOCAL_TABLE_SOURCE, **key: object, + ) -> Iterator[Self]: + originals: dict[str, list[SegmentManifest]] = {} + for segment in segments: + originals.setdefault(segment.sha256, []).append(segment) + for row in cls.stream_keys(db, tuple( + dict(key, segment_digest=digest) for digest in originals + ), table_source=table_source): + span = row.context_span + if any(segment.contains_span(span) for segment in originals[span.segment_sha256]): + yield row + + @classmethod + def retain_segments( + cls, original: AttachedPracticeRebuild, segments: Iterable[SegmentManifest], + ) -> int: + # Restored requests may repeat large original prefixes. Consume one + # segment at a time, not a second materialized context corpus. + members = cls.members_with(cls) + return sum(row.retain_original(original) + for segment in segments + for member in members + for row in member.for_segments(original.destination, (segment,), + table_source=original.source)) + + @dataclass(frozen=True, kw_only=True) -class SpanAnnotationsRow(CoordinatorRow): +class SpanAnnotationsRow(ContextSpanRows, CoordinatorRow): label: ModelLabel created_at_ms: int id: Annotated[RowId, Column(primary_key=True, auto_increment=True)] = field( @@ -78,22 +125,17 @@ def effective(cls, rows: tuple[SpanAnnotationsRow, ...]): # Row ids order successive corrections from the same authority. Nominal # inheritance, not sequence, decides human versus model authority. latest = {} - for row in sorted(rows, key=lambda value: value.id): + for row in sorted(rows, key=lambda value: int(value.id)): latest[type(row.label)] = row.label return SpanLabel.effective(tuple(latest.values())) - @classmethod - def for_digests(cls, db, digests, classifier): - if not digests: - return () - return cls.select(db, - where='segment_digest IN (' + ','.join('?' for _ in digests) + ')', - parameters=tuple(digests), order_by=("id",), - classifier=classifier.classifier, classifier_pin=classifier.pin) + @property + def context_span(self) -> ContextSpan: + return self.label.span @dataclass(frozen=True, kw_only=True) -class AnnotationRequestsRow(CoordinatorRow): +class AnnotationRequestsRow(ContextSpanRows, CoordinatorRow): request: DisclosureRequest outcome: AnnotationOutcome created_at_ms: int @@ -137,6 +179,33 @@ class AnnotationRequestsRow(CoordinatorRow): unique = (SpanAnnotationsRow.address,) indexes = (Index(("grant_reference", "created_at_ms")),) + @property + def context_span(self) -> ContextSpan: + return self.request.span + + @classmethod + def retain_required_rows( + cls, original: AttachedPracticeRebuild, schema: type[TableSchema], + ) -> int: + """Keep original grant usage and each request's complete answer/correction group.""" + changes = super().retain_required_rows(original, schema) + changes += original.retain_keys(cls, tuple( + {"grant_reference": row.request.grant} for row in cls.stream(original.destination) + )) + # One natural address can name multiple correction rows; it is not a + # foreign key. Compare the actual declared generated columns, without + # making storage import a reader to reconstruct their domain values. + related = " AND ".join( + f"{SpanAnnotationsRow.qualified(name)} IS {cls.qualified(name)}" + for name in SpanAnnotationsRow.address + ) + changes += original.retain_rows( + SpanAnnotationsRow, + where=f"EXISTS (SELECT 1 FROM main.{LOCAL_TABLE_SOURCE.qualified(cls.declared_name)} " + f"WHERE {related})", + ) + return changes + @classmethod def requests_since(cls, db, reference: MessageReference, since: int) -> int: return cls.count(db, where='created_at_ms>?', parameters=(since,), grant_reference=reference) diff --git a/src/agent_comms/coordination_tables/assignments.py b/src/agent_comms/coordination_tables/assignments.py index 25b8225162..aeadf074b9 100644 --- a/src/agent_comms/coordination_tables/assignments.py +++ b/src/agent_comms/coordination_tables/assignments.py @@ -2,8 +2,10 @@ from __future__ import annotations +import sqlite3 +from collections.abc import Iterator, Sequence from dataclasses import dataclass, replace -from typing import Annotated, ClassVar +from typing import Annotated, ClassVar, Self from agent_comms.assignment_states import AssignmentState, EngagedAssignment from agent_comms.frozen_recipient import FrozenRecipient @@ -25,6 +27,7 @@ from agent_comms.nominal.field_codec import AssignmentId, MessageId, SnapshotExclude, ThreadLookup, WireName, projected from agent_comms.message_reference import MessageReference from agent_comms.typed_table import ( + LOCAL_TABLE_SOURCE, Column, FirstValueState, ForeignKey, @@ -40,6 +43,7 @@ NoDelete, NonDecreasing, RequiredColumn, + TableSource, WriteOnceColumns, ) from agent_comms.wake_policy import ( @@ -126,6 +130,26 @@ def require_same_acceptance(self, candidate: WakeAssignment) -> None: def source(self) -> MessageReference: return MessageReference(self.wire_seq, self.message_id) + @classmethod + def for_harness_sources( + cls, db: sqlite3.Connection, recipient_lookup: ThreadLookup, + sources: Sequence[MessageReference], *, + table_source: TableSource = LOCAL_TABLE_SOURCE, + ) -> Iterator[Self]: + """Existing claims for these original messages, including settled claims. + + Sequence finds candidates through the declared unique key; the complete + reference must still agree. A conflicting message refuses rather than + disappearing from selection. An attempt need not already have a claim. + """ + claims = cls.select_keys(db, tuple( + {"recipient_lookup": recipient_lookup, "wire_seq": source.seq} for source in sources + ), table_source=table_source) + for claim in claims: + if claim.source not in sources: + raise IdentityConflict("harness delivery names another message at this sequence") + yield claim + def require_engaged_binding(self) -> AssignmentBinding: engagement = self.lifecycle.require_engagement() return AssignmentBinding( diff --git a/src/agent_comms/coordination_tables/executions.py b/src/agent_comms/coordination_tables/executions.py index 5472b8ef7a..027c6fffd2 100644 --- a/src/agent_comms/coordination_tables/executions.py +++ b/src/agent_comms/coordination_tables/executions.py @@ -55,6 +55,7 @@ NonDecreasing, NonIncreasing, OrdinalTransitions, + RetainsChildren, TextStorage, ) from agent_comms.wake_candidate_index import ProjectionUnavailableError @@ -275,6 +276,7 @@ def references(cls): @dataclass(frozen=True, slots=True) class ExecutionRecord( + RetainsChildren, LifecycleTransitions, OrdinalTransitions, IncrementingColumn, diff --git a/src/agent_comms/coordination_tables/harness_deliveries.py b/src/agent_comms/coordination_tables/harness_deliveries.py index c1726c4d84..92fbcaaff3 100644 --- a/src/agent_comms/coordination_tables/harness_deliveries.py +++ b/src/agent_comms/coordination_tables/harness_deliveries.py @@ -14,19 +14,26 @@ from agent_comms.coordination_errors import IdentityConflict from agent_comms.coordination_schema import CoordinatorRow +from agent_comms.coordination_tables.assignments import WakeAssignment from agent_comms.coordination_tables.participants import Participants from agent_comms.nominal.field_codec import MessageId, ThreadLookup, TranscriptRecordId -from agent_comms.harness_delivery_states import MonitorEvidence, Outcome, UnrecordedMonitor +from agent_comms.harness_delivery_states import ( + MonitorEvidence, Outcome, UnconfirmedOutcome, UnrecordedMonitor, +) from agent_comms.harness_install import HarnessSession +from agent_comms.input_origin import HarnessAttemptOrigin from agent_comms.message_reference import MessageReference -from agent_comms.typed_table import Column, ForeignKey, FrozenColumn, ImmutableRows, Index, NoDelete +from agent_comms.typed_table import ( + LOCAL_TABLE_SOURCE, AttachedPracticeRebuild, Column, ForeignKey, FrozenColumn, + ImmutableRows, Index, NoDelete, RetainsChildren, TableSchema, TableSource, +) if TYPE_CHECKING: from agent_comms.assignment_store import AssignmentStore @dataclass(frozen=True, slots=True) -class HarnessDelivery(NoDelete, CoordinatorRow, declared_name="harness_deliveries"): +class HarnessDelivery(RetainsChildren, NoDelete, CoordinatorRow, declared_name="harness_deliveries"): """One original attempt to put a bus message into a recipient's harness session.""" bridge: Annotated[type[HarnessSession], FrozenColumn(primary_key=True)] @@ -45,6 +52,36 @@ def references(cls) -> tuple[ForeignKey, ...]: return (ForeignKey(columns=("recipient_lookup",), target=Participants, target_columns=("participant_lookup",)),) + @classmethod + def retain_attempt( + cls, original: AttachedPracticeRebuild, recipient: ThreadLookup, + attempt: HarnessAttemptOrigin, + ) -> int: + """Keep the original keyed attempt and reject a different full message reference.""" + key = {"bridge": attempt.bridge, "recipient_lookup": recipient, + "message_id": attempt.reference.message_id} + changed = original.retain_keys(cls, (key,)) + retained = cls.require(original.destination, table_source=LOCAL_TABLE_SOURCE, **key) + if retained.reference != attempt.reference: + raise IdentityConflict("Retained harness attempt names another message sequence") + return changed + + @classmethod + def retain_required_rows( + cls, original: AttachedPracticeRebuild, schema: type[TableSchema], + ) -> int: + """Keep original observation parts and every existing claim on these attempts.""" + changed = super().retain_required_rows(original, schema) + for delivery in cls.select(original.destination): + claims = WakeAssignment.for_harness_sources( + original.destination, delivery.recipient_lookup, (delivery.reference,), + table_source=original.source, + ) + changed += original.retain_keys(WakeAssignment, tuple( + {"assignment_id": claim.assignment_id} for claim in claims + )) + return changed + @property def reference(self) -> MessageReference: return MessageReference(self.seq, self.message_id) @@ -110,6 +147,24 @@ def for_recipients(cls, db: sqlite3.Connection, key=lambda row: (row.seq, row.recipient_lookup, row.bridge.declared_name), )) + @classmethod + def unconfirmed_for( + cls, db: sqlite3.Connection, owners: tuple[ThreadLookup, ...], *, + table_source: TableSource = LOCAL_TABLE_SOURCE, + ) -> tuple[Self, ...]: + """Original uncertain attempts for these incarnations, in message order. + + The outcome family owns membership; declared columns own query encoding. + A source attachment changes location, not the stored outcome or identity. + """ + outcomes = Outcome.members_with(UnconfirmedOutcome) + keys = tuple( + {"recipient_lookup": owner, "outcome": outcome} + for owner in owners for outcome in outcomes + ) + return tuple(sorted(cls.select_keys(db, keys, table_source=table_source), + key=lambda row: row.seq)) + @classmethod def latest(cls, db: sqlite3.Connection, bridge: type[HarnessSession], recipient_lookup: ThreadLookup) -> tuple[MessageReference, ...]: diff --git a/src/agent_comms/coordination_tables/publications.py b/src/agent_comms/coordination_tables/publications.py index adf38fb3a8..69fad5de39 100644 --- a/src/agent_comms/coordination_tables/publications.py +++ b/src/agent_comms/coordination_tables/publications.py @@ -28,6 +28,7 @@ Generated, GuardedInserts, ImmutableRows, + RetainsChildren, ValidatedInserts, ) @@ -37,7 +38,7 @@ @dataclass(frozen=True, slots=True) class PublicationIntents( - ValidatedInserts, GuardedInserts, ImmutableRows, CoordinatorRow + RetainsChildren, ValidatedInserts, GuardedInserts, ImmutableRows, CoordinatorRow ): """One obligation route's frozen response: the exact Message it publishes.""" diff --git a/src/agent_comms/goal_attempts.py b/src/agent_comms/goal_attempts.py index 18f867a032..2491714837 100644 --- a/src/agent_comms/goal_attempts.py +++ b/src/agent_comms/goal_attempts.py @@ -76,6 +76,7 @@ WorkDecision, ) from .typed_table import ( + LOCAL_TABLE_SOURCE, Column, Generated, ReconciledFile, @@ -83,6 +84,7 @@ SQLiteJournalMode, StoreWriter, TableReference, + TableSource, TypedTable, ) @@ -212,9 +214,10 @@ def __post_init__(self) -> None: raise ValueError("Generation must be positive.") @classmethod - def _keyed(cls, db: sqlite3.Connection, key: dict[str, object]) -> list[Self]: + def _keyed(cls, db: sqlite3.Connection, key: dict[str, object], *, + table_source: TableSource = LOCAL_TABLE_SOURCE) -> list[Self]: try: - return super()._keyed(db, key) + return super()._keyed(db, key, table_source=table_source) except (sqlite3.Error, MalformedRecord) as error: raise StorageUncertainError("Invalid persisted goal generation.") from error diff --git a/src/agent_comms/input_attempt.py b/src/agent_comms/input_attempt.py index cf29663c95..5b5c2fa7c5 100644 --- a/src/agent_comms/input_attempt.py +++ b/src/agent_comms/input_attempt.py @@ -339,9 +339,13 @@ def digest(self) -> TextDigest: return TextDigest.of(self.source_text) def matches_owner(self, source_owner: ThreadIncarnation) -> bool: - # Input with a recorded author uses that author's creation time. Older - # rows with only a name do not get an inferred thread identity. - return self.origin.matches_owner(self.owner, source_owner) + # The origin owns author evidence; an owned bus key additionally + # retains its original recipient birth. Neither invents a missing birth. + return (self.origin.matches_owner(self.owner, source_owner) + and source_owner.recorded_or( + lambda recorded: self.key.matches_recorded_owner(recorded.created_at), + self.key.matches_unrecorded_owner, + )) def matches_claim(self, claim_scope: int) -> bool: return self.claim_generation == claim_scope diff --git a/src/agent_comms/input_disposition.py b/src/agent_comms/input_disposition.py index dd41a7c38d..0719a385c2 100644 --- a/src/agent_comms/input_disposition.py +++ b/src/agent_comms/input_disposition.py @@ -17,8 +17,7 @@ from .coordination_database import CoordinationStore, ReadableStore from .coordination_tables.harness_deliveries import HarnessDelivery from .errors import RelationViolationError -from .nominal.field_codec import GoalId, InputId, RootId, TurnId -from .harness_delivery_states import Outcome, UnconfirmedOutcome +from .nominal.field_codec import GoalId, InputId, RootId, ThreadLookup, TurnId from .input_attempt import ( UNOBSERVED_DELIVERY_SCOPE, GoalInputDecision, @@ -34,13 +33,14 @@ from .input_key import InputKey, InputKeyText, TurnInputKey from .input_origin import HarnessAttemptOrigin, InputOrigin, UnattributedInputOrigin from .lifecycle_capability import Unresolved -from .locked_store import DocumentChange, PublishedProperty +from .locked_store import CompleteInputReader, DocumentChange, PublishedProperty from .message_reference import MessageReference from .messages import MessageInput from .native_input_proof import NativeContextProof from .reported_turn import ReportedTurnState from .tool_result import ToolResult from .typed_table import ( + AttachedPracticeRebuild, Column, DurableStore, Generated, @@ -248,7 +248,7 @@ def _members(column: str, values) -> tuple[str, tuple]: @dataclass(frozen=True, slots=True) -class InputDispositions(DurableStore): +class InputDispositions(DurableStore, CompleteInputReader): """Indexed durable input rows; each operation reads only the rows it names.""" root: Path @@ -287,19 +287,15 @@ def lookup(self, key: InputKey) -> InputAttempt: def coordinator_reader(self) -> ReadableStore: return ReadableStore(CoordinationStore.path_in(self.root)) + def _harness_deliveries(self, owners: tuple[ThreadLookup, ...]) -> tuple[HarnessDelivery, ...]: + with self.coordinator_reader.reader_snapshot() as db: + return HarnessDelivery.unconfirmed_for(db, owners) + def harness_attempts(self, entries: tuple[RegistryEntry, ...]) -> tuple[ReservedInput, ...]: """Uncertain harness attempts for these entries' frozen incarnations, as the inputs they are. Derived on every read and never stored by reading.""" owners = {entry.thread.incarnation.require_recorded().lookup: entry for entry in entries} - outcomes = Outcome.members_with(UnconfirmedOutcome) - with self.coordinator_reader.reader_snapshot() as db: - attempts = HarnessDelivery.select( - db, - where=(f'"recipient_lookup" IN ({", ".join("?" for _ in owners)}) ' - f'AND "outcome" IN ({", ".join("?" for _ in outcomes)})'), - parameters=(*owners, *(outcome.declared_name for outcome in outcomes)), - order_by=("seq",), - ) + attempts = self._harness_deliveries(tuple(owners)) messages = Bus(Bus.path_in(self.root)).input_messages_for_references( attempt.reference for attempt in attempts) return tuple( @@ -565,3 +561,40 @@ def dismiss(document: InputDocument) -> DocumentChange[InputDocument, InputDocum return self._update(*self._owned_or_attempted(owners, attempted), dismiss).record( *attempted).delivery_overview(owners, scope=scope) + + +@dataclass(frozen=True, slots=True) +class SnapshotInputDispositions(InputDispositions, declared_name=None): + """The same unresolved projection, with harness rows from an active source cut. + + ``root`` names the retained bus and complete suppression ledger. The caller + owns their consistent capture and this attachment's lifetime. This transient + view declares no file/schema, writes no synthetic attempt and grants no replay. + """ + + original: AttachedPracticeRebuild + + def _harness_deliveries(self, owners: tuple[ThreadLookup, ...]) -> tuple[HarnessDelivery, ...]: + return HarnessDelivery.unconfirmed_for( + self.original.destination, owners, table_source=self.original.source, + ) + + def retain_unresolved(self, entries: tuple[RegistryEntry, ...]) -> InputDocument: + """Seed original attempt groups selected by the ordinary unresolved reader. + + The complete copied ledger owns suppression in every state. This offline + selection reuses the same attachment and producer a second time, rather + than restating its key/outcome/suppression decisions. An unresolved ledger + row may keep an additional original attempt witness at its key; it still + wins the projection. No native association is inferred from a scalar id. + """ + selected = self.unresolved(entries) + recipients = {entry.thread.name: entry.thread.incarnation.require_recorded().lookup + for entry in entries} + for attempted in self.harness_attempts(entries): + if attempted.key in selected.rows: + for source in attempted.origin.harness_sources(): + HarnessDelivery.retain_attempt( + self.original, recipients[attempted.owner], source, + ) + return selected diff --git a/src/agent_comms/input_key.py b/src/agent_comms/input_key.py index cd38a58dfd..9cf508ceb7 100644 --- a/src/agent_comms/input_key.py +++ b/src/agent_comms/input_key.py @@ -47,6 +47,13 @@ def public_text(self) -> str: def identify(self, update: InputStartedUpdate) -> InputStartedUpdate: return update + def matches_recorded_owner(self, created_at: float) -> bool: + """A birthless key adds no birth claim to its origin's owner check.""" + return True + + def matches_unrecorded_owner(self) -> bool: + return True + @dataclass(frozen=True) class AcpInputKey(InputKey): @@ -90,6 +97,13 @@ class OwnedBusInputKey(BusInputKey): def text(self) -> str: return super().text() + ":owner:" + self.owner_created.hex() + def matches_recorded_owner(self, created_at: float) -> bool: + """The original recipient's recorded birth must still match.""" + return created_at == self.owner_created + + def matches_unrecorded_owner(self) -> bool: + return False + @dataclass(frozen=True) class TurnInputKey(InputKey): turn_id: TurnId diff --git a/src/agent_comms/input_origin.py b/src/agent_comms/input_origin.py index a9211b20a3..ec2b8e7994 100644 --- a/src/agent_comms/input_origin.py +++ b/src/agent_comms/input_origin.py @@ -61,6 +61,10 @@ def retained_fact(self, source: StoredInput): def require_human(self) -> HumanInputOrigin: raise RelationViolationError("Input lacks an original human author witness") + def harness_sources(self) -> tuple[HarnessAttemptOrigin, ...]: + """An origin without a harness witness contributes no harness sources.""" + return () + def matches_owner(self, recorded_name, incarnation): return recorded_name == incarnation.name @@ -124,6 +128,9 @@ class HarnessAttemptOrigin(InputOrigin): bridge: type[HarnessSession] reference: MessageReference + def harness_sources(self) -> tuple[HarnessAttemptOrigin, ...]: + return (self,) + @dataclass(frozen=True) class HumanInputOrigin(InputOrigin, ClaimScoped): diff --git a/src/agent_comms/installer/practice.py b/src/agent_comms/installer/practice.py index 51137d8fb1..db36562689 100644 --- a/src/agent_comms/installer/practice.py +++ b/src/agent_comms/installer/practice.py @@ -7,15 +7,28 @@ from abc import ABC, abstractmethod from dataclasses import dataclass, field from pathlib import Path - -from agent_comms.nominal.field_codec import boundary_decoder +from math import ceil +from collections.abc import Iterable, Iterator +from contextlib import contextmanager +from agent_comms.context_store import ContextManifestReference, ContextStore +from agent_comms.coordination_database import CoordinationStore +from agent_comms.coordination_tables.annotations import ContextSpanRows +from agent_comms.input_disposition import InputDocument, SnapshotInputDispositions +from agent_comms.native_runtime_input import NativeRuntimeInput +from agent_comms.participant_store import ParticipantRows, ParticipantStore +from agent_comms.registry_document import RegistryEntry +from agent_comms.native_session_reopen import NativeSessionIdentity +from agent_comms.nominal.field_codec import ThreadLookup, boundary_decoder from agent_comms.installer.refusals import UpgradeRefused +from agent_comms.installer.backup import pinned_bus_files, saved_bus_files +from agent_comms.registry_store import RegistryCut, RegistryStore from agent_comms.installer.declared_uv import DeclaredUv from agent_comms.launch_runtime import LaunchRuntime from agent_comms.locked_store import ( - DeclaredFile, LiveFilesRequest, RegistryFile, RetainedCopyRequest, RootFile, UndeclaredFile, + CompleteInputReader, DeclaredFile, InputCaptureFile, LiveFilesRequest, RegistryFile, + RetainedCopyRequest, RootFile, UndeclaredFile, ) -from agent_comms.maintenance_barrier import MaintenanceBarrier +from agent_comms.maintenance_barrier import MaintenanceBarrier, NoMaintenanceHolder from agent_comms.storage_rules import StorageRule from agent_comms.private_path import PrivateDirectoryRole @@ -24,15 +37,28 @@ class PracticeMeasurement: bytes: int + @classmethod + def capture(cls, copy: Path) -> PracticeMeasurement: + """Count the complete materialized tree, including SQLite companions and stages.""" + return cls(sum(path.stat().st_size for path in copy.rglob("*") if path.is_file())) + + def remaining(self, maximum_bytes: int) -> int: + """An explicit selected-input allowance, not a limit on complete backups.""" + if self.bytes > maximum_bytes: + raise UpgradeRefused( + f"Practice copy has {self.bytes} bytes, exceeding {maximum_bytes} allowed bytes") + return maximum_bytes - self.bytes + @classmethod def of(cls, copy: Path, floor: float) -> PracticeMeasurement: """Refuse a copy without room for the rehearsal's second copy above the disk floor. The free space is what a copy can cost; a fixed byte cap beside it refused a 3.01 GiB bus with 29 GiB free and stopped install 45.""" - size = sum(path.stat().st_size for path in copy.rglob("*") if path.is_file()) - if shutil.disk_usage(copy.parent).free < 2 * size + floor: - raise UpgradeRefused(f"Insufficient free space for rehearsal of {size} copied bytes") - return cls(size) + measured = cls.capture(copy) + if shutil.disk_usage(copy.parent).free < 2 * measured.bytes + floor: + raise UpgradeRefused( + f"Insufficient free space for rehearsal of {measured.bytes} copied bytes") + return measured @dataclass(frozen=True) @@ -133,11 +159,164 @@ def copied_names(self) -> tuple[Path, ...]: resource.live(self.root), resource.path, source, ) for source, resource in self.copied.items())) - def include(self, paths: tuple[Path, ...]) -> None: - floor = StorageRule.load(self.root).floor_gib * 2**30 + @property + def floor_bytes(self) -> int: + return ceil(StorageRule.load(self.root).floor_gib * 2**30) + + @property + def available_bytes(self) -> int: + return shutil.disk_usage(self.path.parent).free - self.floor_bytes + + def prepare(self) -> None: + """One space and private-directory owner for whole and selected materialization.""" PrivateDirectoryRole.make_parents(self.path) - if shutil.disk_usage(self.path.parent).free < floor: + if self.available_bytes < 0: raise UpgradeRefused("Insufficient free space before building the practice copy") + + def remaining_bytes(self, maximum_bytes: int) -> int: + """Share one explicit total allowance across incrementally selected stores. + + The current tree, not the copied-resource index, owns used bytes. This + includes retained WAL/groups and data placed there by another copy step. + Caller owns this private copy's serial materialization and wider cut. + """ + return min(PracticeMeasurement.capture(self.path).remaining(maximum_bytes), + self.available_bytes) + + @contextmanager + def capturing_inputs(self) -> Iterator[RegistryCut]: + """Enclose the first and every subsequent selected-input materialization. + + The caller already owns the install's retargeted hold. Materialization + drops its claim, including the exported claim inherited by new children, + so ordinary source writers refuse while the private target stays writable. + Pins prevent last-close WAL checkpoints, not writers. Drain + in-flight transactions separately, releasing each SQL lock before seeking + another or the registry. The hold excludes later non-holder transactions; + source observations additionally reject changes outside that writer path. + Ordinary whole-file include() and complete backups keep their own lifetime. + This supplies a recorded cut, not native launch or write authority. + """ + if self.copied or self.path.exists() or self.path.is_symlink(): + raise UpgradeRefused("Selected input capture must start before materialization") + source, target = self.root.resolve(), self.path.resolve() + if target.is_relative_to(source) or source.is_relative_to(target): + raise UpgradeRefused("Selected input capture requires a separate destination") + barrier = MaintenanceBarrier(RegistryFile.path_in(self.root)) + files = LiveFilesRequest(self.root) + resources = RootFile.owned_files(files, tuple( + owner.path_in(directory) + for owner, directory in InputCaptureFile.declarations_in(self.root) + )) + present = tuple(resource for resource in resources if files.location(resource).exists()) + with (barrier.capturing(LaunchRuntime.running()), + pinned_bus_files(files, present)): + for resource in present: + with resource.owner.writers_drained(files.location(resource)): + pass + with RegistryStore.at(self.root).current_snapshot() as original: + before = saved_bus_files(files, resources) + with NoMaintenanceHolder().acting(): + yield original + if not original.matches_current(): + raise UpgradeRefused("Registry changed during selected input capture") + if saved_bus_files(files, resources) != before: + raise UpgradeRefused("Source stores changed during selected input capture") + + def capture_inputs( + self, owners: tuple[ThreadLookup, ...], *, maximum_bytes: int, + identities: tuple[NativeSessionIdentity, ...] = (), + requests: tuple[ContextManifestReference, ...] = (), + ) -> PracticeMeasurement: + """The selected operation, including its first complete reader copy. + + Resolve births from the original cut, not caller-supplied name/generation + records. No selection callback runs before or outside this lifetime. + The returned measurement describes data, never native launch authority. + """ + with self.capturing_inputs() as original: + entries = tuple(original.snapshot.entry(original.snapshot.born(owner).name) + for owner in owners) + self.prepare() + paths = tuple(owner.path_in(directory) for owner, directory + in CompleteInputReader.declarations_in(self.root)) + files = LiveFilesRequest(self.root) + readers = saved_bus_files(files, RootFile.owned_files(files, paths)) + required = sum(member.size for group in readers.values() for member in group.values()) + if required > self.remaining_bytes(maximum_bytes): + raise UpgradeRefused("Complete input readers exceed the selected copy allowance") + self.include(paths) + self.remaining_bytes(maximum_bytes) + self.include_contexts(identities, maximum_bytes=maximum_bytes, requests=requests) + self.include_inputs(entries, maximum_bytes=maximum_bytes, identities=identities) + measured = PracticeMeasurement.capture(self.path) + measured.remaining(maximum_bytes) + return measured + + def include_contexts( + self, identities: Iterable[NativeSessionIdentity], *, maximum_bytes: int, + requests: Iterable[ContextManifestReference] = (), + ) -> None: + """Explicit selected-context inputs, never a full-file backup. + + Normal include() and the installer's live-root backup remain whole-file + operations. This selection must precede including the context resource; + an existing destination is refused rather than overwritten or reimported. + Select contexts before finalizing coordinator evidence, whose annotation + dependency selection consumes these exact retained segments. + """ + if CoordinationStore.path_in(self.root) in self.copied: + raise UpgradeRefused("Select contexts before finalizing coordinator input evidence") + self.prepare() + source = ContextStore(self.root) + resource, = RootFile.owned_files(LiveFilesRequest(self.root), (source.path,)) + source.retain_contexts( + ContextStore(self.path), identities, requests=requests, + maximum_bytes=self.remaining_bytes(maximum_bytes), + ) + self.copied[source.path] = resource + + def include_inputs( + self, entries: tuple[RegistryEntry, ...], *, maximum_bytes: int, + identities: Iterable[NativeSessionIdentity] = (), + ) -> InputDocument: + """Selected coordinator evidence beside the complete captured input readers. + + include() owns capturing the original bus and ledger; this method never + substitutes a selected ledger or creates suppression evidence. Caller owns + one consistent cut and serial materialization, not native launch authority. + """ + self.prepare() + for owner, directory in CompleteInputReader.declarations_in(self.root): + source = owner.path_in(directory) + if source.exists(): + if source not in self.copied: + raise UpgradeRefused(f"Capture complete {source.name} before selecting input evidence") + elif owner.path_in(self.path).exists(): + raise UpgradeRefused(f"Copied {source.name} has no original reader source") + contexts = ContextStore(self.path) + source_contexts = ContextStore.path_in(self.root) + if contexts.path.exists() and source_contexts not in self.copied: + raise UpgradeRefused("Copied contexts have no captured original reader source") + source = CoordinationStore.path_in(self.root) + resource, = RootFile.owned_files(LiveFilesRequest(self.root), (source,)) + segments = contexts.original_segments() if source_contexts in self.copied else () + with CoordinationStore.retaining_rows( + source, CoordinationStore.path_in(self.path), + maximum_bytes=self.remaining_bytes(maximum_bytes), + original_type=ParticipantRows, + ) as original: + selected = SnapshotInputDispositions(self.path, original).retain_unresolved(entries) + ParticipantStore.retain_owners(original, tuple( + entry.thread.incarnation.require_recorded().lookup for entry in entries + )) + NativeRuntimeInput.retain_sessions(original, identities) + ContextSpanRows.retain_segments(original, segments) + self.copied[source] = resource + return selected + + def include(self, paths: tuple[Path, ...]) -> None: + self.prepare() witnesses = set(MaintenanceBarrier(RegistryFile.path_in(self.root)).witness_paths) for resource in RootFile.owned_files(LiveFilesRequest(self.root), paths): source = resource.live(self.root) @@ -159,5 +338,4 @@ def retain(self, paths: tuple[Path, ...]) -> None: @property def measurement(self) -> PracticeMeasurement: - return PracticeMeasurement.of( - self.path, StorageRule.load(self.root).floor_gib * 2**30) + return PracticeMeasurement.of(self.path, self.floor_bytes) diff --git a/src/agent_comms/installer/probes/inputs.py b/src/agent_comms/installer/probes/inputs.py new file mode 100644 index 0000000000..6b002a7387 --- /dev/null +++ b/src/agent_comms/installer/probes/inputs.py @@ -0,0 +1,35 @@ +"""Selected original inputs captured by the qualified install worker.""" +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Annotated + +from agent_comms.context_store import ContextManifestReference +from agent_comms.installer.interpreter_operation import RuntimeProbe, TargetProbe +from agent_comms.installer.practice import PracticeCopy, PracticeMeasurement +from agent_comms.native_session_reopen import NativeSessionIdentity +from agent_comms.nominal.field_codec import PathText, ThreadLookup + + +@dataclass(frozen=True) +class SelectedInputsProbe(TargetProbe, RuntimeProbe[PracticeMeasurement]): + """Explicit selected capture in the target worker's existing retargeted hold. + + Not the ordinary complete rehearsal, a new hold/stop operation, or native + launch authority. Original coordinates and the allowance belong to this + request; no supplied callback runs outside the captured source lifetime. + """ + + root: Annotated[Path, PathText] + destination: Annotated[Path, PathText] + maximum_bytes: int + owners: tuple[ThreadLookup, ...] = () + identities: tuple[NativeSessionIdentity, ...] = () + requests: tuple[ContextManifestReference, ...] = () + + def run(self) -> PracticeMeasurement: + return PracticeCopy(self.root, self.destination).capture_inputs( + self.owners, maximum_bytes=self.maximum_bytes, + identities=self.identities, requests=self.requests, + ) diff --git a/src/agent_comms/locked_store.py b/src/agent_comms/locked_store.py index 655566d983..6820c47fb0 100644 --- a/src/agent_comms/locked_store.py +++ b/src/agent_comms/locked_store.py @@ -693,6 +693,18 @@ def fresh(cls, root: Path) -> bool: ) +class InputCaptureFile(RootFile): + """A store participating in the original input capture's common source cut.""" + + __slots__ = () + + +class CompleteInputReader(InputCaptureFile): + """An input reader retained whole before selecting context and coordinator rows.""" + + __slots__ = () + + class WireWakeFile(RootFile): """Declared commit signals wake idle readers, unlike lock-file churn.""" diff --git a/src/agent_comms/maintenance_barrier.py b/src/agent_comms/maintenance_barrier.py index 4de9d1f58b..8361c25c64 100644 --- a/src/agent_comms/maintenance_barrier.py +++ b/src/agent_comms/maintenance_barrier.py @@ -106,6 +106,9 @@ def selected[T](held: T, granted: T) -> T: class MaintenanceTarget(SubclassUnion): """A hold without a target grants no additional runtime attachment.""" + def require_capture(self, runtime: LaunchRuntime) -> None: + raise MaintenanceNotHeldError("Selected input capture needs a retargeted maintenance hold") + @abstractmethod def allows_runtime(self, runtime: LaunchRuntime) -> bool: ... @@ -130,6 +133,10 @@ class TargetRuntime(LaunchRuntime, RecordMember, MaintenanceTarget): def allows_runtime(self, runtime: LaunchRuntime) -> bool: return self.prefix == runtime.prefix + def require_capture(self, runtime: LaunchRuntime) -> None: + if not self.allows_runtime(runtime): + raise MaintenanceNotHeldError("Selected input capture needs the held target runtime") + def require_drained_writes_open_to(self, state: MaintenanceState, holder: MaintenanceHolder) -> None: """Retargeted once the outgoing runtime has stopped its owners: from then on only @@ -375,6 +382,9 @@ def require_held_by(self, holder: MaintenanceHolder, change: str) -> None: never enabled maintenance has no holder.""" raise MaintenanceNotHeldError.unheld(change) + def require_capture(self, runtime: LaunchRuntime) -> MaintenanceState: + raise MaintenanceNotHeldError.unheld("Selected input capture") + @dataclass(frozen=True, eq=False) class MaintenanceOperator(ProcessIdentity, declared_name=None): @@ -512,6 +522,11 @@ def require_drained_writes_open_to(self, holder: MaintenanceHolder) -> None: def require_held_by(self, holder: MaintenanceHolder, change: str) -> None: self.phase.require_held_by(self, holder, change) + def require_capture(self, runtime: LaunchRuntime) -> MaintenanceState: + self.require_held_by(MaintenanceHolder.current(), "Selected input capture") + self.target.require_capture(runtime) + return self + def require_draining(self) -> None: if self.phase is not DrainingPhase: raise ValueError('Only an acquired draining phase can be cleared') @@ -634,6 +649,14 @@ def assert_drained_writes_open_unlocked(self) -> None: def read(self) -> MaintenanceStatus: return self.current_unlocked() + @contextmanager + def capturing(self, runtime: LaunchRuntime) -> Iterator[MaintenanceState]: + """Borrow a current retargeted hold; never acquire, refresh or rewrite it.""" + held = self.read().require_capture(runtime) + yield held + if self.read().require_capture(runtime) != held: + raise MaintenanceNotHeldError("Maintenance changed during selected input capture") + def held_by(self, operator: MaintenanceOperator) -> MaintenanceState: """``operator``'s hold as the root records it now, in whatever phase it is in.""" state = self.read() diff --git a/src/agent_comms/messages.py b/src/agent_comms/messages.py index ef3e84b3e3..2aedc5bfb4 100644 --- a/src/agent_comms/messages.py +++ b/src/agent_comms/messages.py @@ -30,7 +30,7 @@ ) from agent_comms.typed_table import ( AppendOnlyTables, Column, DerivedTables, DurableTables, ForeignKey, Index, - ProducedTable, SharedFileTables, TableReference, TypedRow, TypedTable, + LOCAL_TABLE_SOURCE, ProducedTable, SharedFileTables, TableReference, TableSource, TypedRow, TypedTable, ) from .change_report import NoChangeReport, ReportEvidence, SourceRoots @@ -109,9 +109,10 @@ class BusRow(BusTables): at: float @classmethod - def last_seq(cls, db: sqlite3.Connection) -> int: + def last_seq(cls, db: sqlite3.Connection, *, + table_source: TableSource = LOCAL_TABLE_SOURCE) -> int: return FieldCodec.decode(int, db.execute( - cls._sql(f'SELECT COALESCE(MAX({{seq}}), 0) FROM "{cls.declared_name}"') + cls._sql(f'SELECT COALESCE(MAX({{seq}}), 0) FROM {table_source.qualified(cls.declared_name)}') ).fetchone()[0]) @classmethod diff --git a/src/agent_comms/native_input_record.py b/src/agent_comms/native_input_record.py index c13507daf1..93296f5f84 100644 --- a/src/agent_comms/native_input_record.py +++ b/src/agent_comms/native_input_record.py @@ -36,7 +36,7 @@ from .native_integer import PositiveNativeIntegerBounds from .native_session_reopen import PiSessionId from .transcript_routes import InputDisplay, NoDisplayText -from .typed_table import SqlStorage, TextStorage +from .typed_table import LOCAL_TABLE_SOURCE, SqlStorage, TableSource, TextStorage, sql_literal if TYPE_CHECKING: from sqlite3 import Connection @@ -181,17 +181,36 @@ def proves_full_source(self, proof: HistoricalNativeInput) -> bool: @classmethod @abstractmethod - def source_membership_sql(cls) -> str: - """Select the assignments included in this stage's input.""" + def source_membership_sql( + cls, *, table_source: TableSource = LOCAL_TABLE_SOURCE, + ) -> str: + """Select original membership from this query's actual table source.""" + + def source_assignment_ids( + self, db: Connection, input_id: InputId, *, + table_source: TableSource = LOCAL_TABLE_SOURCE, + ) -> tuple[AssignmentId, ...]: + return tuple(row.assignment_id for row in self.source_assignments( + db, input_id, table_source=table_source, + )) + + def source_assignments( + self, db: Connection, input_id: InputId, *, + table_source: TableSource = LOCAL_TABLE_SOURCE, + ) -> tuple[WakeAssignment, ...]: + """Membership leads: a missing claim reaches the required-row decoder. - def source_assignment_ids(self, db: Connection, input_id: InputId) -> tuple[AssignmentId, ...]: + An inner join would silently erase a declared source whose claim was + absent, making corrupted membership look like a smaller valid group. + """ columns = ",".join(f"c.{column}" for column in WakeAssignment.columns()) originals = WakeAssignment.read(db.execute( - f"SELECT {columns} FROM {WakeAssignment.declared_name} c " - f"JOIN ({self.source_membership_sql()}) m ON m.assignment_id=c.assignment_id " + f"SELECT {columns} FROM ({self.source_membership_sql(table_source=table_source)}) m " + f"LEFT JOIN {WakeAssignment.qualified_table(table_source)} c " + "ON m.assignment_id=c.assignment_id " "WHERE m.input_id=? ORDER BY m.ordinal", (input_id,), )) - return tuple(original.assignment_id for original in originals) + return tuple(originals) @abstractmethod def record_sources( @@ -228,11 +247,13 @@ def transcript_events( super().transcript_events(entry, projection, user)] @classmethod - def source_membership_sql(cls) -> str: + def source_membership_sql( + cls, *, table_source: TableSource = LOCAL_TABLE_SOURCE, + ) -> str: from .triage_native_sources import TriageNativeSource return (f"SELECT s.input_id, s.assignment_id, s.ordinal " - f"FROM {TriageNativeSource.declared_name} s") + f"FROM {TriageNativeSource.qualified_table(table_source)} s") def record_sources( self, db: Connection, input_id: InputId, assignments: tuple[WakeAssignment, ...], @@ -272,13 +293,16 @@ class FullNativeExecution(NativeInputExecution): attempt_ordinal: int @classmethod - def source_membership_sql(cls) -> str: + def source_membership_sql( + cls, *, table_source: TableSource = LOCAL_TABLE_SOURCE, + ) -> str: from .native_runtime_input import NativeRuntimeInput return (f"SELECT n.input_id, c.assignment_id, c.ordinal " - f"FROM {NativeRuntimeInput.declared_name} n " - f"JOIN {ExecutionAssignmentLink.declared_name} c ON c.execution_id=n.execution_id " - f"WHERE n.stage='{cls.declared_name}'") + f"FROM {NativeRuntimeInput.qualified_table(table_source)} n " + f"JOIN {ExecutionAssignmentLink.qualified_table(table_source)} c " + "ON c.execution_id=n.execution_id " + f"WHERE n.stage={sql_literal(cls)}") def record_sources( self, db: Connection, input_id: InputId, assignments: tuple[WakeAssignment, ...], diff --git a/src/agent_comms/native_runtime_input.py b/src/agent_comms/native_runtime_input.py index f98a2ed4da..a222e4249d 100644 --- a/src/agent_comms/native_runtime_input.py +++ b/src/agent_comms/native_runtime_input.py @@ -7,6 +7,7 @@ from __future__ import annotations import sqlite3 +from collections.abc import Iterable from dataclasses import dataclass, field from functools import partial from typing import TYPE_CHECKING, Annotated @@ -58,6 +59,8 @@ from .tool_denial import SelectedToolDenied from .turn_item import FreshNativeId from .typed_table import ( + LOCAL_TABLE_SOURCE, + AttachedPracticeRebuild, Column, FrozenColumn, IntegerStorage, @@ -65,8 +68,11 @@ NoDelete, NonDecreasing, RequiredUpdateColumns, + RetainsChildren, StableReferenceColumns, TableReference, + TableSchema, + TableSource, TextStorage, TypedRow, WithoutRowid, @@ -96,6 +102,7 @@ class PublishedReplyRevision(TypedRow): @dataclass(frozen=True) class NativeRuntimeInput( + RetainsChildren, WriteOnceColumns, RequiredUpdateColumns, NoDelete, @@ -150,11 +157,47 @@ def write_once_columns(cls: type[NativeRuntimeInput]) -> tuple[str, ...]: required_update_columns = ('sent_owner_claimed_generation',) @classmethod - def source_membership_sql(cls) -> str: + def source_membership_sql( + cls, *, table_source: TableSource = LOCAL_TABLE_SOURCE, + ) -> str: return " UNION ALL ".join( - member.source_membership_sql() for member in NativeInputExecution.members_with(NativeInputExecution) + member.source_membership_sql(table_source=table_source) + for member in NativeInputExecution.members_with(NativeInputExecution) ) + @classmethod + def retain_sessions( + cls, original: AttachedPracticeRebuild, identities: Iterable[NativeSessionIdentity], + ) -> int: + """Select the original SQL session group, never an input-id-only association. + + A selected journal and recorded session id must both agree. Raw retention + preserves absent context/claim fields; selection grants no native proof. + """ + return original.retain_keys(cls, tuple( + {"session_id": identity.session_id, "session_file": identity.session_file} + for identity in identities + )) + + @classmethod + def retain_required_rows( + cls, original: AttachedPracticeRebuild, schema: type[TableSchema], + ) -> int: + """Keep original triage/cursor children and the stage-owned source claims. + + Triage membership has no declared assignment FK. Its existing stage + query, not a parallel retention join or roster, owns that relationship. + Membership comes from the original attachment; only selected native ids + come from the destination. + """ + changed = super().retain_required_rows(original, schema) + for native in cls.stream(original.destination): + for assignment in native.execution.source_assignments( + original.destination, native.input_id, table_source=original.source, + ): + changed += assignment.retain_original(original) + return changed + def __post_init__(self): self.execution # The selected identity and the later context acknowledgement are separate diff --git a/src/agent_comms/participant_store.py b/src/agent_comms/participant_store.py index 8fc5243a74..f1b9e8daf0 100644 --- a/src/agent_comms/participant_store.py +++ b/src/agent_comms/participant_store.py @@ -2,6 +2,7 @@ from __future__ import annotations +import sqlite3 from dataclasses import dataclass from agent_comms.coordination_errors import ( @@ -21,6 +22,9 @@ Participants, ) from agent_comms.nominal.field_codec import ThreadLookup +from agent_comms.typed_table import ( + LOCAL_TABLE_SOURCE, AttachedPracticeRebuild, RowNotFoundError, TableSchema, TableSource, +) @dataclass(frozen=True, slots=True) @@ -37,6 +41,11 @@ class ParticipantSnapshot: owner_identity: OwnerGenerations pointer: CurrentExecutions + def retain_original(self, original: AttachedPracticeRebuild) -> int: + """The generation and pointer are both required parts of this recorded answer.""" + return (self.owner_identity.retain_original(original) + + self.pointer.retain_original(original)) + class ParticipantStore: def __init__(self, session: CoordinationSession) -> None: @@ -44,17 +53,45 @@ def __init__(self, session: CoordinationSession) -> None: def get(self, lookup: ThreadLookup) -> ParticipantSnapshot: with self.session.read(): - person = Participants.one(self.session._connection, participant_lookup=lookup) - generation = OwnerGenerations.one(self.session._connection, owner_lookup=lookup) - pointer = CurrentExecutions.one(self.session._connection, owner_lookup=lookup) - if person is None or generation is None or pointer is None: - raise ParticipantNotRegistered(f"participant {lookup!r} aggregate is not registered") - return ParticipantSnapshot( - lookup, - person.committed, - generation, - pointer, + return self.read_at(self.session._connection, lookup) + + @staticmethod + def read_at( + db: sqlite3.Connection, lookup: ThreadLookup, *, + table_source: TableSource = LOCAL_TABLE_SOURCE, + ) -> ParticipantSnapshot: + """Read the same aggregate through the caller's already-owned SQL lifetime.""" + try: + person = Participants.require(db, participant_lookup=lookup, table_source=table_source) + generation = OwnerGenerations.require( + db, owner_lookup=lookup, table_source=table_source, ) + pointer = CurrentExecutions.require(db, owner_lookup=lookup, table_source=table_source) + except RowNotFoundError as error: + raise ParticipantNotRegistered( + f"participant {lookup!r} aggregate is not registered", + ) from error + return ParticipantSnapshot(lookup, person.committed, generation, pointer) + + @classmethod + def retain_owners( + cls, original: AttachedPracticeRebuild, lookups: tuple[ThreadLookup, ...], + ) -> int: + """Keep selected original aggregates, not every participant's execution history. + + An absent participant remains absent. A present but incomplete aggregate + follows the ordinary reader's refusal before any file can be published. + """ + changed = 0 + for person in Participants.select_keys( + original.destination, tuple({"participant_lookup": lookup} for lookup in lookups), + table_source=original.source, + ): + snapshot = cls.read_at(original.destination, person.participant_lookup, + table_source=original.source) + changed += person.retain_original(original) + changed += snapshot.retain_original(original) + return changed def register( self, @@ -113,3 +150,13 @@ def advance_generation( generation=expected_generation + 1, ) return Applied(self.get(lookup)) + + +class ParticipantRows(AttachedPracticeRebuild): + """Keep whole participant answers inside the existing selected-file closure.""" + + def retain_referenced_rows(self, schema: type[TableSchema]) -> int: + changed = super().retain_referenced_rows(schema) + return changed + ParticipantStore.retain_owners(self, tuple( + person.participant_lookup for person in Participants.select(self.destination) + )) diff --git a/src/agent_comms/typed_table.py b/src/agent_comms/typed_table.py index bbd0948c23..bf0ca5e839 100644 --- a/src/agent_comms/typed_table.py +++ b/src/agent_comms/typed_table.py @@ -299,9 +299,17 @@ class Reference(ABC): @abstractmethod def reference_sql(self) -> str: ... + @abstractmethod + def foreign_keys(self, columns: tuple[str, ...]) -> tuple[ForeignKey, ...]: + """References made by these source columns; every reference declares its answer.""" + ... + @dataclass(frozen=True) class NoReference(Reference): + def foreign_keys(self, columns: tuple[str, ...]) -> tuple[ForeignKey, ...]: + return () + def reference_sql(self) -> str: return "" @@ -311,6 +319,9 @@ class TableReference(Reference): target: type[TypedTable] target_columns: tuple[str, ...] + def foreign_keys(self, columns: tuple[str, ...]) -> tuple[ForeignKey, ...]: + return (ForeignKey(target=self.target, target_columns=self.target_columns, columns=columns),) + def reference_sql(self) -> str: return ( f" REFERENCES {_identifier(self.target.declared_name)} " @@ -412,6 +423,9 @@ def table_column(self, name: str) -> _Field: class UnallocatedRowId(NullMember): """SQLite allocates this row's AUTOINCREMENT id when the row is inserted.""" + def __int__(self) -> NoReturn: + raise TypeError("SQLite has not allocated this row id") + RowId = int | UnallocatedRowId @@ -472,6 +486,42 @@ def sql(self, owner: type[TypedTable]) -> str: + (" DEFERRABLE INITIALLY DEFERRED" if self.deferred else "") ) + def retain_parents( + self, owner: type[TypedTable], original: AttachedPracticeRebuild, + ) -> int: + """Original parents of selected rows, using SQLite's FK value comparison. + + The parent column is on the left so its affinity/collation governs the + comparison, as it does for the actual constraint. A NULL source key + therefore contributes no parent; encoded SQL values are not decoded as + one domain type and re-encoded as a different target type. + """ + referencing = _identifier(self.target.declared_name + "_referencing") + related = self._related_columns(owner, referencing, _identifier(self.target.declared_name)) + return original.retain_rows( + self.target, + where=f"EXISTS (SELECT 1 FROM main.{_identifier(owner.declared_name)} " + f"AS {referencing} WHERE {related})", + ) + + def _related_columns(self, owner: type[TypedTable], referencing: str, referenced: str) -> str: + """The parent side always supplies the declared FK comparison semantics.""" + return " AND ".join( + f"{referenced}.{self.target._column_list((parent,))} = " + f"{referencing}.{owner._column_list((child,))}" + for child, parent in zip(self.columns, self.target_columns, strict=True) + ) + + def retain_children(self, owner: type[TypedTable], original: AttachedPracticeRebuild) -> int: + """Original children of selected parents, only when a parent requires that group.""" + referenced = _identifier(owner.declared_name + "_referenced") + related = self._related_columns(owner, _identifier(owner.declared_name), referenced) + return original.retain_rows( + owner, + where=f"EXISTS (SELECT 1 FROM main.{_identifier(self.target.declared_name)} " + f"AS {referenced} WHERE {related})", + ) + class SqlProjection: """Scalar and record projections read SQLite's result columns at one boundary.""" @@ -763,12 +813,62 @@ def __str__(self) -> str: return f"{self.name}: stored {self.stored or 'nothing'}; declared {self.declared or 'nothing'}" +class TableSource: + """A query's table location; the default retains SQLite's normal name search.""" + + __slots__ = () + + def qualified(self, name: str) -> str: + return _identifier(name) + + +LOCAL_TABLE_SOURCE = TableSource() + + @dataclass(frozen=True) -class SQLiteDatabase(TypedRow): +class SQLiteDatabase(TypedRow, TableSource): + """An actual attached database owns qualification through its recorded name.""" + seq: int name: str file: str + @classmethod + def named(cls, db: sqlite3.Connection, name: str) -> Self: + database, = cls.read(db.execute( + "SELECT seq,name,file FROM pragma_database_list WHERE name=?", (name,), + )) + return database + + def qualified(self, name: str) -> str: + return f"{_identifier(self.name)}.{_identifier(name)}" + + def limit_bytes(self, db: sqlite3.Connection, maximum_bytes: int) -> None: + """Let SQLite refuse page growth before a selected database exceeds its allowance.""" + size, = SQLitePageSize.read(db.execute(f"PRAGMA {_identifier(self.name)}.page_size")) + pages = maximum_bytes // size.page_size + if pages < 1: + raise RelationViolationError("Selected store has no room for a SQLite page") + limit, = SQLiteMaximumPageCount.read(db.execute( + f"PRAGMA {_identifier(self.name)}.max_page_count={pages}", + )) + if limit.max_page_count > pages: + raise RelationViolationError("Selected database already exceeds its byte allowance") + + +@dataclass(frozen=True) +class SQLitePageSize(TypedRow): + """The page size returned by SQLite, not a copied platform default.""" + + page_size: int + + +@dataclass(frozen=True) +class SQLiteMaximumPageCount(TypedRow): + """SQLite's actual accepted allocation limit; an existing file can force it higher.""" + + max_page_count: int + @dataclass(frozen=True) class SQLiteSchemaObject(TypedRow): @@ -998,26 +1098,121 @@ class PracticeRebuild(SchemaRebuild): database: ClassVar[str] = __qualname__ @contextmanager - def attached(self, db: sqlite3.Connection) -> Iterator[SchemaRebuild]: - with closing(SQLiteFile.read_snapshot(self.path)) as source: - db.execute(f'ATTACH DATABASE ? AS "{self.database}"', - (self.path.resolve().as_uri() + "?mode=ro",)) - try: - yield AttachedPracticeRebuild(source) - finally: - db.execute(f'DETACH DATABASE "{self.database}"') + def attached(self, db: sqlite3.Connection) -> Iterator[AttachedPracticeRebuild]: + db.execute(f'ATTACH DATABASE ? AS "{self.database}"', + (self.path.resolve().as_uri() + "?mode=ro",)) + try: + yield AttachedPracticeRebuild(SQLiteDatabase.named(db, self.database), db) + finally: + db.execute(f'DETACH DATABASE "{self.database}"') @dataclass(frozen=True) class AttachedPracticeRebuild(SchemaRebuild): - source: sqlite3.Connection + source: SQLiteDatabase + destination: sqlite3.Connection + + def retain_referenced_rows(self, schema: type[TableSchema]) -> int: + """Expand this attachment through its file's declared dependency groups.""" + return schema.retain_referenced_rows(self) + + def retain_rows( + self, row: type[TypedTable], *, + where: str = "1", parameters: tuple[object, ...] = (), + ) -> int: + """Copy original writable SQL values through this active source attachment. + + This is a row primitive, not a complete-store preservation certificate. + The caller owns destination freshness, guard/loading lifetime, semantic + dependencies and producer metadata. No stored JSON is reconstructed. + """ + return self._retain_selection( + row, f"FROM {self.source.qualified(row.declared_name)} AS {_identifier(row.declared_name)}", + where=where, parameters=parameters, + ) + + def retain_keys( + self, row: type[TypedTable], keys: Sequence[Mapping[str, object]], + ) -> int: + """The declared indexed-key query supplies raw selected rows, not re-encoded records.""" + return sum(self._retain_selection( + row, requested.join, prefix=f"WITH {requested.values}", + parameters=requested.bindings, + ) for requested in row.requested_keys( + self.destination, keys, table_source=self.source, + )) + + def retain_sequences(self) -> None: + """Copy SQLite's original high-water marks once, including empty selections. + + Allocation belongs to SQLite metadata, not membership in a row-insertion + capability. Only tables present in the selected file's schema participate. + """ + if not SQLiteSchemaObject.read(self.destination.execute( + f"SELECT original.name,original.sql FROM {self.source.qualified('sqlite_master')} original " + "JOIN main.sqlite_master retained ON retained.name=original.name " + "WHERE original.name='sqlite_sequence' " + "AND original.type='table' AND retained.type='table'", + )): + return + source = self.source.qualified("sqlite_sequence") + self.destination.execute( + "INSERT INTO main.sqlite_sequence(name,seq) " + f"SELECT original_sequence.name,original_sequence.seq FROM {source} original_sequence " + "JOIN main.sqlite_master retained_table ON retained_table.name=original_sequence.name " + "AND retained_table.type='table' " + "WHERE NOT EXISTS(SELECT 1 FROM main.sqlite_sequence retained_sequence " + "WHERE retained_sequence.name=original_sequence.name)", + ) + self.destination.execute( + "UPDATE main.sqlite_sequence AS retained_sequence " + f"SET seq=(SELECT seq FROM {source} original_sequence " + "WHERE original_sequence.name=retained_sequence.name) " + f"WHERE EXISTS(SELECT 1 FROM {source} original_sequence " + "WHERE original_sequence.name=retained_sequence.name " + "AND retained_sequence.seq None: + """An unknown source column or trigger is not permission to silently discard it.""" + stored = SQLiteSchemaObject.read(self.destination.execute( + f"SELECT name,sql FROM {self.source.qualified('sqlite_master')} " + "WHERE sql IS NOT NULL AND name NOT LIKE 'sqlite_%'", + )) + differences = SQLiteSchemaObject.differences(stored, schema.declared_schema()) + if differences: + raise schema.schema_error("Retained source schema differs: " + " | ".join( + str(difference) for difference in differences)) + + def _retain_selection( + self, row: type[TypedTable], source: str, *, where: str = "1", + parameters: tuple[object, ...] = (), prefix: str = "", + ) -> int: + keys = tuple(item.name for item in row.retention_key_fields) + fields = tuple(item.name for item in row._writable_fields()) + retained_alias = _identifier(row.declared_name + "_retained") + retained = " AND ".join( + f"{retained_alias}.{row._column_list((key,))} IS {row.qualified(key)}" for key in keys + ) + table = _identifier(row.declared_name) + before = self.destination.total_changes + self.destination.execute( + f"{prefix} INSERT INTO main.{table} ({row._column_list(fields)}) " + f"SELECT {row.qualified(*fields)} {source} " + f"WHERE ({row._sql(where)}) AND NOT EXISTS " + f"(SELECT 1 FROM main.{table} AS {retained_alias} WHERE {retained})", + parameters, + ) + # Cursor.rowcount is -1 for WITH-prefixed INSERTs. The connection owns + # completed changes regardless of which selection grammar supplied them. + return self.destination.total_changes - before def rebuild( self, schema: type[DerivedTables], db: sqlite3.Connection, rows: tuple[type[TypedTable], ...], ) -> None: projections = tuple(row for row in schema.projection_rows() if row in rows) - if not ProjectionProducer.matches(self.source, projections): + if not ProjectionProducer.matches(db, projections, table_source=self.source): raise RelationViolationError("Practice projection producers differ from this runtime") through = max((row.practice_sequence(db, self.source) for row in schema.projection_rows()), default=0) @@ -1025,7 +1220,7 @@ def rebuild( schema.create_tables(db, rows) for row in projections: db.execute(f'INSERT INTO "{row.declared_name}" ' - f'SELECT * FROM "{PracticeRebuild.database}"."{row.declared_name}"') + f'SELECT * FROM {self.source.qualified(row.declared_name)}') schema.replay(db, rows, after=through) @@ -1164,6 +1359,30 @@ class TableSchema(ABC): def schema_rows(cls) -> tuple[type[TypedTable], ...]: return tuple(TypedTable.members_with(cls)) + @classmethod + def retain_referenced_rows( + cls, original: AttachedPracticeRebuild, + ) -> int: + """One declared dependency expansion, including owners' required groups. + + Undeclared natural-key, serialized and cross-store dependencies remain + open; a zero contribution is not a certificate that they do not exist. + """ + rows = cls.schema_rows() + return sum(reference.retain_parents(row, original) + for row in rows for reference in row.foreign_keys()) + sum( + row.retain_required_rows(original, cls) for row in rows + ) + + @classmethod + def retain_referencing_rows( + cls, original: AttachedPracticeRebuild, parent: type[TypedTable], + ) -> int: + """Inverse declared edges for an owner that requires its complete child group.""" + return sum(reference.retain_children(row, original) + for row in cls.schema_rows() for reference in row.foreign_keys() + if reference.target is parent) + @classmethod def declares_schema(cls) -> bool: """Whether this class names a file's tables, not only a parent of such classes.""" @@ -1271,6 +1490,34 @@ def require_readable(cls, db: sqlite3.Connection) -> None: for row in cls.schema_rows(): row.require_decodable_rows(db) + @classmethod + def require_related_rows(cls, db: sqlite3.Connection) -> None: + """Reject stored references without changing their original rows.""" + with closing(SQLiteForeignKeyViolation.iterate( + db.execute('SELECT "table", parent, fkid FROM pragma_foreign_key_check()'), + )) as violations: + for violation in violations: + raise cls.schema_error( + f"{cls.__name__} foreign-key violation: table={violation.table}, " + f"parent={violation.parent}, fkid={violation.fkid}" + ) + + @classmethod + @contextmanager + def loading_rows(cls, db: sqlite3.Connection) -> Iterator[None]: + """Load a fresh declared schema in one transaction, preserving sealed values. + + Every table exists before loading starts; guards are installed only after + all selected rows have loaded. Required children and producer metadata + remain the selector's obligation, not something FK validity proves. + """ + with SQLiteForeignKeys.relations_suspended(db), SQLiteFile.write_transaction(db): + with ExitStack() as loading: + for row in cls.schema_rows(): + loading.enter_context(row.loading(db)) + yield + cls.require_related_rows(db) + @classmethod def create_schema(cls, db: sqlite3.Connection) -> None: cls.create_tables(db, cls.schema_rows()) @@ -1311,14 +1558,7 @@ def reconcile_schema( with SQLiteForeignKeys.relations_suspended(db): with rebuild.attached(db) as attached, SQLiteFile.write_transaction(db): cls.carry_schema(db, rebuild=attached) - with closing(SQLiteForeignKeyViolation.iterate( - db.execute('SELECT "table", parent, fkid FROM pragma_foreign_key_check()'), - )) as violations: - for violation in violations: - raise cls.schema_error( - f"{cls.__name__} foreign-key violation: table={violation.table}, " - f"parent={violation.parent}, fkid={violation.fkid}" - ) + cls.require_related_rows(db) return True @classmethod @@ -2230,9 +2470,21 @@ def _member_declarations(cls) -> tuple[type[DeclaredFamily], ...]: @classmethod def references(cls) -> tuple[ForeignKey, ...]: + """Table-level constraints; column references remain on their fields.""" # A method permits mutually referring row classes without string names. return () + @classmethod + def foreign_keys(cls) -> tuple[ForeignKey, ...]: + """All declared SQL edges, including inherited and generated columns. + + This describes referential integrity, not required dependent children or + references encoded inside a field's value. SQL generation keeps its + existing column/table placement; inspection never reparses that SQL. + """ + return (*cls.references(), *(reference for item in cls._columns + for reference in item.column.references.foreign_keys((item.name,)))) + @classmethod def _computed_columns(cls) -> frozenset[str]: return frozenset(item.wire_name for item in cls._columns if not item.column.writable) @@ -2316,6 +2568,25 @@ def drop(cls, db: sqlite3.Connection) -> None: """Drop the declared table; SQLite drops its indexes with it.""" db.execute(f"DROP TABLE IF EXISTS {_identifier(cls.declared_name)}") + @classmethod + def retain_required_rows( + cls, original: AttachedPracticeRebuild, schema: type[TableSchema], + ) -> int: + """Additional groups this row's declaration requires beyond its FK parents.""" + return 0 + + @classmethod + @contextmanager + def loading(cls, db: sqlite3.Connection) -> Iterator[None]: + """Create the table, load its rows, then install its declared guards. + + The caller owns rollback. No retained row state is rewritten to pass + insertion guards that describe ordinary runtime writes. + """ + db.execute(cls._table_sql()) + yield + cls.create(db, if_not_exists=True) + @classmethod def rebuild(cls, db: sqlite3.Connection, changed: Mapping[str, str] = types.MappingProxyType({})) -> None: @@ -2336,10 +2607,9 @@ def rebuild(cls, db: sqlite3.Connection, db.execute(f"CREATE TEMP TABLE {temporary} AS SELECT {selected} FROM {table}") SQLiteCatalogObject.release_guards(db, cls) cls.drop(db) - db.execute(cls._table_sql()) - columns = cls._column_list(tuple(item.name for item in fields)) - db.execute(f"INSERT INTO {table} ({columns}) SELECT {columns} FROM {temporary}") - cls.create(db, if_not_exists=True) + with cls.loading(db): + columns = cls._column_list(tuple(item.name for item in fields)) + db.execute(f"INSERT INTO {table} ({columns}) SELECT {columns} FROM {temporary}") db.execute(f"DROP TABLE {temporary}") @classmethod @@ -2394,18 +2664,28 @@ def _condition( @classmethod def select_keys(cls, db: sqlite3.Connection, - keys: Sequence[Mapping[str, object]]) -> list[Self]: + keys: Sequence[Mapping[str, object]], *, + table_source: TableSource = LOCAL_TABLE_SOURCE) -> list[Self]: """Rows equal to any of these named keys under the same null-safe IS equality select() uses: the requested keys drive an indexed join, one batch per the connection's own variable limit, each distinct key probed once.""" - return [row for requested in cls.requested_keys(db, keys) for row in cls.iterate(db.execute( - f"WITH {requested.values} " - f"SELECT {cls.qualified(*cls.column_projection())} {requested.join}", - requested.bindings))] + return list(cls.stream_keys(db, keys, table_source=table_source)) + + @classmethod + def stream_keys(cls, db: sqlite3.Connection, + keys: Sequence[Mapping[str, object]], *, + table_source: TableSource = LOCAL_TABLE_SOURCE) -> Generator[Self]: + """Consume the same indexed batches without materializing every matched row.""" + for requested in cls.requested_keys(db, keys, table_source=table_source): + yield from cls.iterate(db.execute( + f"WITH {requested.values} " + f"SELECT {cls.qualified(*cls.column_projection())} {requested.join}", + requested.bindings)) @classmethod def requested_keys(cls, db: sqlite3.Connection, - keys: Sequence[Mapping[str, object]]) -> Iterator[RequestedKeys]: + keys: Sequence[Mapping[str, object]], *, + table_source: TableSource = LOCAL_TABLE_SOURCE) -> Iterator[RequestedKeys]: """The distinct named keys, encoded by their declared columns, in batches that fit the connection's own variable limit; each batch joins this table as ``FROM requested JOIN ON ...`` under select()'s null-safe IS equality.""" @@ -2425,7 +2705,7 @@ def requested_keys(cls, db: sqlite3.Connection, for chunk in batched(bound, batch): yield RequestedKeys( f"requested({','.join(aliases)}) AS (VALUES {','.join(probe for _ in chunk)})", - f"FROM requested JOIN {table} ON {joined}", + f"FROM requested JOIN {table_source.qualified(cls.declared_name)} ON {joined}", tuple(value for _, values in chunk for value in values)) @classmethod @@ -2433,11 +2713,12 @@ def _query( cls, db: sqlite3.Connection, where: str, parameters: tuple, order_by: tuple[str, ...], descending: bool, key: dict[str, object], row_type: type[RelatedRow] | None, limit: int | None, + table_source: TableSource, ) -> Generator[Self | RelatedRow]: condition, parameters = cls._condition(where, parameters, key) direction, selected = " DESC" if descending else "", row_type or cls return selected.iterate(db.execute( - f"SELECT {selected._select_list(cls)} FROM {_identifier(cls.declared_name)} " + f"SELECT {selected._select_list(cls)} FROM {table_source.qualified(cls.declared_name)} " f"WHERE {condition}" + (" ORDER BY " + ", ".join(cls._column_list((name,)) + direction for name in order_by) if order_by else "") + (" LIMIT ?" if limit is not None else ""), @@ -2449,7 +2730,7 @@ def _query( def select( cls, db: sqlite3.Connection, *, row_type: type[RelatedRow], where: str = "1", parameters: tuple = (), order_by: tuple[str, ...] = (), descending: bool = False, - limit: int | None = None, **key: object, + limit: int | None = None, table_source: TableSource = LOCAL_TABLE_SOURCE, **key: object, ) -> list[RelatedRow]: ... @overload @@ -2457,7 +2738,7 @@ def select( def select( cls, db: sqlite3.Connection, *, row_type: None = None, where: str = "1", parameters: tuple = (), order_by: tuple[str, ...] = (), descending: bool = False, - limit: int | None = None, **key: object, + limit: int | None = None, table_source: TableSource = LOCAL_TABLE_SOURCE, **key: object, ) -> list[Self]: ... @classmethod @@ -2470,16 +2751,17 @@ def select( order_by: tuple[str, ...] = (), descending: bool = False, row_type: type[RelatedRow] | None = None, limit: int | None = None, - **key: object, + table_source: TableSource = LOCAL_TABLE_SOURCE, **key: object, ) -> Sequence[Self | RelatedRow]: - return list(cls._query(db, where, parameters, order_by, descending, key, row_type, limit)) + return list(cls._query(db, where, parameters, order_by, descending, key, row_type, limit, + table_source)) @overload @classmethod def stream( cls, db: sqlite3.Connection, *, row_type: type[RelatedRow], where: str = "1", parameters: tuple = (), order_by: tuple[str, ...] = (), descending: bool = False, - limit: int | None = None, **key: object, + limit: int | None = None, table_source: TableSource = LOCAL_TABLE_SOURCE, **key: object, ) -> Generator[RelatedRow]: ... @overload @@ -2487,7 +2769,7 @@ def stream( def stream( cls, db: sqlite3.Connection, *, row_type: None = None, where: str = "1", parameters: tuple = (), order_by: tuple[str, ...] = (), descending: bool = False, - limit: int | None = None, **key: object, + limit: int | None = None, table_source: TableSource = LOCAL_TABLE_SOURCE, **key: object, ) -> Generator[Self]: ... @classmethod @@ -2500,10 +2782,11 @@ def stream( order_by: tuple[str, ...] = (), descending: bool = False, row_type: type[RelatedRow] | None = None, limit: int | None = None, - **key: object, + table_source: TableSource = LOCAL_TABLE_SOURCE, **key: object, ) -> Generator[Self | RelatedRow]: """Rows decoded as the caller consumes them; close it to release the query.""" - return cls._query(db, where, parameters, order_by, descending, key, row_type, limit) + return cls._query(db, where, parameters, order_by, descending, key, row_type, limit, + table_source) @classmethod def wire_record(cls, db: sqlite3.Connection, plan: FieldPlan) -> str: @@ -2511,6 +2794,11 @@ def wire_record(cls, db: sqlite3.Connection, plan: FieldPlan) -> str: item, = cls._fields_named((plan.name,)) return item.wire_record(db, plan, cls.declared_name) + @classmethod + def qualified_table(cls, table_source: TableSource = LOCAL_TABLE_SOURCE) -> str: + """Name this declared table at its SQL boundary, in the supplied database.""" + return table_source.qualified(cls.declared_name) + @classmethod def qualified(cls, *names: str) -> str: """Columns named through this table, for one select joining tables that share names.""" @@ -2532,23 +2820,43 @@ def decode_at(cls, row: tuple[object, ...], offset: int) -> Self: return cls._decode_row(row, {name: offset + index for index, name in enumerate(cls.column_projection())}) + @ClassFact + @staticmethod + def retention_key_fields(cls: type[TypedTable]) -> tuple[_Field, ...]: + """The declaration supplies both raw-copy deduplication and held-row selection.""" + keys = tuple(item for item in cls._columns if item.column.primary_key) + if not keys: + raise RelationViolationError(f"{cls.declared_name} has no declared retention key") + return keys + + def retain_original(self, original: AttachedPracticeRebuild) -> int: + """Keep this original row's raw SQL through its declared key, not reinsertion.""" + keys = self.retention_key_fields + plans = {plan.name: plan for plan in FieldCodec.field_plans(type(self))} + return original.retain_keys(type(self), ({ + item.name: plans[item.name].current(self) for item in keys + },)) + @classmethod - def one(cls, db: sqlite3.Connection, **key: object) -> Self | None: - rows = cls._keyed(db, key) + def one(cls, db: sqlite3.Connection, *, + table_source: TableSource = LOCAL_TABLE_SOURCE, **key: object) -> Self | None: + rows = cls._keyed(db, key, table_source=table_source) return next(iter(rows), None) @classmethod - def require(cls, db: sqlite3.Connection, **key: object) -> Self: + def require(cls, db: sqlite3.Connection, *, + table_source: TableSource = LOCAL_TABLE_SOURCE, **key: object) -> Self: """The one row a reference names; a missing row is corruption.""" - rows = cls._keyed(db, key) + rows = cls._keyed(db, key, table_source=table_source) if not rows: raise RowNotFoundError(f"Expected one {cls.declared_name} for {tuple(key)}") return rows[0] @classmethod - def _keyed(cls, db: sqlite3.Connection, key: dict[str, object]) -> list[Self]: + def _keyed(cls, db: sqlite3.Connection, key: dict[str, object], *, + table_source: TableSource = LOCAL_TABLE_SOURCE) -> list[Self]: """At most one row is stored for a key; a repeated key is corruption.""" - stored = cls.select(db, limit=2, **key) + stored = cls.select(db, limit=2, table_source=table_source, **key) if len(stored) > 1: raise MalformedRecord(f"Expected one {cls.declared_name} for {tuple(key)}") return stored @@ -2580,23 +2888,27 @@ def objects_of( return [item for item in stored if item.name in declared or item.name not in claimed] @classmethod - def present(cls, db: sqlite3.Connection) -> bool: + def present(cls, db: sqlite3.Connection, *, + table_source: TableSource = LOCAL_TABLE_SOURCE) -> bool: return SqlValue.for_annotation(bool).read(db.execute( - "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name=?)", + f"SELECT EXISTS(SELECT 1 FROM {table_source.qualified('sqlite_master')} " + "WHERE type='table' AND name=?)", (cls.declared_name,)))[0] @classmethod - def exists(cls, db: sqlite3.Connection, *, where: str = "1", parameters: tuple = (), **key: object) -> bool: + def exists(cls, db: sqlite3.Connection, *, where: str = "1", parameters: tuple = (), + table_source: TableSource = LOCAL_TABLE_SOURCE, **key: object) -> bool: condition, parameters = cls._condition(where, parameters, key) return SqlValue.for_annotation(bool).read(db.execute( - f"SELECT EXISTS(SELECT 1 FROM {_identifier(cls.declared_name)} WHERE {condition})", + f"SELECT EXISTS(SELECT 1 FROM {table_source.qualified(cls.declared_name)} WHERE {condition})", parameters))[0] @classmethod - def count(cls, db: sqlite3.Connection, *, where: str = "1", parameters: tuple = (), **key: object) -> int: + def count(cls, db: sqlite3.Connection, *, where: str = "1", parameters: tuple = (), + table_source: TableSource = LOCAL_TABLE_SOURCE, **key: object) -> int: condition, parameters = cls._condition(where, parameters, key) return SqlValue.for_annotation(int).read(db.execute( - f"SELECT COUNT(*) FROM {_identifier(cls.declared_name)} WHERE {condition}", parameters))[0] + f"SELECT COUNT(*) FROM {table_source.qualified(cls.declared_name)} WHERE {condition}", parameters))[0] @classmethod @cache @@ -2705,7 +3017,7 @@ def writer_declarations(cls) -> tuple[type[ProjectionOutputs], ...]: if owner.shares_outputs_with(cls)) @classmethod - def practice_sequence(cls, db: sqlite3.Connection, source: sqlite3.Connection) -> int: + def practice_sequence(cls, db: sqlite3.Connection, source: TableSource) -> int: """Only the identity-owning row contributes a verified history prefix.""" return 0 @@ -2825,12 +3137,14 @@ def records_for(cls, rows: tuple[type[ProducedTable], ...]) -> tuple[Self, ...]: return tuple(cls(name, digest) for name, digest in digests.items()) @classmethod - def matches(cls, db: sqlite3.Connection, rows: tuple[type[ProducedTable], ...]) -> bool: + def matches(cls, db: sqlite3.Connection, rows: tuple[type[ProducedTable], ...], *, + table_source: TableSource = LOCAL_TABLE_SOURCE) -> bool: if not rows: return True - if not cls.present(db): + if not cls.present(db, table_source=table_source): return False - stored = {record.projection: record.producer_digest for record in cls.select(db)} + stored = {record.projection: record.producer_digest + for record in cls.select(db, table_source=table_source)} digests = cls.digests_for(rows) return all(stored.get(name) == digest for name, digest in digests.items()) @@ -2892,6 +3206,49 @@ def schema_at(cls, path: Path) -> type[TableSchema]: def schema_change(cls, path: Path) -> SchemaChange: return HeldBusFile(cls, path) + @classmethod + def configure_connection(cls, connection: sqlite3.Connection) -> None: + """Connection behavior needed by this file's declared loading guards.""" + + @classmethod + @contextmanager + def retaining_rows( + cls, source: Path, destination: Path, *, maximum_bytes: int, + original_type: type[AttachedPracticeRebuild] = AttachedPracticeRebuild, + ) -> Iterator[AttachedPracticeRebuild]: + """Publish a fresh selected store only after loading and declared closure succeed. + + The caller supplies seeds and owns the wider cross-store cut. The source + attachment stays read-only in the loading transaction; no runtime writer, + producer regeneration, lifecycle reset or original coordinate is created. + """ + if destination.exists() or destination.is_symlink(): + raise FileExistsError(destination) + schema = cls.schema_at(source) + PrivateDirectoryRole.make_parents(destination.parent) + descriptor, staged = cls.stage_file(destination) + os.close(descriptor) + try: + with closing(cls.writer(staged, isolation_level=None, uri=True)) as db: + cls.configure_connection(db) + SQLiteDatabase.named(db, "main").limit_bytes(db, maximum_bytes) + with PracticeRebuild(source).attached(db) as attached: + original = original_type(attached.source, attached.destination) + with schema.loading_rows(db): + original.require_schema(schema) + yield original + while original.retain_referenced_rows(schema): + pass + original.retain_sequences() + retained_bytes = sum(member.stat().st_size for member in cls.files(staged) + if member.is_file()) + if retained_bytes > maximum_bytes: + raise RelationViolationError("Selected SQLite file group exceeds its byte allowance") + ExclusiveAtomicFile(destination).replace(staged) + finally: + for member in cls.files(staged): + member.unlink(missing_ok=True) + @classmethod def practice_rebuild(cls, path: Path, practice_root: Path) -> SchemaRebuild: """Files without a declared practice replay retain their normal rebuild.""" @@ -3218,6 +3575,20 @@ def __init_subclass__(cls, **kwargs: object) -> None: ) +class RetainsChildren(TypedTable, declared_name=None): + """A retained row requires its whole original child group through declared FK edges.""" + + __slots__ = () + + @classmethod + def retain_required_rows( + cls, original: AttachedPracticeRebuild, schema: type[TableSchema], + ) -> int: + return super().retain_required_rows(original, schema) + schema.retain_referencing_rows( + original, cls, + ) + + class RowidSequenced(TypedTable, declared_name=None): """The table's rows are numbered by SQLite's rowid, and an insert answers its number.""" diff --git a/src/agent_comms/working_memory_annotations.py b/src/agent_comms/working_memory_annotations.py index e85dca11b4..4b47a48c7c 100644 --- a/src/agent_comms/working_memory_annotations.py +++ b/src/agent_comms/working_memory_annotations.py @@ -86,18 +86,15 @@ def for_context( @classmethod def for_segments(cls, source: ReadableStore, segments, classifier) -> tuple[ModelLabel, ...]: """One acquisition serves context and selected-segment readers alike.""" - originals = {} - for segment in segments: - originals.setdefault(segment.sha256, []).append(segment) with cls.reading(source) as db: - rows = SpanAnnotationsRow.for_digests(db, tuple(originals), classifier) + rows = sorted(SpanAnnotationsRow.for_segments( + db, segments, classifier=classifier.classifier, classifier_pin=classifier.pin, + ), key=lambda row: int(row.id)) grouped = {} for row in rows: label = row.label - if any(segment.contains_span(label.span) - for segment in originals[label.span.segment_sha256]): - key = tuple(cls.address(label.span, label.question, label.classifier).values()) - grouped.setdefault(key, []).append(row) + key = tuple(cls.address(label.span, label.question, label.classifier).values()) + grouped.setdefault(key, []).append(row) return tuple(SpanAnnotationsRow.effective(tuple(rows)) for rows in grouped.values()) def reserve(self, request: DisclosureRequest, grant) -> AnnotationRequestsRow: diff --git a/stack/uv.lock b/stack/uv.lock index 2f261b496e..86535f7dcc 100644 --- a/stack/uv.lock +++ b/stack/uv.lock @@ -210,7 +210,7 @@ wheels = [ [[package]] name = "batrachian-toad" version = "0.6.20" -source = { git = "https://github.com/OpenHCSDev/toad.git?branch=main#7805183bf36410b82c1c1b1e327d077c3458634c" } +source = { git = "https://github.com/OpenHCSDev/toad.git?branch=main#9e0452dca33f52f970da7740becc31706be997a2" } dependencies = [ { name = "agent-client-protocol" }, { name = "agent-comms" }, @@ -1029,7 +1029,7 @@ wheels = [ [[package]] name = "textual" version = "8.2.8" -source = { git = "https://github.com/OpenHCSDev/textual.git?branch=main#7c62867cce97d2b0c521c869750413132a315a77" } +source = { git = "https://github.com/OpenHCSDev/textual.git?branch=main#94d743f57df21d80141d0e6aad0be8818e330ac8" } dependencies = [ { name = "markdown-it-py", extra = ["linkify"] }, { name = "mdit-py-plugins" }, diff --git a/tests/test_context_manifest_deltas.py b/tests/test_context_manifest_deltas.py index b78358376d..3ba487a366 100644 --- a/tests/test_context_manifest_deltas.py +++ b/tests/test_context_manifest_deltas.py @@ -3,19 +3,28 @@ from dataclasses import replace import hashlib import sys +from pathlib import Path +from collections.abc import Callable import pytest -from agent_comms.context_store import ContextRequest +from agent_comms.context_store import ( + CapturedContext, ContextEntry, ContextManifestReference, ContextRequest, ContextSegmentVersion, + ContextStore, ContextStoreIdentity, ContextStoreInfo, +) +from agent_comms.errors import RelationViolationError +from agent_comms.installer.practice import PracticeCopy +from agent_comms.typed_table import RowNotFoundError, SQLitePageSize import json +import sqlite3 -from agent_comms.nominal.field_codec import FieldCodec, TurnId +from agent_comms.nominal.field_codec import ContextStoreId, FieldCodec, TurnId from agent_comms.native_session_reopen import NativeSessionIdentity from agent_comms.thread_identity import TurnIdentity from agent_comms.turn_context import ObservedDispatchRequest from agent_comms.turn_context import ( ContextManifest, JournalProvenance, NativeProvenance, RecordedContextPredecessor, - RecordedContextTurn, + RecordedContextTurn, RecordedSegmentManifest, SegmentManifest, SystemLayerSegment, ToolCatalogSegment, TranscriptSegment, ) from bus_record_fixture import context_comms @@ -210,3 +219,129 @@ def test_threads_record_concurrently_and_a_superseded_prior_is_refused(tmp_path) assert comms.contexts.page('Bob', comms.registry).resources[-1].record_id == recorded[1].record_id with pytest.raises(ContextPriorSupersededError): comms.contexts.record_after(alice, context(owners[0], tmp_path, 5, 3)) + + +def test_selected_context_copy_preserves_closure_and_original_sequences(tmp_path: Path) -> None: + comms = context_comms(tmp_path / 'source') + owner = comms.registry.require('Alice') + originals = tuple(comms.contexts.record_after( + UNREAD_CONTEXT, context(owner, tmp_path, count, turn), + ) for turn, count in enumerate((1, 2, 4), start=1)) + selected = originals[1] + reference = ContextManifestReference(selected.source, selected.record_id) + with comms.contexts.snapshot() as db: + expected = comms.contexts.resolve( + db, reference.source, ContextRequest.require(db, id=reference.record_id), + ) + before = ContextStore.digest(comms.contexts.path) + copy = PracticeCopy(comms.root, tmp_path / 'retained') + copy.include_contexts((), requests=(reference, reference), maximum_bytes=2**20) + retained = ContextStore(copy.path) + + with retained.snapshot() as db: + info = ContextStoreInfo.require(db, singleton=1) + assert info.source == selected.source + rows = ContextRequest.select(db, order_by=('id',)) + assert tuple(row.id for row in rows) == tuple(row.record_id for row in originals[:2]) + assert retained.resolve(db, info.source, rows[-1]) == expected + assert ContextEntry.count(db) == 2 + retained_version = max(row.id for row in ContextSegmentVersion.select(db)) + with comms.contexts.snapshot() as db: + source_version = max(row.id for row in ContextSegmentVersion.select(db)) + assert retained_version < source_version + with retained.writing() as db: + # Real row owners allocate after the full original high-water mark, + # not after the greatest ID that happened to be in this selection. + assert ContextRequest.append(db, selected.recorded) == originals[-1].record_id + 1 + fresh = RecordedSegmentManifest(context(owner, tmp_path, 6, 4).segments[2]) + assert ContextSegmentVersion.append(db, fresh) == source_version + 1 + assert ContextStore.digest(comms.contexts.path) == before + assert ContextStore.path_in(comms.root) in copy.copied + assert copy.request.source_root() == comms.root + assert copy.request.root == copy.path + + +def foreign_context_reference( + store: ContextStore, previous: CapturedContext, reference: ContextManifestReference, +) -> ContextManifestReference: + return replace(reference, source=ContextStoreIdentity(ContextStoreId.mint())) + + +def missing_context_reference( + store: ContextStore, previous: CapturedContext, reference: ContextManifestReference, +) -> ContextManifestReference: + return replace(reference, record_id=reference.record_id + 100) + + +def missing_context_predecessor( + store: ContextStore, previous: CapturedContext, reference: ContextManifestReference, +) -> ContextManifestReference: + with store.writing() as db: + ContextRequest.delete(db, id=previous.record_id) + return reference + + +@pytest.mark.parametrize(('change', 'expected'), ( + pytest.param(foreign_context_reference, RelationViolationError, id='foreign-source'), + pytest.param(missing_context_reference, RowNotFoundError, id='missing-record'), + pytest.param(missing_context_predecessor, RowNotFoundError, id='missing-predecessor'), +)) +def test_selected_context_copy_refuses_without_publishing( + tmp_path: Path, + change: Callable[[ContextStore, CapturedContext, ContextManifestReference], ContextManifestReference], + expected: type[Exception], +) -> None: + comms = context_comms(tmp_path / 'source') + owner = comms.registry.require('Alice') + first = comms.contexts.record_after(UNREAD_CONTEXT, context(owner, tmp_path, 1, 1)) + second = comms.contexts.record_after(first, context(owner, tmp_path, 2, 2)) + reference = change(comms.contexts, first, ContextManifestReference(second.source, second.record_id)) + before = ContextStore.digest(comms.contexts.path) + destination = ContextStore(tmp_path / 'retained') + with pytest.raises(expected): + comms.contexts.retain_contexts( + destination, (), requests=(reference,), maximum_bytes=2**20, + ) + assert not destination.path.exists() + assert not tuple(destination.root.iterdir()) + assert ContextStore.digest(comms.contexts.path) == before + + +def test_selected_context_allowance_counts_existing_companions(tmp_path: Path) -> None: + comms = context_comms(tmp_path / 'source') + owner = comms.registry.require('Alice') + selected = comms.contexts.record_after(UNREAD_CONTEXT, context(owner, tmp_path, 1, 1)) + copy = PracticeCopy(comms.root, tmp_path / 'retained') + copy.prepare() + companion = copy.path / 'earlier-store.sqlite3-wal' + original_bytes = b'original retained companion' + companion.write_bytes(original_bytes) + before = ContextStore.digest(comms.contexts.path) + with pytest.raises(RelationViolationError): + copy.include_contexts((), maximum_bytes=len(original_bytes), requests=( + ContextManifestReference(selected.source, selected.record_id), + )) + assert not ContextStore(copy.path).path.exists() + assert tuple(copy.path.iterdir()) == (companion,) + assert companion.read_bytes() == original_bytes + assert not copy.copied + assert ContextStore.digest(comms.contexts.path) == before + + +def test_selected_context_sqlite_allocation_limit_leaves_no_publication(tmp_path: Path) -> None: + comms = context_comms(tmp_path / 'source') + owner = comms.registry.require('Alice') + selected = comms.contexts.record_after(UNREAD_CONTEXT, context(owner, tmp_path, 1, 1)) + copy = PracticeCopy(comms.root, tmp_path / 'retained') + before = ContextStore.digest(comms.contexts.path) + with comms.contexts.snapshot() as db: + page, = SQLitePageSize.read(db.execute('PRAGMA page_size')) + with pytest.raises(sqlite3.OperationalError) as refused: + copy.include_contexts((), maximum_bytes=page.page_size, requests=( + ContextManifestReference(selected.source, selected.record_id), + )) + assert refused.value.sqlite_errorcode == sqlite3.SQLITE_FULL + assert not ContextStore(copy.path).path.exists() + assert not tuple(copy.path.iterdir()) + assert not copy.copied + assert ContextStore.digest(comms.contexts.path) == before diff --git a/tests/test_coordination_response.py b/tests/test_coordination_response.py index 8febcd437e..8e4cc72bdb 100644 --- a/tests/test_coordination_response.py +++ b/tests/test_coordination_response.py @@ -4,6 +4,7 @@ from agent_comms.repo import Checkout import os +from contextlib import closing from dataclasses import dataclass, replace from pathlib import Path @@ -14,6 +15,7 @@ from agent_comms.child_process import ProcessIdentity from agent_comms.comms import Comms, wire from conftest import accept_initial, publish_initial, responder +from agent_comms.coordination_database import CoordinationStore from agent_comms.coordination_errors import ( IdentityConflict, StaleFence, @@ -21,24 +23,27 @@ ) from agent_comms.coordination_response import ( LiveResponseOwner, + PublicationAppendDispatches, prepare_fenced_response, publish_fenced_response, ) -from agent_comms.coordination_tables.executions import NoPointedExecution, WireOrigin +from agent_comms.coordination_tables.executions import ExecutionRecord, NoPointedExecution, WireOrigin +from agent_comms.coordination_tables.publications import PublicationEvidence, PublicationIntents from agent_comms.coordination_tables.responses import ResponseObligation from agent_comms.coordinator import Coordination -from agent_comms.nominal.field_codec import InputId, RootId, ThreadLookup, TurnId +from agent_comms.nominal.field_codec import AssignmentId, InputId, RootId, ThreadLookup, TurnId from agent_comms.lifecycle_capability import Active, Complete from agent_comms.message_bus import MessageBus from agent_comms.assignment_states import CompletedAssignment from agent_comms.coordination_results import AlreadyApplied from agent_comms.execution_states import CompletedExecution from agent_comms.obligation_states import PendingResponse, PublishedResponse, PublishingResponse -from agent_comms.owner_fence import prepare_fence_token +from agent_comms.owner_fence import OwnerFence, prepare_fence_token from agent_comms.publisher import ParticipantPublisher from agent_comms.registration import Registration from agent_comms.selected_source_batch import SelectedSource from agent_comms.threads import Thread +from agent_comms.typed_table import TypedTable from bus_record_fixture import prepare_native_comms # Private bus publication requires POSIX owner/mode ancestry; Windows stat @@ -55,8 +60,8 @@ class Fixture: origin_seq: int reply_target: str owner_lookup: ThreadLookup - assignment_id: str - fence: object + assignment_id: AssignmentId + fence: OwnerFence witness: LiveResponseOwner def close(self) -> None: @@ -367,3 +372,45 @@ def test_response_answers_the_messages_its_turn_took_in(tmp_path: Path) -> None: assert [reply.reference for reply in replies] == [response.reference] finally: case.close() + + +@pytest.mark.parametrize('seed', (PublicationIntents, ExecutionRecord), ids=('intent', 'execution')) +@pytest.mark.parametrize('published', (False, True), ids=('frozen-intent', 'published')) +def test_selected_publication_keeps_original_receipts_and_append_dispatches( + tmp_path: Path, seed: type[TypedTable], published: bool, +) -> None: + case = _ready(tmp_path) + try: + intent = prepare_fenced_response( + case.store, case.bus, case.fence, 'Original retained publication', + exact_target=case.reply_target, owner_witness=case.witness, + ).value + if published: + publish_fenced_response( + case.store, case.bus, case.fence, + exact_target=case.reply_target, owner_witness=case.witness, + ) + snapshot = case.store.get(intent.execution_id) + with case.store.session.read(): + dispatches = tuple(PublicationAppendDispatches.select( + case.store.session._connection, where='execution_id=?', + parameters=(intent.execution_id,), + )) + assert len(snapshot.publication_evidence) == int(published) + assert len(dispatches) == int(published) + source = case.store.session.path + before = CoordinationStore.digest(source) + bus_revision = case.bus.bus.revision() + destination = tmp_path / 'retained-publication' / 'coordination.sqlite3' + with CoordinationStore.retaining_rows( + source, destination, maximum_bytes=8 * 2**20, + ) as original: + original.retain_keys(seed, ({'execution_id': intent.execution_id},)) + with closing(CoordinationStore.read_snapshot(destination)) as db: + assert tuple(PublicationIntents.select(db)) == snapshot.publication_intents + assert tuple(PublicationEvidence.select(db)) == snapshot.publication_evidence + assert tuple(PublicationAppendDispatches.select(db)) == dispatches + assert CoordinationStore.digest(source) == before + assert case.bus.bus.revision() == bus_revision + finally: + case.close() diff --git a/tests/test_harness_receipt_context.py b/tests/test_harness_receipt_context.py index ddb6b14ec3..161e8259d1 100644 --- a/tests/test_harness_receipt_context.py +++ b/tests/test_harness_receipt_context.py @@ -1,10 +1,18 @@ """Interpret actual persisted receipt variants without starting a harness/model.""" +from collections.abc import Callable +from contextlib import closing from dataclasses import replace import pytest from agent_comms.cli_bridges.claude import ClaudeBridge +from agent_comms.bus import Bus +from agent_comms.coordination_database import CoordinationStore from agent_comms.coordination_tables.harness_deliveries import HarnessDelivery +from agent_comms.input_attempt import ReservedInput +from agent_comms.input_disposition import InputDispositions +from agent_comms.installer.practice import PracticeCopy +from agent_comms.nominal.field_codec import InputId, TurnId from agent_comms.coordinator import Coordination from agent_comms.harness_delivery_states import ( AcceptedOutcome, DeliveredOutcome, EmittedOutcome, FilteredOutcome, @@ -59,3 +67,51 @@ def test_only_retained_complete_body_proof_promotes_consumption( assert recorded.monitor == monitor if consumed: assert current.label == current.timed_state(HarnessConsumedProgress.label, 1791964800125) + + +def _record_attempt(ledger: InputDispositions, attempt: ReservedInput) -> None: + ledger.record_originals(attempt) + + +def _start_attempt(ledger: InputDispositions, attempt: ReservedInput) -> None: + _record_attempt(ledger, attempt) + turn, native = TurnId.mint(), InputId.mint() + assert ledger.bind(attempt.key, claim_scope=attempt.claim_generation, + turn_id=turn, native_id=native, text=attempt.source_text) + assert ledger.started(attempt.key, turn_id=turn, native_id=native, text=attempt.source_text) + + +@pytest.mark.parametrize(("record", "retained_attempts"), ( + (lambda ledger, attempt: None, 1), + (_record_attempt, 1), + (_start_attempt, 0), +)) +def test_selected_coordinator_materializer_uses_the_original_unresolved_projection( + tmp_path, record: Callable[[InputDispositions, ReservedInput], None], retained_attempts: int, +): + """Original absence stays visible; a stored completed input cannot resurrect.""" + path, _root_id, comms, initial, people = _root(tmp_path, direct=True) + recipient, = initial.recipients + original = HarnessDelivery(ClaudeBridge, recipient.recipient_lookup, + initial.message.message_id, initial.message.seq, EmittedOutcome) + with Coordination(CoordinationStore.path_in(path)) as store: + with store.session.writing() as db: + assert original.begin(store.assignments, db) + entry = comms.registry.store.snapshot_with_authority().snapshot.entry(people[2].name) + ledger = InputDispositions(path) + attempt, = ledger.harness_attempts((entry,)) + record(ledger, attempt) + expected = ledger.unresolved((entry,)) + coordinator_before = CoordinationStore.path_in(path).read_bytes() + bus_before = comms.bus.bus.revision + copy = PracticeCopy(path, tmp_path / "selected-inputs") + copy.include((Bus.path_in(path), ledger.path)) + selected = copy.include_inputs((entry,), maximum_bytes=3 * 2**30) + assert selected == expected + assert InputDispositions(copy.path).unresolved((entry,)) == expected + with closing(CoordinationStore.read_snapshot(CoordinationStore.path_in(copy.path))) as db: + retained = tuple(HarnessDelivery.select(db)) + assert len(retained) == retained_attempts + assert all(row == original for row in retained) + assert CoordinationStore.path_in(path).read_bytes() == coordinator_before + assert comms.bus.bus.revision == bus_before diff --git a/tests/test_input_documents.py b/tests/test_input_documents.py index bd9eff783e..bc6bbceb4c 100644 --- a/tests/test_input_documents.py +++ b/tests/test_input_documents.py @@ -1,6 +1,7 @@ """Typed saved boundaries and durable acceptance precede process-local scheduling.""" from agent_comms.repo import Checkout +from agent_comms.comms import wire import json import sqlite3 from contextlib import ExitStack, closing @@ -127,6 +128,9 @@ def _insert_raw(store: InputDispositions, key: InputKey, attempt: dict) -> None: def test_saved_discriminator_and_optional_notices_roundtrip_without_replay(tmp_path): goal_id = GoalId.mint() + comms = wire(tmp_path) + comms.registry.declare(Thread('worker', frozenset(), Checkout(tmp_path))) + entry = comms.registry.snapshot().entry('worker') store = InputDispositions(tmp_path) from agent_comms.text_digest import TextDigest @@ -156,7 +160,7 @@ def test_saved_discriminator_and_optional_notices_roundtrip_without_replay(tmp_p assert row.reviewed_for_goal(goal_id) assert FieldCodec.encode(row) == saved assert store.in_turn(TURN).rows == {OLD_KEY: row} - assert store.unresolved(frozenset({"worker"})).rows == {OLD_KEY: row} + assert store.unresolved((entry,)).rows == {OLD_KEY: row} assert not store.bind( row.key, claim_scope=1, @@ -281,6 +285,22 @@ def test_original_native_user_receipts_include_group_and_corrected_final_only(tm ) +def test_owned_bus_input_keeps_its_original_recipient_birth() -> None: + from agent_comms.input_key import OwnedBusInputKey + from agent_comms.thread_identity import UnrecordedThreadIncarnation + + row = ReservedInput( + OwnedBusInputKey(7, 10.0), 7, "owner", 4, "owner", "original input", + ) + assert row.matches_owner(RecordedThreadIncarnation("owner", 10.0)) + assert not row.matches_owner(RecordedThreadIncarnation("owner", 20.0)) + assert not row.matches_owner(RecordedThreadIncarnation("other", 10.0)) + assert not row.matches_owner(UnrecordedThreadIncarnation("owner")) + assert row.queued_text(RecordedThreadIncarnation("owner", 10.0), 4) == "original input" + with pytest.raises(RelationViolationError): + row.queued_text(RecordedThreadIncarnation("owner", 20.0), 4) + + def test_distinct_lifecycle_and_missing_state_never_supply_sent_evidence(tmp_path): store = InputDispositions(tmp_path) missing = stored_inputs(store.root).lookup(MISSING_KEY) @@ -430,6 +450,9 @@ def test_started_delivery_and_original_ingress_use_distinct_owned_relations( def test_reads_select_only_the_rows_their_index_names(tmp_path): + comms = wire(tmp_path) + comms.registry.declare(Thread('owner', frozenset(), Checkout(tmp_path))) + entry = comms.registry.snapshot().entry('owner') store = InputDispositions(tmp_path) for key, owner in ((A_KEY, "owner"), (B_KEY, "owner"), (C_KEY, "other")): assert store.record(key, seq=None, owner=owner, claim_scope=1, target=owner, text=str(key)) @@ -444,7 +467,7 @@ def test_reads_select_only_the_rows_their_index_names(tmp_path): A_KEY, turn_id=TURN, native_id=InputId("a" * 32), text="sent" ) assert store.in_turn(TURN).rows.keys() == {A_KEY} - assert store.unresolved(frozenset({"owner"})).rows.keys() == {B_KEY} + assert store.unresolved((entry,)).rows.keys() == {B_KEY} assert store.read((C_KEY, MISSING_KEY)).rows.keys() == {C_KEY} with closing(sqlite3.connect(store.path)) as connection: plans = [connection.execute("EXPLAIN QUERY PLAN SELECT * FROM input_disposition WHERE " + where, diff --git a/tests/test_maintenance_barrier.py b/tests/test_maintenance_barrier.py index 33f3992742..e997047b44 100644 --- a/tests/test_maintenance_barrier.py +++ b/tests/test_maintenance_barrier.py @@ -252,12 +252,13 @@ def test_rename_and_stopped_same_pid_cannot_reactivate_under_gate(tmp_path: Path comms.registry.declare(owner) comms.registry.rename("owner", "renamed") comms.registry.unregister("renamed") + stopped = comms.registry.snapshot().entry("renamed") comms.owners.maintenance.begin(MaintenanceOperator.current()) with pytest.raises(RelationViolationError): comms.owners.start_many(("owner",)) with pytest.raises(RelationViolationError): comms.registry.register(comms.registry.require("renamed"), new_owner=True) - assert comms.registry.require("renamed").same_local_process(owner.require_process()) + assert comms.registry.snapshot().entry("renamed") == stopped def test_cross_process_claim_races_pause_at_registry_lock(tmp_path: Path) -> None: diff --git a/tests/test_retained_annotations.py b/tests/test_retained_annotations.py new file mode 100644 index 0000000000..7bba5df83b --- /dev/null +++ b/tests/test_retained_annotations.py @@ -0,0 +1,252 @@ +"""Selected original annotation data/refusals, not external disclosure/native proof.""" +from collections.abc import Iterator +from dataclasses import dataclass, replace +from pathlib import Path + +import pytest + +from agent_comms.bus import Bus +from agent_comms.comms import wire +from agent_comms.context_store import ( + UNREAD_CONTEXT, CapturedContext, ContextManifestReference, ContextStore, +) +from agent_comms.coordination_database import CoordinationStore, ReadableStore +from agent_comms.coordination_tables.annotations import AnnotationRequestsRow, SpanAnnotationsRow +from agent_comms.coordinator import StandaloneCoordination +from agent_comms.input_disposition import InputDispositions +from agent_comms.installer.refusals import UpgradeRefused +from agent_comms.installer.practice import PracticeCopy, PracticeMeasurement +from agent_comms.installer.commands import InstallCommands +from agent_comms.installer.probes.inputs import SelectedInputsProbe +from agent_comms.launch_runtime import LaunchRuntime +from agent_comms.maintenance_barrier import MaintenanceBarrier, MaintenanceOperator +from agent_comms.message_reference import MessageReference +from agent_comms.nominal.field_codec import MessageId, TurnId +from agent_comms.repo import Checkout +from agent_comms.task_sources import AnnotationDisclosureGrant, ProjectTaskScope +from agent_comms.thread_identity import RecordedThreadIncarnation, TurnIdentity +from agent_comms.turn_context import ( + ContextManifest, ContextSpan, ContributionCoordinates, ObservedDispatchRequest, + RecordedContextTurn, SystemLayerSegment, ToolCatalogSegment, UnattributedProvenance, +) +from agent_comms.turn_item import PiRequestKey +from agent_comms.working_memory_annotations import PreviouslyRequestedAnnotation, WorkingMemoryAnnotations +from agent_comms.working_memory_disclosure import DisclosureState +from agent_comms.working_memory_labels import ( + HumanLabel, JevChoice, JevClassifier, JevResponse, JevUsage, ModelLabel, QuestionVersion, +) +from agent_comms.working_memory_questions import KindQuestion, OtherSpan, SpanAnswer +from agent_comms.working_memory_requests import ( + AnnotationBudgetExhausted, DisclosureRequest, FailedAnnotationOutcome, +) +from working_memory_fixture import local_system + + +@dataclass(frozen=True) +class AnnotationSource: + store: StandaloneCoordination + context: CapturedContext + grant: AnnotationDisclosureGrant + selected_requests: tuple[DisclosureRequest, ...] + + @property + def path(self) -> Path: + return self.store.session.path + + @property + def reference(self) -> ContextManifestReference: + return ContextManifestReference(self.context.source, self.context.record_id) + + +def completed_label( + store: StandaloneCoordination, request: DisclosureRequest, + grant: AnnotationDisclosureGrant, identifier: str, +) -> ModelLabel: + original = store.annotations.reserve(request, grant) + answers = request.question.question.answer_family.members_with(SpanAnswer) + choice = JevChoice('choice', OtherSpan.declared_name, 0.5, + {answer.declared_name: 1 / len(answers) for answer in answers}) + response = JevResponse(identifier, request.classifier.pin, 'fixture', + {request.question.question.declared_name: choice}, JevUsage(0, 3, 1)) + label = choice.label(request.span, request.question, request.classifier, response) + store.annotations.complete(original, response, label) + return label + + +def disclosed_request( + segment: SystemLayerSegment, span: ContextSpan, grant: MessageReference, +) -> DisclosureRequest: + return DisclosureRequest.capture( + span, JevClassifier.version(), KindQuestion, + DisclosureState.capture(segment, span, KindQuestion), grant, + ) + + +@pytest.fixture +def annotation_source(tmp_path: Path) -> Iterator[AnnotationSource]: + root = tmp_path / 'annotation-source' + comms = wire(root) + # The ordinary bus owner supplies the original empty bus, not a reminted + # destination source. Nothing here sends a message or contacts a provider. + with comms.bus.bus.reading(), InputDispositions(root).snapshot(): + pass + owner = RecordedThreadIncarnation('Fixture', 17001.0) + author = RecordedThreadIncarnation('FixtureHuman', 17002.0) + reference = MessageReference(1, MessageId.mint('stored-annotation-grant')) + grant = AnnotationDisclosureGrant( + scope=ProjectTaskScope(project=Checkout(tmp_path)), subject=reference, + source_user=author, recipient=owner, per_hour=8, segments=(SystemLayerSegment,), + ) + texts = ('Still submitted.\n', 'Recorded failure.\n', 'Original answer.') + positions = [] + offset = 0 + for text in texts: + positions.append(ContributionCoordinates.capture( + SystemLayerSegment, (UnattributedProvenance(),), offset, text, + )) + offset += len(text.encode()) + segment = local_system(''.join(texts), tuple(positions)) + spans = segment.public_spans() + manifest = ContextManifest( + owner, RecordedContextTurn(TurnId('1' * 32), TurnIdentity(owner, 1)), + (segment.measured_manifest(),), 'fixture-counter', + ObservedDispatchRequest(PiRequestKey('fixture-annotation-context')), + ) + captured = ContextStore(root).record_after(UNREAD_CONTEXT, manifest) + with StandaloneCoordination(str(CoordinationStore.path_in(root)), clock_ms=lambda: 1000) as store: + requests = tuple(disclosed_request(segment, span, reference) for span in spans) + store.annotations.reserve(requests[0], grant) + failed = store.annotations.reserve(requests[1], grant) + store.annotations.fail(failed, FailedAnnotationOutcome('recorded external failure')) + model = completed_label(store, requests[2], grant, 'fixture-answer') + # Stored correction data tests precedence, not human/publication authority. + correction = HumanLabel.correct(model, OtherSpan, author) + old_request = replace(requests[2], question=QuestionVersion(KindQuestion, 'b' * 64)) + completed_label(store, old_request, grant, 'fixture-old-question') + other_version = replace(model.classifier, pin=model.classifier.pin + '-fixture-other') + other_reference = MessageReference(2, MessageId.mint('stored-other-classifier-grant')) + other_grant = replace(grant, subject=other_reference, classifier=other_version) + other_request = replace(requests[2], classifier=other_version, grant=other_reference) + completed_label(store, other_request, other_grant, 'fixture-other-classifier') + other_segment = local_system('Not in the selected context.') + other_span, = other_segment.public_spans() + other_usage = disclosed_request(other_segment, other_span, reference) + completed_label(store, other_usage, grant, 'fixture-same-grant-other-context') + unused_reference = MessageReference(3, MessageId.mint('stored-unused-grant')) + unused_grant = replace(grant, subject=unused_reference) + store.annotations.reserve(replace(other_usage, + question=QuestionVersion(KindQuestion, 'c' * 64), grant=unused_reference), unused_grant) + foreign_span = ContextSpan(segment.sha256, ContributionCoordinates.capture( + ToolCatalogSegment, (UnattributedProvenance(),), 1, 't', + )) + assert not segment.measured_manifest().contains_span(foreign_span) + with store.session.writing() as db: + SpanAnnotationsRow(label=correction, created_at_ms=1000).insert(db) + SpanAnnotationsRow(label=replace(model, span=foreign_span), created_at_ms=1000).insert(db) + yield AnnotationSource(store, captured, grant, (*requests, old_request, other_request)) + + +def test_selected_context_copy_preserves_answers_versions_outcomes_and_grant_usage( + annotation_source: AnnotationSource, tmp_path: Path, +) -> None: + source = annotation_source + before = CoordinationStore.digest(source.path) + contexts_before = ContextStore(source.path.parent).path.read_bytes() + expected = WorkingMemoryAnnotations.for_context( + ReadableStore(source.path), (source.context.value,), JevClassifier.version(), + ) + assert len(expected) == 2 + copy = PracticeCopy(source.path.parent, tmp_path / 'selected-annotations') + copy.include((Bus.path_in(source.path.parent), InputDispositions.path_in(source.path.parent))) + copy.include_contexts((), requests=(source.reference,), maximum_bytes=8 * 2**20) + copy.include_inputs((), maximum_bytes=8 * 2**20) + retained_path = CoordinationStore.path_in(copy.path) + assert WorkingMemoryAnnotations.for_context( + ReadableStore(retained_path), (source.context.value,), JevClassifier.version(), + ) == expected + with source.store.session.read(): + db = source.store.session._connection + originals = tuple(AnnotationRequestsRow.select(db, order_by=('id',))) + labels = tuple(SpanAnnotationsRow.select(db, order_by=('id',))) + usage = AnnotationRequestsRow.requests_since(db, source.grant.subject, 0) + with StandaloneCoordination(str(retained_path)) as retained: + with retained.session.read(): + db = retained.session._connection + rows = tuple(AnnotationRequestsRow.select(db, order_by=('id',))) + assert rows == tuple(row for row in originals + if row.request.grant in {request.grant for request in source.selected_requests}) + assert tuple(SpanAnnotationsRow.select(db, order_by=('id',))) == tuple( + row for row in labels if row.label.span.coordinates.kind is SystemLayerSegment) + assert AnnotationRequestsRow.requests_since(db, source.grant.subject, 0) == usage == 5 + for request in source.selected_requests[:3]: + with pytest.raises(PreviouslyRequestedAnnotation): + retained.annotations.reserve(request, source.grant) + # The exact grant-wide count, including the unselected context, decides + # the same budget refusal on source and retained data; no request is sent. + exhausted = replace(source.grant, per_hour=usage) + with pytest.raises(AnnotationBudgetExhausted): + exhausted.require_budget(usage) + assert CoordinationStore.digest(source.path) == before + assert ContextStore(source.path.parent).path.read_bytes() == contexts_before + + +def test_effective_annotations_require_an_original_allocated_row_id( + annotation_source: AnnotationSource, +) -> None: + source = annotation_source + with source.store.session.read(): + stored = SpanAnnotationsRow.select( + source.store.session._connection, order_by=('id',), + )[0] + assert SpanAnnotationsRow.effective((stored,)) == stored.label + unstored = SpanAnnotationsRow(label=stored.label, created_at_ms=stored.created_at_ms) + with pytest.raises(TypeError): + SpanAnnotationsRow.effective((unstored,)) + + +def test_context_selection_cannot_follow_finalized_coordinator_evidence( + annotation_source: AnnotationSource, tmp_path: Path, +) -> None: + source = annotation_source + before = CoordinationStore.digest(source.path) + copy = PracticeCopy(source.path.parent, tmp_path / 'wrong-selection-order') + copy.include((Bus.path_in(source.path.parent), InputDispositions.path_in(source.path.parent))) + copy.include_inputs((), maximum_bytes=8 * 2**20) + with pytest.raises(UpgradeRefused): + copy.include_contexts((), requests=(source.reference,), maximum_bytes=8 * 2**20) + assert not ContextStore(copy.path).path.exists() + assert CoordinationStore.digest(source.path) == before + + +def test_selected_annotation_materialization_writes_only_the_private_target( + annotation_source: AnnotationSource, tmp_path: Path, +) -> None: + source = annotation_source + root = source.path.parent + before = CoordinationStore.digest(source.path) + contexts_before = ContextStore.digest(ContextStore.path_in(root)) + expected = WorkingMemoryAnnotations.for_context( + ReadableStore(source.path), (source.context.value,), JevClassifier.version(), + ) + comms = wire(root) + gate = MaintenanceBarrier(comms.registry.store.path) + held = gate.retarget(gate.begin(MaintenanceOperator.current()), LaunchRuntime.running()) + copy = PracticeCopy(root, tmp_path / 'captured-annotations') + try: + probe = SelectedInputsProbe( + root, copy.path, 8 * 2**20, requests=(source.reference,), + ) + with held.holding(), InstallCommands() as commands: + measured = commands.probe( + probe, LaunchRuntime.running().interpreter, cwd=tmp_path, + ) + assert measured == PracticeMeasurement.capture(copy.path) + contexts = ContextStore(copy.path).original_segments() + retained = WorkingMemoryAnnotations.for_segments( + ReadableStore(CoordinationStore.path_in(copy.path)), contexts, JevClassifier.version(), + ) + assert retained == expected + assert CoordinationStore.digest(source.path) == before + assert ContextStore.digest(ContextStore.path_in(root)) == contexts_before + finally: + gate.clear(held) diff --git a/tests/test_retained_capture.py b/tests/test_retained_capture.py new file mode 100644 index 0000000000..50a13f902c --- /dev/null +++ b/tests/test_retained_capture.py @@ -0,0 +1,233 @@ +"""Selected capture borrows a real hold and drains real store transactions.""" +from collections.abc import Callable +from concurrent.futures import ThreadPoolExecutor +from contextlib import AbstractContextManager, nullcontext +from pathlib import Path +from threading import Event + +import pytest + +from agent_comms.bus import Bus +from agent_comms.comms import Comms, wire +from agent_comms.context_store import ContextStore +from agent_comms.coordination_database import CoordinationStore +from agent_comms.errors import RelationViolationError +from agent_comms.input_disposition import InputDispositions +from agent_comms.input_key import TurnInputKey +from agent_comms.installer.practice import PracticeCopy +from agent_comms.installer.refusals import UpgradeRefused +from agent_comms.installer.probes.inputs import SelectedInputsProbe +from agent_comms.launch_runtime import LaunchRuntime +from agent_comms.maintenance_barrier import ( + MaintenanceBarrier, MaintenanceNotHeldError, MaintenanceOperator, NoMaintenanceHolder, +) +from agent_comms.nominal.field_codec import TurnId +from agent_comms.repo import Checkout +from agent_comms.threads import Thread +from agent_comms.typed_table import WalStore +from bus_record_fixture import prepare_native_comms + + +type StoreOpening = Callable[[Path], AbstractContextManager[WalStore | CoordinationStore]] + + +@pytest.fixture +def capture_source(tmp_path: Path) -> Comms: + comms = prepare_native_comms(wire(tmp_path / 'source')) + comms.registry.declare(Thread('capture-owner', frozenset(), Checkout(tmp_path), created_at=17001.0)) + with comms.bus.bus.reading(), InputDispositions(comms.root).writing(), ContextStore(comms.root).writing(): + pass + with CoordinationStore(CoordinationStore.path_in(comms.root)): + pass + return comms + + +def test_capture_refuses_a_destination_inside_its_source( + capture_source: Comms, +) -> None: + copy = PracticeCopy(capture_source.root, capture_source.root / 'nested-copy') + with pytest.raises(UpgradeRefused, match='separate destination'), copy.capturing_inputs(): + pytest.fail('capture accepted a destination inside its donor') + assert not copy.path.exists() + + +def test_selected_probe_refuses_oversized_complete_readers_before_copying( + capture_source: Comms, tmp_path: Path, +) -> None: + source = capture_source + gate = MaintenanceBarrier(source.registry.store.path) + held = gate.retarget(gate.begin(MaintenanceOperator.current()), LaunchRuntime.running()) + destination = tmp_path / 'too-small' + try: + with held.holding(), pytest.raises(UpgradeRefused, match='Complete input readers'): + SelectedInputsProbe(source.root, destination, 1).run() + assert not Bus.path_in(destination).exists() + assert not InputDispositions.path_in(destination).exists() + finally: + gate.clear(held) + + +stores = pytest.mark.parametrize('opening', ( + pytest.param(lambda root: nullcontext(Bus(Bus.path_in(root))), id='bus'), + pytest.param(lambda root: nullcontext(InputDispositions(root)), id='ledger'), + pytest.param(lambda root: nullcontext(ContextStore(root)), id='contexts'), + pytest.param(lambda root: CoordinationStore(CoordinationStore.path_in(root)), id='coordinator'), +)) + + +def test_selected_capture_refuses_an_open_or_only_draining_root( + capture_source: Comms, tmp_path: Path, +) -> None: + source = capture_source + copy = PracticeCopy(source.root, tmp_path / 'selected') + gate = MaintenanceBarrier(source.registry.store.path) + with pytest.raises(MaintenanceNotHeldError), copy.capturing_inputs(): + pytest.fail('open root supplied a capture cut') + held = gate.begin(MaintenanceOperator.current()) + try: + with held.holding(), pytest.raises(MaintenanceNotHeldError), copy.capturing_inputs(): + pytest.fail('unretargeted drain supplied a capture cut') + finally: + gate.clear(held) + assert not copy.path.exists() + + +def test_selected_capture_requires_holder_and_actual_target_runtime( + capture_source: Comms, tmp_path: Path, +) -> None: + source = capture_source + copy = PracticeCopy(source.root, tmp_path / 'selected') + gate = MaintenanceBarrier(source.registry.store.path) + held = gate.retarget(gate.begin(MaintenanceOperator.current()), + LaunchRuntime(tmp_path / 'other-runtime/bin/python')) + try: + with held.holding(), pytest.raises(MaintenanceNotHeldError), copy.capturing_inputs(): + pytest.fail('another runtime supplied a capture cut') + held = gate.retarget(held, LaunchRuntime.running()) + with NoMaintenanceHolder().selected(), pytest.raises(MaintenanceNotHeldError), copy.capturing_inputs(): + pytest.fail('non-holder supplied a capture cut') + finally: + gate.clear(held) + assert not copy.path.exists() + + +@stores +def test_capture_waits_for_each_already_admitted_transaction( + capture_source: Comms, tmp_path: Path, opening: StoreOpening, +) -> None: + source = capture_source + admitted, release, started, captured = Event(), Event(), Event(), Event() + copy = PracticeCopy(source.root, tmp_path / 'selected') + gate = MaintenanceBarrier(source.registry.store.path) + + def writer() -> None: + with NoMaintenanceHolder().selected(), opening(source.root) as store, store.writing() as db: + db.execute('SELECT 1').fetchone() + admitted.set() + assert release.wait(5), 'controller did not release the admitted writer' + + with ThreadPoolExecutor(max_workers=2) as workers: + pending = workers.submit(writer) + try: + assert admitted.wait(5), 'writer did not enter its ordinary transaction' + held = gate.retarget(gate.begin(MaintenanceOperator.current()), LaunchRuntime.running()) + + def capture() -> None: + with held.holding(): + started.set() + with copy.capturing_inputs(): + captured.set() + + taking = workers.submit(capture) + assert started.wait(5) + assert not captured.wait(0.05), 'capture passed an admitted SQL writer' + release.set() + pending.result(timeout=5) + taking.result(timeout=5) + assert captured.is_set() + gate.clear(held) + finally: + release.set() + + +@stores +def test_capture_refuses_later_non_holder_store_writes( + capture_source: Comms, tmp_path: Path, opening: StoreOpening, +) -> None: + source = capture_source + gate = MaintenanceBarrier(source.registry.store.path) + held = gate.retarget(gate.begin(MaintenanceOperator.current()), LaunchRuntime.running()) + copy = PracticeCopy(source.root, tmp_path / 'selected') + try: + with held.holding(), copy.capturing_inputs(): + with NoMaintenanceHolder().selected(), pytest.raises(RelationViolationError): + with opening(source.root) as store, store.writing(): + pytest.fail('later non-holder writer entered the captured store') + finally: + gate.clear(held) + + +def test_capture_keeps_original_registry_association_through_first_copy( + capture_source: Comms, tmp_path: Path, +) -> None: + source = capture_source + expected = source.registry.snapshot().entry('capture-owner') + before = source.bus.bus.digest(source.bus.bus.path) + gate = MaintenanceBarrier(source.registry.store.path) + held = gate.retarget(gate.begin(MaintenanceOperator.current()), LaunchRuntime.running()) + copy = PracticeCopy(source.root, tmp_path / 'selected') + try: + with held.holding(), copy.capturing_inputs() as original: + assert original.snapshot.entry('capture-owner') == expected + assert original.matches_current() + copy.include((Bus.path_in(source.root), InputDispositions.path_in(source.root))) + assert Bus.digest(Bus.path_in(copy.path)) == before + assert source.registry.snapshot().entry('capture-owner') == expected + assert source.bus.bus.digest(source.bus.bus.path) == before + with held.holding(), pytest.raises(UpgradeRefused), copy.capturing_inputs(): + pytest.fail('capture started after its first source copy') + finally: + gate.clear(held) + + +def test_capture_rejects_a_holder_side_effect_instead_of_accepting_mixed_sources( + capture_source: Comms, tmp_path: Path, +) -> None: + source = capture_source + gate = MaintenanceBarrier(source.registry.store.path) + held = gate.retarget(gate.begin(MaintenanceOperator.current()), LaunchRuntime.running()) + ledger = InputDispositions(source.root) + key = TurnInputKey(TurnId('a' * 32)) + copy = PracticeCopy(source.root, tmp_path / 'selected') + try: + with held.holding(), pytest.raises(RelationViolationError): + with copy.capturing_inputs(): + copy.include((ledger.path,)) + ledger.record(key, seq=None, owner='capture-owner', claim_scope=1, + target='capture-owner', text='holder wrote after copying') + assert key not in ledger.read((key,)).rows + assert key not in InputDispositions(copy.path).read((key,)).rows + finally: + gate.clear(held) + + +def test_capture_borrows_one_hold_generation_and_releases_resources_on_failure( + capture_source: Comms, tmp_path: Path, +) -> None: + source = capture_source + gate = MaintenanceBarrier(source.registry.store.path) + runtime = LaunchRuntime.running() + held = gate.retarget(gate.begin(MaintenanceOperator.current()), runtime) + copy = PracticeCopy(source.root, tmp_path / 'selected') + try: + with held.holding(), pytest.raises(MaintenanceNotHeldError): + with gate.capturing(runtime): + held = gate.retarget(held, runtime) + with held.holding(), pytest.raises(RuntimeError, match='stop materialization'): + with copy.capturing_inputs(): + raise RuntimeError('stop materialization') + finally: + gate.clear(held) + with InputDispositions(source.root).writing(): + pass + assert not copy.path.exists() diff --git a/tests/test_retained_coordinator_groups.py b/tests/test_retained_coordinator_groups.py new file mode 100644 index 0000000000..a4f1f9d3b6 --- /dev/null +++ b/tests/test_retained_coordinator_groups.py @@ -0,0 +1,384 @@ +"""Original harness groups survive selected loading without replaying their lifecycle.""" +from collections.abc import Callable, Iterator +from contextlib import AbstractContextManager, closing +from dataclasses import dataclass, replace +from pathlib import Path +from uuid import uuid4 + +import pytest + +from agent_comms.assignment_states import FullPendingAssignment, TriagePendingAssignment +from agent_comms.attempt_start import AttemptStart +from agent_comms.comms import wire +from agent_comms.cli_bridges.claude import ClaudeBridge +from agent_comms.cli_bridges.codex import CodexBridge +from agent_comms.coordination_database import CoordinationStore +from agent_comms.coordination_errors import IdentityConflict, ParticipantNotRegistered +from agent_comms.coordination_tables.assignments import WakeAssignment +from agent_comms.coordination_tables.harness_deliveries import HarnessDelivery, HarnessObservedPart +from agent_comms.coordination_tables.executions import ( + AcpOrigin, CurrentExecutions, ExecutionRecord, +) +from agent_comms.coordination_tables.participants import OwnerGenerations, Participants +from agent_comms.coordination_tables.recovery import RecoveryAudit +from agent_comms.coordinator import StandaloneCoordination +from agent_comms.harness_delivery_states import EmittedOutcome, ReceivingMonitor +from agent_comms.message_reference import MessageReference +from agent_comms.native_input_record import FullNativeExecution, TriageNativeExecution +from agent_comms.native_runtime_input import NativeRuntimeInput +from agent_comms.nominal.field_codec import AssignmentId, MessageId, ThreadLookup, TranscriptRecordIdText +from agent_comms.owner_fence import prepare_fence_token +from agent_comms.participant_store import ParticipantRows, ParticipantStore +from agent_comms.reason_codes import GivenReasonCode, NoDetail +from agent_comms.recovery_states import ModelStalledRecovery +from agent_comms.triage_native_sources import TriageNativeSource +from agent_comms.typed_table import AttachedPracticeRebuild, NoDelete, TypedTable, UnallocatedRowId +from agent_comms.wake_policy import FullWake, MessageAudience +from bus_record_fixture import ( + FIRST_INPUT_ID, SECOND_INPUT_ID, THIRD_INPUT_ID, RECIPIENT_LOOKUP, prepare_native_comms, +) + + +def retaining_coordinator( + source: Path, destination: Path, +) -> AbstractContextManager[AttachedPracticeRebuild]: + """Use the real coordinator expansion with one shared test-copy allowance.""" + return CoordinationStore.retaining_rows( + source, destination, maximum_bytes=8 * 2**20, original_type=ParticipantRows, + ) + + +@dataclass(frozen=True) +class HarnessSource: + store: StandaloneCoordination + deliveries: tuple[HarnessDelivery, ...] + parts: tuple[HarnessObservedPart, ...] + claims: tuple[WakeAssignment, ...] + + @property + def path(self) -> Path: + return self.store.session.path + + +@pytest.fixture +def harness_source(tmp_path: Path) -> Iterator[HarnessSource]: + path = tmp_path / 'source' / 'coordination.sqlite3' + with StandaloneCoordination(str(path), clock_ms=lambda: 1000) as store: + store.participants.register(RECIPIENT_LOOKUP, committed=True) + claim = WakeAssignment( + assignment_id=AssignmentId.cohort(b'original-retained-claim'), + recipient='Alice', recipient_lookup=RECIPIENT_LOOKUP, + wire_seq=1, message_id=MessageId.mint('original-first-message'), + audience=MessageAudience.DIRECT, + lifecycle=FullPendingAssignment.load(FullWake(), None, None, None), + accepted_at_ms=1000, updated_at_ms=1000, + ) + store.assignments.accept(claim) + monitors = (ReceivingMonitor(2), ReceivingMonitor(1)) + deliveries = ( + HarnessDelivery(ClaudeBridge, RECIPIENT_LOOKUP, claim.message_id, 1, + EmittedOutcome, monitors[0]), + HarnessDelivery(ClaudeBridge, RECIPIENT_LOOKUP, + MessageId.mint('original-second-message'), 2, + EmittedOutcome, monitors[1]), + ) + parts = tuple( + HarnessObservedPart(delivery.bridge, delivery.recipient_lookup, delivery.message_id, + number, 'Sender', f'original body {delivery.seq}/{number}', + TranscriptRecordIdText.decode(str(uuid4())), + 10.0 + delivery.seq + number / 10) + for delivery, monitor in zip(deliveries, monitors, strict=True) + for number in range(1, monitor.total + 1) + ) + with store.session.writing() as db: + for delivery in deliveries: + delivery.begin(store.assignments, db) + for part in parts: + part.insert(db) + # Keep the original post-handoff state and revision, not a reconstructed + # initial claim or a transition performed while materializing the copy. + yield HarnessSource(store, deliveries, parts, + (store.assignments.get(claim.assignment_id),)) + + +@pytest.mark.parametrize('selected_index', (0, 1), ids=('existing-claim', 'no-claim')) +def test_retained_harness_group_keeps_exact_parts_and_existing_claims( + harness_source: HarnessSource, tmp_path: Path, selected_index: int, +) -> None: + selected = harness_source.deliveries[selected_index] + destination = tmp_path / 'retained' / 'coordination.sqlite3' + before = CoordinationStore.digest(harness_source.path) + with retaining_coordinator(harness_source.path, destination) as original: + original.retain_keys(HarnessDelivery, ({ + 'bridge': selected.bridge, 'recipient_lookup': selected.recipient_lookup, + 'message_id': selected.message_id, + },)) + with StandaloneCoordination(str(destination)) as retained: + with retained.session.read(): + db = retained.session._connection + assert tuple(HarnessDelivery.select(db)) == (selected,) + assert tuple(HarnessObservedPart.select(db, order_by=('number',))) == tuple( + part for part in harness_source.parts if part.message_id == selected.message_id + ) + assert tuple(WakeAssignment.select(db)) == tuple( + claim for claim in harness_source.claims if claim.source == selected.reference + ) + assert CoordinationStore.digest(harness_source.path) == before + + +def test_harness_claim_query_refuses_conflicting_full_reference(harness_source: HarnessSource) -> None: + selected = harness_source.deliveries[0] + wrong = MessageReference(selected.seq, MessageId.mint('foreign-message-at-same-sequence')) + with harness_source.store.session.read(): + db = harness_source.store.session._connection + assert tuple(WakeAssignment.for_harness_sources( + db, selected.recipient_lookup, (selected.reference, selected.reference), + )) == harness_source.claims + assert tuple(WakeAssignment.for_harness_sources(db, selected.recipient_lookup, ())) == () + with pytest.raises(IdentityConflict): + tuple(WakeAssignment.for_harness_sources(db, selected.recipient_lookup, (wrong,))) + + +def test_retained_harness_conflict_refuses_without_publication( + harness_source: HarnessSource, tmp_path: Path, +) -> None: + # A foreign producer can leave a SQL-valid attempt whose natural-key claim + # names another message. Retention must refuse it, not filter the claim away. + conflicting = replace(harness_source.deliveries[0], bridge=CodexBridge, + message_id=MessageId.mint('foreign-source-message')) + with harness_source.store.session.writing() as db: + conflicting.insert(db) + before = CoordinationStore.digest(harness_source.path) + destination = tmp_path / 'retained' / 'coordination.sqlite3' + with pytest.raises(IdentityConflict): + with retaining_coordinator(harness_source.path, destination) as original: + original.retain_keys(HarnessDelivery, ({ + 'bridge': conflicting.bridge, 'recipient_lookup': conflicting.recipient_lookup, + 'message_id': conflicting.message_id, + },)) + assert not destination.exists() + assert not tuple(destination.parent.iterdir()) + assert CoordinationStore.digest(harness_source.path) == before + + +@dataclass(frozen=True) +class NativeGroupSource: + store: StandaloneCoordination + native: tuple[NativeRuntimeInput, ...] + claims: tuple[WakeAssignment, ...] + + @property + def path(self) -> Path: + return self.store.session.path + + +@pytest.fixture +def native_group_source(tmp_path: Path) -> Iterator[NativeGroupSource]: + comms = prepare_native_comms(wire(tmp_path / 'native-source')) + with StandaloneCoordination(str(comms.root / 'coordination.sqlite3'), + clock_ms=lambda: 1000) as store: + store.participants.register(RECIPIENT_LOOKUP, committed=True) + claims = tuple(WakeAssignment( + assignment_id=AssignmentId.cohort(f'original-triage-{number}'.encode()), + recipient='Alice', recipient_lookup=RECIPIENT_LOOKUP, wire_seq=number, + message_id=MessageId.mint(f'original-native-message-{number}'), + audience=MessageAudience.DIRECT, lifecycle=TriagePendingAssignment(), + accepted_at_ms=1000, updated_at_ms=1000, + ) for number in (1, 2)) + for claim in claims: + store.assignments.accept(claim) + store.create('original-claimless-full', AcpOrigin, RECIPIENT_LOOKUP, 1) + stages = (TriageNativeExecution(), TriageNativeExecution(), + FullNativeExecution('original-claimless-full', 1)) + native = tuple(NativeRuntimeInput( + input_id=input_id, stage=type(stage), **stage.binding_fields(), + owner_lookup=RECIPIENT_LOOKUP, owner_generation=1, + owner_token_digest='e' * 64, + ) for input_id, stage in zip( + (FIRST_INPUT_ID, SECOND_INPUT_ID, THIRD_INPUT_ID), stages, strict=True, + )) + with store.session.writing() as db: + for row, members in zip(native, ((claims[0],), (claims[1],), ()), strict=True): + row.insert(db) + row.execution.record_sources(db, row.input_id, members) + # These are original reserved SQL groups, not actual native admission or + # a running practice root. No provider or native process is started. + yield NativeGroupSource(store, native, claims) + + +@pytest.mark.parametrize('seed,keys,selected_index,claim_count,execution_ids', ( + (NativeRuntimeInput, {'input_id': FIRST_INPUT_ID}, 0, 1, ()), + (ExecutionRecord, {'execution_id': 'original-claimless-full'}, 2, 0, ('original-claimless-full',)), +), ids=('triage-input', 'full-execution')) +def test_retained_native_group_uses_original_membership_and_children( + native_group_source: NativeGroupSource, tmp_path: Path, seed: type[TypedTable], + keys: dict[str, object], selected_index: int, claim_count: int, execution_ids: tuple[str, ...], +) -> None: + source = native_group_source + selected = source.native[selected_index] + expected_claims = source.claims[:claim_count] + before = CoordinationStore.digest(source.path) + destination = tmp_path / 'native-retained' / 'coordination.sqlite3' + with retaining_coordinator(source.path, destination) as original: + original.retain_keys(seed, (keys,)) + with closing(CoordinationStore.read_snapshot(destination)) as db: + assert tuple(NativeRuntimeInput.select(db)) == (selected,) + assert tuple(WakeAssignment.select(db)) == expected_claims + assert tuple(TriageNativeSource.select(db)) == tuple( + TriageNativeSource(selected.input_id, ordinal, claim.assignment_id) + for ordinal, claim in enumerate(expected_claims) + ) + assert tuple(row.execution_id for row in ExecutionRecord.select(db)) == execution_ids + assert ParticipantStore.read_at(db, RECIPIENT_LOOKUP) == source.store.participants.get(RECIPIENT_LOOKUP) + assert CoordinationStore.digest(source.path) == before + + +def test_selected_participant_keeps_its_generation_and_current_pointer( + native_group_source: NativeGroupSource, tmp_path: Path, +) -> None: + source = native_group_source + source.store.participants.advance_generation(RECIPIENT_LOOKUP, expected_generation=1) + unrelated = ThreadLookup('9' * 32) + absent = ThreadLookup('8' * 32) + source.store.participants.register(unrelated, committed=True) + source.store.create('selected-current', AcpOrigin, RECIPIENT_LOOKUP, 1) + source.store.mark_pending('selected-current', expected_revision=1) + source.store.start(AttemptStart( + 'selected-current', 1, RECIPIENT_LOOKUP, 2, prepare_fence_token(), + expected_execution_revision=2, expected_pointer_revision=0, + )) + expected = source.store.participants.get(RECIPIENT_LOOKUP) + before = CoordinationStore.digest(source.path) + destination = tmp_path / 'participant-retained' / 'coordination.sqlite3' + with retaining_coordinator(source.path, destination) as original: + with pytest.raises(ParticipantNotRegistered): + ParticipantStore.read_at(original.destination, RECIPIENT_LOOKUP) + assert ParticipantStore.read_at( + original.destination, RECIPIENT_LOOKUP, table_source=original.source, + ) == expected + ParticipantStore.retain_owners(original, (RECIPIENT_LOOKUP, absent)) + with closing(CoordinationStore.read_snapshot(destination)) as db: + assert ParticipantStore.read_at(db, RECIPIENT_LOOKUP) == expected + assert tuple(Participants.select(db)) == (Participants( + participant_lookup=RECIPIENT_LOOKUP, committed=True, + ),) + assert tuple(row.execution_id for row in ExecutionRecord.select(db)) == ('selected-current',) + assert not NativeRuntimeInput.select(db) + assert not WakeAssignment.select(db) + assert expected.owner_identity.generation == 2 + assert expected.pointer.pointer_revision == 1 + assert CoordinationStore.digest(source.path) == before + + +@pytest.mark.parametrize('missing', (OwnerGenerations, CurrentExecutions), + ids=('missing-generation', 'missing-pointer')) +@pytest.mark.parametrize('retain_seed', ( + lambda original, delivery: ParticipantStore.retain_owners( + original, (delivery.recipient_lookup,), + ), + lambda original, delivery: delivery.retain_original(original), +), ids=('direct-owner', 'fk-discovered-owner')) +def test_incomplete_original_participant_refuses_selected_publication( + native_group_source: NativeGroupSource, tmp_path: Path, missing: type[TypedTable], + retain_seed: Callable[[AttachedPracticeRebuild, HarnessDelivery], int], +) -> None: + source = native_group_source + lookup = ThreadLookup('7' * 32) + delivery = HarnessDelivery( + ClaudeBridge, lookup, MessageId.mint('incomplete-owner-message'), 7, EmittedOutcome, + ) + source.store.participants.register(lookup, committed=True) + source.store.participants.get(lookup) # The original aggregate starts valid. + with source.store.session.writing() as db: + delivery.insert(db) + # Explicit foreign corruption, not an admitted coordinator transition: + # remove only this type's delete guard, leaving insertion membership intact, + # then restore the exact declared schema before the retained reader sees it. + delete_guards = NoDelete.mutation_guards(missing) + for guard in delete_guards: + db.execute(f'DROP TRIGGER "{guard.name}"') + missing.delete(db, owner_lookup=lookup) + for guard in delete_guards: + db.execute(guard.sql) + CoordinationStore.schema_at(source.path).require_schema(db) + with pytest.raises(ParticipantNotRegistered): + source.store.participants.get(lookup) + before = CoordinationStore.digest(source.path) + destination = tmp_path / 'incomplete-participant' / 'coordination.sqlite3' + with pytest.raises(ParticipantNotRegistered): + with retaining_coordinator(source.path, destination) as original: + retain_seed(original, delivery) + assert not destination.exists() + assert not tuple(destination.parent.iterdir()) + assert CoordinationStore.digest(source.path) == before + + +@pytest.mark.parametrize('keys,selected_count', ( + (({'execution_id': 'sequence-selected'},), 1), + ((), 0), +), ids=('lower-audit', 'empty-selection')) +def test_selected_store_keeps_sqlite_high_water_without_rowid_mixin( + harness_source: HarnessSource, tmp_path: Path, + keys: tuple[dict[str, object], ...], selected_count: int, +) -> None: + source = harness_source + other = ThreadLookup('6' * 32) + source.store.participants.register(other, committed=True) + for execution, owner in (('sequence-selected', RECIPIENT_LOOKUP), ('sequence-omitted', other)): + source.store.create(execution, AcpOrigin, owner, 1) + source.store.mark_pending(execution, expected_revision=1) + source.store.start(AttemptStart( + execution, 1, owner, 1, prepare_fence_token(), + expected_execution_revision=2, expected_pointer_revision=0, + )) + # Actual stored audit rows exercise allocation, not model-stall or native proof. + with source.store.session.writing() as db: + cursor = RecoveryAudit( + execution_id=execution, kind=ModelStalledRecovery, + reason_code=GivenReasonCode('sequence_control'), sanitized_detail=NoDetail(), + attempt=1, elapsed_ms=0, observed_at_ms=1000, + ).insert(db) + source_high_water = cursor.lastrowid + assert source_high_water is not None + with source.store.session.read(): + audits = tuple(RecoveryAudit.select(source.store.session._connection, order_by=('audit_id',))) + before = CoordinationStore.digest(source.path) + destination = tmp_path / 'sequence-retained' / 'coordination.sqlite3' + with retaining_coordinator(source.path, destination) as original: + original.retain_keys(ExecutionRecord, keys) + with StandaloneCoordination(str(destination), clock_ms=lambda: 1000) as retained: + with retained.session.read(): + assert tuple(RecoveryAudit.select(retained.session._connection, order_by=('audit_id',))) == audits[:selected_count] + fresh_owner = ThreadLookup('5' * 32) + retained.participants.register(fresh_owner, committed=True) + retained.create('sequence-new', AcpOrigin, fresh_owner, 1) + retained.mark_pending('sequence-new', expected_revision=1) + retained.start(AttemptStart( + 'sequence-new', 1, fresh_owner, 1, prepare_fence_token(), + expected_execution_revision=2, expected_pointer_revision=0, + )) + with retained.session.writing() as db: + fresh = replace(audits[0], execution_id='sequence-new', audit_id=UnallocatedRowId()) + assert fresh.insert(db).lastrowid == source_high_water + 1 + assert CoordinationStore.digest(source.path) == before + + +def test_native_membership_query_uses_the_supplied_attachment( + native_group_source: NativeGroupSource, tmp_path: Path, +) -> None: + source = native_group_source + selected = source.native[0] + before = CoordinationStore.digest(source.path) + destination = tmp_path / 'membership-query' / 'coordination.sqlite3' + with retaining_coordinator(source.path, destination) as original: + # Both table sets exist. Ordinary lookup sees the empty destination; + # qualification must keep every join on the original attachment. + assert selected.execution.source_assignments(original.destination, selected.input_id) == () + assert selected.execution.source_assignments( + original.destination, selected.input_id, table_source=original.source, + ) == (source.claims[0],) + assert selected.execution.source_assignment_ids(original.destination, selected.input_id) == () + assert selected.execution.source_assignment_ids( + original.destination, selected.input_id, table_source=original.source, + ) == (source.claims[0].assignment_id,) + assert CoordinationStore.digest(source.path) == before diff --git a/tests/test_working_memory_read.py b/tests/test_working_memory_read.py index a6b78d3ad7..8462f1e5f2 100644 --- a/tests/test_working_memory_read.py +++ b/tests/test_working_memory_read.py @@ -1,6 +1,5 @@ """Read/correct original annotation addresses without reader-side installation.""" from dataclasses import replace -import hashlib import pytest from agent_comms.coordination_database import ReadableStore @@ -13,14 +12,7 @@ from agent_comms.working_memory_labels import ( JevClassifier, ) - - -def local_system(text, sources=()): - """Unattested local content; no SDK/provider observation is claimed.""" - raw = text.encode() - return SystemLayerSegment(provenance=(UnattributedProvenance(),), content=text, - tokens=0, sha256=hashlib.sha256(raw).hexdigest(), utf8_bytes=len(raw), - source_spans=sources) +from working_memory_fixture import local_system def test_readers_do_not_create_a_missing_store(tmp_path): diff --git a/tests/working_memory_fixture.py b/tests/working_memory_fixture.py new file mode 100644 index 0000000000..0f6e18fcc5 --- /dev/null +++ b/tests/working_memory_fixture.py @@ -0,0 +1,17 @@ +"""Unattested local annotation data, never an SDK/provider observation.""" +import hashlib + +from agent_comms.turn_context import ( + ContributionCoordinates, SystemLayerSegment, UnattributedProvenance, +) + + +def local_system( + text: str, sources: tuple[ContributionCoordinates, ...] = (), +) -> SystemLayerSegment: + raw = text.encode() + return SystemLayerSegment( + provenance=(UnattributedProvenance(),), content=text, tokens=0, + sha256=hashlib.sha256(raw).hexdigest(), utf8_bytes=len(raw), + source_spans=sources, + )