Fix remote lifetime capture and bounded movement digests

Author: KSemenenko

README.md

@@ -96,7 +96,7 @@ The corrected [working database functional gates](docs/Features/DatabaseComposit
 
 The earlier resource-lifetime and analytics-root repair checkpoint passed its strict solution build, all **45 selected native whole-operation tests**, all **three selected process/native recovery tests**, and the actual SDK/MCP RF3 authorization scenario through all-three-node same-root cold restart. These are local development proofs. Earlier scenarios and failures retain their original source scope; whole-product and Linux qualification remain open.
 
-The later integrated native resource-lifetime command/read and controlled blob outcome source passes the full strict Release build with zero warnings/errors and the full canonical format verifier. Its focused native unit, recovery, RF3 and public functional qualification remains open; these source checks do not refresh the earlier runtime results.
+The later integrated native resource-lifetime command/read and controlled blob outcome source passes the full strict Release build with zero warnings/errors. Its latest selected native test run is **49 passed / 11 failed / 0 skipped**, improving from42/60 after the remote-document repair. Remaining failures cover distributed-search phase serialization, a scoped-budget test counter and a supporting movement fixture budget. Full formatting passed before these runtime repairs; current RF3 and public functional qualification remains open. These checks do not refresh earlier runtime results.
 
 The separate [all-model cold-reopen pair](docs/Features/DatabaseComposition/FunctionalDatabaseColdReopen.md) also passed on `57cea624`: **one node in 104.61 seconds, then RF3 in 118.73 seconds**. After all original nodes restarted through Aspire, both clients replayed the original SQL, mixed-model and blob-stage commands with complete receipt/value checks, then verified persisted grant revocation, repair and a healthy write. These tests keep the same physical roots and credentials. They use three cold-only identities and do not claim the normal gate's eight-user concurrency.
 

docs/implementation/status.json

@@ -2,36 +2,66 @@
   "RESOURCE-LIFETIME-NATIVE-20261016": {
     "title": "Integrated native resource lifetime command/read and controlled blob outcome checkpoint",
     "status": "in_progress",
-    "implementation": "Canonical native source integrated; full strict Release compiler/analyzers and canonical full formatter passed. Runtime qualification on this integrated source remains pending.",
+    "implementation": "Integrated source passes the full strict Release build; fresh focused native cohort improved from42/60 to49/60. Eleven failures in three groups remain and are being repaired; runtime/public qualification is open.",
     "scope": [
       "D1 canonical resource pairs and mandatory command coverage",
       "Native document/query/ANN source and target authority",
       "Effect-sensitive controlled blob OutcomeV2 and bounded native carriers",
       "Aspire-owned RF3 fixture image lifetime and exact17 local admission controls"
     ],
     "strictBuild": {
-      "label": "root-r544-test-bindings-final-release",
+      "label": "root-r545-runtime-repair-release",
       "result": "passed",
-      "elapsedSeconds": 364.02593612507917,
+      "elapsedSeconds": 748.201546916971,
       "errors": 0,
       "warnings": 0,
       "sourceDrift": 0,
-      "stdoutSha256": "dc48573d7b95d83fcddfe5c66332517e8fa29f369160d778e1ea45daae41156d",
-      "completionSha256": "39f680eefbe95342d8266d383cc3efc9b9c7679b8fc8e6cf89f09ac978a6509a"
+      "stdoutSha256": "d902546b398feacd6394651b76997e9782a17cfb7554643b3499f76f4b3aaf71",
+      "completionSha256": "d5436d455c3af73c5173fe975de17a3a70a5973d944920f6fbd6bc8c51c27936"
     },
     "fullFormatVerify": {
       "label": "root-r543-token-order-final-format-verify",
       "result": "passed",
       "elapsedSeconds": 430.19493220793083,
       "sourceDrift": 0,
       "compiledDrift": 0,
-      "completionSha256": "d5ad537e35db482c432e9a8f787dff0b4df518451451b39dfe70eefcaa7f81c4"
+      "completionSha256": "d5ad537e35db482c432e9a8f787dff0b4df518451451b39dfe70eefcaa7f81c4",
+      "scope": "Passed on the pre-runtime-repair source; no fresh full-format result is claimed for the later six-file runtime repair."
     },
-    "focusedUnit": "OPEN: selected60 is source prediction only; native TUnit TRX and functional coverage required.",
-    "processRecovery": "OPEN: three selected real process regressions require a fresh native run.",
+    "focusedUnit": "FAILED: fresh native TUnit selected60,49passed,11failed,0skipped; seven original remote-document failures repaired. Query phase serialization5, scoped-budget counter oracle4 and overconstrained supporting Move budget2 remain open.",
+    "processRecovery": "R538 passed3/3 before the later six-file runtime repair. Its original source identity remains authoritative; no fresh current-source recovery qualification is claimed.",
     "publicQualification": "OPEN: actual native RF3 and fresh matched one-node/three-node functional and cold flows required after product changes. Earlier e292 normal and57cea624 cold results retain their own recorded source scope.",
     "limits": "No full SQL, resource DROP/recreate, Linux, power-loss, endurance, performance or release completion claim. Original105 remain20done85in_progress.",
-    "next": "Run selected native unit and recovery flows, then actual SDK/MCP RF3 and matched functional/cold gates; D2 authoritative conservative resource owner inventory is separate prepared-only work until qualified adoption."
+    "next": "Complete the three confirmed remaining runtime groups as one repair batch, join independent native D2 source/tests and cold-user extension, then execute focused native/process and actual SDK/MCP ONE/RF3 functional and cold gates. Public DROP/recreate remains open.",
+    "previousStrictBuild": {
+      "label": "root-r544-test-bindings-final-release",
+      "result": "passed",
+      "elapsedSeconds": 364.02593612507917,
+      "errors": 0,
+      "warnings": 0,
+      "sourceDrift": 0,
+      "stdoutSha256": "dc48573d7b95d83fcddfe5c66332517e8fa29f369160d778e1ea45daae41156d",
+      "completionSha256": "39f680eefbe95342d8266d383cc3efc9b9c7679b8fc8e6cf89f09ac978a6509a"
+    },
+    "focusedUnitRun": {
+      "label": "root-r546-runtime-repair-unit",
+      "result": "failed",
+      "total": 60,
+      "passed": 49,
+      "failed": 11,
+      "skipped": 0,
+      "elapsedSeconds": 109.84697304177098,
+      "sourceDrift": 0,
+      "consumedImageDrift": 0,
+      "stdoutSha256": "2dc131115abb57d6c6cc4e41e0d55826d6ede1c7803fe1f673419806d6e006ea",
+      "predecessor": {
+        "label": "root-r537-final-lifetime-unit",
+        "passed": 42,
+        "failed": 18
+      },
+      "concurrency": "PredecessorR537 actual overlapping native TRX intervals50; configured native maximum remains50.",
+      "scope": "Full source and consumed image identities frozen; only unrelated nonconsumed IntegrationTests assembly was allowed to finish its separate compile."
+    }
   },
   "FUNCTIONAL-COLD-20261016": {
     "title": "Native all-model same-root cold-reopen functional pair",

src/KeyLoad.Core/Features/ClusterRouting/Queries/PartitionMoveFamilyCapture.cs

@@ -9,6 +9,8 @@ namespace KeyLoad.Core.Features.ClusterRouting.Queries;
 
 internal static class PartitionMoveFamilyCapture
 {
+    private const int ControlOwnedPageRecords = 1;
+
     internal static PartitionMoveImageFamily Capture(IKeyValueView view, PartitionMoveSourceFenceRecord fence,
         string family, ReadExecutionBudget work, PartitionMoveCaptureBudget captureBudget, DatabaseLimits limits, int maximumPageBytes,
         long maximumImageBytes, ImmutableArray<PartitionMoveImagePage>.Builder pages, ref long retained)
@@ -24,11 +26,13 @@ internal static PartitionMoveImageFamily Capture(IKeyValueView view, PartitionMo
         long count = PartitionMoveProtocol.EmptyCount;
         long rawBytes = PartitionMoveProtocol.EmptyCount;
         ReadOnlyMemory<byte> after = default;
+        var controlOwned = ControlOwned(family);
         while (true)
         {
             work.Check();
             var remaining = maximumImageBytes - retained;
-            var page = PartitionRecordPageReader.Read(view, fence.Partition, family, limits.MaxBatchMutations,
+            var page = PartitionRecordPageReader.Read(view, fence.Partition, family,
+                controlOwned ? ControlOwnedPageRecords : limits.MaxBatchMutations,
                 Math.Min(maximumPageBytes, remaining), limits.MaxQueryReadBytes, captureBudget.Observe,
                 after, work.Cancellation);
             foreach (var record in page.Records)
@@ -43,7 +47,7 @@ internal static PartitionMoveImageFamily Capture(IKeyValueView view, PartitionMo
                 count = checked(count + PartitionMoveProtocol.SequenceStep);
                 rawBytes = checked(rawBytes + record.Key.Length + record.Value.Length);
             }
-            if (!ControlOwned(family) && !page.Records.IsEmpty)
+            if (!controlOwned && !page.Records.IsEmpty)
             {
                 var captured = new PartitionMoveImagePage(PartitionMoveProtocol.Version, fence.MoveId,
                     fence.Partition, family, pages.Count, page.Records, PartitionMoveImageDigest.Records(page.Records));

src/KeyLoad.Core/Features/DocumentStorage/Queries/RemoteDocumentLifetimeReadAdmission.cs

@@ -66,7 +66,9 @@ internal void ValidateRemoteDocumentReadV2(RemoteDocumentReadFrameV2 original, R
         var directory = PhysicalOwnerDirectorySerialization.Read(view)
             ?? throw Errors.Fail(ErrorCode.OwnershipLost, RemoteDocumentFenceChanged);
         var destination = directory.Owners.Single(entry => entry.Owner.PhysicalShardId == placement.PhysicalShardId);
-        var resource = Resource(view, reference.Partition, reference.Collection, ResourceKind.Collection);
+        RequireNoUnpublishedPartitionMoveTarget(view, reference.Partition);
+        // This source cut supplies authorization metadata for the admitted remote data owner.
+        var resource = BindResourceDefinition(view, reference.Partition, reference.Collection, ResourceKind.Collection);
         var coverage = PartitionControlLifetimeCoverage.Capture(view, reference.Partition, [resource], Limits.MaxBatchBytes);
         var route = new RemoteDocumentReadFenceV1(principal.Id, principal.TenantId, principal.PolicyEpoch,
             placement, directory.Revision, destination);

tests/KeyLoad.UnitTests/Features/QueryExecution/Cases/QueryLifetimeNativeTests.cs

@@ -94,8 +94,12 @@ await QueryLifetimeNativeAssertions.UnchangedAsync(flow, async () =>
                 {
                     Coverage = null!
                 };
-                await QueryLifetimeNativeAssertions.RefusedAsync(() => _ = flow.Query(captured: missing), ErrorCode.OwnershipLost);
-                await QueryLifetimeNativeAssertions.RefusedAsync(() => _ = flow.Statistics(missing), ErrorCode.OwnershipLost);
+                await QueryLifetimeNativeAssertions.RefusedAsync(() => _ = flow.Query(captured: missing), ErrorCode.Corruption);
+                await QueryLifetimeNativeAssertions.RefusedAsync(() => _ = flow.Statistics(missing), ErrorCode.Corruption);
+                await QueryLifetimeNativeAssertions.RefusedAsync(() => _ = flow.Query(captured: missing, nativeRoundTrip: false),
+                    ErrorCode.OwnershipLost);
+                await QueryLifetimeNativeAssertions.RefusedAsync(() => _ = flow.Statistics(missing, nativeRoundTrip: false),
+                    ErrorCode.OwnershipLost);
                 var malformed = original with
                 {
                     Coverage = original.Coverage with

tests/KeyLoad.UnitTests/Features/QueryExecution/Helpers/QueryLifetimeNativeFlow.cs

@@ -80,7 +80,7 @@ internal ResourceLifetimeCoverageV1 CreateOtherLifetime()
     }
 
     internal PartitionQueryLeafResultV1 Query(string path = QueryLifetimeNativeProtocol.Star,
-        PartitionQueryLifetimeMetadata? captured = null, Predicate? filter = null)
+        PartitionQueryLifetimeMetadata? captured = null, Predicate? filter = null, bool nativeRoundTrip = true)
     {
         var metadata = captured ?? Capture();
         var database = fixture.Destination.Database;
@@ -94,14 +94,16 @@ internal PartitionQueryLeafResultV1 Query(string path = QueryLifetimeNativeProto
         var input = new PartitionQueryOwnedLeafRequestV2(new(plan,
             PhysicalOwnerDirectoryWholeFlow.Destination.Owner, RemoteDocumentNativeFixture.Tenant),
             metadata.Definition, metadata.Coverage, metadata.Principal);
-        var roundTrip = NativeSerialization.Deserialize<PartitionQueryOwnedLeafRequestV2>(NativeSerialization.Serialize(input));
+        var roundTrip = nativeRoundTrip
+            ? NativeSerialization.Deserialize<PartitionQueryOwnedLeafRequestV2>(NativeSerialization.Serialize(input)) : input;
         return PartitionQueryLifetimeLeafExecution.Execute(engine, RemoteDocumentNativeFixture.Reader, roundTrip,
             PhysicalOwnerDirectoryWholeFlow.Destination.Owner, database.EvaluationClock,
             database.EvaluationClock.GetUtcNow() + TimeSpan.FromSeconds(database.Limits.QueryDeadlineSeconds), CancellationToken.None);
     }
 
     internal DistributedSearchLeafResultV1 Statistics(PartitionQueryLifetimeMetadata? captured = null,
-        string field = RemoteDocumentNativeFixture.Title, string text = QueryLifetimeNativeProtocol.Text)
+        string field = RemoteDocumentNativeFixture.Title, string text = QueryLifetimeNativeProtocol.Text,
+        bool nativeRoundTrip = true)
     {
         var metadata = captured ?? Capture();
         var database = fixture.Destination.Database;
@@ -112,9 +114,10 @@ internal DistributedSearchLeafResultV1 Statistics(PartitionQueryLifetimeMetadata
             PhysicalOwnerDirectoryWholeFlow.Destination.Owner, RemoteDocumentNativeFixture.Tenant,
             database.Limits.MaxQueryReadBytes, database.Limits.MaxScanRecords,
             QueryResultBudgetPolicy.Resolve(database.Limits, UnitExecutionOptions.QueryExecution().Value)
-                / QueryLifetimeNativeProtocol.GrantDivisor, null, null, null, null, []);
+                / QueryLifetimeNativeProtocol.GrantDivisor, null, null, null, null, default);
         var input = new DistributedSearchOwnedLeafV2(original, metadata.Definition, metadata.Coverage, metadata.Principal);
-        var roundTrip = NativeSerialization.Deserialize<DistributedSearchOwnedLeafV2>(NativeSerialization.Serialize(input));
+        var roundTrip = nativeRoundTrip
+            ? NativeSerialization.Deserialize<DistributedSearchOwnedLeafV2>(NativeSerialization.Serialize(input)) : input;
         return DistributedSearchOwnedLeafExecution.ExecuteV2(database, RemoteDocumentNativeFixture.Reader,
             roundTrip, PhysicalOwnerDirectoryWholeFlow.Destination.Owner, UnitExecutionOptions.QueryExecution(),
             database.EvaluationClock, database.EvaluationClock.GetUtcNow() + TimeSpan.FromSeconds(database.Limits.QueryDeadlineSeconds),

tests/KeyLoad.UnitTests/Features/ResourceExecution/Assertions/ResourcePairBindingAssertions.cs

@@ -58,12 +58,12 @@ await EqualAsync(new DocumentResult(new(state.Source.Partition, Documents, value
             var page = database.ReadStream(Root, stream);
             await EqualAsync(new StreamHead(FirstRevision, FirstRevision, FirstRevision), page.Head);
             await EqualAsync(new EventRecord(stream, FirstRevision, value.Sequence,
-                new(value.Id, EventType, EventJson), value.Time), page.Events.Single());
+                new(value.Id, EventType, StoredEventJson), value.Time), page.Events.Single());
             await EqualAsync(new StreamPage(stream, new(FirstRevision, FirstRevision, FirstRevision),
-                [new(stream, FirstRevision, value.Sequence, new(value.Id, EventType, EventJson), value.Time)],
+                [new(stream, FirstRevision, value.Sequence, new(value.Id, EventType, StoredEventJson), value.Time)],
                 database.Store.Position, false, null, database.Store.Position), page);
             await EqualAsync(new MessageInspection(new(value.Id, MessageState.Ready, Zero, FirstRevision, value.Sequence, null, null),
-                QueueJson, Headers), database.InspectMessage(Root, new(state.Source.Partition, Queue), value.Id)!);
+                StoredQueueJson, Headers), database.InspectMessage(Root, new(state.Source.Partition, Queue), value.Id)!);
         }
         foreach (var row in state.Rows)
         {

tests/KeyLoad.UnitTests/Features/ResourceExecution/Contracts/ResourcePairBindingProtocol.cs

@@ -30,7 +30,9 @@ internal static class ResourcePairBindingProtocol
     internal const string LifetimeSpace = "resource-lifetime-v1";
     internal const string DocumentJson = """{ "amount": 17, "raw": "\u0061" }""";
     internal const string EventJson = """{"kind":"Created","amount":17}""";
+    internal const string StoredEventJson = """{"amount":17,"kind":"Created"}""";
     internal const string QueueJson = """{"job":"pair","amount":17}""";
+    internal const string StoredQueueJson = """{"amount":17,"job":"pair"}""";
     internal const string EventType = "Created";
     internal const string Headers = "{}";
     internal const string PutKind = "putDocument";