diff --git a/.agentworkforce/trajectories/completed/2026-08/traj_0c4k1s4nyzg3.trace.json b/.agentworkforce/trajectories/completed/2026-08/traj_0c4k1s4nyzg3.trace.json new file mode 100644 index 000000000..fbc3a761d --- /dev/null +++ b/.agentworkforce/trajectories/completed/2026-08/traj_0c4k1s4nyzg3.trace.json @@ -0,0 +1,32 @@ +{ + "version": "1.0.0", + "id": "99d12cd8-b320-49c4-8ea6-25391f5313eb", + "timestamp": "2026-08-25T07:27:52.373Z", + "trajectory": "traj_0c4k1s4nyzg3", + "files": [ + { + "path": "CHANGELOG.md", + "conversations": [{ "contributor": { "type": "ai" }, "ranges": [] }] + }, + { + "path": "crates/broker/src/fleet_wire.rs", + "conversations": [{ "contributor": { "type": "ai" }, "ranges": [] }] + }, + { + "path": "crates/broker/src/node_control.rs", + "conversations": [{ "contributor": { "type": "ai" }, "ranges": [] }] + }, + { + "path": "crates/broker/src/runtime/api.rs", + "conversations": [{ "contributor": { "type": "ai" }, "ranges": [] }] + }, + { + "path": "crates/broker/src/runtime/fleet.rs", + "conversations": [{ "contributor": { "type": "ai" }, "ranges": [] }] + }, + { + "path": "crates/broker/src/runtime/maintenance.rs", + "conversations": [{ "contributor": { "type": "ai" }, "ranges": [] }] + } + ] +} diff --git a/.agentworkforce/trajectories/completed/2026-08/traj_0c4k1s4nyzg3/summary.md b/.agentworkforce/trajectories/completed/2026-08/traj_0c4k1s4nyzg3/summary.md new file mode 100644 index 000000000..3ae89f314 --- /dev/null +++ b/.agentworkforce/trajectories/completed/2026-08/traj_0c4k1s4nyzg3/summary.md @@ -0,0 +1,33 @@ +# Trajectory: Fix relay #1602 authoritative live-worker inventory + +> **Status:** ✅ Completed +> **Task:** AgentWorkforce/relay#1602 +> **Confidence:** 93% +> **Started:** August 25, 2026 at 08:48 AM +> **Completed:** August 25, 2026 at 09:27 AM + +--- + +## Summary + +Unified fleet heartbeat live-agent names with the authoritative reconnect inventory, added divergence and reconnect regressions, and validated broker/TypeScript behavior. + +**Approach:** Standard approach + +--- + +## Key Decisions + +### Use retained fleet_inventory as the sole broker-side identity/name set for inventory.sync and heartbeat live-agent capability +- **Chose:** Use retained fleet_inventory as the sole broker-side identity/name set for inventory.sync and heartbeat live-agent capability +- **Reasoning:** inventory carries authoritative immutable agent IDs, is populated only after registration or audited read-only reconciliation of live PTYs, survives node-control reconnects, and preserves multi-node claim arbitration; deriving heartbeat names from every worker-registry key is the divergent unauthorised projection that caused #1602 + +--- + +## Chapters + +### 1. Work +*Agent: default* + +- Use retained fleet_inventory as the sole broker-side identity/name set for inventory.sync and heartbeat live-agent capability: Use retained fleet_inventory as the sole broker-side identity/name set for inventory.sync and heartbeat live-agent capability +- Issue 1602 regression now proves heartbeat authorization names and reconnect inventory derive from one retained live-worker vector. Existing reconciliation remains responsible for adopting live PTYs without registration or token rotation; immutable-id/name mismatch guards preserve multi-node claim semantics. diff --git a/.agentworkforce/trajectories/completed/2026-08/traj_0c4k1s4nyzg3/trajectory.json b/.agentworkforce/trajectories/completed/2026-08/traj_0c4k1s4nyzg3/trajectory.json new file mode 100644 index 000000000..2798d0642 --- /dev/null +++ b/.agentworkforce/trajectories/completed/2026-08/traj_0c4k1s4nyzg3/trajectory.json @@ -0,0 +1,87 @@ +{ + "id": "traj_0c4k1s4nyzg3", + "version": 1, + "task": { + "title": "Fix relay #1602 authoritative live-worker inventory", + "source": { + "system": "plain", + "id": "AgentWorkforce/relay#1602" + } + }, + "status": "completed", + "startedAt": "2026-08-25T06:48:55.958Z", + "completedAt": "2026-08-25T07:27:52.373Z", + "agents": [ + { + "name": "default", + "role": "lead", + "joinedAt": "2026-08-25T06:57:16.671Z" + } + ], + "chapters": [ + { + "id": "chap_5lb3egoeu1rr", + "title": "Work", + "agentName": "default", + "startedAt": "2026-08-25T06:57:16.671Z", + "endedAt": "2026-08-25T07:27:52.373Z", + "events": [ + { + "ts": 1787641036674, + "type": "decision", + "content": "Use retained fleet_inventory as the sole broker-side identity/name set for inventory.sync and heartbeat live-agent capability: Use retained fleet_inventory as the sole broker-side identity/name set for inventory.sync and heartbeat live-agent capability", + "raw": { + "question": "Use retained fleet_inventory as the sole broker-side identity/name set for inventory.sync and heartbeat live-agent capability", + "chosen": "Use retained fleet_inventory as the sole broker-side identity/name set for inventory.sync and heartbeat live-agent capability", + "alternatives": [], + "reasoning": "inventory carries authoritative immutable agent IDs, is populated only after registration or audited read-only reconciliation of live PTYs, survives node-control reconnects, and preserves multi-node claim arbitration; deriving heartbeat names from every worker-registry key is the divergent unauthorised projection that caused #1602" + }, + "significance": "high" + }, + { + "ts": 1787642870261, + "type": "reflection", + "content": "Issue 1602 regression now proves heartbeat authorization names and reconnect inventory derive from one retained live-worker vector. Existing reconciliation remains responsible for adopting live PTYs without registration or token rotation; immutable-id/name mismatch guards preserve multi-node claim semantics.", + "raw": { + "focalPoints": [ + "authoritative-inventory", + "reconnects", + "identity-safety" + ], + "confidence": 0.93 + }, + "significance": "high", + "tags": [ + "focal:authoritative-inventory", + "focal:reconnects", + "focal:identity-safety", + "confidence:0.93" + ] + } + ] + } + ], + "retrospective": { + "summary": "Unified fleet heartbeat live-agent names with the authoritative reconnect inventory, added divergence and reconnect regressions, and validated broker/TypeScript behavior.", + "approach": "Standard approach", + "confidence": 0.93 + }, + "commits": [ + "fee5ee2d65e0b4a9a4f43f619c90c587e9440fcd" + ], + "filesChanged": [ + "CHANGELOG.md", + "crates/broker/src/fleet_wire.rs", + "crates/broker/src/node_control.rs", + "crates/broker/src/runtime/api.rs", + "crates/broker/src/runtime/fleet.rs", + "crates/broker/src/runtime/maintenance.rs" + ], + "projectId": "AgentWorkforce/relay", + "tags": [], + "_trace": { + "startRef": "191e5f14a343431ec42282c1103c9f014854ac0b", + "endRef": "fee5ee2d65e0b4a9a4f43f619c90c587e9440fcd", + "traceId": "99d12cd8-b320-49c4-8ea6-25391f5313eb" + } +} diff --git a/.agentworkforce/trajectories/completed/2026-08/traj_5ellsvzejuxq.trace.json b/.agentworkforce/trajectories/completed/2026-08/traj_5ellsvzejuxq.trace.json new file mode 100644 index 000000000..6779ea168 --- /dev/null +++ b/.agentworkforce/trajectories/completed/2026-08/traj_5ellsvzejuxq.trace.json @@ -0,0 +1,20 @@ +{ + "version": "1.0.0", + "id": "60d228aa-be88-43f6-b386-88de5bd5d1ef", + "timestamp": "2026-08-25T08:03:24.303Z", + "trajectory": "traj_5ellsvzejuxq", + "files": [ + { + "path": "crates/broker/src/node_control.rs", + "conversations": [{ "contributor": { "type": "ai" }, "ranges": [] }] + }, + { + "path": "crates/broker/src/runtime/fleet.rs", + "conversations": [{ "contributor": { "type": "ai" }, "ranges": [] }] + }, + { + "path": "crates/broker/src/worker.rs", + "conversations": [{ "contributor": { "type": "ai" }, "ranges": [] }] + } + ] +} diff --git a/.agentworkforce/trajectories/completed/2026-08/traj_5ellsvzejuxq/summary.md b/.agentworkforce/trajectories/completed/2026-08/traj_5ellsvzejuxq/summary.md new file mode 100644 index 000000000..c7edcd2e9 --- /dev/null +++ b/.agentworkforce/trajectories/completed/2026-08/traj_5ellsvzejuxq/summary.md @@ -0,0 +1,32 @@ +# Trajectory: Repair parentless adopted PTYs for relay #1602 + +> **Status:** ✅ Completed +> **Task:** AgentWorkforce/relay#1602 review +> **Confidence:** 97% +> **Started:** August 25, 2026 at 10:03 AM +> **Completed:** August 25, 2026 at 10:03 AM + +--- + +## Summary + +Included parentless live adopted PTYs in identity-safe reconciliation and proved the real WorkerRegistry-to-reconnect path end to end. + +**Approach:** Standard approach + +--- + +## Key Decisions + +### Do not use the transient parent marker as roster eligibility +- **Chose:** Do not use the transient parent marker as roster eligibility +- **Reasoning:** The #1555 reconciler shipped before the incident but excluded parentless live handles before identity lookup. Adopted/migrated PTYs can lose that spawn metadata; the existing read-only workspace-scoped lookup, exact-name check, and immutable-ID guard are the safe eligibility boundary. + +--- + +## Chapters + +### 1. Work +*Agent: default* + +- Do not use the transient parent marker as roster eligibility: Do not use the transient parent marker as roster eligibility diff --git a/.agentworkforce/trajectories/completed/2026-08/traj_5ellsvzejuxq/trajectory.json b/.agentworkforce/trajectories/completed/2026-08/traj_5ellsvzejuxq/trajectory.json new file mode 100644 index 000000000..e37cc1b2c --- /dev/null +++ b/.agentworkforce/trajectories/completed/2026-08/traj_5ellsvzejuxq/trajectory.json @@ -0,0 +1,64 @@ +{ + "id": "traj_5ellsvzejuxq", + "version": 1, + "task": { + "title": "Repair parentless adopted PTYs for relay #1602", + "source": { + "system": "plain", + "id": "AgentWorkforce/relay#1602 review" + } + }, + "status": "completed", + "startedAt": "2026-08-25T08:03:20.298Z", + "completedAt": "2026-08-25T08:03:24.303Z", + "agents": [ + { + "name": "default", + "role": "lead", + "joinedAt": "2026-08-25T08:03:22.360Z" + } + ], + "chapters": [ + { + "id": "chap_127rww64racw", + "title": "Work", + "agentName": "default", + "startedAt": "2026-08-25T08:03:22.360Z", + "endedAt": "2026-08-25T08:03:24.303Z", + "events": [ + { + "ts": 1787645002362, + "type": "decision", + "content": "Do not use the transient parent marker as roster eligibility: Do not use the transient parent marker as roster eligibility", + "raw": { + "question": "Do not use the transient parent marker as roster eligibility", + "chosen": "Do not use the transient parent marker as roster eligibility", + "alternatives": [], + "reasoning": "The #1555 reconciler shipped before the incident but excluded parentless live handles before identity lookup. Adopted/migrated PTYs can lose that spawn metadata; the existing read-only workspace-scoped lookup, exact-name check, and immutable-ID guard are the safe eligibility boundary." + }, + "significance": "high" + } + ] + } + ], + "retrospective": { + "summary": "Included parentless live adopted PTYs in identity-safe reconciliation and proved the real WorkerRegistry-to-reconnect path end to end.", + "approach": "Standard approach", + "confidence": 0.97 + }, + "commits": [ + "75f92f4474dc621a7b8e461e121c24605c40bccc" + ], + "filesChanged": [ + "crates/broker/src/node_control.rs", + "crates/broker/src/runtime/fleet.rs", + "crates/broker/src/worker.rs" + ], + "projectId": "AgentWorkforce/relay", + "tags": [], + "_trace": { + "startRef": "4a695b26e9022756f1729be31b929fa61535b0a4", + "endRef": "75f92f4474dc621a7b8e461e121c24605c40bccc", + "traceId": "60d228aa-be88-43f6-b386-88de5bd5d1ef" + } +} diff --git a/.agentworkforce/trajectories/completed/2026-08/traj_bd1m78v7jn8i.trace.json b/.agentworkforce/trajectories/completed/2026-08/traj_bd1m78v7jn8i.trace.json new file mode 100644 index 000000000..2853d136b --- /dev/null +++ b/.agentworkforce/trajectories/completed/2026-08/traj_bd1m78v7jn8i.trace.json @@ -0,0 +1,25 @@ +{ + "version": "1.0.0", + "id": "c8b672d1-2524-4908-9b2d-04666fc8da44", + "timestamp": "2026-08-25T20:50:56.814Z", + "trajectory": "traj_bd1m78v7jn8i", + "files": [ + { + "path": ".agentworkforce/trajectories/completed/2026-08/traj_dmqicqrgy5c1/trajectory.json", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 64, + "end_line": 71, + "revision": "6a8cb6c9a7ff5966ccc7a1b8a3452f62eb6c7a0e" + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/.agentworkforce/trajectories/completed/2026-08/traj_bd1m78v7jn8i/summary.md b/.agentworkforce/trajectories/completed/2026-08/traj_bd1m78v7jn8i/summary.md new file mode 100644 index 000000000..35dfa8d78 --- /dev/null +++ b/.agentworkforce/trajectories/completed/2026-08/traj_bd1m78v7jn8i/summary.md @@ -0,0 +1,39 @@ +# Trajectory: Repair PR 1611 Trail ownership metadata + +> **Status:** ✅ Completed +> **Task:** #1611 +> **Confidence:** 99% +> **Started:** August 25, 2026 at 10:49 PM +> **Completed:** August 25, 2026 at 10:50 PM + +--- + +## Summary + +Verified Cubic's trajectory-ownership finding, regenerated traj_dmqicqrgy5c1 through Trail's validated storage renderer so its filesChanged list names its own outputs instead of a sibling trajectory, and validated the generated record. + +**Approach:** Standard approach + +--- + +## Key Decisions + +### Use Trail's TrajectoryClient/FileStorage renderer for the ownership repair +- **Chose:** Use Trail's TrajectoryClient/FileStorage renderer for the ownership repair +- **Reasoning:** The completed artifact is generated output. Preserve its task, events, trace reference, commits, and retrospective; replace only the two sibling trajectory ownership paths with traj_dmqicqrgy5c1's corresponding renderer-owned summary.md and trajectory.json paths, then let Trail validate and rewrite both generated outputs. + +--- + +## Chapters + +### 1. Initial work +*Agent: relay-1611-finish-direct-0825* + +- Use Trail's TrajectoryClient/FileStorage renderer for the ownership repair: Use Trail's TrajectoryClient/FileStorage renderer for the ownership repair + +--- + +## Artifacts + +**Commits:** 6a8cb6c9a +**Files changed:** 1 diff --git a/.agentworkforce/trajectories/completed/2026-08/traj_bd1m78v7jn8i/trajectory.json b/.agentworkforce/trajectories/completed/2026-08/traj_bd1m78v7jn8i/trajectory.json new file mode 100644 index 000000000..e63b44aab --- /dev/null +++ b/.agentworkforce/trajectories/completed/2026-08/traj_bd1m78v7jn8i/trajectory.json @@ -0,0 +1,62 @@ +{ + "id": "traj_bd1m78v7jn8i", + "version": 1, + "task": { + "title": "Repair PR 1611 Trail ownership metadata", + "source": { + "system": "plain", + "id": "#1611" + } + }, + "status": "completed", + "startedAt": "2026-08-25T20:49:26.982Z", + "completedAt": "2026-08-25T20:50:56.501Z", + "agents": [ + { + "name": "relay-1611-finish-direct-0825", + "role": "lead", + "joinedAt": "2026-08-25T20:49:27.092Z" + } + ], + "chapters": [ + { + "id": "chap_gap43hed57dk", + "title": "Initial work", + "agentName": "relay-1611-finish-direct-0825", + "startedAt": "2026-08-25T20:49:27.092Z", + "endedAt": "2026-08-25T20:50:56.501Z", + "events": [ + { + "ts": 1787690998511, + "type": "decision", + "content": "Use Trail's TrajectoryClient/FileStorage renderer for the ownership repair: Use Trail's TrajectoryClient/FileStorage renderer for the ownership repair", + "raw": { + "question": "Use Trail's TrajectoryClient/FileStorage renderer for the ownership repair", + "chosen": "Use Trail's TrajectoryClient/FileStorage renderer for the ownership repair", + "alternatives": [], + "reasoning": "The completed artifact is generated output. Preserve its task, events, trace reference, commits, and retrospective; replace only the two sibling trajectory ownership paths with traj_dmqicqrgy5c1's corresponding renderer-owned summary.md and trajectory.json paths, then let Trail validate and rewrite both generated outputs." + }, + "significance": "high" + } + ] + } + ], + "retrospective": { + "summary": "Verified Cubic's trajectory-ownership finding, regenerated traj_dmqicqrgy5c1 through Trail's validated storage renderer so its filesChanged list names its own outputs instead of a sibling trajectory, and validated the generated record.", + "approach": "Standard approach", + "confidence": 0.99 + }, + "commits": [ + "6a8cb6c9a" + ], + "filesChanged": [ + ".agentworkforce/trajectories/completed/2026-08/traj_dmqicqrgy5c1/trajectory.json" + ], + "projectId": "AgentWorkforce/relay", + "tags": [], + "_trace": { + "startRef": "0be89c3d513fc3f5eea115665513cd10072988b0", + "endRef": "6a8cb6c9a7ff5966ccc7a1b8a3452f62eb6c7a0e", + "traceId": "c8b672d1-2524-4908-9b2d-04666fc8da44" + } +} \ No newline at end of file diff --git a/.agentworkforce/trajectories/completed/2026-08/traj_dmqicqrgy5c1.trace.json b/.agentworkforce/trajectories/completed/2026-08/traj_dmqicqrgy5c1.trace.json new file mode 100644 index 000000000..0d11544e9 --- /dev/null +++ b/.agentworkforce/trajectories/completed/2026-08/traj_dmqicqrgy5c1.trace.json @@ -0,0 +1,516 @@ +{ + "version": "1.0.0", + "id": "5e19ca7a-8ee1-4fbd-84cc-170bf89080a2", + "timestamp": "2026-08-25T19:56:08.566Z", + "trajectory": "traj_dmqicqrgy5c1", + "files": [ + { + "path": ".agentworkforce/trajectories/completed/2026-08/traj_yr3f5u3r6zdz/summary.md", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 54, + "revision": "d0b5207386a9eb4068befdc3d7bb3bd98da9694d" + } + ] + } + ] + }, + { + "path": ".agentworkforce/trajectories/completed/2026-08/traj_yr3f5u3r6zdz/trajectory.json", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 128, + "revision": "d0b5207386a9eb4068befdc3d7bb3bd98da9694d" + } + ] + } + ] + }, + { + "path": ".github/pull_request_template.md", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 7, + "end_line": 21, + "revision": "d0b5207386a9eb4068befdc3d7bb3bd98da9694d" + } + ] + } + ] + }, + { + "path": ".github/workflows/relayflow-pr-proof.yml", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 113, + "revision": "d0b5207386a9eb4068befdc3d7bb3bd98da9694d" + } + ] + } + ] + }, + { + "path": ".gitignore", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 81, + "end_line": 87, + "revision": "d0b5207386a9eb4068befdc3d7bb3bd98da9694d" + }, + { + "start_line": 92, + "end_line": 98, + "revision": "d0b5207386a9eb4068befdc3d7bb3bd98da9694d" + } + ] + } + ] + }, + { + "path": "packages/cloud/src/api-client.test.ts", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 41, + "revision": "d0b5207386a9eb4068befdc3d7bb3bd98da9694d" + } + ] + } + ] + }, + { + "path": "packages/cloud/src/api-client.ts", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 42, + "end_line": 93, + "revision": "d0b5207386a9eb4068befdc3d7bb3bd98da9694d" + }, + { + "start_line": 288, + "end_line": 294, + "revision": "d0b5207386a9eb4068befdc3d7bb3bd98da9694d" + } + ] + } + ] + }, + { + "path": "packages/cloud/src/auth.test.ts", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 219, + "end_line": 243, + "revision": "d0b5207386a9eb4068befdc3d7bb3bd98da9694d" + } + ] + } + ] + }, + { + "path": "packages/cloud/src/auth.ts", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 825, + "end_line": 840, + "revision": "d0b5207386a9eb4068befdc3d7bb3bd98da9694d" + } + ] + } + ] + }, + { + "path": "packages/cloud/src/workflows.test.ts", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 9, + "end_line": 19, + "revision": "d0b5207386a9eb4068befdc3d7bb3bd98da9694d" + }, + { + "start_line": 517, + "end_line": 553, + "revision": "d0b5207386a9eb4068befdc3d7bb3bd98da9694d" + }, + { + "start_line": 797, + "end_line": 822, + "revision": "d0b5207386a9eb4068befdc3d7bb3bd98da9694d" + } + ] + } + ] + }, + { + "path": "packages/cloud/src/workflows.ts", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 7, + "end_line": 13, + "revision": "d0b5207386a9eb4068befdc3d7bb3bd98da9694d" + }, + { + "start_line": 59, + "end_line": 66, + "revision": "d0b5207386a9eb4068befdc3d7bb3bd98da9694d" + }, + { + "start_line": 241, + "end_line": 247, + "revision": "d0b5207386a9eb4068befdc3d7bb3bd98da9694d" + }, + { + "start_line": 272, + "end_line": 281, + "revision": "d0b5207386a9eb4068befdc3d7bb3bd98da9694d" + }, + { + "start_line": 287, + "end_line": 312, + "revision": "d0b5207386a9eb4068befdc3d7bb3bd98da9694d" + }, + { + "start_line": 410, + "end_line": 416, + "revision": "d0b5207386a9eb4068befdc3d7bb3bd98da9694d" + }, + { + "start_line": 418, + "end_line": 423, + "revision": "d0b5207386a9eb4068befdc3d7bb3bd98da9694d" + }, + { + "start_line": 449, + "end_line": 455, + "revision": "d0b5207386a9eb4068befdc3d7bb3bd98da9694d" + }, + { + "start_line": 485, + "end_line": 491, + "revision": "d0b5207386a9eb4068befdc3d7bb3bd98da9694d" + }, + { + "start_line": 508, + "end_line": 515, + "revision": "d0b5207386a9eb4068befdc3d7bb3bd98da9694d" + }, + { + "start_line": 534, + "end_line": 541, + "revision": "d0b5207386a9eb4068befdc3d7bb3bd98da9694d" + }, + { + "start_line": 556, + "end_line": 566, + "revision": "d0b5207386a9eb4068befdc3d7bb3bd98da9694d" + }, + { + "start_line": 583, + "end_line": 589, + "revision": "d0b5207386a9eb4068befdc3d7bb3bd98da9694d" + }, + { + "start_line": 594, + "end_line": 600, + "revision": "d0b5207386a9eb4068befdc3d7bb3bd98da9694d" + }, + { + "start_line": 622, + "end_line": 633, + "revision": "d0b5207386a9eb4068befdc3d7bb3bd98da9694d" + }, + { + "start_line": 640, + "end_line": 648, + "revision": "d0b5207386a9eb4068befdc3d7bb3bd98da9694d" + }, + { + "start_line": 666, + "end_line": 690, + "revision": "d0b5207386a9eb4068befdc3d7bb3bd98da9694d" + } + ] + } + ] + }, + { + "path": "scripts/pr-proof/cloud-storage.mjs", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 101, + "revision": "d0b5207386a9eb4068befdc3d7bb3bd98da9694d" + } + ] + } + ] + }, + { + "path": "scripts/pr-proof/contract.mjs", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 326, + "revision": "d0b5207386a9eb4068befdc3d7bb3bd98da9694d" + } + ] + } + ] + }, + { + "path": "scripts/pr-proof/prepare.mjs", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 253, + "revision": "d0b5207386a9eb4068befdc3d7bb3bd98da9694d" + } + ] + } + ] + }, + { + "path": "scripts/pr-proof/process-runner.mjs", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 198, + "revision": "d0b5207386a9eb4068befdc3d7bb3bd98da9694d" + } + ] + } + ] + }, + { + "path": "scripts/pr-proof/report-status.mjs", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 199, + "revision": "d0b5207386a9eb4068befdc3d7bb3bd98da9694d" + } + ] + } + ] + }, + { + "path": "scripts/pr-proof/run-arm.mjs", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 210, + "revision": "d0b5207386a9eb4068befdc3d7bb3bd98da9694d" + } + ] + } + ] + }, + { + "path": "scripts/pr-proof/run-cloud.mjs", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 310, + "revision": "d0b5207386a9eb4068befdc3d7bb3bd98da9694d" + } + ] + } + ] + }, + { + "path": "scripts/pr-proof/verify-evidence.mjs", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 104, + "revision": "d0b5207386a9eb4068befdc3d7bb3bd98da9694d" + } + ] + } + ] + }, + { + "path": "tests/fixtures/pr-proof-contract.test.ts", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 753, + "revision": "d0b5207386a9eb4068befdc3d7bb3bd98da9694d" + } + ] + } + ] + }, + { + "path": "tests/relayflows/cases/1602-parentless-worker-inventory/case.json", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 20, + "revision": "d0b5207386a9eb4068befdc3d7bb3bd98da9694d" + } + ] + } + ] + }, + { + "path": "tests/relayflows/cases/1602-parentless-worker-inventory/run.mjs", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 485, + "revision": "d0b5207386a9eb4068befdc3d7bb3bd98da9694d" + } + ] + } + ] + }, + { + "path": "tests/relayflows/cases/README.md", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 147, + "revision": "d0b5207386a9eb4068befdc3d7bb3bd98da9694d" + } + ] + } + ] + }, + { + "path": "workflows/pr-proof.ts", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 88, + "revision": "d0b5207386a9eb4068befdc3d7bb3bd98da9694d" + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/.agentworkforce/trajectories/completed/2026-08/traj_dmqicqrgy5c1/summary.md b/.agentworkforce/trajectories/completed/2026-08/traj_dmqicqrgy5c1/summary.md new file mode 100644 index 000000000..140b2cc00 --- /dev/null +++ b/.agentworkforce/trajectories/completed/2026-08/traj_dmqicqrgy5c1/summary.md @@ -0,0 +1,44 @@ +# Trajectory: Finish PR 1611 with parentless worker RelayFlow proof + +> **Status:** ✅ Completed +> **Task:** #1611 +> **Confidence:** 98% +> **Started:** August 25, 2026 at 08:43 PM +> **Completed:** August 25, 2026 at 09:56 PM + +--- + +## Summary + +Merged RelayFlow infrastructure from main; added an isolated base/head proof for issue 1602 that exercises the production WorkerRegistry, fleet reconciliation, and node-control reconnect paths; hardened the landed proof-contract cleanup test; verified exact bug/fixed signatures plus full broker, clippy, formatting, TypeScript, and typecheck gates. + +**Approach:** Standard approach + +--- + +## Key Decisions + +### Merge origin/main into the existing PR branch instead of rebasing +- **Chose:** Merge origin/main into the existing PR branch instead of rebasing +- **Reasoning:** The landed proof contract must exist in the head checkout for local and Cloud execution; a merge preserves the existing reviewed PR history and avoids force-pushing or overwriting newer work. + +### Harden the landed process-tree test startup budget from 100 ms to 500 ms +- **Chose:** Harden the landed process-tree test startup budget from 100 ms to 500 ms +- **Reasoning:** The exact test failed three times because the spawned Node process was killed before it could print its child PID; the assertion is about process-group cleanup, not sub-100-ms startup. A 500-ms trigger preserves the semantic check and removes the scheduling race exposed on this host. + +--- + +## Chapters + +### 1. Work +*Agent: default* + +- Merge origin/main into the existing PR branch instead of rebasing: Merge origin/main into the existing PR branch instead of rebasing +- Harden the landed process-tree test startup budget from 100 ms to 500 ms: Harden the landed process-tree test startup budget from 100 ms to 500 ms + +--- + +## Artifacts + +**Commits:** d0b520738, bd0407c5d, 639158c8c +**Files changed:** 24 diff --git a/.agentworkforce/trajectories/completed/2026-08/traj_dmqicqrgy5c1/trajectory.json b/.agentworkforce/trajectories/completed/2026-08/traj_dmqicqrgy5c1/trajectory.json new file mode 100644 index 000000000..59519fa96 --- /dev/null +++ b/.agentworkforce/trajectories/completed/2026-08/traj_dmqicqrgy5c1/trajectory.json @@ -0,0 +1,99 @@ +{ + "id": "traj_dmqicqrgy5c1", + "version": 1, + "task": { + "title": "Finish PR 1611 with parentless worker RelayFlow proof", + "source": { + "system": "plain", + "id": "#1611" + } + }, + "status": "completed", + "startedAt": "2026-08-25T18:43:19.160Z", + "completedAt": "2026-08-25T19:56:08.227Z", + "agents": [ + { + "name": "default", + "role": "lead", + "joinedAt": "2026-08-25T18:43:50.155Z" + } + ], + "chapters": [ + { + "id": "chap_omlenooq0im3", + "title": "Work", + "agentName": "default", + "startedAt": "2026-08-25T18:43:50.155Z", + "endedAt": "2026-08-25T19:56:08.227Z", + "events": [ + { + "ts": 1787683430157, + "type": "decision", + "content": "Merge origin/main into the existing PR branch instead of rebasing: Merge origin/main into the existing PR branch instead of rebasing", + "raw": { + "question": "Merge origin/main into the existing PR branch instead of rebasing", + "chosen": "Merge origin/main into the existing PR branch instead of rebasing", + "alternatives": [], + "reasoning": "The landed proof contract must exist in the head checkout for local and Cloud execution; a merge preserves the existing reviewed PR history and avoids force-pushing or overwriting newer work." + }, + "significance": "high" + }, + { + "ts": 1787686357227, + "type": "decision", + "content": "Harden the landed process-tree test startup budget from 100 ms to 500 ms: Harden the landed process-tree test startup budget from 100 ms to 500 ms", + "raw": { + "question": "Harden the landed process-tree test startup budget from 100 ms to 500 ms", + "chosen": "Harden the landed process-tree test startup budget from 100 ms to 500 ms", + "alternatives": [], + "reasoning": "The exact test failed three times because the spawned Node process was killed before it could print its child PID; the assertion is about process-group cleanup, not sub-100-ms startup. A 500-ms trigger preserves the semantic check and removes the scheduling race exposed on this host." + }, + "significance": "high" + } + ] + } + ], + "retrospective": { + "summary": "Merged RelayFlow infrastructure from main; added an isolated base/head proof for issue 1602 that exercises the production WorkerRegistry, fleet reconciliation, and node-control reconnect paths; hardened the landed proof-contract cleanup test; verified exact bug/fixed signatures plus full broker, clippy, formatting, TypeScript, and typecheck gates.", + "approach": "Standard approach", + "confidence": 0.98 + }, + "commits": [ + "d0b520738", + "bd0407c5d", + "639158c8c" + ], + "filesChanged": [ + ".agentworkforce/trajectories/completed/2026-08/traj_dmqicqrgy5c1/summary.md", + ".agentworkforce/trajectories/completed/2026-08/traj_dmqicqrgy5c1/trajectory.json", + ".github/pull_request_template.md", + ".github/workflows/relayflow-pr-proof.yml", + ".gitignore", + "packages/cloud/src/api-client.test.ts", + "packages/cloud/src/api-client.ts", + "packages/cloud/src/auth.test.ts", + "packages/cloud/src/auth.ts", + "packages/cloud/src/workflows.test.ts", + "packages/cloud/src/workflows.ts", + "scripts/pr-proof/cloud-storage.mjs", + "scripts/pr-proof/contract.mjs", + "scripts/pr-proof/prepare.mjs", + "scripts/pr-proof/process-runner.mjs", + "scripts/pr-proof/report-status.mjs", + "scripts/pr-proof/run-arm.mjs", + "scripts/pr-proof/run-cloud.mjs", + "scripts/pr-proof/verify-evidence.mjs", + "tests/fixtures/pr-proof-contract.test.ts", + "tests/relayflows/cases/1602-parentless-worker-inventory/case.json", + "tests/relayflows/cases/1602-parentless-worker-inventory/run.mjs", + "tests/relayflows/cases/README.md", + "workflows/pr-proof.ts" + ], + "projectId": "AgentWorkforce/relay", + "tags": [], + "_trace": { + "startRef": "50eec9efc1f09ceee8ebb8c62f833e7004b19cf9", + "endRef": "d0b5207386a9eb4068befdc3d7bb3bd98da9694d", + "traceId": "5e19ca7a-8ee1-4fbd-84cc-170bf89080a2" + } +} \ No newline at end of file diff --git a/.agentworkforce/trajectories/completed/2026-08/traj_r79zmxc4sagy.trace.json b/.agentworkforce/trajectories/completed/2026-08/traj_r79zmxc4sagy.trace.json new file mode 100644 index 000000000..0e51bd346 --- /dev/null +++ b/.agentworkforce/trajectories/completed/2026-08/traj_r79zmxc4sagy.trace.json @@ -0,0 +1,127 @@ +{ + "version": "1.0.0", + "id": "6b089b86-9d34-4b0a-9786-31a4242fd7b8", + "timestamp": "2026-08-25T12:51:26.496Z", + "trajectory": "traj_r79zmxc4sagy", + "files": [ + { + "path": ".agentworkforce/trajectories/completed/2026-08/traj_0c4k1s4nyzg3.trace.json", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 32, + "revision": "9f907408fede5a2315b6662739f62c6e5e32e30a" + } + ] + } + ] + }, + { + "path": ".agentworkforce/trajectories/completed/2026-08/traj_0c4k1s4nyzg3/trajectory.json", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 66, + "end_line": 87, + "revision": "9f907408fede5a2315b6662739f62c6e5e32e30a" + } + ] + } + ] + }, + { + "path": ".agentworkforce/trajectories/completed/2026-08/traj_5ellsvzejuxq.trace.json", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 20, + "revision": "9f907408fede5a2315b6662739f62c6e5e32e30a" + } + ] + } + ] + }, + { + "path": ".agentworkforce/trajectories/completed/2026-08/traj_5ellsvzejuxq/trajectory.json", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 46, + "end_line": 64, + "revision": "9f907408fede5a2315b6662739f62c6e5e32e30a" + } + ] + } + ] + }, + { + "path": ".agentworkforce/trajectories/completed/2026-08/traj_zotcff3t4aap.trace.json", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 20, + "revision": "9f907408fede5a2315b6662739f62c6e5e32e30a" + } + ] + } + ] + }, + { + "path": ".agentworkforce/trajectories/completed/2026-08/traj_zotcff3t4aap/trajectory.json", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 46, + "end_line": 64, + "revision": "9f907408fede5a2315b6662739f62c6e5e32e30a" + } + ] + } + ] + }, + { + "path": "crates/broker/src/runtime/fleet.rs", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 3459, + "end_line": 3465, + "revision": "9f907408fede5a2315b6662739f62c6e5e32e30a" + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/.agentworkforce/trajectories/completed/2026-08/traj_r79zmxc4sagy/summary.md b/.agentworkforce/trajectories/completed/2026-08/traj_r79zmxc4sagy/summary.md new file mode 100644 index 000000000..182438b8b --- /dev/null +++ b/.agentworkforce/trajectories/completed/2026-08/traj_r79zmxc4sagy/summary.md @@ -0,0 +1,39 @@ +# Trajectory: Finish relay PR #1611 review and merge-readiness + +> **Status:** ✅ Completed +> **Task:** AgentWorkforce/relay#1602 PR #1611 +> **Confidence:** 96% +> **Started:** August 25, 2026 at 02:38 PM +> **Completed:** August 25, 2026 at 02:51 PM + +--- + +## Summary + +Audited PR #1611 production invariants, Unix-gated its Unix child fixture, and repaired incomplete trajectory commit/file/trace metadata. + +**Approach:** Exact-head GitHub review reconciliation followed by minimal review-backed edits and focused regression validation. + +--- + +## Key Decisions + +### Keep the production repair unchanged and close only verified review gaps +- **Chose:** Keep the production repair unchanged and close only verified review gaps +- **Reasoning:** The exact-head audit confirms the live parentless registry path, workspace-key read-only identity lookup, exact-name and immutable-ID guards, zero registration/token rotation, and initial plus reconnect projections already satisfy #1602; the only valid remaining code defect is the Unix-only child fixture, while #1612-dependent RelayFlow work cannot be based on an unmerged contract. + +--- + +## Chapters + +### 1. Work +*Agent: default* + +- Keep the production repair unchanged and close only verified review gaps: Keep the production repair unchanged and close only verified review gaps + +--- + +## Artifacts + +**Commits:** 9f907408f +**Files changed:** 7 diff --git a/.agentworkforce/trajectories/completed/2026-08/traj_r79zmxc4sagy/trajectory.json b/.agentworkforce/trajectories/completed/2026-08/traj_r79zmxc4sagy/trajectory.json new file mode 100644 index 000000000..b0a8f7f3d --- /dev/null +++ b/.agentworkforce/trajectories/completed/2026-08/traj_r79zmxc4sagy/trajectory.json @@ -0,0 +1,68 @@ +{ + "id": "traj_r79zmxc4sagy", + "version": 1, + "task": { + "title": "Finish relay PR #1611 review and merge-readiness", + "source": { + "system": "plain", + "id": "AgentWorkforce/relay#1602 PR #1611" + } + }, + "status": "completed", + "startedAt": "2026-08-25T12:38:31.715Z", + "completedAt": "2026-08-25T12:51:26.153Z", + "agents": [ + { + "name": "default", + "role": "lead", + "joinedAt": "2026-08-25T12:51:23.830Z" + } + ], + "chapters": [ + { + "id": "chap_0i19byrbfmw5", + "title": "Work", + "agentName": "default", + "startedAt": "2026-08-25T12:51:23.830Z", + "endedAt": "2026-08-25T12:51:26.153Z", + "events": [ + { + "ts": 1787662283832, + "type": "decision", + "content": "Keep the production repair unchanged and close only verified review gaps: Keep the production repair unchanged and close only verified review gaps", + "raw": { + "question": "Keep the production repair unchanged and close only verified review gaps", + "chosen": "Keep the production repair unchanged and close only verified review gaps", + "alternatives": [], + "reasoning": "The exact-head audit confirms the live parentless registry path, workspace-key read-only identity lookup, exact-name and immutable-ID guards, zero registration/token rotation, and initial plus reconnect projections already satisfy #1602; the only valid remaining code defect is the Unix-only child fixture, while #1612-dependent RelayFlow work cannot be based on an unmerged contract." + }, + "significance": "high" + } + ] + } + ], + "retrospective": { + "summary": "Audited PR #1611 production invariants, Unix-gated its Unix child fixture, and repaired incomplete trajectory commit/file/trace metadata.", + "approach": "Exact-head GitHub review reconciliation followed by minimal review-backed edits and focused regression validation.", + "confidence": 0.96 + }, + "commits": [ + "9f907408f" + ], + "filesChanged": [ + ".agentworkforce/trajectories/completed/2026-08/traj_0c4k1s4nyzg3.trace.json", + ".agentworkforce/trajectories/completed/2026-08/traj_0c4k1s4nyzg3/trajectory.json", + ".agentworkforce/trajectories/completed/2026-08/traj_5ellsvzejuxq.trace.json", + ".agentworkforce/trajectories/completed/2026-08/traj_5ellsvzejuxq/trajectory.json", + ".agentworkforce/trajectories/completed/2026-08/traj_zotcff3t4aap.trace.json", + ".agentworkforce/trajectories/completed/2026-08/traj_zotcff3t4aap/trajectory.json", + "crates/broker/src/runtime/fleet.rs" + ], + "projectId": "AgentWorkforce/relay", + "tags": [], + "_trace": { + "startRef": "75f92f4474dc621a7b8e461e121c24605c40bccc", + "endRef": "9f907408fede5a2315b6662739f62c6e5e32e30a", + "traceId": "6b089b86-9d34-4b0a-9786-31a4242fd7b8" + } +} \ No newline at end of file diff --git a/.agentworkforce/trajectories/completed/2026-08/traj_zotcff3t4aap.trace.json b/.agentworkforce/trajectories/completed/2026-08/traj_zotcff3t4aap.trace.json new file mode 100644 index 000000000..f72d41541 --- /dev/null +++ b/.agentworkforce/trajectories/completed/2026-08/traj_zotcff3t4aap.trace.json @@ -0,0 +1,20 @@ +{ + "version": "1.0.0", + "id": "2bc5d818-b00d-483a-9bf0-922e95ac2de4", + "timestamp": "2026-08-25T07:49:18.312Z", + "trajectory": "traj_zotcff3t4aap", + "files": [ + { + "path": "crates/broker/src/fleet_wire.rs", + "conversations": [{ "contributor": { "type": "ai" }, "ranges": [] }] + }, + { + "path": "crates/broker/src/node_control.rs", + "conversations": [{ "contributor": { "type": "ai" }, "ranges": [] }] + }, + { + "path": "crates/broker/src/runtime/fleet.rs", + "conversations": [{ "contributor": { "type": "ai" }, "ranges": [] }] + } + ] +} diff --git a/.agentworkforce/trajectories/completed/2026-08/traj_zotcff3t4aap/summary.md b/.agentworkforce/trajectories/completed/2026-08/traj_zotcff3t4aap/summary.md new file mode 100644 index 000000000..08577e66e --- /dev/null +++ b/.agentworkforce/trajectories/completed/2026-08/traj_zotcff3t4aap/summary.md @@ -0,0 +1,32 @@ +# Trajectory: Strengthen relay #1602 adopted-worker regression + +> **Status:** ✅ Completed +> **Task:** AgentWorkforce/relay#1602 review +> **Confidence:** 96% +> **Started:** August 25, 2026 at 09:49 AM +> **Completed:** August 25, 2026 at 09:49 AM + +--- + +## Summary + +Reworked issue 1602 coverage so an adopted live PTY is reconciled with its immutable identity and remains in inventory.sync and heartbeat across reconnect. + +**Approach:** Standard approach + +--- + +## Key Decisions + +### Test the repaired union, not equality of incomplete projections +- **Chose:** Test the repaired union, not equality of incomplete projections +- **Reasoning:** Incident evidence established the heartbeat-only PTY was live; the regression must first prove identity-safe reconciliation adds it to inventory, then prove heartbeat and reconnect retain that repaired set. + +--- + +## Chapters + +### 1. Work +*Agent: default* + +- Test the repaired union, not equality of incomplete projections: Test the repaired union, not equality of incomplete projections diff --git a/.agentworkforce/trajectories/completed/2026-08/traj_zotcff3t4aap/trajectory.json b/.agentworkforce/trajectories/completed/2026-08/traj_zotcff3t4aap/trajectory.json new file mode 100644 index 000000000..a8a42bf2b --- /dev/null +++ b/.agentworkforce/trajectories/completed/2026-08/traj_zotcff3t4aap/trajectory.json @@ -0,0 +1,64 @@ +{ + "id": "traj_zotcff3t4aap", + "version": 1, + "task": { + "title": "Strengthen relay #1602 adopted-worker regression", + "source": { + "system": "plain", + "id": "AgentWorkforce/relay#1602 review" + } + }, + "status": "completed", + "startedAt": "2026-08-25T07:49:14.501Z", + "completedAt": "2026-08-25T07:49:18.312Z", + "agents": [ + { + "name": "default", + "role": "lead", + "joinedAt": "2026-08-25T07:49:16.488Z" + } + ], + "chapters": [ + { + "id": "chap_t8cp1nj1gc4a", + "title": "Work", + "agentName": "default", + "startedAt": "2026-08-25T07:49:16.488Z", + "endedAt": "2026-08-25T07:49:18.312Z", + "events": [ + { + "ts": 1787644156489, + "type": "decision", + "content": "Test the repaired union, not equality of incomplete projections: Test the repaired union, not equality of incomplete projections", + "raw": { + "question": "Test the repaired union, not equality of incomplete projections", + "chosen": "Test the repaired union, not equality of incomplete projections", + "alternatives": [], + "reasoning": "Incident evidence established the heartbeat-only PTY was live; the regression must first prove identity-safe reconciliation adds it to inventory, then prove heartbeat and reconnect retain that repaired set." + }, + "significance": "high" + } + ] + } + ], + "retrospective": { + "summary": "Reworked issue 1602 coverage so an adopted live PTY is reconciled with its immutable identity and remains in inventory.sync and heartbeat across reconnect.", + "approach": "Standard approach", + "confidence": 0.96 + }, + "commits": [ + "4a695b26e9022756f1729be31b929fa61535b0a4" + ], + "filesChanged": [ + "crates/broker/src/fleet_wire.rs", + "crates/broker/src/node_control.rs", + "crates/broker/src/runtime/fleet.rs" + ], + "projectId": "AgentWorkforce/relay", + "tags": [], + "_trace": { + "startRef": "fee5ee2d65e0b4a9a4f43f619c90c587e9440fcd", + "endRef": "4a695b26e9022756f1729be31b929fa61535b0a4", + "traceId": "2bc5d818-b00d-483a-9bf0-922e95ac2de4" + } +} diff --git a/CHANGELOG.md b/CHANGELOG.md index a71c23ff2..aa0308939 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,11 @@ All notable changes to Agent Relay will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [Unreleased - Patch] + +### Fixed + +- Fleet node heartbeats now derive their live-agent names from the same authoritative reconnect inventory as `inventory.sync`, preventing live PTYs from becoming unattachable after roster drift. ## [11.8.3] - 2026-08-24 diff --git a/crates/broker/src/fleet_wire.rs b/crates/broker/src/fleet_wire.rs index db105f154..195b83b52 100644 --- a/crates/broker/src/fleet_wire.rs +++ b/crates/broker/src/fleet_wire.rs @@ -88,10 +88,11 @@ pub struct FleetCapability { pub metadata: Option>, } -/// Reserved node capability carrying the broker's live WorkerName set in its -/// `metadata.names` array. It rides the existing heartbeat descriptor refresh, -/// so it does not depend on per-agent provider registration or a separate -/// control-plane write. +/// Reserved node capability carrying the broker's authoritative reconnect +/// inventory names in its `metadata.names` array. It rides the existing +/// heartbeat descriptor refresh while deriving from the same live-worker set +/// as `inventory.sync`, so heartbeat authorization and roster reconciliation +/// cannot disagree. pub const LIVE_AGENT_CAPABILITY_NAME: &str = "relay:live-agents:v1"; /// Provider identity carried on connection-scoped node frames. `name` is the @@ -175,8 +176,8 @@ pub struct NodeHeartbeat { pub name: String, pub node_id: String, // The broker appends reserved live-agent capabilities on every heartbeat; - // their names come directly from its worker registry, independently of - // provider registration and inventory.sync. + // their names come from the retained, identity-bearing inventory also sent + // by inventory.sync after live-worker reconciliation repairs adopted PTYs. pub capabilities: Vec, pub max_agents: u32, pub version: String, diff --git a/crates/broker/src/node_control.rs b/crates/broker/src/node_control.rs index 02707eaa9..fb509530f 100644 --- a/crates/broker/src/node_control.rs +++ b/crates/broker/src/node_control.rs @@ -466,7 +466,6 @@ pub(crate) struct FleetLoadSnapshot { pub(crate) active_agents: u32, pub(crate) max_agents: u32, pub(crate) handlers_live: bool, - pub(crate) active_agent_names: Vec, } impl FleetLoadSnapshot { @@ -487,7 +486,7 @@ impl FleetLoadSnapshot { /// /// `last_heartbeat_at` is intentionally NOT set — the engine stamps receipt /// time server-side as the single source of truth for liveness. - fn heartbeat(&self, node: &NodeRegister) -> NodeHeartbeat { + fn heartbeat(&self, node: &NodeRegister, inventory: &[InventoryAgent]) -> NodeHeartbeat { let load = if self.max_agents == 0 { // Relaycast releases before relaycast#307 require a numeric load. // Keep emitting the legacy value until the engine accepts an @@ -503,7 +502,14 @@ impl FleetLoadSnapshot { .filter(|capability| capability.name != LIVE_AGENT_CAPABILITY_NAME) .cloned() .collect(); - let mut active_agent_names = self.active_agent_names.clone(); + // The retained inventory is the broker's identity-bearing projection of + // its live fleet workers: runtime reconciliation repairs adopted PTYs + // into it without registration or token rotation, and node control keeps + // it across reconnects. Deriving this capability from the same slice + // makes heartbeat authorization and inventory.sync converge on the + // repaired worker set instead of maintaining independent projections. + let mut active_agent_names: Vec<_> = + inventory.iter().map(|agent| agent.name.clone()).collect(); active_agent_names.sort(); active_agent_names.dedup(); capabilities.push(FleetCapability { @@ -1775,7 +1781,7 @@ async fn run_connected_once( } if send_wire( &mut sink, - &BrokerToRelaycast::NodeHeartbeat(load.heartbeat(&node_register)), + &BrokerToRelaycast::NodeHeartbeat(load.heartbeat(&node_register, inventory)), ) .await .is_err() @@ -1828,7 +1834,7 @@ async fn run_connected_once( *load = next; } Some(FleetControlCommand::HeartbeatNow) => { - if send_wire(&mut sink, &BrokerToRelaycast::NodeHeartbeat(load.heartbeat(&node_register))).await.is_err() { + if send_wire(&mut sink, &BrokerToRelaycast::NodeHeartbeat(load.heartbeat(&node_register, inventory))).await.is_err() { return ControlRunResult::Disconnected; } } @@ -1882,7 +1888,7 @@ async fn run_connected_once( drain_agent_registrations(&mut pending_agent_registrations, "node_control_disconnected"); return ControlRunResult::Disconnected; } - if send_wire(&mut sink, &BrokerToRelaycast::NodeHeartbeat(load.heartbeat(&node_register))).await.is_err() { + if send_wire(&mut sink, &BrokerToRelaycast::NodeHeartbeat(load.heartbeat(&node_register, inventory))).await.is_err() { drain_agent_registrations(&mut pending_agent_registrations, "node_control_disconnected"); return ControlRunResult::Disconnected; } @@ -2155,7 +2161,7 @@ pub(crate) fn delivery_ack(agent: impl Into, up_to_seq: u64) -> BrokerTo } #[cfg(test)] -mod tests { +pub(crate) mod tests { use std::sync::{ atomic::{AtomicUsize, Ordering}, Arc, @@ -3744,7 +3750,6 @@ mod tests { active_agents: 1, max_agents: 4, handlers_live: true, - active_agent_names: vec!["agent-a".to_string()], }; let server = tokio::spawn(async move { @@ -3825,7 +3830,6 @@ mod tests { active_agents: 0, max_agents: 4, handlers_live: true, - active_agent_names: Vec::new(), }; let server = tokio::spawn(async move { @@ -4055,7 +4059,27 @@ mod tests { } #[tokio::test] - async fn node_control_reconnect_sends_inventory_sync() { + async fn repaired_inventory_drives_heartbeat_and_survives_reconnect() { + assert_repaired_inventory_drives_heartbeat_and_survives_reconnect(vec![ + InventoryAgent { + agent_id: "agent-inventory-id".to_string(), + name: "inventory-worker".to_string(), + invocation_id: Some("inv-1".to_string()), + session_ref: Some("session-1".to_string()), + }, + InventoryAgent { + agent_id: "agent-adopted-id".to_string(), + name: "adopted-worker".to_string(), + invocation_id: None, + session_ref: Some("session-adopted".to_string()), + }, + ]) + .await; + } + + pub(crate) async fn assert_repaired_inventory_drives_heartbeat_and_survives_reconnect( + repaired_inventory: Vec, + ) { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let ws_url = format!("ws://{}/v1/node/ws", listener.local_addr().unwrap()); let (command_tx, command_rx) = mpsc::channel(32); @@ -4093,10 +4117,34 @@ mod tests { )); match next_non_heartbeat_node_to_server(&mut ws).await { BrokerToRelaycast::InventorySync(sync) => { - assert_eq!(sync.agents.len(), 1); - assert_eq!(sync.agents[0].name, "agent-a"); + let mut names: Vec<_> = sync + .agents + .iter() + .map(|agent| agent.name.as_str()) + .collect(); + names.sort_unstable(); + assert_eq!(names, ["adopted-worker", "inventory-worker"]); } - other => panic!("expected inventory update before reconnect, got {other:?}"), + other => panic!("expected repaired inventory update, got {other:?}"), + } + match next_node_to_server(&mut ws).await { + BrokerToRelaycast::NodeHeartbeat(heartbeat) => { + let names = heartbeat + .capabilities + .iter() + .find(|capability| capability.name == LIVE_AGENT_CAPABILITY_NAME) + .and_then(|capability| capability.metadata.as_ref()) + .and_then(|metadata| metadata.get("names")); + assert_eq!( + names, + Some(&serde_json::json!([ + "adopted-worker", + "inventory-worker" + ])), + "the adopted live worker must enter heartbeat authorization after inventory repair" + ); + } + other => panic!("expected heartbeat after repaired inventory, got {other:?}"), } ws.close(None).await.unwrap(); @@ -4108,11 +4156,35 @@ mod tests { )); match next_non_heartbeat_node_to_server(&mut ws).await { BrokerToRelaycast::InventorySync(sync) => { - assert_eq!(sync.agents.len(), 1); - assert_eq!(sync.agents[0].name, "agent-a"); + let mut names: Vec<_> = sync + .agents + .iter() + .map(|agent| agent.name.as_str()) + .collect(); + names.sort_unstable(); + assert_eq!(names, ["adopted-worker", "inventory-worker"]); + assert!(sync.agents.iter().any(|agent| { + agent.name == "adopted-worker" && agent.agent_id == "agent-adopted-id" + })); } other => panic!("expected inventory.sync, got {other:?}"), } + match next_node_to_server(&mut ws).await { + BrokerToRelaycast::NodeHeartbeat(heartbeat) => { + let names = heartbeat + .capabilities + .iter() + .find(|capability| capability.name == LIVE_AGENT_CAPABILITY_NAME) + .and_then(|capability| capability.metadata.as_ref()) + .and_then(|metadata| metadata.get("names")); + assert_eq!( + names, + Some(&serde_json::json!(["adopted-worker", "inventory-worker"])), + "a reconnect heartbeat must retain the repaired live-worker inventory" + ); + } + other => panic!("expected reconnect heartbeat, got {other:?}"), + } }); command_tx @@ -4123,12 +4195,19 @@ mod tests { .await .unwrap(); command_tx - .send(FleetControlCommand::UpdateInventory(vec![InventoryAgent { - agent_id: "agt-1".to_string(), - name: "agent-a".to_string(), - invocation_id: Some("inv-1".to_string()), - session_ref: Some("session-1".to_string()), - }])) + .send(FleetControlCommand::UpdateInventory(repaired_inventory)) + .await + .unwrap(); + command_tx + .send(FleetControlCommand::UpdateLoad(FleetLoadSnapshot { + active_agents: 2, + max_agents: 4, + handlers_live: true, + })) + .await + .unwrap(); + command_tx + .send(FleetControlCommand::HeartbeatNow) .await .unwrap(); @@ -4197,9 +4276,24 @@ mod tests { active_agents: 3, max_agents: 4, handlers_live: true, - active_agent_names: vec!["worker-b".to_string(), "worker-a".to_string()], } - .heartbeat(®ister); + .heartbeat( + ®ister, + &[ + InventoryAgent { + agent_id: "agent-b".to_string(), + name: "worker-b".to_string(), + invocation_id: None, + session_ref: None, + }, + InventoryAgent { + agent_id: "agent-a".to_string(), + name: "worker-a".to_string(), + invocation_id: None, + session_ref: None, + }, + ], + ); assert_eq!(measured.load, Some(0.75)); let live_agent_capability = measured .capabilities @@ -4218,9 +4312,8 @@ mod tests { active_agents: 25, max_agents: 0, handlers_live: true, - active_agent_names: Vec::new(), } - .heartbeat(®ister); + .heartbeat(®ister, &[]); assert_eq!(unbounded.load, Some(0.0)); let value = serde_json::to_value(BrokerToRelaycast::NodeHeartbeat(unbounded)).unwrap(); diff --git a/crates/broker/src/runtime/api.rs b/crates/broker/src/runtime/api.rs index 4535da971..d1b797f38 100644 --- a/crates/broker/src/runtime/api.rs +++ b/crates/broker/src/runtime/api.rs @@ -965,11 +965,6 @@ impl BrokerRuntime { super::fleet::publish_fleet_load_snapshot( fleet_control_tx, u32::try_from(workers.workers.len()).unwrap_or(u32::MAX), - workers - .workers - .keys() - .map(|name| name.as_str().to_string()) - .collect(), fleet_max_agents, fleet_handlers_live, true, @@ -1072,11 +1067,6 @@ impl BrokerRuntime { super::fleet::publish_fleet_load_snapshot( fleet_control_tx, u32::try_from(workers.workers.len()).unwrap_or(u32::MAX), - workers - .workers - .keys() - .map(|name| name.as_str().to_string()) - .collect(), fleet_max_agents, fleet_handlers_live, true, diff --git a/crates/broker/src/runtime/fleet.rs b/crates/broker/src/runtime/fleet.rs index 00e4f8500..f91445828 100644 --- a/crates/broker/src/runtime/fleet.rs +++ b/crates/broker/src/runtime/fleet.rs @@ -1489,19 +1489,12 @@ impl BrokerRuntime { async fn publish_fleet_load(&self, heartbeat_now: bool) { let active_agents = u32::try_from(self.workers.workers.len()).unwrap_or(u32::MAX); - let active_agent_names = self - .workers - .workers - .keys() - .map(|name| name.as_str().to_string()) - .collect(); // The broker provider's capacity handlers (spawn/release) are live for as // long as its connection is up, so `handlers_live` is unconditionally true // here — a connected broker can always place work. publish_fleet_load_snapshot( &self.fleet_control_tx, active_agents, - active_agent_names, self.fleet_max_agents, true, heartbeat_now, @@ -1830,7 +1823,6 @@ pub(super) async fn register_node_agent_token( pub(super) async fn publish_fleet_load_snapshot( fleet_control_tx: &mpsc::Sender, active_agents: u32, - active_agent_names: Vec, max_agents: u32, handlers_live: bool, heartbeat_now: bool, @@ -1840,7 +1832,6 @@ pub(super) async fn publish_fleet_load_snapshot( active_agents, max_agents, handlers_live, - active_agent_names, })) { tracing::warn!(error = %error, "fleet load update queue is unavailable; periodic heartbeat will retry"); @@ -3280,7 +3271,7 @@ mod tests { tokio::time::timeout( Duration::from_millis(50), - publish_fleet_load_snapshot(&tx, 1, vec!["worker-a".to_string()], 4, true, true), + publish_fleet_load_snapshot(&tx, 1, 4, true, true), ) .await .expect("load publication must not stall the runtime API actor"); @@ -3468,18 +3459,67 @@ mod tests { } } + #[cfg(unix)] #[tokio::test] - async fn reconciliation_restores_a_live_worker_missing_from_inventory_without_reregistering() { + async fn reconciliation_adds_an_adopted_live_worker_without_reregistering() { + let temp = tempfile::tempdir().expect("worker registry tempdir"); + let (event_tx, _event_rx) = mpsc::channel::(4); + let mut workers = WorkerRegistry::new( + event_tx, + Vec::new(), + temp.path().join("worker-logs"), + Instant::now(), + ); + let mut child_command = tokio::process::Command::new("sh"); + child_command + .args(["-c", "sleep 30"]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .kill_on_drop(true); + let child = child_command.spawn().expect("live adopted PTY fixture"); + let adopted_name = WorkerName::from("adopted-worker"); + let mut adopted_spec = test_agent_spec(Some("session-adopted"), None); + adopted_spec.name = adopted_name.clone(); + let (worker_command_tx, _worker_command_rx) = mpsc::channel(4); + workers.workers.insert( + adopted_name.clone(), + WorkerHandle { + generation: Uuid::from_u128(101), + spec: adopted_spec, + // Reproduce the incident seam: an adopted/migrated live PTY no + // longer has the original fleet spawn parent marker. + parent: None, + workspace_id: None, + child, + command_tx: worker_command_tx, + harness_pid: None, + spawned_at: Instant::now(), + ready_at: Some(Instant::now()), + last_activity_at: Instant::now(), + context_budget_pct: None, + state: crate::worker::AgentWorkState::Idle, + exit_reason: None, + }, + ); + let live_workers = workers.live_fleet_inventory_candidates(); + assert_eq!(live_workers.len(), 1); + assert_eq!(live_workers[0].name, adopted_name); + assert_eq!( + live_workers[0].session_ref.as_deref(), + Some("session-adopted") + ); + let server = MockServer::start(); let lookup = server.mock(|when, then| { when.method(GET) - .path("/v1/agents/live-worker") + .path("/v1/agents/adopted-worker") .header("authorization", "Bearer rk_live_test"); then.status(200).json_body(serde_json::json!({ "ok": true, "data": { - "id": "agent-live-id", - "name": "live-worker", + "id": "agent-adopted-id", + "name": "adopted-worker", "type": "agent", "status": "offline", "persona": null, @@ -3499,7 +3539,15 @@ mod tests { let relaycast_http = RelaycastHttpClient::new(Some(server.base_url()), "rk_live_test", "broker", "claude"); let (tx, mut rx) = mpsc::channel(2); - let mut inventory = HashMap::new(); + let mut inventory = HashMap::from([( + WorkerName::from("inventory-worker"), + InventoryAgent { + agent_id: "agent-inventory-id".to_string(), + name: "inventory-worker".to_string(), + invocation_id: Some("inv-inventory".to_string()), + session_ref: Some("session-inventory".to_string()), + }, + )]); let mut delivery_book = FleetDeliveryBook::default(); let mut retry_after = HashMap::new(); @@ -3509,30 +3557,35 @@ mod tests { &mut delivery_book, &mut inventory, &mut retry_after, - vec![live_fleet_worker("live-worker", Some("session-live"), 101)], + live_workers, Instant::now(), ) .await; assert_eq!(repaired, 1, "the live orphan must be restored"); assert_eq!( - inventory.get(&WorkerName::from("live-worker")), + inventory.get(&WorkerName::from("adopted-worker")), Some(&InventoryAgent { - agent_id: "agent-live-id".to_string(), - name: "live-worker".to_string(), + agent_id: "agent-adopted-id".to_string(), + name: "adopted-worker".to_string(), invocation_id: None, - session_ref: Some("session-live".to_string()), + session_ref: Some("session-adopted".to_string()), }) ); assert_eq!( - delivery_book.active_agent_id("live-worker"), - Some("agent-live-id") + delivery_book.active_agent_id("adopted-worker"), + Some("agent-adopted-id") ); match rx.recv().await { Some(FleetControlCommand::UpdateInventory(agents)) => { - assert_eq!(agents.len(), 1); - assert_eq!(agents[0].name, "live-worker"); - assert_eq!(agents[0].agent_id, "agent-live-id"); + assert_eq!(agents.len(), 2); + assert!(agents.iter().any(|agent| { + agent.name == "inventory-worker" && agent.agent_id == "agent-inventory-id" + })); + assert!(agents.iter().any(|agent| { + agent.name == "adopted-worker" && agent.agent_id == "agent-adopted-id" + })); + crate::node_control::tests::assert_repaired_inventory_drives_heartbeat_and_survives_reconnect(agents).await; } other => panic!("expected repaired inventory snapshot, got {other:?}"), } @@ -4021,12 +4074,11 @@ mod tests { async fn publish_fleet_load_snapshot_emits_immediate_heartbeat_after_release() { let (tx, mut rx) = mpsc::channel(4); - publish_fleet_load_snapshot(&tx, 1, vec!["worker-a".to_string()], 4, true, true).await; + publish_fleet_load_snapshot(&tx, 1, 4, true, true).await; match rx.recv().await { Some(FleetControlCommand::UpdateLoad(load)) => { assert_eq!(load.active_agents, 1); - assert_eq!(load.active_agent_names, vec!["worker-a"]); assert_eq!(load.max_agents, 4); assert!(load.handlers_live); } diff --git a/crates/broker/src/runtime/maintenance.rs b/crates/broker/src/runtime/maintenance.rs index 2d69f1547..9019130bd 100644 --- a/crates/broker/src/runtime/maintenance.rs +++ b/crates/broker/src/runtime/maintenance.rs @@ -729,11 +729,6 @@ impl BrokerRuntime { super::fleet::publish_fleet_load_snapshot( fleet_control_tx, u32::try_from(workers.workers.len()).unwrap_or(u32::MAX), - workers - .workers - .keys() - .map(|name| name.as_str().to_string()) - .collect(), fleet_max_agents, fleet_handlers_live, true, diff --git a/crates/broker/src/worker.rs b/crates/broker/src/worker.rs index d99ff9172..9dec99d40 100644 --- a/crates/broker/src/worker.rs +++ b/crates/broker/src/worker.rs @@ -511,15 +511,17 @@ impl WorkerRegistry { } } - /// Return the live broker-owned workers that must remain present in the - /// Relaycast reconnect inventory. The parent marker is set by the two - /// production spawn surfaces (Dashboard and Relaycast); workers without it - /// are local-only and must not cause a fleet identity lookup. + /// Return every live broker-owned worker that may need to be restored to the + /// Relaycast reconnect inventory. Adopted and migrated PTYs can outlive the + /// spawn metadata that supplied `parent`, so that marker cannot decide + /// roster membership. Reconciliation performs a read-only, workspace-scoped + /// identity lookup and only adopts an existing matching Relaycast identity; + /// it never registers a local-only worker or rotates a token. pub(crate) fn live_fleet_inventory_candidates(&self) -> Vec { self.workers .iter() .filter_map(|(name, handle)| { - if handle.parent.is_none() || !self.is_worker_live(name) { + if !self.is_worker_live(name) { return None; } let session_ref = handle.spec.session_id.clone().or_else(|| { diff --git a/tests/fixtures/pr-proof-contract.test.ts b/tests/fixtures/pr-proof-contract.test.ts index 9a1ced05c..7eba32de0 100644 --- a/tests/fixtures/pr-proof-contract.test.ts +++ b/tests/fixtures/pr-proof-contract.test.ts @@ -459,7 +459,10 @@ describe('process timeout contract', () => { ].join(''); const result = await runProcess(process.execPath, ['-e', script], { echo: false, - timeoutMs: 100, + // This timeout measures when cleanup begins, not child startup. Leave + // enough room for a contended runner to spawn and report the descendant + // PID so the assertions exercise process-group termination itself. + timeoutMs: 500, terminationGraceMs: 100, }); const descendantPid = Number(result.stdout.trim()); diff --git a/tests/relayflows/cases/1602-parentless-worker-inventory/case.json b/tests/relayflows/cases/1602-parentless-worker-inventory/case.json new file mode 100644 index 000000000..6c7ced728 --- /dev/null +++ b/tests/relayflows/cases/1602-parentless-worker-inventory/case.json @@ -0,0 +1,20 @@ +{ + "version": 1, + "id": "1602-parentless-worker-inventory", + "kind": "bugfix", + "title": "Parentless adopted workers remain in reconnect inventory", + "runner": { + "command": ["node", "tests/relayflows/cases/1602-parentless-worker-inventory/run.mjs"] + }, + "timeoutSeconds": 1800, + "expected": { + "base": { + "outcome": "bug", + "signature": "parentless_worker_excluded_from_reconnect_inventory" + }, + "head": { + "outcome": "fixed", + "signature": "parentless_worker_in_sync_and_heartbeat_after_reconnect" + } + } +} diff --git a/tests/relayflows/cases/1602-parentless-worker-inventory/run.mjs b/tests/relayflows/cases/1602-parentless-worker-inventory/run.mjs new file mode 100644 index 000000000..5ede5a470 --- /dev/null +++ b/tests/relayflows/cases/1602-parentless-worker-inventory/run.mjs @@ -0,0 +1,485 @@ +#!/usr/bin/env node + +import { spawn } from 'node:child_process'; +import { constants as fsConstants } from 'node:fs'; +import { access, readFile, readdir, realpath, writeFile } from 'node:fs/promises'; +import path from 'node:path'; + +const CASE_ID = '1602-parentless-worker-inventory'; +const MARKER = 'RELAY_PR_PROOF_OBSERVATION='; +const PROBE_TEST = 'runtime::fleet::relayflow_1602_probe::relayflow_1602_parentless_worker_inventory_probe'; + +function requiredEnvironment(name) { + const value = process.env[name]?.trim(); + if (!value) throw new Error(`${name} is required`); + return value; +} + +function boundedAppend(current, chunk, maximum = 2 * 1024 * 1024) { + const next = current + chunk; + return next.length <= maximum ? next : next.slice(-maximum); +} + +function proofChildEnvironment() { + const env = { + ...process.env, + CARGO_INCREMENTAL: '0', + CARGO_PROFILE_DEV_DEBUG: '0', + CARGO_PROFILE_TEST_DEBUG: '0', + CARGO_TERM_COLOR: 'never', + RUST_BACKTRACE: '1', + }; + for (const key of Object.keys(env)) { + if (/^GIT_CONFIG_(?:COUNT|KEY_\d+|VALUE_\d+)$/.test(key) || key.startsWith('RELAY_ATTEST_')) { + delete env[key]; + } + } + return env; +} + +async function isExecutable(filePath) { + try { + await access(filePath, fsConstants.X_OK); + return true; + } catch { + return false; + } +} + +async function resolveCargo() { + const pathEntries = (process.env.PATH ?? '').split(path.delimiter).filter(Boolean); + for (const entry of pathEntries) { + const candidate = path.join(entry, 'cargo'); + if (!(await isExecutable(candidate))) continue; + const resolved = await realpath(candidate).catch(() => candidate); + if (path.basename(resolved) === 'cargo') return resolved; + } + + const homes = new Set(); + for (const entry of pathEntries) { + const normalized = entry.replaceAll('\\', '/'); + for (const suffix of ['/.local/share/mise/shims', '/.cargo/bin']) { + if (normalized.endsWith(suffix)) homes.add(normalized.slice(0, -suffix.length)); + } + } + if (process.env.RUSTUP_HOME) { + homes.add(path.dirname(path.resolve(process.env.RUSTUP_HOME))); + } + + for (const home of homes) { + const toolchains = path.join(home, '.rustup', 'toolchains'); + const entries = await readdir(toolchains).catch(() => []); + for (const entry of entries.sort().reverse()) { + const candidate = path.join(toolchains, entry, 'bin', 'cargo'); + if (await isExecutable(candidate)) return candidate; + } + } + + for (const candidate of ['/usr/local/cargo/bin/cargo', '/opt/rust/bin/cargo']) { + if (await isExecutable(candidate)) return candidate; + } + throw new Error('could not resolve a real Cargo executable outside a version-manager shim'); +} + +function run(command, args, options = {}) { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { + cwd: options.cwd, + env: options.env ?? process.env, + stdio: ['ignore', 'pipe', 'pipe'], + }); + let stdout = ''; + let stderr = ''; + child.stdout.on('data', (chunk) => { + const text = chunk.toString('utf8'); + stdout = boundedAppend(stdout, text); + process.stdout.write(text); + }); + child.stderr.on('data', (chunk) => { + const text = chunk.toString('utf8'); + stderr = boundedAppend(stderr, text); + process.stderr.write(text); + }); + child.on('error', reject); + child.on('close', (code, signal) => resolve({ code: code ?? 1, signal, stdout, stderr })); + }); +} + +const PROBE_SOURCE = String.raw` + +#[cfg(all(test, unix))] +mod relayflow_1602_probe { + use super::*; + use crate::fleet_wire::{ + BrokerToRelaycast, InventoryAgent, LIVE_AGENT_CAPABILITY_NAME, + }; + use crate::node_control::{ + run_node_control_client, FleetControlCommand, FleetControlConfig, + FleetControlEvent, FleetDeliveryBook, + }; + use crate::protocol::{AgentRuntime, AgentSpec, NodeManifest}; + use crate::relaycast::RelaycastHttpClient; + use crate::worker::{AgentWorkState, WorkerEvent, WorkerHandle, WorkerRegistry}; + use futures_util::{SinkExt, StreamExt}; + use httpmock::prelude::*; + use std::process::Stdio; + use tokio::net::{TcpListener, TcpStream}; + use tokio::sync::mpsc; + use tokio_tungstenite::{accept_async, tungstenite::Message, WebSocketStream}; + + const ADOPTED_NAME: &str = "adopted-worker"; + const ADOPTED_ID: &str = "agent-adopted-id"; + + fn adopted_spec() -> AgentSpec { + AgentSpec { + name: WorkerName::from(ADOPTED_NAME), + runtime: AgentRuntime::Pty, + provider: None, + cli: Some("codex".to_string()), + session_id: Some("session-adopted".to_string()), + harness_config: None, + model: None, + cwd: None, + team: None, + shadow_of: None, + shadow_mode: None, + args: Vec::new(), + channels: Vec::new(), + restart_policy: None, + } + } + + fn manifest() -> NodeManifest { + NodeManifest { + name: "relayflow-1602".to_string(), + node_id: None, + capabilities: Vec::new(), + max_agents: Some(4), + tags: None, + repo_keys: None, + version: Some("relayflow-1602".to_string()), + } + } + + async fn next_frame(ws: &mut WebSocketStream) -> BrokerToRelaycast { + loop { + match ws.next().await.expect("node-control frame").expect("valid websocket frame") { + Message::Text(text) => return serde_json::from_str(&text).expect("valid node frame"), + Message::Ping(payload) => ws.send(Message::Pong(payload)).await.expect("pong"), + Message::Close(frame) => panic!("node-control socket closed early: {frame:?}"), + _ => {} + } + } + } + + fn inventory_has_adopted(sync: &crate::fleet_wire::InventorySync) -> bool { + sync.agents + .iter() + .any(|agent| agent.name == ADOPTED_NAME && agent.agent_id == ADOPTED_ID) + } + + fn heartbeat_has_adopted(heartbeat: &crate::fleet_wire::NodeHeartbeat) -> bool { + heartbeat + .capabilities + .iter() + .find(|capability| capability.name == LIVE_AGENT_CAPABILITY_NAME) + .and_then(|capability| capability.metadata.as_ref()) + .and_then(|metadata| metadata.get("names")) + .and_then(serde_json::Value::as_array) + .is_some_and(|names| names.iter().any(|name| name.as_str() == Some(ADOPTED_NAME))) + } + + async fn observe_first_connection(ws: &mut WebSocketStream) { + let mut saw_register = false; + let mut saw_initial_inventory = false; + let mut saw_initial_heartbeat = false; + let mut saw_repaired_inventory = false; + let mut saw_repaired_heartbeat = false; + + while !(saw_repaired_inventory && saw_repaired_heartbeat) { + match next_frame(ws).await { + BrokerToRelaycast::NodeRegister(_) => saw_register = true, + BrokerToRelaycast::InventorySync(sync) => { + if inventory_has_adopted(&sync) { + assert_eq!(sync.agents.len(), 2, "repair must retain the existing worker"); + saw_repaired_inventory = true; + } else { + assert!(sync.agents.is_empty(), "initial inventory must begin empty"); + saw_initial_inventory = true; + } + } + BrokerToRelaycast::NodeHeartbeat(heartbeat) => { + if heartbeat_has_adopted(&heartbeat) { + saw_repaired_heartbeat = true; + } else { + saw_initial_heartbeat = true; + } + } + _ => {} + } + } + + assert!(saw_register && saw_initial_inventory && saw_initial_heartbeat); + } + + async fn observe_reconnect(ws: &mut WebSocketStream) { + let mut saw_register = false; + let mut saw_inventory = false; + let mut saw_heartbeat = false; + while !(saw_register && saw_inventory && saw_heartbeat) { + match next_frame(ws).await { + BrokerToRelaycast::NodeRegister(_) => saw_register = true, + BrokerToRelaycast::InventorySync(sync) => { + assert!(inventory_has_adopted(&sync), "reconnect sync lost immutable adopted id"); + assert_eq!(sync.agents.len(), 2, "reconnect sync lost retained inventory"); + saw_inventory = true; + } + BrokerToRelaycast::NodeHeartbeat(heartbeat) => { + assert!(heartbeat_has_adopted(&heartbeat), "reconnect heartbeat lost adopted worker"); + saw_heartbeat = true; + } + _ => {} + } + } + } + + #[tokio::test] + async fn relayflow_1602_parentless_worker_inventory_probe() { + let temp = tempfile::tempdir().expect("worker registry tempdir"); + let (event_tx, _event_rx) = mpsc::channel::(4); + let mut workers = WorkerRegistry::new( + event_tx, + Vec::new(), + temp.path().join("worker-logs"), + Instant::now(), + ); + let mut child_command = tokio::process::Command::new("sh"); + child_command + .args(["-c", "sleep 30"]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .kill_on_drop(true); + let child = child_command.spawn().expect("live adopted PTY fixture"); + let adopted_name = WorkerName::from(ADOPTED_NAME); + let (worker_command_tx, _worker_command_rx) = mpsc::channel(4); + workers.workers.insert( + adopted_name.clone(), + WorkerHandle { + generation: Uuid::from_u128(1602), + spec: adopted_spec(), + parent: None, + workspace_id: None, + child, + command_tx: worker_command_tx, + harness_pid: None, + spawned_at: Instant::now(), + ready_at: Some(Instant::now()), + last_activity_at: Instant::now(), + context_budget_pct: None, + state: AgentWorkState::Idle, + exit_reason: None, + }, + ); + + let live_workers = workers.live_fleet_inventory_candidates(); + if live_workers.is_empty() { + println!( + "RELAY_PR_PROOF_OBSERVATION={{\"outcome\":\"bug\",\"signature\":\"parentless_worker_excluded_from_reconnect_inventory\",\"details\":\"The exact target's production WorkerRegistry excluded a live parentless adopted PTY before identity-safe reconciliation, so it could not enter reconnect inventory.\"}}" + ); + return; + } + assert_eq!(live_workers.len(), 1, "probe owns exactly one live worker"); + assert_eq!(live_workers[0].name, adopted_name); + + let relaycast = MockServer::start(); + let lookup = relaycast.mock(|when, then| { + when.method(GET) + .path("/v1/agents/adopted-worker") + .header("authorization", "Bearer rk_live_test"); + then.status(200).json_body(serde_json::json!({ + "ok": true, + "data": { + "id": ADOPTED_ID, + "name": ADOPTED_NAME, + "type": "agent", + "status": "offline", + "persona": null, + "metadata": {} + } + })); + }); + let registration = relaycast.mock(|when, then| { + when.method(POST).path("/v1/agents"); + then.status(500).json_body(serde_json::json!({ + "ok": false, + "error": { "code": "must_not_register", "message": "must not register" } + })); + }); + + let listener = TcpListener::bind("127.0.0.1:0").await.expect("node listener"); + let ws_url = format!("ws://{}/v1/node/ws", listener.local_addr().unwrap()); + let (command_tx, command_rx) = mpsc::channel(32); + let (control_event_tx, mut control_event_rx) = mpsc::channel(32); + tokio::spawn(run_node_control_client( + FleetControlConfig { + ws_url, + node_token: Some("nt_test".to_string()), + node_id: "node-relayflow-1602".to_string(), + node_name: "relayflow-1602".to_string(), + broker_version: "broker/relayflow-1602".to_string(), + token_minter: None, + session_token: None, + read_idle_timeout: None, + }, + command_rx, + control_event_tx, + )); + + let server = tokio::spawn(async move { + let (first_stream, _) = listener.accept().await.expect("first node connection"); + let mut first = accept_async(first_stream).await.expect("first websocket"); + observe_first_connection(&mut first).await; + first.close(None).await.expect("close first connection"); + + let (second_stream, _) = listener.accept().await.expect("reconnect"); + let mut second = accept_async(second_stream).await.expect("second websocket"); + observe_reconnect(&mut second).await; + second.close(None).await.expect("close second connection"); + }); + + command_tx + .send(FleetControlCommand::RegisterNode { + manifest: manifest(), + resume_cursor: None, + }) + .await + .expect("register node command"); + assert_eq!( + tokio::time::timeout(Duration::from_secs(5), control_event_rx.recv()) + .await + .expect("node connect timeout"), + Some(FleetControlEvent::Connected) + ); + + let relaycast_http = RelaycastHttpClient::new( + Some(relaycast.base_url()), + "rk_live_test", + "broker", + "claude", + ); + let mut inventory = HashMap::from([( + WorkerName::from("inventory-worker"), + InventoryAgent { + agent_id: "agent-inventory-id".to_string(), + name: "inventory-worker".to_string(), + invocation_id: Some("inv-inventory".to_string()), + session_ref: Some("session-inventory".to_string()), + }, + )]); + let mut delivery_book = FleetDeliveryBook::default(); + let mut retry_after = HashMap::new(); + let repaired = reconcile_fleet_inventory_with_live_workers( + &command_tx, + &relaycast_http, + &mut delivery_book, + &mut inventory, + &mut retry_after, + live_workers, + Instant::now(), + ) + .await; + assert_eq!(repaired, 1, "identity-safe reconciliation must repair the adopted worker"); + assert_eq!(delivery_book.active_agent_id(ADOPTED_NAME), Some(ADOPTED_ID)); + command_tx + .send(FleetControlCommand::HeartbeatNow) + .await + .expect("immediate repaired heartbeat"); + + tokio::time::timeout(Duration::from_secs(12), server) + .await + .expect("initial and reconnect observations timed out") + .expect("node server task"); + lookup.assert_hits(1); + registration.assert_hits(0); + let _ = command_tx.send(FleetControlCommand::Shutdown).await; + + println!( + "RELAY_PR_PROOF_OBSERVATION={{\"outcome\":\"fixed\",\"signature\":\"parentless_worker_in_sync_and_heartbeat_after_reconnect\",\"details\":\"The exact target included a real live parentless PTY, preserved agent-adopted-id via one read-only name lookup and zero registrations, then emitted that worker in inventory.sync and relay:live-agents:v1 before and after reconnect.\"}}" + ); + } +} +`; + +async function main() { + if (process.platform === 'win32') { + throw new Error('The parentless live-PTY proof requires a Unix process fixture'); + } + const arm = requiredEnvironment('RELAY_PR_PROOF_ARM'); + if (arm !== 'base' && arm !== 'head') throw new Error(`unsupported proof arm: ${arm}`); + const targetDir = path.resolve(requiredEnvironment('RELAY_PR_PROOF_TARGET_DIR')); + const resultPath = path.resolve(requiredEnvironment('RELAY_PR_PROOF_RESULT_PATH')); + const expectedTargetSha = process.env.RELAY_PR_PROOF_TARGET_SHA?.trim(); + + const revision = await run('git', ['rev-parse', 'HEAD'], { cwd: targetDir }); + if (revision.code !== 0) throw new Error('could not resolve the target checkout SHA'); + const actualTargetSha = revision.stdout.trim(); + if (expectedTargetSha && actualTargetSha !== expectedTargetSha) { + throw new Error(`target provenance mismatch: expected ${expectedTargetSha}, got ${actualTargetSha}`); + } + + const fleetPath = path.join(targetDir, 'crates', 'broker', 'src', 'runtime', 'fleet.rs'); + const original = await readFile(fleetPath, 'utf8'); + if (original.includes('mod relayflow_1602_probe')) { + throw new Error('target checkout unexpectedly already contains the external probe module'); + } + + let testResult; + try { + await writeFile(fleetPath, `${original}${PROBE_SOURCE}`); + const cargo = await resolveCargo(); + const childEnvironment = proofChildEnvironment(); + childEnvironment.RUSTC = path.join(path.dirname(cargo), 'rustc'); + childEnvironment.RUSTDOC = path.join(path.dirname(cargo), 'rustdoc'); + testResult = await run( + cargo, + [ + 'test', + '--locked', + '-p', + 'agent-relay-broker', + '--lib', + PROBE_TEST, + '--', + '--exact', + '--nocapture', + '--test-threads=1', + ], + { cwd: targetDir, env: childEnvironment } + ); + } finally { + await writeFile(fleetPath, original); + } + + if (testResult.code !== 0) { + throw new Error( + `production-path probe failed with exit ${testResult.code}${testResult.signal ? ` (${testResult.signal})` : ''}` + ); + } + const markerLine = testResult.stdout.split(/\r?\n/).find((line) => line.includes(MARKER)); + if (!markerLine) throw new Error('production-path probe did not emit a structured observation'); + const observation = JSON.parse(markerLine.slice(markerLine.indexOf(MARKER) + MARKER.length)); + if (!observation || typeof observation !== 'object') { + throw new Error('production-path probe emitted an invalid observation'); + } + + await writeFile( + resultPath, + `${JSON.stringify({ version: 1, caseId: CASE_ID, arm, ...observation }, null, 2)}\n` + ); +} + +main().catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; +});