diff --git a/.agentworkforce/trajectories/completed/2026-08/traj_iv95g9euppbw.trace.json b/.agentworkforce/trajectories/completed/2026-08/traj_iv95g9euppbw.trace.json new file mode 100644 index 000000000..fa9830991 --- /dev/null +++ b/.agentworkforce/trajectories/completed/2026-08/traj_iv95g9euppbw.trace.json @@ -0,0 +1,42 @@ +{ + "version": "1.0.0", + "id": "f1c6df14-5543-4d93-ba01-81d22ed56675", + "timestamp": "2026-08-26T00:39:15.360Z", + "trajectory": "traj_iv95g9euppbw", + "files": [ + { + "path": ".agentworkforce/trajectories/active/traj_iv95g9euppbw/trajectory.json", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 50, + "revision": "8e46228dea1e0ab93b37e2fb5c9dc2754f2eb323" + } + ] + } + ] + }, + { + "path": "tests/relayflows/cases/1591-application-ack-reconnect/run.mjs", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 85, + "end_line": 113, + "revision": "8e46228dea1e0ab93b37e2fb5c9dc2754f2eb323" + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/.agentworkforce/trajectories/completed/2026-08/traj_iv95g9euppbw/summary.md b/.agentworkforce/trajectories/completed/2026-08/traj_iv95g9euppbw/summary.md new file mode 100644 index 000000000..ff7173bd7 --- /dev/null +++ b/.agentworkforce/trajectories/completed/2026-08/traj_iv95g9euppbw/summary.md @@ -0,0 +1,39 @@ +# Trajectory: Stabilize PR 1610 proof shutdown + +> **Status:** ✅ Completed +> **Task:** PR-1610 +> **Confidence:** 95% +> **Started:** August 26, 2026 at 02:38 AM +> **Completed:** August 26, 2026 at 02:39 AM + +--- + +## Summary + +Removed the reconnect observation race from the application-ack RelayFlow harness. + +**Approach:** Kept the first socket responsive until closure, accepted the second connection deterministically, then aborted the client only for harness cleanup. + +--- + +## Key Decisions + +### Keep the reconnected loopback WebSocket alive until the harness captures the reconnect +- **Chose:** Keep the reconnected loopback WebSocket alive until the harness captures the reconnect +- **Reasoning:** The harness only needs to deterministically observe a second public node-control connection; aborting the client afterward avoids an irrelevant shutdown race. + +--- + +## Chapters + +### 1. Work +*Agent: default* + +- Keep the reconnected loopback WebSocket alive until the harness captures the reconnect: Keep the reconnected loopback WebSocket alive until the harness captures the reconnect + +--- + +## Artifacts + +**Commits:** 8e46228de +**Files changed:** 2 diff --git a/.agentworkforce/trajectories/completed/2026-08/traj_iv95g9euppbw/trajectory.json b/.agentworkforce/trajectories/completed/2026-08/traj_iv95g9euppbw/trajectory.json new file mode 100644 index 000000000..839644caf --- /dev/null +++ b/.agentworkforce/trajectories/completed/2026-08/traj_iv95g9euppbw/trajectory.json @@ -0,0 +1,63 @@ +{ + "id": "traj_iv95g9euppbw", + "version": 1, + "task": { + "title": "Stabilize PR 1610 proof shutdown", + "source": { + "system": "plain", + "id": "PR-1610" + } + }, + "status": "completed", + "startedAt": "2026-08-26T00:38:05.964Z", + "completedAt": "2026-08-26T00:39:14.893Z", + "agents": [ + { + "name": "default", + "role": "lead", + "joinedAt": "2026-08-26T00:38:21.120Z" + } + ], + "chapters": [ + { + "id": "chap_f2ig35c45rtp", + "title": "Work", + "agentName": "default", + "startedAt": "2026-08-26T00:38:21.120Z", + "endedAt": "2026-08-26T00:39:14.893Z", + "events": [ + { + "ts": 1787704701149, + "type": "decision", + "content": "Keep the reconnected loopback WebSocket alive until the harness captures the reconnect: Keep the reconnected loopback WebSocket alive until the harness captures the reconnect", + "raw": { + "question": "Keep the reconnected loopback WebSocket alive until the harness captures the reconnect", + "chosen": "Keep the reconnected loopback WebSocket alive until the harness captures the reconnect", + "alternatives": [], + "reasoning": "The harness only needs to deterministically observe a second public node-control connection; aborting the client afterward avoids an irrelevant shutdown race." + }, + "significance": "high" + } + ] + } + ], + "retrospective": { + "summary": "Removed the reconnect observation race from the application-ack RelayFlow harness.", + "approach": "Kept the first socket responsive until closure, accepted the second connection deterministically, then aborted the client only for harness cleanup.", + "confidence": 0.95 + }, + "commits": [ + "8e46228de" + ], + "filesChanged": [ + ".agentworkforce/trajectories/active/traj_iv95g9euppbw/trajectory.json", + "tests/relayflows/cases/1591-application-ack-reconnect/run.mjs" + ], + "projectId": "AgentWorkforce/relay", + "tags": [], + "_trace": { + "startRef": "4a3e03106837e71d9d1f223ae727300449e68431", + "endRef": "8e46228dea1e0ab93b37e2fb5c9dc2754f2eb323", + "traceId": "f1c6df14-5543-4d93-ba01-81d22ed56675" + } +} \ No newline at end of file diff --git a/.agentworkforce/trajectories/completed/2026-08/traj_l0imiqugdc9k.trace.json b/.agentworkforce/trajectories/completed/2026-08/traj_l0imiqugdc9k.trace.json new file mode 100644 index 000000000..0a7b68a00 --- /dev/null +++ b/.agentworkforce/trajectories/completed/2026-08/traj_l0imiqugdc9k.trace.json @@ -0,0 +1,52 @@ +{ + "version": "1.0.0", + "id": "f0436409-faf2-4c70-8a78-761f272534b1", + "timestamp": "2026-08-26T00:37:34.286Z", + "trajectory": "traj_l0imiqugdc9k", + "files": [ + { + "path": ".agentworkforce/trajectories/active/traj_l0imiqugdc9k/trajectory.json", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 50, + "revision": "dbdea083abe058d5f94574b893103191f59d6c8b" + } + ] + } + ] + }, + { + "path": "crates/broker/src/node_control.rs", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1729, + "end_line": 1735, + "revision": "dbdea083abe058d5f94574b893103191f59d6c8b" + }, + { + "start_line": 1738, + "end_line": 1779, + "revision": "dbdea083abe058d5f94574b893103191f59d6c8b" + }, + { + "start_line": 3287, + "end_line": 3306, + "revision": "dbdea083abe058d5f94574b893103191f59d6c8b" + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/.agentworkforce/trajectories/completed/2026-08/traj_l0imiqugdc9k/summary.md b/.agentworkforce/trajectories/completed/2026-08/traj_l0imiqugdc9k/summary.md new file mode 100644 index 000000000..cbf637810 --- /dev/null +++ b/.agentworkforce/trajectories/completed/2026-08/traj_l0imiqugdc9k/summary.md @@ -0,0 +1,39 @@ +# Trajectory: Address PR 1610 ordered liveness probe review + +> **Status:** ✅ Completed +> **Task:** PR-1610 +> **Confidence:** 95% +> **Started:** August 26, 2026 at 02:34 AM +> **Completed:** August 26, 2026 at 02:37 AM + +--- + +## Summary + +Preserved newer inventory probes across older acknowledgements and added an A-acknowledged/B-rejected regression. + +**Approach:** Tracked inventory probe IDs in send order and covered the ordered acknowledgement/rejection sequence with a focused regression test. + +--- + +## Key Decisions + +### Track pending inventory probes in send order and drain only through the acknowledged probe +- **Chose:** Track pending inventory probes in send order and drain only through the acknowledged probe +- **Reasoning:** Relaycast serializes node control work; acknowledging A proves older probes but not newer B, whose later rejection must still force reconnect. + +--- + +## Chapters + +### 1. Work +*Agent: default* + +- Track pending inventory probes in send order and drain only through the acknowledged probe: Track pending inventory probes in send order and drain only through the acknowledged probe + +--- + +## Artifacts + +**Commits:** dbdea083a +**Files changed:** 2 diff --git a/.agentworkforce/trajectories/completed/2026-08/traj_l0imiqugdc9k/trajectory.json b/.agentworkforce/trajectories/completed/2026-08/traj_l0imiqugdc9k/trajectory.json new file mode 100644 index 000000000..af9b60959 --- /dev/null +++ b/.agentworkforce/trajectories/completed/2026-08/traj_l0imiqugdc9k/trajectory.json @@ -0,0 +1,63 @@ +{ + "id": "traj_l0imiqugdc9k", + "version": 1, + "task": { + "title": "Address PR 1610 ordered liveness probe review", + "source": { + "system": "plain", + "id": "PR-1610" + } + }, + "status": "completed", + "startedAt": "2026-08-26T00:34:00.303Z", + "completedAt": "2026-08-26T00:37:33.497Z", + "agents": [ + { + "name": "default", + "role": "lead", + "joinedAt": "2026-08-26T00:34:05.830Z" + } + ], + "chapters": [ + { + "id": "chap_osj8m7sxxbfj", + "title": "Work", + "agentName": "default", + "startedAt": "2026-08-26T00:34:05.830Z", + "endedAt": "2026-08-26T00:37:33.497Z", + "events": [ + { + "ts": 1787704445834, + "type": "decision", + "content": "Track pending inventory probes in send order and drain only through the acknowledged probe: Track pending inventory probes in send order and drain only through the acknowledged probe", + "raw": { + "question": "Track pending inventory probes in send order and drain only through the acknowledged probe", + "chosen": "Track pending inventory probes in send order and drain only through the acknowledged probe", + "alternatives": [], + "reasoning": "Relaycast serializes node control work; acknowledging A proves older probes but not newer B, whose later rejection must still force reconnect." + }, + "significance": "high" + } + ] + } + ], + "retrospective": { + "summary": "Preserved newer inventory probes across older acknowledgements and added an A-acknowledged/B-rejected regression.", + "approach": "Tracked inventory probe IDs in send order and covered the ordered acknowledgement/rejection sequence with a focused regression test.", + "confidence": 0.95 + }, + "commits": [ + "dbdea083a" + ], + "filesChanged": [ + ".agentworkforce/trajectories/active/traj_l0imiqugdc9k/trajectory.json", + "crates/broker/src/node_control.rs" + ], + "projectId": "AgentWorkforce/relay", + "tags": [], + "_trace": { + "startRef": "5971b921932d6a0f0baffafa0b1fdb9239bd3fc5", + "endRef": "dbdea083abe058d5f94574b893103191f59d6c8b", + "traceId": "f0436409-faf2-4c70-8a78-761f272534b1" + } +} \ No newline at end of file diff --git a/.agentworkforce/trajectories/completed/2026-08/traj_lg1p32dy7d6e.trace.json b/.agentworkforce/trajectories/completed/2026-08/traj_lg1p32dy7d6e.trace.json new file mode 100644 index 000000000..cbb1355de --- /dev/null +++ b/.agentworkforce/trajectories/completed/2026-08/traj_lg1p32dy7d6e.trace.json @@ -0,0 +1,879 @@ +{ + "version": "1.0.0", + "id": "5fec40ab-a5ed-4f06-9183-e23ff31366a9", + "timestamp": "2026-08-25T22:23:32.130Z", + "trajectory": "traj_lg1p32dy7d6e", + "files": [ + { + "path": ".agentworkforce/trajectories/completed/2026-08/traj_yr3f5u3r6zdz/summary.md", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 54, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + } + ] + } + ] + }, + { + "path": ".agentworkforce/trajectories/completed/2026-08/traj_yr3f5u3r6zdz/trajectory.json", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 128, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + } + ] + } + ] + }, + { + "path": ".github/pull_request_template.md", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 7, + "end_line": 21, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + } + ] + } + ] + }, + { + "path": ".github/workflows/relayflow-pr-proof.yml", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 113, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + } + ] + } + ] + }, + { + "path": ".gitignore", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 81, + "end_line": 87, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + }, + { + "start_line": 92, + "end_line": 98, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + } + ] + } + ] + }, + { + "path": "CHANGELOG.md", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 12, + "end_line": 23, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + } + ] + } + ] + }, + { + "path": "package.json", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 6, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + } + ] + } + ] + }, + { + "path": "packages/brand/package.json", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 6, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + } + ] + } + ] + }, + { + "path": "packages/broker-darwin-arm64/package.json", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 6, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + } + ] + } + ] + }, + { + "path": "packages/broker-darwin-x64/package.json", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 6, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + } + ] + } + ] + }, + { + "path": "packages/broker-linux-arm64/package.json", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 6, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + } + ] + } + ] + }, + { + "path": "packages/broker-linux-x64/package.json", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 6, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + } + ] + } + ] + }, + { + "path": "packages/broker-win32-x64/package.json", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 6, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + } + ] + } + ] + }, + { + "path": "packages/cli/package.json", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 6, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + }, + { + "start_line": 43, + "end_line": 56, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + } + ] + } + ] + }, + { + "path": "packages/cloud/package.json", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 6, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + }, + { + "start_line": 62, + "end_line": 68, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + } + ] + } + ] + }, + { + "path": "packages/cloud/src/api-client.test.ts", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 41, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + } + ] + } + ] + }, + { + "path": "packages/cloud/src/api-client.ts", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 42, + "end_line": 93, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + }, + { + "start_line": 288, + "end_line": 294, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + } + ] + } + ] + }, + { + "path": "packages/cloud/src/auth.test.ts", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 219, + "end_line": 243, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + } + ] + } + ] + }, + { + "path": "packages/cloud/src/auth.ts", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 825, + "end_line": 840, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + } + ] + } + ] + }, + { + "path": "packages/cloud/src/workflows.test.ts", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 9, + "end_line": 19, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + }, + { + "start_line": 517, + "end_line": 553, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + }, + { + "start_line": 797, + "end_line": 822, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + } + ] + } + ] + }, + { + "path": "packages/cloud/src/workflows.ts", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 7, + "end_line": 13, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + }, + { + "start_line": 59, + "end_line": 66, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + }, + { + "start_line": 241, + "end_line": 247, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + }, + { + "start_line": 272, + "end_line": 281, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + }, + { + "start_line": 287, + "end_line": 312, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + }, + { + "start_line": 410, + "end_line": 416, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + }, + { + "start_line": 418, + "end_line": 423, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + }, + { + "start_line": 449, + "end_line": 455, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + }, + { + "start_line": 485, + "end_line": 491, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + }, + { + "start_line": 508, + "end_line": 515, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + }, + { + "start_line": 534, + "end_line": 541, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + }, + { + "start_line": 556, + "end_line": 566, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + }, + { + "start_line": 583, + "end_line": 589, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + }, + { + "start_line": 594, + "end_line": 600, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + }, + { + "start_line": 622, + "end_line": 633, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + }, + { + "start_line": 640, + "end_line": 648, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + }, + { + "start_line": 666, + "end_line": 690, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + } + ] + } + ] + }, + { + "path": "packages/config/package.json", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 6, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + } + ] + } + ] + }, + { + "path": "packages/evals/package.json", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 6, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + }, + { + "start_line": 71, + "end_line": 78, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + } + ] + } + ] + }, + { + "path": "packages/fleet/package.json", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 6, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + }, + { + "start_line": 26, + "end_line": 33, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + } + ] + } + ] + }, + { + "path": "packages/harness-driver/package.json", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 6, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + }, + { + "start_line": 56, + "end_line": 71, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + } + ] + } + ] + }, + { + "path": "packages/harnesses/package.json", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 6, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + }, + { + "start_line": 26, + "end_line": 33, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + } + ] + } + ] + }, + { + "path": "packages/integration-prompts/package.json", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 6, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + } + ] + } + ] + }, + { + "path": "packages/policy/package.json", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 6, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + }, + { + "start_line": 25, + "end_line": 31, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + } + ] + } + ] + }, + { + "path": "packages/sdk-py/pyproject.toml", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 4, + "end_line": 10, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + } + ] + } + ] + }, + { + "path": "packages/sdk/package.json", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 6, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + } + ] + } + ] + }, + { + "path": "packages/session/package.json", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 6, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + } + ] + } + ] + }, + { + "path": "packages/utils/package.json", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 6, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + }, + { + "start_line": 109, + "end_line": 115, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + } + ] + } + ] + }, + { + "path": "scripts/pr-proof/cloud-storage.mjs", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 101, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + } + ] + } + ] + }, + { + "path": "scripts/pr-proof/contract.mjs", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 326, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + } + ] + } + ] + }, + { + "path": "scripts/pr-proof/prepare.mjs", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 253, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + } + ] + } + ] + }, + { + "path": "scripts/pr-proof/process-runner.mjs", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 198, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + } + ] + } + ] + }, + { + "path": "scripts/pr-proof/report-status.mjs", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 199, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + } + ] + } + ] + }, + { + "path": "scripts/pr-proof/run-arm.mjs", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 210, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + } + ] + } + ] + }, + { + "path": "scripts/pr-proof/run-cloud.mjs", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 310, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + } + ] + } + ] + }, + { + "path": "scripts/pr-proof/verify-evidence.mjs", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 104, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + } + ] + } + ] + }, + { + "path": "tests/fixtures/pr-proof-contract.test.ts", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 750, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + } + ] + } + ] + }, + { + "path": "tests/relayflows/cases/README.md", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 147, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + } + ] + } + ] + }, + { + "path": "workflows/pr-proof.ts", + "conversations": [ + { + "contributor": { + "type": "ai" + }, + "ranges": [ + { + "start_line": 1, + "end_line": 88, + "revision": "054a3284b78d379f62feb03899cdf238aa539a54" + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/.agentworkforce/trajectories/completed/2026-08/traj_lg1p32dy7d6e/summary.md b/.agentworkforce/trajectories/completed/2026-08/traj_lg1p32dy7d6e/summary.md new file mode 100644 index 000000000..5b884c8a3 --- /dev/null +++ b/.agentworkforce/trajectories/completed/2026-08/traj_lg1p32dy7d6e/summary.md @@ -0,0 +1,40 @@ +# Trajectory: Fix PR 1610 reconnect backoff and add RelayFlow proof + +> **Status:** ✅ Completed +> **Task:** PR-1610 +> **Confidence:** 78% +> **Started:** August 25, 2026 at 09:57 PM +> **Completed:** August 25, 2026 at 10:23 PM + +--- + +## Summary + +Rebased PR 1610 onto main, preserved exponential backoff until successful correlated inventory acknowledgement, added readiness/error regressions, and added the single 1591 application-ack reconnect RelayFlow case with a head-authored loopback production-client runner. + +**Approach:** Standard approach + +--- + +## Key Decisions + +### Reset reconnect backoff only after a successful correlated inventory.sync reply +- **Chose:** Reset reconnect backoff only after a successful correlated inventory.sync reply +- **Reasoning:** A WebSocket handshake or correlated error proves transport/application reachability but does not establish a healthy ready control session; pre-ready failures must retain exponential outage backoff. + +--- + +## Chapters + +### 1. Work +*Agent: default* + +- Reset reconnect backoff only after a successful correlated inventory.sync reply: Reset reconnect backoff only after a successful correlated inventory.sync reply +- Implementation and single RelayFlow case are complete; Rust library cargo check and formatting pass. Full lib-test and discriminating runner execution are locally resource-blocked because rustc is SIGKILLed during code generation in the 5 GiB filesystem environment. + +--- + +## Artifacts + +**Commits:** 054a3284b, 10fc5ce1c, 639158c8c +**Files changed:** 43 diff --git a/.agentworkforce/trajectories/completed/2026-08/traj_lg1p32dy7d6e/trajectory.json b/.agentworkforce/trajectories/completed/2026-08/traj_lg1p32dy7d6e/trajectory.json new file mode 100644 index 000000000..5886690fd --- /dev/null +++ b/.agentworkforce/trajectories/completed/2026-08/traj_lg1p32dy7d6e/trajectory.json @@ -0,0 +1,118 @@ +{ + "id": "traj_lg1p32dy7d6e", + "version": 1, + "task": { + "title": "Fix PR 1610 reconnect backoff and add RelayFlow proof", + "source": { + "system": "plain", + "id": "PR-1610" + } + }, + "status": "completed", + "startedAt": "2026-08-25T21:57:29.360Z", + "completedAt": "2026-08-25T22:23:32.094Z", + "agents": [ + { + "name": "default", + "role": "lead", + "joinedAt": "2026-08-25T22:23:18.074Z" + } + ], + "chapters": [ + { + "id": "chap_hte7kuu3p2rl", + "title": "Work", + "agentName": "default", + "startedAt": "2026-08-25T22:23:18.074Z", + "endedAt": "2026-08-25T22:23:32.094Z", + "events": [ + { + "ts": 1787696598075, + "type": "decision", + "content": "Reset reconnect backoff only after a successful correlated inventory.sync reply: Reset reconnect backoff only after a successful correlated inventory.sync reply", + "raw": { + "question": "Reset reconnect backoff only after a successful correlated inventory.sync reply", + "chosen": "Reset reconnect backoff only after a successful correlated inventory.sync reply", + "alternatives": [], + "reasoning": "A WebSocket handshake or correlated error proves transport/application reachability but does not establish a healthy ready control session; pre-ready failures must retain exponential outage backoff." + }, + "significance": "high" + }, + { + "ts": 1787696599112, + "type": "reflection", + "content": "Implementation and single RelayFlow case are complete; Rust library cargo check and formatting pass. Full lib-test and discriminating runner execution are locally resource-blocked because rustc is SIGKILLed during code generation in the 5 GiB filesystem environment.", + "raw": { + "confidence": 0.78 + }, + "significance": "high", + "tags": [ + "confidence:0.78" + ] + } + ] + } + ], + "retrospective": { + "summary": "Rebased PR 1610 onto main, preserved exponential backoff until successful correlated inventory acknowledgement, added readiness/error regressions, and added the single 1591 application-ack reconnect RelayFlow case with a head-authored loopback production-client runner.", + "approach": "Standard approach", + "confidence": 0.78 + }, + "commits": [ + "054a3284b", + "10fc5ce1c", + "639158c8c" + ], + "filesChanged": [ + ".agentworkforce/trajectories/completed/2026-08/traj_yr3f5u3r6zdz/summary.md", + ".agentworkforce/trajectories/completed/2026-08/traj_yr3f5u3r6zdz/trajectory.json", + ".github/pull_request_template.md", + ".github/workflows/relayflow-pr-proof.yml", + ".gitignore", + "CHANGELOG.md", + "package.json", + "packages/brand/package.json", + "packages/broker-darwin-arm64/package.json", + "packages/broker-darwin-x64/package.json", + "packages/broker-linux-arm64/package.json", + "packages/broker-linux-x64/package.json", + "packages/broker-win32-x64/package.json", + "packages/cli/package.json", + "packages/cloud/package.json", + "packages/cloud/src/api-client.test.ts", + "packages/cloud/src/api-client.ts", + "packages/cloud/src/auth.test.ts", + "packages/cloud/src/auth.ts", + "packages/cloud/src/workflows.test.ts", + "packages/cloud/src/workflows.ts", + "packages/config/package.json", + "packages/evals/package.json", + "packages/fleet/package.json", + "packages/harness-driver/package.json", + "packages/harnesses/package.json", + "packages/integration-prompts/package.json", + "packages/policy/package.json", + "packages/sdk-py/pyproject.toml", + "packages/sdk/package.json", + "packages/session/package.json", + "packages/utils/package.json", + "scripts/pr-proof/cloud-storage.mjs", + "scripts/pr-proof/contract.mjs", + "scripts/pr-proof/prepare.mjs", + "scripts/pr-proof/process-runner.mjs", + "scripts/pr-proof/report-status.mjs", + "scripts/pr-proof/run-arm.mjs", + "scripts/pr-proof/run-cloud.mjs", + "scripts/pr-proof/verify-evidence.mjs", + "tests/fixtures/pr-proof-contract.test.ts", + "tests/relayflows/cases/README.md", + "workflows/pr-proof.ts" + ], + "projectId": "AgentWorkforce/relay", + "tags": [], + "_trace": { + "startRef": "44ddb17dcafcdefa347cb31bb132a3dc4e9aa846", + "endRef": "054a3284b78d379f62feb03899cdf238aa539a54", + "traceId": "5fec40ab-a5ed-4f06-9183-e23ff31366a9" + } +} \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index d6b7c92f3..531a77c7b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,12 @@ All notable changes to Agent Relay will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [Unreleased - Patch] + +### Fixed + +- Fleet nodes now reconnect when the control-plane application stops acknowledging requests even if the WebSocket remains open. +- Offline fleet node records now report the active-agent count as unknown instead of zero. ## [11.8.4] - 2026-08-25 diff --git a/crates/broker/src/node_control.rs b/crates/broker/src/node_control.rs index 02707eaa9..8b002d01f 100644 --- a/crates/broker/src/node_control.rs +++ b/crates/broker/src/node_control.rs @@ -46,6 +46,13 @@ const INVENTORY_REFRESH_INTERVAL: Duration = Duration::from_secs(60); /// heartbeat intervals so three consecutive lost pings are tolerated before a /// reconnect. const READ_IDLE_TIMEOUT: Duration = Duration::from_secs(48); +/// Bound every node-control WebSocket handshake so a half-open TCP/TLS path +/// cannot stall the reconnect loop before it reaches the capped backoff. +const CONNECT_TIMEOUT: Duration = Duration::from_secs(15); +/// Bound each control-plane write. A full send buffer otherwise parks the +/// entire select loop and prevents both transport and application deadlines +/// from being observed. +const WRITE_TIMEOUT: Duration = Duration::from_secs(10); const INITIAL_RECONNECT_DELAY: Duration = Duration::from_secs(1); const MAX_RECONNECT_DELAY: Duration = Duration::from_secs(30); const REGISTER_AGENT_PENDING_TTL: Duration = Duration::from_secs(300); @@ -1609,13 +1616,19 @@ pub(crate) async fn run_node_control_client( if matches!(result, ControlRunResult::Shutdown) { return; } - if matches!(result, ControlRunResult::Disconnected) { - // A real connection was established and then dropped, so the current - // token authenticated successfully. Reset the 401 counter only here — - // NOT after a successful re-mint — so a tight loop where each mint - // succeeds but the engine keeps 401-ing `/v1/node/ws` still - // accumulates toward the cap instead of resetting on every iteration. + let application_ready = matches!( + result, + ControlRunResult::Disconnected { + application_ready: true + } + ); + if application_ready { + // A correlated inventory.sync acknowledgement proves the engine's + // application processed this session. Reset outage state only after + // that proof — a transport handshake followed by a pre-ready drop + // must preserve both the 401 history and exponential backoff. consecutive_unauthorized = 0; + reconnect_delay = INITIAL_RECONNECT_DELAY; } if matches!(result, ControlRunResult::Unauthorized) { // The engine rejected our current node token. Re-mint a fresh one @@ -1660,15 +1673,36 @@ pub(crate) async fn run_node_control_client( ); } } + let transition = match result { + ControlRunResult::ConnectFailed => "connect_failed", + ControlRunResult::Disconnected { .. } => "disconnected", + ControlRunResult::Unauthorized => "unauthorized", + ControlRunResult::Shutdown => unreachable!("shutdown returned above"), + }; + tracing::warn!( + target = "relay_broker::fleet", + node_id = %config.node_id, + transition, + reconnect_delay_ms = reconnect_delay.as_millis(), + "node-control transition: unhealthy; reconnect scheduled" + ); let _ = event_tx.send(FleetControlEvent::Disconnected).await; tokio::time::sleep(reconnect_delay).await; - reconnect_delay = (reconnect_delay * 2).min(MAX_RECONNECT_DELAY); + if !application_ready { + reconnect_delay = (reconnect_delay * 2).min(MAX_RECONNECT_DELAY); + } } } #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum ControlRunResult { - Disconnected, + /// The WebSocket session never completed its transport handshake. + ConnectFailed, + Disconnected { + /// True only after a correlated inventory.sync acknowledgement proved + /// that the engine application processed this connection. + application_ready: bool, + }, /// The `/v1/node/ws` handshake was rejected with HTTP 401/Unauthorized, /// i.e. the current node token is stale or scoped to a different /// workspace/engine and must be re-minted before retrying. @@ -1687,6 +1721,66 @@ fn connect_error_is_unauthorized(error: &tokio_tungstenite::tungstenite::Error) ) } +/// Application-level liveness is proven by an engine reply to the correlated +/// `inventory.sync` request the broker already sends periodically. WebSocket +/// pong traffic is deliberately excluded: an intermediary or a socket task can +/// keep answering pings even after the node-control application stops applying +/// heartbeats and inventory. +struct ApplicationLiveness { + deadline: Duration, + last_acknowledged: Instant, + pending_inventory_syncs: VecDeque, + ready: bool, +} + +impl ApplicationLiveness { + fn new(deadline: Duration) -> Self { + Self { + deadline, + last_acknowledged: Instant::now(), + pending_inventory_syncs: VecDeque::new(), + ready: false, + } + } + + fn track_inventory_sync(&mut self, id: String) { + self.pending_inventory_syncs.push_back(id); + } + + /// Returns `Some(true)` for the first successful application acknowledgement, + /// `Some(false)` for later ones, and `None` for unrelated replies. + fn acknowledge(&mut self, id: &str) -> Option { + let acknowledged_index = self + .pending_inventory_syncs + .iter() + .position(|pending_id| pending_id == id)?; + self.pending_inventory_syncs.drain(..=acknowledged_index); + let became_ready = !self.ready; + self.ready = true; + self.last_acknowledged = Instant::now(); + // Relaycast serializes control work for a node, so this reply proves all + // older probes were processed. Preserve newer probes: their later error + // replies must still replace an unhealthy control session. + Some(became_ready) + } + + fn reject(&mut self, id: &str) -> bool { + let Some(index) = self + .pending_inventory_syncs + .iter() + .position(|pending_id| pending_id == id) + else { + return false; + }; + self.pending_inventory_syncs.remove(index); + true + } + + fn idle(&self) -> Duration { + self.last_acknowledged.elapsed() + } +} + async fn run_connected_once( config: &FleetControlConfig, command_rx: &mut mpsc::Receiver, @@ -1697,10 +1791,10 @@ async fn run_connected_once( inventory_refresh_interval: Duration, ) -> ControlRunResult { let Some(mut node_register) = registration.clone() else { - return ControlRunResult::Disconnected; + return ControlRunResult::ConnectFailed; }; let Some(node_token) = config.node_token.as_deref() else { - return ControlRunResult::Disconnected; + return ControlRunResult::ConnectFailed; }; // A fresh provider instance per connection: reconnecting with a new @@ -1718,7 +1812,7 @@ async fn run_connected_once( Ok(request) => request, Err(error) => { tracing::warn!(target = "relay_broker::fleet", error = %error, "invalid fleet node ws url"); - return ControlRunResult::Disconnected; + return ControlRunResult::ConnectFailed; } }; let header = format!("Bearer {}", node_token.trim()); @@ -1728,7 +1822,7 @@ async fn run_connected_once( } Err(error) => { tracing::warn!(target = "relay_broker::fleet", error = %error, "invalid fleet node token header"); - return ControlRunResult::Disconnected; + return ControlRunResult::ConnectFailed; } } @@ -1747,19 +1841,47 @@ async fn run_connected_once( } } - let (ws, _) = match tokio_tungstenite::connect_async(request).await { - Ok(connected) => connected, - Err(error) => { + let (ws, _) = match tokio::time::timeout( + CONNECT_TIMEOUT, + tokio_tungstenite::connect_async(request), + ) + .await + { + Ok(Ok(connected)) => connected, + Ok(Err(error)) => { tracing::warn!(target = "relay_broker::fleet", url = %config.ws_url, error = %error, "fleet node ws connect failed"); if connect_error_is_unauthorized(&error) { return ControlRunResult::Unauthorized; } - return ControlRunResult::Disconnected; + return ControlRunResult::ConnectFailed; + } + Err(_) => { + tracing::warn!( + target = "relay_broker::fleet", + url = %config.ws_url, + timeout_secs = CONNECT_TIMEOUT.as_secs(), + "fleet node ws connect attempt timed out" + ); + return ControlRunResult::ConnectFailed; } }; + tracing::info!( + target = "relay_broker::fleet", + node_id = %config.node_id, + "node-control transition: transport connected; awaiting application acknowledgement" + ); let _ = event_tx.send(FleetControlEvent::Connected).await; let (mut sink, mut stream) = ws.split(); let mut pending_agent_registrations: HashMap = HashMap::new(); + let read_idle_timeout = config.read_idle_timeout.unwrap_or(READ_IDLE_TIMEOUT); + // inventory.sync is the existing request/reply application probe. Allow two + // complete refresh periods before declaring it unacknowledged, while never + // making the application deadline tighter than the transport deadline. + let application_liveness_timeout = inventory_refresh_interval + .checked_mul(2) + .unwrap_or(Duration::MAX) + .max(read_idle_timeout); + let mut application_liveness = ApplicationLiveness::new(application_liveness_timeout); if send_wire( &mut sink, @@ -1768,10 +1890,21 @@ async fn run_connected_once( .await .is_err() { - return ControlRunResult::Disconnected; + return ControlRunResult::Disconnected { + application_ready: false, + }; } - if !send_inventory_sync(&mut sink, inventory, &mut pending_agent_registrations).await { - return ControlRunResult::Disconnected; + if !send_inventory_sync( + &mut sink, + inventory, + &mut pending_agent_registrations, + &mut application_liveness, + ) + .await + { + return ControlRunResult::Disconnected { + application_ready: false, + }; } if send_wire( &mut sink, @@ -1780,13 +1913,14 @@ async fn run_connected_once( .await .is_err() { - return ControlRunResult::Disconnected; + return ControlRunResult::Disconnected { + application_ready: false, + }; } // The idle check runs on the heartbeat tick, so the tick must be shorter // than the window it polices; an overridden (test) window keeps that ratio. - let read_idle_timeout_value = config.read_idle_timeout.unwrap_or(READ_IDLE_TIMEOUT); - let mut heartbeat = tokio::time::interval(HEARTBEAT_INTERVAL.min(read_idle_timeout_value / 4)); + let mut heartbeat = tokio::time::interval(HEARTBEAT_INTERVAL.min(read_idle_timeout / 4)); heartbeat.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); let mut inventory_refresh = tokio::time::interval(inventory_refresh_interval); inventory_refresh.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); @@ -1794,7 +1928,6 @@ async fn run_connected_once( // already renewed the lease, so schedule the first refresh one full period // from now instead of duplicating it on connection setup. inventory_refresh.tick().await; - let read_idle_timeout = read_idle_timeout_value; let mut last_inbound = Instant::now(); loop { @@ -1809,7 +1942,7 @@ async fn run_connected_once( node_register = next.clone(); *registration = Some(next.clone()); if send_wire(&mut sink, &BrokerToRelaycast::NodeRegister(next)).await.is_err() { - return ControlRunResult::Disconnected; + return ControlRunResult::Disconnected { application_ready: application_liveness.ready }; } } Some(FleetControlCommand::UpdateInventory(next)) => { @@ -1818,10 +1951,11 @@ async fn run_connected_once( &mut sink, inventory, &mut pending_agent_registrations, + &mut application_liveness, ) .await { - return ControlRunResult::Disconnected; + return ControlRunResult::Disconnected { application_ready: application_liveness.ready }; } } Some(FleetControlCommand::UpdateLoad(next)) => { @@ -1829,12 +1963,12 @@ async fn run_connected_once( } Some(FleetControlCommand::HeartbeatNow) => { if send_wire(&mut sink, &BrokerToRelaycast::NodeHeartbeat(load.heartbeat(&node_register))).await.is_err() { - return ControlRunResult::Disconnected; + return ControlRunResult::Disconnected { application_ready: application_liveness.ready }; } } Some(FleetControlCommand::Send(message)) => { if send_wire(&mut sink, &message).await.is_err() { - return ControlRunResult::Disconnected; + return ControlRunResult::Disconnected { application_ready: application_liveness.ready }; } } Some(FleetControlCommand::RegisterAgent { mut request, reply }) => { @@ -1852,7 +1986,7 @@ async fn run_connected_once( ); if send_wire(&mut sink, &BrokerToRelaycast::AgentRegister(request)).await.is_err() { drain_agent_registrations(&mut pending_agent_registrations, "node_control_disconnected"); - return ControlRunResult::Disconnected; + return ControlRunResult::Disconnected { application_ready: application_liveness.ready }; } } Some(FleetControlCommand::Shutdown) | None => { @@ -1880,17 +2014,31 @@ async fn run_connected_once( "no inbound node-control frame within the read-idle window; reconnecting" ); drain_agent_registrations(&mut pending_agent_registrations, "node_control_disconnected"); - return ControlRunResult::Disconnected; + return ControlRunResult::Disconnected { application_ready: application_liveness.ready }; + } + let application_idle = application_liveness.idle(); + if application_idle >= application_liveness.deadline { + tracing::warn!( + target = "relay_broker::fleet", + node_id = %config.node_id, + application_idle_ms = application_idle.as_millis(), + application_deadline_ms = application_liveness.deadline.as_millis(), + transport_idle_ms = idle.as_millis(), + pending_inventory_syncs = application_liveness.pending_inventory_syncs.len(), + "node-control application acknowledgement deadline exceeded while transport remained active; reconnecting" + ); + drain_agent_registrations(&mut pending_agent_registrations, "node_control_disconnected"); + return ControlRunResult::Disconnected { application_ready: application_liveness.ready }; } if send_wire(&mut sink, &BrokerToRelaycast::NodeHeartbeat(load.heartbeat(&node_register))).await.is_err() { drain_agent_registrations(&mut pending_agent_registrations, "node_control_disconnected"); - return ControlRunResult::Disconnected; + return ControlRunResult::Disconnected { application_ready: application_liveness.ready }; } // Guarantees the peer owes us a frame every interval, so an idle // engine is distinguishable from a dead connection. - if sink.send(Message::Ping(Vec::new())).await.is_err() { + if send_ws_frame(&mut sink, Message::Ping(Vec::new())).await.is_err() { drain_agent_registrations(&mut pending_agent_registrations, "node_control_disconnected"); - return ControlRunResult::Disconnected; + return ControlRunResult::Disconnected { application_ready: application_liveness.ready }; } } _ = inventory_refresh.tick() => { @@ -1898,32 +2046,42 @@ async fn run_connected_once( &mut sink, inventory, &mut pending_agent_registrations, + &mut application_liveness, ) .await { - return ControlRunResult::Disconnected; + return ControlRunResult::Disconnected { application_ready: application_liveness.ready }; } } message = stream.next() => { let Some(message) = message else { drain_agent_registrations(&mut pending_agent_registrations, "node_control_disconnected"); - return ControlRunResult::Disconnected; + return ControlRunResult::Disconnected { application_ready: application_liveness.ready }; }; let message = match message { Ok(message) => message, Err(error) => { tracing::warn!(target = "relay_broker::fleet", error = %error, "fleet node ws read failed"); drain_agent_registrations(&mut pending_agent_registrations, "node_control_disconnected"); - return ControlRunResult::Disconnected; + return ControlRunResult::Disconnected { application_ready: application_liveness.ready }; } }; // Any frame proves the peer is still there — including the pong // answering our ping, which is the only traffic a healthy but // idle engine is guaranteed to send. last_inbound = Instant::now(); - if !handle_server_message(message, event_tx, &mut pending_agent_registrations, &mut sink).await { + if !handle_server_message( + message, + event_tx, + &mut pending_agent_registrations, + &mut application_liveness, + &config.node_id, + &mut sink, + ) + .await + { drain_agent_registrations(&mut pending_agent_registrations, "node_control_disconnected"); - return ControlRunResult::Disconnected; + return ControlRunResult::Disconnected { application_ready: application_liveness.ready }; } } } @@ -1934,16 +2092,18 @@ async fn send_inventory_sync( sink: &mut S, inventory: &[InventoryAgent], pending_agent_registrations: &mut HashMap, + application_liveness: &mut ApplicationLiveness, ) -> bool where S: Sink + Unpin, S::Error: std::error::Error + Send + Sync + 'static, { + let request_id = format!("inventory_sync_{}", Uuid::new_v4().simple()); if send_wire( sink, &BrokerToRelaycast::InventorySync(InventorySync { v: FLEET_WIRE_VERSION, - id: None, + id: Some(request_id.clone()), agents: inventory.to_vec(), }), ) @@ -1953,6 +2113,7 @@ where drain_agent_registrations(pending_agent_registrations, "node_control_disconnected"); return false; } + application_liveness.track_inventory_sync(request_id); true } @@ -1961,6 +2122,8 @@ async fn handle_server_message( message: Message, event_tx: &mpsc::Sender, pending_agent_registrations: &mut HashMap, + application_liveness: &mut ApplicationLiveness, + node_id: &str, sink: &mut S, ) -> bool where @@ -1970,7 +2133,21 @@ where match message { Message::Text(text) => match serde_json::from_str::(&text) { Ok(RelaycastToBroker::Reply(reply)) => { - complete_agent_registration(reply, pending_agent_registrations, sink).await + match application_liveness.acknowledge(&reply.id) { + Some(became_ready) => { + if became_ready { + tracing::info!( + target = "relay_broker::fleet", + node_id, + "node-control transition: application acknowledgement received; control link ready" + ); + } + true + } + None => { + complete_agent_registration(reply, pending_agent_registrations, sink).await + } + } } Ok(RelaycastToBroker::Error(error)) => { // Surface every engine rejection at error level. A node.register or @@ -1984,12 +2161,16 @@ where id = %error.id, "engine rejected a node control frame" ); + let rejected_liveness_probe = application_liveness.reject(&error.id); fail_agent_registration( &error.id, format!("{}: {}", error.code, error.message), pending_agent_registrations, ); - true + // A reply proves the application is responsive, but rejecting + // the authoritative inventory probe means the control session + // is not healthy enough to advertise; replace it immediately. + !rejected_liveness_probe } Ok(other) => event_tx .send(FleetControlEvent::Message(other)) @@ -2016,11 +2197,10 @@ where S::Error: std::error::Error + Send + Sync + 'static, { let request_id = reply.id.clone(); - // The engine replies to every node-control request (`node.register`, - // `inventory.sync`, ...) with a `reply` frame, but only `agent.register` - // replies correspond to a pending registration. Those non-agent replies - // carry a fresh engine-minted snowflake id (the broker sends those frames - // without an `id`), so they never match `request_id`. To stay robust we: + // The engine replies to node-control requests such as `node.register`, but + // only `agent.register` replies correspond to a pending registration. + // Correlated `inventory.sync` replies have already been consumed by the + // application-liveness tracker above. To stay robust we: // 1. match on the echoed request id (the happy path), then // 2. fall back to matching the validated reply `data.name` against a // pending entry (covers an engine that drops/regenerates the id), and @@ -2044,7 +2224,7 @@ where tracing::debug!( target = "relay_broker::fleet", id = %request_id, - "node-control reply did not match a pending agent.register (likely a node.register/inventory.sync reply)" + "node-control reply did not match a pending agent.register (likely a node.register reply)" ); return true; }; @@ -2141,8 +2321,28 @@ where S::Error: std::error::Error + Send + Sync + 'static, { let text = serde_json::to_string(message)?; - sink.send(Message::Text(text)).await?; - Ok(()) + send_ws_frame(sink, Message::Text(text)).await +} + +async fn send_ws_frame(sink: &mut S, message: Message) -> Result<()> +where + S: Sink + Unpin, + S::Error: std::error::Error + Send + Sync + 'static, +{ + match tokio::time::timeout(WRITE_TIMEOUT, sink.send(message)).await { + Ok(result) => { + result?; + Ok(()) + } + Err(_) => { + tracing::warn!( + target = "relay_broker::fleet", + timeout_secs = WRITE_TIMEOUT.as_secs(), + "fleet node websocket write timed out; treating control link as unhealthy" + ); + Err(anyhow::anyhow!("fleet node websocket write timed out")) + } + } } pub(crate) fn delivery_ack(agent: impl Into, up_to_seq: u64) -> BrokerToRelaycast { @@ -3071,6 +3271,36 @@ mod tests { assert!(seen.contains("dup")); } + #[test] + fn application_readiness_requires_successful_correlated_inventory_reply() { + let mut liveness = ApplicationLiveness::new(Duration::from_secs(1)); + liveness.track_inventory_sync("inventory-rejected".to_string()); + assert!(liveness.reject("inventory-rejected")); + assert!( + !liveness.ready, + "an error reply must not make the link ready" + ); + + liveness.track_inventory_sync("inventory-acknowledged".to_string()); + assert_eq!(liveness.acknowledge("unrelated"), None); + assert_eq!(liveness.acknowledge("inventory-acknowledged"), Some(true)); + assert!(liveness.ready); + } + + #[test] + fn acknowledged_probe_preserves_newer_probe_for_rejection() { + let mut liveness = ApplicationLiveness::new(Duration::from_secs(1)); + liveness.track_inventory_sync("inventory-a".to_string()); + liveness.track_inventory_sync("inventory-b".to_string()); + + assert_eq!(liveness.acknowledge("inventory-a"), Some(true)); + assert_eq!( + liveness.pending_inventory_syncs, + VecDeque::from(["inventory-b".to_string()]) + ); + assert!(liveness.reject("inventory-b")); + } + #[test] fn expire_agent_registrations_bounds_pending_map() { let created_at = Instant::now(); @@ -3803,7 +4033,12 @@ mod tests { .await .expect("mock node-control session should finish"); - assert_eq!(result, ControlRunResult::Disconnected); + assert_eq!( + result, + ControlRunResult::Disconnected { + application_ready: false + } + ); server.await.unwrap(); } @@ -3881,10 +4116,268 @@ mod tests { .await .expect("mock node-control session should finish"); - assert_eq!(result, ControlRunResult::Disconnected); + assert_eq!( + result, + ControlRunResult::Disconnected { + application_ready: false + } + ); server.await.unwrap(); } + /// Regression for relay#1591: transport traffic must not mask an application + /// control-plane failure. The peer keeps the TCP/WebSocket connection open, + /// continuously polls it (so tungstenite answers every WebSocket ping with a + /// pong), and drains every node frame, but never acknowledges an application + /// request. The node-control session must still declare the link dead. + #[tokio::test] + async fn node_control_disconnects_when_application_acks_stop_but_socket_stays_live() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let ws_url = format!("ws://{}/v1/node/ws", listener.local_addr().unwrap()); + let (_command_tx, mut command_rx) = mpsc::channel(4); + let (event_tx, _event_rx) = mpsc::channel(4); + let mut registration = Some(build_node_register( + &test_manifest(), + "node-test", + "host-test", + "broker/test", + None, + )); + let mut inventory = Vec::new(); + let mut load = FleetLoadSnapshot { + active_agents: 3, + max_agents: 4, + handlers_live: true, + active_agent_names: vec![ + "agent-a".to_string(), + "agent-b".to_string(), + "agent-c".to_string(), + ], + }; + + let server = tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + let mut ws = accept_async(stream).await.unwrap(); + let mut saw_inventory = false; + let mut saw_heartbeat = false; + + while let Some(frame) = ws.next().await { + let Ok(frame) = frame else { break }; + match frame { + Message::Text(text) => match serde_json::from_str::(&text) + .expect("valid node-control frame") + { + BrokerToRelaycast::InventorySync(_) => saw_inventory = true, + BrokerToRelaycast::NodeHeartbeat(heartbeat) => { + saw_heartbeat = true; + assert_eq!(heartbeat.active_agents, 3); + } + _ => {} + }, + Message::Close(_) => break, + _ => {} + } + } + + assert!(saw_inventory, "client must send an application request"); + assert!(saw_heartbeat, "client process must keep heartbeating"); + }); + + let result = tokio::time::timeout( + Duration::from_secs(2), + run_connected_once( + &FleetControlConfig { + ws_url, + node_token: Some("nt_test".to_string()), + node_id: "node-test".to_string(), + node_name: "host-test".to_string(), + broker_version: "broker/test".to_string(), + token_minter: None, + session_token: None, + read_idle_timeout: Some(Duration::from_millis(400)), + }, + &mut command_rx, + &event_tx, + &mut registration, + &mut inventory, + &mut load, + Duration::from_millis(100), + ), + ) + .await + .expect("application-level liveness deadline did not fire"); + + assert_eq!( + result, + ControlRunResult::Disconnected { + application_ready: false + } + ); + server.await.unwrap(); + } + + /// Must-not-fire control arm for the application deadline above. Under the + /// same 400ms deadline and 100ms probe cadence, correlated inventory replies + /// keep the session healthy for several complete deadline windows. + #[tokio::test] + async fn node_control_stays_connected_while_application_acks_continue() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let ws_url = format!("ws://{}/v1/node/ws", listener.local_addr().unwrap()); + let (command_tx, mut command_rx) = mpsc::channel(4); + let (event_tx, _event_rx) = mpsc::channel(4); + let mut registration = Some(build_node_register( + &test_manifest(), + "node-test", + "host-test", + "broker/test", + None, + )); + let mut inventory = Vec::new(); + let mut load = FleetLoadSnapshot { + active_agents: 3, + max_agents: 4, + handlers_live: true, + active_agent_names: vec![ + "agent-a".to_string(), + "agent-b".to_string(), + "agent-c".to_string(), + ], + }; + + let server = tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + let mut ws = accept_async(stream).await.unwrap(); + while let Some(frame) = ws.next().await { + let Ok(frame) = frame else { break }; + let Message::Text(text) = frame else { continue }; + let BrokerToRelaycast::InventorySync(sync) = + serde_json::from_str::(&text) + .expect("valid node-control frame") + else { + continue; + }; + let id = sync.id.expect("inventory liveness probe id"); + if ws + .send(Message::Text( + serde_json::to_string(&RelaycastToBroker::Reply( + crate::fleet_wire::Reply { + v: FLEET_WIRE_VERSION, + id, + ok: true, + data: json!({ "reconciled": sync.agents.len() }), + }, + )) + .unwrap(), + )) + .await + .is_err() + { + break; + } + } + }); + + let shutdown = tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(1200)).await; + command_tx + .send(FleetControlCommand::Shutdown) + .await + .unwrap(); + }); + let result = tokio::time::timeout( + Duration::from_secs(2), + run_connected_once( + &FleetControlConfig { + ws_url, + node_token: Some("nt_test".to_string()), + node_id: "node-test".to_string(), + node_name: "host-test".to_string(), + broker_version: "broker/test".to_string(), + token_minter: None, + session_token: None, + read_idle_timeout: Some(Duration::from_millis(400)), + }, + &mut command_rx, + &event_tx, + &mut registration, + &mut inventory, + &mut load, + Duration::from_millis(100), + ), + ) + .await + .expect("acknowledged application link should remain connected"); + + assert_eq!(result, ControlRunResult::Shutdown); + shutdown.await.unwrap(); + server.await.unwrap(); + } + + /// Transport handshakes do not make a control session healthy. If the peer + /// repeatedly drops each socket before acknowledging inventory.sync, the + /// reconnect delay must continue growing instead of resetting to one second. + #[tokio::test] + async fn pre_ready_disconnects_preserve_exponential_reconnect_backoff() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let ws_url = format!("ws://{}/v1/node/ws", listener.local_addr().unwrap()); + let (command_tx, command_rx) = mpsc::channel(8); + let (event_tx, _event_rx) = mpsc::channel(8); + + let client = tokio::spawn(run_node_control_client( + FleetControlConfig { + ws_url, + node_token: Some("nt_test".to_string()), + node_id: "node-test".to_string(), + node_name: "host-test".to_string(), + broker_version: "broker/test".to_string(), + token_minter: None, + session_token: None, + read_idle_timeout: Some(Duration::from_millis(400)), + }, + command_rx, + event_tx, + )); + command_tx + .send(FleetControlCommand::RegisterNode { + manifest: test_manifest(), + resume_cursor: None, + }) + .await + .unwrap(); + + let started = Instant::now(); + let mut accepted_at = Vec::new(); + let mut shutdown_ws = None; + for attempt in 0..3 { + let (stream, _) = tokio::time::timeout(Duration::from_secs(5), listener.accept()) + .await + .expect("client did not make the next reconnect attempt") + .unwrap(); + let ws = accept_async(stream).await.unwrap(); + accepted_at.push(started.elapsed()); + if attempt < 2 { + drop(ws); + } else { + shutdown_ws = Some(ws); + command_tx + .send(FleetControlCommand::Shutdown) + .await + .unwrap(); + } + } + + client.await.unwrap(); + drop(shutdown_ws); + assert!( + accepted_at[1].saturating_sub(accepted_at[0]) >= Duration::from_millis(900), + "first pre-ready failure must retain the one-second backoff: {accepted_at:?}" + ); + assert!( + accepted_at[2].saturating_sub(accepted_at[1]) >= Duration::from_millis(1800), + "second pre-ready failure must grow to the two-second backoff: {accepted_at:?}" + ); + } + /// A blackholed `/v1/node/ws` — the socket still accepts writes, but the /// engine never sends another frame — must be detected and reconnected. /// diff --git a/packages/sdk/package.json b/packages/sdk/package.json index 62083ec95..6b666dbe9 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -54,7 +54,7 @@ "build": "node -e \"require('node:fs').rmSync('dist',{recursive:true,force:true})\" && npx tsc -p tsconfig.build.json", "build:full": "npm run build", "check": "tsc -p tsconfig.json --noEmit", - "test": "vitest run src/__tests__/activity-reducer.test.ts src/__tests__/agent-relay.test.ts src/__tests__/facade.test.ts src/__tests__/listeners.test.ts src/__tests__/integrations.test.ts src/__tests__/messaging.test.ts src/__tests__/delivery-actions.test.ts src/__tests__/relaycast-errors.test.ts src/__tests__/register-action-relay.test.ts src/__tests__/thin-client.test.ts src/__tests__/typed-action-handle.test.ts src/__tests__/webhooks.test.ts src/__tests__/event-fanin.test.ts src/__tests__/observer-source.test.ts", + "test": "vitest run src/__tests__/activity-reducer.test.ts src/__tests__/agent-relay.test.ts src/__tests__/facade.test.ts src/__tests__/listeners.test.ts src/__tests__/integrations.test.ts src/__tests__/messaging.test.ts src/__tests__/relaycast-translate.test.ts src/__tests__/delivery-actions.test.ts src/__tests__/relaycast-errors.test.ts src/__tests__/register-action-relay.test.ts src/__tests__/thin-client.test.ts src/__tests__/typed-action-handle.test.ts src/__tests__/webhooks.test.ts src/__tests__/event-fanin.test.ts src/__tests__/observer-source.test.ts", "test:types": "vitest run --typecheck.only --typecheck.tsconfig tsconfig.typetest.json", "prepack": "npm run build" }, diff --git a/packages/sdk/src/__tests__/messaging.test.ts b/packages/sdk/src/__tests__/messaging.test.ts index 5df00834d..1e8f267e1 100644 --- a/packages/sdk/src/__tests__/messaging.test.ts +++ b/packages/sdk/src/__tests__/messaging.test.ts @@ -515,13 +515,13 @@ describe('RelaycastMessagingClient', () => { name: 'builder-2', status: 'offline', live: false, - activeAgents: 0, handlersLive: false, maxAgents: 4, load: 0, lastHeartbeatAt: '2026-06-16T09:55:00.000Z', createdAt: '2026-06-16T08:00:00.000Z', }); + expect(nodes[1].activeAgents).toBeUndefined(); expect(nodes[2]).toMatchObject({ name: 'builder-3', status: 'unknown', @@ -549,13 +549,14 @@ describe('RelaycastMessagingClient', () => { expect(toRelayNode({ name: 'builder-6', tags: ['factory'] }).repoKeys).toBeUndefined(); expect(toRelayNode({ name: 'builder-unbounded', max_agents: 0, load: null }).load).toBeUndefined(); - await expect(client.nodes.get('builder-2')).resolves.toMatchObject({ + const offlineNode = await client.nodes.get('builder-2'); + expect(offlineNode).toMatchObject({ name: 'builder-2', status: 'offline', live: false, - activeAgents: 0, load: 0, }); + expect(offlineNode.activeAgents).toBeUndefined(); }); it('delegates write operations through an agent client and normalizes responses', async () => { diff --git a/packages/sdk/src/__tests__/relaycast-translate.test.ts b/packages/sdk/src/__tests__/relaycast-translate.test.ts new file mode 100644 index 000000000..0437481df --- /dev/null +++ b/packages/sdk/src/__tests__/relaycast-translate.test.ts @@ -0,0 +1,29 @@ +import { describe, expect, it } from 'vitest'; + +import { toRelayNode } from '../messaging/relaycast-translate.js'; + +describe('toRelayNode fleet liveness', () => { + it('does not present an unreachable node active-agent count as zero', () => { + const node = toRelayNode({ + name: 'finn-mini', + status: 'offline', + live: false, + active_agents: 0, + capabilities: [], + }); + + expect(node.activeAgents).toBeUndefined(); + }); + + it('preserves a measured zero for a reachable node', () => { + const node = toRelayNode({ + name: 'finn-mini', + status: 'online', + live: true, + active_agents: 0, + capabilities: [], + }); + + expect(node.activeAgents).toBe(0); + }); +}); diff --git a/packages/sdk/src/messaging/relaycast-translate.ts b/packages/sdk/src/messaging/relaycast-translate.ts index 6f6d9e814..c584b76cc 100644 --- a/packages/sdk/src/messaging/relaycast-translate.ts +++ b/packages/sdk/src/messaging/relaycast-translate.ts @@ -141,16 +141,24 @@ export function toRelayCapability(raw: unknown): RelayCapability { export function toRelayNode(raw: unknown): RelayNode { const node = (raw ?? {}) as Record; const rawStatus = readStr(node, 'status'); + const status = rawStatus === 'online' || rawStatus === 'offline' ? rawStatus : 'unknown'; + const live = readBoolean(node, 'live'); + // Relaycast retains the last numeric provider load in node history and may + // surface zero after the provider is unreachable. Zero is a measurement, not + // a synonym for unknown, so omit it whenever liveness says the node cannot be + // queried authoritatively. + const activeAgents = + live === false || status === 'offline' ? undefined : readNumber(node, 'activeAgents', 'active_agents'); return { id: readStr(node, 'id', 'node_id'), nodeId: readStr(node, 'nodeId', 'node_id'), name: readStr(node, 'name') ?? '', - status: rawStatus === 'online' || rawStatus === 'offline' ? rawStatus : 'unknown', - live: readBoolean(node, 'live'), + status, + live, capabilities: Array.isArray(node.capabilities) ? node.capabilities.map(toRelayNodeCapability) : [], repoKeys: readRepoKeys(node), maxAgents: readNumber(node, 'maxAgents', 'max_agents'), - activeAgents: readNumber(node, 'activeAgents', 'active_agents'), + activeAgents, handlersLive: readBoolean(node, 'handlersLive', 'handlers_live'), load: readNumber(node, 'load'), lastHeartbeatAt: readStr(node, 'lastHeartbeatAt', 'last_heartbeat_at'), diff --git a/tests/relayflows/cases/1591-application-ack-reconnect/case.json b/tests/relayflows/cases/1591-application-ack-reconnect/case.json new file mode 100644 index 000000000..765ebbd66 --- /dev/null +++ b/tests/relayflows/cases/1591-application-ack-reconnect/case.json @@ -0,0 +1,20 @@ +{ + "version": 1, + "id": "1591-application-ack-reconnect", + "kind": "bugfix", + "title": "Reconnect when application acknowledgements stop", + "runner": { + "command": ["node", "tests/relayflows/cases/1591-application-ack-reconnect/run.mjs"] + }, + "timeoutSeconds": 900, + "expected": { + "base": { + "outcome": "bug", + "signature": "application_ack_stall_not_detected" + }, + "head": { + "outcome": "fixed", + "signature": "application_ack_stall_reconnects" + } + } +} diff --git a/tests/relayflows/cases/1591-application-ack-reconnect/run.mjs b/tests/relayflows/cases/1591-application-ack-reconnect/run.mjs new file mode 100644 index 000000000..56a2b9288 --- /dev/null +++ b/tests/relayflows/cases/1591-application-ack-reconnect/run.mjs @@ -0,0 +1,179 @@ +import { execFileSync, spawnSync } from 'node:child_process'; +import { appendFileSync, cpSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; + +const CASE_ID = '1591-application-ack-reconnect'; +const MARKER = 'RELAYFLOW_APPLICATION_ACK_RECONNECTED='; + +function fail(message) { + throw new Error(`[${CASE_ID}] ${message}`); +} + +const arm = process.env.RELAY_PR_PROOF_ARM; +if (arm !== 'base' && arm !== 'head') fail(`invalid RELAY_PR_PROOF_ARM: ${arm ?? ''}`); + +const targetDir = resolve(process.env.RELAY_PR_PROOF_TARGET_DIR ?? ''); +const resultPath = resolve(process.env.RELAY_PR_PROOF_RESULT_PATH ?? ''); +if (!process.env.RELAY_PR_PROOF_TARGET_DIR) fail('RELAY_PR_PROOF_TARGET_DIR is required'); +if (!process.env.RELAY_PR_PROOF_RESULT_PATH) fail('RELAY_PR_PROOF_RESULT_PATH is required'); + +const expectedSha = + arm === 'base' ? process.env.RELAY_PR_PROOF_BASE_SHA : process.env.RELAY_PR_PROOF_HEAD_SHA; +if (!expectedSha) fail(`missing expected ${arm} SHA`); +const targetSha = execFileSync('git', ['-C', targetDir, 'rev-parse', 'HEAD'], { + encoding: 'utf8', +}).trim(); +if (targetSha !== expectedSha) fail(`target SHA ${targetSha} does not match ${arm} SHA ${expectedSha}`); + +const scratch = mkdtempSync(join(tmpdir(), `relayflow-${CASE_ID}-`)); +const checkout = join(scratch, 'target'); + +try { + cpSync(targetDir, checkout, { + recursive: true, + filter: (source) => + !source.includes(`${join(targetDir, '.git')}`) && !source.includes(`${join(targetDir, 'target')}`), + }); + + const sourcePath = join(checkout, 'crates/broker/src/node_control.rs'); + const source = readFileSync(sourcePath, 'utf8'); + const cadence = 'const INVENTORY_REFRESH_INTERVAL: Duration = Duration::from_secs(60);'; + if (!source.includes(cadence)) fail('base/head-common inventory refresh constant was not found'); + writeFileSync( + sourcePath, + source.replace(cadence, 'const INVENTORY_REFRESH_INTERVAL: Duration = Duration::from_millis(100);') + ); + appendFileSync( + sourcePath, + String.raw` + +pub async fn relayflow_application_ack_reconnect_probe() -> bool { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let ws_url = format!("ws://{}/v1/node/ws", listener.local_addr().unwrap()); + let (command_tx, command_rx) = tokio::sync::mpsc::channel(8); + let (event_tx, _event_rx) = tokio::sync::mpsc::channel(8); + let client = tokio::spawn(run_node_control_client( + FleetControlConfig { + ws_url, + node_token: Some("nt_relayflow".to_string()), + node_id: "node-relayflow".to_string(), + node_name: "relayflow-loopback".to_string(), + broker_version: "relayflow/head-authored".to_string(), + token_minter: None, + session_token: None, + read_idle_timeout: Some(Duration::from_millis(400)), + }, + command_rx, + event_tx, + )); + command_tx + .send(FleetControlCommand::RegisterNode { + manifest: NodeManifest { + name: "relayflow-loopback".to_string(), + node_id: Some("node-relayflow".to_string()), + capabilities: Vec::new(), + max_agents: Some(1), + tags: None, + repo_keys: None, + version: Some("relayflow/head-authored".to_string()), + }, + resume_cursor: None, + }) + .await + .unwrap(); + + let (stream, _) = listener.accept().await.unwrap(); + let mut first = tokio_tungstenite::accept_async(stream).await.unwrap(); + let mut shutdown_ws = None; + let reconnected = tokio::time::timeout(Duration::from_millis(2500), async { + loop { + match first.next().await { + Some(Ok(Message::Ping(payload))) => { + if first.send(Message::Pong(payload)).await.is_err() { + break; + } + } + Some(Ok(_)) => {} + Some(Err(_)) | None => break, + } + } + let (stream, _) = listener.accept().await.unwrap(); + shutdown_ws = Some(tokio_tungstenite::accept_async(stream).await.unwrap()); + true + }) + .await + .unwrap_or(false); + + client.abort(); + let _ = client.await; + drop(shutdown_ws); + reconnected +} +` + ); + + const libPath = join(checkout, 'crates/broker/src/lib.rs'); + const libSource = readFileSync(libPath, 'utf8'); + const privateModule = 'pub(crate) mod node_control;'; + if (!libSource.includes(privateModule)) fail('base/head-common node_control module was not found'); + writeFileSync(libPath, libSource.replace(privateModule, 'pub mod node_control;')); + appendFileSync( + join(checkout, 'crates/broker/Cargo.toml'), + '\n[[bin]]\nname = "relayflow-application-ack-reconnect"\npath = "src/bin/relayflow_application_ack_reconnect.rs"\n' + ); + const binDir = join(checkout, 'crates/broker/src/bin'); + mkdirSync(binDir, { recursive: true }); + writeFileSync( + join(binDir, 'relayflow_application_ack_reconnect.rs'), + `#[tokio::main]\nasync fn main() {\n let reconnected = relay_broker::node_control::relayflow_application_ack_reconnect_probe().await;\n println!("${MARKER}{}", if reconnected { "true" } else { "false" });\n}\n` + ); + + const run = spawnSync( + 'cargo', + ['run', '-p', 'agent-relay-broker', '--bin', 'relayflow-application-ack-reconnect'], + { + cwd: checkout, + encoding: 'utf8', + timeout: 600_000, + maxBuffer: 4 * 1024 * 1024, + env: { + ...process.env, + CARGO_BUILD_JOBS: process.env.CARGO_BUILD_JOBS ?? '1', + CARGO_INCREMENTAL: process.env.CARGO_INCREMENTAL ?? '0', + CARGO_PROFILE_DEV_DEBUG: process.env.CARGO_PROFILE_DEV_DEBUG ?? '0', + RUSTFLAGS: process.env.RUSTFLAGS ?? '-C debuginfo=0 -C codegen-units=256', + }, + } + ); + process.stdout.write(run.stdout ?? ''); + process.stderr.write(run.stderr ?? ''); + if (run.error) fail(`cargo run infrastructure failure: ${run.error.message}`); + if (run.status !== 0) fail(`cargo run infrastructure failure (exit ${run.status})`); + + const matches = [...(run.stdout ?? '').matchAll(new RegExp(`${MARKER}(true|false)`, 'g'))]; + if (matches.length !== 1) fail(`expected exactly one observation marker, found ${matches.length}`); + const reconnected = matches[0][1] === 'true'; + const observation = reconnected + ? { + version: 1, + caseId: CASE_ID, + arm, + outcome: 'fixed', + signature: 'application_ack_stall_reconnects', + details: + 'The production node-control client reconnected while loopback WebSocket pongs continued and inventory.sync acknowledgements were withheld.', + } + : { + version: 1, + caseId: CASE_ID, + arm, + outcome: 'bug', + signature: 'application_ack_stall_not_detected', + details: + 'The production node-control client did not reconnect while loopback WebSocket pongs continued and inventory.sync acknowledgements were withheld.', + }; + writeFileSync(resultPath, `${JSON.stringify(observation, null, 2)}\n`); +} finally { + rmSync(scratch, { recursive: true, force: true }); +}