From 98af8282ef442d6a59f0700c3d5faa0e7986eb83 Mon Sep 17 00:00:00 2001 From: dnth Date: Thu, 15 Oct 2026 23:59:34 +0800 Subject: [PATCH 1/9] test(ci): rebalance portable serial shards Refresh all serial shard weights from green-run timing artifacts, diagnose the 20-minute cancellations as aggregate runtime, and document the balanced membership. --- .github/workflows/ci.yml | 6 +- bin/fm-test-run.sh | 314 +++++++++++++++++--------------- docs/fm-test-portable-shards.md | 263 ++++++++++++++++++++++++-- 3 files changed, 415 insertions(+), 168 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f9465522f50..7109bfae723 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -192,9 +192,9 @@ jobs: tests-portable-serial: name: Behavior portable serial ${{ matrix.shard }} runs-on: ubuntu-latest - # Refreshed timings plus conservative hints for grown tests estimate ~84 min - # of serial work, balanced over six runners at ~14 min each. Keep the cap - # as a hang tripwire with runner-load margin; refresh hints per + # Refreshed measurements total 86m19.574s and project 14m23.259s–14m23.270s + # per shard. Keep this 20-minute cap as a hang tripwire with about 5m36.7s + # for setup and runner-load variance; refresh hints per # docs/fm-test-portable-shards.md when a shard approaches it. timeout-minutes: 20 strategy: diff --git a/bin/fm-test-run.sh b/bin/fm-test-run.sh index 7de253f58e6..d161cc2ed45 100755 --- a/bin/fm-test-run.sh +++ b/bin/fm-test-run.sh @@ -382,158 +382,172 @@ list_portable_serial() { # procedure. portable_serial_weight_hints() { cat <<'EOF' -tests/fm-afk-inject-e2e.test.sh 26008 -tests/fm-afk-pi-herdr-return-e2e.test.sh 47 -tests/fm-afk-return.test.sh 1517 -tests/fm-ask-user-authority.test.sh 138 -tests/fm-backend-cmux-smoke.test.sh 30 -tests/fm-backend-cmux.test.sh 2950 -tests/fm-backend-herdr-focus-flash-e2e.test.sh 21 -tests/fm-backend-herdr-server-detach.test.sh 90000 -tests/fm-backend-orca.test.sh 19416 -tests/fm-backend-tmux-smoke.test.sh 315 -tests/fm-backend-zellij-smoke.test.sh 14 -tests/fm-backend-zellij.test.sh 5188 -tests/fm-backend.test.sh 19862 -tests/fm-backlog-atomicity.test.sh 78496 -tests/fm-backlog-handoff.test.sh 3343 -tests/fm-bearings-snapshot.test.sh 66118 -tests/fm-boat-lifecycle.test.sh 52034 -tests/fm-boat-omp-auth.test.sh 58941 -tests/fm-boat-routing.test.sh 200000 -tests/fm-bootstrap.test.sh 62184 -tests/fm-busy-adapter-wiring.test.sh 17306 -tests/fm-busy-state.test.sh 2821 -tests/fm-calm-pi-extension.test.sh 260 -tests/fm-check-unregister.test.sh 356 -tests/fm-classify-decision-key.test.sh 703 -tests/fm-claude-stop-autoarm-live-e2e.test.sh 17 -tests/fm-claude-stop-autoarm.test.sh 60519 -tests/fm-cleanup-skill.test.sh 72 -tests/fm-codex-continuity-live-e2e.test.sh 19 -tests/fm-daemon.test.sh 61329 -tests/fm-devin-adapter.test.sh 280 -tests/fm-devin-herdr-live-e2e.test.sh 13 -tests/fm-dispatch-resolve.test.sh 3995 -tests/fm-documentation-audiences.test.sh 708 -tests/fm-ext-bridge.test.sh 35693 -tests/fm-fleet-snapshot-view.test.sh 13508 -tests/fm-fleet-sync.test.sh 32815 -tests/fm-gate-refuse.test.sh 9270 -tests/fm-gitignore-config.test.sh 25 -tests/fm-gotmp.test.sh 713 -tests/fm-grok-continuity-live-e2e.test.sh 12 +tests/fm-afk-inject-e2e.test.sh 25557 +tests/fm-afk-pi-herdr-return-e2e.test.sh 76 +tests/fm-afk-return.test.sh 1513 +tests/fm-ask-user-authority.test.sh 170 +tests/fm-backend-cmux-smoke.test.sh 22 +tests/fm-backend-cmux.test.sh 3245 +tests/fm-backend-herdr-focus-flash-e2e.test.sh 14 +tests/fm-backend-herdr-server-detach.test.sh 2631 +tests/fm-backend-orca.test.sh 16662 +tests/fm-backend-tmux-smoke.test.sh 191 +tests/fm-backend-zellij-smoke.test.sh 17 +tests/fm-backend-zellij.test.sh 5810 +tests/fm-backend.test.sh 20363 +tests/fm-backlog-atomicity.test.sh 63738 +tests/fm-backlog-handoff.test.sh 2814 +tests/fm-bearings-snapshot.test.sh 61847 +tests/fm-boat-lane-runner.test.sh 120149 +tests/fm-boat-lifecycle.test.sh 49402 +tests/fm-boat-omp-auth.test.sh 62421 +tests/fm-boat-routing.test.sh 27470 +tests/fm-bootstrap.test.sh 72800 +tests/fm-busy-adapter-wiring.test.sh 18965 +tests/fm-busy-state.test.sh 2834 +tests/fm-calm-pi-extension.test.sh 256 +tests/fm-check-unregister.test.sh 361 +tests/fm-classify-decision-key.test.sh 542 +tests/fm-claude-stop-autoarm-live-e2e.test.sh 14 +tests/fm-claude-stop-autoarm.test.sh 60541 +tests/fm-cleanup-skill.test.sh 58 +tests/fm-codex-continuity-live-e2e.test.sh 21 +tests/fm-daemon.test.sh 50403 +tests/fm-devin-adapter.test.sh 284 +tests/fm-devin-herdr-live-e2e.test.sh 20 +tests/fm-dispatch-resolve.test.sh 6418 +tests/fm-documentation-audiences.test.sh 879 +tests/fm-ext-ask.test.sh 48612 +tests/fm-ext-bridge.test.sh 61703 +tests/fm-ext-decision.test.sh 116759 +tests/fm-ext-plugin-dest.test.sh 8528 +tests/fm-ext-thread-context.test.sh 17972 +tests/fm-ext-topic-threads.test.sh 81072 +tests/fm-ext-waiting-pin.test.sh 124489 +tests/fm-ext-watch.test.sh 7673 +tests/fm-fleet-snapshot-view.test.sh 13346 +tests/fm-fleet-sync.test.sh 33947 +tests/fm-gate-refuse.test.sh 9734 +tests/fm-gitignore-config.test.sh 27 +tests/fm-gotmp.test.sh 908 +tests/fm-grok-continuity-live-e2e.test.sh 21 tests/fm-grok-stop-live-e2e.test.sh 21 -tests/fm-guard-stale-banner.test.sh 4698 -tests/fm-herdr-session-cleanup.test.sh 4832 -tests/fm-herdr-submit-confirm-live-e2e.test.sh 21 -tests/fm-hermes-harness.test.sh 146669 -tests/fm-hermes-live-e2e.test.sh 28 -tests/fm-idle-sleep-watch.test.sh 8000 -tests/fm-idle-sleep.test.sh 15000 -tests/fm-inbox-result.test.sh 7377 -tests/fm-inbox.test.sh 7022 -tests/fm-kimi-harness.test.sh 16139 -tests/fm-omp-branch-bash.test.sh 44 -tests/fm-omp-branch-live-e2e.test.sh 12 -tests/fm-omp-branch-supervision.test.sh 4373 -tests/fm-omp-branch-types.test.sh 24 -tests/fm-omp-compact-adviser-live-e2e.test.sh 20 -tests/fm-omp-compact-adviser.test.sh 298 -tests/fm-omp-fleet-hooks.test.sh 2407 -tests/fm-omp-harness.test.sh 1764 -tests/fm-omp-herdr-exit-live-e2e.test.sh 16 +tests/fm-guard-stale-banner.test.sh 4737 +tests/fm-herdr-session-cleanup.test.sh 5318 +tests/fm-herdr-submit-confirm-live-e2e.test.sh 18 +tests/fm-hermes-harness.test.sh 147035 +tests/fm-hermes-live-e2e.test.sh 42 +tests/fm-idle-sleep-watch.test.sh 4199 +tests/fm-idle-sleep.test.sh 12252 +tests/fm-inbox-result.test.sh 10147 +tests/fm-inbox.test.sh 8356 +tests/fm-kimi-harness.test.sh 17704 +tests/fm-landing.test.sh 34429 +tests/fm-main-ci-watch.test.sh 40258 +tests/fm-merge-local.test.sh 2363 +tests/fm-omp-branch-bash.test.sh 35 +tests/fm-omp-branch-live-e2e.test.sh 14 +tests/fm-omp-branch-supervision.test.sh 37983 +tests/fm-omp-branch-types.test.sh 15 +tests/fm-omp-compact-adviser-live-e2e.test.sh 14 +tests/fm-omp-compact-adviser.test.sh 1592 +tests/fm-omp-fleet-hooks.test.sh 2522 +tests/fm-omp-harness.test.sh 66551 +tests/fm-omp-herdr-exit-live-e2e.test.sh 21 tests/fm-omp-herdr-live-e2e.test.sh 19 -tests/fm-omp-primary-live-e2e.test.sh 18 -tests/fm-omp-primary.test.sh 45746 -tests/fm-omp-relaunch-guard.test.sh 4276 -tests/fm-omp-relaunch-tmux-live-e2e.test.sh 21 -tests/fm-omp-secondmate-live-e2e.test.sh 21 -tests/fm-omp-secondmate.test.sh 600000 -tests/fm-omp-task-inbox-doorbell.test.sh 20577 -tests/fm-omp-update.test.sh 448 -tests/fm-omp-worker-tmux-live-e2e.test.sh 17 -tests/fm-on.test.sh 9627 -tests/fm-opencode-primary-live-e2e.test.sh 16 -tests/fm-operational-input.test.sh 176 -tests/fm-pending-reply-sleep-reconcile.test.sh 243 -tests/fm-pending-reply.test.sh 19949 -tests/fm-pi-compatible-family.test.sh 56 -tests/fm-pi-primary-live-e2e.test.sh 21 -tests/fm-pi-watch-extension.test.sh 22031 -tests/fm-pr-check-handoff.test.sh 42228 -tests/fm-pr-check-security.test.sh 158907 -tests/fm-prepush-guard.test.sh 19208 -tests/fm-procevent-when.test.sh 24358 -tests/fm-procevent.test.sh 61916 -tests/fm-public-followup.test.sh 35559 -tests/fm-quota-array-dispatch-live-e2e.test.sh 12 -tests/fm-receipt-check.test.sh 11636 -tests/fm-receipt.test.sh 1331 -tests/fm-reflect-skill.test.sh 372 -tests/fm-remote-backlog-handoff.test.sh 15173 -tests/fm-remote-doctor.test.sh 6203 -tests/fm-remote-entrypoint.test.sh 208 -tests/fm-remote-job.test.sh 48694 -tests/fm-remote-reply.test.sh 99366 -tests/fm-remote-secondmate-lifecycle-e2e.test.sh 700000 -tests/fm-remote-secondmate-parent-binding.test.sh 15892 -tests/fm-remote-secondmate-sleep-reconcile.test.sh 3000 -tests/fm-remote-secondmate-trace-context.test.sh 49391 -tests/fm-runpod-lifecycle.test.sh 56695 -tests/fm-runpod-omp-auth.test.sh 1012 -tests/fm-runpod-pod-boot.test.sh 45589 -tests/fm-runpod-routing.test.sh 28030 -tests/fm-secondmate-harness.test.sh 151987 -tests/fm-secondmate-lifecycle-e2e.test.sh 5372 -tests/fm-secondmate-liveness.test.sh 20126 -tests/fm-secondmate-restart.test.sh 31003 -tests/fm-secondmate-safety.test.sh 55903 -tests/fm-secondmate-sync.test.sh 17159 -tests/fm-send-inbox-doorbell-live-e2e.test.sh 18 -tests/fm-send-inbox.test.sh 37394 -tests/fm-send-resolve-key.test.sh 12632 -tests/fm-send-secondmate-marker-herdr-e2e.test.sh 48 -tests/fm-send-secondmate-marker.test.sh 4320 -tests/fm-send-turn-start.test.sh 11143 -tests/fm-session-lock-ancestry.test.sh 1284 -tests/fm-session-start.test.sh 77942 -tests/fm-sessionstart-nudge.test.sh 284 -tests/fm-shared-captain-inheritance.test.sh 4259 -tests/fm-spawn-dispatch-profile.test.sh 193113 -tests/fm-spawn-pool-base-freshen.test.sh 35139 -tests/fm-spawn-relaunch-dead-endpoint.test.sh 31467 -tests/fm-spawn-worktree-settle.test.sh 5788 -tests/fm-stall-recovery.test.sh 48 -tests/fm-startup-memory-budget.test.sh 7593 -tests/fm-subagent-pretool-check.test.sh 709 -tests/fm-supervision-events.test.sh 462 -tests/fm-tangle-guard.test.sh 8797 -tests/fm-task-delivery.test.sh 9829 -tests/fm-task-inbox.test.sh 37450 -tests/fm-teardown-endpoint-safety.test.sh 1797 -tests/fm-teardown.test.sh 107078 -tests/fm-test-fixture-cleanup.test.sh 542 -tests/fm-test-isolation-proof.test.sh 426 -tests/fm-todo-project.test.sh 6065 -tests/fm-trace-context-lib.test.sh 1424 -tests/fm-trace-context-spawn.test.sh 46250 -tests/fm-treehouse-orphan-recovery.test.sh 125 -tests/fm-treehouse-status-read-only.test.sh 225 -tests/fm-treehouse-sweep.test.sh 4886 -tests/fm-turnend-guard.test.sh 20258 -tests/fm-update.test.sh 10591 -tests/fm-vendor-auth-probe.test.sh 42632 -tests/fm-wake-daemon-lifecycle-e2e.test.sh 19948 -tests/fm-wake-drain-open-decisions-cursor.test.sh 22217 -tests/fm-wake-drain-open-decisions.test.sh 3443 -tests/fm-wake-drain-unread-status.test.sh 14508 -tests/fm-wake-queue.test.sh 72347 -tests/fm-watch-arm.test.sh 66052 -tests/fm-watch-checkpoint.test.sh 4768 -tests/fm-watch-recovery-loop.test.sh 67796 -tests/fm-watch-triage.test.sh 407955 -tests/fm-watcher-lock.test.sh 38168 +tests/fm-omp-primary-live-e2e.test.sh 16 +tests/fm-omp-primary.test.sh 61145 +tests/fm-omp-relaunch-guard.test.sh 5403 +tests/fm-omp-relaunch-tmux-live-e2e.test.sh 15 +tests/fm-omp-secondmate-live-e2e.test.sh 20 +tests/fm-omp-secondmate.test.sh 103752 +tests/fm-omp-task-inbox-doorbell.test.sh 19102 +tests/fm-omp-update.test.sh 497 +tests/fm-omp-worker-tmux-live-e2e.test.sh 16 +tests/fm-on.test.sh 8384 +tests/fm-opencode-primary-live-e2e.test.sh 19 +tests/fm-operational-input.test.sh 179 +tests/fm-pending-reply-sleep-reconcile.test.sh 178 +tests/fm-pending-reply.test.sh 11519 +tests/fm-pi-compatible-family.test.sh 58 +tests/fm-pi-primary-live-e2e.test.sh 14 +tests/fm-pi-watch-extension.test.sh 22138 +tests/fm-pr-check-handoff.test.sh 16266 +tests/fm-pr-check-security.test.sh 137546 +tests/fm-prepush-guard.test.sh 21813 +tests/fm-procevent-when.test.sh 19951 +tests/fm-procevent.test.sh 65023 +tests/fm-public-followup.test.sh 37602 +tests/fm-quota-array-dispatch-live-e2e.test.sh 16 +tests/fm-receipt-check.test.sh 6298 +tests/fm-receipt.test.sh 1469 +tests/fm-reflect-skill.test.sh 480 +tests/fm-remote-backlog-handoff.test.sh 19810 +tests/fm-remote-doctor.test.sh 4824 +tests/fm-remote-entrypoint.test.sh 199 +tests/fm-remote-job.test.sh 49058 +tests/fm-remote-omp-update-restart-live-e2e.test.sh 19 +tests/fm-remote-reply.test.sh 85746 +tests/fm-remote-secondmate-lifecycle-e2e.test.sh 660575 +tests/fm-remote-secondmate-parent-binding.test.sh 12099 +tests/fm-remote-secondmate-sleep-reconcile.test.sh 1354 +tests/fm-remote-secondmate-trace-context.test.sh 59443 +tests/fm-remote-secondmate-update-restart.test.sh 38017 +tests/fm-runpod-lifecycle.test.sh 51268 +tests/fm-runpod-omp-auth.test.sh 988 +tests/fm-runpod-pod-boot.test.sh 43745 +tests/fm-runpod-routing.test.sh 37972 +tests/fm-secondmate-harness.test.sh 149950 +tests/fm-secondmate-lifecycle-e2e.test.sh 5351 +tests/fm-secondmate-liveness.test.sh 23406 +tests/fm-secondmate-restart.test.sh 37021 +tests/fm-secondmate-safety.test.sh 46061 +tests/fm-secondmate-sync.test.sh 37220 +tests/fm-send-inbox-doorbell-live-e2e.test.sh 23 +tests/fm-send-inbox.test.sh 40599 +tests/fm-send-resolve-key.test.sh 20275 +tests/fm-send-secondmate-marker-herdr-e2e.test.sh 52 +tests/fm-send-secondmate-marker.test.sh 6023 +tests/fm-send-turn-start.test.sh 11269 +tests/fm-session-lock-ancestry.test.sh 1524 +tests/fm-session-start.test.sh 69482 +tests/fm-sessionstart-nudge.test.sh 407 +tests/fm-shared-captain-inheritance.test.sh 13607 +tests/fm-spawn-dispatch-profile.test.sh 209130 +tests/fm-spawn-pool-base-freshen.test.sh 35517 +tests/fm-spawn-relaunch-dead-endpoint.test.sh 35325 +tests/fm-spawn-worktree-settle.test.sh 11206 +tests/fm-stall-recovery.test.sh 46 +tests/fm-startup-memory-budget.test.sh 7617 +tests/fm-subagent-pretool-check.test.sh 777 +tests/fm-supervision-events.test.sh 634 +tests/fm-tangle-guard.test.sh 9372 +tests/fm-task-delivery.test.sh 15612 +tests/fm-task-inbox.test.sh 35336 +tests/fm-tasks-axi.test.sh 16934 +tests/fm-teardown-endpoint-safety.test.sh 30753 +tests/fm-teardown.test.sh 140801 +tests/fm-test-fixture-cleanup.test.sh 515 +tests/fm-test-isolation-proof.test.sh 473 +tests/fm-todo-project.test.sh 9253 +tests/fm-trace-context-lib.test.sh 1626 +tests/fm-trace-context-spawn.test.sh 41182 +tests/fm-treehouse-orphan-recovery.test.sh 157 +tests/fm-treehouse-status-read-only.test.sh 230 +tests/fm-treehouse-sweep.test.sh 4598 +tests/fm-turnend-guard.test.sh 17221 +tests/fm-update.test.sh 7384 +tests/fm-vendor-auth-probe.test.sh 42704 +tests/fm-wake-daemon-lifecycle-e2e.test.sh 60312 +tests/fm-wake-drain-open-decisions-cursor.test.sh 22524 +tests/fm-wake-drain-open-decisions.test.sh 17306 +tests/fm-wake-drain-unread-status.test.sh 24217 +tests/fm-wake-queue.test.sh 100988 +tests/fm-watch-arm.test.sh 66908 +tests/fm-watch-checkpoint.test.sh 5271 +tests/fm-watch-recovery-loop.test.sh 64620 +tests/fm-watch-triage.test.sh 410055 +tests/fm-watcher-lock.test.sh 67720 EOF } diff --git a/docs/fm-test-portable-shards.md b/docs/fm-test-portable-shards.md index 3d45a084456..cc09ce2852b 100644 --- a/docs/fm-test-portable-shards.md +++ b/docs/fm-test-portable-shards.md @@ -51,37 +51,270 @@ The two parallel lanes use longest-processing-time assignment from those measure `portable-serial` includes every `tests/*.test.sh` that is neither proven-isolated nor `real-herdr-gated`. It keeps watcher, lock, AFK, real tmux, daemon, secondmate lifecycle, bootstrap, live-harness opt-in, GUI-backend, and other unproven work serial. -Membership is derived rather than enumerated, so a newly added test lands here by default. +The portable-serial test set is derived rather than enumerated, so a newly added test lands here by default. +The per-shard membership snapshot below records the generated assignment for the current timing hints. ## Portable serial CI shards -On green CI run [30725985757](https://github.com/kunchenguid/firstmate/actions/runs/30725985757), that remainder accumulated 19m04s of script time against a 20-minute job timeout. -On [PR 1495](https://github.com/kunchenguid/firstmate/pull/1495), its main step ran about 19m51s before the job was cancelled at that boundary. -`portable-serial-of` splits it across `n` separate CI runners. +The refreshed hints use all 166 completed serial-script measurements from green run [38746979839 attempt 2](https://github.com/dnth/firstmate/actions/runs/38746979839/attempts/2), totaling 5,179,574 ms (86m19.6s). +The measured durations replace the older estimates and default weights in `portable_serial_weight_hints`. +`portable-serial-of` splits that remainder across `n` separate CI runners. Each shard is still strictly serial in itself, and separate runners mean no two of these stateful scripts ever share a machine, so the split needs no concurrency isolation proof. `bin/fm-test-run.sh` owns `n` and refuses any lane whose `of` disagrees with it. `.github/workflows/ci.yml` derives the same `n` from `strategy.job-total` rather than a literal, so changing the shard count in either file without the other fails the lane loudly instead of leaving part of the required suite unrun. Assignment is longest-processing-time bin packing over per-script duration hints embedded in `bin/fm-test-run.sh`. -The hints were refreshed on 2026-10-04 from 116 completed, passing script timings in attempt 3 of [run 37159651011](https://github.com/dnth/firstmate/actions/runs/37159651011/attempts/3). -Shards 3 and 4 produced timing artifacts; shards 1 and 2 hit the 20-minute job cap, so their completed `FM_TEST_END` log records supply the measurements. -Existing hints for uncompleted scripts remain from [run 33904204200](https://github.com/dnth/firstmate/actions/runs/33904204200). -The four grown tests use conservative operator-supplied estimates: remote lifecycle 700000 ms, OMP secondmate 600000 ms, Boat routing 200000 ms, and Herdr server detach 90000 ms. -These estimates deliberately exceed this attempt's observed times for those scripts to budget for the reported slower runs. -At that refresh, the remainder estimated about 83.9 minutes; even a balanced four-shard partition would exceed the cap, so six runners now share the work. -A script with no hint gets the conservative `PORTABLE_SERIAL_DEFAULT_WEIGHT_MS` default. Hints only affect balance: the coverage guard keeps the partition complete and disjoint whatever they say, so a stale hint costs a slower shard rather than lost coverage. +A script with no hint gets the conservative `PORTABLE_SERIAL_DEFAULT_WEIGHT_MS` default. + +### Timeout diagnosis + +The recent 20-minute cancellations came from aggregate serial work, not a hanging `fm-watch-checkpoint.test.sh`. +The cancelled PR run is [38753521432](https://github.com/dnth/firstmate/actions/runs/38753521432). +Its first attempt ended during `fm-on.test.sh` after the job reached its timeout, before the checkpoint test began. +Its second attempt reached the checkpoint test about five seconds before cancellation, after the preceding serial scripts consumed the available job time. +On main run [38746979839 attempt 1](https://github.com/dnth/firstmate/actions/runs/38746979839/attempts/1), all 28 scripts emitted a 19m55.325s summary before the job was cancelled near its 20-minute limit. +The passing main attempt completed the same shard's 28 scripts in 17m20.456s. + +| Run / attempt | Measured shard-3 script spans (UTC) | Result | +|---|---|---| +| [Main 38746979839 attempt 1](https://github.com/dnth/firstmate/actions/runs/38746979839/attempts/1), cancelled | `fm-watch-triage`: 13:56:29–14:03:33 (423,952 ms); `fm-ext-decision`: 14:10:55–14:13:28 (153,704 ms); `fm-watch-checkpoint`: 14:16:03–14:16:10 (5,993 ms). | All scripts completed; the 19m55.325s suite summary preceded cancellation. | +| [PR 38753521432 attempt 1](https://github.com/dnth/firstmate/actions/runs/38753521432/attempts/1), cancelled | `fm-watch-triage`: 14:40:54–14:48:14 (440,356 ms); `fm-ext-decision`: 14:55:56–14:58:39 (163,088 ms); `fm-on` began 15:00:52 and the job cancelled at 15:01:00. | Cancelled in the final `fm-on` test; checkpoint did not start. | +| [PR 38753521432 attempt 2](https://github.com/dnth/firstmate/actions/runs/38753521432/attempts/2), cancelled | `fm-watch-triage`: 15:05:34–15:12:49 (435,200 ms); `fm-ext-decision`: 15:20:20–15:22:55 (155,491 ms); `fm-watch-checkpoint` began at 15:25:32 and the job cancelled at 15:25:37. | The checkpoint log reached its wake acknowledgement before cancellation; no test-end marker was emitted. | +| [Main 38746979839 attempt 2](https://github.com/dnth/firstmate/actions/runs/38746979839/attempts/2), passed | `fm-watch-triage`: 14:22:32–14:29:22 (410,055 ms); `fm-ext-decision`: 14:35:28–14:37:25 (116,759 ms); `fm-watch-checkpoint`: 14:39:36–14:39:41 (5,271 ms). | All 28 scripts completed in 17m20.456s. | + +The checkpoint completed in 5.271s and 5.993s on the passing and cancelled main attempts, respectively. +The cancelled PR attempt that reached it spent only about five seconds there before the 20-minute job limit, while the other cancelled PR attempt ended in `fm-on`. +The measured long tests vary across runs but none approaches the job limit; the cause is cumulative shard work. +The fix rebalances measured work without changing tests or raising the 20-minute hang tripwire. + +### Rebalanced duration profile + +The refreshed longest-processing-time assignment projects 14m23.259s–14m23.270s per shard from the green run's measured script durations. +The balanced six-runner matrix leaves about 5m36.7s for setup and runner-load variance before the 20-minute timeout. + +| Shard | Scripts | Projected script time | +|---|---:|---:| +| 1 of 6 | 23 | 14:23.260 | +| 2 of 6 | 27 | 14:23.263 | +| 3 of 6 | 30 | 14:23.262 | +| 4 of 6 | 30 | 14:23.260 | +| 5 of 6 | 26 | 14:23.259 | +| 6 of 6 | 30 | 14:23.270 | +| Total | 166 | 86:19.574 | + +The longest measured script is `tests/fm-remote-secondmate-lifecycle-e2e.test.sh` at 660,575 ms (11m00.575s), which is the single-script lower bound for any shard count. +The current shard memberships below are generated from the same hints. + +### Current shard membership + +#### `portable-serial-1of6` — 23 scripts + +```text +tests/fm-remote-secondmate-lifecycle-e2e.test.sh +tests/fm-omp-branch-supervision.test.sh +tests/fm-spawn-pool-base-freshen.test.sh +tests/fm-teardown-endpoint-safety.test.sh +tests/fm-send-resolve-key.test.sh +tests/fm-kimi-harness.test.sh +tests/fm-task-delivery.test.sh +tests/fm-send-turn-start.test.sh +tests/fm-inbox-result.test.sh +tests/fm-ext-watch.test.sh +tests/fm-backend-zellij.test.sh +tests/fm-guard-stale-banner.test.sh +tests/fm-backend-herdr-server-detach.test.sh +tests/fm-runpod-omp-auth.test.sh +tests/fm-subagent-pretool-check.test.sh +tests/fm-reflect-skill.test.sh +tests/fm-pending-reply-sleep-reconcile.test.sh +tests/fm-send-secondmate-marker-herdr-e2e.test.sh +tests/fm-gitignore-config.test.sh +tests/fm-codex-continuity-live-e2e.test.sh +tests/fm-omp-secondmate-live-e2e.test.sh +tests/fm-backend-zellij-smoke.test.sh +tests/fm-backend-herdr-focus-flash-e2e.test.sh +``` + +#### `portable-serial-2of6` — 27 scripts + +```text +tests/fm-watch-triage.test.sh +tests/fm-omp-harness.test.sh +tests/fm-bearings-snapshot.test.sh +tests/fm-remote-secondmate-trace-context.test.sh +tests/fm-ext-ask.test.sh +tests/fm-send-inbox.test.sh +tests/fm-secondmate-sync.test.sh +tests/fm-landing.test.sh +tests/fm-wake-drain-open-decisions-cursor.test.sh +tests/fm-procevent-when.test.sh +tests/fm-backend-orca.test.sh +tests/fm-idle-sleep.test.sh +tests/fm-tangle-guard.test.sh +tests/fm-on.test.sh +tests/fm-secondmate-lifecycle-e2e.test.sh +tests/fm-remote-doctor.test.sh +tests/fm-omp-fleet-hooks.test.sh +tests/fm-remote-secondmate-sleep-reconcile.test.sh +tests/fm-supervision-events.test.sh +tests/fm-devin-adapter.test.sh +tests/fm-treehouse-status-read-only.test.sh +tests/fm-pi-compatible-family.test.sh +tests/fm-omp-branch-bash.test.sh +tests/fm-omp-herdr-exit-live-e2e.test.sh +tests/fm-remote-omp-update-restart-live-e2e.test.sh +tests/fm-quota-array-dispatch-live-e2e.test.sh +tests/fm-omp-compact-adviser-live-e2e.test.sh +``` + +#### `portable-serial-3of6` — 30 scripts + +```text +tests/fm-spawn-dispatch-profile.test.sh +tests/fm-ext-decision.test.sh +tests/fm-ext-topic-threads.test.sh +tests/fm-watch-arm.test.sh +tests/fm-boat-omp-auth.test.sh +tests/fm-wake-daemon-lifecycle-e2e.test.sh +tests/fm-remote-job.test.sh +tests/fm-trace-context-spawn.test.sh +tests/fm-public-followup.test.sh +tests/fm-fleet-sync.test.sh +tests/fm-secondmate-liveness.test.sh +tests/fm-omp-task-inbox-doorbell.test.sh +tests/fm-turnend-guard.test.sh +tests/fm-pending-reply.test.sh +tests/fm-spawn-worktree-settle.test.sh +tests/fm-dispatch-resolve.test.sh +tests/fm-receipt-check.test.sh +tests/fm-backend-cmux.test.sh +tests/fm-busy-state.test.sh +tests/fm-omp-compact-adviser.test.sh +tests/fm-gotmp.test.sh +tests/fm-omp-update.test.sh +tests/fm-calm-pi-extension.test.sh +tests/fm-remote-entrypoint.test.sh +tests/fm-cleanup-skill.test.sh +tests/fm-hermes-live-e2e.test.sh +tests/fm-grok-stop-live-e2e.test.sh +tests/fm-opencode-primary-live-e2e.test.sh +tests/fm-omp-worker-tmux-live-e2e.test.sh +tests/fm-omp-branch-live-e2e.test.sh +``` -Current serial shard membership is derived by `bin/fm-test-run.sh`; its header owns the inspection commands, and its weight hints own the balancing inputs. +#### `portable-serial-4of6` — 30 scripts -The single longest estimate, `tests/fm-remote-secondmate-lifecycle-e2e.test.sh` at 700000 ms, is the floor for any shard count. +```text +tests/fm-secondmate-harness.test.sh +tests/fm-boat-lane-runner.test.sh +tests/fm-omp-secondmate.test.sh +tests/fm-watcher-lock.test.sh +tests/fm-watch-recovery-loop.test.sh +tests/fm-claude-stop-autoarm.test.sh +tests/fm-daemon.test.sh +tests/fm-vendor-auth-probe.test.sh +tests/fm-main-ci-watch.test.sh +tests/fm-spawn-relaunch-dead-endpoint.test.sh +tests/fm-wake-drain-unread-status.test.sh +tests/fm-pi-watch-extension.test.sh +tests/fm-remote-backlog-handoff.test.sh +tests/fm-pr-check-handoff.test.sh +tests/fm-fleet-snapshot-view.test.sh +tests/fm-todo-project.test.sh +tests/fm-update.test.sh +tests/fm-omp-relaunch-guard.test.sh +tests/fm-watch-checkpoint.test.sh +tests/fm-trace-context-lib.test.sh +tests/fm-session-lock-ancestry.test.sh +tests/fm-documentation-audiences.test.sh +tests/fm-check-unregister.test.sh +tests/fm-backend-tmux-smoke.test.sh +tests/fm-afk-pi-herdr-return-e2e.test.sh +tests/fm-send-inbox-doorbell-live-e2e.test.sh +tests/fm-grok-continuity-live-e2e.test.sh +tests/fm-omp-herdr-live-e2e.test.sh +tests/fm-omp-primary-live-e2e.test.sh +tests/fm-claude-stop-autoarm-live-e2e.test.sh +``` + +#### `portable-serial-5of6` — 26 scripts + +```text +tests/fm-hermes-harness.test.sh +tests/fm-ext-waiting-pin.test.sh +tests/fm-wake-queue.test.sh +tests/fm-session-start.test.sh +tests/fm-backlog-atomicity.test.sh +tests/fm-omp-primary.test.sh +tests/fm-boat-lifecycle.test.sh +tests/fm-runpod-pod-boot.test.sh +tests/fm-remote-secondmate-update-restart.test.sh +tests/fm-secondmate-restart.test.sh +tests/fm-afk-inject-e2e.test.sh +tests/fm-prepush-guard.test.sh +tests/fm-ext-thread-context.test.sh +tests/fm-wake-drain-open-decisions.test.sh +tests/fm-shared-captain-inheritance.test.sh +tests/fm-ext-plugin-dest.test.sh +tests/fm-startup-memory-budget.test.sh +tests/fm-send-secondmate-marker.test.sh +tests/fm-treehouse-sweep.test.sh +tests/fm-merge-local.test.sh +tests/fm-afk-return.test.sh +tests/fm-classify-decision-key.test.sh +tests/fm-sessionstart-nudge.test.sh +tests/fm-operational-input.test.sh +tests/fm-treehouse-orphan-recovery.test.sh +tests/fm-omp-relaunch-tmux-live-e2e.test.sh +``` + +#### `portable-serial-6of6` — 30 scripts + +```text +tests/fm-teardown.test.sh +tests/fm-pr-check-security.test.sh +tests/fm-remote-reply.test.sh +tests/fm-bootstrap.test.sh +tests/fm-procevent.test.sh +tests/fm-ext-bridge.test.sh +tests/fm-runpod-lifecycle.test.sh +tests/fm-secondmate-safety.test.sh +tests/fm-runpod-routing.test.sh +tests/fm-task-inbox.test.sh +tests/fm-boat-routing.test.sh +tests/fm-backend.test.sh +tests/fm-busy-adapter-wiring.test.sh +tests/fm-tasks-axi.test.sh +tests/fm-remote-secondmate-parent-binding.test.sh +tests/fm-gate-refuse.test.sh +tests/fm-inbox.test.sh +tests/fm-herdr-session-cleanup.test.sh +tests/fm-idle-sleep-watch.test.sh +tests/fm-backlog-handoff.test.sh +tests/fm-receipt.test.sh +tests/fm-test-fixture-cleanup.test.sh +tests/fm-test-isolation-proof.test.sh +tests/fm-ask-user-authority.test.sh +tests/fm-stall-recovery.test.sh +tests/fm-backend-cmux-smoke.test.sh +tests/fm-devin-herdr-live-e2e.test.sh +tests/fm-herdr-submit-confirm-live-e2e.test.sh +tests/fm-omp-branch-types.test.sh +tests/fm-pi-primary-live-e2e.test.sh +``` -Refresh the hints by downloading the per-shard timing artifacts from a green CI run and replacing the `portable_serial_weight_hints` table in `bin/fm-test-run.sh` with the measured `path`/`duration_ms` pairs: +To refresh after a green run, replace the full hint table with measured per-script pairs and regenerate the membership snapshot below from the same hints. +Copy the generated assignments into the snapshot and run the partition guard. ```sh gh run download -R dnth/firstmate --pattern 'fm-test-timing-portable-serial-*' -D /tmp/fm-serial jq -r '.scripts[] | [.path, .duration_ms] | @tsv' /tmp/fm-serial/*.json | LC_ALL=C sort +for i in 1 2 3 4 5 6; do + printf 'portable-serial-%sof6\n' "$i" + bin/fm-test-run.sh --list --lane "portable-serial-${i}of6" +done bin/fm-test-run.sh --check-coverage ``` @@ -107,7 +340,7 @@ Portable shards, each portable serial shard, and the Herdr lane upload runner-ge | Job | timeout-minutes | Rationale | |---|---:|---| | portable parallel 1/2 | 10 | The measured shard sums are about three minutes and the timeout is a hang tripwire. | -| portable serial 1-6 | 20 | Each rebalanced shard estimates about 14 minutes of script time, leaving about six minutes for setup and runner-load variance while still tripping on a wedged test. | +| portable serial 1-6 | 20 | Each rebalanced shard projects 14m23.259s–14m23.270s of script time, leaving 5m36.730s–5m36.741s for setup and runner-load variance while still tripping on a wedged test. | | Herdr | 40 | The real-Herdr lane keeps its dedicated timeout. | Timeouts are hang tripwires rather than expected healthy durations. From 0b2502957e110ebb1689140bde1655bda5570f48 Mon Sep 17 00:00:00 2001 From: dnth Date: Fri, 16 Oct 2026 00:14:02 +0800 Subject: [PATCH 2/9] no-mistakes(review): Complete cancelled-attempt timing evidence and rebalance rationale --- docs/fm-test-portable-shards.md | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/docs/fm-test-portable-shards.md b/docs/fm-test-portable-shards.md index cc09ce2852b..2d2c722d635 100644 --- a/docs/fm-test-portable-shards.md +++ b/docs/fm-test-portable-shards.md @@ -74,6 +74,7 @@ The recent 20-minute cancellations came from aggregate serial work, not a hangin The cancelled PR run is [38753521432](https://github.com/dnth/firstmate/actions/runs/38753521432). Its first attempt ended during `fm-on.test.sh` after the job reached its timeout, before the checkpoint test began. Its second attempt reached the checkpoint test about five seconds before cancellation, after the preceding serial scripts consumed the available job time. +Its third attempt ended during `fm-wake-drain-unread-status.test.sh` after its wake acknowledgement, about ten seconds after that test began. On main run [38746979839 attempt 1](https://github.com/dnth/firstmate/actions/runs/38746979839/attempts/1), all 28 scripts emitted a 19m55.325s summary before the job was cancelled near its 20-minute limit. The passing main attempt completed the same shard's 28 scripts in 17m20.456s. @@ -82,18 +83,37 @@ The passing main attempt completed the same shard's 28 scripts in 17m20.456s. | [Main 38746979839 attempt 1](https://github.com/dnth/firstmate/actions/runs/38746979839/attempts/1), cancelled | `fm-watch-triage`: 13:56:29–14:03:33 (423,952 ms); `fm-ext-decision`: 14:10:55–14:13:28 (153,704 ms); `fm-watch-checkpoint`: 14:16:03–14:16:10 (5,993 ms). | All scripts completed; the 19m55.325s suite summary preceded cancellation. | | [PR 38753521432 attempt 1](https://github.com/dnth/firstmate/actions/runs/38753521432/attempts/1), cancelled | `fm-watch-triage`: 14:40:54–14:48:14 (440,356 ms); `fm-ext-decision`: 14:55:56–14:58:39 (163,088 ms); `fm-on` began 15:00:52 and the job cancelled at 15:01:00. | Cancelled in the final `fm-on` test; checkpoint did not start. | | [PR 38753521432 attempt 2](https://github.com/dnth/firstmate/actions/runs/38753521432/attempts/2), cancelled | `fm-watch-triage`: 15:05:34–15:12:49 (435,200 ms); `fm-ext-decision`: 15:20:20–15:22:55 (155,491 ms); `fm-watch-checkpoint` began at 15:25:32 and the job cancelled at 15:25:37. | The checkpoint log reached its wake acknowledgement before cancellation; no test-end marker was emitted. | +| [PR 38753521432 attempt 3](https://github.com/dnth/firstmate/actions/runs/38753521432/attempts/3), cancelled | `fm-watch-triage`: 15:30:20–15:37:47 (446,890 ms); `fm-ext-decision`: 15:45:36–15:48:25 (169,627 ms); `fm-wake-drain-unread-status` began at 15:50:11 and the job cancelled at 15:50:21. | Cancelled after the wake acknowledgement in `fm-wake-drain-unread-status`. | | [Main 38746979839 attempt 2](https://github.com/dnth/firstmate/actions/runs/38746979839/attempts/2), passed | `fm-watch-triage`: 14:22:32–14:29:22 (410,055 ms); `fm-ext-decision`: 14:35:28–14:37:25 (116,759 ms); `fm-watch-checkpoint`: 14:39:36–14:39:41 (5,271 ms). | All 28 scripts completed in 17m20.456s. | The checkpoint completed in 5.271s and 5.993s on the passing and cancelled main attempts, respectively. -The cancelled PR attempt that reached it spent only about five seconds there before the 20-minute job limit, while the other cancelled PR attempt ended in `fm-on`. +All three cancelled PR attempts reached the 20-minute limit in different tests: `fm-on`, `fm-watch-checkpoint`, and `fm-wake-drain-unread-status`, so no single test hung. The measured long tests vary across runs but none approaches the job limit; the cause is cumulative shard work. The fix rebalances measured work without changing tests or raising the 20-minute hang tripwire. +Recent green `fm-watch-triage.test.sh` measurements provide the comparison for attempt 3's 446,890 ms: + +| Green run | Duration (ms) | +|---|---:| +| [38540024417](https://github.com/dnth/firstmate/actions/runs/38540024417) | 406,842 | +| [38546582306](https://github.com/dnth/firstmate/actions/runs/38546582306) | 414,725 | +| [38551009549](https://github.com/dnth/firstmate/actions/runs/38551009549) | 419,614 | +| [38535146206](https://github.com/dnth/firstmate/actions/runs/38535146206) | 421,505 | +| [38746979839 attempt 2](https://github.com/dnth/firstmate/actions/runs/38746979839/attempts/2) | 410,055 | + +The 446,890 ms measurement exceeds the recent green maximum of 421,505 ms by 25,385 ms, about 6%, so it is not a large outlier. +Separately, [run 38759616524](https://github.com/dnth/firstmate/actions/runs/38759616524) failed the stale-suppressor assertion after 100,083 ms. +That is a separate correctness signal, not evidence that a 447-second `fm-watch-triage` run caused the timeout; the test assertions remain unchanged. + ### Rebalanced duration profile The refreshed longest-processing-time assignment projects 14m23.259s–14m23.270s per shard from the green run's measured script durations. The balanced six-runner matrix leaves about 5m36.7s for setup and runner-load variance before the 20-minute timeout. +For the PR rationale, `portable-serial-2of6`, which contains `fm-watch-triage.test.sh`, projects 863,263 ms using its 410,055 ms green measurement. +Substituting attempt 3's 446,890 ms yields 863,263 - 410,055 + 446,890 = 900,098 ms (15m00.098s), leaving 299,902 ms (4m59.902s) below the unchanged 20-minute cap. +This remains a safe margin for setup and runner-load variance, and the timeout cause remains aggregate runtime. + | Shard | Scripts | Projected script time | |---|---:|---:| | 1 of 6 | 23 | 14:23.260 | From 9e07cc75f8aa021bb57cdc9997cbcf5ef8427e3a Mon Sep 17 00:00:00 2001 From: dnth Date: Fri, 16 Oct 2026 00:20:31 +0800 Subject: [PATCH 3/9] no-mistakes(document): Correct stale shard comments and documentation reference --- .github/workflows/ci.yml | 6 ++---- bin/fm-test-run.sh | 8 +++----- docs/fm-test-portable-shards.md | 2 +- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7109bfae723..83aa8e5f93d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -192,10 +192,8 @@ jobs: tests-portable-serial: name: Behavior portable serial ${{ matrix.shard }} runs-on: ubuntu-latest - # Refreshed measurements total 86m19.574s and project 14m23.259s–14m23.270s - # per shard. Keep this 20-minute cap as a hang tripwire with about 5m36.7s - # for setup and runner-load variance; refresh hints per - # docs/fm-test-portable-shards.md when a shard approaches it. + # Keep this cap as a hang tripwire; when a shard approaches it, follow the + # balance evidence and refresh procedure in docs/fm-test-portable-shards.md. timeout-minutes: 20 strategy: # Every shard reports so one failure never hides another shard's result. diff --git a/bin/fm-test-run.sh b/bin/fm-test-run.sh index d161cc2ed45..bc195ba4f7b 100755 --- a/bin/fm-test-run.sh +++ b/bin/fm-test-run.sh @@ -94,9 +94,7 @@ JOBS_MAX=8 # One owner: CI lane names carry this count and are refused when they disagree. PORTABLE_SERIAL_SHARDS=6 -# Balance hint for a portable-serial script with no measured duration, close to -# the measured per-script mean so a newly added test neither starves nor -# overloads the shard it lands in. +# Fallback balance hint for a portable-serial script with no measured duration. PORTABLE_SERIAL_DEFAULT_WEIGHT_MS=20000 usage() { @@ -375,8 +373,8 @@ list_portable_serial() { done < <(all_repo_tests) } -# Portable-serial duration hints in milliseconds, from CI artifacts, completed -# log records, and conservative estimates documented in docs/fm-test-portable-shards.md. +# Portable-serial measured duration hints in milliseconds; source evidence is +# documented in docs/fm-test-portable-shards.md. # These affect balance only: the shard partition stays complete and disjoint, so a # stale hint costs balance rather than coverage. That doc owns the refresh # procedure. diff --git a/docs/fm-test-portable-shards.md b/docs/fm-test-portable-shards.md index 2d2c722d635..9cbea8c51a5 100644 --- a/docs/fm-test-portable-shards.md +++ b/docs/fm-test-portable-shards.md @@ -325,7 +325,7 @@ tests/fm-omp-branch-types.test.sh tests/fm-pi-primary-live-e2e.test.sh ``` -To refresh after a green run, replace the full hint table with measured per-script pairs and regenerate the membership snapshot below from the same hints. +To refresh after a green run, replace the full hint table with measured per-script pairs and regenerate the membership snapshot above from the same hints. Copy the generated assignments into the snapshot and run the partition guard. ```sh From 9144f281d413bffe5fa2a24e5d2d3f5bab96bcd0 Mon Sep 17 00:00:00 2001 From: dnth Date: Fri, 16 Oct 2026 01:06:01 +0800 Subject: [PATCH 4/9] no-mistakes(ci): Captain, fixed the reproduced reply-capture race by selecting each acknowledgement by correlation token. The original failing case and later cleanup cases now pass. Partition guard, focused ShellCheck, and whitespace checks pass. Full lifecycle verification stopped later because existing /tmp/fm-fallback-omp permissions are unsafe; that directory was left untouched --- tests/fm-remote-secondmate-lifecycle-e2e.test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fm-remote-secondmate-lifecycle-e2e.test.sh b/tests/fm-remote-secondmate-lifecycle-e2e.test.sh index dd0062d7b78..68d8a2e8f46 100755 --- a/tests/fm-remote-secondmate-lifecycle-e2e.test.sh +++ b/tests/fm-remote-secondmate-lifecycle-e2e.test.sh @@ -970,7 +970,7 @@ resolve_ios_pending() { remote_env "$ROOT/bin/fm-procevent.sh" start "$SID" >/dev/null 2>&1 || true pending_result="" n=200 while [ -z "$pending_result" ] && [ "$n" -gt 0 ]; do - pending_result=$(find "$PARENT/state/procevent-inbox" -name "$SID.*.result" ! -name "$SID.1.result" ! -name "$SID.2.result" ! -name "$SID.3.result" -print | sort | tail -1) + pending_result=$(grep -lF "done [corr=$pending_corr]:" "$PARENT/state/procevent-inbox/$SID".*.result 2>/dev/null | head -1 || true) [ -n "$pending_result" ] || { n=$((n - 1)); sleep 0.1; } done [ -n "$pending_result" ] \ From 24d0de30efefca363891f69a58169239b582ff22 Mon Sep 17 00:00:00 2001 From: dnth Date: Fri, 16 Oct 2026 02:00:53 +0800 Subject: [PATCH 5/9] fix(ci): balance portable serial tests across seven shards Use the measured 91-minute serial suite from PR run 38774522507 to distribute work across seven CI runners. Refresh the duration hints and membership snapshot while keeping each shard serial and the 20-minute timeout unchanged. --- .github/workflows/ci.yml | 2 +- bin/fm-test-run.sh | 328 ++++++++++++++-------------- docs/fm-test-portable-shards.md | 365 +++++++++++++++++--------------- 3 files changed, 359 insertions(+), 336 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 83aa8e5f93d..c3e072160ed 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -199,7 +199,7 @@ jobs: # Every shard reports so one failure never hides another shard's result. fail-fast: false matrix: - shard: [1, 2, 3, 4, 5, 6] + shard: [1, 2, 3, 4, 5, 6, 7] steps: - uses: actions/checkout@v6 with: diff --git a/bin/fm-test-run.sh b/bin/fm-test-run.sh index bc195ba4f7b..afc6dd3034a 100755 --- a/bin/fm-test-run.sh +++ b/bin/fm-test-run.sh @@ -92,7 +92,7 @@ JOBS_MAX=8 # How many separate-runner shards the portable serial remainder splits into. # One owner: CI lane names carry this count and are refused when they disagree. -PORTABLE_SERIAL_SHARDS=6 +PORTABLE_SERIAL_SHARDS=7 # Fallback balance hint for a portable-serial script with no measured duration. PORTABLE_SERIAL_DEFAULT_WEIGHT_MS=20000 @@ -380,172 +380,172 @@ list_portable_serial() { # procedure. portable_serial_weight_hints() { cat <<'EOF' -tests/fm-afk-inject-e2e.test.sh 25557 +tests/fm-afk-inject-e2e.test.sh 26538 tests/fm-afk-pi-herdr-return-e2e.test.sh 76 -tests/fm-afk-return.test.sh 1513 -tests/fm-ask-user-authority.test.sh 170 -tests/fm-backend-cmux-smoke.test.sh 22 -tests/fm-backend-cmux.test.sh 3245 -tests/fm-backend-herdr-focus-flash-e2e.test.sh 14 -tests/fm-backend-herdr-server-detach.test.sh 2631 -tests/fm-backend-orca.test.sh 16662 -tests/fm-backend-tmux-smoke.test.sh 191 -tests/fm-backend-zellij-smoke.test.sh 17 -tests/fm-backend-zellij.test.sh 5810 -tests/fm-backend.test.sh 20363 -tests/fm-backlog-atomicity.test.sh 63738 -tests/fm-backlog-handoff.test.sh 2814 -tests/fm-bearings-snapshot.test.sh 61847 -tests/fm-boat-lane-runner.test.sh 120149 -tests/fm-boat-lifecycle.test.sh 49402 -tests/fm-boat-omp-auth.test.sh 62421 -tests/fm-boat-routing.test.sh 27470 -tests/fm-bootstrap.test.sh 72800 -tests/fm-busy-adapter-wiring.test.sh 18965 -tests/fm-busy-state.test.sh 2834 -tests/fm-calm-pi-extension.test.sh 256 -tests/fm-check-unregister.test.sh 361 -tests/fm-classify-decision-key.test.sh 542 -tests/fm-claude-stop-autoarm-live-e2e.test.sh 14 -tests/fm-claude-stop-autoarm.test.sh 60541 -tests/fm-cleanup-skill.test.sh 58 -tests/fm-codex-continuity-live-e2e.test.sh 21 -tests/fm-daemon.test.sh 50403 -tests/fm-devin-adapter.test.sh 284 -tests/fm-devin-herdr-live-e2e.test.sh 20 -tests/fm-dispatch-resolve.test.sh 6418 -tests/fm-documentation-audiences.test.sh 879 -tests/fm-ext-ask.test.sh 48612 -tests/fm-ext-bridge.test.sh 61703 -tests/fm-ext-decision.test.sh 116759 -tests/fm-ext-plugin-dest.test.sh 8528 -tests/fm-ext-thread-context.test.sh 17972 -tests/fm-ext-topic-threads.test.sh 81072 -tests/fm-ext-waiting-pin.test.sh 124489 -tests/fm-ext-watch.test.sh 7673 -tests/fm-fleet-snapshot-view.test.sh 13346 -tests/fm-fleet-sync.test.sh 33947 -tests/fm-gate-refuse.test.sh 9734 -tests/fm-gitignore-config.test.sh 27 -tests/fm-gotmp.test.sh 908 -tests/fm-grok-continuity-live-e2e.test.sh 21 -tests/fm-grok-stop-live-e2e.test.sh 21 -tests/fm-guard-stale-banner.test.sh 4737 -tests/fm-herdr-session-cleanup.test.sh 5318 -tests/fm-herdr-submit-confirm-live-e2e.test.sh 18 -tests/fm-hermes-harness.test.sh 147035 -tests/fm-hermes-live-e2e.test.sh 42 -tests/fm-idle-sleep-watch.test.sh 4199 -tests/fm-idle-sleep.test.sh 12252 -tests/fm-inbox-result.test.sh 10147 -tests/fm-inbox.test.sh 8356 -tests/fm-kimi-harness.test.sh 17704 -tests/fm-landing.test.sh 34429 -tests/fm-main-ci-watch.test.sh 40258 -tests/fm-merge-local.test.sh 2363 -tests/fm-omp-branch-bash.test.sh 35 -tests/fm-omp-branch-live-e2e.test.sh 14 -tests/fm-omp-branch-supervision.test.sh 37983 -tests/fm-omp-branch-types.test.sh 15 -tests/fm-omp-compact-adviser-live-e2e.test.sh 14 -tests/fm-omp-compact-adviser.test.sh 1592 -tests/fm-omp-fleet-hooks.test.sh 2522 -tests/fm-omp-harness.test.sh 66551 -tests/fm-omp-herdr-exit-live-e2e.test.sh 21 +tests/fm-afk-return.test.sh 1524 +tests/fm-ask-user-authority.test.sh 154 +tests/fm-backend-cmux-smoke.test.sh 34 +tests/fm-backend-cmux.test.sh 3751 +tests/fm-backend-herdr-focus-flash-e2e.test.sh 22 +tests/fm-backend-herdr-server-detach.test.sh 2649 +tests/fm-backend-orca.test.sh 19716 +tests/fm-backend-tmux-smoke.test.sh 250 +tests/fm-backend-zellij-smoke.test.sh 22 +tests/fm-backend-zellij.test.sh 7129 +tests/fm-backend.test.sh 23256 +tests/fm-backlog-atomicity.test.sh 63215 +tests/fm-backlog-handoff.test.sh 3410 +tests/fm-bearings-snapshot.test.sh 81131 +tests/fm-boat-lane-runner.test.sh 119102 +tests/fm-boat-lifecycle.test.sh 44937 +tests/fm-boat-omp-auth.test.sh 62445 +tests/fm-boat-routing.test.sh 26287 +tests/fm-bootstrap.test.sh 71450 +tests/fm-busy-adapter-wiring.test.sh 19619 +tests/fm-busy-state.test.sh 2879 +tests/fm-calm-pi-extension.test.sh 265 +tests/fm-check-unregister.test.sh 436 +tests/fm-classify-decision-key.test.sh 501 +tests/fm-claude-stop-autoarm-live-e2e.test.sh 20 +tests/fm-claude-stop-autoarm.test.sh 60780 +tests/fm-cleanup-skill.test.sh 74 +tests/fm-codex-continuity-live-e2e.test.sh 20 +tests/fm-daemon.test.sh 60677 +tests/fm-devin-adapter.test.sh 349 +tests/fm-devin-herdr-live-e2e.test.sh 21 +tests/fm-dispatch-resolve.test.sh 8747 +tests/fm-documentation-audiences.test.sh 878 +tests/fm-ext-ask.test.sh 48724 +tests/fm-ext-bridge.test.sh 82962 +tests/fm-ext-decision.test.sh 161682 +tests/fm-ext-plugin-dest.test.sh 7190 +tests/fm-ext-thread-context.test.sh 18046 +tests/fm-ext-topic-threads.test.sh 100660 +tests/fm-ext-waiting-pin.test.sh 103172 +tests/fm-ext-watch.test.sh 7522 +tests/fm-fleet-snapshot-view.test.sh 12998 +tests/fm-fleet-sync.test.sh 36580 +tests/fm-gate-refuse.test.sh 11343 +tests/fm-gitignore-config.test.sh 29 +tests/fm-gotmp.test.sh 1168 +tests/fm-grok-continuity-live-e2e.test.sh 19 +tests/fm-grok-stop-live-e2e.test.sh 20 +tests/fm-guard-stale-banner.test.sh 5898 +tests/fm-herdr-session-cleanup.test.sh 6849 +tests/fm-herdr-submit-confirm-live-e2e.test.sh 22 +tests/fm-hermes-harness.test.sh 151616 +tests/fm-hermes-live-e2e.test.sh 44 +tests/fm-idle-sleep-watch.test.sh 4560 +tests/fm-idle-sleep.test.sh 13959 +tests/fm-inbox-result.test.sh 9828 +tests/fm-inbox.test.sh 9840 +tests/fm-kimi-harness.test.sh 18045 +tests/fm-landing.test.sh 43476 +tests/fm-main-ci-watch.test.sh 46302 +tests/fm-merge-local.test.sh 2502 +tests/fm-omp-branch-bash.test.sh 41 +tests/fm-omp-branch-live-e2e.test.sh 20 +tests/fm-omp-branch-supervision.test.sh 37015 +tests/fm-omp-branch-types.test.sh 21 +tests/fm-omp-compact-adviser-live-e2e.test.sh 20 +tests/fm-omp-compact-adviser.test.sh 1697 +tests/fm-omp-fleet-hooks.test.sh 2743 +tests/fm-omp-harness.test.sh 66500 +tests/fm-omp-herdr-exit-live-e2e.test.sh 19 tests/fm-omp-herdr-live-e2e.test.sh 19 -tests/fm-omp-primary-live-e2e.test.sh 16 -tests/fm-omp-primary.test.sh 61145 -tests/fm-omp-relaunch-guard.test.sh 5403 -tests/fm-omp-relaunch-tmux-live-e2e.test.sh 15 +tests/fm-omp-primary-live-e2e.test.sh 19 +tests/fm-omp-primary.test.sh 61283 +tests/fm-omp-relaunch-guard.test.sh 5534 +tests/fm-omp-relaunch-tmux-live-e2e.test.sh 14 tests/fm-omp-secondmate-live-e2e.test.sh 20 -tests/fm-omp-secondmate.test.sh 103752 -tests/fm-omp-task-inbox-doorbell.test.sh 19102 -tests/fm-omp-update.test.sh 497 -tests/fm-omp-worker-tmux-live-e2e.test.sh 16 -tests/fm-on.test.sh 8384 -tests/fm-opencode-primary-live-e2e.test.sh 19 -tests/fm-operational-input.test.sh 179 -tests/fm-pending-reply-sleep-reconcile.test.sh 178 -tests/fm-pending-reply.test.sh 11519 -tests/fm-pi-compatible-family.test.sh 58 -tests/fm-pi-primary-live-e2e.test.sh 14 -tests/fm-pi-watch-extension.test.sh 22138 -tests/fm-pr-check-handoff.test.sh 16266 -tests/fm-pr-check-security.test.sh 137546 -tests/fm-prepush-guard.test.sh 21813 -tests/fm-procevent-when.test.sh 19951 -tests/fm-procevent.test.sh 65023 -tests/fm-public-followup.test.sh 37602 -tests/fm-quota-array-dispatch-live-e2e.test.sh 16 -tests/fm-receipt-check.test.sh 6298 -tests/fm-receipt.test.sh 1469 -tests/fm-reflect-skill.test.sh 480 -tests/fm-remote-backlog-handoff.test.sh 19810 -tests/fm-remote-doctor.test.sh 4824 -tests/fm-remote-entrypoint.test.sh 199 -tests/fm-remote-job.test.sh 49058 -tests/fm-remote-omp-update-restart-live-e2e.test.sh 19 -tests/fm-remote-reply.test.sh 85746 -tests/fm-remote-secondmate-lifecycle-e2e.test.sh 660575 -tests/fm-remote-secondmate-parent-binding.test.sh 12099 -tests/fm-remote-secondmate-sleep-reconcile.test.sh 1354 -tests/fm-remote-secondmate-trace-context.test.sh 59443 -tests/fm-remote-secondmate-update-restart.test.sh 38017 -tests/fm-runpod-lifecycle.test.sh 51268 -tests/fm-runpod-omp-auth.test.sh 988 -tests/fm-runpod-pod-boot.test.sh 43745 -tests/fm-runpod-routing.test.sh 37972 -tests/fm-secondmate-harness.test.sh 149950 -tests/fm-secondmate-lifecycle-e2e.test.sh 5351 -tests/fm-secondmate-liveness.test.sh 23406 -tests/fm-secondmate-restart.test.sh 37021 -tests/fm-secondmate-safety.test.sh 46061 -tests/fm-secondmate-sync.test.sh 37220 -tests/fm-send-inbox-doorbell-live-e2e.test.sh 23 -tests/fm-send-inbox.test.sh 40599 -tests/fm-send-resolve-key.test.sh 20275 -tests/fm-send-secondmate-marker-herdr-e2e.test.sh 52 -tests/fm-send-secondmate-marker.test.sh 6023 -tests/fm-send-turn-start.test.sh 11269 -tests/fm-session-lock-ancestry.test.sh 1524 -tests/fm-session-start.test.sh 69482 -tests/fm-sessionstart-nudge.test.sh 407 -tests/fm-shared-captain-inheritance.test.sh 13607 -tests/fm-spawn-dispatch-profile.test.sh 209130 -tests/fm-spawn-pool-base-freshen.test.sh 35517 -tests/fm-spawn-relaunch-dead-endpoint.test.sh 35325 -tests/fm-spawn-worktree-settle.test.sh 11206 -tests/fm-stall-recovery.test.sh 46 -tests/fm-startup-memory-budget.test.sh 7617 -tests/fm-subagent-pretool-check.test.sh 777 -tests/fm-supervision-events.test.sh 634 -tests/fm-tangle-guard.test.sh 9372 -tests/fm-task-delivery.test.sh 15612 -tests/fm-task-inbox.test.sh 35336 -tests/fm-tasks-axi.test.sh 16934 -tests/fm-teardown-endpoint-safety.test.sh 30753 -tests/fm-teardown.test.sh 140801 -tests/fm-test-fixture-cleanup.test.sh 515 -tests/fm-test-isolation-proof.test.sh 473 -tests/fm-todo-project.test.sh 9253 -tests/fm-trace-context-lib.test.sh 1626 -tests/fm-trace-context-spawn.test.sh 41182 -tests/fm-treehouse-orphan-recovery.test.sh 157 -tests/fm-treehouse-status-read-only.test.sh 230 -tests/fm-treehouse-sweep.test.sh 4598 -tests/fm-turnend-guard.test.sh 17221 -tests/fm-update.test.sh 7384 -tests/fm-vendor-auth-probe.test.sh 42704 -tests/fm-wake-daemon-lifecycle-e2e.test.sh 60312 -tests/fm-wake-drain-open-decisions-cursor.test.sh 22524 -tests/fm-wake-drain-open-decisions.test.sh 17306 -tests/fm-wake-drain-unread-status.test.sh 24217 -tests/fm-wake-queue.test.sh 100988 -tests/fm-watch-arm.test.sh 66908 -tests/fm-watch-checkpoint.test.sh 5271 -tests/fm-watch-recovery-loop.test.sh 64620 -tests/fm-watch-triage.test.sh 410055 -tests/fm-watcher-lock.test.sh 67720 +tests/fm-omp-secondmate.test.sh 113332 +tests/fm-omp-task-inbox-doorbell.test.sh 20456 +tests/fm-omp-update.test.sh 689 +tests/fm-omp-worker-tmux-live-e2e.test.sh 20 +tests/fm-on.test.sh 9816 +tests/fm-opencode-primary-live-e2e.test.sh 20 +tests/fm-operational-input.test.sh 154 +tests/fm-pending-reply-sleep-reconcile.test.sh 229 +tests/fm-pending-reply.test.sh 15024 +tests/fm-pi-compatible-family.test.sh 70 +tests/fm-pi-primary-live-e2e.test.sh 21 +tests/fm-pi-watch-extension.test.sh 22125 +tests/fm-pr-check-handoff.test.sh 15330 +tests/fm-pr-check-security.test.sh 163835 +tests/fm-prepush-guard.test.sh 19861 +tests/fm-procevent-when.test.sh 21536 +tests/fm-procevent.test.sh 69535 +tests/fm-public-followup.test.sh 47380 +tests/fm-quota-array-dispatch-live-e2e.test.sh 19 +tests/fm-receipt-check.test.sh 6254 +tests/fm-receipt.test.sh 1854 +tests/fm-reflect-skill.test.sh 473 +tests/fm-remote-backlog-handoff.test.sh 20170 +tests/fm-remote-doctor.test.sh 5787 +tests/fm-remote-entrypoint.test.sh 263 +tests/fm-remote-job.test.sh 48956 +tests/fm-remote-omp-update-restart-live-e2e.test.sh 20 +tests/fm-remote-reply.test.sh 82089 +tests/fm-remote-secondmate-lifecycle-e2e.test.sh 657637 +tests/fm-remote-secondmate-parent-binding.test.sh 15757 +tests/fm-remote-secondmate-sleep-reconcile.test.sh 1287 +tests/fm-remote-secondmate-trace-context.test.sh 59252 +tests/fm-remote-secondmate-update-restart.test.sh 30187 +tests/fm-runpod-lifecycle.test.sh 58778 +tests/fm-runpod-omp-auth.test.sh 1192 +tests/fm-runpod-pod-boot.test.sh 43135 +tests/fm-runpod-routing.test.sh 37449 +tests/fm-secondmate-harness.test.sh 170105 +tests/fm-secondmate-lifecycle-e2e.test.sh 6278 +tests/fm-secondmate-liveness.test.sh 20034 +tests/fm-secondmate-restart.test.sh 36334 +tests/fm-secondmate-safety.test.sh 58742 +tests/fm-secondmate-sync.test.sh 40216 +tests/fm-send-inbox-doorbell-live-e2e.test.sh 22 +tests/fm-send-inbox.test.sh 40199 +tests/fm-send-resolve-key.test.sh 22866 +tests/fm-send-secondmate-marker-herdr-e2e.test.sh 49 +tests/fm-send-secondmate-marker.test.sh 4337 +tests/fm-send-turn-start.test.sh 11083 +tests/fm-session-lock-ancestry.test.sh 1513 +tests/fm-session-start.test.sh 65773 +tests/fm-sessionstart-nudge.test.sh 303 +tests/fm-shared-captain-inheritance.test.sh 13503 +tests/fm-spawn-dispatch-profile.test.sh 249954 +tests/fm-spawn-pool-base-freshen.test.sh 35286 +tests/fm-spawn-relaunch-dead-endpoint.test.sh 35039 +tests/fm-spawn-worktree-settle.test.sh 11488 +tests/fm-stall-recovery.test.sh 48 +tests/fm-startup-memory-budget.test.sh 6514 +tests/fm-subagent-pretool-check.test.sh 968 +tests/fm-supervision-events.test.sh 628 +tests/fm-tangle-guard.test.sh 10406 +tests/fm-task-delivery.test.sh 19399 +tests/fm-task-inbox.test.sh 37373 +tests/fm-tasks-axi.test.sh 18227 +tests/fm-teardown-endpoint-safety.test.sh 30883 +tests/fm-teardown.test.sh 137584 +tests/fm-test-fixture-cleanup.test.sh 681 +tests/fm-test-isolation-proof.test.sh 588 +tests/fm-todo-project.test.sh 9125 +tests/fm-trace-context-lib.test.sh 1997 +tests/fm-trace-context-spawn.test.sh 45140 +tests/fm-treehouse-orphan-recovery.test.sh 111 +tests/fm-treehouse-status-read-only.test.sh 223 +tests/fm-treehouse-sweep.test.sh 4347 +tests/fm-turnend-guard.test.sh 21290 +tests/fm-update.test.sh 9518 +tests/fm-vendor-auth-probe.test.sh 42927 +tests/fm-wake-daemon-lifecycle-e2e.test.sh 69696 +tests/fm-wake-drain-open-decisions-cursor.test.sh 26536 +tests/fm-wake-drain-open-decisions.test.sh 12358 +tests/fm-wake-drain-unread-status.test.sh 30489 +tests/fm-wake-queue.test.sh 85314 +tests/fm-watch-arm.test.sh 85430 +tests/fm-watch-checkpoint.test.sh 5858 +tests/fm-watch-recovery-loop.test.sh 66814 +tests/fm-watch-triage.test.sh 428164 +tests/fm-watcher-lock.test.sh 46975 EOF } diff --git a/docs/fm-test-portable-shards.md b/docs/fm-test-portable-shards.md index 9cbea8c51a5..220e9ac00e6 100644 --- a/docs/fm-test-portable-shards.md +++ b/docs/fm-test-portable-shards.md @@ -56,7 +56,7 @@ The per-shard membership snapshot below records the generated assignment for the ## Portable serial CI shards -The refreshed hints use all 166 completed serial-script measurements from green run [38746979839 attempt 2](https://github.com/dnth/firstmate/actions/runs/38746979839/attempts/2), totaling 5,179,574 ms (86m19.6s). +The refreshed hints use all 166 completed serial-script measurements from PR run [38774522507](https://github.com/dnth/firstmate/actions/runs/38774522507), totaling 5,461,112 ms (91m01.112s). The measured durations replace the older estimates and default weights in `portable_serial_weight_hints`. `portable-serial-of` splits that remainder across `n` separate CI runners. Each shard is still strictly serial in itself, and separate runners mean no two of these stateful scripts ever share a machine, so the split needs no concurrency isolation proof. @@ -85,244 +85,267 @@ The passing main attempt completed the same shard's 28 scripts in 17m20.456s. | [PR 38753521432 attempt 2](https://github.com/dnth/firstmate/actions/runs/38753521432/attempts/2), cancelled | `fm-watch-triage`: 15:05:34–15:12:49 (435,200 ms); `fm-ext-decision`: 15:20:20–15:22:55 (155,491 ms); `fm-watch-checkpoint` began at 15:25:32 and the job cancelled at 15:25:37. | The checkpoint log reached its wake acknowledgement before cancellation; no test-end marker was emitted. | | [PR 38753521432 attempt 3](https://github.com/dnth/firstmate/actions/runs/38753521432/attempts/3), cancelled | `fm-watch-triage`: 15:30:20–15:37:47 (446,890 ms); `fm-ext-decision`: 15:45:36–15:48:25 (169,627 ms); `fm-wake-drain-unread-status` began at 15:50:11 and the job cancelled at 15:50:21. | Cancelled after the wake acknowledgement in `fm-wake-drain-unread-status`. | | [Main 38746979839 attempt 2](https://github.com/dnth/firstmate/actions/runs/38746979839/attempts/2), passed | `fm-watch-triage`: 14:22:32–14:29:22 (410,055 ms); `fm-ext-decision`: 14:35:28–14:37:25 (116,759 ms); `fm-watch-checkpoint`: 14:39:36–14:39:41 (5,271 ms). | All 28 scripts completed in 17m20.456s. | +| [PR 38774522507](https://github.com/dnth/firstmate/actions/runs/38774522507), passing six-shard run | `fm-spawn-dispatch-profile`: 17:06:27–17:10:37 (249,954 ms); `fm-ext-decision`: 17:10:37–17:13:18 (161,682 ms); `fm-ext-topic-threads`: 17:13:18–17:14:59 (100,660 ms); `fm-watch-arm`: 17:14:59–17:16:25 (85,430 ms). | The 30 scripts summed to 1,022,126 ms (17m02.126s); job wall time was 17m17s. | The checkpoint completed in 5.271s and 5.993s on the passing and cancelled main attempts, respectively. All three cancelled PR attempts reached the 20-minute limit in different tests: `fm-on`, `fm-watch-checkpoint`, and `fm-wake-drain-unread-status`, so no single test hung. -The measured long tests vary across runs but none approaches the job limit; the cause is cumulative shard work. -The fix rebalances measured work without changing tests or raising the 20-minute hang tripwire. +The six-shard PR run 38774522507 completed every script, but its 91m01.112s total exceeds six times the ~14-minute budget (84 minutes); shard 3 summed to 17m02.126s and took 17m17s wall time. +The slowest script in shard 3 was `fm-spawn-dispatch-profile.test.sh` at 249,954 ms (4m09.954s), so the shard overrun is aggregate growth, not a single hung test. +Seven shards are required to keep the refreshed per-script assignment near the documented budget; test assertions and the 20-minute cap remain unchanged. Recent green `fm-watch-triage.test.sh` measurements provide the comparison for attempt 3's 446,890 ms: -| Green run | Duration (ms) | +| Passing run | Duration (ms) | |---|---:| | [38540024417](https://github.com/dnth/firstmate/actions/runs/38540024417) | 406,842 | | [38546582306](https://github.com/dnth/firstmate/actions/runs/38546582306) | 414,725 | | [38551009549](https://github.com/dnth/firstmate/actions/runs/38551009549) | 419,614 | | [38535146206](https://github.com/dnth/firstmate/actions/runs/38535146206) | 421,505 | | [38746979839 attempt 2](https://github.com/dnth/firstmate/actions/runs/38746979839/attempts/2) | 410,055 | +| [PR 38774522507](https://github.com/dnth/firstmate/actions/runs/38774522507) | 428,164 | -The 446,890 ms measurement exceeds the recent green maximum of 421,505 ms by 25,385 ms, about 6%, so it is not a large outlier. +The 446,890 ms result exceeds the latest passing measurement in this table, 428,164 ms, by 18,726 ms (about 4.4%), so it is not a large outlier. +PR run 38774522507 measured `fm-watch-triage.test.sh` at 428,164 ms (7m08.164s), within the observed range. Separately, [run 38759616524](https://github.com/dnth/firstmate/actions/runs/38759616524) failed the stale-suppressor assertion after 100,083 ms. That is a separate correctness signal, not evidence that a 447-second `fm-watch-triage` run caused the timeout; the test assertions remain unchanged. ### Rebalanced duration profile -The refreshed longest-processing-time assignment projects 14m23.259s–14m23.270s per shard from the green run's measured script durations. -The balanced six-runner matrix leaves about 5m36.7s for setup and runner-load variance before the 20-minute timeout. +The six-shard PR run measured 5,461,112 ms (91m01.112s) across 166 serial scripts, averaging 15m10.185s per shard. +Seven shards are needed because six shards cannot fit that measured total within the ~14-minute budget. -For the PR rationale, `portable-serial-2of6`, which contains `fm-watch-triage.test.sh`, projects 863,263 ms using its 410,055 ms green measurement. -Substituting attempt 3's 446,890 ms yields 863,263 - 410,055 + 446,890 = 900,098 ms (15m00.098s), leaving 299,902 ms (4m59.902s) below the unchanged 20-minute cap. -This remains a safe margin for setup and runner-load variance, and the timeout cause remains aggregate runtime. +PR run 38774522507 measured `fm-watch-triage.test.sh` at 428,164 ms (7m08.164s); the earlier PR attempt 3 measured 446,890 ms (7m26.890s). +If `fm-watch-triage.test.sh` reaches 446,890 ms again, shard 2 rises by only 18,726 ms to 798,888 ms (13m18.888s), still within the ~14-minute budget. +The seven-shard assignment covers the observed variation without raising the 20-minute hang tripwire. + +| Shard | Scripts | Measured six-shard script sum | Six-shard CI job wall | +|---|---:|---:|---:| +| 1 of 6 | 23 | 14:28.264 | 14:46 | +| 2 of 6 | 27 | 15:27.115 | 15:44 | +| 3 of 6 | 30 | 17:02.126 | 17:17 | +| 4 of 6 | 30 | 14:58.449 | 15:13 | +| 5 of 6 | 26 | 13:22.769 | 13:36 | +| 6 of 6 | 30 | 15:42.389 | 15:59 | +| Total | 166 | 91:01.112 | — | + +The seven-shard LPT assignment uses those same per-script measurements and balances within 15 ms. | Shard | Scripts | Projected script time | |---|---:|---:| -| 1 of 6 | 23 | 14:23.260 | -| 2 of 6 | 27 | 14:23.263 | -| 3 of 6 | 30 | 14:23.262 | -| 4 of 6 | 30 | 14:23.260 | -| 5 of 6 | 26 | 14:23.259 | -| 6 of 6 | 30 | 14:23.270 | -| Total | 166 | 86:19.574 | - -The longest measured script is `tests/fm-remote-secondmate-lifecycle-e2e.test.sh` at 660,575 ms (11m00.575s), which is the single-script lower bound for any shard count. +| 1 of 7 | 17 | 13:00.156 | +| 2 of 7 | 23 | 13:00.162 | +| 3 of 7 | 23 | 13:00.151 | +| 4 of 7 | 25 | 13:00.154 | +| 5 of 7 | 26 | 13:00.161 | +| 6 of 7 | 26 | 13:00.162 | +| 7 of 7 | 26 | 13:00.166 | +| Total | 166 | 91:01.112 | + +The longest measured script is `tests/fm-remote-secondmate-lifecycle-e2e.test.sh` at 657,637 ms (10m57.637s), which is the single-script lower bound for any shard count. + The current shard memberships below are generated from the same hints. ### Current shard membership -#### `portable-serial-1of6` — 23 scripts +#### `portable-serial-1of7` — 17 scripts ```text tests/fm-remote-secondmate-lifecycle-e2e.test.sh -tests/fm-omp-branch-supervision.test.sh -tests/fm-spawn-pool-base-freshen.test.sh -tests/fm-teardown-endpoint-safety.test.sh -tests/fm-send-resolve-key.test.sh -tests/fm-kimi-harness.test.sh -tests/fm-task-delivery.test.sh -tests/fm-send-turn-start.test.sh -tests/fm-inbox-result.test.sh -tests/fm-ext-watch.test.sh -tests/fm-backend-zellij.test.sh -tests/fm-guard-stale-banner.test.sh -tests/fm-backend-herdr-server-detach.test.sh +tests/fm-wake-drain-unread-status.test.sh +tests/fm-pi-watch-extension.test.sh +tests/fm-backend-orca.test.sh +tests/fm-pr-check-handoff.test.sh +tests/fm-gate-refuse.test.sh +tests/fm-todo-project.test.sh +tests/fm-secondmate-lifecycle-e2e.test.sh +tests/fm-treehouse-sweep.test.sh +tests/fm-omp-compact-adviser.test.sh tests/fm-runpod-omp-auth.test.sh -tests/fm-subagent-pretool-check.test.sh -tests/fm-reflect-skill.test.sh -tests/fm-pending-reply-sleep-reconcile.test.sh -tests/fm-send-secondmate-marker-herdr-e2e.test.sh -tests/fm-gitignore-config.test.sh -tests/fm-codex-continuity-live-e2e.test.sh -tests/fm-omp-secondmate-live-e2e.test.sh -tests/fm-backend-zellij-smoke.test.sh -tests/fm-backend-herdr-focus-flash-e2e.test.sh +tests/fm-test-isolation-proof.test.sh +tests/fm-operational-input.test.sh +tests/fm-cleanup-skill.test.sh +tests/fm-send-inbox-doorbell-live-e2e.test.sh +tests/fm-omp-branch-live-e2e.test.sh +tests/fm-omp-herdr-exit-live-e2e.test.sh ``` -#### `portable-serial-2of6` — 27 scripts +#### `portable-serial-2of7` — 23 scripts ```text tests/fm-watch-triage.test.sh tests/fm-omp-harness.test.sh -tests/fm-bearings-snapshot.test.sh -tests/fm-remote-secondmate-trace-context.test.sh -tests/fm-ext-ask.test.sh -tests/fm-send-inbox.test.sh -tests/fm-secondmate-sync.test.sh -tests/fm-landing.test.sh -tests/fm-wake-drain-open-decisions-cursor.test.sh +tests/fm-runpod-lifecycle.test.sh +tests/fm-main-ci-watch.test.sh +tests/fm-task-inbox.test.sh +tests/fm-secondmate-restart.test.sh +tests/fm-backend.test.sh tests/fm-procevent-when.test.sh -tests/fm-backend-orca.test.sh -tests/fm-idle-sleep.test.sh +tests/fm-task-delivery.test.sh +tests/fm-wake-drain-open-decisions.test.sh tests/fm-tangle-guard.test.sh -tests/fm-on.test.sh -tests/fm-secondmate-lifecycle-e2e.test.sh -tests/fm-remote-doctor.test.sh -tests/fm-omp-fleet-hooks.test.sh -tests/fm-remote-secondmate-sleep-reconcile.test.sh -tests/fm-supervision-events.test.sh -tests/fm-devin-adapter.test.sh -tests/fm-treehouse-status-read-only.test.sh -tests/fm-pi-compatible-family.test.sh +tests/fm-ext-plugin-dest.test.sh +tests/fm-guard-stale-banner.test.sh +tests/fm-busy-state.test.sh +tests/fm-receipt.test.sh +tests/fm-gotmp.test.sh +tests/fm-check-unregister.test.sh +tests/fm-ask-user-authority.test.sh +tests/fm-afk-pi-herdr-return-e2e.test.sh tests/fm-omp-branch-bash.test.sh -tests/fm-omp-herdr-exit-live-e2e.test.sh -tests/fm-remote-omp-update-restart-live-e2e.test.sh -tests/fm-quota-array-dispatch-live-e2e.test.sh -tests/fm-omp-compact-adviser-live-e2e.test.sh +tests/fm-omp-branch-types.test.sh +tests/fm-omp-secondmate-live-e2e.test.sh +tests/fm-omp-primary-live-e2e.test.sh ``` -#### `portable-serial-3of6` — 30 scripts +#### `portable-serial-3of7` — 23 scripts ```text tests/fm-spawn-dispatch-profile.test.sh -tests/fm-ext-decision.test.sh tests/fm-ext-topic-threads.test.sh -tests/fm-watch-arm.test.sh -tests/fm-boat-omp-auth.test.sh -tests/fm-wake-daemon-lifecycle-e2e.test.sh -tests/fm-remote-job.test.sh +tests/fm-bearings-snapshot.test.sh +tests/fm-backlog-atomicity.test.sh +tests/fm-secondmate-safety.test.sh tests/fm-trace-context-spawn.test.sh -tests/fm-public-followup.test.sh +tests/fm-runpod-routing.test.sh tests/fm-fleet-sync.test.sh -tests/fm-secondmate-liveness.test.sh -tests/fm-omp-task-inbox-doorbell.test.sh -tests/fm-turnend-guard.test.sh -tests/fm-pending-reply.test.sh -tests/fm-spawn-worktree-settle.test.sh -tests/fm-dispatch-resolve.test.sh -tests/fm-receipt-check.test.sh -tests/fm-backend-cmux.test.sh -tests/fm-busy-state.test.sh -tests/fm-omp-compact-adviser.test.sh -tests/fm-gotmp.test.sh -tests/fm-omp-update.test.sh +tests/fm-boat-routing.test.sh +tests/fm-remote-backlog-handoff.test.sh +tests/fm-tasks-axi.test.sh +tests/fm-fleet-snapshot-view.test.sh +tests/fm-inbox.test.sh +tests/fm-ext-watch.test.sh +tests/fm-omp-relaunch-guard.test.sh +tests/fm-send-secondmate-marker.test.sh +tests/fm-remote-secondmate-sleep-reconcile.test.sh +tests/fm-supervision-events.test.sh tests/fm-calm-pi-extension.test.sh -tests/fm-remote-entrypoint.test.sh -tests/fm-cleanup-skill.test.sh -tests/fm-hermes-live-e2e.test.sh -tests/fm-grok-stop-live-e2e.test.sh -tests/fm-opencode-primary-live-e2e.test.sh -tests/fm-omp-worker-tmux-live-e2e.test.sh -tests/fm-omp-branch-live-e2e.test.sh +tests/fm-treehouse-orphan-recovery.test.sh +tests/fm-backend-cmux-smoke.test.sh +tests/fm-codex-continuity-live-e2e.test.sh +tests/fm-remote-omp-update-restart-live-e2e.test.sh ``` -#### `portable-serial-4of6` — 30 scripts +#### `portable-serial-4of7` — 25 scripts ```text tests/fm-secondmate-harness.test.sh -tests/fm-boat-lane-runner.test.sh -tests/fm-omp-secondmate.test.sh -tests/fm-watcher-lock.test.sh -tests/fm-watch-recovery-loop.test.sh -tests/fm-claude-stop-autoarm.test.sh -tests/fm-daemon.test.sh -tests/fm-vendor-auth-probe.test.sh -tests/fm-main-ci-watch.test.sh -tests/fm-spawn-relaunch-dead-endpoint.test.sh -tests/fm-wake-drain-unread-status.test.sh -tests/fm-pi-watch-extension.test.sh -tests/fm-remote-backlog-handoff.test.sh -tests/fm-pr-check-handoff.test.sh -tests/fm-fleet-snapshot-view.test.sh -tests/fm-todo-project.test.sh -tests/fm-update.test.sh -tests/fm-omp-relaunch-guard.test.sh +tests/fm-ext-waiting-pin.test.sh +tests/fm-watch-arm.test.sh +tests/fm-bootstrap.test.sh +tests/fm-session-start.test.sh +tests/fm-remote-job.test.sh +tests/fm-public-followup.test.sh +tests/fm-runpod-pod-boot.test.sh +tests/fm-omp-branch-supervision.test.sh +tests/fm-afk-inject-e2e.test.sh +tests/fm-omp-task-inbox-doorbell.test.sh +tests/fm-kimi-harness.test.sh +tests/fm-shared-captain-inheritance.test.sh +tests/fm-on.test.sh +tests/fm-herdr-session-cleanup.test.sh tests/fm-watch-checkpoint.test.sh -tests/fm-trace-context-lib.test.sh -tests/fm-session-lock-ancestry.test.sh +tests/fm-backlog-handoff.test.sh +tests/fm-afk-return.test.sh tests/fm-documentation-audiences.test.sh -tests/fm-check-unregister.test.sh -tests/fm-backend-tmux-smoke.test.sh -tests/fm-afk-pi-herdr-return-e2e.test.sh -tests/fm-send-inbox-doorbell-live-e2e.test.sh +tests/fm-classify-decision-key.test.sh +tests/fm-pending-reply-sleep-reconcile.test.sh +tests/fm-pi-compatible-family.test.sh +tests/fm-herdr-submit-confirm-live-e2e.test.sh +tests/fm-grok-stop-live-e2e.test.sh tests/fm-grok-continuity-live-e2e.test.sh -tests/fm-omp-herdr-live-e2e.test.sh -tests/fm-omp-primary-live-e2e.test.sh -tests/fm-claude-stop-autoarm-live-e2e.test.sh ``` -#### `portable-serial-5of6` — 26 scripts +#### `portable-serial-5of7` — 26 scripts ```text -tests/fm-hermes-harness.test.sh -tests/fm-ext-waiting-pin.test.sh +tests/fm-pr-check-security.test.sh +tests/fm-omp-secondmate.test.sh tests/fm-wake-queue.test.sh -tests/fm-session-start.test.sh -tests/fm-backlog-atomicity.test.sh -tests/fm-omp-primary.test.sh +tests/fm-wake-daemon-lifecycle-e2e.test.sh +tests/fm-boat-omp-auth.test.sh +tests/fm-remote-secondmate-trace-context.test.sh tests/fm-boat-lifecycle.test.sh -tests/fm-runpod-pod-boot.test.sh +tests/fm-send-inbox.test.sh +tests/fm-teardown-endpoint-safety.test.sh tests/fm-remote-secondmate-update-restart.test.sh -tests/fm-secondmate-restart.test.sh -tests/fm-afk-inject-e2e.test.sh tests/fm-prepush-guard.test.sh -tests/fm-ext-thread-context.test.sh -tests/fm-wake-drain-open-decisions.test.sh -tests/fm-shared-captain-inheritance.test.sh -tests/fm-ext-plugin-dest.test.sh -tests/fm-startup-memory-budget.test.sh -tests/fm-send-secondmate-marker.test.sh -tests/fm-treehouse-sweep.test.sh -tests/fm-merge-local.test.sh -tests/fm-afk-return.test.sh -tests/fm-classify-decision-key.test.sh -tests/fm-sessionstart-nudge.test.sh -tests/fm-operational-input.test.sh -tests/fm-treehouse-orphan-recovery.test.sh -tests/fm-omp-relaunch-tmux-live-e2e.test.sh +tests/fm-remote-secondmate-parent-binding.test.sh +tests/fm-pending-reply.test.sh +tests/fm-inbox-result.test.sh +tests/fm-backend-zellij.test.sh +tests/fm-remote-doctor.test.sh +tests/fm-backend-cmux.test.sh +tests/fm-session-lock-ancestry.test.sh +tests/fm-omp-update.test.sh +tests/fm-devin-adapter.test.sh +tests/fm-remote-entrypoint.test.sh +tests/fm-stall-recovery.test.sh +tests/fm-backend-herdr-focus-flash-e2e.test.sh +tests/fm-devin-herdr-live-e2e.test.sh +tests/fm-omp-compact-adviser-live-e2e.test.sh +tests/fm-omp-herdr-live-e2e.test.sh ``` -#### `portable-serial-6of6` — 30 scripts +#### `portable-serial-6of7` — 26 scripts ```text -tests/fm-teardown.test.sh -tests/fm-pr-check-security.test.sh -tests/fm-remote-reply.test.sh -tests/fm-bootstrap.test.sh -tests/fm-procevent.test.sh +tests/fm-ext-decision.test.sh +tests/fm-boat-lane-runner.test.sh tests/fm-ext-bridge.test.sh -tests/fm-runpod-lifecycle.test.sh -tests/fm-secondmate-safety.test.sh -tests/fm-runpod-routing.test.sh -tests/fm-task-inbox.test.sh -tests/fm-boat-routing.test.sh -tests/fm-backend.test.sh +tests/fm-procevent.test.sh +tests/fm-omp-primary.test.sh +tests/fm-daemon.test.sh +tests/fm-landing.test.sh +tests/fm-secondmate-sync.test.sh +tests/fm-spawn-relaunch-dead-endpoint.test.sh +tests/fm-send-resolve-key.test.sh +tests/fm-turnend-guard.test.sh tests/fm-busy-adapter-wiring.test.sh -tests/fm-tasks-axi.test.sh -tests/fm-remote-secondmate-parent-binding.test.sh -tests/fm-gate-refuse.test.sh -tests/fm-inbox.test.sh -tests/fm-herdr-session-cleanup.test.sh +tests/fm-spawn-worktree-settle.test.sh +tests/fm-send-turn-start.test.sh +tests/fm-dispatch-resolve.test.sh tests/fm-idle-sleep-watch.test.sh -tests/fm-backlog-handoff.test.sh -tests/fm-receipt.test.sh -tests/fm-test-fixture-cleanup.test.sh -tests/fm-test-isolation-proof.test.sh -tests/fm-ask-user-authority.test.sh -tests/fm-stall-recovery.test.sh -tests/fm-backend-cmux-smoke.test.sh -tests/fm-devin-herdr-live-e2e.test.sh -tests/fm-herdr-submit-confirm-live-e2e.test.sh -tests/fm-omp-branch-types.test.sh +tests/fm-omp-fleet-hooks.test.sh +tests/fm-trace-context-lib.test.sh +tests/fm-subagent-pretool-check.test.sh +tests/fm-reflect-skill.test.sh +tests/fm-treehouse-status-read-only.test.sh +tests/fm-send-secondmate-marker-herdr-e2e.test.sh +tests/fm-gitignore-config.test.sh tests/fm-pi-primary-live-e2e.test.sh +tests/fm-opencode-primary-live-e2e.test.sh +tests/fm-omp-relaunch-tmux-live-e2e.test.sh +``` + +#### `portable-serial-7of7` — 26 scripts + +```text +tests/fm-hermes-harness.test.sh +tests/fm-teardown.test.sh +tests/fm-remote-reply.test.sh +tests/fm-watch-recovery-loop.test.sh +tests/fm-claude-stop-autoarm.test.sh +tests/fm-ext-ask.test.sh +tests/fm-watcher-lock.test.sh +tests/fm-vendor-auth-probe.test.sh +tests/fm-spawn-pool-base-freshen.test.sh +tests/fm-wake-drain-open-decisions-cursor.test.sh +tests/fm-secondmate-liveness.test.sh +tests/fm-ext-thread-context.test.sh +tests/fm-idle-sleep.test.sh +tests/fm-update.test.sh +tests/fm-startup-memory-budget.test.sh +tests/fm-receipt-check.test.sh +tests/fm-backend-herdr-server-detach.test.sh +tests/fm-merge-local.test.sh +tests/fm-test-fixture-cleanup.test.sh +tests/fm-sessionstart-nudge.test.sh +tests/fm-backend-tmux-smoke.test.sh +tests/fm-hermes-live-e2e.test.sh +tests/fm-backend-zellij-smoke.test.sh +tests/fm-claude-stop-autoarm-live-e2e.test.sh +tests/fm-omp-worker-tmux-live-e2e.test.sh +tests/fm-quota-array-dispatch-live-e2e.test.sh ``` To refresh after a green run, replace the full hint table with measured per-script pairs and regenerate the membership snapshot above from the same hints. @@ -331,9 +354,9 @@ Copy the generated assignments into the snapshot and run the partition guard. ```sh gh run download -R dnth/firstmate --pattern 'fm-test-timing-portable-serial-*' -D /tmp/fm-serial jq -r '.scripts[] | [.path, .duration_ms] | @tsv' /tmp/fm-serial/*.json | LC_ALL=C sort -for i in 1 2 3 4 5 6; do - printf 'portable-serial-%sof6\n' "$i" - bin/fm-test-run.sh --list --lane "portable-serial-${i}of6" +for i in 1 2 3 4 5 6 7; do + printf 'portable-serial-%sof7\n' "$i" + bin/fm-test-run.sh --list --lane "portable-serial-${i}of7" done bin/fm-test-run.sh --check-coverage ``` @@ -360,7 +383,7 @@ Portable shards, each portable serial shard, and the Herdr lane upload runner-ge | Job | timeout-minutes | Rationale | |---|---:|---| | portable parallel 1/2 | 10 | The measured shard sums are about three minutes and the timeout is a hang tripwire. | -| portable serial 1-6 | 20 | Each rebalanced shard projects 14m23.259s–14m23.270s of script time, leaving 5m36.730s–5m36.741s for setup and runner-load variance while still tripping on a wedged test. | +| portable serial 1-7 | 20 | The seven-shard assignment projects 13m00.151s–13m00.166s from PR run 38774522507, leaving about 6m59.834s for setup and runner-load variance while keeping the timeout as a hang tripwire. | | Herdr | 40 | The real-Herdr lane keeps its dedicated timeout. | Timeouts are hang tripwires rather than expected healthy durations. From de1fbf146aeaed6c90b6b821037141efa84b50ab Mon Sep 17 00:00:00 2001 From: dnth Date: Fri, 16 Oct 2026 02:07:32 +0800 Subject: [PATCH 6/9] no-mistakes(document): Restore five-run timing baseline comparison --- docs/fm-test-portable-shards.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/fm-test-portable-shards.md b/docs/fm-test-portable-shards.md index 220e9ac00e6..fbeeeca97b7 100644 --- a/docs/fm-test-portable-shards.md +++ b/docs/fm-test-portable-shards.md @@ -104,8 +104,8 @@ Recent green `fm-watch-triage.test.sh` measurements provide the comparison for a | [38746979839 attempt 2](https://github.com/dnth/firstmate/actions/runs/38746979839/attempts/2) | 410,055 | | [PR 38774522507](https://github.com/dnth/firstmate/actions/runs/38774522507) | 428,164 | -The 446,890 ms result exceeds the latest passing measurement in this table, 428,164 ms, by 18,726 ms (about 4.4%), so it is not a large outlier. -PR run 38774522507 measured `fm-watch-triage.test.sh` at 428,164 ms (7m08.164s), within the observed range. +The 446,890 ms result exceeds the maximum of the five preceding green runs, 421,505 ms, by 25,385 ms (about 6%), so it is not a large outlier. +Against the newer passing measurement in the final row, the difference narrows to 18,726 ms (about 4.4%). Separately, [run 38759616524](https://github.com/dnth/firstmate/actions/runs/38759616524) failed the stale-suppressor assertion after 100,083 ms. That is a separate correctness signal, not evidence that a 447-second `fm-watch-triage` run caused the timeout; the test assertions remain unchanged. From 3ae8c75809751d9f5daccda50bed2a9cb4011d15 Mon Sep 17 00:00:00 2001 From: dnth Date: Fri, 16 Oct 2026 02:41:31 +0800 Subject: [PATCH 7/9] revert(test): restore sequence-based reply selection Keep this PR limited to the portable serial shard rebalance. The correlation-ID behavior change is a separate fix and can be reviewed on its own. --- tests/fm-remote-secondmate-lifecycle-e2e.test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fm-remote-secondmate-lifecycle-e2e.test.sh b/tests/fm-remote-secondmate-lifecycle-e2e.test.sh index 68d8a2e8f46..dd0062d7b78 100755 --- a/tests/fm-remote-secondmate-lifecycle-e2e.test.sh +++ b/tests/fm-remote-secondmate-lifecycle-e2e.test.sh @@ -970,7 +970,7 @@ resolve_ios_pending() { remote_env "$ROOT/bin/fm-procevent.sh" start "$SID" >/dev/null 2>&1 || true pending_result="" n=200 while [ -z "$pending_result" ] && [ "$n" -gt 0 ]; do - pending_result=$(grep -lF "done [corr=$pending_corr]:" "$PARENT/state/procevent-inbox/$SID".*.result 2>/dev/null | head -1 || true) + pending_result=$(find "$PARENT/state/procevent-inbox" -name "$SID.*.result" ! -name "$SID.1.result" ! -name "$SID.2.result" ! -name "$SID.3.result" -print | sort | tail -1) [ -n "$pending_result" ] || { n=$((n - 1)); sleep 0.1; } done [ -n "$pending_result" ] \ From 65068dbddb8b255f7aa0a80db7dfbe708e917b56 Mon Sep 17 00:00:00 2001 From: dnth Date: Fri, 16 Oct 2026 02:50:10 +0800 Subject: [PATCH 8/9] Revert "revert(test): restore sequence-based reply selection" This reverts commit 3ae8c75809751d9f5daccda50bed2a9cb4011d15. --- tests/fm-remote-secondmate-lifecycle-e2e.test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fm-remote-secondmate-lifecycle-e2e.test.sh b/tests/fm-remote-secondmate-lifecycle-e2e.test.sh index dd0062d7b78..68d8a2e8f46 100755 --- a/tests/fm-remote-secondmate-lifecycle-e2e.test.sh +++ b/tests/fm-remote-secondmate-lifecycle-e2e.test.sh @@ -970,7 +970,7 @@ resolve_ios_pending() { remote_env "$ROOT/bin/fm-procevent.sh" start "$SID" >/dev/null 2>&1 || true pending_result="" n=200 while [ -z "$pending_result" ] && [ "$n" -gt 0 ]; do - pending_result=$(find "$PARENT/state/procevent-inbox" -name "$SID.*.result" ! -name "$SID.1.result" ! -name "$SID.2.result" ! -name "$SID.3.result" -print | sort | tail -1) + pending_result=$(grep -lF "done [corr=$pending_corr]:" "$PARENT/state/procevent-inbox/$SID".*.result 2>/dev/null | head -1 || true) [ -n "$pending_result" ] || { n=$((n - 1)); sleep 0.1; } done [ -n "$pending_result" ] \ From e3c3551d39ed0101f3d3d22f11213455824150af Mon Sep 17 00:00:00 2001 From: dnth Date: Fri, 16 Oct 2026 02:59:01 +0800 Subject: [PATCH 9/9] no-mistakes(document): Document correlation-ID test fix and CI acceptance requirements --- docs/fm-test-portable-shards.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/fm-test-portable-shards.md b/docs/fm-test-portable-shards.md index fbeeeca97b7..4b43435cc46 100644 --- a/docs/fm-test-portable-shards.md +++ b/docs/fm-test-portable-shards.md @@ -109,6 +109,14 @@ Against the newer passing measurement in the final row, the difference narrows t Separately, [run 38759616524](https://github.com/dnth/firstmate/actions/runs/38759616524) failed the stale-suppressor assertion after 100,083 ms. That is a separate correctness signal, not evidence that a 447-second `fm-watch-triage` run caused the timeout; the test assertions remain unchanged. +### Required test fix + +`resolve_ios_pending` in [fm-remote-secondmate-lifecycle-e2e.test.sh](../tests/fm-remote-secondmate-lifecycle-e2e.test.sh) now selects the captured acknowledgement by the pending request's correlation ID. +The previous selection excluded the first three result files and picked the last sorted result filename, which did not reliably identify the reply for that request. +Handling an unrelated result could leave an inheritance acknowledgement unresolved before the merged-PR reply assertion. +[Run 38770719267](https://github.com/dnth/firstmate/actions/runs/38770719267) failed that assertion without the correlation-ID selection; [run 38785253557](https://github.com/dnth/firstmate/actions/runs/38785253557) passed with it. +The fix preserves the assertions and changes only which captured reply is handled. + ### Rebalanced duration profile The six-shard PR run measured 5,461,112 ms (91m01.112s) across 166 serial scripts, averaging 15m10.185s per shard. @@ -117,6 +125,7 @@ Seven shards are needed because six shards cannot fit that measured total within PR run 38774522507 measured `fm-watch-triage.test.sh` at 428,164 ms (7m08.164s); the earlier PR attempt 3 measured 446,890 ms (7m26.890s). If `fm-watch-triage.test.sh` reaches 446,890 ms again, shard 2 rises by only 18,726 ms to 798,888 ms (13m18.888s), still within the ~14-minute budget. The seven-shard assignment covers the observed variation without raising the 20-minute hang tripwire. +CI acceptance for this change requires all seven actual serial jobs to complete without cancellation and with each job's wall time at most 15 minutes; projected script sums alone do not satisfy that requirement. | Shard | Scripts | Measured six-shard script sum | Six-shard CI job wall | |---|---:|---:|---:|