diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f9465522f50..c3e072160ed 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -192,16 +192,14 @@ jobs: tests-portable-serial: name: Behavior portable serial ${{ matrix.shard }} runs-on: ubuntu-latest - # Refreshed timings plus conservative hints for grown tests estimate ~84 min - # of serial work, balanced over six runners at ~14 min each. Keep the cap - # as a hang tripwire with runner-load margin; refresh hints per - # docs/fm-test-portable-shards.md when a shard approaches it. + # Keep this cap as a hang tripwire; when a shard approaches it, follow the + # balance evidence and refresh procedure in docs/fm-test-portable-shards.md. timeout-minutes: 20 strategy: # Every shard reports so one failure never hides another shard's result. fail-fast: false matrix: - shard: [1, 2, 3, 4, 5, 6] + shard: [1, 2, 3, 4, 5, 6, 7] steps: - uses: actions/checkout@v6 with: diff --git a/bin/fm-test-run.sh b/bin/fm-test-run.sh index 7de253f58e6..afc6dd3034a 100755 --- a/bin/fm-test-run.sh +++ b/bin/fm-test-run.sh @@ -92,11 +92,9 @@ JOBS_MAX=8 # How many separate-runner shards the portable serial remainder splits into. # One owner: CI lane names carry this count and are refused when they disagree. -PORTABLE_SERIAL_SHARDS=6 +PORTABLE_SERIAL_SHARDS=7 -# Balance hint for a portable-serial script with no measured duration, close to -# the measured per-script mean so a newly added test neither starves nor -# overloads the shard it lands in. +# Fallback balance hint for a portable-serial script with no measured duration. PORTABLE_SERIAL_DEFAULT_WEIGHT_MS=20000 usage() { @@ -375,165 +373,179 @@ list_portable_serial() { done < <(all_repo_tests) } -# Portable-serial duration hints in milliseconds, from CI artifacts, completed -# log records, and conservative estimates documented in docs/fm-test-portable-shards.md. +# Portable-serial measured duration hints in milliseconds; source evidence is +# documented in docs/fm-test-portable-shards.md. # These affect balance only: the shard partition stays complete and disjoint, so a # stale hint costs balance rather than coverage. That doc owns the refresh # procedure. portable_serial_weight_hints() { cat <<'EOF' -tests/fm-afk-inject-e2e.test.sh 26008 -tests/fm-afk-pi-herdr-return-e2e.test.sh 47 -tests/fm-afk-return.test.sh 1517 -tests/fm-ask-user-authority.test.sh 138 -tests/fm-backend-cmux-smoke.test.sh 30 -tests/fm-backend-cmux.test.sh 2950 -tests/fm-backend-herdr-focus-flash-e2e.test.sh 21 -tests/fm-backend-herdr-server-detach.test.sh 90000 -tests/fm-backend-orca.test.sh 19416 -tests/fm-backend-tmux-smoke.test.sh 315 -tests/fm-backend-zellij-smoke.test.sh 14 -tests/fm-backend-zellij.test.sh 5188 -tests/fm-backend.test.sh 19862 -tests/fm-backlog-atomicity.test.sh 78496 -tests/fm-backlog-handoff.test.sh 3343 -tests/fm-bearings-snapshot.test.sh 66118 -tests/fm-boat-lifecycle.test.sh 52034 -tests/fm-boat-omp-auth.test.sh 58941 -tests/fm-boat-routing.test.sh 200000 -tests/fm-bootstrap.test.sh 62184 -tests/fm-busy-adapter-wiring.test.sh 17306 -tests/fm-busy-state.test.sh 2821 -tests/fm-calm-pi-extension.test.sh 260 -tests/fm-check-unregister.test.sh 356 -tests/fm-classify-decision-key.test.sh 703 -tests/fm-claude-stop-autoarm-live-e2e.test.sh 17 -tests/fm-claude-stop-autoarm.test.sh 60519 -tests/fm-cleanup-skill.test.sh 72 -tests/fm-codex-continuity-live-e2e.test.sh 19 -tests/fm-daemon.test.sh 61329 -tests/fm-devin-adapter.test.sh 280 -tests/fm-devin-herdr-live-e2e.test.sh 13 -tests/fm-dispatch-resolve.test.sh 3995 -tests/fm-documentation-audiences.test.sh 708 -tests/fm-ext-bridge.test.sh 35693 -tests/fm-fleet-snapshot-view.test.sh 13508 -tests/fm-fleet-sync.test.sh 32815 -tests/fm-gate-refuse.test.sh 9270 -tests/fm-gitignore-config.test.sh 25 -tests/fm-gotmp.test.sh 713 -tests/fm-grok-continuity-live-e2e.test.sh 12 -tests/fm-grok-stop-live-e2e.test.sh 21 -tests/fm-guard-stale-banner.test.sh 4698 -tests/fm-herdr-session-cleanup.test.sh 4832 -tests/fm-herdr-submit-confirm-live-e2e.test.sh 21 -tests/fm-hermes-harness.test.sh 146669 -tests/fm-hermes-live-e2e.test.sh 28 -tests/fm-idle-sleep-watch.test.sh 8000 -tests/fm-idle-sleep.test.sh 15000 -tests/fm-inbox-result.test.sh 7377 -tests/fm-inbox.test.sh 7022 -tests/fm-kimi-harness.test.sh 16139 -tests/fm-omp-branch-bash.test.sh 44 -tests/fm-omp-branch-live-e2e.test.sh 12 -tests/fm-omp-branch-supervision.test.sh 4373 -tests/fm-omp-branch-types.test.sh 24 +tests/fm-afk-inject-e2e.test.sh 26538 +tests/fm-afk-pi-herdr-return-e2e.test.sh 76 +tests/fm-afk-return.test.sh 1524 +tests/fm-ask-user-authority.test.sh 154 +tests/fm-backend-cmux-smoke.test.sh 34 +tests/fm-backend-cmux.test.sh 3751 +tests/fm-backend-herdr-focus-flash-e2e.test.sh 22 +tests/fm-backend-herdr-server-detach.test.sh 2649 +tests/fm-backend-orca.test.sh 19716 +tests/fm-backend-tmux-smoke.test.sh 250 +tests/fm-backend-zellij-smoke.test.sh 22 +tests/fm-backend-zellij.test.sh 7129 +tests/fm-backend.test.sh 23256 +tests/fm-backlog-atomicity.test.sh 63215 +tests/fm-backlog-handoff.test.sh 3410 +tests/fm-bearings-snapshot.test.sh 81131 +tests/fm-boat-lane-runner.test.sh 119102 +tests/fm-boat-lifecycle.test.sh 44937 +tests/fm-boat-omp-auth.test.sh 62445 +tests/fm-boat-routing.test.sh 26287 +tests/fm-bootstrap.test.sh 71450 +tests/fm-busy-adapter-wiring.test.sh 19619 +tests/fm-busy-state.test.sh 2879 +tests/fm-calm-pi-extension.test.sh 265 +tests/fm-check-unregister.test.sh 436 +tests/fm-classify-decision-key.test.sh 501 +tests/fm-claude-stop-autoarm-live-e2e.test.sh 20 +tests/fm-claude-stop-autoarm.test.sh 60780 +tests/fm-cleanup-skill.test.sh 74 +tests/fm-codex-continuity-live-e2e.test.sh 20 +tests/fm-daemon.test.sh 60677 +tests/fm-devin-adapter.test.sh 349 +tests/fm-devin-herdr-live-e2e.test.sh 21 +tests/fm-dispatch-resolve.test.sh 8747 +tests/fm-documentation-audiences.test.sh 878 +tests/fm-ext-ask.test.sh 48724 +tests/fm-ext-bridge.test.sh 82962 +tests/fm-ext-decision.test.sh 161682 +tests/fm-ext-plugin-dest.test.sh 7190 +tests/fm-ext-thread-context.test.sh 18046 +tests/fm-ext-topic-threads.test.sh 100660 +tests/fm-ext-waiting-pin.test.sh 103172 +tests/fm-ext-watch.test.sh 7522 +tests/fm-fleet-snapshot-view.test.sh 12998 +tests/fm-fleet-sync.test.sh 36580 +tests/fm-gate-refuse.test.sh 11343 +tests/fm-gitignore-config.test.sh 29 +tests/fm-gotmp.test.sh 1168 +tests/fm-grok-continuity-live-e2e.test.sh 19 +tests/fm-grok-stop-live-e2e.test.sh 20 +tests/fm-guard-stale-banner.test.sh 5898 +tests/fm-herdr-session-cleanup.test.sh 6849 +tests/fm-herdr-submit-confirm-live-e2e.test.sh 22 +tests/fm-hermes-harness.test.sh 151616 +tests/fm-hermes-live-e2e.test.sh 44 +tests/fm-idle-sleep-watch.test.sh 4560 +tests/fm-idle-sleep.test.sh 13959 +tests/fm-inbox-result.test.sh 9828 +tests/fm-inbox.test.sh 9840 +tests/fm-kimi-harness.test.sh 18045 +tests/fm-landing.test.sh 43476 +tests/fm-main-ci-watch.test.sh 46302 +tests/fm-merge-local.test.sh 2502 +tests/fm-omp-branch-bash.test.sh 41 +tests/fm-omp-branch-live-e2e.test.sh 20 +tests/fm-omp-branch-supervision.test.sh 37015 +tests/fm-omp-branch-types.test.sh 21 tests/fm-omp-compact-adviser-live-e2e.test.sh 20 -tests/fm-omp-compact-adviser.test.sh 298 -tests/fm-omp-fleet-hooks.test.sh 2407 -tests/fm-omp-harness.test.sh 1764 -tests/fm-omp-herdr-exit-live-e2e.test.sh 16 +tests/fm-omp-compact-adviser.test.sh 1697 +tests/fm-omp-fleet-hooks.test.sh 2743 +tests/fm-omp-harness.test.sh 66500 +tests/fm-omp-herdr-exit-live-e2e.test.sh 19 tests/fm-omp-herdr-live-e2e.test.sh 19 -tests/fm-omp-primary-live-e2e.test.sh 18 -tests/fm-omp-primary.test.sh 45746 -tests/fm-omp-relaunch-guard.test.sh 4276 -tests/fm-omp-relaunch-tmux-live-e2e.test.sh 21 -tests/fm-omp-secondmate-live-e2e.test.sh 21 -tests/fm-omp-secondmate.test.sh 600000 -tests/fm-omp-task-inbox-doorbell.test.sh 20577 -tests/fm-omp-update.test.sh 448 -tests/fm-omp-worker-tmux-live-e2e.test.sh 17 -tests/fm-on.test.sh 9627 -tests/fm-opencode-primary-live-e2e.test.sh 16 -tests/fm-operational-input.test.sh 176 -tests/fm-pending-reply-sleep-reconcile.test.sh 243 -tests/fm-pending-reply.test.sh 19949 -tests/fm-pi-compatible-family.test.sh 56 +tests/fm-omp-primary-live-e2e.test.sh 19 +tests/fm-omp-primary.test.sh 61283 +tests/fm-omp-relaunch-guard.test.sh 5534 +tests/fm-omp-relaunch-tmux-live-e2e.test.sh 14 +tests/fm-omp-secondmate-live-e2e.test.sh 20 +tests/fm-omp-secondmate.test.sh 113332 +tests/fm-omp-task-inbox-doorbell.test.sh 20456 +tests/fm-omp-update.test.sh 689 +tests/fm-omp-worker-tmux-live-e2e.test.sh 20 +tests/fm-on.test.sh 9816 +tests/fm-opencode-primary-live-e2e.test.sh 20 +tests/fm-operational-input.test.sh 154 +tests/fm-pending-reply-sleep-reconcile.test.sh 229 +tests/fm-pending-reply.test.sh 15024 +tests/fm-pi-compatible-family.test.sh 70 tests/fm-pi-primary-live-e2e.test.sh 21 -tests/fm-pi-watch-extension.test.sh 22031 -tests/fm-pr-check-handoff.test.sh 42228 -tests/fm-pr-check-security.test.sh 158907 -tests/fm-prepush-guard.test.sh 19208 -tests/fm-procevent-when.test.sh 24358 -tests/fm-procevent.test.sh 61916 -tests/fm-public-followup.test.sh 35559 -tests/fm-quota-array-dispatch-live-e2e.test.sh 12 -tests/fm-receipt-check.test.sh 11636 -tests/fm-receipt.test.sh 1331 -tests/fm-reflect-skill.test.sh 372 -tests/fm-remote-backlog-handoff.test.sh 15173 -tests/fm-remote-doctor.test.sh 6203 -tests/fm-remote-entrypoint.test.sh 208 -tests/fm-remote-job.test.sh 48694 -tests/fm-remote-reply.test.sh 99366 -tests/fm-remote-secondmate-lifecycle-e2e.test.sh 700000 -tests/fm-remote-secondmate-parent-binding.test.sh 15892 -tests/fm-remote-secondmate-sleep-reconcile.test.sh 3000 -tests/fm-remote-secondmate-trace-context.test.sh 49391 -tests/fm-runpod-lifecycle.test.sh 56695 -tests/fm-runpod-omp-auth.test.sh 1012 -tests/fm-runpod-pod-boot.test.sh 45589 -tests/fm-runpod-routing.test.sh 28030 -tests/fm-secondmate-harness.test.sh 151987 -tests/fm-secondmate-lifecycle-e2e.test.sh 5372 -tests/fm-secondmate-liveness.test.sh 20126 -tests/fm-secondmate-restart.test.sh 31003 -tests/fm-secondmate-safety.test.sh 55903 -tests/fm-secondmate-sync.test.sh 17159 -tests/fm-send-inbox-doorbell-live-e2e.test.sh 18 -tests/fm-send-inbox.test.sh 37394 -tests/fm-send-resolve-key.test.sh 12632 -tests/fm-send-secondmate-marker-herdr-e2e.test.sh 48 -tests/fm-send-secondmate-marker.test.sh 4320 -tests/fm-send-turn-start.test.sh 11143 -tests/fm-session-lock-ancestry.test.sh 1284 -tests/fm-session-start.test.sh 77942 -tests/fm-sessionstart-nudge.test.sh 284 -tests/fm-shared-captain-inheritance.test.sh 4259 -tests/fm-spawn-dispatch-profile.test.sh 193113 -tests/fm-spawn-pool-base-freshen.test.sh 35139 -tests/fm-spawn-relaunch-dead-endpoint.test.sh 31467 -tests/fm-spawn-worktree-settle.test.sh 5788 +tests/fm-pi-watch-extension.test.sh 22125 +tests/fm-pr-check-handoff.test.sh 15330 +tests/fm-pr-check-security.test.sh 163835 +tests/fm-prepush-guard.test.sh 19861 +tests/fm-procevent-when.test.sh 21536 +tests/fm-procevent.test.sh 69535 +tests/fm-public-followup.test.sh 47380 +tests/fm-quota-array-dispatch-live-e2e.test.sh 19 +tests/fm-receipt-check.test.sh 6254 +tests/fm-receipt.test.sh 1854 +tests/fm-reflect-skill.test.sh 473 +tests/fm-remote-backlog-handoff.test.sh 20170 +tests/fm-remote-doctor.test.sh 5787 +tests/fm-remote-entrypoint.test.sh 263 +tests/fm-remote-job.test.sh 48956 +tests/fm-remote-omp-update-restart-live-e2e.test.sh 20 +tests/fm-remote-reply.test.sh 82089 +tests/fm-remote-secondmate-lifecycle-e2e.test.sh 657637 +tests/fm-remote-secondmate-parent-binding.test.sh 15757 +tests/fm-remote-secondmate-sleep-reconcile.test.sh 1287 +tests/fm-remote-secondmate-trace-context.test.sh 59252 +tests/fm-remote-secondmate-update-restart.test.sh 30187 +tests/fm-runpod-lifecycle.test.sh 58778 +tests/fm-runpod-omp-auth.test.sh 1192 +tests/fm-runpod-pod-boot.test.sh 43135 +tests/fm-runpod-routing.test.sh 37449 +tests/fm-secondmate-harness.test.sh 170105 +tests/fm-secondmate-lifecycle-e2e.test.sh 6278 +tests/fm-secondmate-liveness.test.sh 20034 +tests/fm-secondmate-restart.test.sh 36334 +tests/fm-secondmate-safety.test.sh 58742 +tests/fm-secondmate-sync.test.sh 40216 +tests/fm-send-inbox-doorbell-live-e2e.test.sh 22 +tests/fm-send-inbox.test.sh 40199 +tests/fm-send-resolve-key.test.sh 22866 +tests/fm-send-secondmate-marker-herdr-e2e.test.sh 49 +tests/fm-send-secondmate-marker.test.sh 4337 +tests/fm-send-turn-start.test.sh 11083 +tests/fm-session-lock-ancestry.test.sh 1513 +tests/fm-session-start.test.sh 65773 +tests/fm-sessionstart-nudge.test.sh 303 +tests/fm-shared-captain-inheritance.test.sh 13503 +tests/fm-spawn-dispatch-profile.test.sh 249954 +tests/fm-spawn-pool-base-freshen.test.sh 35286 +tests/fm-spawn-relaunch-dead-endpoint.test.sh 35039 +tests/fm-spawn-worktree-settle.test.sh 11488 tests/fm-stall-recovery.test.sh 48 -tests/fm-startup-memory-budget.test.sh 7593 -tests/fm-subagent-pretool-check.test.sh 709 -tests/fm-supervision-events.test.sh 462 -tests/fm-tangle-guard.test.sh 8797 -tests/fm-task-delivery.test.sh 9829 -tests/fm-task-inbox.test.sh 37450 -tests/fm-teardown-endpoint-safety.test.sh 1797 -tests/fm-teardown.test.sh 107078 -tests/fm-test-fixture-cleanup.test.sh 542 -tests/fm-test-isolation-proof.test.sh 426 -tests/fm-todo-project.test.sh 6065 -tests/fm-trace-context-lib.test.sh 1424 -tests/fm-trace-context-spawn.test.sh 46250 -tests/fm-treehouse-orphan-recovery.test.sh 125 -tests/fm-treehouse-status-read-only.test.sh 225 -tests/fm-treehouse-sweep.test.sh 4886 -tests/fm-turnend-guard.test.sh 20258 -tests/fm-update.test.sh 10591 -tests/fm-vendor-auth-probe.test.sh 42632 -tests/fm-wake-daemon-lifecycle-e2e.test.sh 19948 -tests/fm-wake-drain-open-decisions-cursor.test.sh 22217 -tests/fm-wake-drain-open-decisions.test.sh 3443 -tests/fm-wake-drain-unread-status.test.sh 14508 -tests/fm-wake-queue.test.sh 72347 -tests/fm-watch-arm.test.sh 66052 -tests/fm-watch-checkpoint.test.sh 4768 -tests/fm-watch-recovery-loop.test.sh 67796 -tests/fm-watch-triage.test.sh 407955 -tests/fm-watcher-lock.test.sh 38168 +tests/fm-startup-memory-budget.test.sh 6514 +tests/fm-subagent-pretool-check.test.sh 968 +tests/fm-supervision-events.test.sh 628 +tests/fm-tangle-guard.test.sh 10406 +tests/fm-task-delivery.test.sh 19399 +tests/fm-task-inbox.test.sh 37373 +tests/fm-tasks-axi.test.sh 18227 +tests/fm-teardown-endpoint-safety.test.sh 30883 +tests/fm-teardown.test.sh 137584 +tests/fm-test-fixture-cleanup.test.sh 681 +tests/fm-test-isolation-proof.test.sh 588 +tests/fm-todo-project.test.sh 9125 +tests/fm-trace-context-lib.test.sh 1997 +tests/fm-trace-context-spawn.test.sh 45140 +tests/fm-treehouse-orphan-recovery.test.sh 111 +tests/fm-treehouse-status-read-only.test.sh 223 +tests/fm-treehouse-sweep.test.sh 4347 +tests/fm-turnend-guard.test.sh 21290 +tests/fm-update.test.sh 9518 +tests/fm-vendor-auth-probe.test.sh 42927 +tests/fm-wake-daemon-lifecycle-e2e.test.sh 69696 +tests/fm-wake-drain-open-decisions-cursor.test.sh 26536 +tests/fm-wake-drain-open-decisions.test.sh 12358 +tests/fm-wake-drain-unread-status.test.sh 30489 +tests/fm-wake-queue.test.sh 85314 +tests/fm-watch-arm.test.sh 85430 +tests/fm-watch-checkpoint.test.sh 5858 +tests/fm-watch-recovery-loop.test.sh 66814 +tests/fm-watch-triage.test.sh 428164 +tests/fm-watcher-lock.test.sh 46975 EOF } diff --git a/docs/fm-test-portable-shards.md b/docs/fm-test-portable-shards.md index 3d45a084456..4b43435cc46 100644 --- a/docs/fm-test-portable-shards.md +++ b/docs/fm-test-portable-shards.md @@ -51,37 +51,322 @@ The two parallel lanes use longest-processing-time assignment from those measure `portable-serial` includes every `tests/*.test.sh` that is neither proven-isolated nor `real-herdr-gated`. It keeps watcher, lock, AFK, real tmux, daemon, secondmate lifecycle, bootstrap, live-harness opt-in, GUI-backend, and other unproven work serial. -Membership is derived rather than enumerated, so a newly added test lands here by default. +The portable-serial test set is derived rather than enumerated, so a newly added test lands here by default. +The per-shard membership snapshot below records the generated assignment for the current timing hints. ## Portable serial CI shards -On green CI run [30725985757](https://github.com/kunchenguid/firstmate/actions/runs/30725985757), that remainder accumulated 19m04s of script time against a 20-minute job timeout. -On [PR 1495](https://github.com/kunchenguid/firstmate/pull/1495), its main step ran about 19m51s before the job was cancelled at that boundary. -`portable-serial-of` splits it across `n` separate CI runners. +The refreshed hints use all 166 completed serial-script measurements from PR run [38774522507](https://github.com/dnth/firstmate/actions/runs/38774522507), totaling 5,461,112 ms (91m01.112s). +The measured durations replace the older estimates and default weights in `portable_serial_weight_hints`. +`portable-serial-of` splits that remainder across `n` separate CI runners. Each shard is still strictly serial in itself, and separate runners mean no two of these stateful scripts ever share a machine, so the split needs no concurrency isolation proof. `bin/fm-test-run.sh` owns `n` and refuses any lane whose `of` disagrees with it. `.github/workflows/ci.yml` derives the same `n` from `strategy.job-total` rather than a literal, so changing the shard count in either file without the other fails the lane loudly instead of leaving part of the required suite unrun. Assignment is longest-processing-time bin packing over per-script duration hints embedded in `bin/fm-test-run.sh`. -The hints were refreshed on 2026-10-04 from 116 completed, passing script timings in attempt 3 of [run 37159651011](https://github.com/dnth/firstmate/actions/runs/37159651011/attempts/3). -Shards 3 and 4 produced timing artifacts; shards 1 and 2 hit the 20-minute job cap, so their completed `FM_TEST_END` log records supply the measurements. -Existing hints for uncompleted scripts remain from [run 33904204200](https://github.com/dnth/firstmate/actions/runs/33904204200). -The four grown tests use conservative operator-supplied estimates: remote lifecycle 700000 ms, OMP secondmate 600000 ms, Boat routing 200000 ms, and Herdr server detach 90000 ms. -These estimates deliberately exceed this attempt's observed times for those scripts to budget for the reported slower runs. -At that refresh, the remainder estimated about 83.9 minutes; even a balanced four-shard partition would exceed the cap, so six runners now share the work. -A script with no hint gets the conservative `PORTABLE_SERIAL_DEFAULT_WEIGHT_MS` default. Hints only affect balance: the coverage guard keeps the partition complete and disjoint whatever they say, so a stale hint costs a slower shard rather than lost coverage. +A script with no hint gets the conservative `PORTABLE_SERIAL_DEFAULT_WEIGHT_MS` default. + +### Timeout diagnosis + +The recent 20-minute cancellations came from aggregate serial work, not a hanging `fm-watch-checkpoint.test.sh`. +The cancelled PR run is [38753521432](https://github.com/dnth/firstmate/actions/runs/38753521432). +Its first attempt ended during `fm-on.test.sh` after the job reached its timeout, before the checkpoint test began. +Its second attempt reached the checkpoint test about five seconds before cancellation, after the preceding serial scripts consumed the available job time. +Its third attempt ended during `fm-wake-drain-unread-status.test.sh` after its wake acknowledgement, about ten seconds after that test began. +On main run [38746979839 attempt 1](https://github.com/dnth/firstmate/actions/runs/38746979839/attempts/1), all 28 scripts emitted a 19m55.325s summary before the job was cancelled near its 20-minute limit. +The passing main attempt completed the same shard's 28 scripts in 17m20.456s. + +| Run / attempt | Measured shard-3 script spans (UTC) | Result | +|---|---|---| +| [Main 38746979839 attempt 1](https://github.com/dnth/firstmate/actions/runs/38746979839/attempts/1), cancelled | `fm-watch-triage`: 13:56:29–14:03:33 (423,952 ms); `fm-ext-decision`: 14:10:55–14:13:28 (153,704 ms); `fm-watch-checkpoint`: 14:16:03–14:16:10 (5,993 ms). | All scripts completed; the 19m55.325s suite summary preceded cancellation. | +| [PR 38753521432 attempt 1](https://github.com/dnth/firstmate/actions/runs/38753521432/attempts/1), cancelled | `fm-watch-triage`: 14:40:54–14:48:14 (440,356 ms); `fm-ext-decision`: 14:55:56–14:58:39 (163,088 ms); `fm-on` began 15:00:52 and the job cancelled at 15:01:00. | Cancelled in the final `fm-on` test; checkpoint did not start. | +| [PR 38753521432 attempt 2](https://github.com/dnth/firstmate/actions/runs/38753521432/attempts/2), cancelled | `fm-watch-triage`: 15:05:34–15:12:49 (435,200 ms); `fm-ext-decision`: 15:20:20–15:22:55 (155,491 ms); `fm-watch-checkpoint` began at 15:25:32 and the job cancelled at 15:25:37. | The checkpoint log reached its wake acknowledgement before cancellation; no test-end marker was emitted. | +| [PR 38753521432 attempt 3](https://github.com/dnth/firstmate/actions/runs/38753521432/attempts/3), cancelled | `fm-watch-triage`: 15:30:20–15:37:47 (446,890 ms); `fm-ext-decision`: 15:45:36–15:48:25 (169,627 ms); `fm-wake-drain-unread-status` began at 15:50:11 and the job cancelled at 15:50:21. | Cancelled after the wake acknowledgement in `fm-wake-drain-unread-status`. | +| [Main 38746979839 attempt 2](https://github.com/dnth/firstmate/actions/runs/38746979839/attempts/2), passed | `fm-watch-triage`: 14:22:32–14:29:22 (410,055 ms); `fm-ext-decision`: 14:35:28–14:37:25 (116,759 ms); `fm-watch-checkpoint`: 14:39:36–14:39:41 (5,271 ms). | All 28 scripts completed in 17m20.456s. | +| [PR 38774522507](https://github.com/dnth/firstmate/actions/runs/38774522507), passing six-shard run | `fm-spawn-dispatch-profile`: 17:06:27–17:10:37 (249,954 ms); `fm-ext-decision`: 17:10:37–17:13:18 (161,682 ms); `fm-ext-topic-threads`: 17:13:18–17:14:59 (100,660 ms); `fm-watch-arm`: 17:14:59–17:16:25 (85,430 ms). | The 30 scripts summed to 1,022,126 ms (17m02.126s); job wall time was 17m17s. | + +The checkpoint completed in 5.271s and 5.993s on the passing and cancelled main attempts, respectively. +All three cancelled PR attempts reached the 20-minute limit in different tests: `fm-on`, `fm-watch-checkpoint`, and `fm-wake-drain-unread-status`, so no single test hung. +The six-shard PR run 38774522507 completed every script, but its 91m01.112s total exceeds six times the ~14-minute budget (84 minutes); shard 3 summed to 17m02.126s and took 17m17s wall time. +The slowest script in shard 3 was `fm-spawn-dispatch-profile.test.sh` at 249,954 ms (4m09.954s), so the shard overrun is aggregate growth, not a single hung test. +Seven shards are required to keep the refreshed per-script assignment near the documented budget; test assertions and the 20-minute cap remain unchanged. + +Recent green `fm-watch-triage.test.sh` measurements provide the comparison for attempt 3's 446,890 ms: + +| Passing run | Duration (ms) | +|---|---:| +| [38540024417](https://github.com/dnth/firstmate/actions/runs/38540024417) | 406,842 | +| [38546582306](https://github.com/dnth/firstmate/actions/runs/38546582306) | 414,725 | +| [38551009549](https://github.com/dnth/firstmate/actions/runs/38551009549) | 419,614 | +| [38535146206](https://github.com/dnth/firstmate/actions/runs/38535146206) | 421,505 | +| [38746979839 attempt 2](https://github.com/dnth/firstmate/actions/runs/38746979839/attempts/2) | 410,055 | +| [PR 38774522507](https://github.com/dnth/firstmate/actions/runs/38774522507) | 428,164 | + +The 446,890 ms result exceeds the maximum of the five preceding green runs, 421,505 ms, by 25,385 ms (about 6%), so it is not a large outlier. +Against the newer passing measurement in the final row, the difference narrows to 18,726 ms (about 4.4%). +Separately, [run 38759616524](https://github.com/dnth/firstmate/actions/runs/38759616524) failed the stale-suppressor assertion after 100,083 ms. +That is a separate correctness signal, not evidence that a 447-second `fm-watch-triage` run caused the timeout; the test assertions remain unchanged. + +### Required test fix + +`resolve_ios_pending` in [fm-remote-secondmate-lifecycle-e2e.test.sh](../tests/fm-remote-secondmate-lifecycle-e2e.test.sh) now selects the captured acknowledgement by the pending request's correlation ID. +The previous selection excluded the first three result files and picked the last sorted result filename, which did not reliably identify the reply for that request. +Handling an unrelated result could leave an inheritance acknowledgement unresolved before the merged-PR reply assertion. +[Run 38770719267](https://github.com/dnth/firstmate/actions/runs/38770719267) failed that assertion without the correlation-ID selection; [run 38785253557](https://github.com/dnth/firstmate/actions/runs/38785253557) passed with it. +The fix preserves the assertions and changes only which captured reply is handled. + +### Rebalanced duration profile + +The six-shard PR run measured 5,461,112 ms (91m01.112s) across 166 serial scripts, averaging 15m10.185s per shard. +Seven shards are needed because six shards cannot fit that measured total within the ~14-minute budget. + +PR run 38774522507 measured `fm-watch-triage.test.sh` at 428,164 ms (7m08.164s); the earlier PR attempt 3 measured 446,890 ms (7m26.890s). +If `fm-watch-triage.test.sh` reaches 446,890 ms again, shard 2 rises by only 18,726 ms to 798,888 ms (13m18.888s), still within the ~14-minute budget. +The seven-shard assignment covers the observed variation without raising the 20-minute hang tripwire. +CI acceptance for this change requires all seven actual serial jobs to complete without cancellation and with each job's wall time at most 15 minutes; projected script sums alone do not satisfy that requirement. + +| Shard | Scripts | Measured six-shard script sum | Six-shard CI job wall | +|---|---:|---:|---:| +| 1 of 6 | 23 | 14:28.264 | 14:46 | +| 2 of 6 | 27 | 15:27.115 | 15:44 | +| 3 of 6 | 30 | 17:02.126 | 17:17 | +| 4 of 6 | 30 | 14:58.449 | 15:13 | +| 5 of 6 | 26 | 13:22.769 | 13:36 | +| 6 of 6 | 30 | 15:42.389 | 15:59 | +| Total | 166 | 91:01.112 | — | + +The seven-shard LPT assignment uses those same per-script measurements and balances within 15 ms. + +| Shard | Scripts | Projected script time | +|---|---:|---:| +| 1 of 7 | 17 | 13:00.156 | +| 2 of 7 | 23 | 13:00.162 | +| 3 of 7 | 23 | 13:00.151 | +| 4 of 7 | 25 | 13:00.154 | +| 5 of 7 | 26 | 13:00.161 | +| 6 of 7 | 26 | 13:00.162 | +| 7 of 7 | 26 | 13:00.166 | +| Total | 166 | 91:01.112 | + +The longest measured script is `tests/fm-remote-secondmate-lifecycle-e2e.test.sh` at 657,637 ms (10m57.637s), which is the single-script lower bound for any shard count. + +The current shard memberships below are generated from the same hints. -Current serial shard membership is derived by `bin/fm-test-run.sh`; its header owns the inspection commands, and its weight hints own the balancing inputs. +### Current shard membership -The single longest estimate, `tests/fm-remote-secondmate-lifecycle-e2e.test.sh` at 700000 ms, is the floor for any shard count. +#### `portable-serial-1of7` — 17 scripts + +```text +tests/fm-remote-secondmate-lifecycle-e2e.test.sh +tests/fm-wake-drain-unread-status.test.sh +tests/fm-pi-watch-extension.test.sh +tests/fm-backend-orca.test.sh +tests/fm-pr-check-handoff.test.sh +tests/fm-gate-refuse.test.sh +tests/fm-todo-project.test.sh +tests/fm-secondmate-lifecycle-e2e.test.sh +tests/fm-treehouse-sweep.test.sh +tests/fm-omp-compact-adviser.test.sh +tests/fm-runpod-omp-auth.test.sh +tests/fm-test-isolation-proof.test.sh +tests/fm-operational-input.test.sh +tests/fm-cleanup-skill.test.sh +tests/fm-send-inbox-doorbell-live-e2e.test.sh +tests/fm-omp-branch-live-e2e.test.sh +tests/fm-omp-herdr-exit-live-e2e.test.sh +``` + +#### `portable-serial-2of7` — 23 scripts + +```text +tests/fm-watch-triage.test.sh +tests/fm-omp-harness.test.sh +tests/fm-runpod-lifecycle.test.sh +tests/fm-main-ci-watch.test.sh +tests/fm-task-inbox.test.sh +tests/fm-secondmate-restart.test.sh +tests/fm-backend.test.sh +tests/fm-procevent-when.test.sh +tests/fm-task-delivery.test.sh +tests/fm-wake-drain-open-decisions.test.sh +tests/fm-tangle-guard.test.sh +tests/fm-ext-plugin-dest.test.sh +tests/fm-guard-stale-banner.test.sh +tests/fm-busy-state.test.sh +tests/fm-receipt.test.sh +tests/fm-gotmp.test.sh +tests/fm-check-unregister.test.sh +tests/fm-ask-user-authority.test.sh +tests/fm-afk-pi-herdr-return-e2e.test.sh +tests/fm-omp-branch-bash.test.sh +tests/fm-omp-branch-types.test.sh +tests/fm-omp-secondmate-live-e2e.test.sh +tests/fm-omp-primary-live-e2e.test.sh +``` + +#### `portable-serial-3of7` — 23 scripts + +```text +tests/fm-spawn-dispatch-profile.test.sh +tests/fm-ext-topic-threads.test.sh +tests/fm-bearings-snapshot.test.sh +tests/fm-backlog-atomicity.test.sh +tests/fm-secondmate-safety.test.sh +tests/fm-trace-context-spawn.test.sh +tests/fm-runpod-routing.test.sh +tests/fm-fleet-sync.test.sh +tests/fm-boat-routing.test.sh +tests/fm-remote-backlog-handoff.test.sh +tests/fm-tasks-axi.test.sh +tests/fm-fleet-snapshot-view.test.sh +tests/fm-inbox.test.sh +tests/fm-ext-watch.test.sh +tests/fm-omp-relaunch-guard.test.sh +tests/fm-send-secondmate-marker.test.sh +tests/fm-remote-secondmate-sleep-reconcile.test.sh +tests/fm-supervision-events.test.sh +tests/fm-calm-pi-extension.test.sh +tests/fm-treehouse-orphan-recovery.test.sh +tests/fm-backend-cmux-smoke.test.sh +tests/fm-codex-continuity-live-e2e.test.sh +tests/fm-remote-omp-update-restart-live-e2e.test.sh +``` + +#### `portable-serial-4of7` — 25 scripts + +```text +tests/fm-secondmate-harness.test.sh +tests/fm-ext-waiting-pin.test.sh +tests/fm-watch-arm.test.sh +tests/fm-bootstrap.test.sh +tests/fm-session-start.test.sh +tests/fm-remote-job.test.sh +tests/fm-public-followup.test.sh +tests/fm-runpod-pod-boot.test.sh +tests/fm-omp-branch-supervision.test.sh +tests/fm-afk-inject-e2e.test.sh +tests/fm-omp-task-inbox-doorbell.test.sh +tests/fm-kimi-harness.test.sh +tests/fm-shared-captain-inheritance.test.sh +tests/fm-on.test.sh +tests/fm-herdr-session-cleanup.test.sh +tests/fm-watch-checkpoint.test.sh +tests/fm-backlog-handoff.test.sh +tests/fm-afk-return.test.sh +tests/fm-documentation-audiences.test.sh +tests/fm-classify-decision-key.test.sh +tests/fm-pending-reply-sleep-reconcile.test.sh +tests/fm-pi-compatible-family.test.sh +tests/fm-herdr-submit-confirm-live-e2e.test.sh +tests/fm-grok-stop-live-e2e.test.sh +tests/fm-grok-continuity-live-e2e.test.sh +``` + +#### `portable-serial-5of7` — 26 scripts + +```text +tests/fm-pr-check-security.test.sh +tests/fm-omp-secondmate.test.sh +tests/fm-wake-queue.test.sh +tests/fm-wake-daemon-lifecycle-e2e.test.sh +tests/fm-boat-omp-auth.test.sh +tests/fm-remote-secondmate-trace-context.test.sh +tests/fm-boat-lifecycle.test.sh +tests/fm-send-inbox.test.sh +tests/fm-teardown-endpoint-safety.test.sh +tests/fm-remote-secondmate-update-restart.test.sh +tests/fm-prepush-guard.test.sh +tests/fm-remote-secondmate-parent-binding.test.sh +tests/fm-pending-reply.test.sh +tests/fm-inbox-result.test.sh +tests/fm-backend-zellij.test.sh +tests/fm-remote-doctor.test.sh +tests/fm-backend-cmux.test.sh +tests/fm-session-lock-ancestry.test.sh +tests/fm-omp-update.test.sh +tests/fm-devin-adapter.test.sh +tests/fm-remote-entrypoint.test.sh +tests/fm-stall-recovery.test.sh +tests/fm-backend-herdr-focus-flash-e2e.test.sh +tests/fm-devin-herdr-live-e2e.test.sh +tests/fm-omp-compact-adviser-live-e2e.test.sh +tests/fm-omp-herdr-live-e2e.test.sh +``` + +#### `portable-serial-6of7` — 26 scripts + +```text +tests/fm-ext-decision.test.sh +tests/fm-boat-lane-runner.test.sh +tests/fm-ext-bridge.test.sh +tests/fm-procevent.test.sh +tests/fm-omp-primary.test.sh +tests/fm-daemon.test.sh +tests/fm-landing.test.sh +tests/fm-secondmate-sync.test.sh +tests/fm-spawn-relaunch-dead-endpoint.test.sh +tests/fm-send-resolve-key.test.sh +tests/fm-turnend-guard.test.sh +tests/fm-busy-adapter-wiring.test.sh +tests/fm-spawn-worktree-settle.test.sh +tests/fm-send-turn-start.test.sh +tests/fm-dispatch-resolve.test.sh +tests/fm-idle-sleep-watch.test.sh +tests/fm-omp-fleet-hooks.test.sh +tests/fm-trace-context-lib.test.sh +tests/fm-subagent-pretool-check.test.sh +tests/fm-reflect-skill.test.sh +tests/fm-treehouse-status-read-only.test.sh +tests/fm-send-secondmate-marker-herdr-e2e.test.sh +tests/fm-gitignore-config.test.sh +tests/fm-pi-primary-live-e2e.test.sh +tests/fm-opencode-primary-live-e2e.test.sh +tests/fm-omp-relaunch-tmux-live-e2e.test.sh +``` + +#### `portable-serial-7of7` — 26 scripts + +```text +tests/fm-hermes-harness.test.sh +tests/fm-teardown.test.sh +tests/fm-remote-reply.test.sh +tests/fm-watch-recovery-loop.test.sh +tests/fm-claude-stop-autoarm.test.sh +tests/fm-ext-ask.test.sh +tests/fm-watcher-lock.test.sh +tests/fm-vendor-auth-probe.test.sh +tests/fm-spawn-pool-base-freshen.test.sh +tests/fm-wake-drain-open-decisions-cursor.test.sh +tests/fm-secondmate-liveness.test.sh +tests/fm-ext-thread-context.test.sh +tests/fm-idle-sleep.test.sh +tests/fm-update.test.sh +tests/fm-startup-memory-budget.test.sh +tests/fm-receipt-check.test.sh +tests/fm-backend-herdr-server-detach.test.sh +tests/fm-merge-local.test.sh +tests/fm-test-fixture-cleanup.test.sh +tests/fm-sessionstart-nudge.test.sh +tests/fm-backend-tmux-smoke.test.sh +tests/fm-hermes-live-e2e.test.sh +tests/fm-backend-zellij-smoke.test.sh +tests/fm-claude-stop-autoarm-live-e2e.test.sh +tests/fm-omp-worker-tmux-live-e2e.test.sh +tests/fm-quota-array-dispatch-live-e2e.test.sh +``` -Refresh the hints by downloading the per-shard timing artifacts from a green CI run and replacing the `portable_serial_weight_hints` table in `bin/fm-test-run.sh` with the measured `path`/`duration_ms` pairs: +To refresh after a green run, replace the full hint table with measured per-script pairs and regenerate the membership snapshot above from the same hints. +Copy the generated assignments into the snapshot and run the partition guard. ```sh gh run download -R dnth/firstmate --pattern 'fm-test-timing-portable-serial-*' -D /tmp/fm-serial jq -r '.scripts[] | [.path, .duration_ms] | @tsv' /tmp/fm-serial/*.json | LC_ALL=C sort +for i in 1 2 3 4 5 6 7; do + printf 'portable-serial-%sof7\n' "$i" + bin/fm-test-run.sh --list --lane "portable-serial-${i}of7" +done bin/fm-test-run.sh --check-coverage ``` @@ -107,7 +392,7 @@ Portable shards, each portable serial shard, and the Herdr lane upload runner-ge | Job | timeout-minutes | Rationale | |---|---:|---| | portable parallel 1/2 | 10 | The measured shard sums are about three minutes and the timeout is a hang tripwire. | -| portable serial 1-6 | 20 | Each rebalanced shard estimates about 14 minutes of script time, leaving about six minutes for setup and runner-load variance while still tripping on a wedged test. | +| portable serial 1-7 | 20 | The seven-shard assignment projects 13m00.151s–13m00.166s from PR run 38774522507, leaving about 6m59.834s for setup and runner-load variance while keeping the timeout as a hang tripwire. | | Herdr | 40 | The real-Herdr lane keeps its dedicated timeout. | Timeouts are hang tripwires rather than expected healthy durations. diff --git a/tests/fm-remote-secondmate-lifecycle-e2e.test.sh b/tests/fm-remote-secondmate-lifecycle-e2e.test.sh index dd0062d7b78..68d8a2e8f46 100755 --- a/tests/fm-remote-secondmate-lifecycle-e2e.test.sh +++ b/tests/fm-remote-secondmate-lifecycle-e2e.test.sh @@ -970,7 +970,7 @@ resolve_ios_pending() { remote_env "$ROOT/bin/fm-procevent.sh" start "$SID" >/dev/null 2>&1 || true pending_result="" n=200 while [ -z "$pending_result" ] && [ "$n" -gt 0 ]; do - pending_result=$(find "$PARENT/state/procevent-inbox" -name "$SID.*.result" ! -name "$SID.1.result" ! -name "$SID.2.result" ! -name "$SID.3.result" -print | sort | tail -1) + pending_result=$(grep -lF "done [corr=$pending_corr]:" "$PARENT/state/procevent-inbox/$SID".*.result 2>/dev/null | head -1 || true) [ -n "$pending_result" ] || { n=$((n - 1)); sleep 0.1; } done [ -n "$pending_result" ] \