fix(fleet-watch): retry transport failures before paging UNREACHABLE (#972)

scripts/fleet-bridge-watch.sh

@@ -34,6 +34,10 @@
 #                    sweep is scheduled separately with its own timeout, since
 #                    only 3 of 12 nodes have agent-cron and the rest would
 #                    otherwise never be checked for harness drift.
+#   CCC_FLEET_RETRIES extra attempts per node on transport failure (default 2,
+#                    capped at 5; only UNREACHABLE is retried — a node that
+#                    answers, even DOWN, is judged on its single answer)
+#   CCC_FLEET_RETRY_DELAY seconds between attempts (default 10, capped at 120)
 set -u
 
 NODES="${CCC_FLEET_NODES:-seoseo dungae sogyo nosuk bangtong yukson soonwook gwakga jingun gongmyoung gongyung daegyo}"
@@ -116,6 +120,20 @@ else
 fi
 PROBE_EOF
 
+# One unanswered probe is a transport blip, not a health signal: on 2026-07-31
+# and 2026-08-01 single SSH failures paged UNREACHABLE for nodes that were fine
+# minutes later, and a 42% failure rate taught the fleet to ignore the watch —
+# the one real outage (daegyo 2026-08-06, #968) looked identical to the noise
+# (#972). Retry transport failures with a delay so a blip never reaches the
+# notification path; a persistent failure still ends UNREACHABLE after the
+# retries, and a node that answers is never re-asked.
+RETRIES="${CCC_FLEET_RETRIES:-2}"
+case "$RETRIES" in ''|*[!0-9]*) RETRIES=2 ;; esac
+[ "$RETRIES" -le 5 ] || RETRIES=5
+RETRY_DELAY="${CCC_FLEET_RETRY_DELAY:-10}"
+case "$RETRY_DELAY" in ''|*[!0-9]*) RETRY_DELAY=10 ;; esac
+[ "$RETRY_DELAY" -le 120 ] || RETRY_DELAY=120
+
 fail=0
 for node in $NODES; do
   # The flag is prepended to the piped script rather than passed as an ssh
@@ -124,11 +142,18 @@ for node in $NODES; do
   payload="CCC_FLEET_DOCTOR=${CCC_FLEET_DOCTOR:-0}
 $PROBE"
   if [ "${CCC_FLEET_DOCTOR:-0}" = "1" ]; then node_budget=90; else node_budget=30; fi
-  if [ "$node" = "$SELF" ]; then
-    out=$(printf '%s' "$payload" | sh -s 2>/dev/null)
-  else
-    out=$(printf '%s' "$payload" | timeout "$node_budget" "$SSH_BIN" -o BatchMode=yes -o ConnectTimeout=8 "$node" sh -s 2>/dev/null)
-  fi
+  attempt=0
+  while :; do
+    if [ "$node" = "$SELF" ]; then
+      out=$(printf '%s' "$payload" | sh -s 2>/dev/null)
+    else
+      out=$(printf '%s' "$payload" | timeout "$node_budget" "$SSH_BIN" -o BatchMode=yes -o ConnectTimeout=8 "$node" sh -s 2>/dev/null)
+    fi
+    [ -n "$out" ] && break
+    attempt=$((attempt + 1))
+    [ "$attempt" -gt "$RETRIES" ] && break
+    [ "$RETRY_DELAY" -gt 0 ] && sleep "$RETRY_DELAY"
+  done
 
   if [ -z "$out" ]; then
     echo "UNREACHABLE $node"; fail=1; continue

scripts/fleet-bridge-watch.test.sh

@@ -39,6 +39,7 @@ reply() { # <node> <runtime> <avail> <unit>
 run() { # <nodes>
   OUT="$TMP/out"; RC=0
   CCC_FLEET_NODES="$1" CCC_FLEET_SSH="$STUB" CCC_FLEET_SELF=_never_ \
+    CCC_FLEET_RETRY_DELAY=0 \
     bash "$SC" >"$OUT" 2>&1 || RC=$?
 }
 
@@ -184,5 +185,65 @@ ok "overridden root reports OK" 'grep -q "^OK beta (/srv/ccc-node)" "$OUT"'
 ok "no hardcoded node->path table" \
   '! grep -nE "^(check|[a-z]+) +(seoseo|yukson|sogyo|nosuk|dungae) +.*(/opt/ccc-node|/root/ccc-node)" "$SC"'
 
+# ---- transport retry (#972) ------------------------------------------------
+# Flaky stub: fails while $TMP/flaky/<node> holds a positive counter, then
+# answers from reply/<node>. Every invocation is logged to $TMP/calls so the
+# tests can count attempts per node.
+FLAKY="$TMP/ssh-flaky"
+cat > "$FLAKY" <<FLAKYEOF
+#!$(command -v bash)
+cat >/dev/null
+node=""
+for a in "\$@"; do case "\$a" in -o|-*) ;; sh|-s) ;; *) node="\$a" ;; esac; done
+printf '%s\n' "\$node" >> "$TMP/calls"
+budget="$TMP/flaky/\$node"
+left=0; [ -f "\$budget" ] && left=\$(cat "\$budget")
+if [ "\$left" -gt 0 ] 2>/dev/null; then printf '%s\n' \$((left - 1)) > "\$budget"; exit 255; fi
+f="$TMP/reply/\$node"
+[ -f "\$f" ] || exit 255
+cat "\$f"
+FLAKYEOF
+chmod +x "$FLAKY"
+mkdir -p "$TMP/flaky"
+: > "$TMP/calls"
+
+run_flaky() { # <nodes> <retries>
+  OUT="$TMP/out"; RC=0
+  CCC_FLEET_NODES="$1" CCC_FLEET_SSH="$FLAKY" CCC_FLEET_SELF=_never_ \
+    CCC_FLEET_RETRIES="$2" CCC_FLEET_RETRY_DELAY=0 \
+    bash "$SC" >"$OUT" 2>&1 || RC=$?
+}
+
+# One blip, then healthy: the retry absorbs it and the node reports OK.
+reply blip /opt/ccc-node yes /opt/ccc-node
+printf '1\n' > "$TMP/flaky/blip"
+run_flaky "blip" 2
+okc "$RC" 0 "single blip recovered by retry exits 0"
+ok "blip node reports OK, never UNREACHABLE" 'grep -q "^OK blip" "$OUT" && ! grep -q "^UNREACHABLE blip" "$OUT"'
+ok "blip took exactly two attempts" '[ "$(grep -c "^blip$" "$TMP/calls")" = 2 ]'
+
+# Persistent transport failure: still UNREACHABLE, reported once, after
+# retries+1 attempts.
+: > "$TMP/calls"
+run_flaky "ghost2" 2
+okc "$RC" 1 "persistent failure exits nonzero"
+ok "persistent failure reported once" '[ "$(grep -c "^UNREACHABLE ghost2" "$OUT")" = 1 ]'
+ok "persistent failure used every attempt" '[ "$(grep -c "^ghost2$" "$TMP/calls")" = 3 ]'
+
+# DOWN is a real answer, not a transport failure: never retried.
+: > "$TMP/calls"
+reply sick /opt/ccc-node no /opt/ccc-node
+run_flaky "sick" 2
+okc "$RC" 1 "down node exits nonzero"
+ok "down node reported, never retried" 'grep -q "^DOWN sick" "$OUT" && [ "$(grep -c "^sick$" "$TMP/calls")" = 1 ]'
+
+# CCC_FLEET_RETRIES=0 keeps the single-attempt contract.
+: > "$TMP/calls"
+printf '1\n' > "$TMP/flaky/blip0"
+reply blip0 /opt/ccc-node yes /opt/ccc-node
+run_flaky "blip0" 0
+okc "$RC" 1 "retries disabled still fails on one blip"
+ok "retries disabled means exactly one attempt" '[ "$(grep -c "^blip0$" "$TMP/calls")" = 1 ] && grep -q "^UNREACHABLE blip0" "$OUT"'
+
 echo "----"; echo "PASS=$pass FAIL=$fail"
 [ "$fail" = 0 ]