diff --git a/scripts/fleet-bridge-watch.sh b/scripts/fleet-bridge-watch.sh index dc06fc82..585de7a3 100755 --- a/scripts/fleet-bridge-watch.sh +++ b/scripts/fleet-bridge-watch.sh @@ -34,6 +34,10 @@ # sweep is scheduled separately with its own timeout, since # only 3 of 12 nodes have agent-cron and the rest would # otherwise never be checked for harness drift. +# CCC_FLEET_RETRIES extra attempts per node on transport failure (default 2, +# capped at 5; only UNREACHABLE is retried — a node that +# answers, even DOWN, is judged on its single answer) +# CCC_FLEET_RETRY_DELAY seconds between attempts (default 10, capped at 120) set -u NODES="${CCC_FLEET_NODES:-seoseo dungae sogyo nosuk bangtong yukson soonwook gwakga jingun gongmyoung gongyung daegyo}" @@ -116,6 +120,20 @@ else fi PROBE_EOF +# One unanswered probe is a transport blip, not a health signal: on 2026-07-31 +# and 2026-08-01 single SSH failures paged UNREACHABLE for nodes that were fine +# minutes later, and a 42% failure rate taught the fleet to ignore the watch — +# the one real outage (daegyo 2026-08-06, #968) looked identical to the noise +# (#972). Retry transport failures with a delay so a blip never reaches the +# notification path; a persistent failure still ends UNREACHABLE after the +# retries, and a node that answers is never re-asked. +RETRIES="${CCC_FLEET_RETRIES:-2}" +case "$RETRIES" in ''|*[!0-9]*) RETRIES=2 ;; esac +[ "$RETRIES" -le 5 ] || RETRIES=5 +RETRY_DELAY="${CCC_FLEET_RETRY_DELAY:-10}" +case "$RETRY_DELAY" in ''|*[!0-9]*) RETRY_DELAY=10 ;; esac +[ "$RETRY_DELAY" -le 120 ] || RETRY_DELAY=120 + fail=0 for node in $NODES; do # The flag is prepended to the piped script rather than passed as an ssh @@ -124,11 +142,18 @@ for node in $NODES; do payload="CCC_FLEET_DOCTOR=${CCC_FLEET_DOCTOR:-0} $PROBE" if [ "${CCC_FLEET_DOCTOR:-0}" = "1" ]; then node_budget=90; else node_budget=30; fi - if [ "$node" = "$SELF" ]; then - out=$(printf '%s' "$payload" | sh -s 2>/dev/null) - else - out=$(printf '%s' "$payload" | timeout "$node_budget" "$SSH_BIN" -o BatchMode=yes -o ConnectTimeout=8 "$node" sh -s 2>/dev/null) - fi + attempt=0 + while :; do + if [ "$node" = "$SELF" ]; then + out=$(printf '%s' "$payload" | sh -s 2>/dev/null) + else + out=$(printf '%s' "$payload" | timeout "$node_budget" "$SSH_BIN" -o BatchMode=yes -o ConnectTimeout=8 "$node" sh -s 2>/dev/null) + fi + [ -n "$out" ] && break + attempt=$((attempt + 1)) + [ "$attempt" -gt "$RETRIES" ] && break + [ "$RETRY_DELAY" -gt 0 ] && sleep "$RETRY_DELAY" + done if [ -z "$out" ]; then echo "UNREACHABLE $node"; fail=1; continue diff --git a/scripts/fleet-bridge-watch.test.sh b/scripts/fleet-bridge-watch.test.sh index f8fd2e82..946ef63e 100755 --- a/scripts/fleet-bridge-watch.test.sh +++ b/scripts/fleet-bridge-watch.test.sh @@ -39,6 +39,7 @@ reply() { # run() { # OUT="$TMP/out"; RC=0 CCC_FLEET_NODES="$1" CCC_FLEET_SSH="$STUB" CCC_FLEET_SELF=_never_ \ + CCC_FLEET_RETRY_DELAY=0 \ bash "$SC" >"$OUT" 2>&1 || RC=$? } @@ -184,5 +185,65 @@ ok "overridden root reports OK" 'grep -q "^OK beta (/srv/ccc-node)" "$OUT"' ok "no hardcoded node->path table" \ '! grep -nE "^(check|[a-z]+) +(seoseo|yukson|sogyo|nosuk|dungae) +.*(/opt/ccc-node|/root/ccc-node)" "$SC"' +# ---- transport retry (#972) ------------------------------------------------ +# Flaky stub: fails while $TMP/flaky/ holds a positive counter, then +# answers from reply/. Every invocation is logged to $TMP/calls so the +# tests can count attempts per node. +FLAKY="$TMP/ssh-flaky" +cat > "$FLAKY" </dev/null +node="" +for a in "\$@"; do case "\$a" in -o|-*) ;; sh|-s) ;; *) node="\$a" ;; esac; done +printf '%s\n' "\$node" >> "$TMP/calls" +budget="$TMP/flaky/\$node" +left=0; [ -f "\$budget" ] && left=\$(cat "\$budget") +if [ "\$left" -gt 0 ] 2>/dev/null; then printf '%s\n' \$((left - 1)) > "\$budget"; exit 255; fi +f="$TMP/reply/\$node" +[ -f "\$f" ] || exit 255 +cat "\$f" +FLAKYEOF +chmod +x "$FLAKY" +mkdir -p "$TMP/flaky" +: > "$TMP/calls" + +run_flaky() { # + OUT="$TMP/out"; RC=0 + CCC_FLEET_NODES="$1" CCC_FLEET_SSH="$FLAKY" CCC_FLEET_SELF=_never_ \ + CCC_FLEET_RETRIES="$2" CCC_FLEET_RETRY_DELAY=0 \ + bash "$SC" >"$OUT" 2>&1 || RC=$? +} + +# One blip, then healthy: the retry absorbs it and the node reports OK. +reply blip /opt/ccc-node yes /opt/ccc-node +printf '1\n' > "$TMP/flaky/blip" +run_flaky "blip" 2 +okc "$RC" 0 "single blip recovered by retry exits 0" +ok "blip node reports OK, never UNREACHABLE" 'grep -q "^OK blip" "$OUT" && ! grep -q "^UNREACHABLE blip" "$OUT"' +ok "blip took exactly two attempts" '[ "$(grep -c "^blip$" "$TMP/calls")" = 2 ]' + +# Persistent transport failure: still UNREACHABLE, reported once, after +# retries+1 attempts. +: > "$TMP/calls" +run_flaky "ghost2" 2 +okc "$RC" 1 "persistent failure exits nonzero" +ok "persistent failure reported once" '[ "$(grep -c "^UNREACHABLE ghost2" "$OUT")" = 1 ]' +ok "persistent failure used every attempt" '[ "$(grep -c "^ghost2$" "$TMP/calls")" = 3 ]' + +# DOWN is a real answer, not a transport failure: never retried. +: > "$TMP/calls" +reply sick /opt/ccc-node no /opt/ccc-node +run_flaky "sick" 2 +okc "$RC" 1 "down node exits nonzero" +ok "down node reported, never retried" 'grep -q "^DOWN sick" "$OUT" && [ "$(grep -c "^sick$" "$TMP/calls")" = 1 ]' + +# CCC_FLEET_RETRIES=0 keeps the single-attempt contract. +: > "$TMP/calls" +printf '1\n' > "$TMP/flaky/blip0" +reply blip0 /opt/ccc-node yes /opt/ccc-node +run_flaky "blip0" 0 +okc "$RC" 1 "retries disabled still fails on one blip" +ok "retries disabled means exactly one attempt" '[ "$(grep -c "^blip0$" "$TMP/calls")" = 1 ] && grep -q "^UNREACHABLE blip0" "$OUT"' + echo "----"; echo "PASS=$pass FAIL=$fail" [ "$fail" = 0 ]