From 4c8545c47b808be87fbec80923519ad920de68f3 Mon Sep 17 00:00:00 2001 From: jinon86 <247078695+jinon86@users.noreply.github.com> Date: Thu, 6 Aug 2026 01:49:02 +0000 Subject: [PATCH] fix(self-update): run external restart inside audit/notify boundary + runtime recovery tick (#971) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The hand-chained cron line (ccc-self-update run; rc=11 -> start.sh --restart; exit 0) detected its own restart failure, reported it to a log file, and then discarded the signal: daegyo stayed down 4h15m until the daily fleet watch noticed. The 05:45 retry slot could never help because it only retries the update, never the restart. - new operator-owned hooks: self-update.restart-cmd (external restart command) and self-update.health-cmd (runtime probe), same trust model as self-update.services - degraded path (code changed, nothing restarted): run the restart command inside the script — outcome audited/notified, failure exits 7 with the recovery snapshot retained instead of a silent exit 0 - up-to-date tick: with both hooks configured, verify runtime health and attempt one recovery restart when the runtime is down, so the second daily slot can recover an updated-but-down node - docs: supported replacement for the chained cron line Closes #971 --- docs/self-update.md | 31 +++++++- scripts/ccc-self-update.sh | 136 ++++++++++++++++++++++++++++---- scripts/ccc-self-update.test.sh | 56 +++++++++++++ 3 files changed, 206 insertions(+), 17 deletions(-) diff --git a/docs/self-update.md b/docs/self-update.md index e49b40da..8316c13c 100644 --- a/docs/self-update.md +++ b/docs/self-update.md @@ -103,6 +103,31 @@ and a no-services-allowlist degraded run (11) do not raise on-failure alerts — only real aborts do. To verify registration: `agent-cron.sh list | grep self-update`. +### External restart command (#971) + +Hosts where systemd cannot restart the bridge (Termux `start.sh`, user-scoped +units) previously chained `start.sh --restart` off the cron line with a +trailing `exit 0` — a failed restart was detected, reported to a log file, +and then thrown away (daegyo 2026-08-06: 4h15m silent outage). The supported +replacement is two operator-owned files: + +```sh +cat > ~/.claude/self-update.restart-cmd <<'EOF' +$HOME/ccc-node/bridge/start.sh --path "$HOME" --restart -d +EOF +cat > ~/.claude/self-update.health-cmd <<'EOF' +pgrep -f telegram_bot >/dev/null +EOF +``` + +With a restart command configured, the degraded path (code changed, no +allowlisted service) runs it inside the script: the outcome is audited, +notified, and a failure exits `7` with the recovery snapshot retained. With a +health probe also configured, every up-to-date tick verifies runtime health +and attempts one recovery restart when the runtime is down — the second daily +slot can therefore recover an updated-but-down node. The cron line becomes a +plain `ccc-self-update.sh run` with no shell chaining. + ## Knobs | Env | Default | Meaning | @@ -111,6 +136,9 @@ self-update`. | `CCC_SELF_UPDATE_BRANCH` | `main` | branch the node must be on | | `CCC_SELF_UPDATE_SERVICES` | `~/.claude/self-update.services` | allowlist path | | `CCC_SELF_UPDATE_SYSTEMCTL` | `systemctl` | service manager command (tests inject a fake) | +| `CCC_SELF_UPDATE_RESTART_CMD` | `~/.claude/self-update.restart-cmd` | external restart command for hosts where systemd cannot reach the bridge (Termux, user-scoped). Runs inside the audit/notify boundary (#971) | +| `CCC_SELF_UPDATE_HEALTH_CMD` | `~/.claude/self-update.health-cmd` | runtime health probe (exit 0 = healthy); with a restart command configured, an up-to-date tick that finds the runtime down attempts one recovery restart | +| `CCC_SELF_UPDATE_RESTART_WAIT_SECONDS` | `60` | health-poll budget after an external restart | | `CCC_SELF_UPDATE_HEALTH_FILE` | `~/.telegram_bot/health.json` | bridge health file the idle gate reads | | `CCC_SELF_UPDATE_HEALTH_FRESH_SECONDS` | `90` | max age of `health.json` for its workload to count | | `CCC_SELF_UPDATE_BUSY_MAX_SECONDS` | `1800` | never defer for a task older than this | @@ -118,7 +146,8 @@ self-update`. Exit codes: 0 ok/up-to-date · 3 lock held · 4 precondition failed · 5 fetch/ff failed · 6 setup/snapshot failed (repo and managed artifacts were verified -rolled back, or setup never started) · 7 service restart failure · 8 deferred +rolled back, or setup never started) · 7 service restart failure, external +restart-cmd failure, or failed runtime recovery · 8 deferred (bridge busy — retry next tick) · 9 repository or installed-artifact rollback was degraded · 10 successful-update recovery snapshot cleanup failed · 11 degraded — code updated but no allowlisted service restarted (services diff --git a/scripts/ccc-self-update.sh b/scripts/ccc-self-update.sh index 6cff6676..ed2239a0 100755 --- a/scripts/ccc-self-update.sh +++ b/scripts/ccc-self-update.sh @@ -11,6 +11,15 @@ # write: # ~/.claude/self-update.services ([user:|system:]unit per line, # comments) # ~/.claude/self-update.repo (optional: absolute repo path override) +# ~/.claude/self-update.restart-cmd (optional: one external restart command +# for hosts where systemd cannot reach the bridge, e.g. Termux +# `bridge/start.sh --path "$HOME" --restart -d`. Runs INSIDE this script's +# audit/notify boundary so its failure can never be discarded the way the +# hand-chained cron `... ; exit 0` discarded it on daegyo (#971).) +# ~/.claude/self-update.health-cmd (optional: one runtime health probe, exit +# 0 = healthy. With both files present, an up-to-date tick that finds the +# runtime DOWN attempts one recovery restart — so the second daily slot +# can recover an updated-but-down node (#971).) # # Procedure (run): # 1. take a lock; resolve the repo (env > repo file > script location > ~/ccc-node) @@ -35,9 +44,10 @@ # (1800 — never defer a task older than this), CCC_SELF_UPDATE_MAX_DEFER_SECONDS # (3600 — cap total deferral so continuous load can't starve updates). # Fail-open (missing/unreadable/stale health → proceed); --force bypasses. -# Exit: 0 = up-to-date or updated cleanly; 8 = deferred (bridge busy); 11 = -# degraded (code updated but no service restarted — allowlist missing/empty, -# runtime may be stale); other non-zero = aborted (reason logged). +# Exit: 0 = up-to-date or updated cleanly; 7 = a restart (allowlisted service +# or external restart-cmd) or a recovery attempt failed; 8 = deferred +# (bridge busy); 11 = degraded (code updated but nothing restarted and no +# restart-cmd configured); other non-zero = aborted (reason logged). set -uo pipefail CLAUDE_DIR="${CCC_CLAUDE_DIR:-${HOME:-/root}/.claude}" @@ -48,6 +58,9 @@ LOCK="$STATE_DIR/self-update.lock" SPOOL="${CCC_PUSH_SPOOL:-$STATE_DIR/telegram-spool}" SERVICES_FILE="${CCC_SELF_UPDATE_SERVICES:-$CLAUDE_DIR/self-update.services}" REPO_FILE="$CLAUDE_DIR/self-update.repo" +RESTART_CMD_FILE="${CCC_SELF_UPDATE_RESTART_CMD_FILE:-$CLAUDE_DIR/self-update.restart-cmd}" +HEALTH_CMD_FILE="${CCC_SELF_UPDATE_HEALTH_CMD_FILE:-$CLAUDE_DIR/self-update.health-cmd}" +RESTART_WAIT_SECONDS="${CCC_SELF_UPDATE_RESTART_WAIT_SECONDS:-60}" BRANCH="${CCC_SELF_UPDATE_BRANCH:-main}" SYSTEMCTL="${CCC_SELF_UPDATE_SYSTEMCTL:-systemctl}" @@ -100,6 +113,55 @@ audit() { # >> "$LOG" 2>/dev/null } +read_operator_cmd() { # — first non-comment, non-blank line (operator-owned) + [ -f "$1" ] || return 1 + local line + while IFS= read -r line; do + line="${line%%#*}" + line="$(printf '%s' "$line" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" + [ -n "$line" ] && { printf '%s' "$line"; return 0; } + done < "$1" + return 1 +} + +resolve_restart_cmd() { + if [ -n "${CCC_SELF_UPDATE_RESTART_CMD:-}" ]; then printf '%s' "$CCC_SELF_UPDATE_RESTART_CMD"; return 0; fi + read_operator_cmd "$RESTART_CMD_FILE" +} + +resolve_health_cmd() { + if [ -n "${CCC_SELF_UPDATE_HEALTH_CMD:-}" ]; then printf '%s' "$CCC_SELF_UPDATE_HEALTH_CMD"; return 0; fi + read_operator_cmd "$HEALTH_CMD_FILE" +} + +# Run the operator's external restart command INSIDE the audit/notify boundary. +# Outcome: health-cmd poll (when configured, up to RESTART_WAIT_SECONDS), else +# the command's own exit code. Returns 0 = runtime back, 1 = still down. +run_external_restart() { + local rcmd hcmd rc waited + rcmd="$(resolve_restart_cmd)" || return 1 + hcmd="$(resolve_health_cmd || true)" + log "external-restart begin" + if command -v timeout >/dev/null 2>&1; then + timeout 180 bash -c "$rcmd" >>"$LOG" 2>&1 + else + bash -c "$rcmd" >>"$LOG" 2>&1 + fi + rc=$? + log "external-restart exit=$rc" + if [ -n "$hcmd" ]; then + waited=0 + until bash -c "$hcmd" >>"$LOG" 2>&1; do + waited=$((waited + 3)) + [ "$waited" -ge "$RESTART_WAIT_SECONDS" ] && { log "external-restart health-timeout waited=${waited}s"; return 1; } + sleep 3 + done + log "external-restart healthy waited=${waited}s" + return 0 + fi + return "$rc" +} + snapshot_installed_artifacts() { local existing=() item ccc_validate_managed_artifacts "self-update:" "$CLAUDE_DIR" "$HERMES_ROOT" "${CCC_MANAGED_PATHS[@]}" || return 1 @@ -337,6 +399,30 @@ CHANGED=false [ "$OLD_SHA" != "$NEW_SHA" ] && CHANGED=true if [ "$CHANGED" = "false" ] && [ "$FORCE" != "1" ]; then + # Second-slot runtime recovery (#971): code is current, but an earlier + # chained restart may have failed and left the runtime down. When the + # operator configured both a health probe and an external restart command, + # verify runtime health and attempt ONE recovery restart — with the outcome + # audited and notified, never discarded. + if hcmd="$(resolve_health_cmd)" && resolve_restart_cmd >/dev/null 2>&1; then + if bash -c "$hcmd" >>"$LOG" 2>&1; then + log "done result=up-to-date sha=$NEW_SHA runtime=healthy" + say "self-update: already up to date ($(git -C "$REPO" rev-parse --short HEAD))" + exit 0 + fi + SHORT_CUR="$(git -C "$REPO" rev-parse --short HEAD 2>/dev/null)" + log "runtime unhealthy at up-to-date tick; attempting recovery restart" + if run_external_restart; then + audit "runtime-recovered" "$OLD_SHA" "$NEW_SHA" "$CHANGED" true '[{"name":"external-restart","ok":true,"scope":"external"}]' + notify "self-update ${SHORT_CUR}: 코드는 최신이나 런타임 다운 감지 — 외부 재시작으로 복구 완료. ~/.claude/state/self-update.log" "recovered-$NEW_SHA" + say "self-update: code up to date but runtime was down; recovered via external restart" + exit 0 + fi + audit "runtime-down" "$OLD_SHA" "$NEW_SHA" "$CHANGED" true '[{"name":"external-restart","ok":false,"scope":"external"}]' + notify "self-update ${SHORT_CUR} 경고: 코드는 최신이나 런타임이 다운 상태이며 복구 재시작도 실패했습니다. 브리지가 남아있는지 즉시 확인 필요. ~/.claude/state/self-update.log" "runtime-down-$NEW_SHA" + say "self-update: code up to date but runtime is DOWN and the recovery restart failed" >&2 + exit 7 + fi log "done result=up-to-date sha=$NEW_SHA" say "self-update: already up to date ($(git -C "$REPO" rev-parse --short HEAD))" exit 0 @@ -456,6 +542,37 @@ if [ "$FAILED" -gt 0 ]; then say "self-update: updated to $SHORT_NEW but $FAILED service(s) failed to restart; recovery snapshot retained at $INSTALL_SNAPSHOT_DIR" >&2 exit 7 fi +# Per #910: code changed but NO service was restarted (services allowlist file +# missing or empty). With an operator-configured external restart command +# (#971, e.g. Termux start.sh), run it HERE — inside the audit/notify boundary +# — instead of letting a hand-chained cron line discard its failure. Success +# falls through to the shared snapshot-cleanup/ok path; failure keeps the +# recovery snapshot, notifies, and exits non-zero. Without a configured +# command, report degraded (not ok) and exit non-zero so it cannot read as +# success. +if [ "$CHANGED" = "true" ] && [ "$RESTARTED" -eq 0 ]; then + if resolve_restart_cmd >/dev/null 2>&1; then + if run_external_restart; then + RESTARTED=1 + SERVICES_JSON="$(printf '%s' "$SERVICES_JSON" | jq -c '. + [{"name":"external-restart","ok":true,"scope":"external"}]')" + log "external-restart ok; proceeding to cleanup" + else + KEEP_INSTALL_SNAPSHOT=1 + SERVICES_JSON="$(printf '%s' "$SERVICES_JSON" | jq -c '. + [{"name":"external-restart","ok":false,"scope":"external"}]')" + audit "restart-failures" "$OLD_SHA" "$NEW_SHA" "$CHANGED" "$SETUP_OK" "$SERVICES_JSON" + log "recovery snapshot=$INSTALL_SNAPSHOT_DIR oldSha=$OLD_SHA reason=external-restart-failure" + notify "self-update ${SHORT_NEW}: 코드 갱신 후 외부 재시작 명령이 실패했습니다 — 브리지가 남아있는지 즉시 확인 필요. 롤백 자료 보존: ${INSTALL_SNAPSHOT_DIR}. ~/.claude/state/self-update.log" "fail-$NEW_SHA" + say "self-update: updated to $SHORT_NEW but the external restart command failed; recovery snapshot retained at $INSTALL_SNAPSHOT_DIR" >&2 + exit 7 + fi + else + audit "degraded-no-services" "$OLD_SHA" "$NEW_SHA" "$CHANGED" "$SETUP_OK" "$SERVICES_JSON" + notify "self-update ${SHORT_NEW}: 코드 갱신됐으나 재시작된 서비스 없음 (허용목록 누락/비어있음 의심). 실행 중 프로세스가 옛 코드일 수 있음 — self-update.services 확인 필요. ~/.claude/state/self-update.log" "degraded-$NEW_SHA" + say "self-update: degraded — ${OLD_SHA:0:7} → ${SHORT_NEW}, services restarted: 0 (no allowlisted services; runtime may be stale)" >&2 + exit 11 + fi +fi + if ! rm -rf -- "$INSTALL_SNAPSHOT_DIR"; then # Do not turn a failed private-snapshot cleanup into a reported success. # Keep the path available to the operator (and prevent the EXIT trap from @@ -469,19 +586,6 @@ if ! rm -rf -- "$INSTALL_SNAPSHOT_DIR"; then fi INSTALL_SNAPSHOT_DIR="" -# Per #910: code changed but NO service was restarted (services allowlist file -# missing or empty). Running processes still hold OLD code — silent code/runtime -# drift that previously reported result:"ok" / "services restarted: 0". Report -# it as degraded (not ok) and exit non-zero so it cannot read as success. -# (FAILED==0 is guaranteed here — the FAILED>0 path exited 7 above — so a zero -# restart count with a change means nothing was even attempted.) -if [ "$CHANGED" = "true" ] && [ "$RESTARTED" -eq 0 ]; then - audit "degraded-no-services" "$OLD_SHA" "$NEW_SHA" "$CHANGED" "$SETUP_OK" "$SERVICES_JSON" - notify "self-update ${SHORT_NEW}: 코드 갱신됐으나 재시작된 서비스 없음 (허용목록 누락/비어있음 의심). 실행 중 프로세스가 옛 코드일 수 있음 — self-update.services 확인 필요. ~/.claude/state/self-update.log" "degraded-$NEW_SHA" - say "self-update: degraded — ${OLD_SHA:0:7} → ${SHORT_NEW}, services restarted: 0 (no allowlisted services; runtime may be stale)" >&2 - exit 11 -fi - audit "ok" "$OLD_SHA" "$NEW_SHA" "$CHANGED" "$SETUP_OK" "$SERVICES_JSON" if [ "$CHANGED" = "true" ]; then notify "self-update 완료: ${OLD_SHA:0:7} → ${SHORT_NEW}, 서비스 ${RESTARTED}개 재시작." "ok-$NEW_SHA" diff --git a/scripts/ccc-self-update.test.sh b/scripts/ccc-self-update.test.sh index 47264b7c..d438762e 100755 --- a/scripts/ccc-self-update.test.sh +++ b/scripts/ccc-self-update.test.sh @@ -22,6 +22,8 @@ export GIT_AUTHOR_NAME=t GIT_AUTHOR_EMAIL=t@t GIT_COMMITTER_NAME=t GIT_COMMITTER # actively serving a session every fixture update defers (rc=8) and the suite # mass-fails. Point it at a nonexistent fixture path instead (fail-open). unset CCC_SELF_UPDATE_BRANCH CCC_SELF_UPDATE_SERVICES +unset CCC_SELF_UPDATE_RESTART_CMD CCC_SELF_UPDATE_HEALTH_CMD +unset CCC_SELF_UPDATE_RESTART_CMD_FILE CCC_SELF_UPDATE_HEALTH_CMD_FILE unset CCC_SELF_UPDATE_HEALTH_FRESH_SECONDS unset CCC_SELF_UPDATE_BUSY_MAX_SECONDS CCC_SELF_UPDATE_MAX_DEFER_SECONDS export CCC_SELF_UPDATE_HEALTH_FILE="$TMP/no-such-health.json" @@ -71,6 +73,7 @@ run_selfup() { CCC_HERMES_DIR="$HERMES" \ CCC_SELF_UPDATE_BRIDGE_PROJECT_ROOT="$TMP/project" \ CCC_SELF_UPDATE_REPO="$REPO" CCC_SELF_UPDATE_SYSTEMCTL="$FAKEBIN/fakesystemctl" \ + CCC_SELF_UPDATE_RESTART_WAIT_SECONDS=3 \ CCC_NODE=testnode bash "$SELFUP" "$@" } @@ -107,6 +110,59 @@ ok "degraded notification warns of stale runtime" 'grep -rh "재시작된 서비 # restore the allowlist so later sections restart normally printf '%s\n' 'hermes-broker' 'a2a-worker' > "$CLAUDE/self-update.services" +# --- #971: external restart-cmd runs INSIDE the audit/notify boundary --------- +# (1) changed + no allowlist + restart-cmd succeeds -> ok, not degraded. +echo drift2 > "$TMP/seed/file2.txt" +git -C "$TMP/seed" add -A && git -C "$TMP/seed" commit -qm four && git -C "$TMP/seed" push -q origin main +rm -f "$CLAUDE/self-update.services" +printf 'touch %s\n' "$TMP/external-restarted.marker" > "$CLAUDE/self-update.restart-cmd" +out="$(run_selfup run 2>&1)"; rc=$? +ok "external restart-cmd on change exits 0 (not degraded 11)" '[ "$rc" = 0 ]' +ok "external restart-cmd actually ran" '[ -f "$TMP/external-restarted.marker" ]' +ok "external restart audited as ok with external scope" \ + 'grep -q "\"result\":\"ok\"" "$STATE/self-update.log" && grep -q "\"name\":\"external-restart\",\"ok\":true" "$STATE/self-update.log"' + +# (2) changed + no allowlist + restart-cmd FAILS -> rc 7 + failure notified, +# never silently discarded (the daegyo cron `exit 0` bug). +echo drift3 > "$TMP/seed/file3.txt" +git -C "$TMP/seed" add -A && git -C "$TMP/seed" commit -qm five && git -C "$TMP/seed" push -q origin main +printf '%s\n' 'exit 1' > "$CLAUDE/self-update.restart-cmd" +out="$(run_selfup run 2>&1)"; rc=$? +ok "failing external restart-cmd exits 7" '[ "$rc" = 7 ]' +ok "failing external restart audited as restart-failures" \ + 'grep -q "\"result\":\"restart-failures\"" "$STATE/self-update.log" && grep -q "\"name\":\"external-restart\",\"ok\":false" "$STATE/self-update.log"' +ok "failing external restart notifies immediately" 'grep -rh "외부 재시작 명령이 실패" "$TMP/spool" >/dev/null 2>&1' +ok "external restart failure retains recovery snapshot" 'compgen -G "$STATE/self-update-install-rollback.*" >/dev/null' +rm -rf "$STATE"/self-update-install-rollback.* + +# (3) up-to-date but runtime DOWN + health/restart-cmd -> recovery restart. +printf '[ -f %s ]\n' "$TMP/runtime-healthy" > "$CLAUDE/self-update.health-cmd" +printf 'touch %s\n' "$TMP/runtime-healthy" > "$CLAUDE/self-update.restart-cmd" +out="$(run_selfup run 2>&1)"; rc=$? +ok "up-to-date with down runtime recovers (rc 0)" '[ "$rc" = 0 ]' +ok "recovery restart ran and runtime is healthy" '[ -f "$TMP/runtime-healthy" ]' +ok "recovery audited as runtime-recovered" 'grep -q "\"result\":\"runtime-recovered\"" "$STATE/self-update.log"' +ok "recovery notified" 'grep -rh "런타임 다운 감지" "$TMP/spool" >/dev/null 2>&1' + +# (4) up-to-date and runtime healthy -> no recovery attempt at all. +printf 'touch %s\n' "$TMP/restart-should-not-run" > "$CLAUDE/self-update.restart-cmd" +out="$(run_selfup run 2>&1)"; rc=$? +ok "healthy up-to-date tick exits 0 without recovery" '[ "$rc" = 0 ] && [ ! -e "$TMP/restart-should-not-run" ]' +ok "healthy tick does not audit a second recovery" '[ "$(grep -c "runtime-recovered" "$STATE/self-update.log")" = 1 ]' + +# (5) up-to-date, runtime down, recovery FAILS -> rc 7 + notified. +rm -f "$TMP/runtime-healthy" +printf '[ -f %s ]\n' "$TMP/never-healthy" > "$CLAUDE/self-update.health-cmd" +printf '%s\n' 'exit 1' > "$CLAUDE/self-update.restart-cmd" +out="$(run_selfup run 2>&1)"; rc=$? +ok "failed recovery exits 7" '[ "$rc" = 7 ]' +ok "failed recovery audited as runtime-down" 'grep -q "\"result\":\"runtime-down\"" "$STATE/self-update.log"' +ok "failed recovery notifies" 'grep -rh "복구 재시작도 실패" "$TMP/spool" >/dev/null 2>&1' + +# cleanup: restore the allowlist and drop the external-cmd fixtures. +rm -f "$CLAUDE/self-update.restart-cmd" "$CLAUDE/self-update.health-cmd" +printf '%s\n' 'hermes-broker' 'a2a-worker' > "$CLAUDE/self-update.services" + # A target commit must not restart into existing invalid node-local bridge # timeout settings; it rolls back before systemctl touches the allowlisted unit. mkdir -p "$TMP/seed/bridge"