From a9145b8eddcd6d6a16f282f5487ef4ae5070d04b Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 15 Sep 2026 02:13:53 +0000 Subject: [PATCH 1/3] redgate: point hooks at the handlers that exist; sort runs in C locale MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two fixes lifted out of #115 so they do not wait on 76k lines. hooks.json referenced ${CLAUDE_PLUGIN_ROOT}/hooks-handlers/.sh, but both handlers live under hooks/hooks-handlers/, and ${CLAUDE_PLUGIN_ROOT} is the plugin dir on an installed plugin (agent-compiler's own hooks.json resolves the same way). No plugins/redgate/hooks-handlers/ directory exists. So on every installed copy the PreToolUse write guard — the hook that stops a write reaching a ratified run's contract while its round is in MIDDLE — and the SessionStart announcer silently never ran. The cheap tier was green the whole time because nothing resolved handler paths against the filesystem. criteria-index.sh sorted run dirs with the host locale. Its own header promises deterministic output and --check compares against it, so an en_US host can order slugs differently from CI and report drift that is not there. LC_ALL=C pins it. New cheap-tier section 2b resolves every ${CLAUDE_PLUGIN_ROOT}/ in every plugins/*/hooks/hooks.json to a file under plugins/

/, and fails closed if the walk finds nothing. It is resolved against the filesystem, not pattern-matched: voice's hooks.json uses the identical-looking hooks-handlers/ path and is CORRECT, because voice keeps its handler directly under the plugin root. Verified: cheap tier 1295 passed / 0 failed (1290 on main + 5 handler paths). Against main's pre-fix hooks.json the new section reports both redgate hooks as missing (1293/2). Two further mutations — an agent-compiler handler renamed away, and the walk pointed at an empty glob — each go red. Not verified here: the locale ordering difference. No en_US locale is installed in this container; LC_ALL=en_US.UTF-8 falls back to C and sorts identically, so the one-line fix rests on sort(1) semantics and #115's report of a red cheap tier on en_US hosts, not on a reproduction. The deep tier's path filter matches criteria-index.sh, but redgate ships no pier pack, so that check will report green because its leg is skipped, not because it ran. No SKILL.md, command, or skill references/ touched; demonstration discipline does not apply. No eval tier, job, or pack added, removed, or re-scoped, so docs/testing.md is unchanged. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01XHYtoYrtWhTGy59GC9uJGd --- evals/cheap/run.sh | 42 +++++++++++++++++++ plugins/redgate/hooks/hooks.json | 4 +- .../scripts/criteria-index.sh | 2 +- 3 files changed, 45 insertions(+), 3 deletions(-) diff --git a/evals/cheap/run.sh b/evals/cheap/run.sh index 19e7582e..847d8e7a 100755 --- a/evals/cheap/run.sh +++ b/evals/cheap/run.sh @@ -43,6 +43,48 @@ while IFS= read -r j; do ok "$j"; else bad "$j (invalid JSON)"; fi done < <(find . -name '*.json' -not -path './node_modules/*' -type f | sort) +# --- 2b. hooks.json handler paths resolve to real files --------------------- +# Every `${CLAUDE_PLUGIN_ROOT}/` in a plugin's hooks.json must be a file +# under plugins/

/, because ${CLAUDE_PLUGIN_ROOT} IS the plugin dir on an +# installed plugin. Nothing checked this, and redgate shipped with both handlers +# one directory too high (`hooks-handlers/` for files that live under +# `hooks/hooks-handlers/`): the PreToolUse write guard — the hook that stops a +# write reaching a ratified run's contract mid-round — silently never ran on any +# installed copy, and the cheap tier stayed green throughout. Found by the +# agentic framework in #115, fixed in its own PR so it does not wait on 76k +# lines. voice's identical-looking path is CORRECT (its handler really is at +# plugins/voice/hooks-handlers/), which is exactly why this is resolved against +# the filesystem rather than pattern-matched. +# Coupled: break any handler path, or make the walk find nothing, and this goes red. +group "hooks.json handler paths resolve under their plugin root" +hook_paths_seen=0 +while IFS=$'\t' read -r verdict plugin event rel; do + [ -n "$verdict" ] || continue + hook_paths_seen=$((hook_paths_seen+1)) + if [ "$verdict" = "OK" ]; then ok "$plugin hooks.json $event -> $rel" + else bad "$plugin hooks.json $event -> \${CLAUDE_PLUGIN_ROOT}/$rel does not exist under plugins/$plugin/ — that hook never fires on an installed plugin"; fi +done < <(python3 - "$REPO_ROOT" <<'PYH' +import glob, json, os, re, sys +root = sys.argv[1] +for f in sorted(glob.glob(os.path.join(root, "plugins", "*", "hooks", "hooks.json"))): + plugin = f.split(os.sep)[-3] + try: + d = json.load(open(f)) + except Exception: + continue # section 2 already fails invalid JSON + for event, entries in (d.get("hooks") or {}).items(): + for e in entries or []: + for h in e.get("hooks") or []: + for rel in re.findall(r"\$\{CLAUDE_PLUGIN_ROOT\}/([^\"'\s]+)", h.get("command", "")): + p = os.path.join(root, "plugins", plugin, rel) + print("\t".join(["OK" if os.path.isfile(p) else "BAD", plugin, event, rel])) +PYH +) +# A pass must mean paths were actually resolved, not that the walk found nothing. +if [ "$hook_paths_seen" -eq 0 ]; then + bad "hooks.json walk resolved ZERO handler paths — the check would be green without checking anything" +fi + # --- 3. Marketplace <-> plugin wiring -------------------------------------- group "marketplace wiring" python3 - "$REPO_ROOT" <<'PY' diff --git a/plugins/redgate/hooks/hooks.json b/plugins/redgate/hooks/hooks.json index b367731b..15134414 100644 --- a/plugins/redgate/hooks/hooks.json +++ b/plugins/redgate/hooks/hooks.json @@ -7,7 +7,7 @@ "hooks": [ { "type": "command", - "command": "bash \"${CLAUDE_PLUGIN_ROOT}/hooks-handlers/guard-redgate-paths.sh\"" + "command": "bash \"${CLAUDE_PLUGIN_ROOT}/hooks/hooks-handlers/guard-redgate-paths.sh\"" } ] } @@ -18,7 +18,7 @@ "hooks": [ { "type": "command", - "command": "bash \"${CLAUDE_PLUGIN_ROOT}/hooks-handlers/session-start.sh\"" + "command": "bash \"${CLAUDE_PLUGIN_ROOT}/hooks/hooks-handlers/session-start.sh\"" } ] } diff --git a/plugins/redgate/skills/criteria-contract/scripts/criteria-index.sh b/plugins/redgate/skills/criteria-contract/scripts/criteria-index.sh index c4bc361c..5d598dd6 100755 --- a/plugins/redgate/skills/criteria-contract/scripts/criteria-index.sh +++ b/plugins/redgate/skills/criteria-contract/scripts/criteria-index.sh @@ -57,7 +57,7 @@ emit() { printf '| run | # | status | layers | statement |\n' printf '|---|---|---|---|---|\n' local run slug crit - for run in $(ls -d "$RG"/*/ 2>/dev/null | sort); do + for run in $(ls -d "$RG"/*/ 2>/dev/null | LC_ALL=C sort); do [ -f "$run/manifest" ] || continue # not a run dir (or synthetic) [ -f "$run/CRITERIA.md" ] || continue slug="$(basename "$run")" From 2af3101b13b72cebcf294fdd9379b965bc47ab5a Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 15 Sep 2026 02:22:44 +0000 Subject: [PATCH 2/3] cheap tier 2b: contain handler paths inside the plugin; no-hooks root is nothing-to-check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two corrections to the section added in a9145b8, both found by running it against something other than the tree it was written on. 1. The counterfeit corpus's synthetic marketplace ships no hooks.json at all, and the section's "zero paths resolved" fail-closed turned that root red: the corpus's own calibration guard reported "baseline plugin is NOT green — corpus is miscalibrated, every rejection below is meaningless" and the required counterfeit tier failed on CI. Reproduced locally (24 passed / 1 failed) before touching anything. The section now counts hooks.json FILES separately from extracted PATHS: no files is a legitimate "nothing to check", the same PASS shape every other section uses for an absent surface; files present but zero paths extracted still fails closed, because that is the walk breaking, not the plugins being clean. 2. The resolver checked isfile() only, while the section's name promises containment. `${CLAUDE_PLUGIN_ROOT}/../shared/x.sh`, or a symlink out of the plugin, would report OK when the target exists even though the installed command resolves outside the plugin root. Copilot's finding on #136. Both paths are now realpath'd and the target must share the plugin dir as its commonpath — the same idiom the relative-link resolver further down already uses. Verified, in order: counterfeit harness on the fixed file: 25 passed / 0 failed, baseline green, all 18 counterfeits rejected by their expected gate (was 24 / 1). cheap tier on the repo: 1295 passed / 0 failed, unchanged. Five mutations against the final section, each restored after: main's pre-fix redgate hooks.json -> 2 FAIL (1293 / 2) agent-compiler handler renamed away -> 1 FAIL (1294 / 1) ../ escape to a REAL file in redgate -> 1 FAIL (1294 / 1) isfile alone said OK extraction regex broken, 3 files kept -> 1 FAIL "3 file(s) present but the walk extracted ZERO handler paths" glob pointed at a root with no hooks -> PASS "nothing to check" (1291 / 0) The regex mutation's first attempt did not apply (a sed pattern that missed the heredoc's escaping left the file untouched and the tier at 1295 / 0); that was a null test, not evidence, and is recorded here so the second, applied attempt is the one that counts. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01XHYtoYrtWhTGy59GC9uJGd --- evals/cheap/run.sh | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/evals/cheap/run.sh b/evals/cheap/run.sh index 847d8e7a..a81239d0 100755 --- a/evals/cheap/run.sh +++ b/evals/cheap/run.sh @@ -55,19 +55,24 @@ done < <(find . -name '*.json' -not -path './node_modules/*' -type f | sort) # lines. voice's identical-looking path is CORRECT (its handler really is at # plugins/voice/hooks-handlers/), which is exactly why this is resolved against # the filesystem rather than pattern-matched. -# Coupled: break any handler path, or make the walk find nothing, and this goes red. +# Coupled: break any handler path, or make the walk extract nothing from files +# that exist, and this goes red. A root with NO hooks.json at all (the +# counterfeit corpus's synthetic marketplace, for one) is legitimately "nothing +# to check", same as every other section — its first push turned that root red. group "hooks.json handler paths resolve under their plugin root" -hook_paths_seen=0 +hook_files_seen=0; hook_paths_seen=0 while IFS=$'\t' read -r verdict plugin event rel; do [ -n "$verdict" ] || continue + if [ "$verdict" = "FILE" ]; then hook_files_seen=$((hook_files_seen+1)); continue; fi hook_paths_seen=$((hook_paths_seen+1)) if [ "$verdict" = "OK" ]; then ok "$plugin hooks.json $event -> $rel" - else bad "$plugin hooks.json $event -> \${CLAUDE_PLUGIN_ROOT}/$rel does not exist under plugins/$plugin/ — that hook never fires on an installed plugin"; fi + else bad "$plugin hooks.json $event -> \${CLAUDE_PLUGIN_ROOT}/$rel is not a file INSIDE plugins/$plugin/ — that hook never fires on an installed plugin, or fires something outside it"; fi done < <(python3 - "$REPO_ROOT" <<'PYH' import glob, json, os, re, sys root = sys.argv[1] for f in sorted(glob.glob(os.path.join(root, "plugins", "*", "hooks", "hooks.json"))): plugin = f.split(os.sep)[-3] + print("\t".join(["FILE", plugin, "", ""])) try: d = json.load(open(f)) except Exception: @@ -76,13 +81,25 @@ for f in sorted(glob.glob(os.path.join(root, "plugins", "*", "hooks", "hooks.jso for e in entries or []: for h in e.get("hooks") or []: for rel in re.findall(r"\$\{CLAUDE_PLUGIN_ROOT\}/([^\"'\s]+)", h.get("command", "")): - p = os.path.join(root, "plugins", plugin, rel) - print("\t".join(["OK" if os.path.isfile(p) else "BAD", plugin, event, rel])) + # Containment, not just existence: canonicalize and require the + # target to sit INSIDE the plugin dir, as the link resolver does + # further down. `../shared/x.sh` or a symlink out of the plugin + # would otherwise pass isfile() while the installed command + # resolves outside ${CLAUDE_PLUGIN_ROOT}. (Copilot, PR #136) + pdir = os.path.realpath(os.path.join(root, "plugins", plugin)) + p = os.path.realpath(os.path.join(pdir, rel)) + inside = os.path.commonpath([pdir, p]) == pdir + print("\t".join(["OK" if (inside and os.path.isfile(p)) else "BAD", plugin, event, rel])) PYH ) -# A pass must mean paths were actually resolved, not that the walk found nothing. -if [ "$hook_paths_seen" -eq 0 ]; then - bad "hooks.json walk resolved ZERO handler paths — the check would be green without checking anything" +# A pass must mean paths were actually resolved from the files that exist — +# hooks.json present but zero ${CLAUDE_PLUGIN_ROOT} paths extracted means the +# walk broke, not that the plugins are clean. No files at all is a different, +# legitimate case. +if [ "$hook_files_seen" -eq 0 ]; then + ok "hooks.json: no plugin ships hooks in this root — nothing to check" +elif [ "$hook_paths_seen" -eq 0 ]; then + bad "hooks.json: $hook_files_seen file(s) present but the walk extracted ZERO handler paths — the check would be green without checking anything" fi # --- 3. Marketplace <-> plugin wiring -------------------------------------- From 1bf15a69fac00e2c2a546193fe87b853a5d27a0a Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 15 Sep 2026 02:41:02 +0000 Subject: [PATCH 3/3] cheap tier 2b: a scanner crash or malformed hook entry fails closed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex's finding on #136, confirmed by reproduction before touching anything: the handler-path scanner ran in process substitution, so its exit status was discarded. A manifest that is valid JSON but not the shape the walk expects — `"command": null` — raised TypeError, the remaining manifests were never scanned, and because an EARLIER manifest had already emitted records the zero-paths guard stayed quiet. On the unmodified file, nulling voice's command made the `voice hooks.json SessionStart` line simply vanish; section 2b reported no failure and the required tier exited 0 (1294 / 0), while stderr carried the traceback nobody reads. Nulling agent-compiler's instead went red only incidentally, via the zero-paths guard, and misreported "1 file(s)" when three exist — the scan had been silently truncated. Three layers now, each verified on its own: * Every shape the walk touches is checked, not assumed. Nonsense in a valid-JSON manifest emits a MALFORMED record naming the plugin and event instead of raising, and the walk continues to the next manifest. * The scanner writes to a temp file and its exit status is captured; a nonzero one is a recorded FAIL saying an unknown number of manifests were never checked. * The temp file itself fails closed if it cannot be created, and the "nothing to check" PASS moved inside the scanner-ran branch, so no silent-scanner path can produce that green line. Verified against the final file: null command, last manifest -> FAIL "MALFORMED hook entry (a hook has no string "command")" 1294 / 1, exit 1 null command, first manifest -> same FAIL, and redgate + voice still scanned after it 1294 / 1 crash on every manifest -> FAIL "scanner exited 1" + zero-paths FAIL crash on the LAST manifest -> FAIL "scanner exited 1" 1294 / 1 (the exact earlier-records shape that used to pass) "hooks" not an object / event not a list / entry's hooks not a list -> one MALFORMED FAIL each TMPDIR unwritable -> FAIL, and no "nothing to check" green The five prior mutations unchanged: main's pre-fix redgate (2 FAIL), handler renamed (1), ../ escape to a real file (1), regex broken (now reports "3 file(s)", not 1 — the walk no longer truncates), no-hooks root (PASS). cheap tier 1295 / 0 unchanged; counterfeit harness 25 / 0, baseline green. The pre-fix reproduction and the full mutation sweep were run by a delegated agent; the fixed-file cases above, both roots, and the hooks.json files being byte-identical to HEAD were re-verified independently before this commit. Survey, not fixed here: sections 10 (per-plugin safety-pack discovery) and 12 (install-smoke coverage) feed `while read` from the same `< <(python3 …)` shape and neither asserts the enumerated count against marketplace.json, so a scanner crash after N plugins would silently run only N packs and stay green. Section 10 is the one this repo's safety story rests on. Follow-up. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01XHYtoYrtWhTGy59GC9uJGd --- evals/cheap/run.sh | 73 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 60 insertions(+), 13 deletions(-) diff --git a/evals/cheap/run.sh b/evals/cheap/run.sh index a81239d0..27e4a5b8 100755 --- a/evals/cheap/run.sh +++ b/evals/cheap/run.sh @@ -60,16 +60,30 @@ done < <(find . -name '*.json' -not -path './node_modules/*' -type f | sort) # counterfeit corpus's synthetic marketplace, for one) is legitimately "nothing # to check", same as every other section — its first push turned that root red. group "hooks.json handler paths resolve under their plugin root" +# The scanner's exit status is CAPTURED, not discarded. It used to run in process +# substitution, where a crash is invisible: a manifest containing something the +# walk did not expect (`"command": null` — valid JSON, so section 2 is happy) +# raised, the remaining manifests were never scanned, and because an EARLIER +# manifest had already emitted records the zero-paths guard below stayed quiet. +# Section 2b then reported no failure over files it never read. (Codex, PR #136) +# Coupled: make the scanner exit nonzero — crash it, or delete its interpreter — +# and this goes red instead of green-by-silence. hook_files_seen=0; hook_paths_seen=0 -while IFS=$'\t' read -r verdict plugin event rel; do - [ -n "$verdict" ] || continue - if [ "$verdict" = "FILE" ]; then hook_files_seen=$((hook_files_seen+1)); continue; fi - hook_paths_seen=$((hook_paths_seen+1)) - if [ "$verdict" = "OK" ]; then ok "$plugin hooks.json $event -> $rel" - else bad "$plugin hooks.json $event -> \${CLAUDE_PLUGIN_ROOT}/$rel is not a file INSIDE plugins/$plugin/ — that hook never fires on an installed plugin, or fires something outside it"; fi -done < <(python3 - "$REPO_ROOT" <<'PYH' +hook_scan_out="$(mktemp "${TMPDIR:-/tmp}/cheap-hooks-scan.XXXXXX" 2>/dev/null || true)" +if [ -z "$hook_scan_out" ] || [ ! -w "$hook_scan_out" ]; then + bad "hooks.json: could not create the handler-path scanner's output file — section 2b cannot run, and a section that cannot run is not a section that passed" +else +python3 - "$REPO_ROOT" >"$hook_scan_out" <<'PYH' import glob, json, os, re, sys root = sys.argv[1] + +def emit(verdict, plugin, event, rel): + # Fields are tab-separated and read back by a bash `read`; tab is IFS + # WHITESPACE there, so runs of tabs collapse and an empty field would shift + # every later one. Never emit an empty event/rel on a record the shell + # destructures past field 2. + print("\t".join([verdict, plugin, event, rel])) + for f in sorted(glob.glob(os.path.join(root, "plugins", "*", "hooks", "hooks.json"))): plugin = f.split(os.sep)[-3] print("\t".join(["FILE", plugin, "", ""])) @@ -77,10 +91,27 @@ for f in sorted(glob.glob(os.path.join(root, "plugins", "*", "hooks", "hooks.jso d = json.load(open(f)) except Exception: continue # section 2 already fails invalid JSON - for event, entries in (d.get("hooks") or {}).items(): - for e in entries or []: + # Every shape below is CHECKED rather than assumed. A manifest can be valid + # JSON and still be nonsense to this walk; the answer to nonsense is a + # recorded BAD naming the plugin and event, never a traceback. + hooks = (d.get("hooks") if isinstance(d, dict) else None) or {} + if not isinstance(hooks, dict): + emit("MALFORMED", plugin, "", '"hooks" is not an object') + continue + for event, entries in hooks.items(): + if not isinstance(entries, list): + emit("MALFORMED", plugin, event or "", "the event's value is not a list of entries") + continue + for e in entries: + if not isinstance(e, dict) or not isinstance(e.get("hooks") or [], list): + emit("MALFORMED", plugin, event or "", "an entry is not an object carrying a list of hooks") + continue for h in e.get("hooks") or []: - for rel in re.findall(r"\$\{CLAUDE_PLUGIN_ROOT\}/([^\"'\s]+)", h.get("command", "")): + cmd = h.get("command") if isinstance(h, dict) else None + if not isinstance(cmd, str): + emit("MALFORMED", plugin, event or "", 'a hook has no string "command"') + continue + for rel in re.findall(r"\$\{CLAUDE_PLUGIN_ROOT\}/([^\"'\s]+)", cmd): # Containment, not just existence: canonicalize and require the # target to sit INSIDE the plugin dir, as the link resolver does # further down. `../shared/x.sh` or a symlink out of the plugin @@ -89,18 +120,34 @@ for f in sorted(glob.glob(os.path.join(root, "plugins", "*", "hooks", "hooks.jso pdir = os.path.realpath(os.path.join(root, "plugins", plugin)) p = os.path.realpath(os.path.join(pdir, rel)) inside = os.path.commonpath([pdir, p]) == pdir - print("\t".join(["OK" if (inside and os.path.isfile(p)) else "BAD", plugin, event, rel])) + emit("OK" if (inside and os.path.isfile(p)) else "BAD", plugin, event, rel) PYH -) +hook_scan_rc=$? +while IFS=$'\t' read -r verdict plugin event rel; do + [ -n "$verdict" ] || continue + if [ "$verdict" = "FILE" ]; then hook_files_seen=$((hook_files_seen+1)); continue; fi + # A MALFORMED record counts here too: it still proves the walk reached this + # manifest's entries, which is what the zero-paths guard below is asking. + hook_paths_seen=$((hook_paths_seen+1)) + if [ "$verdict" = "OK" ]; then ok "$plugin hooks.json $event -> $rel" + elif [ "$verdict" = "MALFORMED" ]; then bad "$plugin hooks.json $event — MALFORMED hook entry ($rel): valid JSON, but no handler path can be resolved from it, so this hook is unverifiable" + else bad "$plugin hooks.json $event -> \${CLAUDE_PLUGIN_ROOT}/$rel is not a file INSIDE plugins/$plugin/ — that hook never fires on an installed plugin, or fires something outside it"; fi +done < "$hook_scan_out" +rm -f "$hook_scan_out" +if [ "$hook_scan_rc" -ne 0 ]; then + bad "hooks.json: the handler-path scanner exited $hook_scan_rc — it died partway through the walk, so an unknown number of manifests were never checked at all" +fi # A pass must mean paths were actually resolved from the files that exist — # hooks.json present but zero ${CLAUDE_PLUGIN_ROOT} paths extracted means the # walk broke, not that the plugins are clean. No files at all is a different, -# legitimate case. +# legitimate case. It lives INSIDE the scanner-ran branch: "no files" may only +# be concluded from a walk that actually happened. if [ "$hook_files_seen" -eq 0 ]; then ok "hooks.json: no plugin ships hooks in this root — nothing to check" elif [ "$hook_paths_seen" -eq 0 ]; then bad "hooks.json: $hook_files_seen file(s) present but the walk extracted ZERO handler paths — the check would be green without checking anything" fi +fi # --- 3. Marketplace <-> plugin wiring -------------------------------------- group "marketplace wiring"