diff --git a/AGENTS.md b/AGENTS.md index 22092f2..265b8c9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,26 +4,25 @@ This repository is the `entwining-mods` Claude Code marketplace. Each mod is a p ## Layout -- Development tooling lives only at the repository root: a `package.json` with a lockfile inside a plugin folder makes Claude Code install dependencies on every user's machine. -- An install copies only the plugin folder, so anything a mod needs at run time or a user needs to read stays inside it. -- `plugins//.claude-plugin/types/` is written by the engine when it loads the mod and holds the machine's MCP tool list; it stays untracked. -- A mod's README is written for the people who install it; what only someone changing the mod needs goes in its AGENTS.md. +Development tooling lives only at the repository root: a `package.json` with a lockfile inside a plugin folder makes Claude Code install dependencies on every user's machine. An install copies only the plugin folder, so anything a mod needs at run time or a user needs to read stays inside it. `plugins//.claude-plugin/types/` is written by the engine when it loads the mod and holds the machine's MCP tool list; it stays untracked. + +A mod's README is written for the people who install it; what only someone changing the mod needs goes in its AGENTS.md. ## Code -- Use what the mods API, Bun, or the standard library already provides before writing a helper, and add no type, wrapper, or abstraction whose only purpose is to look tidy. -- A guard, fallback, or `catch` handles a failure a real session meets or a documented engine behavior, and comes with the test the Checks section asks for; otherwise leave it out. A case only a constructed input reaches goes in the README's known gaps instead, unless it breaks a promise the README makes. -- A file stays within 200 lines of code (`max-lines`, which skips blank lines and comments, so blank lines that set blocks apart cost nothing; a script's `*.generated.ts` output is not linted). Split it along what its parts do rather than to fit the count; a mod's tests follow the same split, with shared fixtures in `tests/` files not named `*.test.ts`. -- Comments state the reason the code cannot show, not what it does or what it replaced. -- `.oxlintrc.json` enforces the cases of these that a lint rule can catch. +Use what the mods API, Bun, or the standard library already provides before writing a helper, and add no type, wrapper, or abstraction whose only purpose is to look tidy. + +A guard, fallback, or `catch` handles a failure a real session meets or a documented engine behavior, and comes with the test the Checks section asks for; otherwise leave it out. A case only a constructed input reaches goes in the README's known gaps instead, unless it breaks a promise the README makes. + +A file stays within 200 lines of code (`max-lines`, which skips blank lines and comments, so blank lines that set blocks apart cost nothing; a script's `*.generated.ts` output is not linted). Split it along what its parts do rather than to fit the count; a mod's tests follow the same split, with shared fixtures in `tests/` files not named `*.test.ts`. + +Comments state the reason the code cannot show, not what it does or what it replaced. `.oxlintrc.json` enforces the cases of these that a lint rule can catch. ## Checks `bun run check` runs what CI runs, apart from CI regenerating preallow's gitleaks rules and failing when they differ from the committed file, so a Renovate update of the gitleaks release `scripts/gitleaks.ts` pins waits for someone to run `bun run gitleaks` on its branch. It runs `oxfmt --check`, `oxlint`, `claude plugin validate` for the marketplace and each plugin, `claude plugin test`, `bun run test:sources`, which fails when preallow's output scan finds a credential in one of the plugin's own files, since plugin tests cannot read files, and `bun run typecheck:core`, which checks every mod against the plugin API's core declarations that `scripts/core-types.ts` takes from the pinned engine's binary. That check is what keeps an engine update, which Renovate merges once CI passes, from adding a field to an answer a mod reads, such as preallow's `OWNS_ASK`, unnoticed. Tool records are declared only in the types an interactive session writes, so `tool-results.d.ts` types the ones a mod reads, as loosely as it reads them; add a record there when a mod starts reading one. `bun run typecheck` checks a mod against all the engine's declarations (see the README); run it whenever you load a mod from the checkout. -- Tests use synthetic inputs only. This repository is public: no transcripts, logs, real repository names, hostnames, or credentials. Build credential-shaped test strings at run time, such as `` `ghp_${'x'.repeat(36)}` ``, so the source does not look like a leaked key. -- Every safeguard has a test that fails when the safeguard is removed. After adding or changing one, remove it, watch a test fail, and restore it. -- Each test covers a behavior no other test fails on; merge or delete one that fails on the same bug as another, and delete one whose behavior the code no longer has. +Tests use synthetic inputs only. This repository is public: no transcripts, logs, real repository names, hostnames, or credentials. Build credential-shaped test strings at run time, such as `` `ghp_${'x'.repeat(36)}` ``, so the source does not look like a leaked key. Every safeguard has a test that fails when the safeguard is removed. After adding or changing one, remove it, watch a test fail, and restore it. Each test covers a behavior no other test fails on; merge or delete one that fails on the same bug as another, and delete one whose behavior the code no longer has. ## Commits and releases diff --git a/README.md b/README.md index 6c289f7..443ebfb 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,9 @@ Claude Code plugins built on function hooks, published as the `entwining-mods` marketplace. -| Mod | What it does | -| ---------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------- | -| [preallow](plugins/preallow) | In auto mode, a model judge runs the steps of your task without the classifier, leaving irreversible effects you did not explicitly ask for to it | +| Mod | What it does | +| ---------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [preallow](plugins/preallow) | In auto mode, a model judge allows the steps of your task without the classifier, leaving the rest to it, such as a leak or an irreversible effect you did not explicitly ask for | ## Install diff --git a/plugins/preallow/.claude-plugin/plugin.json b/plugins/preallow/.claude-plugin/plugin.json index 1e2a1e5..164a3ae 100644 --- a/plugins/preallow/.claude-plugin/plugin.json +++ b/plugins/preallow/.claude-plugin/plugin.json @@ -11,7 +11,7 @@ "model": { "type": "string", "title": "Judge model", - "description": "The model the judge of every call runs on, passed to the model client as you write it: an alias or a full model id. Leave empty for the plugin's default pair, haiku with a second judge on opus. If you set it and leave the second judge's model empty, there is no second judge." + "description": "The model the judge of every call runs on, passed to the model client as you write it: an alias or a full model id. Leave empty for the plugin's default pair, haiku with a second judge on opus. If you set it and leave Escalation model empty, there is no second judge." }, "effort": { "type": "string", @@ -21,7 +21,7 @@ "escalation_model": { "type": "string", "title": "Escalation model", - "description": "The model of a second judge for every call the first judge did not allow, failures included; it judges the call again from the start, and its verdict is final. Passed as you write it. Leave empty for opus when the judge model is empty too, or for no second judge when you set the judge model." + "description": "The model of a second judge for every call the first judge did not allow, failures included; it judges the call again from the start, and its verdict is final. Passed as you write it. Leave empty for opus when Judge model is empty too, or for no second judge when you set Judge model." }, "escalation_effort": { "type": "string", diff --git a/plugins/preallow/AGENTS.md b/plugins/preallow/AGENTS.md index c46058b..b7b7a8a 100644 --- a/plugins/preallow/AGENTS.md +++ b/plugins/preallow/AGENTS.md @@ -2,13 +2,22 @@ The README states what the plugin allows and what it leaves to native approval; keep it in step with the code. -- The plugin only loosens. `tool.check` turns the core's `ask` into `allow` or returns the core's answer unchanged; it never touches an `allow`, a `deny`, or an ask whose result names a decider other than the classifier (`OWNS_ASK` in `hooks/register.ts`), or whose input names a host as `approver`. Preserve this ownership boundary when the engine adds metadata. An unknown field leaves the ask to the judge, since sending every ask with an unknown field to the classifier would bring back the false denials the plugin exists to remove. A permission hook that throws or overruns its budget is skipped and the core's answer stands. -- A failed output scan withholds the output of a call the plugin allowed; the call has already run, so recovery must not execute it again. Every pattern in `hooks/withhold.ts` stays linear in the output's length, and `scripts/gitleaks.ts` strips the name gitleaks matches before a value under a keyword, which backtracks for seconds over a crafted megabyte, so a scan ends within the budget instead of withholding large outputs; the test that scans every rule's keyword fails when it comes back. A pattern needs a key's body, not only the words that name its format, and `scripts/gitleaks.ts` appends a token body to a gitleaks rule that is only a literal prefix, since the plugin's own files name every format; `bun run test:sources` fails when reading one of them would be withheld. -- `hooks/gitleaks.generated.ts` is written by `bun run gitleaks` (`scripts/gitleaks.ts`, which pins the gitleaks release and rewrites its Go regexes for JavaScript); change the script and regenerate instead of editing the file. -- One judge for every tool it is asked about: no per-tool or per-command policy branches or allowlists in its policy or around it, and the policy text names no project, command, or wording taken from real data. A tool whose verdict the engine keeps whatever a hook answers (`RESERVED_TOOLS` in `hooks/register.ts`) never reaches the judge, since a judge call there spends seconds on an allow the engine rejects or ignores; add a tool there only for that engine behavior, as a `rejected` log line or the engine's documentation shows it. A Bash, PowerShell, or command-monitor call requesting a non-empty `allowed_domains` list also stays with the classifier, whose approval registers the network grants. Nor does a call whose input asks to run in the background, since its output arrives where the withholding scan cannot see it; a tool whose output arrives later without such a flag (`Monitor`) is a known gap in the README until the engine signals it. -- The judge chooses what to check; `hooks/facts.ts`, with `hooks/paths.ts`, `hooks/refs.ts`, and `hooks/git.ts`, answers from a fixed set of queries that only read, stay offline, and carry a time limit, and takes the judge's strings only as absolute paths or ref names that cannot pose as options. A new query keeps all of that, with a test that fails without it. -- Only a reply that ends with a JSON object answering `allow` allows: text may come before it, and nothing but a closing code fence after it, and no conflicting verdict values may appear anywhere in the reply. One answering `check` with one to four well-formed items gets one more round with the facts; anything else, including no reply, is `review`. Facts that cannot be read, and a second `check`, are `review` too. -- The user's authority comes from `prompt.submit` with a `composer` or `bridge` origin and from the person's answer in an `AskUserQuestion` result, never an answer the dialog gave while the person was away. The answer reaches the judge as the engine gives it, with the questions from the result, which are what the person was shown: the plugin does not split or match it against the options, since the engine's answer is display text a label can also contain. A chosen option authorizes only the action and target it states, as `JUDGE_SYSTEM` says. Never from assistant text, other tools' results, files, skill bodies, or other sessions. Do not widen what counts as the user's words. They are kept in `$.store` under the session id, so a resume keeps them and `/clear`, which goes on under a new id, starts without them. A conversation is kept bounded by its JSON size (`addTurn` in `hooks/messages.ts`), so 50 fit the store whatever they hold, and the judge reads all of it for every call, since leaving out a limit lets through a call the user ruled out; each write replaces it whole: a judge reading meanwhile sees it before or after the message, never without it; never rebuild them from `$.session.messages`, which does not say where a message came from. The judge prompt is one JSON document, so text inside a message cannot pose as another field; it is sent as two blocks, the kept conversation first and marked for the prompt cache, then the call's fields (`judgePrompt` in `hooks/judge.ts`), so every call until the user writes again opens the same way and reads the conversation, and on gateway routes the policy, from the cache. -- The `model`, `effort`, `escalation_model`, and `escalation_effort` settings reach `$.model.complete` as the user wrote them: no aliases, option lists, or validation. -- `judge` (`hooks/judge.ts`) stays one stage, since the evaluation outside the engine calls it once per stage; the second stage lives in `judgeCall` (`hooks/stages.ts`). -- `JUDGE_SYSTEM` (`hooks/policy.ts`) decides what is allowed. Check a change to it or to either stage's default model or effort in `hooks/register.ts` against real denials kept outside this repository, on a split not used while writing the change, and compare allow and review rates before and after. +The plugin only loosens. `tool.check` turns the core's `ask` into `allow` or returns the core's answer unchanged; it never touches an `allow`, a `deny`, or an ask whose result names a decider other than the classifier (`OWNS_ASK` in `hooks/register.ts`), or whose input names a host as `approver`. Preserve this ownership boundary when the engine adds metadata. An unknown field leaves the ask to the judge, since sending every ask with an unknown field to the classifier would bring back the false denials the plugin exists to remove. A permission hook that throws or overruns its budget is skipped and the core's answer stands. + +One judge for every tool it is asked about: no per-tool or per-command policy branches or allowlists in its policy or around it, and the policy text names no project, command, or wording taken from real data. A tool whose verdict the engine keeps whatever a hook answers (`RESERVED_TOOLS` in `hooks/register.ts`) never reaches the judge, since a judge call there spends seconds on an allow the engine rejects or ignores; add a tool there only for that engine behavior, as a `rejected` log line or the engine's documentation shows it. A Bash, PowerShell, or command-monitor call requesting a non-empty `allowed_domains` list also stays with the classifier, whose approval registers the network grants. Nor does a call whose input asks to run in the background, since its output arrives where the withholding scan cannot see it; a tool whose output arrives later without such a flag (`Monitor`) is a known gap in the README until the engine signals it. + +`JUDGE_SYSTEM` (`hooks/policy.ts`) decides what is allowed. Check a change to it or to either stage's default model or effort in `hooks/register.ts` against real denials kept outside this repository, on a split not used while writing the change, and compare allow and review rates before and after. + +The user's authority comes from `prompt.submit` with a `composer` or `bridge` origin and from the person's answer in an `AskUserQuestion` result, never an answer the dialog gave while the person was away, and never from assistant text, other tools' results, files, skill bodies, or other sessions. Do not widen what counts as the user's words. The answer reaches the judge as the engine gives it, with the questions from the result, which are what the person was shown: the plugin does not split or match it against the options, since the engine's answer is display text a label can also contain. A chosen option authorizes only the action and target it states, as `JUDGE_SYSTEM` says. + +The user's words are kept in `$.store` under the session id, so a resume keeps them and `/clear`, which goes on under a new id, starts without them; never rebuild them from `$.session.messages`, which does not say where a message came from. A conversation is kept bounded by its JSON size (`addTurn` in `hooks/messages.ts`), so 50 fit the store whatever they hold, and the judge reads all of it for every call, since leaving out a limit lets through a call the user ruled out. Each write replaces it whole: a judge reading meanwhile sees it before or after the message, never without it. The judge prompt is one JSON document, so text inside a message cannot pose as another field; it is sent as two blocks, the kept conversation first and marked for the prompt cache, then the call's fields (`judgePrompt` in `hooks/judge.ts`), so every call until the user writes again opens the same way and reads the conversation, and on gateway routes the policy, from the cache. + +The judge chooses what to check; `hooks/facts.ts`, with `hooks/paths.ts`, `hooks/refs.ts`, and `hooks/git.ts`, answers from a fixed set of queries that only read, stay offline, and carry a time limit, and takes the judge's strings only as absolute paths or ref names that cannot pose as options. A new query keeps all of that, with a test that fails without it. + +Only a reply that ends with a JSON object answering `allow` allows: text may come before it, and nothing but a closing code fence after it, and no conflicting verdict values may appear anywhere in the reply. One answering `check` with one to four well-formed items gets one more round with the facts; anything else, including no reply, is `review`. Facts that cannot be read, and a second `check`, are `review` too. + +`judge` (`hooks/judge.ts`) stays one stage, since the evaluation outside the engine calls it once per stage; the second stage lives in `judgeCall` (`hooks/stages.ts`). The `model`, `effort`, `escalation_model`, and `escalation_effort` settings reach `$.model.complete` as the user wrote them: no aliases, option lists, or validation. + +A failed output scan withholds the output of a call the plugin allowed; the call has already run, so recovery must not execute it again. Every pattern in `hooks/withhold.ts` stays linear in the output's length, and `scripts/gitleaks.ts` strips the name gitleaks matches before a value under a keyword, which backtracks for seconds over a crafted megabyte, so a scan ends within the budget instead of withholding large outputs; the test that scans every rule's keyword fails when it comes back. A pattern needs a key's body, not only the words that name its format, and `scripts/gitleaks.ts` appends a token body to a gitleaks rule that is only a literal prefix, since the plugin's own files name every format; `bun run test:sources` fails when reading one of them would be withheld. + +`hooks/gitleaks.generated.ts` is written by `bun run gitleaks` (`scripts/gitleaks.ts`, which pins the gitleaks release and rewrites its Go regexes for JavaScript); change the script and regenerate instead of editing the file. diff --git a/plugins/preallow/CHANGELOG.md b/plugins/preallow/CHANGELOG.md index 7ac0c95..9430ade 100644 --- a/plugins/preallow/CHANGELOG.md +++ b/plugins/preallow/CHANGELOG.md @@ -4,9 +4,9 @@ - The judge runs on `haiku` at `high` effort instead of `opus` at `medium`, and a second judge on `opus` at `medium` judges again, from the start, every call the first did not allow, including one it gave no verdict on; the second judge's verdict is final. The new `escalation_model` and `escalation_effort` settings choose the second judge; if you set `model` and leave `escalation_model` empty, there is none. Each `judged` line in the log keeps the first judge's fields and gives the second judge's in `escalation`. - The judge's instructions are rewritten as decisions taken in order: leaks first, then whether the call serves your task and respects what you ruled out, then work on your own repositories, then whether each effect can be undone, and last whether your words explicitly cover the irreversible ones. They describe each field the judge reads and each fact a check returns, including what a fact does not establish, such as file times saying nothing about who made a file. The wording was chosen by evaluating it against real calls. -- A judge model call that comes back without a reply, or with an API error that may clear on its own, such as an overloaded server, is made once more two seconds later instead of sending the tool call to the classifier. Each judge's 90 seconds now cover all its model calls for one tool call, retries and the call after a check included. +- A judge model call that comes back without a reply, or with an API error that may clear on its own, such as an overloaded server, is made once more two seconds later, while the judge has time, instead of the judge giving no verdict. Each judge's 90 seconds now cover all its model calls for one tool call, retries and the call after a check included. - A call auto mode denied, made again with the same input before you write again, goes to the classifier instead of the judge, which could allow it on the second try. The log records it as `repeated`. -- The judge reads all your kept messages, up to 64,000 characters, for every call, instead of up to 24,000 characters chosen for each call by wording and by the words they share with it, which could leave out a prohibition phrased without the words selection looked for. Your messages open the judge's prompt and are marked for the prompt cache with its instructions, so calls until you write again read both from the cache, also on Bedrock, Vertex, Foundry, or behind a gateway. +- The judge reads all your kept messages, up to 64,000 characters as JSON, for every call, instead of up to 24,000 characters chosen for each call by wording and by the words they share with it, which could leave out a prohibition phrased without the words selection looked for. Your messages open the judge's prompt and are marked for the prompt cache with its instructions, so calls until you write again can read both from the cache, also on Bedrock, Vertex, Foundry, or behind a gateway. - When the judge checks a ref with commits no other ref holds, it is also told how many of them have an equivalent patch in the branch the ref follows or in a remote's default branch, as when a pull request was merged by rebase, so deleting a branch whose work is already merged need not go to the classifier. - Text that names a private key's format without the key, such as the header of a PEM or PuTTY key file, an age key's prefix, or the fixed start of a short-lived Amazon Bedrock key, is no longer withheld; the key's body has to be there. Reading the plugin's own source was withheld for this. - The gitleaks rules for Slack webhook and Sidekiq URLs match a dot in their hostnames only where the URL has one, not any character. diff --git a/plugins/preallow/README.md b/plugins/preallow/README.md index 42d6bc2..6ad2e7c 100644 --- a/plugins/preallow/README.md +++ b/plugins/preallow/README.md @@ -2,6 +2,20 @@ A Claude Code plugin for auto mode, requiring Claude Code 2.1.308 or later. When the core permission check would hand a tool call to the auto-mode classifier, preallow asks a model judge first, and runs the call without the classifier when the judge allows it. +The plugin only loosens: it never changes an `allow` or a `deny`, and disabling it restores Claude Code's native approval flow. [What the judge allows](#what-the-judge-allows) follows one policy for every tool, and [some calls never reach it](#what-it-leaves-to-native-approval). + +## Install + +```sh +claude plugin install preallow --marketplace Entwining/mods +``` + +This installs for your user. To pick a scope interactively, run `/plugin install preallow --marketplace Entwining/mods` in a Claude Code session. + +Claude Code does not update plugins from third-party marketplaces by default. In `/plugin`, select `entwining-mods` on the Marketplaces tab and choose **Enable auto-update**, or run `claude plugin update preallow@entwining-mods` manually. Start a new session after installing or updating. + +## What the judge allows + The judge applies one policy to every tool, Bash, file edits, subagents, messages, and MCP tools alike: giving the agent a task authorizes the steps it takes toward that task, whether or not you named them. It sends a call on to the classifier only when: - **the call would leak information**: publish or send your personal information, your company's information, credentials or tokens to people or systems that should not have it, or print a secret into the conversation. This overrides everything below, including your own instructions. @@ -11,79 +25,100 @@ The judge applies one policy to every tool, Bash, file edits, subagents, message - **the call does what you told the agent not to do** in this conversation, unless a later message of yours lifts that, even when the step is reversible. A prohibition covers what it names: asking for a value without showing it to you keeps it out of the agent's reply, not out of a tool result the task needs. - **the judge cannot tell** whether the effect can be undone, or whether your task covers it. -Everything else is allowed, including commands outside the sandbox, loosening permission or sandbox settings as a step of the task, and retries. The plugin only loosens: it never changes an `allow` or a `deny`, and disabling it restores Claude Code's native approval flow. +Everything else is allowed, including commands outside the sandbox, loosening permission or sandbox settings as a step of the task, and retries. + +## What it leaves to native approval + +- Any mode other than `auto`, and a subagent's calls until the subagent's mode has been reported. +- An `ask` from a settings rule, from a hook in your settings or another plugin, from a setting's own check (the read block, `permissions.blockReadsOutsideWorkingDirectories`, which asks you rather than the classifier), from an organization's ceiling for the tool, or one put to a host that answers the session's asks. +- A call run in the background (`run_in_background`), such as a dev server or a watcher: its output reaches the model after the call returns, where the plugin cannot scan it. +- A Bash, PowerShell, or command-monitor call with a non-empty `allowed_domains` list: the classifier reviews the command and registers approved network grants; a hook allow approves the command without granting the hosts. +- A call whose output the plugin withheld, or whose allow the engine rejected, when the same tool runs again with the same input. +- A call auto mode denied after the judge left it to review, when the agent makes it again with the same input, in the same directory and, for a subagent, with the same brief, before you write again. Judging it again could only repeat the review or undo the denial with nothing new from you; once you have written, the call is judged again with your new words. +- A tool whose verdict the engine keeps whatever a hook answers: a subagent's report to its parent (`SubagentHandback`), which only the classifier may allow, and a question to you or a plan for your approval (`AskUserQuestion`, `ExitPlanMode`), which wait for you. The judge is not asked, since its allow would be rejected or ignored. + +## Settings + +Set them in `/config` or when you install; an emptied field counts as unset. -When whether a call can be undone turns on the state of files or git refs, the judge first asks to check them, naming up to four paths or refs. The plugin answers from a fixed set of read-only queries: for a path, whether it exists, how many of its files changed before the session started, and, inside a git work tree, how many files are tracked and the uncommitted, untracked, and ignored files under it; for a ref, its commit, how many commits no other ref holds and how many of those have an equivalent patch (the same change, whitespace aside) in the branch the ref follows or in a remote's default branch, which refs contain it, and when the remote was last fetched. Patches are compared one commit at a time, so commits squashed into one when merged, or rebased with conflicts resolved, find no equivalent even when their work is there, and merge commits are not compared; an equivalent patch also does not show that the change is still there, or that the commits' messages and authors are kept anywhere. The git commands run without optional locks, an fsmonitor, or any transport, five seconds each. The judge then decides once more with what they found; when a check cannot run or the judge is still unsure, it does not allow the call. +- `model`: the model the first judge runs on, an alias or a full model id. Default `haiku`. +- `effort`: how hard the first judge thinks. Default `high`. +- `escalation_model`: the model of a second judge, for every call the first did not allow, including those it gave no verdict on, such as when its model call failed. Default `opus` when `model` is unset too; if you set `model` and leave this empty, there is no second judge. +- `escalation_effort`: how hard the second judge thinks. Default `medium`. Set alone, it does not add a second judge. + +The second judge reads the same call with the same instructions and starts over: it asks for checks of its own and does not see the first judge's verdict, checks, or facts. Its verdict is final, so a call goes to the classifier only when both judges leave it there. + +The plugin passes what you write to the model client unchanged; if the client refuses a value, the judge it belongs to gives no verdict on any call, and the `failure` in each `judged` line of the log says why. + +The defaults were chosen by evaluating the policy on real calls auto mode had sent to the classifier, labeled by hand, on a split used while choosing them and on one held out until the end. On both, the two judges allowed no call the policy rules out, apart from [reads whose output held a secret](#known-gaps), the same as `opus` at `medium` alone. Compared with it, they sent fewer of the calls they should allow to the classifier on the first split and one more on the held-out one, at about a seventh of the cost. Other models were not evaluated. ## What the judge reads -- The tool's name and its whole input, since the call runs with all of it. A call whose input passes 100,000 characters goes to the classifier without asking the judge. -- Your own messages in this conversation: what you typed at the terminal or sent through Remote Control, including slash command lines, and your answers to the agent's questions (`AskUserQuestion`): the questions and options as you were shown them, the options you chose, your notes, and anything you typed instead. A chosen option authorizes the action on the target it states, nothing wider; a question that resolved while you were away holds no answer of yours. Task notifications, other sessions' messages, channel relays, and the agent's own text never count as your words. Your messages are kept per conversation in the plugin's store, up to 64,000 characters (the first message and the newest that fit, each cut to its head and tail past 16,000), so a conversation you resume still has them, and `/clear` starts with none; the store keeps the 50 conversations you wrote in most recently. The judge reads all your kept messages for every call, with a note where the store let messages go. -- For a subagent's call, the brief its parent agent gave it, up to 8,000 characters. The brief carries your authorization only where it quotes you or restates your task. +- The tool's name and its whole input, since the call runs with all of it. A call whose input passes 100,000 characters as JSON goes to the classifier without asking either judge. +- Your own messages in this conversation: what you typed at the terminal or sent through Remote Control, including slash command lines, and your answers to the agent's questions (`AskUserQuestion`): the questions and options as you were shown them, the options you chose, your notes, and anything you typed instead. A chosen option authorizes the action on the target it states, nothing wider; a question that resolved while you were away holds no answer of yours. Task notifications, other sessions' messages, channel relays, and the agent's own text never count as your words. +- For a subagent's call, the brief its parent agent gave it, cut to its head and tail past 8,000 characters. Your words quoted in it count as yours; the parent's restatement of your task shows what the task is, but cannot supply the explicit words an irreversible effect needs or lift a prohibition of yours. - The directory the call's loop runs in, and what the checks it asked for found. -The judge runs through your session's own model client, one call per tool call it is asked about and one more after a check; with the default settings, a second judge judges again every call the first did not allow (see [Settings](#settings)). A model call that comes back without a reply, or with an API error Claude Code says may clear on its own (overload, a rate limit, or a server error), is made once more two seconds later; one cut short or refused for what it asked is not. Its instructions, the same for every call, and your kept messages, the same until you write again, open its prompt and are marked for the prompt cache, so a call soon after another, and the call after a check, can read them from the cache instead of paying for them in full. The verdict is the JSON object that ends the judge's reply; a call the judge does not answer within 90 seconds, counting every model call it makes for that tool call, a reply that does not end with a verdict or that states different verdicts along the way, and anything other than `allow` are not allowed, and go to the second judge if there is one, with 90 seconds of its own, or else to the classifier. +Your messages are kept per conversation in the plugin's store, so a conversation you resume still has them, and `/clear` starts with none; the store keeps the 50 conversations you wrote in most recently. A conversation keeps your first message and the newest that fit 64,000 characters, each message cut to its head and tail past 16,000. Both limits count the messages as JSON, in which quotes, backslashes, and control characters take more than one character. The judge reads all your kept messages for every call, with a note where the store let messages go. -## Settings +## How a judge decides -- `model`: the model the judge of every call runs on, an alias or a full model id. Default `haiku`. -- `effort`: how hard that judge thinks. Default `high`. -- `escalation_model`: the model of a second judge, for the calls the first did not allow: those it answered `review`, was still unsure of after a check, or could not get the facts for, and those it gave no verdict on, such as when its model call failed. Default `opus` when `model` is unset too; if you set `model` and leave this empty, there is no second judge. -- `escalation_effort`: how hard the second judge thinks. Default `medium`. Set alone, it does not add a second judge. +When whether a call can be undone turns on the state of files or git refs, a judge first asks to check them, naming up to four paths or refs, and then decides once more with what they found. The plugin answers from a fixed set of read-only, offline queries: -The second judge reads the same call with the same instructions and starts over: it asks for checks of its own and does not see the first judge's verdict, checks, or facts. Its verdict is final, so a call goes to the classifier only when both judges leave it there. This is not the round after a check, in which one judge decides once more with the facts it asked for; each judge can take that round. +- For a path: whether it exists, its kind, whether it is a symbolic link, and when it last changed; for a directory that is not a link, how many of its files changed before the session started, up to a bound the answer says it reached; and inside a git work tree, how many files are tracked and the uncommitted, untracked, and ignored files under it. A file's time does not show who made it or whether a copy exists: copying or extracting a file can keep an older time. +- For a ref: its commit, which refs contain it, how many commits no other ref holds, and how many of those have an equivalent patch (the same change, whitespace aside) in the branch the ref follows or in a remote's default branch; for a remote-tracking ref, also when the repository last fetched, from any of its worktrees, which need not be the fetch that updated that ref. -Set them in `/config` or when you install. The plugin passes what you write to the model client unchanged; if the client refuses a value, the judge it belongs to gives no verdict on any call, and each `judged` line in the log says why. The defaults were chosen by evaluating the policy on real calls auto mode had sent to the classifier, labeled by hand, on a split used while choosing them and on one held out until the end. On both, the two judges allowed no call the policy rules out, apart from reads whose output held a secret (see [Known gaps](#known-gaps)), the same as `opus` at `medium` alone. Compared with it, they sent fewer of the calls they should allow to the classifier on the first split and one more on the held-out one, at about a seventh of the cost. Other models were not evaluated. +Patches are compared one commit at a time, so commits squashed into one when merged, or rebased with conflicts resolved, find no equivalent even when their work is there, and merge commits are not compared; an equivalent patch also does not show that the change is still there, or that the commits' messages and authors are kept anywhere. The git commands run without optional locks, an fsmonitor, or any transport, five seconds each. When a check cannot run or the judge is still unsure after it, the judge does not allow the call. -## What it leaves to native approval +Each judge runs through your session's own model client: one model call per tool call, and one more after a check. A model call that comes back without a reply, or with an API error Claude Code says may clear on its own (overload, a rate limit, or a server error), is made once more two seconds later while the judge has time; one cut short or refused for what it asked is not. A judge has 90 seconds for a tool call, counting all its model calls, waits, and checks. -- Any mode other than `auto`, and a subagent's calls until the subagent's mode has been reported. -- An `ask` from a settings rule, from a hook in your settings or another plugin, from a setting's own check (the read block, `permissions.blockReadsOutsideWorkingDirectories`, which asks you rather than the classifier), from an organization's ceiling for the tool, or one put to a host that answers the session's asks. -- A call run in the background (`run_in_background`), such as a dev server or a watcher: its output reaches the model after the call returns, where the plugin cannot scan it. -- A Bash, PowerShell, or command-monitor call with a non-empty `allowed_domains` list: the classifier reviews the command and registers approved network grants; a hook allow approves the command without granting the hosts. -- A call whose output the plugin withheld, or whose allow the engine rejected, when it runs again. -- A call auto mode denied after the judge left it to review, when the agent makes it again with the same input, in the same directory and, for a subagent, with the same brief, before you write again. Judging it again could only repeat the review or undo the denial with nothing new from you; once you have written, the call is judged again with your new words. -- A tool whose verdict the engine keeps whatever a hook answers: a subagent's report to its parent (`SubagentHandback`), which only the classifier may allow, and a question to you or a plan for your approval (`AskUserQuestion`, `ExitPlanMode`), which wait for you. The judge is not asked, since its allow would be rejected or ignored. +The verdict is the JSON object that ends the judge's reply: text may come before it, and only a closing code fence after it. A reply that does not end with a verdict or that states different verdicts along the way, a call the judge does not answer in time, and anything other than `allow` are not allowed, and go to the second judge if there is one, or else to the classifier. -## Output it withholds +The policy and your kept messages open every judge's prompt and are marked for the prompt cache, so a call soon after another, and the call after a check, can read them from the cache until you write again; the `usage` in each `judged` line shows what was read and written. -The judge decides before the call runs and cannot see what it will print. When what the model would read from a call the plugin allowed, including the full output a large result leaves in a file, a result another plugin's hook gave in the tool's place, and notes a hook adds beside the result, holds what looks like a credential (a private key, a token in a provider format that [gitleaks](https://github.com/gitleaks/gitleaks) knows or one starting `sk-`, a JSON web token, a netrc or registry login, a session cookie, or a password or token in a URL, header, or assignment), the plugin withholds it, and the same call run again goes to the classifier. The call has already been made, so the agent is warned that running it again repeats its effects. A call that moves to the background has its hand-off scanned in the same way, including the output returned before the move; a withheld hand-off warns that withholding did not stop the task, and gives its scanned task ID to follow, or asks you to inspect `/tasks` when that ID cannot be read safely. The result is also withheld when the scan itself fails or runs out of time. +## Output it withholds -## Install +The judge decides before the call runs and cannot see what it will print. When what the model would read from a call the plugin allowed, including the full output a large result leaves in a file, a result another plugin's hook gave in the tool's place, and notes a hook adds beside the result, holds what looks like a credential (a private key, a token in a provider format that [gitleaks](https://github.com/gitleaks/gitleaks) knows or one starting `sk-`, a JSON web token, a netrc or registry login, a session cookie, or a password or token in a URL, header, or assignment), the plugin withholds it. The result is also withheld when the scan itself fails or runs out of time. -```sh -claude plugin install preallow --marketplace Entwining/mods -``` +The call has already been made, and withholding its output does not undo it: the agent is warned that running it again repeats its effects, and the same call run again goes to the classifier. A call that moves to the background has its hand-off scanned in the same way, including the output returned before the move; a withheld hand-off warns that withholding did not stop the task, and gives its scanned task ID to follow, or asks you to inspect `/tasks` when that ID cannot be read safely. -This installs for your user. To pick a scope interactively, run `/plugin install preallow --marketplace Entwining/mods` in a Claude Code session. +## Reading the log -Claude Code does not update plugins from third-party marketplaces by default. In `/plugin`, select `entwining-mods` on the Marketplaces tab and choose **Enable auto-update**, or run `claude plugin update preallow@entwining-mods` manually. Start a new session after installing or updating. +The plugin logs one JSON line per event to `~/.claude/preallow/.jsonl`, keyed by `tool_use_id`, with its time in `at` and the Claude Code version in `engine`; a line that names the tool also names the subagent that made the call, if one did, in `agentId`. Past about 1 MB the log goes on in `.1.jsonl`, `.2.jsonl`, and so on. Tool inputs are logged as given, unscanned, so a credential in a command or a file edit lands in the log. The log stays on this machine until you delete the folder. -## Reading the log +To find calls the judge should not have allowed, read the `judged` lines whose deciding verdict is `allow`: `escalation.verdict` where the line has an `escalation`, `verdict` otherwise. -The plugin logs one JSON line per event to `~/.claude/preallow/.jsonl`, keyed by `tool_use_id`, with its time in `at` and the Claude Code version in `engine`; a line that names the tool also names the subagent that made the call, if one did, in `agentId`; past about 1 MB the log goes on in `.1.jsonl`, `.2.jsonl`, and so on: +The events; all but `rejected`, `denied`, and `withheld` also give why the core asked, in `coreReason`: -- `owned`: an `ask` its decider owns, left to it without asking the judge (see What it leaves to native approval), with the field that named the decider in `ownedBy` and its value, and why the core asked. -- `judged`: a call the judge was asked about, or whose input was too long to ask it about: the tool, its input (up to 2,000 characters), why the core asked (`coreReason`), the first judge's verdict (`allow`, or `review` when it did not allow the call, whether it answered so or not), the reason (the judge's, or why it gave none), whether the judge's model answered (`isAnswered`, absent when the input was too long to ask it, and false when its time ran out before the call after a check) and, when it did not, why in `failure` (`empty-reply`, `api-error` with Claude Code's kind of error and the HTTP status, `aborted` when the time limit ran out or the turn was interrupted, or the model client's refusal of the request), the failures it made a model call again after (`retried`), what it asked to check (`checks`, listed even when the checks could not run) and what they found (`facts`), how long it took, the `model` and `effort` settings it ran on, and in `usage` what each model call cost, one entry per call with the four token counts the API reports (`input_tokens` counts only the input neither read from nor written to the cache). When a second judge ran, `escalation` holds the same fields for it, from `verdict` to `usage`, and its verdict decided the call. -- `reserved`: a call of a tool whose verdict the engine keeps, left to it without asking the judge, with why the core asked. -- `background`: a call run in the background, left to the classifier without asking the judge, with why the core asked. -- `network`: a sandbox command requesting per-command domain grants, left to the classifier without asking the judge, with why the core asked. +- `judged`: a call the judge was asked about, or whose input was too long to ask it about, with the tool, its input (up to 2,000 characters), and the first judge's fields listed below. When a second judge ran, `escalation` holds the same fields for it, and its verdict decided the call. +- `owned`: an `ask` its decider owns, left to it without asking the judge (see [What it leaves to native approval](#what-it-leaves-to-native-approval)), with the field that named the decider in `ownedBy` and its value. +- `reserved`: a call of a tool whose verdict the engine keeps, left to it without asking the judge. +- `background`: a call run in the background, left to the classifier without asking the judge. +- `network`: a sandbox command requesting per-command domain grants, left to the classifier without asking the judge. +- `repeated`: a call auto mode denied, made again unchanged before you wrote again, left to the classifier without asking the judge. - `rejected`: the engine rejected the plugin's allow of a call because only the classifier may allow that tool, with the start of its message in `reason`. The call failed, and the same call run again goes to the classifier; that tool belongs with the reserved ones, so report it. -- `repeated`: a call auto mode denied, made again unchanged before you wrote again, left to the classifier without asking the judge, with why the core asked. - `denied`: auto mode denied a call the judge left to review, with the start of its reason in `reason`; auto mode can deny without a classifier verdict. The matching `judged` line says why the judge did not allow it. - `withheld`: the plugin withheld the output of a call it allowed, and why in `reason` (never the output). -To find calls the judge should not have allowed, read the `judged` lines whose deciding verdict is `allow`: `escalation.verdict` where the line has an `escalation`, `verdict` otherwise. Tool inputs are logged as given, unscanned, so a credential in a command or a file edit lands in the log. The log stays on this machine until you delete the folder. +A judge's fields in a `judged` line: + +- `verdict`: `allow`, or `review` when it did not allow the call, whether it answered so or not; and `reason`, the judge's, or why it gave none. +- `isAnswered`: whether the judge's model answered; absent when the input was too long to ask it, and false when its time ran out before the call after a check. +- `failure`, when it did not answer: `empty-reply`, `api-error` with Claude Code's kind of error and the HTTP status, `aborted` when the time limit ran out or the turn was interrupted, or the model client's refusal of the request. +- `retried`: the failures it made a model call again after. +- `checks`: what it asked to check, listed even when the checks could not run; and `facts`, what they found. +- `ms`, how long it took, and the `model` and `effort` settings it ran on. +- `usage`: what each model call cost, one entry per call with the four token counts the API reports; `input_tokens` counts only the input neither read from nor written to the cache. ## Known gaps - A long conversation can lose a prohibition through the storage limit described in [What the judge reads](#what-the-judge-reads): past 64,000 characters the plugin keeps your first message and the newest that fit, and cuts a message over 16,000 characters to its head and tail. Repeat an important prohibition in a short message when it matters. -- On Bedrock, Vertex, Foundry, or behind a gateway, Claude Code finds cached text again only while the prompt opens the same way, and the judge's prompt opens with your kept messages, so after each message you write the judge's next call writes its instructions to the cache again along with your messages; the `usage` in each `judged` line shows what was read and written. +- On Bedrock, Vertex, Foundry, or behind a gateway, Claude Code finds cached text again only while the prompt opens the same way, and the judge's prompt opens with your kept messages, so after each message you write the judge's next call writes its instructions to the cache again along with your messages. - A conversation forked into a new session, or one you last wrote in before the 50 most recent, starts with none of your earlier messages, so the judge may allow a call your earlier words ruled out. - `/rewind` does not take your messages back from the judge: the plugin learns of no rewind, so a request you rewound still counts as yours. To withdraw one, say so in a new message. - A call that only reads, such as printing a log, listing processes, showing proxy variables, or capturing a terminal pane, does not show that its output will hold a secret, so the judge allows it as it would any read; keeping the secret out of the conversation is left to the [output scan](#output-it-withholds), with the limits below. - The credential scan knows a fixed list of formats (`hooks/withhold.ts` and gitleaks' provider rules); a credential in any other format, such as a password of letters only, or a value under a name the list does not know, slips through, and so do a token holding the word `false`, which gitleaks' allowlist reads as a boolean, and a value in a Kubernetes Secret that follows an empty one. JSON escaping can also hide a quoted password's shape. A private key slips through when more than about 200 characters of headers come before its body, or when a body wrapped narrower than 40 characters has indented or blank lines. -- Some text that is not a credential is taken for one, such as a private key's armor around 40 or more letters and digits that are not its body, a placeholder spread over lines for example, or, under gitleaks' rules, any 40-character hex string, a git commit hash included, in an output that also holds the word `sourcegraph`; the output is withheld and the call runs again under the classifier. +- Some text that is not a credential is taken for one, such as a private key's armor around 40 or more letters and digits that are not its body, a placeholder spread over lines for example, or, under gitleaks' rules, almost any 40-character hex string, a git commit hash included, in an output that also holds the word `sourcegraph`; the output is withheld, and the same call run again goes to the classifier. - For a call moved to the background while it runs, the plugin scans only the hand-off. Output delivered later, including task notifications, is outside that scan; a later read is scanned only if the plugin allows that call. - A `Monitor` call starts a watch whose events reach the model later, one line at a time; the judge may allow it, and the plugin does not scan those events. - A call the plugin allowed runs again under the classifier after a withheld output only when its input is the same; a rerun with any input change is judged again. diff --git a/plugins/preallow/hooks/facts.ts b/plugins/preallow/hooks/facts.ts index 3f8fe6a..113777d 100644 --- a/plugins/preallow/hooks/facts.ts +++ b/plugins/preallow/hooks/facts.ts @@ -17,7 +17,7 @@ export type Host = { // The directory relative paths resolve from, and `~`. cwd: string; home: string | undefined; - // When the session began: a file changed before then predates the task. + // When the session began, which file modification times are counted against. startedAt: number; }; diff --git a/plugins/preallow/hooks/judge.ts b/plugins/preallow/hooks/judge.ts index 177a08e..4d1a85f 100644 --- a/plugins/preallow/hooks/judge.ts +++ b/plugins/preallow/hooks/judge.ts @@ -62,8 +62,8 @@ export type JudgeHost = { // longer than this goes to the classifier unjudged. const INPUT_CHARS = 100_000; const BRIEF_CHARS = 8000; -// A judge that has not answered by then, retries included, sends the call to -// the classifier. +// A judge that has not answered by then, retries included, answers review, +// which the second stage, if there is one, judges again. const JUDGE_TIMEOUT_MS = 90_000; // Empty replies come between replies to prompts of the same size seconds // apart, so a call made again shortly after usually gets one. The wait, unlike diff --git a/plugins/preallow/hooks/messages.ts b/plugins/preallow/hooks/messages.ts index e6a6b68..30252bb 100644 --- a/plugins/preallow/hooks/messages.ts +++ b/plugins/preallow/hooks/messages.ts @@ -26,8 +26,8 @@ export function keptMessages({ turns, leftOut }: UserMessages): readonly string[ return [first!, `[${leftOut} messages left out]`, ...rest]; } -// The head and tail of a long text, which hold a message's request and its -// last word. +// The head and tail of a long text, where a message usually puts its request +// and its last word. export function clip(text: string, max: number): string { if (text.length <= max) return text; const head = Math.floor(max * 0.75); diff --git a/plugins/preallow/hooks/paths.ts b/plugins/preallow/hooks/paths.ts index 8a09dbb..d4a8671 100644 --- a/plugins/preallow/hooks/paths.ts +++ b/plugins/preallow/hooks/paths.ts @@ -1,5 +1,5 @@ // Facts about a path the judge names: whether it exists, how many of its -// files predate the session, and its git status. +// files changed before the session began, and its git status. import type { Host } from './facts.ts'; import { git, gitOk } from './git.ts'; diff --git a/plugins/preallow/hooks/refs.ts b/plugins/preallow/hooks/refs.ts index c3c03ff..da58bc6 100644 --- a/plugins/preallow/hooks/refs.ts +++ b/plugins/preallow/hooks/refs.ts @@ -110,8 +110,9 @@ export async function refFacts(host: Host, repo: string, ref: string) { } // A fetch updates the remote refs every worktree shares but writes FETCH_HEAD -// in the worktree it ran in, so the refs are as fresh as the newest FETCH_HEAD -// among the main worktree and the linked ones. +// in the worktree it ran in, so the newest FETCH_HEAD among the main worktree +// and the linked ones dates the last fetch, which need not have updated the +// ref asked about. async function lastFetch(host: Host, dir: string): Promise { const common = ( await gitOk(host.run, dir, ['rev-parse', '--path-format=absolute', '--git-common-dir']) diff --git a/plugins/preallow/tests/withhold.test.ts b/plugins/preallow/tests/withhold.test.ts index 4be5f35..2a0ab1c 100644 --- a/plugins/preallow/tests/withhold.test.ts +++ b/plugins/preallow/tests/withhold.test.ts @@ -7,10 +7,11 @@ import { world } from './world.ts'; // Values are built at run time, so no scanner takes the source for a leak. const RANDOM = 'q7Rk2Vx9Lm4Np8Ts3Wz6Yb1Hc5'; const FAKE_TOKEN = `sk-${RANDOM}`; -// A value spread evenly over its alphabet, as uniform as a generated token. const B62 = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'; const BECH32 = 'QPZRY9X8GF2TVDW0S3JN54KHCE6MUA7L'; const DIGITS = '1234567890'; +// A value spread evenly over its alphabet, so it passes the entropy check of +// gitleaks' rules as a generated token does. const mixed = (n: number, alphabet = B62) => Array.from({ length: n }, (_, i) => alphabet[(i * 7) % alphabet.length]).join(''); diff --git a/plugins/preallow/tests/world.ts b/plugins/preallow/tests/world.ts index 6b0cbb3..5c940cc 100644 --- a/plugins/preallow/tests/world.ts +++ b/plugins/preallow/tests/world.ts @@ -67,7 +67,8 @@ export const LOG = '/home/u/.claude/preallow/s1.jsonl'; export const REJECTION = "Only the auto-mode classifier can allow mcp__srv__handoff: an allow of type 'hook' is not the classifier's verdict"; -// What the judge model's nth call costs, distinct for each call. +// What the judge model's nth call costs, distinct for each call so a usage +// entry dropped or counted twice shows. export const usageOf = (n: number) => ({ input_tokens: n, output_tokens: 10 * n,