From 25e1a000e68b586bd611a605bf7438ecea646b27 Mon Sep 17 00:00:00 2001 From: Mohid Makhdoomi Date: Sat, 18 Jul 2026 02:07:17 -0400 Subject: [PATCH 01/23] add .codex/skills/ --- .codex/skills/afx/SKILL.md | 179 +++ .../without_skill/output.txt | 3 + .../without_skill/output.txt | 3 + .../without_skill/output.txt | 3 + .../without_skill/output.txt | 3 + .../without_skill/output.txt | 3 + .codex/skills/afx/evals/evals.json | 35 + .codex/skills/arch-init/SKILL.md | 62 + .codex/skills/codev/SKILL.md | 59 + .codex/skills/consult/SKILL.md | 96 ++ .codex/skills/forge/SKILL.md | 68 + .codex/skills/generate-image/SKILL.md | 45 + .codex/skills/porch/SKILL.md | 83 ++ .codex/skills/skill-creator/LICENSE.txt | 202 +++ .codex/skills/skill-creator/SKILL.md | 479 ++++++ .../skills/skill-creator/agents/analyzer.md | 274 ++++ .../skills/skill-creator/agents/comparator.md | 202 +++ .codex/skills/skill-creator/agents/grader.md | 223 +++ .../skill-creator/assets/eval_review.html | 146 ++ .../eval-viewer/generate_review.py | 471 ++++++ .../skill-creator/eval-viewer/viewer.html | 1325 +++++++++++++++++ .../skill-creator/references/schemas.md | 430 ++++++ .../skills/skill-creator/scripts/__init__.py | 0 .../scripts/aggregate_benchmark.py | 401 +++++ .../skill-creator/scripts/generate_report.py | 326 ++++ .../scripts/improve_description.py | 248 +++ .../skill-creator/scripts/package_skill.py | 136 ++ .../skill-creator/scripts/quick_validate.py | 103 ++ .../skills/skill-creator/scripts/run_eval.py | 310 ++++ .../skills/skill-creator/scripts/run_loop.py | 332 +++++ .codex/skills/skill-creator/scripts/utils.py | 47 + .codex/skills/team/SKILL.md | 129 ++ .codex/skills/update-arch-docs/SKILL.md | 132 ++ 33 files changed, 6558 insertions(+) create mode 100644 .codex/skills/afx/SKILL.md create mode 100644 .codex/skills/afx/af-workspace/iteration-1/eval-1-spawn-naming/without_skill/output.txt create mode 100644 .codex/skills/afx/af-workspace/iteration-1/eval-2-cleanup-format/without_skill/output.txt create mode 100644 .codex/skills/afx/af-workspace/iteration-1/eval-3-send-with-file/without_skill/output.txt create mode 100644 .codex/skills/afx/af-workspace/iteration-1/eval-4-tower-restart/without_skill/output.txt create mode 100644 .codex/skills/afx/af-workspace/iteration-1/eval-5-tick-amends/without_skill/output.txt create mode 100644 .codex/skills/afx/evals/evals.json create mode 100644 .codex/skills/arch-init/SKILL.md create mode 100644 .codex/skills/codev/SKILL.md create mode 100644 .codex/skills/consult/SKILL.md create mode 100644 .codex/skills/forge/SKILL.md create mode 100644 .codex/skills/generate-image/SKILL.md create mode 100644 .codex/skills/porch/SKILL.md create mode 100644 .codex/skills/skill-creator/LICENSE.txt create mode 100644 .codex/skills/skill-creator/SKILL.md create mode 100644 .codex/skills/skill-creator/agents/analyzer.md create mode 100644 .codex/skills/skill-creator/agents/comparator.md create mode 100644 .codex/skills/skill-creator/agents/grader.md create mode 100644 .codex/skills/skill-creator/assets/eval_review.html create mode 100644 .codex/skills/skill-creator/eval-viewer/generate_review.py create mode 100644 .codex/skills/skill-creator/eval-viewer/viewer.html create mode 100644 .codex/skills/skill-creator/references/schemas.md create mode 100644 .codex/skills/skill-creator/scripts/__init__.py create mode 100755 .codex/skills/skill-creator/scripts/aggregate_benchmark.py create mode 100755 .codex/skills/skill-creator/scripts/generate_report.py create mode 100755 .codex/skills/skill-creator/scripts/improve_description.py create mode 100755 .codex/skills/skill-creator/scripts/package_skill.py create mode 100755 .codex/skills/skill-creator/scripts/quick_validate.py create mode 100755 .codex/skills/skill-creator/scripts/run_eval.py create mode 100755 .codex/skills/skill-creator/scripts/run_loop.py create mode 100644 .codex/skills/skill-creator/scripts/utils.py create mode 100644 .codex/skills/team/SKILL.md create mode 100644 .codex/skills/update-arch-docs/SKILL.md diff --git a/.codex/skills/afx/SKILL.md b/.codex/skills/afx/SKILL.md new file mode 100644 index 000000000..2b365b5ea --- /dev/null +++ b/.codex/skills/afx/SKILL.md @@ -0,0 +1,179 @@ +--- +name: afx +description: Agent Farm CLI — the tool for spawning builders, managing Tower, workspaces, and cron tasks. ALWAYS consult this skill BEFORE running any `afx` command to get the exact syntax. This prevents wasting time guessing flags that don't exist. Use this whenever you need to spawn a builder, check status, send messages, clean up worktrees, manage Tower, or run cron tasks. If you're about to type `afx` followed by anything, check here first. +--- + +# Agent Farm CLI + +## afx spawn + +Spawns a new builder in an isolated git worktree. + +``` +afx spawn [number] [options] +``` + +**The ONLY flags that exist:** + +| Flag | Description | +|------|-------------| +| `--protocol ` | Protocol: spir, aspir, air, bugfix, tick, maintain, experiment. **Required for numbered spawns.** | +| `--task ` | Ad-hoc task (no issue number needed) | +| `--shell` | Bare Claude session | +| `--worktree` | Bare worktree session | +| `--amends ` | Original spec number (TICK only) | +| `--files ` | Context files, comma-separated. **Requires `--task`.** | +| `--no-comment` | Skip commenting on the GitHub issue | +| `--force` | Skip dirty-worktree and collision checks | +| `--soft` | Soft mode (AI follows protocol, you verify) | +| `--strict` | Strict mode (porch orchestrates) — this is the default | +| `--resume` | Resume builder in existing worktree | +| `--no-role` | Skip loading role prompt | + +**There is NO `-t`, `--title`, `--name`, or `--branch` flag.** The branch name is auto-generated from the issue title. + +**Examples:** +```bash +afx spawn 42 --protocol spir # SPIR builder for issue #42 +afx spawn 42 --protocol aspir # ASPIR (autonomous, no human gates) +afx spawn 42 --protocol air # AIR (small features) +afx spawn 42 --protocol bugfix # Bugfix +afx spawn 42 --protocol tick --amends 30 # TICK amendment to spec 30 +afx spawn 42 --protocol spir --soft # Soft mode +afx spawn 42 --resume # Resume existing builder +afx spawn --task "fix the flaky test" # Ad-hoc task (no issue) +afx spawn 42 --protocol spir --force # Skip dirty-worktree check +``` + +**Pre-spawn checklist:** +1. `git status` — worktree must be clean (or use `--force`) +2. Commit specs/plans first — builders branch from HEAD and can't see uncommitted files +3. `--protocol` is required for numbered spawns + +## afx send + +Sends a message to a running builder. + +``` +afx send [builder] [message] +``` + +| Flag | Description | +|------|-------------| +| `--all` | Send to all builders | +| `--file ` | Include file content | +| `--interrupt` | Send Ctrl+C first | +| `--raw` | Skip structured formatting | +| `--no-enter` | Don't press Enter after message | + +```bash +afx send 0042 "PR approved, please merge" +afx send 0585 "check the test output" --file /tmp/test-results.txt +``` + +**Addressing forms** — the recipient isn't only a builder ID: + +| Form | Routes to | +|------|-----------| +| `afx send 0585 "…"` | Builder `0585` (zero-padded). | +| `afx send architect "…"` | From a builder: its spawning architect. From an architect: the `main` architect (else first registered). | +| `afx send architect: "…"` | A specific named architect (sibling-architect messaging). Architects can address any architect; a builder may only use its own `spawnedByArchitect`. | +| `afx send :architect "…"` | The architect in **another workspace** (cross-workspace). | + +```bash +afx send architect:ob-refine "PR-iter-2 feedback ready" # sibling architect, same workspace +afx send taqwabench:architect "use porch.checks, don't fork protocol.json" # cross-workspace +``` + +**Find the exact workspace name first** — guessing fails with `NOT_FOUND`. The name is the basename of the workspace path, which can differ from how someone refers to it. List active workspaces via the Tower API: + +```bash +curl -s http://localhost:4100/api/workspaces | python3 -m json.tool # → each .name is a valid address +``` + +## afx cleanup + +Removes a builder's worktree and branch after work is done. + +``` +afx cleanup [options] +``` + +| Flag | Description | +|------|-------------| +| `-p, --project ` | Builder project ID (no leading zeros: `585` not `0585`) | +| `-i, --issue ` | Cleanup bugfix builder by issue number | +| `-t, --task ` | Cleanup task builder (e.g., `task-bEPd`) | +| `-f, --force` | Force cleanup even if branch not merged | + +```bash +afx cleanup -p 585 # Clean up project 585 +afx cleanup -p 585 -f # Force (unmerged branch) +``` + +**Note:** `afx cleanup` uses plain numbers (`585`), not zero-padded (`0585`). But `afx send` uses zero-padded IDs (`0585`). + +## afx status + +```bash +afx status # Show all builders and workspace status +``` + +No flags needed. Shows Tower status, workspace, and all active builders. + +## afx whoami + +```bash +afx whoami # Report this terminal's identity (workspace, type, name) +afx whoami --json # Same, as a single JSON object +``` + +Resolves identity from Tower/global.db's perspective: builder-worktree cwd +match first, then the Tower-injected `CODEV_ARCHITECT_NAME` env var. Exits 1 +with an explanation when identity cannot be determined — it never guesses and +never defaults to `main` (issue #1094). Builders also get an `architect:` +field naming their spawning architect when recorded. Works without Tower. + +## afx tower + +```bash +afx tower start # Start Tower on port 4100 +afx tower stop # Stop Tower +afx tower log # Tail Tower logs +afx tower status # Check daemon and cloud connection status +afx tower connect # Connect to Codev Cloud +afx tower disconnect # Disconnect from Codev Cloud +``` + +There is NO `afx tower restart` — use `afx tower stop && afx tower start`. + +## afx workspace + +```bash +afx workspace start # Start workspace for current project +afx workspace stop # Stop workspace processes +``` + +`afx dash` is a deprecated alias — use `afx workspace` instead. + +## afx cron + +```bash +afx cron list # List all cron tasks +afx cron status # Check task status +afx cron run # Run immediately +afx cron enable # Enable +afx cron disable # Disable +``` + +There is NO `afx cron add` — create YAML files in `.af-cron/` directly. + +## Other commands + +```bash +afx open # Open file in annotation viewer (NOT system open) +afx shell # Spawn utility shell +afx attach # Attach to running builder terminal +afx rename # Rename current shell session +afx architect # Start architect session in current terminal +``` diff --git a/.codex/skills/afx/af-workspace/iteration-1/eval-1-spawn-naming/without_skill/output.txt b/.codex/skills/afx/af-workspace/iteration-1/eval-1-spawn-naming/without_skill/output.txt new file mode 100644 index 000000000..e965b166e --- /dev/null +++ b/.codex/skills/afx/af-workspace/iteration-1/eval-1-spawn-naming/without_skill/output.txt @@ -0,0 +1,3 @@ +Error: Claude Code cannot be launched inside another Claude Code session. +Nested sessions share runtime resources and will crash all active sessions. +To bypass this check, unset the CLAUDECODE environment variable. diff --git a/.codex/skills/afx/af-workspace/iteration-1/eval-2-cleanup-format/without_skill/output.txt b/.codex/skills/afx/af-workspace/iteration-1/eval-2-cleanup-format/without_skill/output.txt new file mode 100644 index 000000000..e965b166e --- /dev/null +++ b/.codex/skills/afx/af-workspace/iteration-1/eval-2-cleanup-format/without_skill/output.txt @@ -0,0 +1,3 @@ +Error: Claude Code cannot be launched inside another Claude Code session. +Nested sessions share runtime resources and will crash all active sessions. +To bypass this check, unset the CLAUDECODE environment variable. diff --git a/.codex/skills/afx/af-workspace/iteration-1/eval-3-send-with-file/without_skill/output.txt b/.codex/skills/afx/af-workspace/iteration-1/eval-3-send-with-file/without_skill/output.txt new file mode 100644 index 000000000..e965b166e --- /dev/null +++ b/.codex/skills/afx/af-workspace/iteration-1/eval-3-send-with-file/without_skill/output.txt @@ -0,0 +1,3 @@ +Error: Claude Code cannot be launched inside another Claude Code session. +Nested sessions share runtime resources and will crash all active sessions. +To bypass this check, unset the CLAUDECODE environment variable. diff --git a/.codex/skills/afx/af-workspace/iteration-1/eval-4-tower-restart/without_skill/output.txt b/.codex/skills/afx/af-workspace/iteration-1/eval-4-tower-restart/without_skill/output.txt new file mode 100644 index 000000000..e965b166e --- /dev/null +++ b/.codex/skills/afx/af-workspace/iteration-1/eval-4-tower-restart/without_skill/output.txt @@ -0,0 +1,3 @@ +Error: Claude Code cannot be launched inside another Claude Code session. +Nested sessions share runtime resources and will crash all active sessions. +To bypass this check, unset the CLAUDECODE environment variable. diff --git a/.codex/skills/afx/af-workspace/iteration-1/eval-5-tick-amends/without_skill/output.txt b/.codex/skills/afx/af-workspace/iteration-1/eval-5-tick-amends/without_skill/output.txt new file mode 100644 index 000000000..e965b166e --- /dev/null +++ b/.codex/skills/afx/af-workspace/iteration-1/eval-5-tick-amends/without_skill/output.txt @@ -0,0 +1,3 @@ +Error: Claude Code cannot be launched inside another Claude Code session. +Nested sessions share runtime resources and will crash all active sessions. +To bypass this check, unset the CLAUDECODE environment variable. diff --git a/.codex/skills/afx/evals/evals.json b/.codex/skills/afx/evals/evals.json new file mode 100644 index 000000000..9bff5419e --- /dev/null +++ b/.codex/skills/afx/evals/evals.json @@ -0,0 +1,35 @@ +{ + "skill_name": "af", + "evals": [ + { + "id": 1, + "prompt": "Spawn a builder for issue 71 using the ASPIR protocol. I want to call it e2e-staging-tests.", + "expected_output": "Should run `af spawn 71 --protocol aspir` without trying -t, --title, --name, or any flag for naming. Should note that branch names are auto-generated.", + "files": [] + }, + { + "id": 2, + "prompt": "Clean up the builder for project 42. The branch wasn't merged yet so force it.", + "expected_output": "Should run `af cleanup -p 42 -f` (plain number, not 0042).", + "files": [] + }, + { + "id": 3, + "prompt": "Send a message to builder 42 telling it to check the failing test and include the test output file at /tmp/test-results.txt", + "expected_output": "Should run `af send 0042 \"check the failing test\" --file /tmp/test-results.txt` (zero-padded ID for send).", + "files": [] + }, + { + "id": 4, + "prompt": "I need to restart Tower to pick up new code changes.", + "expected_output": "Should run `af tower stop && af tower start`. Should NOT try `af tower restart` which doesn't exist.", + "files": [] + }, + { + "id": 5, + "prompt": "Spawn a TICK builder for issue 88 that amends spec 45. There are uncommitted changes but they're not relevant.", + "expected_output": "Should run `af spawn 88 --protocol tick --amends 45 --force`. Must include both --amends and --force.", + "files": [] + } + ] +} diff --git a/.codex/skills/arch-init/SKILL.md b/.codex/skills/arch-init/SKILL.md new file mode 100644 index 000000000..26cacde9d --- /dev/null +++ b/.codex/skills/arch-init/SKILL.md @@ -0,0 +1,62 @@ +--- +name: arch-init +description: Adopt an architect identity and recover its state from codev/state/.md. Use when an architect terminal needs to (re)establish which architect it is — after a restart, context loss, or session handoff — or when the user says "/arch-init", "you are the X architect", or "recover your architect state". Identity resolves via `afx whoami` (an explicit name argument overrides); if neither resolves, ask the human — never guess. +argument-hint: "[name] (e.g. main; omit to auto-detect via afx whoami)" +--- + +# /arch-init — become architect `` and recover state + +You are an **architect agent** in a codev workspace. This command tells you +which architect you are and where your durable state lives, so you can resume +mid-stream. + +`$ARGUMENTS` is the architect name (e.g. `main`, or a sibling architect's +name in a multi-architect workspace). + +## What to do + +1. **Resolve your name.** + - If `$ARGUMENTS` is non-empty, that is your name — the human named you + explicitly, which removes all identity-resolution risk. **Validate it + first**: an architect name must match `[a-z][a-z0-9-]*` and be at most + 64 characters (lowercase letters, digits, hyphens; starts with a + letter). Reject anything else — slashes, `..`, uppercase, spaces — and + tell the human the rule. Never build a file path from an unvalidated + name (path-traversal guard). + - If `$ARGUMENTS` is empty, run `afx whoami` and read its output: + - `type: architect` → adopt the reported `name`. + - `type: builder` → STOP. This terminal is a builder, not an architect; + report the mismatch to the human and do not adopt an architect + identity. + - Non-zero exit (identity unknown) → STOP and ask the human which + architect you are. Do NOT guess, and do NOT default to `main` — + adopting the wrong identity and writing to another architect's state + file is the exact failure this command exists to prevent. + +2. **Read your state file: `codev/state/.md`** (relative to the + workspace root). + - If it does not exist: list the architect state files in `codev/state/` + — **excluding `*_thread.md` files**, which are builder thread logs that + share the directory — tell the human the file is missing, and ask + whether to start a fresh state file for ``. Do not fabricate + state. + - The state file is authoritative free text. It typically opens with a + role banner and may carry resume instructions; follow whatever it says. + +3. **Confirm identity + orient, then follow the state file.** In one tight + block, report: who you now are (name + one-line role from the banner, if + present), the file you read, and the current-state / open-loops summary + from the most recent dated section (or the file's leading content if it + has no dated sections). Then carry out whatever the state file says to do + on resume. Do not invent a new agenda — resume the one the state file + describes. + +## Guardrails (architect-wide; the state file may add more) + +- **Never auto-approve porch gates.** A gate notification is for the human, + not you. +- **Touch only your own builders / spawns / filings.** Sibling architects own + theirs. +- **Never `cd` into a builder worktree**; use `git -C` + absolute paths. +- **Stay on the default branch at the workspace root**; verify with + `git branch` if unsure. diff --git a/.codex/skills/codev/SKILL.md b/.codex/skills/codev/SKILL.md new file mode 100644 index 000000000..0a0e001e9 --- /dev/null +++ b/.codex/skills/codev/SKILL.md @@ -0,0 +1,59 @@ +--- +name: codev +description: Codev project management CLI — init, adopt, update, and doctor commands. Check this skill before running any `codev` command (except `consult`, `porch`, or `afx` which have their own skills). Use when setting up new projects, adding codev to existing repos, updating framework files, or diagnosing missing dependencies. +--- + +# codev - Project Management CLI + +## Commands + +``` +codev init [project-name] Create a new codev project directory +codev adopt Add codev to the current directory +codev update Update protocols, roles, skills from package +codev doctor Check system dependencies +``` + +## codev init + +Creates a **new directory** with the full codev structure: specs/, plans/, reviews/, protocols, CLAUDE.md, AGENTS.md, .claude/skills/, .codev/config.json, .gitignore. + +```bash +codev init my-app # Interactive setup +codev init my-app -y # Non-interactive with defaults +``` + +## codev adopt + +Adds codev to the **current directory** (existing project). Run from the project root. If CLAUDE.md or AGENTS.md already exists, creates `.codev-new` versions and spawns Claude to merge. + +```bash +codev adopt # Interactive +codev adopt -y # Skip conflict prompts +``` + +## codev update + +Updates framework files (protocols, roles, skills) from the installed `@cluesmith/codev` package. **Never touches user data** (specs, plans, reviews). Creates `.codev-new` versions for customized files. + +```bash +codev update # Interactive update +codev update --dry-run # Preview changes +codev update --force # Overwrite all framework files +codev update --agent # Agent mode: JSON stdout, no interactive merge +``` + +## codev doctor + +Checks all required dependencies: Node.js (>=18), git (>=2.5), gh (authenticated), and at least one AI CLI (Claude, Gemini, or Codex). + +```bash +codev doctor +``` + +## Common mistakes + +- There is NO `codev tower` command — use `afx tower start/stop` +- `codev init` creates a new directory — use `codev adopt` for existing projects +- Always run `codev adopt` and `codev update` from the project root +- `codev update` only updates framework files — it never touches specs/plans/reviews diff --git a/.codex/skills/consult/SKILL.md b/.codex/skills/consult/SKILL.md new file mode 100644 index 000000000..4720b0dd1 --- /dev/null +++ b/.codex/skills/consult/SKILL.md @@ -0,0 +1,96 @@ +--- +name: consult +description: AI consultation CLI — query Gemini, Codex, or Claude for reviews and analysis. ALWAYS check this skill before running any `consult` command. Use when reviewing specs, plans, implementations, or PRs with external models, running parallel 3-way reviews (cmap), or checking consultation stats. The `-m` model flag is always required except for `consult stats`. +--- + +# consult - AI Consultation CLI + +Query external AI models for reviews and analysis. Supports Gemini, Codex, and Claude. + +## Synopsis + +``` +consult -m [options] +consult stats [options] +``` + +The `-m` / `--model` flag is **always required** except for `consult stats`. + +## Models + +| Flag value | Alias | Notes | +|------------|-------|-------| +| `gemini` | `pro` | Antigravity CLI (`agy`); agentic file access (`--sandbox`), OAuth login; skips non-blockingly if unavailable | +| `codex` | `gpt` | Thorough (~200-250s), shell exploration | +| `claude` | `opus` | Agent SDK with tool use (~60-120s) | + +## All flags + +``` +-m, --model Model to use (required except stats) +--prompt Inline prompt (general mode) +--prompt-file Prompt file path (general mode) +--protocol Protocol: spir, aspir, air, bugfix, maintain +-t, --type Review type (see below) +--issue Issue number (required in architect context) +--output Save result to file +--plan-phase Scope review to a plan phase (porch use) +--context Context file with feedback (porch use) +--project-id Project ID for metrics (porch use) +--days Stats: limit to last N days (default: 30) +--project Stats: filter by project ID +--last Stats: show last N invocations +--json Stats: output as JSON +``` + +## Review types (`--type`) + +| Type | When to use | +|------|-------------| +| `spec` | Review a specification for completeness | +| `plan` | Review an implementation plan | +| `impl` | Review code implementation | +| `pr` | Review a pull request before merge | +| `phase` | Phase-scoped review (builder context only) | +| `integration` | Architect's integration review of a PR | + +## Usage patterns + +**General query:** +```bash +consult -m gemini --prompt "What's the best way to structure auth?" +consult -m codex --prompt-file review-checklist.md +``` + +**Protocol review:** +```bash +consult -m gemini --type spec --issue 42 +consult -m codex --type plan --issue 42 +consult -m claude --type integration --issue 42 +``` + +**3-way parallel review (cmap):** +Always use `--output` for background runs — without it, results go to a temp file that may be garbage-collected. + +```bash +consult -m gemini --type integration --issue 42 --output /tmp/cmap-gemini-42.md & +consult -m codex --type integration --issue 42 --output /tmp/cmap-codex-42.md & +consult -m claude --type integration --issue 42 --output /tmp/cmap-claude-42.md & +wait +``` + +**Stats:** +```bash +consult stats # 30-day summary +consult stats --days 7 --json # Last 7 days as JSON +consult stats --project 42 # Filter by project +``` + +## Rules + +- `-m` is required for all non-stats commands +- `--prompt` and `--type` are mutually exclusive (different modes) +- `--prompt` and `--prompt-file` are mutually exclusive +- `--protocol` requires `--type` +- From architect context (outside `.builders/`), `--issue` is required for protocol reviews +- From builder context (inside `.builders/`), project auto-detects from porch state diff --git a/.codex/skills/forge/SKILL.md b/.codex/skills/forge/SKILL.md new file mode 100644 index 000000000..be2c2062f --- /dev/null +++ b/.codex/skills/forge/SKILL.md @@ -0,0 +1,68 @@ +# Forge Concept Commands + +Forge concept commands decouple codev from direct `gh` CLI calls. Each GitHub operation is routed through a configurable external command. + +## Concepts + +| Concept | Env Vars | Description | +|---------|----------|-------------| +| `issue-view` | `CODEV_ISSUE_ID` | Fetch issue details (JSON) | +| `pr-list` | — | List open PRs (JSON array) | +| `issue-list` | — | List issues (JSON array) | +| `issue-comment` | `CODEV_ISSUE_ID`, `CODEV_COMMENT_BODY` | Post a comment on an issue | +| `pr-exists` | `CODEV_BRANCH_NAME` | Check if a PR exists for a branch | +| `recently-closed` | `CODEV_SINCE_DATE` (optional) | List recently closed issues | +| `recently-merged` | `CODEV_SINCE_DATE` (optional) | List recently merged PRs | +| `user-identity` | — | Get current user's handle (plain text) | +| `team-activity` | `CODEV_GRAPHQL_QUERY` | Run a batched GraphQL query | +| `on-it-timestamps` | `CODEV_ISSUE_NUMBERS`, `CODEV_GRAPHQL_QUERY`, `CODEV_REPO_OWNER`, `CODEV_REPO_NAME` | Get "on it" comment timestamps | +| `pr-merge` | `CODEV_PR_NUMBER` | Merge a PR | +| `pr-search` | `CODEV_SEARCH_QUERY` | Search PRs (JSON array) | +| `pr-view` | `CODEV_PR_NUMBER`, `CODEV_INCLUDE_COMMENTS` (optional) | View PR details (JSON or text) | +| `pr-diff` | `CODEV_PR_NUMBER`, `CODEV_DIFF_NAME_ONLY` (optional) | Get PR diff | +| `auth-status` | — | Check forge authentication status | + +## Configuration + +In `.codev/config.json`: + +```json +{ + "forge": { + "provider": "gitlab", + "issue-comment": "my-custom-script $CODEV_ISSUE_ID" + } +} +``` + +### Resolution order +1. Manual concept override in `forge` section +2. Provider preset (if `provider` is set) +3. Default (GitHub via `gh` CLI) + +### Providers + +Built-in presets: `github` (default), `gitlab` (via `glab`), `gitea` (via `tea`). + +**Note:** Non-GitHub presets are best-effort. Output schemas may differ from GitHub's JSON contracts. Non-conforming JSON returns `null` — consumers handle this gracefully. Override individual concepts if a preset doesn't match your CLI version. + +### Disabling concepts + +Set a concept to `null` to disable it: +```json +{ + "forge": { + "team-activity": null + } +} +``` + +## Validation + +Run `codev doctor` to see forge concept status, provider, and validation results. + +## Code + +- **Dispatcher**: `packages/codev/src/lib/forge.ts` +- **Contracts**: `packages/codev/src/lib/forge-contracts.ts` +- **Spec**: `codev/specs/589-non-github-repository-support.md` diff --git a/.codex/skills/generate-image/SKILL.md b/.codex/skills/generate-image/SKILL.md new file mode 100644 index 000000000..aed0276b9 --- /dev/null +++ b/.codex/skills/generate-image/SKILL.md @@ -0,0 +1,45 @@ +--- +name: generate-image +description: AI image generation via Gemini. Use when the user wants to generate, create, or make an image, or when you need to create visual assets like logos, diagrams, or illustrations. Requires GEMINI_API_KEY or GOOGLE_API_KEY. +--- + +# generate-image - AI Image Generation + +Uses Google Gemini to generate images from text prompts. + +## Synopsis + +``` +codev generate-image "" [options] +``` + +Note: this is a `codev` subcommand, not standalone. + +## All flags + +``` +-o, --output Output file path (default: output.png) +-r, --resolution Resolution: 1K, 2K, 4K (default: 1K) +-a, --aspect Aspect ratio (default: 1:1) +--ref Reference image (repeatable, max 14) +``` + +## Aspect ratios + +`1:1` | `16:9` | `9:16` | `3:4` | `4:3` | `3:2` | `2:3` + +## Examples + +```bash +codev generate-image "A sunset over mountains" +codev generate-image "A futuristic city" -r 4K -a 16:9 -o city.png +codev generate-image "Same style but with cats" --ref style.png --ref layout.png +codev generate-image prompt.txt -o result.png # Prompt from .txt file +``` + +## Notes + +- Prompt must be quoted if it contains spaces +- Prompt can be a `.txt` file path (auto-detected by extension) +- Reference images must exist on disk +- Requires `GEMINI_API_KEY` or `GOOGLE_API_KEY` environment variable diff --git a/.codex/skills/porch/SKILL.md b/.codex/skills/porch/SKILL.md new file mode 100644 index 000000000..24dd63d9e --- /dev/null +++ b/.codex/skills/porch/SKILL.md @@ -0,0 +1,83 @@ +--- +name: porch +description: Protocol orchestrator CLI — drives SPIR, ASPIR, AIR, TICK, and BUGFIX protocols via a state machine. ALWAYS check this skill before running any `porch` command. Use when you need to check project status, approve gates, signal phase completion, or manage protocol state. Also use when a builder asks about gate approvals or phase transitions. +--- + +# porch - Protocol Orchestrator + +Porch manages the state machine behind development protocols. It tracks phases, gates, consultations, and transitions. + +## Commands + +``` +porch status [id] Show current project state and phase +porch run [id] Run the protocol loop (strict mode) +porch next [id] Get next tasks as JSON +porch done [id] Signal current phase is complete +porch check [id] Run checks for current phase +porch gate [id] Request human approval at a gate +porch approve --a-human-explicitly-approved-this +porch rollback Rewind to an earlier phase +porch init Initialize a new project +``` + +Project ID auto-detects from worktree path when inside a builder worktree. + +## Gate approvals + +Gates are human-only approval checkpoints. The `--a-human-explicitly-approved-this` flag is **required** — it exists to prevent AI agents from auto-approving. + +| Gate | Protocol | When | +|------|----------|------| +| `spec-approval` | SPIR | After spec is written | +| `plan-approval` | SPIR | After plan is written | +| `pr` | SPIR, TICK, AIR | After PR is created | + +```bash +porch approve 42 spec-approval --a-human-explicitly-approved-this +porch approve 42 plan-approval --a-human-explicitly-approved-this +porch approve 42 pr --a-human-explicitly-approved-this +``` + +**ASPIR and BUGFIX have no spec/plan gates** — they run autonomously through those phases. + +## Checking pending gates + +```bash +porch pending # List all gates waiting for approval +``` + +## Critical rules + +- **Builders must NEVER call `porch approve`** — only humans approve gates +- **Never edit `status.yaml` directly** — porch manages all state +- Builders signal completion with `porch done`, not `porch approve` +- `porch run` is for strict mode only — soft mode builders follow the protocol document manually +- When running `porch approve` from the architect, use a subshell if you need worktree context: `(cd /path/to/worktree && porch approve ...)` + +## Language-agnostic checks (`porch.checks` override) + +Protocol check commands default to the **npm** toolchain (`npm run build`, `npm test`, etc.) baked into each `protocol.json`. A non-Node project (Python/uv, Go, Rust) will BLOCK at the first implement-phase check because `npm` can never pass there — even when the code is green under its real test runner. + +**You do NOT need to fork `protocol.json`.** Override check commands per-name in `.codev/config.json` under `porch.checks`. Each entry takes `command` (replace), `cwd` (replace), and/or `skip: true`. This covers both phase checks **and** the `phase_completion` predicates (`build_succeeds` / `tests_pass`). + +```jsonc +// .codev/config.json — Python/uv example +{ + "porch": { + "checks": { + "build": { "command": "uv build" }, + "test": { "command": "uv run pytest" }, + "build_succeeds": { "command": "uv build" }, + "tests_pass": { "command": "uv run pytest" }, + "e2e_tests": { "skip": true } + } + } +} +``` + +Keys are the **check names** from the protocol: `build` / `test` / `e2e_tests` live in phases; `build_succeeds` / `tests_pass` live in `phase_completion`. An override key that matches no check in the protocol prints a warning. Shipped by Spec #550. + +## State storage + +Project state lives in `codev/projects/-/status.yaml`, managed automatically by porch. The status file tracks current phase, gate states, consultation results, and timestamps. diff --git a/.codex/skills/skill-creator/LICENSE.txt b/.codex/skills/skill-creator/LICENSE.txt new file mode 100644 index 000000000..7a4a3ea24 --- /dev/null +++ b/.codex/skills/skill-creator/LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/.codex/skills/skill-creator/SKILL.md b/.codex/skills/skill-creator/SKILL.md new file mode 100644 index 000000000..942bfe896 --- /dev/null +++ b/.codex/skills/skill-creator/SKILL.md @@ -0,0 +1,479 @@ +--- +name: skill-creator +description: Create new skills, modify and improve existing skills, and measure skill performance. Use when users want to create a skill from scratch, update or optimize an existing skill, run evals to test a skill, benchmark skill performance with variance analysis, or optimize a skill's description for better triggering accuracy. +--- + +# Skill Creator + +A skill for creating new skills and iteratively improving them. + +At a high level, the process of creating a skill goes like this: + +- Decide what you want the skill to do and roughly how it should do it +- Write a draft of the skill +- Create a few test prompts and run claude-with-access-to-the-skill on them +- Help the user evaluate the results both qualitatively and quantitatively + - While the runs happen in the background, draft some quantitative evals if there aren't any (if there are some, you can either use as is or modify if you feel something needs to change about them). Then explain them to the user (or if they already existed, explain the ones that already exist) + - Use the `eval-viewer/generate_review.py` script to show the user the results for them to look at, and also let them look at the quantitative metrics +- Rewrite the skill based on feedback from the user's evaluation of the results (and also if there are any glaring flaws that become apparent from the quantitative benchmarks) +- Repeat until you're satisfied +- Expand the test set and try again at larger scale + +Your job when using this skill is to figure out where the user is in this process and then jump in and help them progress through these stages. So for instance, maybe they're like "I want to make a skill for X". You can help narrow down what they mean, write a draft, write the test cases, figure out how they want to evaluate, run all the prompts, and repeat. + +On the other hand, maybe they already have a draft of the skill. In this case you can go straight to the eval/iterate part of the loop. + +Of course, you should always be flexible and if the user is like "I don't need to run a bunch of evaluations, just vibe with me", you can do that instead. + +Then after the skill is done (but again, the order is flexible), you can also run the skill description improver, which we have a whole separate script for, to optimize the triggering of the skill. + +Cool? Cool. + +## Communicating with the user + +The skill creator is liable to be used by people across a wide range of familiarity with coding jargon. If you haven't heard (and how could you, it's only very recently that it started), there's a trend now where the power of Claude is inspiring plumbers to open up their terminals, parents and grandparents to google "how to install npm". On the other hand, the bulk of users are probably fairly computer-literate. + +So please pay attention to context cues to understand how to phrase your communication! In the default case, just to give you some idea: + +- "evaluation" and "benchmark" are borderline, but OK +- for "JSON" and "assertion" you want to see serious cues from the user that they know what those things are before using them without explaining them + +It's OK to briefly explain terms if you're in doubt, and feel free to clarify terms with a short definition if you're unsure if the user will get it. + +--- + +## Creating a skill + +### Capture Intent + +Start by understanding the user's intent. The current conversation might already contain a workflow the user wants to capture (e.g., they say "turn this into a skill"). If so, extract answers from the conversation history first — the tools used, the sequence of steps, corrections the user made, input/output formats observed. The user may need to fill the gaps, and should confirm before proceeding to the next step. + +1. What should this skill enable Claude to do? +2. When should this skill trigger? (what user phrases/contexts) +3. What's the expected output format? +4. Should we set up test cases to verify the skill works? Skills with objectively verifiable outputs (file transforms, data extraction, code generation, fixed workflow steps) benefit from test cases. Skills with subjective outputs (writing style, art) often don't need them. Suggest the appropriate default based on the skill type, but let the user decide. + +### Interview and Research + +Proactively ask questions about edge cases, input/output formats, example files, success criteria, and dependencies. Wait to write test prompts until you've got this part ironed out. + +Check available MCPs - if useful for research (searching docs, finding similar skills, looking up best practices), research in parallel via subagents if available, otherwise inline. Come prepared with context to reduce burden on the user. + +### Write the SKILL.md + +Based on the user interview, fill in these components: + +- **name**: Skill identifier +- **description**: When to trigger, what it does. This is the primary triggering mechanism - include both what the skill does AND specific contexts for when to use it. All "when to use" info goes here, not in the body. Note: currently Claude has a tendency to "undertrigger" skills -- to not use them when they'd be useful. To combat this, please make the skill descriptions a little bit "pushy". So for instance, instead of "How to build a simple fast dashboard to display internal Anthropic data.", you might write "How to build a simple fast dashboard to display internal Anthropic data. Make sure to use this skill whenever the user mentions dashboards, data visualization, internal metrics, or wants to display any kind of company data, even if they don't explicitly ask for a 'dashboard.'" +- **compatibility**: Required tools, dependencies (optional, rarely needed) +- **the rest of the skill :)** + +### Skill Writing Guide + +#### Anatomy of a Skill + +``` +skill-name/ +├── SKILL.md (required) +│ ├── YAML frontmatter (name, description required) +│ └── Markdown instructions +└── Bundled Resources (optional) + ├── scripts/ - Executable code for deterministic/repetitive tasks + ├── references/ - Docs loaded into context as needed + └── assets/ - Files used in output (templates, icons, fonts) +``` + +#### Progressive Disclosure + +Skills use a three-level loading system: +1. **Metadata** (name + description) - Always in context (~100 words) +2. **SKILL.md body** - In context whenever skill triggers (<500 lines ideal) +3. **Bundled resources** - As needed (unlimited, scripts can execute without loading) + +These word counts are approximate and you can feel free to go longer if needed. + +**Key patterns:** +- Keep SKILL.md under 500 lines; if you're approaching this limit, add an additional layer of hierarchy along with clear pointers about where the model using the skill should go next to follow up. +- Reference files clearly from SKILL.md with guidance on when to read them +- For large reference files (>300 lines), include a table of contents + +**Domain organization**: When a skill supports multiple domains/frameworks, organize by variant: +``` +cloud-deploy/ +├── SKILL.md (workflow + selection) +└── references/ + ├── aws.md + ├── gcp.md + └── azure.md +``` +Claude reads only the relevant reference file. + +#### Principle of Lack of Surprise + +This goes without saying, but skills must not contain malware, exploit code, or any content that could compromise system security. A skill's contents should not surprise the user in their intent if described. Don't go along with requests to create misleading skills or skills designed to facilitate unauthorized access, data exfiltration, or other malicious activities. Things like a "roleplay as an XYZ" are OK though. + +#### Writing Patterns + +Prefer using the imperative form in instructions. + +**Defining output formats** - You can do it like this: +```markdown +## Report structure +ALWAYS use this exact template: +# [Title] +## Executive summary +## Key findings +## Recommendations +``` + +**Examples pattern** - It's useful to include examples. You can format them like this (but if "Input" and "Output" are in the examples you might want to deviate a little): +```markdown +## Commit message format +**Example 1:** +Input: Added user authentication with JWT tokens +Output: feat(auth): implement JWT-based authentication +``` + +### Writing Style + +Try to explain to the model why things are important in lieu of heavy-handed musty MUSTs. Use theory of mind and try to make the skill general and not super-narrow to specific examples. Start by writing a draft and then look at it with fresh eyes and improve it. + +### Test Cases + +After writing the skill draft, come up with 2-3 realistic test prompts — the kind of thing a real user would actually say. Share them with the user: [you don't have to use this exact language] "Here are a few test cases I'd like to try. Do these look right, or do you want to add more?" Then run them. + +Save test cases to `evals/evals.json`. Don't write assertions yet — just the prompts. You'll draft assertions in the next step while the runs are in progress. + +```json +{ + "skill_name": "example-skill", + "evals": [ + { + "id": 1, + "prompt": "User's task prompt", + "expected_output": "Description of expected result", + "files": [] + } + ] +} +``` + +See `references/schemas.md` for the full schema (including the `assertions` field, which you'll add later). + +## Running and evaluating test cases + +This section is one continuous sequence — don't stop partway through. Do NOT use `/skill-test` or any other testing skill. + +Put results in `-workspace/` as a sibling to the skill directory. Within the workspace, organize results by iteration (`iteration-1/`, `iteration-2/`, etc.) and within that, each test case gets a directory (`eval-0/`, `eval-1/`, etc.). Don't create all of this upfront — just create directories as you go. + +### Step 1: Spawn all runs (with-skill AND baseline) in the same turn + +For each test case, spawn two subagents in the same turn — one with the skill, one without. This is important: don't spawn the with-skill runs first and then come back for baselines later. Launch everything at once so it all finishes around the same time. + +**With-skill run:** + +``` +Execute this task: +- Skill path: +- Task: +- Input files: +- Save outputs to: /iteration-/eval-/with_skill/outputs/ +- Outputs to save: +``` + +**Baseline run** (same prompt, but the baseline depends on context): +- **Creating a new skill**: no skill at all. Same prompt, no skill path, save to `without_skill/outputs/`. +- **Improving an existing skill**: the old version. Before editing, snapshot the skill (`cp -r /skill-snapshot/`), then point the baseline subagent at the snapshot. Save to `old_skill/outputs/`. + +Write an `eval_metadata.json` for each test case (assertions can be empty for now). Give each eval a descriptive name based on what it's testing — not just "eval-0". Use this name for the directory too. If this iteration uses new or modified eval prompts, create these files for each new eval directory — don't assume they carry over from previous iterations. + +```json +{ + "eval_id": 0, + "eval_name": "descriptive-name-here", + "prompt": "The user's task prompt", + "assertions": [] +} +``` + +### Step 2: While runs are in progress, draft assertions + +Don't just wait for the runs to finish — you can use this time productively. Draft quantitative assertions for each test case and explain them to the user. If assertions already exist in `evals/evals.json`, review them and explain what they check. + +Good assertions are objectively verifiable and have descriptive names — they should read clearly in the benchmark viewer so someone glancing at the results immediately understands what each one checks. Subjective skills (writing style, design quality) are better evaluated qualitatively — don't force assertions onto things that need human judgment. + +Update the `eval_metadata.json` files and `evals/evals.json` with the assertions once drafted. Also explain to the user what they'll see in the viewer — both the qualitative outputs and the quantitative benchmark. + +### Step 3: As runs complete, capture timing data + +When each subagent task completes, you receive a notification containing `total_tokens` and `duration_ms`. Save this data immediately to `timing.json` in the run directory: + +```json +{ + "total_tokens": 84852, + "duration_ms": 23332, + "total_duration_seconds": 23.3 +} +``` + +This is the only opportunity to capture this data — it comes through the task notification and isn't persisted elsewhere. Process each notification as it arrives rather than trying to batch them. + +### Step 4: Grade, aggregate, and launch the viewer + +Once all runs are done: + +1. **Grade each run** — spawn a grader subagent (or grade inline) that reads `agents/grader.md` and evaluates each assertion against the outputs. Save results to `grading.json` in each run directory. The grading.json expectations array must use the fields `text`, `passed`, and `evidence` (not `name`/`met`/`details` or other variants) — the viewer depends on these exact field names. For assertions that can be checked programmatically, write and run a script rather than eyeballing it — scripts are faster, more reliable, and can be reused across iterations. + +2. **Aggregate into benchmark** — run the aggregation script from the skill-creator directory: + ```bash + python -m scripts.aggregate_benchmark /iteration-N --skill-name + ``` + This produces `benchmark.json` and `benchmark.md` with pass_rate, time, and tokens for each configuration, with mean ± stddev and the delta. If generating benchmark.json manually, see `references/schemas.md` for the exact schema the viewer expects. +Put each with_skill version before its baseline counterpart. + +3. **Do an analyst pass** — read the benchmark data and surface patterns the aggregate stats might hide. See `agents/analyzer.md` (the "Analyzing Benchmark Results" section) for what to look for — things like assertions that always pass regardless of skill (non-discriminating), high-variance evals (possibly flaky), and time/token tradeoffs. + +4. **Launch the viewer** with both qualitative outputs and quantitative data: + ```bash + nohup python /eval-viewer/generate_review.py \ + /iteration-N \ + --skill-name "my-skill" \ + --benchmark /iteration-N/benchmark.json \ + > /dev/null 2>&1 & + VIEWER_PID=$! + ``` + For iteration 2+, also pass `--previous-workspace /iteration-`. + + **Cowork / headless environments:** If `webbrowser.open()` is not available or the environment has no display, use `--static ` to write a standalone HTML file instead of starting a server. Feedback will be downloaded as a `feedback.json` file when the user clicks "Submit All Reviews". After download, copy `feedback.json` into the workspace directory for the next iteration to pick up. + +Note: please use generate_review.py to create the viewer; there's no need to write custom HTML. + +5. **Tell the user** something like: "I've opened the results in your browser. There are two tabs — 'Outputs' lets you click through each test case and leave feedback, 'Benchmark' shows the quantitative comparison. When you're done, come back here and let me know." + +### What the user sees in the viewer + +The "Outputs" tab shows one test case at a time: +- **Prompt**: the task that was given +- **Output**: the files the skill produced, rendered inline where possible +- **Previous Output** (iteration 2+): collapsed section showing last iteration's output +- **Formal Grades** (if grading was run): collapsed section showing assertion pass/fail +- **Feedback**: a textbox that auto-saves as they type +- **Previous Feedback** (iteration 2+): their comments from last time, shown below the textbox + +The "Benchmark" tab shows the stats summary: pass rates, timing, and token usage for each configuration, with per-eval breakdowns and analyst observations. + +Navigation is via prev/next buttons or arrow keys. When done, they click "Submit All Reviews" which saves all feedback to `feedback.json`. + +### Step 5: Read the feedback + +When the user tells you they're done, read `feedback.json`: + +```json +{ + "reviews": [ + {"run_id": "eval-0-with_skill", "feedback": "the chart is missing axis labels", "timestamp": "..."}, + {"run_id": "eval-1-with_skill", "feedback": "", "timestamp": "..."}, + {"run_id": "eval-2-with_skill", "feedback": "perfect, love this", "timestamp": "..."} + ], + "status": "complete" +} +``` + +Empty feedback means the user thought it was fine. Focus your improvements on the test cases where the user had specific complaints. + +Kill the viewer server when you're done with it: + +```bash +kill $VIEWER_PID 2>/dev/null +``` + +--- + +## Improving the skill + +This is the heart of the loop. You've run the test cases, the user has reviewed the results, and now you need to make the skill better based on their feedback. + +### How to think about improvements + +1. **Generalize from the feedback.** The big picture thing that's happening here is that we're trying to create skills that can be used a million times (maybe literally, maybe even more who knows) across many different prompts. Here you and the user are iterating on only a few examples over and over again because it helps move faster. The user knows these examples in and out and it's quick for them to assess new outputs. But if the skill you and the user are codeveloping works only for those examples, it's useless. Rather than put in fiddly overfitty changes, or oppressively constrictive MUSTs, if there's some stubborn issue, you might try branching out and using different metaphors, or recommending different patterns of working. It's relatively cheap to try and maybe you'll land on something great. + +2. **Keep the prompt lean.** Remove things that aren't pulling their weight. Make sure to read the transcripts, not just the final outputs — if it looks like the skill is making the model waste a bunch of time doing things that are unproductive, you can try getting rid of the parts of the skill that are making it do that and seeing what happens. + +3. **Explain the why.** Try hard to explain the **why** behind everything you're asking the model to do. Today's LLMs are *smart*. They have good theory of mind and when given a good harness can go beyond rote instructions and really make things happen. Even if the feedback from the user is terse or frustrated, try to actually understand the task and why the user is writing what they wrote, and what they actually wrote, and then transmit this understanding into the instructions. If you find yourself writing ALWAYS or NEVER in all caps, or using super rigid structures, that's a yellow flag — if possible, reframe and explain the reasoning so that the model understands why the thing you're asking for is important. That's a more humane, powerful, and effective approach. + +4. **Look for repeated work across test cases.** Read the transcripts from the test runs and notice if the subagents all independently wrote similar helper scripts or took the same multi-step approach to something. If all 3 test cases resulted in the subagent writing a `create_docx.py` or a `build_chart.py`, that's a strong signal the skill should bundle that script. Write it once, put it in `scripts/`, and tell the skill to use it. This saves every future invocation from reinventing the wheel. + +This task is pretty important (we are trying to create billions a year in economic value here!) and your thinking time is not the blocker; take your time and really mull things over. I'd suggest writing a draft revision and then looking at it anew and making improvements. Really do your best to get into the head of the user and understand what they want and need. + +### The iteration loop + +After improving the skill: + +1. Apply your improvements to the skill +2. Rerun all test cases into a new `iteration-/` directory, including baseline runs. If you're creating a new skill, the baseline is always `without_skill` (no skill) — that stays the same across iterations. If you're improving an existing skill, use your judgment on what makes sense as the baseline: the original version the user came in with, or the previous iteration. +3. Launch the reviewer with `--previous-workspace` pointing at the previous iteration +4. Wait for the user to review and tell you they're done +5. Read the new feedback, improve again, repeat + +Keep going until: +- The user says they're happy +- The feedback is all empty (everything looks good) +- You're not making meaningful progress + +--- + +## Advanced: Blind comparison + +For situations where you want a more rigorous comparison between two versions of a skill (e.g., the user asks "is the new version actually better?"), there's a blind comparison system. Read `agents/comparator.md` and `agents/analyzer.md` for the details. The basic idea is: give two outputs to an independent agent without telling it which is which, and let it judge quality. Then analyze why the winner won. + +This is optional, requires subagents, and most users won't need it. The human review loop is usually sufficient. + +--- + +## Description Optimization + +The description field in SKILL.md frontmatter is the primary mechanism that determines whether Claude invokes a skill. After creating or improving a skill, offer to optimize the description for better triggering accuracy. + +### Step 1: Generate trigger eval queries + +Create 20 eval queries — a mix of should-trigger and should-not-trigger. Save as JSON: + +```json +[ + {"query": "the user prompt", "should_trigger": true}, + {"query": "another prompt", "should_trigger": false} +] +``` + +The queries must be realistic and something a Claude Code or Claude.ai user would actually type. Not abstract requests, but requests that are concrete and specific and have a good amount of detail. For instance, file paths, personal context about the user's job or situation, column names and values, company names, URLs. A little bit of backstory. Some might be in lowercase or contain abbreviations or typos or casual speech. Use a mix of different lengths, and focus on edge cases rather than making them clear-cut (the user will get a chance to sign off on them). + +Bad: `"Format this data"`, `"Extract text from PDF"`, `"Create a chart"` + +Good: `"ok so my boss just sent me this xlsx file (its in my downloads, called something like 'Q4 sales final FINAL v2.xlsx') and she wants me to add a column that shows the profit margin as a percentage. The revenue is in column C and costs are in column D i think"` + +For the **should-trigger** queries (8-10), think about coverage. You want different phrasings of the same intent — some formal, some casual. Include cases where the user doesn't explicitly name the skill or file type but clearly needs it. Throw in some uncommon use cases and cases where this skill competes with another but should win. + +For the **should-not-trigger** queries (8-10), the most valuable ones are the near-misses — queries that share keywords or concepts with the skill but actually need something different. Think adjacent domains, ambiguous phrasing where a naive keyword match would trigger but shouldn't, and cases where the query touches on something the skill does but in a context where another tool is more appropriate. + +The key thing to avoid: don't make should-not-trigger queries obviously irrelevant. "Write a fibonacci function" as a negative test for a PDF skill is too easy — it doesn't test anything. The negative cases should be genuinely tricky. + +### Step 2: Review with user + +Present the eval set to the user for review using the HTML template: + +1. Read the template from `assets/eval_review.html` +2. Replace the placeholders: + - `__EVAL_DATA_PLACEHOLDER__` → the JSON array of eval items (no quotes around it — it's a JS variable assignment) + - `__SKILL_NAME_PLACEHOLDER__` → the skill's name + - `__SKILL_DESCRIPTION_PLACEHOLDER__` → the skill's current description +3. Write to a temp file (e.g., `/tmp/eval_review_.html`) and open it: `open /tmp/eval_review_.html` +4. The user can edit queries, toggle should-trigger, add/remove entries, then click "Export Eval Set" +5. The file downloads to `~/Downloads/eval_set.json` — check the Downloads folder for the most recent version in case there are multiple (e.g., `eval_set (1).json`) + +This step matters — bad eval queries lead to bad descriptions. + +### Step 3: Run the optimization loop + +Tell the user: "This will take some time — I'll run the optimization loop in the background and check on it periodically." + +Save the eval set to the workspace, then run in the background: + +```bash +python -m scripts.run_loop \ + --eval-set \ + --skill-path \ + --model \ + --max-iterations 5 \ + --verbose +``` + +Use the model ID from your system prompt (the one powering the current session) so the triggering test matches what the user actually experiences. + +While it runs, periodically tail the output to give the user updates on which iteration it's on and what the scores look like. + +This handles the full optimization loop automatically. It splits the eval set into 60% train and 40% held-out test, evaluates the current description (running each query 3 times to get a reliable trigger rate), then calls Claude with extended thinking to propose improvements based on what failed. It re-evaluates each new description on both train and test, iterating up to 5 times. When it's done, it opens an HTML report in the browser showing the results per iteration and returns JSON with `best_description` — selected by test score rather than train score to avoid overfitting. + +### How skill triggering works + +Understanding the triggering mechanism helps design better eval queries. Skills appear in Claude's `available_skills` list with their name + description, and Claude decides whether to consult a skill based on that description. The important thing to know is that Claude only consults skills for tasks it can't easily handle on its own — simple, one-step queries like "read this PDF" may not trigger a skill even if the description matches perfectly, because Claude can handle them directly with basic tools. Complex, multi-step, or specialized queries reliably trigger skills when the description matches. + +This means your eval queries should be substantive enough that Claude would actually benefit from consulting a skill. Simple queries like "read file X" are poor test cases — they won't trigger skills regardless of description quality. + +### Step 4: Apply the result + +Take `best_description` from the JSON output and update the skill's SKILL.md frontmatter. Show the user before/after and report the scores. + +--- + +### Package and Present (only if `present_files` tool is available) + +Check whether you have access to the `present_files` tool. If you don't, skip this step. If you do, package the skill and present the .skill file to the user: + +```bash +python -m scripts.package_skill +``` + +After packaging, direct the user to the resulting `.skill` file path so they can install it. + +--- + +## Claude.ai-specific instructions + +In Claude.ai, the core workflow is the same (draft → test → review → improve → repeat), but because Claude.ai doesn't have subagents, some mechanics change. Here's what to adapt: + +**Running test cases**: No subagents means no parallel execution. For each test case, read the skill's SKILL.md, then follow its instructions to accomplish the test prompt yourself. Do them one at a time. This is less rigorous than independent subagents (you wrote the skill and you're also running it, so you have full context), but it's a useful sanity check — and the human review step compensates. Skip the baseline runs — just use the skill to complete the task as requested. + +**Reviewing results**: If you can't open a browser (e.g., Claude.ai's VM has no display, or you're on a remote server), skip the browser reviewer entirely. Instead, present results directly in the conversation. For each test case, show the prompt and the output. If the output is a file the user needs to see (like a .docx or .xlsx), save it to the filesystem and tell them where it is so they can download and inspect it. Ask for feedback inline: "How does this look? Anything you'd change?" + +**Benchmarking**: Skip the quantitative benchmarking — it relies on baseline comparisons which aren't meaningful without subagents. Focus on qualitative feedback from the user. + +**The iteration loop**: Same as before — improve the skill, rerun the test cases, ask for feedback — just without the browser reviewer in the middle. You can still organize results into iteration directories on the filesystem if you have one. + +**Description optimization**: This section requires the `claude` CLI tool (specifically `claude -p`) which is only available in Claude Code. Skip it if you're on Claude.ai. + +**Blind comparison**: Requires subagents. Skip it. + +**Packaging**: The `package_skill.py` script works anywhere with Python and a filesystem. On Claude.ai, you can run it and the user can download the resulting `.skill` file. + +--- + +## Cowork-Specific Instructions + +If you're in Cowork, the main things to know are: + +- You have subagents, so the main workflow (spawn test cases in parallel, run baselines, grade, etc.) all works. (However, if you run into severe problems with timeouts, it's OK to run the test prompts in series rather than parallel.) +- You don't have a browser or display, so when generating the eval viewer, use `--static ` to write a standalone HTML file instead of starting a server. Then proffer a link that the user can click to open the HTML in their browser. +- For whatever reason, the Cowork setup seems to disincline Claude from generating the eval viewer after running the tests, so just to reiterate: whether you're in Cowork or in Claude Code, after running tests, you should always generate the eval viewer for the human to look at examples before revising the skill yourself and trying to make corrections, using `generate_review.py` (not writing your own boutique html code). Sorry in advance but I'm gonna go all caps here: GENERATE THE EVAL VIEWER *BEFORE* evaluating inputs yourself. You want to get them in front of the human ASAP! +- Feedback works differently: since there's no running server, the viewer's "Submit All Reviews" button will download `feedback.json` as a file. You can then read it from there (you may have to request access first). +- Packaging works — `package_skill.py` just needs Python and a filesystem. +- Description optimization (`run_loop.py` / `run_eval.py`) should work in Cowork just fine since it uses `claude -p` via subprocess, not a browser, but please save it until you've fully finished making the skill and the user agrees it's in good shape. + +--- + +## Reference files + +The agents/ directory contains instructions for specialized subagents. Read them when you need to spawn the relevant subagent. + +- `agents/grader.md` — How to evaluate assertions against outputs +- `agents/comparator.md` — How to do blind A/B comparison between two outputs +- `agents/analyzer.md` — How to analyze why one version beat another + +The references/ directory has additional documentation: +- `references/schemas.md` — JSON structures for evals.json, grading.json, etc. + +--- + +Repeating one more time the core loop here for emphasis: + +- Figure out what the skill is about +- Draft or edit the skill +- Run claude-with-access-to-the-skill on test prompts +- With the user, evaluate the outputs: + - Create benchmark.json and run `eval-viewer/generate_review.py` to help the user review them + - Run quantitative evals +- Repeat until you and the user are satisfied +- Package the final skill and return it to the user. + +Please add steps to your TodoList, if you have such a thing, to make sure you don't forget. If you're in Cowork, please specifically put "Create evals JSON and run `eval-viewer/generate_review.py` so human can review test cases" in your TodoList to make sure it happens. + +Good luck! diff --git a/.codex/skills/skill-creator/agents/analyzer.md b/.codex/skills/skill-creator/agents/analyzer.md new file mode 100644 index 000000000..14e41d606 --- /dev/null +++ b/.codex/skills/skill-creator/agents/analyzer.md @@ -0,0 +1,274 @@ +# Post-hoc Analyzer Agent + +Analyze blind comparison results to understand WHY the winner won and generate improvement suggestions. + +## Role + +After the blind comparator determines a winner, the Post-hoc Analyzer "unblids" the results by examining the skills and transcripts. The goal is to extract actionable insights: what made the winner better, and how can the loser be improved? + +## Inputs + +You receive these parameters in your prompt: + +- **winner**: "A" or "B" (from blind comparison) +- **winner_skill_path**: Path to the skill that produced the winning output +- **winner_transcript_path**: Path to the execution transcript for the winner +- **loser_skill_path**: Path to the skill that produced the losing output +- **loser_transcript_path**: Path to the execution transcript for the loser +- **comparison_result_path**: Path to the blind comparator's output JSON +- **output_path**: Where to save the analysis results + +## Process + +### Step 1: Read Comparison Result + +1. Read the blind comparator's output at comparison_result_path +2. Note the winning side (A or B), the reasoning, and any scores +3. Understand what the comparator valued in the winning output + +### Step 2: Read Both Skills + +1. Read the winner skill's SKILL.md and key referenced files +2. Read the loser skill's SKILL.md and key referenced files +3. Identify structural differences: + - Instructions clarity and specificity + - Script/tool usage patterns + - Example coverage + - Edge case handling + +### Step 3: Read Both Transcripts + +1. Read the winner's transcript +2. Read the loser's transcript +3. Compare execution patterns: + - How closely did each follow their skill's instructions? + - What tools were used differently? + - Where did the loser diverge from optimal behavior? + - Did either encounter errors or make recovery attempts? + +### Step 4: Analyze Instruction Following + +For each transcript, evaluate: +- Did the agent follow the skill's explicit instructions? +- Did the agent use the skill's provided tools/scripts? +- Were there missed opportunities to leverage skill content? +- Did the agent add unnecessary steps not in the skill? + +Score instruction following 1-10 and note specific issues. + +### Step 5: Identify Winner Strengths + +Determine what made the winner better: +- Clearer instructions that led to better behavior? +- Better scripts/tools that produced better output? +- More comprehensive examples that guided edge cases? +- Better error handling guidance? + +Be specific. Quote from skills/transcripts where relevant. + +### Step 6: Identify Loser Weaknesses + +Determine what held the loser back: +- Ambiguous instructions that led to suboptimal choices? +- Missing tools/scripts that forced workarounds? +- Gaps in edge case coverage? +- Poor error handling that caused failures? + +### Step 7: Generate Improvement Suggestions + +Based on the analysis, produce actionable suggestions for improving the loser skill: +- Specific instruction changes to make +- Tools/scripts to add or modify +- Examples to include +- Edge cases to address + +Prioritize by impact. Focus on changes that would have changed the outcome. + +### Step 8: Write Analysis Results + +Save structured analysis to `{output_path}`. + +## Output Format + +Write a JSON file with this structure: + +```json +{ + "comparison_summary": { + "winner": "A", + "winner_skill": "path/to/winner/skill", + "loser_skill": "path/to/loser/skill", + "comparator_reasoning": "Brief summary of why comparator chose winner" + }, + "winner_strengths": [ + "Clear step-by-step instructions for handling multi-page documents", + "Included validation script that caught formatting errors", + "Explicit guidance on fallback behavior when OCR fails" + ], + "loser_weaknesses": [ + "Vague instruction 'process the document appropriately' led to inconsistent behavior", + "No script for validation, agent had to improvise and made errors", + "No guidance on OCR failure, agent gave up instead of trying alternatives" + ], + "instruction_following": { + "winner": { + "score": 9, + "issues": [ + "Minor: skipped optional logging step" + ] + }, + "loser": { + "score": 6, + "issues": [ + "Did not use the skill's formatting template", + "Invented own approach instead of following step 3", + "Missed the 'always validate output' instruction" + ] + } + }, + "improvement_suggestions": [ + { + "priority": "high", + "category": "instructions", + "suggestion": "Replace 'process the document appropriately' with explicit steps: 1) Extract text, 2) Identify sections, 3) Format per template", + "expected_impact": "Would eliminate ambiguity that caused inconsistent behavior" + }, + { + "priority": "high", + "category": "tools", + "suggestion": "Add validate_output.py script similar to winner skill's validation approach", + "expected_impact": "Would catch formatting errors before final output" + }, + { + "priority": "medium", + "category": "error_handling", + "suggestion": "Add fallback instructions: 'If OCR fails, try: 1) different resolution, 2) image preprocessing, 3) manual extraction'", + "expected_impact": "Would prevent early failure on difficult documents" + } + ], + "transcript_insights": { + "winner_execution_pattern": "Read skill -> Followed 5-step process -> Used validation script -> Fixed 2 issues -> Produced output", + "loser_execution_pattern": "Read skill -> Unclear on approach -> Tried 3 different methods -> No validation -> Output had errors" + } +} +``` + +## Guidelines + +- **Be specific**: Quote from skills and transcripts, don't just say "instructions were unclear" +- **Be actionable**: Suggestions should be concrete changes, not vague advice +- **Focus on skill improvements**: The goal is to improve the losing skill, not critique the agent +- **Prioritize by impact**: Which changes would most likely have changed the outcome? +- **Consider causation**: Did the skill weakness actually cause the worse output, or is it incidental? +- **Stay objective**: Analyze what happened, don't editorialize +- **Think about generalization**: Would this improvement help on other evals too? + +## Categories for Suggestions + +Use these categories to organize improvement suggestions: + +| Category | Description | +|----------|-------------| +| `instructions` | Changes to the skill's prose instructions | +| `tools` | Scripts, templates, or utilities to add/modify | +| `examples` | Example inputs/outputs to include | +| `error_handling` | Guidance for handling failures | +| `structure` | Reorganization of skill content | +| `references` | External docs or resources to add | + +## Priority Levels + +- **high**: Would likely change the outcome of this comparison +- **medium**: Would improve quality but may not change win/loss +- **low**: Nice to have, marginal improvement + +--- + +# Analyzing Benchmark Results + +When analyzing benchmark results, the analyzer's purpose is to **surface patterns and anomalies** across multiple runs, not suggest skill improvements. + +## Role + +Review all benchmark run results and generate freeform notes that help the user understand skill performance. Focus on patterns that wouldn't be visible from aggregate metrics alone. + +## Inputs + +You receive these parameters in your prompt: + +- **benchmark_data_path**: Path to the in-progress benchmark.json with all run results +- **skill_path**: Path to the skill being benchmarked +- **output_path**: Where to save the notes (as JSON array of strings) + +## Process + +### Step 1: Read Benchmark Data + +1. Read the benchmark.json containing all run results +2. Note the configurations tested (with_skill, without_skill) +3. Understand the run_summary aggregates already calculated + +### Step 2: Analyze Per-Assertion Patterns + +For each expectation across all runs: +- Does it **always pass** in both configurations? (may not differentiate skill value) +- Does it **always fail** in both configurations? (may be broken or beyond capability) +- Does it **always pass with skill but fail without**? (skill clearly adds value here) +- Does it **always fail with skill but pass without**? (skill may be hurting) +- Is it **highly variable**? (flaky expectation or non-deterministic behavior) + +### Step 3: Analyze Cross-Eval Patterns + +Look for patterns across evals: +- Are certain eval types consistently harder/easier? +- Do some evals show high variance while others are stable? +- Are there surprising results that contradict expectations? + +### Step 4: Analyze Metrics Patterns + +Look at time_seconds, tokens, tool_calls: +- Does the skill significantly increase execution time? +- Is there high variance in resource usage? +- Are there outlier runs that skew the aggregates? + +### Step 5: Generate Notes + +Write freeform observations as a list of strings. Each note should: +- State a specific observation +- Be grounded in the data (not speculation) +- Help the user understand something the aggregate metrics don't show + +Examples: +- "Assertion 'Output is a PDF file' passes 100% in both configurations - may not differentiate skill value" +- "Eval 3 shows high variance (50% ± 40%) - run 2 had an unusual failure that may be flaky" +- "Without-skill runs consistently fail on table extraction expectations (0% pass rate)" +- "Skill adds 13s average execution time but improves pass rate by 50%" +- "Token usage is 80% higher with skill, primarily due to script output parsing" +- "All 3 without-skill runs for eval 1 produced empty output" + +### Step 6: Write Notes + +Save notes to `{output_path}` as a JSON array of strings: + +```json +[ + "Assertion 'Output is a PDF file' passes 100% in both configurations - may not differentiate skill value", + "Eval 3 shows high variance (50% ± 40%) - run 2 had an unusual failure", + "Without-skill runs consistently fail on table extraction expectations", + "Skill adds 13s average execution time but improves pass rate by 50%" +] +``` + +## Guidelines + +**DO:** +- Report what you observe in the data +- Be specific about which evals, expectations, or runs you're referring to +- Note patterns that aggregate metrics would hide +- Provide context that helps interpret the numbers + +**DO NOT:** +- Suggest improvements to the skill (that's for the improvement step, not benchmarking) +- Make subjective quality judgments ("the output was good/bad") +- Speculate about causes without evidence +- Repeat information already in the run_summary aggregates diff --git a/.codex/skills/skill-creator/agents/comparator.md b/.codex/skills/skill-creator/agents/comparator.md new file mode 100644 index 000000000..80e00eb45 --- /dev/null +++ b/.codex/skills/skill-creator/agents/comparator.md @@ -0,0 +1,202 @@ +# Blind Comparator Agent + +Compare two outputs WITHOUT knowing which skill produced them. + +## Role + +The Blind Comparator judges which output better accomplishes the eval task. You receive two outputs labeled A and B, but you do NOT know which skill produced which. This prevents bias toward a particular skill or approach. + +Your judgment is based purely on output quality and task completion. + +## Inputs + +You receive these parameters in your prompt: + +- **output_a_path**: Path to the first output file or directory +- **output_b_path**: Path to the second output file or directory +- **eval_prompt**: The original task/prompt that was executed +- **expectations**: List of expectations to check (optional - may be empty) + +## Process + +### Step 1: Read Both Outputs + +1. Examine output A (file or directory) +2. Examine output B (file or directory) +3. Note the type, structure, and content of each +4. If outputs are directories, examine all relevant files inside + +### Step 2: Understand the Task + +1. Read the eval_prompt carefully +2. Identify what the task requires: + - What should be produced? + - What qualities matter (accuracy, completeness, format)? + - What would distinguish a good output from a poor one? + +### Step 3: Generate Evaluation Rubric + +Based on the task, generate a rubric with two dimensions: + +**Content Rubric** (what the output contains): +| Criterion | 1 (Poor) | 3 (Acceptable) | 5 (Excellent) | +|-----------|----------|----------------|---------------| +| Correctness | Major errors | Minor errors | Fully correct | +| Completeness | Missing key elements | Mostly complete | All elements present | +| Accuracy | Significant inaccuracies | Minor inaccuracies | Accurate throughout | + +**Structure Rubric** (how the output is organized): +| Criterion | 1 (Poor) | 3 (Acceptable) | 5 (Excellent) | +|-----------|----------|----------------|---------------| +| Organization | Disorganized | Reasonably organized | Clear, logical structure | +| Formatting | Inconsistent/broken | Mostly consistent | Professional, polished | +| Usability | Difficult to use | Usable with effort | Easy to use | + +Adapt criteria to the specific task. For example: +- PDF form → "Field alignment", "Text readability", "Data placement" +- Document → "Section structure", "Heading hierarchy", "Paragraph flow" +- Data output → "Schema correctness", "Data types", "Completeness" + +### Step 4: Evaluate Each Output Against the Rubric + +For each output (A and B): + +1. **Score each criterion** on the rubric (1-5 scale) +2. **Calculate dimension totals**: Content score, Structure score +3. **Calculate overall score**: Average of dimension scores, scaled to 1-10 + +### Step 5: Check Assertions (if provided) + +If expectations are provided: + +1. Check each expectation against output A +2. Check each expectation against output B +3. Count pass rates for each output +4. Use expectation scores as secondary evidence (not the primary decision factor) + +### Step 6: Determine the Winner + +Compare A and B based on (in priority order): + +1. **Primary**: Overall rubric score (content + structure) +2. **Secondary**: Assertion pass rates (if applicable) +3. **Tiebreaker**: If truly equal, declare a TIE + +Be decisive - ties should be rare. One output is usually better, even if marginally. + +### Step 7: Write Comparison Results + +Save results to a JSON file at the path specified (or `comparison.json` if not specified). + +## Output Format + +Write a JSON file with this structure: + +```json +{ + "winner": "A", + "reasoning": "Output A provides a complete solution with proper formatting and all required fields. Output B is missing the date field and has formatting inconsistencies.", + "rubric": { + "A": { + "content": { + "correctness": 5, + "completeness": 5, + "accuracy": 4 + }, + "structure": { + "organization": 4, + "formatting": 5, + "usability": 4 + }, + "content_score": 4.7, + "structure_score": 4.3, + "overall_score": 9.0 + }, + "B": { + "content": { + "correctness": 3, + "completeness": 2, + "accuracy": 3 + }, + "structure": { + "organization": 3, + "formatting": 2, + "usability": 3 + }, + "content_score": 2.7, + "structure_score": 2.7, + "overall_score": 5.4 + } + }, + "output_quality": { + "A": { + "score": 9, + "strengths": ["Complete solution", "Well-formatted", "All fields present"], + "weaknesses": ["Minor style inconsistency in header"] + }, + "B": { + "score": 5, + "strengths": ["Readable output", "Correct basic structure"], + "weaknesses": ["Missing date field", "Formatting inconsistencies", "Partial data extraction"] + } + }, + "expectation_results": { + "A": { + "passed": 4, + "total": 5, + "pass_rate": 0.80, + "details": [ + {"text": "Output includes name", "passed": true}, + {"text": "Output includes date", "passed": true}, + {"text": "Format is PDF", "passed": true}, + {"text": "Contains signature", "passed": false}, + {"text": "Readable text", "passed": true} + ] + }, + "B": { + "passed": 3, + "total": 5, + "pass_rate": 0.60, + "details": [ + {"text": "Output includes name", "passed": true}, + {"text": "Output includes date", "passed": false}, + {"text": "Format is PDF", "passed": true}, + {"text": "Contains signature", "passed": false}, + {"text": "Readable text", "passed": true} + ] + } + } +} +``` + +If no expectations were provided, omit the `expectation_results` field entirely. + +## Field Descriptions + +- **winner**: "A", "B", or "TIE" +- **reasoning**: Clear explanation of why the winner was chosen (or why it's a tie) +- **rubric**: Structured rubric evaluation for each output + - **content**: Scores for content criteria (correctness, completeness, accuracy) + - **structure**: Scores for structure criteria (organization, formatting, usability) + - **content_score**: Average of content criteria (1-5) + - **structure_score**: Average of structure criteria (1-5) + - **overall_score**: Combined score scaled to 1-10 +- **output_quality**: Summary quality assessment + - **score**: 1-10 rating (should match rubric overall_score) + - **strengths**: List of positive aspects + - **weaknesses**: List of issues or shortcomings +- **expectation_results**: (Only if expectations provided) + - **passed**: Number of expectations that passed + - **total**: Total number of expectations + - **pass_rate**: Fraction passed (0.0 to 1.0) + - **details**: Individual expectation results + +## Guidelines + +- **Stay blind**: DO NOT try to infer which skill produced which output. Judge purely on output quality. +- **Be specific**: Cite specific examples when explaining strengths and weaknesses. +- **Be decisive**: Choose a winner unless outputs are genuinely equivalent. +- **Output quality first**: Assertion scores are secondary to overall task completion. +- **Be objective**: Don't favor outputs based on style preferences; focus on correctness and completeness. +- **Explain your reasoning**: The reasoning field should make it clear why you chose the winner. +- **Handle edge cases**: If both outputs fail, pick the one that fails less badly. If both are excellent, pick the one that's marginally better. diff --git a/.codex/skills/skill-creator/agents/grader.md b/.codex/skills/skill-creator/agents/grader.md new file mode 100644 index 000000000..558ab05c0 --- /dev/null +++ b/.codex/skills/skill-creator/agents/grader.md @@ -0,0 +1,223 @@ +# Grader Agent + +Evaluate expectations against an execution transcript and outputs. + +## Role + +The Grader reviews a transcript and output files, then determines whether each expectation passes or fails. Provide clear evidence for each judgment. + +You have two jobs: grade the outputs, and critique the evals themselves. A passing grade on a weak assertion is worse than useless — it creates false confidence. When you notice an assertion that's trivially satisfied, or an important outcome that no assertion checks, say so. + +## Inputs + +You receive these parameters in your prompt: + +- **expectations**: List of expectations to evaluate (strings) +- **transcript_path**: Path to the execution transcript (markdown file) +- **outputs_dir**: Directory containing output files from execution + +## Process + +### Step 1: Read the Transcript + +1. Read the transcript file completely +2. Note the eval prompt, execution steps, and final result +3. Identify any issues or errors documented + +### Step 2: Examine Output Files + +1. List files in outputs_dir +2. Read/examine each file relevant to the expectations. If outputs aren't plain text, use the inspection tools provided in your prompt — don't rely solely on what the transcript says the executor produced. +3. Note contents, structure, and quality + +### Step 3: Evaluate Each Assertion + +For each expectation: + +1. **Search for evidence** in the transcript and outputs +2. **Determine verdict**: + - **PASS**: Clear evidence the expectation is true AND the evidence reflects genuine task completion, not just surface-level compliance + - **FAIL**: No evidence, or evidence contradicts the expectation, or the evidence is superficial (e.g., correct filename but empty/wrong content) +3. **Cite the evidence**: Quote the specific text or describe what you found + +### Step 4: Extract and Verify Claims + +Beyond the predefined expectations, extract implicit claims from the outputs and verify them: + +1. **Extract claims** from the transcript and outputs: + - Factual statements ("The form has 12 fields") + - Process claims ("Used pypdf to fill the form") + - Quality claims ("All fields were filled correctly") + +2. **Verify each claim**: + - **Factual claims**: Can be checked against the outputs or external sources + - **Process claims**: Can be verified from the transcript + - **Quality claims**: Evaluate whether the claim is justified + +3. **Flag unverifiable claims**: Note claims that cannot be verified with available information + +This catches issues that predefined expectations might miss. + +### Step 5: Read User Notes + +If `{outputs_dir}/user_notes.md` exists: +1. Read it and note any uncertainties or issues flagged by the executor +2. Include relevant concerns in the grading output +3. These may reveal problems even when expectations pass + +### Step 6: Critique the Evals + +After grading, consider whether the evals themselves could be improved. Only surface suggestions when there's a clear gap. + +Good suggestions test meaningful outcomes — assertions that are hard to satisfy without actually doing the work correctly. Think about what makes an assertion *discriminating*: it passes when the skill genuinely succeeds and fails when it doesn't. + +Suggestions worth raising: +- An assertion that passed but would also pass for a clearly wrong output (e.g., checking filename existence but not file content) +- An important outcome you observed — good or bad — that no assertion covers at all +- An assertion that can't actually be verified from the available outputs + +Keep the bar high. The goal is to flag things the eval author would say "good catch" about, not to nitpick every assertion. + +### Step 7: Write Grading Results + +Save results to `{outputs_dir}/../grading.json` (sibling to outputs_dir). + +## Grading Criteria + +**PASS when**: +- The transcript or outputs clearly demonstrate the expectation is true +- Specific evidence can be cited +- The evidence reflects genuine substance, not just surface compliance (e.g., a file exists AND contains correct content, not just the right filename) + +**FAIL when**: +- No evidence found for the expectation +- Evidence contradicts the expectation +- The expectation cannot be verified from available information +- The evidence is superficial — the assertion is technically satisfied but the underlying task outcome is wrong or incomplete +- The output appears to meet the assertion by coincidence rather than by actually doing the work + +**When uncertain**: The burden of proof to pass is on the expectation. + +### Step 8: Read Executor Metrics and Timing + +1. If `{outputs_dir}/metrics.json` exists, read it and include in grading output +2. If `{outputs_dir}/../timing.json` exists, read it and include timing data + +## Output Format + +Write a JSON file with this structure: + +```json +{ + "expectations": [ + { + "text": "The output includes the name 'John Smith'", + "passed": true, + "evidence": "Found in transcript Step 3: 'Extracted names: John Smith, Sarah Johnson'" + }, + { + "text": "The spreadsheet has a SUM formula in cell B10", + "passed": false, + "evidence": "No spreadsheet was created. The output was a text file." + }, + { + "text": "The assistant used the skill's OCR script", + "passed": true, + "evidence": "Transcript Step 2 shows: 'Tool: Bash - python ocr_script.py image.png'" + } + ], + "summary": { + "passed": 2, + "failed": 1, + "total": 3, + "pass_rate": 0.67 + }, + "execution_metrics": { + "tool_calls": { + "Read": 5, + "Write": 2, + "Bash": 8 + }, + "total_tool_calls": 15, + "total_steps": 6, + "errors_encountered": 0, + "output_chars": 12450, + "transcript_chars": 3200 + }, + "timing": { + "executor_duration_seconds": 165.0, + "grader_duration_seconds": 26.0, + "total_duration_seconds": 191.0 + }, + "claims": [ + { + "claim": "The form has 12 fillable fields", + "type": "factual", + "verified": true, + "evidence": "Counted 12 fields in field_info.json" + }, + { + "claim": "All required fields were populated", + "type": "quality", + "verified": false, + "evidence": "Reference section was left blank despite data being available" + } + ], + "user_notes_summary": { + "uncertainties": ["Used 2023 data, may be stale"], + "needs_review": [], + "workarounds": ["Fell back to text overlay for non-fillable fields"] + }, + "eval_feedback": { + "suggestions": [ + { + "assertion": "The output includes the name 'John Smith'", + "reason": "A hallucinated document that mentions the name would also pass — consider checking it appears as the primary contact with matching phone and email from the input" + }, + { + "reason": "No assertion checks whether the extracted phone numbers match the input — I observed incorrect numbers in the output that went uncaught" + } + ], + "overall": "Assertions check presence but not correctness. Consider adding content verification." + } +} +``` + +## Field Descriptions + +- **expectations**: Array of graded expectations + - **text**: The original expectation text + - **passed**: Boolean - true if expectation passes + - **evidence**: Specific quote or description supporting the verdict +- **summary**: Aggregate statistics + - **passed**: Count of passed expectations + - **failed**: Count of failed expectations + - **total**: Total expectations evaluated + - **pass_rate**: Fraction passed (0.0 to 1.0) +- **execution_metrics**: Copied from executor's metrics.json (if available) + - **output_chars**: Total character count of output files (proxy for tokens) + - **transcript_chars**: Character count of transcript +- **timing**: Wall clock timing from timing.json (if available) + - **executor_duration_seconds**: Time spent in executor subagent + - **total_duration_seconds**: Total elapsed time for the run +- **claims**: Extracted and verified claims from the output + - **claim**: The statement being verified + - **type**: "factual", "process", or "quality" + - **verified**: Boolean - whether the claim holds + - **evidence**: Supporting or contradicting evidence +- **user_notes_summary**: Issues flagged by the executor + - **uncertainties**: Things the executor wasn't sure about + - **needs_review**: Items requiring human attention + - **workarounds**: Places where the skill didn't work as expected +- **eval_feedback**: Improvement suggestions for the evals (only when warranted) + - **suggestions**: List of concrete suggestions, each with a `reason` and optionally an `assertion` it relates to + - **overall**: Brief assessment — can be "No suggestions, evals look solid" if nothing to flag + +## Guidelines + +- **Be objective**: Base verdicts on evidence, not assumptions +- **Be specific**: Quote the exact text that supports your verdict +- **Be thorough**: Check both transcript and output files +- **Be consistent**: Apply the same standard to each expectation +- **Explain failures**: Make it clear why evidence was insufficient +- **No partial credit**: Each expectation is pass or fail, not partial diff --git a/.codex/skills/skill-creator/assets/eval_review.html b/.codex/skills/skill-creator/assets/eval_review.html new file mode 100644 index 000000000..938ff32ae --- /dev/null +++ b/.codex/skills/skill-creator/assets/eval_review.html @@ -0,0 +1,146 @@ + + + + + + Eval Set Review - __SKILL_NAME_PLACEHOLDER__ + + + + + + +

Eval Set Review: __SKILL_NAME_PLACEHOLDER__

+

Current description: __SKILL_DESCRIPTION_PLACEHOLDER__

+ +
+ + +
+ + + + + + + + + + +
QueryShould TriggerActions
+ +

+ + + + diff --git a/.codex/skills/skill-creator/eval-viewer/generate_review.py b/.codex/skills/skill-creator/eval-viewer/generate_review.py new file mode 100644 index 000000000..7fa597863 --- /dev/null +++ b/.codex/skills/skill-creator/eval-viewer/generate_review.py @@ -0,0 +1,471 @@ +#!/usr/bin/env python3 +"""Generate and serve a review page for eval results. + +Reads the workspace directory, discovers runs (directories with outputs/), +embeds all output data into a self-contained HTML page, and serves it via +a tiny HTTP server. Feedback auto-saves to feedback.json in the workspace. + +Usage: + python generate_review.py [--port PORT] [--skill-name NAME] + python generate_review.py --previous-feedback /path/to/old/feedback.json + +No dependencies beyond the Python stdlib are required. +""" + +import argparse +import base64 +import json +import mimetypes +import os +import re +import signal +import subprocess +import sys +import time +import webbrowser +from functools import partial +from http.server import HTTPServer, BaseHTTPRequestHandler +from pathlib import Path + +# Files to exclude from output listings +METADATA_FILES = {"transcript.md", "user_notes.md", "metrics.json"} + +# Extensions we render as inline text +TEXT_EXTENSIONS = { + ".txt", ".md", ".json", ".csv", ".py", ".js", ".ts", ".tsx", ".jsx", + ".yaml", ".yml", ".xml", ".html", ".css", ".sh", ".rb", ".go", ".rs", + ".java", ".c", ".cpp", ".h", ".hpp", ".sql", ".r", ".toml", +} + +# Extensions we render as inline images +IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".gif", ".svg", ".webp"} + +# MIME type overrides for common types +MIME_OVERRIDES = { + ".svg": "image/svg+xml", + ".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + ".pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation", +} + + +def get_mime_type(path: Path) -> str: + ext = path.suffix.lower() + if ext in MIME_OVERRIDES: + return MIME_OVERRIDES[ext] + mime, _ = mimetypes.guess_type(str(path)) + return mime or "application/octet-stream" + + +def find_runs(workspace: Path) -> list[dict]: + """Recursively find directories that contain an outputs/ subdirectory.""" + runs: list[dict] = [] + _find_runs_recursive(workspace, workspace, runs) + runs.sort(key=lambda r: (r.get("eval_id", float("inf")), r["id"])) + return runs + + +def _find_runs_recursive(root: Path, current: Path, runs: list[dict]) -> None: + if not current.is_dir(): + return + + outputs_dir = current / "outputs" + if outputs_dir.is_dir(): + run = build_run(root, current) + if run: + runs.append(run) + return + + skip = {"node_modules", ".git", "__pycache__", "skill", "inputs"} + for child in sorted(current.iterdir()): + if child.is_dir() and child.name not in skip: + _find_runs_recursive(root, child, runs) + + +def build_run(root: Path, run_dir: Path) -> dict | None: + """Build a run dict with prompt, outputs, and grading data.""" + prompt = "" + eval_id = None + + # Try eval_metadata.json + for candidate in [run_dir / "eval_metadata.json", run_dir.parent / "eval_metadata.json"]: + if candidate.exists(): + try: + metadata = json.loads(candidate.read_text()) + prompt = metadata.get("prompt", "") + eval_id = metadata.get("eval_id") + except (json.JSONDecodeError, OSError): + pass + if prompt: + break + + # Fall back to transcript.md + if not prompt: + for candidate in [run_dir / "transcript.md", run_dir / "outputs" / "transcript.md"]: + if candidate.exists(): + try: + text = candidate.read_text() + match = re.search(r"## Eval Prompt\n\n([\s\S]*?)(?=\n##|$)", text) + if match: + prompt = match.group(1).strip() + except OSError: + pass + if prompt: + break + + if not prompt: + prompt = "(No prompt found)" + + run_id = str(run_dir.relative_to(root)).replace("/", "-").replace("\\", "-") + + # Collect output files + outputs_dir = run_dir / "outputs" + output_files: list[dict] = [] + if outputs_dir.is_dir(): + for f in sorted(outputs_dir.iterdir()): + if f.is_file() and f.name not in METADATA_FILES: + output_files.append(embed_file(f)) + + # Load grading if present + grading = None + for candidate in [run_dir / "grading.json", run_dir.parent / "grading.json"]: + if candidate.exists(): + try: + grading = json.loads(candidate.read_text()) + except (json.JSONDecodeError, OSError): + pass + if grading: + break + + return { + "id": run_id, + "prompt": prompt, + "eval_id": eval_id, + "outputs": output_files, + "grading": grading, + } + + +def embed_file(path: Path) -> dict: + """Read a file and return an embedded representation.""" + ext = path.suffix.lower() + mime = get_mime_type(path) + + if ext in TEXT_EXTENSIONS: + try: + content = path.read_text(errors="replace") + except OSError: + content = "(Error reading file)" + return { + "name": path.name, + "type": "text", + "content": content, + } + elif ext in IMAGE_EXTENSIONS: + try: + raw = path.read_bytes() + b64 = base64.b64encode(raw).decode("ascii") + except OSError: + return {"name": path.name, "type": "error", "content": "(Error reading file)"} + return { + "name": path.name, + "type": "image", + "mime": mime, + "data_uri": f"data:{mime};base64,{b64}", + } + elif ext == ".pdf": + try: + raw = path.read_bytes() + b64 = base64.b64encode(raw).decode("ascii") + except OSError: + return {"name": path.name, "type": "error", "content": "(Error reading file)"} + return { + "name": path.name, + "type": "pdf", + "data_uri": f"data:{mime};base64,{b64}", + } + elif ext == ".xlsx": + try: + raw = path.read_bytes() + b64 = base64.b64encode(raw).decode("ascii") + except OSError: + return {"name": path.name, "type": "error", "content": "(Error reading file)"} + return { + "name": path.name, + "type": "xlsx", + "data_b64": b64, + } + else: + # Binary / unknown — base64 download link + try: + raw = path.read_bytes() + b64 = base64.b64encode(raw).decode("ascii") + except OSError: + return {"name": path.name, "type": "error", "content": "(Error reading file)"} + return { + "name": path.name, + "type": "binary", + "mime": mime, + "data_uri": f"data:{mime};base64,{b64}", + } + + +def load_previous_iteration(workspace: Path) -> dict[str, dict]: + """Load previous iteration's feedback and outputs. + + Returns a map of run_id -> {"feedback": str, "outputs": list[dict]}. + """ + result: dict[str, dict] = {} + + # Load feedback + feedback_map: dict[str, str] = {} + feedback_path = workspace / "feedback.json" + if feedback_path.exists(): + try: + data = json.loads(feedback_path.read_text()) + feedback_map = { + r["run_id"]: r["feedback"] + for r in data.get("reviews", []) + if r.get("feedback", "").strip() + } + except (json.JSONDecodeError, OSError, KeyError): + pass + + # Load runs (to get outputs) + prev_runs = find_runs(workspace) + for run in prev_runs: + result[run["id"]] = { + "feedback": feedback_map.get(run["id"], ""), + "outputs": run.get("outputs", []), + } + + # Also add feedback for run_ids that had feedback but no matching run + for run_id, fb in feedback_map.items(): + if run_id not in result: + result[run_id] = {"feedback": fb, "outputs": []} + + return result + + +def generate_html( + runs: list[dict], + skill_name: str, + previous: dict[str, dict] | None = None, + benchmark: dict | None = None, +) -> str: + """Generate the complete standalone HTML page with embedded data.""" + template_path = Path(__file__).parent / "viewer.html" + template = template_path.read_text() + + # Build previous_feedback and previous_outputs maps for the template + previous_feedback: dict[str, str] = {} + previous_outputs: dict[str, list[dict]] = {} + if previous: + for run_id, data in previous.items(): + if data.get("feedback"): + previous_feedback[run_id] = data["feedback"] + if data.get("outputs"): + previous_outputs[run_id] = data["outputs"] + + embedded = { + "skill_name": skill_name, + "runs": runs, + "previous_feedback": previous_feedback, + "previous_outputs": previous_outputs, + } + if benchmark: + embedded["benchmark"] = benchmark + + data_json = json.dumps(embedded) + + return template.replace("/*__EMBEDDED_DATA__*/", f"const EMBEDDED_DATA = {data_json};") + + +# --------------------------------------------------------------------------- +# HTTP server (stdlib only, zero dependencies) +# --------------------------------------------------------------------------- + +def _kill_port(port: int) -> None: + """Kill any process listening on the given port.""" + try: + result = subprocess.run( + ["lsof", "-ti", f":{port}"], + capture_output=True, text=True, timeout=5, + ) + for pid_str in result.stdout.strip().split("\n"): + if pid_str.strip(): + try: + os.kill(int(pid_str.strip()), signal.SIGTERM) + except (ProcessLookupError, ValueError): + pass + if result.stdout.strip(): + time.sleep(0.5) + except subprocess.TimeoutExpired: + pass + except FileNotFoundError: + print("Note: lsof not found, cannot check if port is in use", file=sys.stderr) + +class ReviewHandler(BaseHTTPRequestHandler): + """Serves the review HTML and handles feedback saves. + + Regenerates the HTML on each page load so that refreshing the browser + picks up new eval outputs without restarting the server. + """ + + def __init__( + self, + workspace: Path, + skill_name: str, + feedback_path: Path, + previous: dict[str, dict], + benchmark_path: Path | None, + *args, + **kwargs, + ): + self.workspace = workspace + self.skill_name = skill_name + self.feedback_path = feedback_path + self.previous = previous + self.benchmark_path = benchmark_path + super().__init__(*args, **kwargs) + + def do_GET(self) -> None: + if self.path == "/" or self.path == "/index.html": + # Regenerate HTML on each request (re-scans workspace for new outputs) + runs = find_runs(self.workspace) + benchmark = None + if self.benchmark_path and self.benchmark_path.exists(): + try: + benchmark = json.loads(self.benchmark_path.read_text()) + except (json.JSONDecodeError, OSError): + pass + html = generate_html(runs, self.skill_name, self.previous, benchmark) + content = html.encode("utf-8") + self.send_response(200) + self.send_header("Content-Type", "text/html; charset=utf-8") + self.send_header("Content-Length", str(len(content))) + self.end_headers() + self.wfile.write(content) + elif self.path == "/api/feedback": + data = b"{}" + if self.feedback_path.exists(): + data = self.feedback_path.read_bytes() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + else: + self.send_error(404) + + def do_POST(self) -> None: + if self.path == "/api/feedback": + length = int(self.headers.get("Content-Length", 0)) + body = self.rfile.read(length) + try: + data = json.loads(body) + if not isinstance(data, dict) or "reviews" not in data: + raise ValueError("Expected JSON object with 'reviews' key") + self.feedback_path.write_text(json.dumps(data, indent=2) + "\n") + resp = b'{"ok":true}' + self.send_response(200) + except (json.JSONDecodeError, OSError, ValueError) as e: + resp = json.dumps({"error": str(e)}).encode() + self.send_response(500) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(resp))) + self.end_headers() + self.wfile.write(resp) + else: + self.send_error(404) + + def log_message(self, format: str, *args: object) -> None: + # Suppress request logging to keep terminal clean + pass + + +def main() -> None: + parser = argparse.ArgumentParser(description="Generate and serve eval review") + parser.add_argument("workspace", type=Path, help="Path to workspace directory") + parser.add_argument("--port", "-p", type=int, default=3117, help="Server port (default: 3117)") + parser.add_argument("--skill-name", "-n", type=str, default=None, help="Skill name for header") + parser.add_argument( + "--previous-workspace", type=Path, default=None, + help="Path to previous iteration's workspace (shows old outputs and feedback as context)", + ) + parser.add_argument( + "--benchmark", type=Path, default=None, + help="Path to benchmark.json to show in the Benchmark tab", + ) + parser.add_argument( + "--static", "-s", type=Path, default=None, + help="Write standalone HTML to this path instead of starting a server", + ) + args = parser.parse_args() + + workspace = args.workspace.resolve() + if not workspace.is_dir(): + print(f"Error: {workspace} is not a directory", file=sys.stderr) + sys.exit(1) + + runs = find_runs(workspace) + if not runs: + print(f"No runs found in {workspace}", file=sys.stderr) + sys.exit(1) + + skill_name = args.skill_name or workspace.name.replace("-workspace", "") + feedback_path = workspace / "feedback.json" + + previous: dict[str, dict] = {} + if args.previous_workspace: + previous = load_previous_iteration(args.previous_workspace.resolve()) + + benchmark_path = args.benchmark.resolve() if args.benchmark else None + benchmark = None + if benchmark_path and benchmark_path.exists(): + try: + benchmark = json.loads(benchmark_path.read_text()) + except (json.JSONDecodeError, OSError): + pass + + if args.static: + html = generate_html(runs, skill_name, previous, benchmark) + args.static.parent.mkdir(parents=True, exist_ok=True) + args.static.write_text(html) + print(f"\n Static viewer written to: {args.static}\n") + sys.exit(0) + + # Kill any existing process on the target port + port = args.port + _kill_port(port) + handler = partial(ReviewHandler, workspace, skill_name, feedback_path, previous, benchmark_path) + try: + server = HTTPServer(("127.0.0.1", port), handler) + except OSError: + # Port still in use after kill attempt — find a free one + server = HTTPServer(("127.0.0.1", 0), handler) + port = server.server_address[1] + + url = f"http://localhost:{port}" + print(f"\n Eval Viewer") + print(f" ─────────────────────────────────") + print(f" URL: {url}") + print(f" Workspace: {workspace}") + print(f" Feedback: {feedback_path}") + if previous: + print(f" Previous: {args.previous_workspace} ({len(previous)} runs)") + if benchmark_path: + print(f" Benchmark: {benchmark_path}") + print(f"\n Press Ctrl+C to stop.\n") + + webbrowser.open(url) + + try: + server.serve_forever() + except KeyboardInterrupt: + print("\nStopped.") + server.server_close() + + +if __name__ == "__main__": + main() diff --git a/.codex/skills/skill-creator/eval-viewer/viewer.html b/.codex/skills/skill-creator/eval-viewer/viewer.html new file mode 100644 index 000000000..6d8e96348 --- /dev/null +++ b/.codex/skills/skill-creator/eval-viewer/viewer.html @@ -0,0 +1,1325 @@ + + + + + + Eval Review + + + + + + + +
+
+
+

Eval Review:

+
Review each output and leave feedback below. Navigate with arrow keys or buttons. When done, copy feedback and paste into Claude Code.
+
+
+
+ + + + + +
+
+ +
+
Prompt
+
+
+
+
+ + +
+
Output
+
+
No output files found
+
+
+ + + + + + + + +
+
Your Feedback
+
+ + + +
+
+
+ + +
+ + +
+
+
No benchmark data available. Run a benchmark to see quantitative results here.
+
+
+
+ + +
+
+

Review Complete

+

Your feedback has been saved. Go back to your Claude Code session and tell Claude you're done reviewing.

+
+ +
+
+
+ + +
+ + + + diff --git a/.codex/skills/skill-creator/references/schemas.md b/.codex/skills/skill-creator/references/schemas.md new file mode 100644 index 000000000..b6eeaa2d4 --- /dev/null +++ b/.codex/skills/skill-creator/references/schemas.md @@ -0,0 +1,430 @@ +# JSON Schemas + +This document defines the JSON schemas used by skill-creator. + +--- + +## evals.json + +Defines the evals for a skill. Located at `evals/evals.json` within the skill directory. + +```json +{ + "skill_name": "example-skill", + "evals": [ + { + "id": 1, + "prompt": "User's example prompt", + "expected_output": "Description of expected result", + "files": ["evals/files/sample1.pdf"], + "expectations": [ + "The output includes X", + "The skill used script Y" + ] + } + ] +} +``` + +**Fields:** +- `skill_name`: Name matching the skill's frontmatter +- `evals[].id`: Unique integer identifier +- `evals[].prompt`: The task to execute +- `evals[].expected_output`: Human-readable description of success +- `evals[].files`: Optional list of input file paths (relative to skill root) +- `evals[].expectations`: List of verifiable statements + +--- + +## history.json + +Tracks version progression in Improve mode. Located at workspace root. + +```json +{ + "started_at": "2026-01-15T10:30:00Z", + "skill_name": "pdf", + "current_best": "v2", + "iterations": [ + { + "version": "v0", + "parent": null, + "expectation_pass_rate": 0.65, + "grading_result": "baseline", + "is_current_best": false + }, + { + "version": "v1", + "parent": "v0", + "expectation_pass_rate": 0.75, + "grading_result": "won", + "is_current_best": false + }, + { + "version": "v2", + "parent": "v1", + "expectation_pass_rate": 0.85, + "grading_result": "won", + "is_current_best": true + } + ] +} +``` + +**Fields:** +- `started_at`: ISO timestamp of when improvement started +- `skill_name`: Name of the skill being improved +- `current_best`: Version identifier of the best performer +- `iterations[].version`: Version identifier (v0, v1, ...) +- `iterations[].parent`: Parent version this was derived from +- `iterations[].expectation_pass_rate`: Pass rate from grading +- `iterations[].grading_result`: "baseline", "won", "lost", or "tie" +- `iterations[].is_current_best`: Whether this is the current best version + +--- + +## grading.json + +Output from the grader agent. Located at `/grading.json`. + +```json +{ + "expectations": [ + { + "text": "The output includes the name 'John Smith'", + "passed": true, + "evidence": "Found in transcript Step 3: 'Extracted names: John Smith, Sarah Johnson'" + }, + { + "text": "The spreadsheet has a SUM formula in cell B10", + "passed": false, + "evidence": "No spreadsheet was created. The output was a text file." + } + ], + "summary": { + "passed": 2, + "failed": 1, + "total": 3, + "pass_rate": 0.67 + }, + "execution_metrics": { + "tool_calls": { + "Read": 5, + "Write": 2, + "Bash": 8 + }, + "total_tool_calls": 15, + "total_steps": 6, + "errors_encountered": 0, + "output_chars": 12450, + "transcript_chars": 3200 + }, + "timing": { + "executor_duration_seconds": 165.0, + "grader_duration_seconds": 26.0, + "total_duration_seconds": 191.0 + }, + "claims": [ + { + "claim": "The form has 12 fillable fields", + "type": "factual", + "verified": true, + "evidence": "Counted 12 fields in field_info.json" + } + ], + "user_notes_summary": { + "uncertainties": ["Used 2023 data, may be stale"], + "needs_review": [], + "workarounds": ["Fell back to text overlay for non-fillable fields"] + }, + "eval_feedback": { + "suggestions": [ + { + "assertion": "The output includes the name 'John Smith'", + "reason": "A hallucinated document that mentions the name would also pass" + } + ], + "overall": "Assertions check presence but not correctness." + } +} +``` + +**Fields:** +- `expectations[]`: Graded expectations with evidence +- `summary`: Aggregate pass/fail counts +- `execution_metrics`: Tool usage and output size (from executor's metrics.json) +- `timing`: Wall clock timing (from timing.json) +- `claims`: Extracted and verified claims from the output +- `user_notes_summary`: Issues flagged by the executor +- `eval_feedback`: (optional) Improvement suggestions for the evals, only present when the grader identifies issues worth raising + +--- + +## metrics.json + +Output from the executor agent. Located at `/outputs/metrics.json`. + +```json +{ + "tool_calls": { + "Read": 5, + "Write": 2, + "Bash": 8, + "Edit": 1, + "Glob": 2, + "Grep": 0 + }, + "total_tool_calls": 18, + "total_steps": 6, + "files_created": ["filled_form.pdf", "field_values.json"], + "errors_encountered": 0, + "output_chars": 12450, + "transcript_chars": 3200 +} +``` + +**Fields:** +- `tool_calls`: Count per tool type +- `total_tool_calls`: Sum of all tool calls +- `total_steps`: Number of major execution steps +- `files_created`: List of output files created +- `errors_encountered`: Number of errors during execution +- `output_chars`: Total character count of output files +- `transcript_chars`: Character count of transcript + +--- + +## timing.json + +Wall clock timing for a run. Located at `/timing.json`. + +**How to capture:** When a subagent task completes, the task notification includes `total_tokens` and `duration_ms`. Save these immediately — they are not persisted anywhere else and cannot be recovered after the fact. + +```json +{ + "total_tokens": 84852, + "duration_ms": 23332, + "total_duration_seconds": 23.3, + "executor_start": "2026-01-15T10:30:00Z", + "executor_end": "2026-01-15T10:32:45Z", + "executor_duration_seconds": 165.0, + "grader_start": "2026-01-15T10:32:46Z", + "grader_end": "2026-01-15T10:33:12Z", + "grader_duration_seconds": 26.0 +} +``` + +--- + +## benchmark.json + +Output from Benchmark mode. Located at `benchmarks//benchmark.json`. + +```json +{ + "metadata": { + "skill_name": "pdf", + "skill_path": "/path/to/pdf", + "executor_model": "claude-sonnet-4-20250514", + "analyzer_model": "most-capable-model", + "timestamp": "2026-01-15T10:30:00Z", + "evals_run": [1, 2, 3], + "runs_per_configuration": 3 + }, + + "runs": [ + { + "eval_id": 1, + "eval_name": "Ocean", + "configuration": "with_skill", + "run_number": 1, + "result": { + "pass_rate": 0.85, + "passed": 6, + "failed": 1, + "total": 7, + "time_seconds": 42.5, + "tokens": 3800, + "tool_calls": 18, + "errors": 0 + }, + "expectations": [ + {"text": "...", "passed": true, "evidence": "..."} + ], + "notes": [ + "Used 2023 data, may be stale", + "Fell back to text overlay for non-fillable fields" + ] + } + ], + + "run_summary": { + "with_skill": { + "pass_rate": {"mean": 0.85, "stddev": 0.05, "min": 0.80, "max": 0.90}, + "time_seconds": {"mean": 45.0, "stddev": 12.0, "min": 32.0, "max": 58.0}, + "tokens": {"mean": 3800, "stddev": 400, "min": 3200, "max": 4100} + }, + "without_skill": { + "pass_rate": {"mean": 0.35, "stddev": 0.08, "min": 0.28, "max": 0.45}, + "time_seconds": {"mean": 32.0, "stddev": 8.0, "min": 24.0, "max": 42.0}, + "tokens": {"mean": 2100, "stddev": 300, "min": 1800, "max": 2500} + }, + "delta": { + "pass_rate": "+0.50", + "time_seconds": "+13.0", + "tokens": "+1700" + } + }, + + "notes": [ + "Assertion 'Output is a PDF file' passes 100% in both configurations - may not differentiate skill value", + "Eval 3 shows high variance (50% ± 40%) - may be flaky or model-dependent", + "Without-skill runs consistently fail on table extraction expectations", + "Skill adds 13s average execution time but improves pass rate by 50%" + ] +} +``` + +**Fields:** +- `metadata`: Information about the benchmark run + - `skill_name`: Name of the skill + - `timestamp`: When the benchmark was run + - `evals_run`: List of eval names or IDs + - `runs_per_configuration`: Number of runs per config (e.g. 3) +- `runs[]`: Individual run results + - `eval_id`: Numeric eval identifier + - `eval_name`: Human-readable eval name (used as section header in the viewer) + - `configuration`: Must be `"with_skill"` or `"without_skill"` (the viewer uses this exact string for grouping and color coding) + - `run_number`: Integer run number (1, 2, 3...) + - `result`: Nested object with `pass_rate`, `passed`, `total`, `time_seconds`, `tokens`, `errors` +- `run_summary`: Statistical aggregates per configuration + - `with_skill` / `without_skill`: Each contains `pass_rate`, `time_seconds`, `tokens` objects with `mean` and `stddev` fields + - `delta`: Difference strings like `"+0.50"`, `"+13.0"`, `"+1700"` +- `notes`: Freeform observations from the analyzer + +**Important:** The viewer reads these field names exactly. Using `config` instead of `configuration`, or putting `pass_rate` at the top level of a run instead of nested under `result`, will cause the viewer to show empty/zero values. Always reference this schema when generating benchmark.json manually. + +--- + +## comparison.json + +Output from blind comparator. Located at `/comparison-N.json`. + +```json +{ + "winner": "A", + "reasoning": "Output A provides a complete solution with proper formatting and all required fields. Output B is missing the date field and has formatting inconsistencies.", + "rubric": { + "A": { + "content": { + "correctness": 5, + "completeness": 5, + "accuracy": 4 + }, + "structure": { + "organization": 4, + "formatting": 5, + "usability": 4 + }, + "content_score": 4.7, + "structure_score": 4.3, + "overall_score": 9.0 + }, + "B": { + "content": { + "correctness": 3, + "completeness": 2, + "accuracy": 3 + }, + "structure": { + "organization": 3, + "formatting": 2, + "usability": 3 + }, + "content_score": 2.7, + "structure_score": 2.7, + "overall_score": 5.4 + } + }, + "output_quality": { + "A": { + "score": 9, + "strengths": ["Complete solution", "Well-formatted", "All fields present"], + "weaknesses": ["Minor style inconsistency in header"] + }, + "B": { + "score": 5, + "strengths": ["Readable output", "Correct basic structure"], + "weaknesses": ["Missing date field", "Formatting inconsistencies", "Partial data extraction"] + } + }, + "expectation_results": { + "A": { + "passed": 4, + "total": 5, + "pass_rate": 0.80, + "details": [ + {"text": "Output includes name", "passed": true} + ] + }, + "B": { + "passed": 3, + "total": 5, + "pass_rate": 0.60, + "details": [ + {"text": "Output includes name", "passed": true} + ] + } + } +} +``` + +--- + +## analysis.json + +Output from post-hoc analyzer. Located at `/analysis.json`. + +```json +{ + "comparison_summary": { + "winner": "A", + "winner_skill": "path/to/winner/skill", + "loser_skill": "path/to/loser/skill", + "comparator_reasoning": "Brief summary of why comparator chose winner" + }, + "winner_strengths": [ + "Clear step-by-step instructions for handling multi-page documents", + "Included validation script that caught formatting errors" + ], + "loser_weaknesses": [ + "Vague instruction 'process the document appropriately' led to inconsistent behavior", + "No script for validation, agent had to improvise" + ], + "instruction_following": { + "winner": { + "score": 9, + "issues": ["Minor: skipped optional logging step"] + }, + "loser": { + "score": 6, + "issues": [ + "Did not use the skill's formatting template", + "Invented own approach instead of following step 3" + ] + } + }, + "improvement_suggestions": [ + { + "priority": "high", + "category": "instructions", + "suggestion": "Replace 'process the document appropriately' with explicit steps", + "expected_impact": "Would eliminate ambiguity that caused inconsistent behavior" + } + ], + "transcript_insights": { + "winner_execution_pattern": "Read skill -> Followed 5-step process -> Used validation script", + "loser_execution_pattern": "Read skill -> Unclear on approach -> Tried 3 different methods" + } +} +``` diff --git a/.codex/skills/skill-creator/scripts/__init__.py b/.codex/skills/skill-creator/scripts/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/.codex/skills/skill-creator/scripts/aggregate_benchmark.py b/.codex/skills/skill-creator/scripts/aggregate_benchmark.py new file mode 100755 index 000000000..3e66e8c10 --- /dev/null +++ b/.codex/skills/skill-creator/scripts/aggregate_benchmark.py @@ -0,0 +1,401 @@ +#!/usr/bin/env python3 +""" +Aggregate individual run results into benchmark summary statistics. + +Reads grading.json files from run directories and produces: +- run_summary with mean, stddev, min, max for each metric +- delta between with_skill and without_skill configurations + +Usage: + python aggregate_benchmark.py + +Example: + python aggregate_benchmark.py benchmarks/2026-01-15T10-30-00/ + +The script supports two directory layouts: + + Workspace layout (from skill-creator iterations): + / + └── eval-N/ + ├── with_skill/ + │ ├── run-1/grading.json + │ └── run-2/grading.json + └── without_skill/ + ├── run-1/grading.json + └── run-2/grading.json + + Legacy layout (with runs/ subdirectory): + / + └── runs/ + └── eval-N/ + ├── with_skill/ + │ └── run-1/grading.json + └── without_skill/ + └── run-1/grading.json +""" + +import argparse +import json +import math +import sys +from datetime import datetime, timezone +from pathlib import Path + + +def calculate_stats(values: list[float]) -> dict: + """Calculate mean, stddev, min, max for a list of values.""" + if not values: + return {"mean": 0.0, "stddev": 0.0, "min": 0.0, "max": 0.0} + + n = len(values) + mean = sum(values) / n + + if n > 1: + variance = sum((x - mean) ** 2 for x in values) / (n - 1) + stddev = math.sqrt(variance) + else: + stddev = 0.0 + + return { + "mean": round(mean, 4), + "stddev": round(stddev, 4), + "min": round(min(values), 4), + "max": round(max(values), 4) + } + + +def load_run_results(benchmark_dir: Path) -> dict: + """ + Load all run results from a benchmark directory. + + Returns dict keyed by config name (e.g. "with_skill"/"without_skill", + or "new_skill"/"old_skill"), each containing a list of run results. + """ + # Support both layouts: eval dirs directly under benchmark_dir, or under runs/ + runs_dir = benchmark_dir / "runs" + if runs_dir.exists(): + search_dir = runs_dir + elif list(benchmark_dir.glob("eval-*")): + search_dir = benchmark_dir + else: + print(f"No eval directories found in {benchmark_dir} or {benchmark_dir / 'runs'}") + return {} + + results: dict[str, list] = {} + + for eval_idx, eval_dir in enumerate(sorted(search_dir.glob("eval-*"))): + metadata_path = eval_dir / "eval_metadata.json" + if metadata_path.exists(): + try: + with open(metadata_path) as mf: + eval_id = json.load(mf).get("eval_id", eval_idx) + except (json.JSONDecodeError, OSError): + eval_id = eval_idx + else: + try: + eval_id = int(eval_dir.name.split("-")[1]) + except ValueError: + eval_id = eval_idx + + # Discover config directories dynamically rather than hardcoding names + for config_dir in sorted(eval_dir.iterdir()): + if not config_dir.is_dir(): + continue + # Skip non-config directories (inputs, outputs, etc.) + if not list(config_dir.glob("run-*")): + continue + config = config_dir.name + if config not in results: + results[config] = [] + + for run_dir in sorted(config_dir.glob("run-*")): + run_number = int(run_dir.name.split("-")[1]) + grading_file = run_dir / "grading.json" + + if not grading_file.exists(): + print(f"Warning: grading.json not found in {run_dir}") + continue + + try: + with open(grading_file) as f: + grading = json.load(f) + except json.JSONDecodeError as e: + print(f"Warning: Invalid JSON in {grading_file}: {e}") + continue + + # Extract metrics + result = { + "eval_id": eval_id, + "run_number": run_number, + "pass_rate": grading.get("summary", {}).get("pass_rate", 0.0), + "passed": grading.get("summary", {}).get("passed", 0), + "failed": grading.get("summary", {}).get("failed", 0), + "total": grading.get("summary", {}).get("total", 0), + } + + # Extract timing — check grading.json first, then sibling timing.json + timing = grading.get("timing", {}) + result["time_seconds"] = timing.get("total_duration_seconds", 0.0) + timing_file = run_dir / "timing.json" + if result["time_seconds"] == 0.0 and timing_file.exists(): + try: + with open(timing_file) as tf: + timing_data = json.load(tf) + result["time_seconds"] = timing_data.get("total_duration_seconds", 0.0) + result["tokens"] = timing_data.get("total_tokens", 0) + except json.JSONDecodeError: + pass + + # Extract metrics if available + metrics = grading.get("execution_metrics", {}) + result["tool_calls"] = metrics.get("total_tool_calls", 0) + if not result.get("tokens"): + result["tokens"] = metrics.get("output_chars", 0) + result["errors"] = metrics.get("errors_encountered", 0) + + # Extract expectations — viewer requires fields: text, passed, evidence + raw_expectations = grading.get("expectations", []) + for exp in raw_expectations: + if "text" not in exp or "passed" not in exp: + print(f"Warning: expectation in {grading_file} missing required fields (text, passed, evidence): {exp}") + result["expectations"] = raw_expectations + + # Extract notes from user_notes_summary + notes_summary = grading.get("user_notes_summary", {}) + notes = [] + notes.extend(notes_summary.get("uncertainties", [])) + notes.extend(notes_summary.get("needs_review", [])) + notes.extend(notes_summary.get("workarounds", [])) + result["notes"] = notes + + results[config].append(result) + + return results + + +def aggregate_results(results: dict) -> dict: + """ + Aggregate run results into summary statistics. + + Returns run_summary with stats for each configuration and delta. + """ + run_summary = {} + configs = list(results.keys()) + + for config in configs: + runs = results.get(config, []) + + if not runs: + run_summary[config] = { + "pass_rate": {"mean": 0.0, "stddev": 0.0, "min": 0.0, "max": 0.0}, + "time_seconds": {"mean": 0.0, "stddev": 0.0, "min": 0.0, "max": 0.0}, + "tokens": {"mean": 0, "stddev": 0, "min": 0, "max": 0} + } + continue + + pass_rates = [r["pass_rate"] for r in runs] + times = [r["time_seconds"] for r in runs] + tokens = [r.get("tokens", 0) for r in runs] + + run_summary[config] = { + "pass_rate": calculate_stats(pass_rates), + "time_seconds": calculate_stats(times), + "tokens": calculate_stats(tokens) + } + + # Calculate delta between the first two configs (if two exist) + if len(configs) >= 2: + primary = run_summary.get(configs[0], {}) + baseline = run_summary.get(configs[1], {}) + else: + primary = run_summary.get(configs[0], {}) if configs else {} + baseline = {} + + delta_pass_rate = primary.get("pass_rate", {}).get("mean", 0) - baseline.get("pass_rate", {}).get("mean", 0) + delta_time = primary.get("time_seconds", {}).get("mean", 0) - baseline.get("time_seconds", {}).get("mean", 0) + delta_tokens = primary.get("tokens", {}).get("mean", 0) - baseline.get("tokens", {}).get("mean", 0) + + run_summary["delta"] = { + "pass_rate": f"{delta_pass_rate:+.2f}", + "time_seconds": f"{delta_time:+.1f}", + "tokens": f"{delta_tokens:+.0f}" + } + + return run_summary + + +def generate_benchmark(benchmark_dir: Path, skill_name: str = "", skill_path: str = "") -> dict: + """ + Generate complete benchmark.json from run results. + """ + results = load_run_results(benchmark_dir) + run_summary = aggregate_results(results) + + # Build runs array for benchmark.json + runs = [] + for config in results: + for result in results[config]: + runs.append({ + "eval_id": result["eval_id"], + "configuration": config, + "run_number": result["run_number"], + "result": { + "pass_rate": result["pass_rate"], + "passed": result["passed"], + "failed": result["failed"], + "total": result["total"], + "time_seconds": result["time_seconds"], + "tokens": result.get("tokens", 0), + "tool_calls": result.get("tool_calls", 0), + "errors": result.get("errors", 0) + }, + "expectations": result["expectations"], + "notes": result["notes"] + }) + + # Determine eval IDs from results + eval_ids = sorted(set( + r["eval_id"] + for config in results.values() + for r in config + )) + + benchmark = { + "metadata": { + "skill_name": skill_name or "", + "skill_path": skill_path or "", + "executor_model": "", + "analyzer_model": "", + "timestamp": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "evals_run": eval_ids, + "runs_per_configuration": 3 + }, + "runs": runs, + "run_summary": run_summary, + "notes": [] # To be filled by analyzer + } + + return benchmark + + +def generate_markdown(benchmark: dict) -> str: + """Generate human-readable benchmark.md from benchmark data.""" + metadata = benchmark["metadata"] + run_summary = benchmark["run_summary"] + + # Determine config names (excluding "delta") + configs = [k for k in run_summary if k != "delta"] + config_a = configs[0] if len(configs) >= 1 else "config_a" + config_b = configs[1] if len(configs) >= 2 else "config_b" + label_a = config_a.replace("_", " ").title() + label_b = config_b.replace("_", " ").title() + + lines = [ + f"# Skill Benchmark: {metadata['skill_name']}", + "", + f"**Model**: {metadata['executor_model']}", + f"**Date**: {metadata['timestamp']}", + f"**Evals**: {', '.join(map(str, metadata['evals_run']))} ({metadata['runs_per_configuration']} runs each per configuration)", + "", + "## Summary", + "", + f"| Metric | {label_a} | {label_b} | Delta |", + "|--------|------------|---------------|-------|", + ] + + a_summary = run_summary.get(config_a, {}) + b_summary = run_summary.get(config_b, {}) + delta = run_summary.get("delta", {}) + + # Format pass rate + a_pr = a_summary.get("pass_rate", {}) + b_pr = b_summary.get("pass_rate", {}) + lines.append(f"| Pass Rate | {a_pr.get('mean', 0)*100:.0f}% ± {a_pr.get('stddev', 0)*100:.0f}% | {b_pr.get('mean', 0)*100:.0f}% ± {b_pr.get('stddev', 0)*100:.0f}% | {delta.get('pass_rate', '—')} |") + + # Format time + a_time = a_summary.get("time_seconds", {}) + b_time = b_summary.get("time_seconds", {}) + lines.append(f"| Time | {a_time.get('mean', 0):.1f}s ± {a_time.get('stddev', 0):.1f}s | {b_time.get('mean', 0):.1f}s ± {b_time.get('stddev', 0):.1f}s | {delta.get('time_seconds', '—')}s |") + + # Format tokens + a_tokens = a_summary.get("tokens", {}) + b_tokens = b_summary.get("tokens", {}) + lines.append(f"| Tokens | {a_tokens.get('mean', 0):.0f} ± {a_tokens.get('stddev', 0):.0f} | {b_tokens.get('mean', 0):.0f} ± {b_tokens.get('stddev', 0):.0f} | {delta.get('tokens', '—')} |") + + # Notes section + if benchmark.get("notes"): + lines.extend([ + "", + "## Notes", + "" + ]) + for note in benchmark["notes"]: + lines.append(f"- {note}") + + return "\n".join(lines) + + +def main(): + parser = argparse.ArgumentParser( + description="Aggregate benchmark run results into summary statistics" + ) + parser.add_argument( + "benchmark_dir", + type=Path, + help="Path to the benchmark directory" + ) + parser.add_argument( + "--skill-name", + default="", + help="Name of the skill being benchmarked" + ) + parser.add_argument( + "--skill-path", + default="", + help="Path to the skill being benchmarked" + ) + parser.add_argument( + "--output", "-o", + type=Path, + help="Output path for benchmark.json (default: /benchmark.json)" + ) + + args = parser.parse_args() + + if not args.benchmark_dir.exists(): + print(f"Directory not found: {args.benchmark_dir}") + sys.exit(1) + + # Generate benchmark + benchmark = generate_benchmark(args.benchmark_dir, args.skill_name, args.skill_path) + + # Determine output paths + output_json = args.output or (args.benchmark_dir / "benchmark.json") + output_md = output_json.with_suffix(".md") + + # Write benchmark.json + with open(output_json, "w") as f: + json.dump(benchmark, f, indent=2) + print(f"Generated: {output_json}") + + # Write benchmark.md + markdown = generate_markdown(benchmark) + with open(output_md, "w") as f: + f.write(markdown) + print(f"Generated: {output_md}") + + # Print summary + run_summary = benchmark["run_summary"] + configs = [k for k in run_summary if k != "delta"] + delta = run_summary.get("delta", {}) + + print(f"\nSummary:") + for config in configs: + pr = run_summary[config]["pass_rate"]["mean"] + label = config.replace("_", " ").title() + print(f" {label}: {pr*100:.1f}% pass rate") + print(f" Delta: {delta.get('pass_rate', '—')}") + + +if __name__ == "__main__": + main() diff --git a/.codex/skills/skill-creator/scripts/generate_report.py b/.codex/skills/skill-creator/scripts/generate_report.py new file mode 100755 index 000000000..959e30a00 --- /dev/null +++ b/.codex/skills/skill-creator/scripts/generate_report.py @@ -0,0 +1,326 @@ +#!/usr/bin/env python3 +"""Generate an HTML report from run_loop.py output. + +Takes the JSON output from run_loop.py and generates a visual HTML report +showing each description attempt with check/x for each test case. +Distinguishes between train and test queries. +""" + +import argparse +import html +import json +import sys +from pathlib import Path + + +def generate_html(data: dict, auto_refresh: bool = False, skill_name: str = "") -> str: + """Generate HTML report from loop output data. If auto_refresh is True, adds a meta refresh tag.""" + history = data.get("history", []) + holdout = data.get("holdout", 0) + title_prefix = html.escape(skill_name + " \u2014 ") if skill_name else "" + + # Get all unique queries from train and test sets, with should_trigger info + train_queries: list[dict] = [] + test_queries: list[dict] = [] + if history: + for r in history[0].get("train_results", history[0].get("results", [])): + train_queries.append({"query": r["query"], "should_trigger": r.get("should_trigger", True)}) + if history[0].get("test_results"): + for r in history[0].get("test_results", []): + test_queries.append({"query": r["query"], "should_trigger": r.get("should_trigger", True)}) + + refresh_tag = ' \n' if auto_refresh else "" + + html_parts = [""" + + + +""" + refresh_tag + """ """ + title_prefix + """Skill Description Optimization + + + + + + +

""" + title_prefix + """Skill Description Optimization

+
+ Optimizing your skill's description. This page updates automatically as Claude tests different versions of your skill's description. Each row is an iteration — a new description attempt. The columns show test queries: green checkmarks mean the skill triggered correctly (or correctly didn't trigger), red crosses mean it got it wrong. The "Train" score shows performance on queries used to improve the description; the "Test" score shows performance on held-out queries the optimizer hasn't seen. When it's done, Claude will apply the best-performing description to your skill. +
+"""] + + # Summary section + best_test_score = data.get('best_test_score') + best_train_score = data.get('best_train_score') + html_parts.append(f""" +
+

Original: {html.escape(data.get('original_description', 'N/A'))}

+

Best: {html.escape(data.get('best_description', 'N/A'))}

+

Best Score: {data.get('best_score', 'N/A')} {'(test)' if best_test_score else '(train)'}

+

Iterations: {data.get('iterations_run', 0)} | Train: {data.get('train_size', '?')} | Test: {data.get('test_size', '?')}

+
+""") + + # Legend + html_parts.append(""" +
+ Query columns: + Should trigger + Should NOT trigger + Train + Test +
+""") + + # Table header + html_parts.append(""" +
+ + + + + + + +""") + + # Add column headers for train queries + for qinfo in train_queries: + polarity = "positive-col" if qinfo["should_trigger"] else "negative-col" + html_parts.append(f' \n') + + # Add column headers for test queries (different color) + for qinfo in test_queries: + polarity = "positive-col" if qinfo["should_trigger"] else "negative-col" + html_parts.append(f' \n') + + html_parts.append(""" + + +""") + + # Find best iteration for highlighting + if test_queries: + best_iter = max(history, key=lambda h: h.get("test_passed") or 0).get("iteration") + else: + best_iter = max(history, key=lambda h: h.get("train_passed", h.get("passed", 0))).get("iteration") + + # Add rows for each iteration + for h in history: + iteration = h.get("iteration", "?") + train_passed = h.get("train_passed", h.get("passed", 0)) + train_total = h.get("train_total", h.get("total", 0)) + test_passed = h.get("test_passed") + test_total = h.get("test_total") + description = h.get("description", "") + train_results = h.get("train_results", h.get("results", [])) + test_results = h.get("test_results", []) + + # Create lookups for results by query + train_by_query = {r["query"]: r for r in train_results} + test_by_query = {r["query"]: r for r in test_results} if test_results else {} + + # Compute aggregate correct/total runs across all retries + def aggregate_runs(results: list[dict]) -> tuple[int, int]: + correct = 0 + total = 0 + for r in results: + runs = r.get("runs", 0) + triggers = r.get("triggers", 0) + total += runs + if r.get("should_trigger", True): + correct += triggers + else: + correct += runs - triggers + return correct, total + + train_correct, train_runs = aggregate_runs(train_results) + test_correct, test_runs = aggregate_runs(test_results) + + # Determine score classes + def score_class(correct: int, total: int) -> str: + if total > 0: + ratio = correct / total + if ratio >= 0.8: + return "score-good" + elif ratio >= 0.5: + return "score-ok" + return "score-bad" + + train_class = score_class(train_correct, train_runs) + test_class = score_class(test_correct, test_runs) + + row_class = "best-row" if iteration == best_iter else "" + + html_parts.append(f""" + + + + +""") + + # Add result for each train query + for qinfo in train_queries: + r = train_by_query.get(qinfo["query"], {}) + did_pass = r.get("pass", False) + triggers = r.get("triggers", 0) + runs = r.get("runs", 0) + + icon = "✓" if did_pass else "✗" + css_class = "pass" if did_pass else "fail" + + html_parts.append(f' \n') + + # Add result for each test query (with different background) + for qinfo in test_queries: + r = test_by_query.get(qinfo["query"], {}) + did_pass = r.get("pass", False) + triggers = r.get("triggers", 0) + runs = r.get("runs", 0) + + icon = "✓" if did_pass else "✗" + css_class = "pass" if did_pass else "fail" + + html_parts.append(f' \n') + + html_parts.append(" \n") + + html_parts.append(""" +
IterTrainTestDescription{html.escape(qinfo["query"])}{html.escape(qinfo["query"])}
{iteration}{train_correct}/{train_runs}{test_correct}/{test_runs}{html.escape(description)}{icon}{triggers}/{runs}{icon}{triggers}/{runs}
+
+""") + + html_parts.append(""" + + +""") + + return "".join(html_parts) + + +def main(): + parser = argparse.ArgumentParser(description="Generate HTML report from run_loop output") + parser.add_argument("input", help="Path to JSON output from run_loop.py (or - for stdin)") + parser.add_argument("-o", "--output", default=None, help="Output HTML file (default: stdout)") + parser.add_argument("--skill-name", default="", help="Skill name to include in the report title") + args = parser.parse_args() + + if args.input == "-": + data = json.load(sys.stdin) + else: + data = json.loads(Path(args.input).read_text()) + + html_output = generate_html(data, skill_name=args.skill_name) + + if args.output: + Path(args.output).write_text(html_output) + print(f"Report written to {args.output}", file=sys.stderr) + else: + print(html_output) + + +if __name__ == "__main__": + main() diff --git a/.codex/skills/skill-creator/scripts/improve_description.py b/.codex/skills/skill-creator/scripts/improve_description.py new file mode 100755 index 000000000..a270777b9 --- /dev/null +++ b/.codex/skills/skill-creator/scripts/improve_description.py @@ -0,0 +1,248 @@ +#!/usr/bin/env python3 +"""Improve a skill description based on eval results. + +Takes eval results (from run_eval.py) and generates an improved description +using Claude with extended thinking. +""" + +import argparse +import json +import re +import sys +from pathlib import Path + +import anthropic + +from scripts.utils import parse_skill_md + + +def improve_description( + client: anthropic.Anthropic, + skill_name: str, + skill_content: str, + current_description: str, + eval_results: dict, + history: list[dict], + model: str, + test_results: dict | None = None, + log_dir: Path | None = None, + iteration: int | None = None, +) -> str: + """Call Claude to improve the description based on eval results.""" + failed_triggers = [ + r for r in eval_results["results"] + if r["should_trigger"] and not r["pass"] + ] + false_triggers = [ + r for r in eval_results["results"] + if not r["should_trigger"] and not r["pass"] + ] + + # Build scores summary + train_score = f"{eval_results['summary']['passed']}/{eval_results['summary']['total']}" + if test_results: + test_score = f"{test_results['summary']['passed']}/{test_results['summary']['total']}" + scores_summary = f"Train: {train_score}, Test: {test_score}" + else: + scores_summary = f"Train: {train_score}" + + prompt = f"""You are optimizing a skill description for a Claude Code skill called "{skill_name}". A "skill" is sort of like a prompt, but with progressive disclosure -- there's a title and description that Claude sees when deciding whether to use the skill, and then if it does use the skill, it reads the .md file which has lots more details and potentially links to other resources in the skill folder like helper files and scripts and additional documentation or examples. + +The description appears in Claude's "available_skills" list. When a user sends a query, Claude decides whether to invoke the skill based solely on the title and on this description. Your goal is to write a description that triggers for relevant queries, and doesn't trigger for irrelevant ones. + +Here's the current description: + +"{current_description}" + + +Current scores ({scores_summary}): + +""" + if failed_triggers: + prompt += "FAILED TO TRIGGER (should have triggered but didn't):\n" + for r in failed_triggers: + prompt += f' - "{r["query"]}" (triggered {r["triggers"]}/{r["runs"]} times)\n' + prompt += "\n" + + if false_triggers: + prompt += "FALSE TRIGGERS (triggered but shouldn't have):\n" + for r in false_triggers: + prompt += f' - "{r["query"]}" (triggered {r["triggers"]}/{r["runs"]} times)\n' + prompt += "\n" + + if history: + prompt += "PREVIOUS ATTEMPTS (do NOT repeat these — try something structurally different):\n\n" + for h in history: + train_s = f"{h.get('train_passed', h.get('passed', 0))}/{h.get('train_total', h.get('total', 0))}" + test_s = f"{h.get('test_passed', '?')}/{h.get('test_total', '?')}" if h.get('test_passed') is not None else None + score_str = f"train={train_s}" + (f", test={test_s}" if test_s else "") + prompt += f'\n' + prompt += f'Description: "{h["description"]}"\n' + if "results" in h: + prompt += "Train results:\n" + for r in h["results"]: + status = "PASS" if r["pass"] else "FAIL" + prompt += f' [{status}] "{r["query"][:80]}" (triggered {r["triggers"]}/{r["runs"]})\n' + if h.get("note"): + prompt += f'Note: {h["note"]}\n' + prompt += "\n\n" + + prompt += f""" + +Skill content (for context on what the skill does): + +{skill_content} + + +Based on the failures, write a new and improved description that is more likely to trigger correctly. When I say "based on the failures", it's a bit of a tricky line to walk because we don't want to overfit to the specific cases you're seeing. So what I DON'T want you to do is produce an ever-expanding list of specific queries that this skill should or shouldn't trigger for. Instead, try to generalize from the failures to broader categories of user intent and situations where this skill would be useful or not useful. The reason for this is twofold: + +1. Avoid overfitting +2. The list might get loooong and it's injected into ALL queries and there might be a lot of skills, so we don't want to blow too much space on any given description. + +Concretely, your description should not be more than about 100-200 words, even if that comes at the cost of accuracy. + +Here are some tips that we've found to work well in writing these descriptions: +- The skill should be phrased in the imperative -- "Use this skill for" rather than "this skill does" +- The skill description should focus on the user's intent, what they are trying to achieve, vs. the implementation details of how the skill works. +- The description competes with other skills for Claude's attention — make it distinctive and immediately recognizable. +- If you're getting lots of failures after repeated attempts, change things up. Try different sentence structures or wordings. + +I'd encourage you to be creative and mix up the style in different iterations since you'll have multiple opportunities to try different approaches and we'll just grab the highest-scoring one at the end. + +Please respond with only the new description text in tags, nothing else.""" + + response = client.messages.create( + model=model, + max_tokens=16000, + thinking={ + "type": "enabled", + "budget_tokens": 10000, + }, + messages=[{"role": "user", "content": prompt}], + ) + + # Extract thinking and text from response + thinking_text = "" + text = "" + for block in response.content: + if block.type == "thinking": + thinking_text = block.thinking + elif block.type == "text": + text = block.text + + # Parse out the tags + match = re.search(r"(.*?)", text, re.DOTALL) + description = match.group(1).strip().strip('"') if match else text.strip().strip('"') + + # Log the transcript + transcript: dict = { + "iteration": iteration, + "prompt": prompt, + "thinking": thinking_text, + "response": text, + "parsed_description": description, + "char_count": len(description), + "over_limit": len(description) > 1024, + } + + # If over 1024 chars, ask the model to shorten it + if len(description) > 1024: + shorten_prompt = f"Your description is {len(description)} characters, which exceeds the hard 1024 character limit. Please rewrite it to be under 1024 characters while preserving the most important trigger words and intent coverage. Respond with only the new description in tags." + shorten_response = client.messages.create( + model=model, + max_tokens=16000, + thinking={ + "type": "enabled", + "budget_tokens": 10000, + }, + messages=[ + {"role": "user", "content": prompt}, + {"role": "assistant", "content": text}, + {"role": "user", "content": shorten_prompt}, + ], + ) + + shorten_thinking = "" + shorten_text = "" + for block in shorten_response.content: + if block.type == "thinking": + shorten_thinking = block.thinking + elif block.type == "text": + shorten_text = block.text + + match = re.search(r"(.*?)", shorten_text, re.DOTALL) + shortened = match.group(1).strip().strip('"') if match else shorten_text.strip().strip('"') + + transcript["rewrite_prompt"] = shorten_prompt + transcript["rewrite_thinking"] = shorten_thinking + transcript["rewrite_response"] = shorten_text + transcript["rewrite_description"] = shortened + transcript["rewrite_char_count"] = len(shortened) + description = shortened + + transcript["final_description"] = description + + if log_dir: + log_dir.mkdir(parents=True, exist_ok=True) + log_file = log_dir / f"improve_iter_{iteration or 'unknown'}.json" + log_file.write_text(json.dumps(transcript, indent=2)) + + return description + + +def main(): + parser = argparse.ArgumentParser(description="Improve a skill description based on eval results") + parser.add_argument("--eval-results", required=True, help="Path to eval results JSON (from run_eval.py)") + parser.add_argument("--skill-path", required=True, help="Path to skill directory") + parser.add_argument("--history", default=None, help="Path to history JSON (previous attempts)") + parser.add_argument("--model", required=True, help="Model for improvement") + parser.add_argument("--verbose", action="store_true", help="Print thinking to stderr") + args = parser.parse_args() + + skill_path = Path(args.skill_path) + if not (skill_path / "SKILL.md").exists(): + print(f"Error: No SKILL.md found at {skill_path}", file=sys.stderr) + sys.exit(1) + + eval_results = json.loads(Path(args.eval_results).read_text()) + history = [] + if args.history: + history = json.loads(Path(args.history).read_text()) + + name, _, content = parse_skill_md(skill_path) + current_description = eval_results["description"] + + if args.verbose: + print(f"Current: {current_description}", file=sys.stderr) + print(f"Score: {eval_results['summary']['passed']}/{eval_results['summary']['total']}", file=sys.stderr) + + client = anthropic.Anthropic() + new_description = improve_description( + client=client, + skill_name=name, + skill_content=content, + current_description=current_description, + eval_results=eval_results, + history=history, + model=args.model, + ) + + if args.verbose: + print(f"Improved: {new_description}", file=sys.stderr) + + # Output as JSON with both the new description and updated history + output = { + "description": new_description, + "history": history + [{ + "description": current_description, + "passed": eval_results["summary"]["passed"], + "failed": eval_results["summary"]["failed"], + "total": eval_results["summary"]["total"], + "results": eval_results["results"], + }], + } + print(json.dumps(output, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/.codex/skills/skill-creator/scripts/package_skill.py b/.codex/skills/skill-creator/scripts/package_skill.py new file mode 100755 index 000000000..f48eac444 --- /dev/null +++ b/.codex/skills/skill-creator/scripts/package_skill.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +""" +Skill Packager - Creates a distributable .skill file of a skill folder + +Usage: + python utils/package_skill.py [output-directory] + +Example: + python utils/package_skill.py skills/public/my-skill + python utils/package_skill.py skills/public/my-skill ./dist +""" + +import fnmatch +import sys +import zipfile +from pathlib import Path +from scripts.quick_validate import validate_skill + +# Patterns to exclude when packaging skills. +EXCLUDE_DIRS = {"__pycache__", "node_modules"} +EXCLUDE_GLOBS = {"*.pyc"} +EXCLUDE_FILES = {".DS_Store"} +# Directories excluded only at the skill root (not when nested deeper). +ROOT_EXCLUDE_DIRS = {"evals"} + + +def should_exclude(rel_path: Path) -> bool: + """Check if a path should be excluded from packaging.""" + parts = rel_path.parts + if any(part in EXCLUDE_DIRS for part in parts): + return True + # rel_path is relative to skill_path.parent, so parts[0] is the skill + # folder name and parts[1] (if present) is the first subdir. + if len(parts) > 1 and parts[1] in ROOT_EXCLUDE_DIRS: + return True + name = rel_path.name + if name in EXCLUDE_FILES: + return True + return any(fnmatch.fnmatch(name, pat) for pat in EXCLUDE_GLOBS) + + +def package_skill(skill_path, output_dir=None): + """ + Package a skill folder into a .skill file. + + Args: + skill_path: Path to the skill folder + output_dir: Optional output directory for the .skill file (defaults to current directory) + + Returns: + Path to the created .skill file, or None if error + """ + skill_path = Path(skill_path).resolve() + + # Validate skill folder exists + if not skill_path.exists(): + print(f"❌ Error: Skill folder not found: {skill_path}") + return None + + if not skill_path.is_dir(): + print(f"❌ Error: Path is not a directory: {skill_path}") + return None + + # Validate SKILL.md exists + skill_md = skill_path / "SKILL.md" + if not skill_md.exists(): + print(f"❌ Error: SKILL.md not found in {skill_path}") + return None + + # Run validation before packaging + print("🔍 Validating skill...") + valid, message = validate_skill(skill_path) + if not valid: + print(f"❌ Validation failed: {message}") + print(" Please fix the validation errors before packaging.") + return None + print(f"✅ {message}\n") + + # Determine output location + skill_name = skill_path.name + if output_dir: + output_path = Path(output_dir).resolve() + output_path.mkdir(parents=True, exist_ok=True) + else: + output_path = Path.cwd() + + skill_filename = output_path / f"{skill_name}.skill" + + # Create the .skill file (zip format) + try: + with zipfile.ZipFile(skill_filename, 'w', zipfile.ZIP_DEFLATED) as zipf: + # Walk through the skill directory, excluding build artifacts + for file_path in skill_path.rglob('*'): + if not file_path.is_file(): + continue + arcname = file_path.relative_to(skill_path.parent) + if should_exclude(arcname): + print(f" Skipped: {arcname}") + continue + zipf.write(file_path, arcname) + print(f" Added: {arcname}") + + print(f"\n✅ Successfully packaged skill to: {skill_filename}") + return skill_filename + + except Exception as e: + print(f"❌ Error creating .skill file: {e}") + return None + + +def main(): + if len(sys.argv) < 2: + print("Usage: python utils/package_skill.py [output-directory]") + print("\nExample:") + print(" python utils/package_skill.py skills/public/my-skill") + print(" python utils/package_skill.py skills/public/my-skill ./dist") + sys.exit(1) + + skill_path = sys.argv[1] + output_dir = sys.argv[2] if len(sys.argv) > 2 else None + + print(f"📦 Packaging skill: {skill_path}") + if output_dir: + print(f" Output directory: {output_dir}") + print() + + result = package_skill(skill_path, output_dir) + + if result: + sys.exit(0) + else: + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/.codex/skills/skill-creator/scripts/quick_validate.py b/.codex/skills/skill-creator/scripts/quick_validate.py new file mode 100755 index 000000000..ed8e1dddc --- /dev/null +++ b/.codex/skills/skill-creator/scripts/quick_validate.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +""" +Quick validation script for skills - minimal version +""" + +import sys +import os +import re +import yaml +from pathlib import Path + +def validate_skill(skill_path): + """Basic validation of a skill""" + skill_path = Path(skill_path) + + # Check SKILL.md exists + skill_md = skill_path / 'SKILL.md' + if not skill_md.exists(): + return False, "SKILL.md not found" + + # Read and validate frontmatter + content = skill_md.read_text() + if not content.startswith('---'): + return False, "No YAML frontmatter found" + + # Extract frontmatter + match = re.match(r'^---\n(.*?)\n---', content, re.DOTALL) + if not match: + return False, "Invalid frontmatter format" + + frontmatter_text = match.group(1) + + # Parse YAML frontmatter + try: + frontmatter = yaml.safe_load(frontmatter_text) + if not isinstance(frontmatter, dict): + return False, "Frontmatter must be a YAML dictionary" + except yaml.YAMLError as e: + return False, f"Invalid YAML in frontmatter: {e}" + + # Define allowed properties + ALLOWED_PROPERTIES = {'name', 'description', 'license', 'allowed-tools', 'metadata', 'compatibility'} + + # Check for unexpected properties (excluding nested keys under metadata) + unexpected_keys = set(frontmatter.keys()) - ALLOWED_PROPERTIES + if unexpected_keys: + return False, ( + f"Unexpected key(s) in SKILL.md frontmatter: {', '.join(sorted(unexpected_keys))}. " + f"Allowed properties are: {', '.join(sorted(ALLOWED_PROPERTIES))}" + ) + + # Check required fields + if 'name' not in frontmatter: + return False, "Missing 'name' in frontmatter" + if 'description' not in frontmatter: + return False, "Missing 'description' in frontmatter" + + # Extract name for validation + name = frontmatter.get('name', '') + if not isinstance(name, str): + return False, f"Name must be a string, got {type(name).__name__}" + name = name.strip() + if name: + # Check naming convention (kebab-case: lowercase with hyphens) + if not re.match(r'^[a-z0-9-]+$', name): + return False, f"Name '{name}' should be kebab-case (lowercase letters, digits, and hyphens only)" + if name.startswith('-') or name.endswith('-') or '--' in name: + return False, f"Name '{name}' cannot start/end with hyphen or contain consecutive hyphens" + # Check name length (max 64 characters per spec) + if len(name) > 64: + return False, f"Name is too long ({len(name)} characters). Maximum is 64 characters." + + # Extract and validate description + description = frontmatter.get('description', '') + if not isinstance(description, str): + return False, f"Description must be a string, got {type(description).__name__}" + description = description.strip() + if description: + # Check for angle brackets + if '<' in description or '>' in description: + return False, "Description cannot contain angle brackets (< or >)" + # Check description length (max 1024 characters per spec) + if len(description) > 1024: + return False, f"Description is too long ({len(description)} characters). Maximum is 1024 characters." + + # Validate compatibility field if present (optional) + compatibility = frontmatter.get('compatibility', '') + if compatibility: + if not isinstance(compatibility, str): + return False, f"Compatibility must be a string, got {type(compatibility).__name__}" + if len(compatibility) > 500: + return False, f"Compatibility is too long ({len(compatibility)} characters). Maximum is 500 characters." + + return True, "Skill is valid!" + +if __name__ == "__main__": + if len(sys.argv) != 2: + print("Usage: python quick_validate.py ") + sys.exit(1) + + valid, message = validate_skill(sys.argv[1]) + print(message) + sys.exit(0 if valid else 1) \ No newline at end of file diff --git a/.codex/skills/skill-creator/scripts/run_eval.py b/.codex/skills/skill-creator/scripts/run_eval.py new file mode 100755 index 000000000..e58c70bea --- /dev/null +++ b/.codex/skills/skill-creator/scripts/run_eval.py @@ -0,0 +1,310 @@ +#!/usr/bin/env python3 +"""Run trigger evaluation for a skill description. + +Tests whether a skill's description causes Claude to trigger (read the skill) +for a set of queries. Outputs results as JSON. +""" + +import argparse +import json +import os +import select +import subprocess +import sys +import time +import uuid +from concurrent.futures import ProcessPoolExecutor, as_completed +from pathlib import Path + +from scripts.utils import parse_skill_md + + +def find_project_root() -> Path: + """Find the project root by walking up from cwd looking for .claude/. + + Mimics how Claude Code discovers its project root, so the command file + we create ends up where claude -p will look for it. + """ + current = Path.cwd() + for parent in [current, *current.parents]: + if (parent / ".claude").is_dir(): + return parent + return current + + +def run_single_query( + query: str, + skill_name: str, + skill_description: str, + timeout: int, + project_root: str, + model: str | None = None, +) -> bool: + """Run a single query and return whether the skill was triggered. + + Creates a command file in .claude/commands/ so it appears in Claude's + available_skills list, then runs `claude -p` with the raw query. + Uses --include-partial-messages to detect triggering early from + stream events (content_block_start) rather than waiting for the + full assistant message, which only arrives after tool execution. + """ + unique_id = uuid.uuid4().hex[:8] + clean_name = f"{skill_name}-skill-{unique_id}" + project_commands_dir = Path(project_root) / ".claude" / "commands" + command_file = project_commands_dir / f"{clean_name}.md" + + try: + project_commands_dir.mkdir(parents=True, exist_ok=True) + # Use YAML block scalar to avoid breaking on quotes in description + indented_desc = "\n ".join(skill_description.split("\n")) + command_content = ( + f"---\n" + f"description: |\n" + f" {indented_desc}\n" + f"---\n\n" + f"# {skill_name}\n\n" + f"This skill handles: {skill_description}\n" + ) + command_file.write_text(command_content) + + cmd = [ + "claude", + "-p", query, + "--output-format", "stream-json", + "--verbose", + "--include-partial-messages", + ] + if model: + cmd.extend(["--model", model]) + + # Remove CLAUDECODE env var to allow nesting claude -p inside a + # Claude Code session. The guard is for interactive terminal conflicts; + # programmatic subprocess usage is safe. + env = {k: v for k, v in os.environ.items() if k != "CLAUDECODE"} + + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + cwd=project_root, + env=env, + ) + + triggered = False + start_time = time.time() + buffer = "" + # Track state for stream event detection + pending_tool_name = None + accumulated_json = "" + + try: + while time.time() - start_time < timeout: + if process.poll() is not None: + remaining = process.stdout.read() + if remaining: + buffer += remaining.decode("utf-8", errors="replace") + break + + ready, _, _ = select.select([process.stdout], [], [], 1.0) + if not ready: + continue + + chunk = os.read(process.stdout.fileno(), 8192) + if not chunk: + break + buffer += chunk.decode("utf-8", errors="replace") + + while "\n" in buffer: + line, buffer = buffer.split("\n", 1) + line = line.strip() + if not line: + continue + + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + + # Early detection via stream events + if event.get("type") == "stream_event": + se = event.get("event", {}) + se_type = se.get("type", "") + + if se_type == "content_block_start": + cb = se.get("content_block", {}) + if cb.get("type") == "tool_use": + tool_name = cb.get("name", "") + if tool_name in ("Skill", "Read"): + pending_tool_name = tool_name + accumulated_json = "" + else: + return False + + elif se_type == "content_block_delta" and pending_tool_name: + delta = se.get("delta", {}) + if delta.get("type") == "input_json_delta": + accumulated_json += delta.get("partial_json", "") + if clean_name in accumulated_json: + return True + + elif se_type in ("content_block_stop", "message_stop"): + if pending_tool_name: + return clean_name in accumulated_json + if se_type == "message_stop": + return False + + # Fallback: full assistant message + elif event.get("type") == "assistant": + message = event.get("message", {}) + for content_item in message.get("content", []): + if content_item.get("type") != "tool_use": + continue + tool_name = content_item.get("name", "") + tool_input = content_item.get("input", {}) + if tool_name == "Skill" and clean_name in tool_input.get("skill", ""): + triggered = True + elif tool_name == "Read" and clean_name in tool_input.get("file_path", ""): + triggered = True + return triggered + + elif event.get("type") == "result": + return triggered + finally: + # Clean up process on any exit path (return, exception, timeout) + if process.poll() is None: + process.kill() + process.wait() + + return triggered + finally: + if command_file.exists(): + command_file.unlink() + + +def run_eval( + eval_set: list[dict], + skill_name: str, + description: str, + num_workers: int, + timeout: int, + project_root: Path, + runs_per_query: int = 1, + trigger_threshold: float = 0.5, + model: str | None = None, +) -> dict: + """Run the full eval set and return results.""" + results = [] + + with ProcessPoolExecutor(max_workers=num_workers) as executor: + future_to_info = {} + for item in eval_set: + for run_idx in range(runs_per_query): + future = executor.submit( + run_single_query, + item["query"], + skill_name, + description, + timeout, + str(project_root), + model, + ) + future_to_info[future] = (item, run_idx) + + query_triggers: dict[str, list[bool]] = {} + query_items: dict[str, dict] = {} + for future in as_completed(future_to_info): + item, _ = future_to_info[future] + query = item["query"] + query_items[query] = item + if query not in query_triggers: + query_triggers[query] = [] + try: + query_triggers[query].append(future.result()) + except Exception as e: + print(f"Warning: query failed: {e}", file=sys.stderr) + query_triggers[query].append(False) + + for query, triggers in query_triggers.items(): + item = query_items[query] + trigger_rate = sum(triggers) / len(triggers) + should_trigger = item["should_trigger"] + if should_trigger: + did_pass = trigger_rate >= trigger_threshold + else: + did_pass = trigger_rate < trigger_threshold + results.append({ + "query": query, + "should_trigger": should_trigger, + "trigger_rate": trigger_rate, + "triggers": sum(triggers), + "runs": len(triggers), + "pass": did_pass, + }) + + passed = sum(1 for r in results if r["pass"]) + total = len(results) + + return { + "skill_name": skill_name, + "description": description, + "results": results, + "summary": { + "total": total, + "passed": passed, + "failed": total - passed, + }, + } + + +def main(): + parser = argparse.ArgumentParser(description="Run trigger evaluation for a skill description") + parser.add_argument("--eval-set", required=True, help="Path to eval set JSON file") + parser.add_argument("--skill-path", required=True, help="Path to skill directory") + parser.add_argument("--description", default=None, help="Override description to test") + parser.add_argument("--num-workers", type=int, default=10, help="Number of parallel workers") + parser.add_argument("--timeout", type=int, default=30, help="Timeout per query in seconds") + parser.add_argument("--runs-per-query", type=int, default=3, help="Number of runs per query") + parser.add_argument("--trigger-threshold", type=float, default=0.5, help="Trigger rate threshold") + parser.add_argument("--model", default=None, help="Model to use for claude -p (default: user's configured model)") + parser.add_argument("--verbose", action="store_true", help="Print progress to stderr") + args = parser.parse_args() + + eval_set = json.loads(Path(args.eval_set).read_text()) + skill_path = Path(args.skill_path) + + if not (skill_path / "SKILL.md").exists(): + print(f"Error: No SKILL.md found at {skill_path}", file=sys.stderr) + sys.exit(1) + + name, original_description, content = parse_skill_md(skill_path) + description = args.description or original_description + project_root = find_project_root() + + if args.verbose: + print(f"Evaluating: {description}", file=sys.stderr) + + output = run_eval( + eval_set=eval_set, + skill_name=name, + description=description, + num_workers=args.num_workers, + timeout=args.timeout, + project_root=project_root, + runs_per_query=args.runs_per_query, + trigger_threshold=args.trigger_threshold, + model=args.model, + ) + + if args.verbose: + summary = output["summary"] + print(f"Results: {summary['passed']}/{summary['total']} passed", file=sys.stderr) + for r in output["results"]: + status = "PASS" if r["pass"] else "FAIL" + rate_str = f"{r['triggers']}/{r['runs']}" + print(f" [{status}] rate={rate_str} expected={r['should_trigger']}: {r['query'][:70]}", file=sys.stderr) + + print(json.dumps(output, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/.codex/skills/skill-creator/scripts/run_loop.py b/.codex/skills/skill-creator/scripts/run_loop.py new file mode 100755 index 000000000..36f9b4e05 --- /dev/null +++ b/.codex/skills/skill-creator/scripts/run_loop.py @@ -0,0 +1,332 @@ +#!/usr/bin/env python3 +"""Run the eval + improve loop until all pass or max iterations reached. + +Combines run_eval.py and improve_description.py in a loop, tracking history +and returning the best description found. Supports train/test split to prevent +overfitting. +""" + +import argparse +import json +import random +import sys +import tempfile +import time +import webbrowser +from pathlib import Path + +import anthropic + +from scripts.generate_report import generate_html +from scripts.improve_description import improve_description +from scripts.run_eval import find_project_root, run_eval +from scripts.utils import parse_skill_md + + +def split_eval_set(eval_set: list[dict], holdout: float, seed: int = 42) -> tuple[list[dict], list[dict]]: + """Split eval set into train and test sets, stratified by should_trigger.""" + random.seed(seed) + + # Separate by should_trigger + trigger = [e for e in eval_set if e["should_trigger"]] + no_trigger = [e for e in eval_set if not e["should_trigger"]] + + # Shuffle each group + random.shuffle(trigger) + random.shuffle(no_trigger) + + # Calculate split points + n_trigger_test = max(1, int(len(trigger) * holdout)) + n_no_trigger_test = max(1, int(len(no_trigger) * holdout)) + + # Split + test_set = trigger[:n_trigger_test] + no_trigger[:n_no_trigger_test] + train_set = trigger[n_trigger_test:] + no_trigger[n_no_trigger_test:] + + return train_set, test_set + + +def run_loop( + eval_set: list[dict], + skill_path: Path, + description_override: str | None, + num_workers: int, + timeout: int, + max_iterations: int, + runs_per_query: int, + trigger_threshold: float, + holdout: float, + model: str, + verbose: bool, + live_report_path: Path | None = None, + log_dir: Path | None = None, +) -> dict: + """Run the eval + improvement loop.""" + project_root = find_project_root() + name, original_description, content = parse_skill_md(skill_path) + current_description = description_override or original_description + + # Split into train/test if holdout > 0 + if holdout > 0: + train_set, test_set = split_eval_set(eval_set, holdout) + if verbose: + print(f"Split: {len(train_set)} train, {len(test_set)} test (holdout={holdout})", file=sys.stderr) + else: + train_set = eval_set + test_set = [] + + client = anthropic.Anthropic() + history = [] + exit_reason = "unknown" + + for iteration in range(1, max_iterations + 1): + if verbose: + print(f"\n{'='*60}", file=sys.stderr) + print(f"Iteration {iteration}/{max_iterations}", file=sys.stderr) + print(f"Description: {current_description}", file=sys.stderr) + print(f"{'='*60}", file=sys.stderr) + + # Evaluate train + test together in one batch for parallelism + all_queries = train_set + test_set + t0 = time.time() + all_results = run_eval( + eval_set=all_queries, + skill_name=name, + description=current_description, + num_workers=num_workers, + timeout=timeout, + project_root=project_root, + runs_per_query=runs_per_query, + trigger_threshold=trigger_threshold, + model=model, + ) + eval_elapsed = time.time() - t0 + + # Split results back into train/test by matching queries + train_queries_set = {q["query"] for q in train_set} + train_result_list = [r for r in all_results["results"] if r["query"] in train_queries_set] + test_result_list = [r for r in all_results["results"] if r["query"] not in train_queries_set] + + train_passed = sum(1 for r in train_result_list if r["pass"]) + train_total = len(train_result_list) + train_summary = {"passed": train_passed, "failed": train_total - train_passed, "total": train_total} + train_results = {"results": train_result_list, "summary": train_summary} + + if test_set: + test_passed = sum(1 for r in test_result_list if r["pass"]) + test_total = len(test_result_list) + test_summary = {"passed": test_passed, "failed": test_total - test_passed, "total": test_total} + test_results = {"results": test_result_list, "summary": test_summary} + else: + test_results = None + test_summary = None + + history.append({ + "iteration": iteration, + "description": current_description, + "train_passed": train_summary["passed"], + "train_failed": train_summary["failed"], + "train_total": train_summary["total"], + "train_results": train_results["results"], + "test_passed": test_summary["passed"] if test_summary else None, + "test_failed": test_summary["failed"] if test_summary else None, + "test_total": test_summary["total"] if test_summary else None, + "test_results": test_results["results"] if test_results else None, + # For backward compat with report generator + "passed": train_summary["passed"], + "failed": train_summary["failed"], + "total": train_summary["total"], + "results": train_results["results"], + }) + + # Write live report if path provided + if live_report_path: + partial_output = { + "original_description": original_description, + "best_description": current_description, + "best_score": "in progress", + "iterations_run": len(history), + "holdout": holdout, + "train_size": len(train_set), + "test_size": len(test_set), + "history": history, + } + live_report_path.write_text(generate_html(partial_output, auto_refresh=True, skill_name=name)) + + if verbose: + def print_eval_stats(label, results, elapsed): + pos = [r for r in results if r["should_trigger"]] + neg = [r for r in results if not r["should_trigger"]] + tp = sum(r["triggers"] for r in pos) + pos_runs = sum(r["runs"] for r in pos) + fn = pos_runs - tp + fp = sum(r["triggers"] for r in neg) + neg_runs = sum(r["runs"] for r in neg) + tn = neg_runs - fp + total = tp + tn + fp + fn + precision = tp / (tp + fp) if (tp + fp) > 0 else 1.0 + recall = tp / (tp + fn) if (tp + fn) > 0 else 1.0 + accuracy = (tp + tn) / total if total > 0 else 0.0 + print(f"{label}: {tp+tn}/{total} correct, precision={precision:.0%} recall={recall:.0%} accuracy={accuracy:.0%} ({elapsed:.1f}s)", file=sys.stderr) + for r in results: + status = "PASS" if r["pass"] else "FAIL" + rate_str = f"{r['triggers']}/{r['runs']}" + print(f" [{status}] rate={rate_str} expected={r['should_trigger']}: {r['query'][:60]}", file=sys.stderr) + + print_eval_stats("Train", train_results["results"], eval_elapsed) + if test_summary: + print_eval_stats("Test ", test_results["results"], 0) + + if train_summary["failed"] == 0: + exit_reason = f"all_passed (iteration {iteration})" + if verbose: + print(f"\nAll train queries passed on iteration {iteration}!", file=sys.stderr) + break + + if iteration == max_iterations: + exit_reason = f"max_iterations ({max_iterations})" + if verbose: + print(f"\nMax iterations reached ({max_iterations}).", file=sys.stderr) + break + + # Improve the description based on train results + if verbose: + print(f"\nImproving description...", file=sys.stderr) + + t0 = time.time() + # Strip test scores from history so improvement model can't see them + blinded_history = [ + {k: v for k, v in h.items() if not k.startswith("test_")} + for h in history + ] + new_description = improve_description( + client=client, + skill_name=name, + skill_content=content, + current_description=current_description, + eval_results=train_results, + history=blinded_history, + model=model, + log_dir=log_dir, + iteration=iteration, + ) + improve_elapsed = time.time() - t0 + + if verbose: + print(f"Proposed ({improve_elapsed:.1f}s): {new_description}", file=sys.stderr) + + current_description = new_description + + # Find the best iteration by TEST score (or train if no test set) + if test_set: + best = max(history, key=lambda h: h["test_passed"] or 0) + best_score = f"{best['test_passed']}/{best['test_total']}" + else: + best = max(history, key=lambda h: h["train_passed"]) + best_score = f"{best['train_passed']}/{best['train_total']}" + + if verbose: + print(f"\nExit reason: {exit_reason}", file=sys.stderr) + print(f"Best score: {best_score} (iteration {best['iteration']})", file=sys.stderr) + + return { + "exit_reason": exit_reason, + "original_description": original_description, + "best_description": best["description"], + "best_score": best_score, + "best_train_score": f"{best['train_passed']}/{best['train_total']}", + "best_test_score": f"{best['test_passed']}/{best['test_total']}" if test_set else None, + "final_description": current_description, + "iterations_run": len(history), + "holdout": holdout, + "train_size": len(train_set), + "test_size": len(test_set), + "history": history, + } + + +def main(): + parser = argparse.ArgumentParser(description="Run eval + improve loop") + parser.add_argument("--eval-set", required=True, help="Path to eval set JSON file") + parser.add_argument("--skill-path", required=True, help="Path to skill directory") + parser.add_argument("--description", default=None, help="Override starting description") + parser.add_argument("--num-workers", type=int, default=10, help="Number of parallel workers") + parser.add_argument("--timeout", type=int, default=30, help="Timeout per query in seconds") + parser.add_argument("--max-iterations", type=int, default=5, help="Max improvement iterations") + parser.add_argument("--runs-per-query", type=int, default=3, help="Number of runs per query") + parser.add_argument("--trigger-threshold", type=float, default=0.5, help="Trigger rate threshold") + parser.add_argument("--holdout", type=float, default=0.4, help="Fraction of eval set to hold out for testing (0 to disable)") + parser.add_argument("--model", required=True, help="Model for improvement") + parser.add_argument("--verbose", action="store_true", help="Print progress to stderr") + parser.add_argument("--report", default="auto", help="Generate HTML report at this path (default: 'auto' for temp file, 'none' to disable)") + parser.add_argument("--results-dir", default=None, help="Save all outputs (results.json, report.html, log.txt) to a timestamped subdirectory here") + args = parser.parse_args() + + eval_set = json.loads(Path(args.eval_set).read_text()) + skill_path = Path(args.skill_path) + + if not (skill_path / "SKILL.md").exists(): + print(f"Error: No SKILL.md found at {skill_path}", file=sys.stderr) + sys.exit(1) + + name, _, _ = parse_skill_md(skill_path) + + # Set up live report path + if args.report != "none": + if args.report == "auto": + timestamp = time.strftime("%Y%m%d_%H%M%S") + live_report_path = Path(tempfile.gettempdir()) / f"skill_description_report_{skill_path.name}_{timestamp}.html" + else: + live_report_path = Path(args.report) + # Open the report immediately so the user can watch + live_report_path.write_text("

Starting optimization loop...

") + webbrowser.open(str(live_report_path)) + else: + live_report_path = None + + # Determine output directory (create before run_loop so logs can be written) + if args.results_dir: + timestamp = time.strftime("%Y-%m-%d_%H%M%S") + results_dir = Path(args.results_dir) / timestamp + results_dir.mkdir(parents=True, exist_ok=True) + else: + results_dir = None + + log_dir = results_dir / "logs" if results_dir else None + + output = run_loop( + eval_set=eval_set, + skill_path=skill_path, + description_override=args.description, + num_workers=args.num_workers, + timeout=args.timeout, + max_iterations=args.max_iterations, + runs_per_query=args.runs_per_query, + trigger_threshold=args.trigger_threshold, + holdout=args.holdout, + model=args.model, + verbose=args.verbose, + live_report_path=live_report_path, + log_dir=log_dir, + ) + + # Save JSON output + json_output = json.dumps(output, indent=2) + print(json_output) + if results_dir: + (results_dir / "results.json").write_text(json_output) + + # Write final HTML report (without auto-refresh) + if live_report_path: + live_report_path.write_text(generate_html(output, auto_refresh=False, skill_name=name)) + print(f"\nReport: {live_report_path}", file=sys.stderr) + + if results_dir and live_report_path: + (results_dir / "report.html").write_text(generate_html(output, auto_refresh=False, skill_name=name)) + + if results_dir: + print(f"Results saved to: {results_dir}", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/.codex/skills/skill-creator/scripts/utils.py b/.codex/skills/skill-creator/scripts/utils.py new file mode 100644 index 000000000..51b6a07dd --- /dev/null +++ b/.codex/skills/skill-creator/scripts/utils.py @@ -0,0 +1,47 @@ +"""Shared utilities for skill-creator scripts.""" + +from pathlib import Path + + + +def parse_skill_md(skill_path: Path) -> tuple[str, str, str]: + """Parse a SKILL.md file, returning (name, description, full_content).""" + content = (skill_path / "SKILL.md").read_text() + lines = content.split("\n") + + if lines[0].strip() != "---": + raise ValueError("SKILL.md missing frontmatter (no opening ---)") + + end_idx = None + for i, line in enumerate(lines[1:], start=1): + if line.strip() == "---": + end_idx = i + break + + if end_idx is None: + raise ValueError("SKILL.md missing frontmatter (no closing ---)") + + name = "" + description = "" + frontmatter_lines = lines[1:end_idx] + i = 0 + while i < len(frontmatter_lines): + line = frontmatter_lines[i] + if line.startswith("name:"): + name = line[len("name:"):].strip().strip('"').strip("'") + elif line.startswith("description:"): + value = line[len("description:"):].strip() + # Handle YAML multiline indicators (>, |, >-, |-) + if value in (">", "|", ">-", "|-"): + continuation_lines: list[str] = [] + i += 1 + while i < len(frontmatter_lines) and (frontmatter_lines[i].startswith(" ") or frontmatter_lines[i].startswith("\t")): + continuation_lines.append(frontmatter_lines[i].strip()) + i += 1 + description = " ".join(continuation_lines) + continue + else: + description = value.strip('"').strip("'") + i += 1 + + return name, description, content diff --git a/.codex/skills/team/SKILL.md b/.codex/skills/team/SKILL.md new file mode 100644 index 000000000..704dde979 --- /dev/null +++ b/.codex/skills/team/SKILL.md @@ -0,0 +1,129 @@ +--- +name: team +description: Team CLI — manage team members, post messages, and run activity updates. ALWAYS check this skill before running any `team` command. Use when listing members, adding members, posting messages, or running team activity updates. Note - `afx team` is deprecated; use `team` directly. +--- + +# team - Team Coordination CLI + +Manage team members and messages for your Codev project. Team data is stored in `codev/team/` and displayed in the Tower dashboard. + +## Synopsis + +``` +team list +team message [-a ] +team update +team add [-n ] [-r ] +``` + +## Commands + +### team list + +List all team members from `codev/team/people/`. + +```bash +team list +``` + +Displays a formatted table with Name, GitHub handle, and Role columns. + +### team message \ + +Post a message to the team message log (`codev/team/messages.md`). + +```bash +team message "Spec 42 is ready for review" +team message "Deployed to staging" -a deploy-bot +``` + +**Options:** +- `-a, --author ` — Override author (default: auto-detected from `gh` CLI or `git config`) + +### team update + +Post an hourly activity summary. Called automatically by cron (`.af-cron/team-update.yaml`) or manually. + +```bash +team update +``` + +Collects notable events from the last hour (builder spawns, gate approvals, PR merges, completed reviews) and posts a summary. Exits silently if no events occurred. + +### team add \ + +Scaffold a new team member file. + +```bash +team add waleedkadous +team add jdoe --name "Jane Doe" --role "Developer" +``` + +**Options:** +- `-n, --name ` — Full name (default: github handle) +- `-r, --role ` — Role (default: "Team Member") + +Creates `codev/team/people/.md` with YAML frontmatter. Handle is normalized to lowercase. Fails if the member already exists. + +## Team Directory Structure + +``` +codev/team/ +├── people/ +│ ├── waleedkadous.md # YAML frontmatter + optional bio +│ └── jdoe.md +└── messages.md # Append-only message log +``` + +## Team Member File Format + +```yaml +--- +name: M Waleed Kadous +github: waleedkadous +role: Lead Architect +--- + +Optional freeform bio or notes (not displayed in Tower). +``` + +**Required frontmatter fields:** +- `name` — Display name +- `github` — GitHub handle (used for API data enrichment) +- `role` — Team role (displayed in Tower) + +## Message Log Format + +```markdown +# Team Messages + + + +--- +**waleedkadous** | 2026-03-09 14:30 UTC +Spec 42 is ready for review. + +--- +**tower-cron** | 2026-03-09 15:00 UTC +Hourly update: Merged PR #123: Add user auth. Gate approved for #42. +``` + +Messages are append-only. Each entry has an author, UTC timestamp, and body text. + +## Deprecation Note + +`afx team` commands still work but print a deprecation warning. Use `team` directly: +- `afx team list` → `team list` +- `afx team message` → `team message` +- `afx team update` → `team update` + +## Setup + +To set up a team directory for a new project: + +```bash +team add first-member --name "First Member" --role "Architect" +team add second-member --name "Second Member" --role "Developer" +``` + +The Team tab appears in Tower when 2+ valid members exist. diff --git a/.codex/skills/update-arch-docs/SKILL.md b/.codex/skills/update-arch-docs/SKILL.md new file mode 100644 index 000000000..9fbcfd727 --- /dev/null +++ b/.codex/skills/update-arch-docs/SKILL.md @@ -0,0 +1,132 @@ +--- +name: update-arch-docs +description: "Audit, prune, and update the project's governance docs — the COLD reference archives `codev/resources/arch.md` and `codev/resources/lessons-learned.md` AND their always-on HOT companions `codev/resources/arch-critical.md` and `codev/resources/lessons-critical.md` (Spec 987, hot/cold two-tier model). Use this skill when running MAINTAIN's arch-doc step, or when asked to update / audit / prune any of those four files. It polices the hot-tier cap (capped facts/lessons + a bounded cold-doc map), enforces displacement (demote to cold when full), keeps each hot file's map accurate, and is opinionated about what NOT to put in each tier (per-spec changelogs, exhaustive enumerations, aspirational state). Two modes: diff-mode (apply a specific change) and audit-mode (propose cuts with reasons). Edits files directly via normal file-edit tooling; no destructive shell commands." +--- + +# update-arch-docs + +This skill maintains the project's governance docs, each split into **two tiers** (Spec 987): + +- **COLD reference archives** — `codev/resources/arch.md` (architecture) and `codev/resources/lessons-learned.md` (durable wisdom). Full, on-demand; grepped/read for depth. +- **HOT always-on companions** — `codev/resources/arch-critical.md` and `codev/resources/lessons-critical.md`. Tiny, **hard-capped**, **always injected** into every porch prompt and into CLAUDE.md/AGENTS.md. Each holds capped facts/lessons **plus a bounded "consult when…" map** of its cold doc's top-level topics. + +It is invoked by the MAINTAIN protocol's documentation step, and ad-hoc whenever someone updates, audits, or prunes any of these files. The skill is opinionated about *what does not belong* in each tier and **polices the hot-tier cap, displacement, and map accuracy**. Use it whenever a doc change touches any of the four files. + +## What this skill does NOT do + +These are the patterns that have, in practice, caused arch.md and lessons-learned.md to grow without bound. Treat them as bright-line rejections during both audit-mode and diff-mode work. + +### In arch.md + +- **Per-file enumerations** that go stale the moment they're written. Document the *shape* of a directory and the handful of load-bearing files; do not list every file. `git ls-files` is authoritative; the doc is for orientation. +- **Per-spec changelog sections** ("Spec 0042 added X, Spec 0073 changed Y"). Architecture is current state, not history. The git log + the spec/review documents own the changelog framing. +- **Specs/plans tables** that mirror the contents of `codev/specs/` and `codev/plans/`. These are duplicative and rot quickly. Link to the directory; do not paginate it into the doc. +- **Aspirational state** ("we plan to…", "in the next phase we'll…"). That belongs in the relevant meta-spec or roadmap doc, not in the architecture body. arch.md describes what *is*, not what *might be*. +- **Date-stamped narrative** ("As of 2026-Q2, the system uses…"). Dates make the doc look fresh while making it harder to maintain. Use git log + commit dates for temporal context. +- **Duplication of meta-spec content**. If a subsystem has its own meta-spec under `codev/architecture/.md` or under `codev/specs/`, arch.md should carry a 1-paragraph summary plus a pointer — not a copy. +- **Retired-component graveyards**. When a component is removed, delete its section. `git log` retains history; an arch.md that describes things that no longer exist is misleading. + +### In the HOT files (`arch-critical.md`, `lessons-critical.md`) + +These are capped, always-on, and **behavior-changers only**. Bright-line rejections: + +- **Anything over the cap.** Each hot file fits in a handful of lines (≈10 single-line facts/lessons + a cold-doc map of ≈12 top-level topics, ≤35 lines). To add an entry, **demote** a weaker one into the cold doc (displacement) — never grow the hot file. +- **Spec-narrow recipes / reference detail.** Those go in the COLD archive (reference), never the hot file. +- **A full/auto table of contents** in the cold-doc map. The map lists only **top-level** cold-doc sections, each with a one-line "consult when…" — never every entry. +- **Multi-paragraph entries.** One line each. + +### In the COLD archives (`arch.md`, `lessons-learned.md`) + +These are the on-demand **reference**. They may hold spec-narrow recipes and deeper detail — the anti-accretion discipline lives in the hot cap, not here. Still avoid: + +- **Multi-paragraph lesson entries**. Split or compress to 1–3 sentences. +- **Duplicate adjacent entries**. Fold variations of the same lesson into one. +- **Spec-numbered narrative framing** ("Lesson from #0468:…"). State the general principle; link the review if needed. + +### In either file (process) + +- **Destructive shell commands**. The skill must not invoke `rm -rf`, `git rm`, or destructive `sed` scripts. All edits go through the normal Edit tool. The MAINTAIN PR diff is the human-confirmation step; that's enough. + +## arch.md vs. lessons-learned.md (two-doc framing) + +Treat the two files as siblings with different purposes. When in doubt about which file a fact belongs in, route by purpose: + +| Purpose | Goes in | +|---|---| +| Current system shape — services, transports, key mental models | `arch.md` | +| Mechanism for a unique subsystem | `arch.md` (subsystem section) — or its own meta-spec if the mechanism is large enough to warrant one, with arch.md keeping a 1-paragraph summary + pointer | +| Pointers ("see meta-spec X for details") | `arch.md` | +| A durable engineering pattern that applies across multiple specs | `lessons-learned.md` | +| A system-shape surprise verified-wrong in production ("looks like X but isn't") | `arch.md` § "Verified-Wrong Assumptions" — *not* lessons-learned.md, because it's a property of the system, not a general pattern | + +The "system-shape surprise" routing is the one most often gotten wrong. If a future reader needs to know "the system *looks* like X but actually does Y", that is system shape and lives in arch.md. If they need to know "we learned that doing X is generally a bad idea", that is engineering wisdom and lives in lessons-learned.md. + +That arch-vs-lessons routing is the **cold-tier** axis. **Orthogonal to it is the hot/cold axis**: once you know a fact is architecture (or a lesson), decide whether it is *behavior-changing enough* to earn a slot in the capped hot companion, or whether it is reference detail for the cold archive. + +## Hot tier: cap, displacement, and map accuracy + +The hot files (`arch-critical.md`, `lessons-critical.md`) are the behavior-changers, and their value depends on staying tiny. When MAINTAIN runs — or any update touches them — enforce: + +- **Cap.** Each hot file fits in a handful of lines: ≈10 single-line facts/lessons **plus** a cold-doc map of ≈12 top-level topics, ≤35 lines total. If an addition would exceed the cap, **demote** the weakest existing entry into the corresponding cold doc rather than growing the hot file. The cap is load-bearing: it is what keeps the hot tier cheap enough to inject into *every* prompt. +- **Map accuracy (bounded AND accurate).** Each hot file ends with a "Map of (consult when…)" listing only the cold doc's **top-level** sections, each with a one-line "consult when…". As cold-doc sections are added / renamed / removed, update the map to match — but keep it top-level only; never expand it into a full table of contents (that re-creates the accretion the hot tier exists to avoid). +- **Behavior-changers only.** A hot entry must be something that should change a decision up front. Reference detail, recipes, and one-offs belong in the cold archive — demote on sight. + +## Sizing by purpose, not by line count + +There is no line-count budget for either file. The right size for each section is determined by what the section needs to do. If a subsystem genuinely has unique mechanism that takes 80 lines to explain clearly, 80 lines is the right size. If a subsystem can be described in 5 lines and a pointer to a meta-spec, 5 lines is right. + +What's *wrong* is bulk that comes from any of the patterns in "What this skill does NOT do" above. Strip those, and the file's size will land where it belongs. + +When proposing a section, ask: "Could a future reader skip this section without losing anything load-bearing?" If yes, the section should not exist. The doc's purpose is orientation, not completeness. + +## Mode: diff-mode (apply a specific change) + +Use diff-mode when the request is specific: "add a section about the new caching layer," "update the Glossary entry for Tower," "remove the reference to the deleted dashboard-server." + +In diff-mode: + +1. Find the smallest section that needs to change. +2. Apply the change via the Edit tool. +3. If the change adds new content, run a quick sanity check against "What this skill does NOT do" before saving — make sure the new content is current state, not aspirational; not duplicative of a meta-spec; not a per-spec changelog entry. +4. Surface the resulting diff for review. + +Diff-mode is fast. It is the right mode for ~80% of post-MAINTAIN documentation work. + +## Mode: audit-mode (identify what to cut) + +Use audit-mode when the request is general: "the doc feels stale," "MAINTAIN says it's time to prune," "review arch.md against the principles." + +In audit-mode: + +1. Read the file end-to-end. +2. For each section in arch.md, run through the per-section pruning checklist (from the MAINTAIN protocol): + - Does it describe *current state*? If aspirational, the section moves to a meta-spec; the arch.md keeps a 1-paragraph summary + pointer (or nothing, if the meta-spec stands on its own). + - Does it duplicate a meta-spec? If yes, replace with a 1-paragraph summary + pointer. + - Is it a per-file enumeration that's gone stale? If yes, prune to the directory shape + a few key files. + - Is it a changelog/narrative section ("Spec 0042 added X")? If yes, absorb the architecturally-relevant facts and remove the spec-numbered framing. + - Is the component still alive? If retired, delete the section entirely. +3. For each entry in the COLD `lessons-learned.md`, run the per-entry checklist: + - Is it terse (1–3 sentences)? If multi-paragraph, split or compress. + - Is the topic section the right one? If filed under "Architecture (continued)" or a spec-numbered section, move it to the right topical home. + - Is it a duplicate of an adjacent entry? If yes, fold them. + - (Spec-narrow recipes are **kept** here as reference — do not cut them just for being spec-narrow. Anti-accretion now lives in the hot cap, not in the cold archive.) + + Then audit each **HOT** file (`arch-critical.md`, `lessons-critical.md`): + - **Cap**: within ≈10 entries + a ≈12-topic map, ≤35 lines? If over, **demote** the weakest entries into the cold doc. + - **Map accuracy**: does every map topic name a real top-level cold-doc section, and is any new/renamed section reflected? Fix drift; keep the map top-level only. + - **Behavior-changers only**: demote any reference detail that crept in. +4. **When in doubt, KEEP.** This rule comes from the MAINTAIN protocol and applies in audit-mode too. A confident cut is better than three speculative ones. Bias toward fewer, higher-confidence proposals with clear rationale; do not chase a maximum cut count. +5. Apply the cuts via the Edit tool — the skill does not produce a "candidate-cuts list and stop." The diff *is* the proposal. The MAINTAIN PR review is the human-confirmation step. +6. Surface a short reason alongside each cut in the run file (`codev/maintain/NNNN.md`) so PR reviewers can evaluate intent ("removed because: per-spec changelog framing"; "compressed because: duplicates the orchestrator meta-spec"). + +Audit-mode is slower than diff-mode and produces larger diffs. Reserve it for explicit audit invocations, not for routine doc updates. + +## Output contract + +The skill commits to the following: + +- It edits the four governance files — `codev/resources/arch.md` / `arch-critical.md` and `codev/resources/lessons-learned.md` / `lessons-critical.md` — directly via the Edit tool. +- It does not invoke destructive shell commands (no `rm -rf`, no `git rm`, no destructive `sed`). File deletions happen only through Edit removing the relevant content; whole-file removal would be a structural change that is out of scope for this skill. +- In audit-mode, every removal is paired with a one-line reason in the MAINTAIN run file's `## Audit Findings` section. Rationale lives there so reviewers can evaluate intent without re-deriving it from the diff. +- The skill does not modify other files (specs, plans, reviews, source code). If a fact belongs somewhere else, the skill flags it; it does not move it. +- The skill does not edit live arch.md or lessons-learned.md content as a side effect of being read or invoked — only when the user/MAINTAIN protocol explicitly asks for an update or audit. From 1c9c000f1a5e1cea83490d7b881bb5a9b5413150 Mon Sep 17 00:00:00 2001 From: Mohid Makhdoomi Date: Sat, 18 Jul 2026 02:26:40 -0400 Subject: [PATCH 02/23] chore(porch): 1196 init pir --- .../status.yaml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 codev/projects/1196-ship-codex-skills-in-the-skele/status.yaml diff --git a/codev/projects/1196-ship-codex-skills-in-the-skele/status.yaml b/codev/projects/1196-ship-codex-skills-in-the-skele/status.yaml new file mode 100644 index 000000000..8f5e51aac --- /dev/null +++ b/codev/projects/1196-ship-codex-skills-in-the-skele/status.yaml @@ -0,0 +1,18 @@ +id: '1196' +title: ship-codex-skills-in-the-skele +protocol: pir +phase: plan +plan_phases: [] +current_plan_phase: null +gates: + plan-approval: + status: pending + dev-approval: + status: pending + pr: + status: pending +iteration: 1 +build_complete: false +history: [] +started_at: '2026-07-18T06:26:40.103Z' +updated_at: '2026-07-18T06:26:40.104Z' From 6bb0f58a874705fae296eccd8dfb7d4247deb568 Mon Sep 17 00:00:00 2001 From: Mohid Makhdoomi Date: Sat, 18 Jul 2026 02:29:45 -0400 Subject: [PATCH 03/23] [PIR #1196] Plan draft --- .../1196-ship-codex-skills-in-the-skele.md | 134 ++++++++++++++++++ codev/state/pir-1196_thread.md | 12 ++ 2 files changed, 146 insertions(+) create mode 100644 codev/plans/1196-ship-codex-skills-in-the-skele.md create mode 100644 codev/state/pir-1196_thread.md diff --git a/codev/plans/1196-ship-codex-skills-in-the-skele.md b/codev/plans/1196-ship-codex-skills-in-the-skele.md new file mode 100644 index 000000000..186d23c91 --- /dev/null +++ b/codev/plans/1196-ship-codex-skills-in-the-skele.md @@ -0,0 +1,134 @@ +# PIR Plan: Ship Codex Skills Through the Scaffold Lifecycle + +## Understanding + +Issue #1196 asks Codev to give Codex users the same on-disk, packaged skill +experience that Claude users receive today. The current lifecycle has a +single-provider implementation: + +- `packages/codev/src/lib/scaffold.ts:358-399` hard-codes both the + `codev-skeleton/.claude/skills/` source and `.claude/skills/` destination. +- `packages/codev/src/commands/init.ts:98-106`, + `packages/codev/src/commands/adopt.ts:134-142`, and + `packages/codev/src/commands/update.ts:218-231` consequently copy and report + only Claude skills. +- `codev-skeleton/` contains no `.codex/skills/` tree, so the package build's + existing whole-skeleton copy cannot ship Codex skills. +- There is no automated comparison of the two provider trees. + +The branch already contains contributor commit `25e1a000`, which mirrors the +self-hosted root `.claude/skills/` tree into `.codex/skills/`. I will preserve +and validate that existing work rather than recreating it. The shipped +skeleton's provider set is intentionally smaller than the self-hosted root +set; parity will be enforced between providers within each context, not +between the root and skeleton contexts. + +Skills are a deliberate exception to the runtime-only framework rule: provider +tools discover them from root `.claude/skills/` or `.codex/skills/`, so they +must be materialized. Their source remains `codev-skeleton/`, obtained through +the existing `getTemplatesDir()` package-skeleton path; no fresh adopter will +be expected to have repository-local framework files. + +## Proposed Change + +1. Add `codev-skeleton/.codex/skills/` as a byte-identical mirror of the + shipped `codev-skeleton/.claude/skills/` set. Keep physical files in both + provider locations because each tool discovers only its own conventional + path. +2. Generalize the scaffold skill copier around an explicit provider list + (`claude`, `codex`) rather than duplicating lifecycle logic. The shared + operation will return provider-qualified copy results so init, adopt, and + update can report exact paths. +3. Have init copy both provider trees into a fresh project. Have adopt and + update process both with per-skill-directory `skipExisting` semantics: + missing skills are added, while any existing provider skill directory is + left untouched. `--force` will not bypass this customization safeguard, + matching the issue and current Claude behavior. +4. Add a recursive parity test covering directory names, relative file names, + and file bytes for both the shipped skeleton provider pair and the + self-hosted root provider pair. Put provider-specific exceptions in one + named, reviewed allowlist; validate that entries are real skill names so an + exception cannot silently become stale. The allowlist starts empty. +5. Expand scaffold/update regression tests to cover fresh Codex installation, + backfilling a missing Codex skill, preserving a customized Codex skill, and + provider-qualified update results/output. +6. Update lifecycle comments/output and user-facing skill/directory + documentation to name both `.claude/skills/` and `.codex/skills/`. Keep + `AGENTS.md` and `CLAUDE.md` byte-identical in both the self-hosted root and + skeleton template pairs. + +## Files to Change + +- `codev-skeleton/.codex/skills/**` — new shipped Codex mirror of every + `codev-skeleton/.claude/skills/**` file. +- `.codex/skills/**` — retain the contributor's existing self-hosted Codex + mirror; update mirrored documentation alongside `.claude/skills/**` where + needed. +- `packages/codev/src/lib/scaffold.ts:345-399` — make skill copying + provider-aware while retaining directory-level preservation semantics. +- `packages/codev/src/commands/init.ts:13-106` — install and report both + provider skill trees for fresh projects. +- `packages/codev/src/commands/adopt.ts:14-142` — add missing skills for both + providers without replacing existing skill directories. +- `packages/codev/src/commands/update.ts:1-13,170-231` — refresh/backfill and + report both provider trees with neutral user-facing wording. +- `packages/codev/src/__tests__/scaffold.test.ts:295-345` — cover dynamic + installation and preservation for Claude and Codex. +- `packages/codev/src/__tests__/update.test.ts:220-275` — cover Codex + backfill/preservation and accept `.codex/` in structured relative paths. +- `packages/codev/src/__tests__/skill-parity.test.ts` — new recursive + provider-parity guard with an explicit exception allowlist. +- `.claude/skills/codev/SKILL.md` and `.codex/skills/codev/SKILL.md` — describe + both skill locations in the self-hosted command reference. +- `codev-skeleton/.claude/skills/codev/SKILL.md` and + `codev-skeleton/.codex/skills/codev/SKILL.md` — same documentation in the + shipped skill trees. +- `AGENTS.md`, `CLAUDE.md`, `codev-skeleton/templates/AGENTS.md`, and + `codev-skeleton/templates/CLAUDE.md` — accurately show Claude and Codex skill + directories while preserving each required identical pair. + +## Risks & Alternatives Considered + +- **Risk: user customizations are overwritten in one provider tree.** + Mitigation: apply `skipExisting` independently to each provider's complete + skill directory and assert customized file contents remain unchanged. +- **Risk: two copied provider trees drift.** Mitigation: compare recursive + path sets and bytes in CI, with a single explicit exception allowlist rather + than informal differences. +- **Risk: tests pass against repository sources but the npm package omits the + new tree.** Mitigation: use the existing `copy-skeleton` build step, then + smoke-test the built CLI against a temporary project so resolution starts + from the embedded package skeleton. +- **Risk: the large pre-existing root `.codex/skills/` commit contains more + skills than the shipped skeleton.** Mitigation: treat self-hosted and shipped + inventories as separate contexts, but require provider parity within each. +- **Alternative: symlink `.codex/skills` to `.claude/skills`.** Rejected + because symlink behavior is less portable in npm packages and adopter + repositories, and it would couple user customizations across providers. +- **Alternative: generate `.codex/skills` only during package build.** + Rejected because the issue explicitly requires the source skeleton tree and + a checked-in parity mechanism; generated-only files would be less visible to + review and repository tests. +- **Alternative: maintain two separate lifecycle implementations.** Rejected + because duplicated copy/preservation logic is another drift surface. + +## Test Plan + +- Unit: run the scaffold tests to prove both real skeleton trees install every + skill into a fresh target. +- Unit: create a target with one customized Codex skill and one missing Codex + skill; prove adopt/update semantics preserve the first and add the second. +- Unit: run the recursive parity guard and prove both provider pairs match + with no exceptions. +- Unit: run update command tests, including structured `newFiles` paths and + dry-run assertions for `.codex/skills/`. +- Build: run the Codev package build so `codev-skeleton/` is copied into the + package's embedded `skeleton/`. +- Integration smoke: invoke the built CLI for fresh init and existing-project + adopt/update in temporary directories; inspect both skill trees, then modify + a Codex `SKILL.md`, remove a different Codex skill, rerun update, and verify + the customized bytes remain while the missing skill returns. +- Documentation guard: assert/diff `AGENTS.md` against `CLAUDE.md`, the two + skeleton templates against each other, and the provider skill trees + recursively. + diff --git a/codev/state/pir-1196_thread.md b/codev/state/pir-1196_thread.md new file mode 100644 index 000000000..ae60ecbbd --- /dev/null +++ b/codev/state/pir-1196_thread.md @@ -0,0 +1,12 @@ +# PIR #1196 thread + +- Resumed the existing `builder/pir-1196` branch in the plan phase. The branch + is based on contributor commit `25e1a000`, which added a byte-identical + self-hosted `.codex/skills/` mirror of the root `.claude/skills/` tree. +- Investigation found that `copySkills()` and all three scaffold lifecycle + commands are hard-coded to `.claude/skills/`; `codev-skeleton/.codex/skills/` + is absent and no provider-parity guard exists. +- Plan decision: materialize both provider paths from the embedded skeleton, + centralize provider iteration, preserve existing skill directories per + provider, and enforce recursive byte parity with an explicit (initially + empty) provider-specific exception allowlist. From 628a74f8b4dd7a593af56c1413647f4d3258447d Mon Sep 17 00:00:00 2001 From: Mohid Makhdoomi Date: Sat, 18 Jul 2026 02:29:49 -0400 Subject: [PATCH 04/23] chore(porch): 1196 plan-approval gate-requested --- codev/projects/1196-ship-codex-skills-in-the-skele/status.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/codev/projects/1196-ship-codex-skills-in-the-skele/status.yaml b/codev/projects/1196-ship-codex-skills-in-the-skele/status.yaml index 8f5e51aac..89477a378 100644 --- a/codev/projects/1196-ship-codex-skills-in-the-skele/status.yaml +++ b/codev/projects/1196-ship-codex-skills-in-the-skele/status.yaml @@ -7,6 +7,7 @@ current_plan_phase: null gates: plan-approval: status: pending + requested_at: '2026-07-18T06:29:49.252Z' dev-approval: status: pending pr: @@ -15,4 +16,4 @@ iteration: 1 build_complete: false history: [] started_at: '2026-07-18T06:26:40.103Z' -updated_at: '2026-07-18T06:26:40.104Z' +updated_at: '2026-07-18T06:29:49.253Z' From 158c254139a7471f38e6149797a9bd4f230e7ff3 Mon Sep 17 00:00:00 2001 From: Mohid Makhdoomi Date: Sat, 18 Jul 2026 02:33:24 -0400 Subject: [PATCH 05/23] chore(porch): 1196 plan-approval gate-approved --- .../projects/1196-ship-codex-skills-in-the-skele/status.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/codev/projects/1196-ship-codex-skills-in-the-skele/status.yaml b/codev/projects/1196-ship-codex-skills-in-the-skele/status.yaml index 89477a378..0428e7ed7 100644 --- a/codev/projects/1196-ship-codex-skills-in-the-skele/status.yaml +++ b/codev/projects/1196-ship-codex-skills-in-the-skele/status.yaml @@ -6,8 +6,9 @@ plan_phases: [] current_plan_phase: null gates: plan-approval: - status: pending + status: approved requested_at: '2026-07-18T06:29:49.252Z' + approved_at: '2026-07-18T06:33:24.369Z' dev-approval: status: pending pr: @@ -16,4 +17,4 @@ iteration: 1 build_complete: false history: [] started_at: '2026-07-18T06:26:40.103Z' -updated_at: '2026-07-18T06:29:49.253Z' +updated_at: '2026-07-18T06:33:24.374Z' From cf42facb1ca6458ce75af97e796e96399b3ee3d4 Mon Sep 17 00:00:00 2001 From: Mohid Makhdoomi Date: Sat, 18 Jul 2026 02:33:29 -0400 Subject: [PATCH 06/23] chore(porch): 1196 implement phase-transition --- .../projects/1196-ship-codex-skills-in-the-skele/status.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/codev/projects/1196-ship-codex-skills-in-the-skele/status.yaml b/codev/projects/1196-ship-codex-skills-in-the-skele/status.yaml index 0428e7ed7..ddbdabac2 100644 --- a/codev/projects/1196-ship-codex-skills-in-the-skele/status.yaml +++ b/codev/projects/1196-ship-codex-skills-in-the-skele/status.yaml @@ -1,7 +1,7 @@ id: '1196' title: ship-codex-skills-in-the-skele protocol: pir -phase: plan +phase: implement plan_phases: [] current_plan_phase: null gates: @@ -17,4 +17,4 @@ iteration: 1 build_complete: false history: [] started_at: '2026-07-18T06:26:40.103Z' -updated_at: '2026-07-18T06:33:24.374Z' +updated_at: '2026-07-18T06:33:29.607Z' From 743a30e3273fdbb3cfdc07bff00fe984113ca37f Mon Sep 17 00:00:00 2001 From: Mohid Makhdoomi Date: Sat, 18 Jul 2026 02:40:37 -0400 Subject: [PATCH 07/23] [PIR #1196] Scaffold Codex skills across lifecycle --- .claude/skills/codev/SKILL.md | 4 +- .codex/skills/codev/SKILL.md | 4 +- AGENTS.md | 5 +- CLAUDE.md | 5 +- codev-skeleton/.claude/skills/codev/SKILL.md | 4 +- codev-skeleton/.codex/skills/afx/SKILL.md | 157 ++++++++++++++++++ .../.codex/skills/arch-init/SKILL.md | 62 +++++++ codev-skeleton/.codex/skills/codev/SKILL.md | 59 +++++++ codev-skeleton/.codex/skills/consult/SKILL.md | 96 +++++++++++ .../.codex/skills/generate-image/SKILL.md | 45 +++++ codev-skeleton/.codex/skills/porch/SKILL.md | 60 +++++++ .../.codex/skills/update-arch-docs/SKILL.md | 132 +++++++++++++++ codev-skeleton/templates/AGENTS.md | 2 + codev-skeleton/templates/CLAUDE.md | 2 + packages/codev/src/commands/adopt.ts | 8 +- packages/codev/src/commands/init.ts | 8 +- packages/codev/src/commands/update.ts | 20 +-- packages/codev/src/lib/scaffold.ts | 63 +++---- 18 files changed, 681 insertions(+), 55 deletions(-) create mode 100644 codev-skeleton/.codex/skills/afx/SKILL.md create mode 100644 codev-skeleton/.codex/skills/arch-init/SKILL.md create mode 100644 codev-skeleton/.codex/skills/codev/SKILL.md create mode 100644 codev-skeleton/.codex/skills/consult/SKILL.md create mode 100644 codev-skeleton/.codex/skills/generate-image/SKILL.md create mode 100644 codev-skeleton/.codex/skills/porch/SKILL.md create mode 100644 codev-skeleton/.codex/skills/update-arch-docs/SKILL.md diff --git a/.claude/skills/codev/SKILL.md b/.claude/skills/codev/SKILL.md index 0a0e001e9..9c7a1b7e1 100644 --- a/.claude/skills/codev/SKILL.md +++ b/.claude/skills/codev/SKILL.md @@ -16,7 +16,7 @@ codev doctor Check system dependencies ## codev init -Creates a **new directory** with the full codev structure: specs/, plans/, reviews/, protocols, CLAUDE.md, AGENTS.md, .claude/skills/, .codev/config.json, .gitignore. +Creates a **new directory** with the full codev structure: specs/, plans/, reviews/, protocols, CLAUDE.md, AGENTS.md, `.claude/skills/`, `.codex/skills/`, .codev/config.json, and .gitignore. ```bash codev init my-app # Interactive setup @@ -34,7 +34,7 @@ codev adopt -y # Skip conflict prompts ## codev update -Updates framework files (protocols, roles, skills) from the installed `@cluesmith/codev` package. **Never touches user data** (specs, plans, reviews). Creates `.codev-new` versions for customized files. +Updates framework files (protocols, roles, skills) from the installed `@cluesmith/codev` package. **Never touches user data** (specs, plans, reviews). Creates `.codev-new` versions for customized files. Missing Claude and Codex skills are added, while existing skill directories are preserved. ```bash codev update # Interactive update diff --git a/.codex/skills/codev/SKILL.md b/.codex/skills/codev/SKILL.md index 0a0e001e9..9c7a1b7e1 100644 --- a/.codex/skills/codev/SKILL.md +++ b/.codex/skills/codev/SKILL.md @@ -16,7 +16,7 @@ codev doctor Check system dependencies ## codev init -Creates a **new directory** with the full codev structure: specs/, plans/, reviews/, protocols, CLAUDE.md, AGENTS.md, .claude/skills/, .codev/config.json, .gitignore. +Creates a **new directory** with the full codev structure: specs/, plans/, reviews/, protocols, CLAUDE.md, AGENTS.md, `.claude/skills/`, `.codex/skills/`, .codev/config.json, and .gitignore. ```bash codev init my-app # Interactive setup @@ -34,7 +34,7 @@ codev adopt -y # Skip conflict prompts ## codev update -Updates framework files (protocols, roles, skills) from the installed `@cluesmith/codev` package. **Never touches user data** (specs, plans, reviews). Creates `.codev-new` versions for customized files. +Updates framework files (protocols, roles, skills) from the installed `@cluesmith/codev` package. **Never touches user data** (specs, plans, reviews). Creates `.codev-new` versions for customized files. Missing Claude and Codex skills are added, while existing skill directories are preserved. ```bash codev update # Interactive update diff --git a/AGENTS.md b/AGENTS.md index 5d7cb53ca..d70bab1da 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -306,7 +306,10 @@ project-root/ │ ├── lessons-learned.md # Engineering wisdom (COLD reference; maintained during MAINTAIN) │ └── lessons-critical.md # Engineering wisdom HOT tier — capped, always-injected (Spec 987) ├── .claude/ -│ └── agents/ # AI agent definitions (custom project agents) +│ ├── agents/ # AI agent definitions (custom project agents) +│ └── skills/ # Claude-native Codev skills +├── .codex/ +│ └── skills/ # Codex-native Codev skills ├── AGENTS.md # Universal AI agent instructions (AGENTS.md standard) ├── CLAUDE.md # This file (Claude Code-specific, identical to AGENTS.md) └── [project code] diff --git a/CLAUDE.md b/CLAUDE.md index 5d7cb53ca..d70bab1da 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -306,7 +306,10 @@ project-root/ │ ├── lessons-learned.md # Engineering wisdom (COLD reference; maintained during MAINTAIN) │ └── lessons-critical.md # Engineering wisdom HOT tier — capped, always-injected (Spec 987) ├── .claude/ -│ └── agents/ # AI agent definitions (custom project agents) +│ ├── agents/ # AI agent definitions (custom project agents) +│ └── skills/ # Claude-native Codev skills +├── .codex/ +│ └── skills/ # Codex-native Codev skills ├── AGENTS.md # Universal AI agent instructions (AGENTS.md standard) ├── CLAUDE.md # This file (Claude Code-specific, identical to AGENTS.md) └── [project code] diff --git a/codev-skeleton/.claude/skills/codev/SKILL.md b/codev-skeleton/.claude/skills/codev/SKILL.md index 0a0e001e9..9c7a1b7e1 100644 --- a/codev-skeleton/.claude/skills/codev/SKILL.md +++ b/codev-skeleton/.claude/skills/codev/SKILL.md @@ -16,7 +16,7 @@ codev doctor Check system dependencies ## codev init -Creates a **new directory** with the full codev structure: specs/, plans/, reviews/, protocols, CLAUDE.md, AGENTS.md, .claude/skills/, .codev/config.json, .gitignore. +Creates a **new directory** with the full codev structure: specs/, plans/, reviews/, protocols, CLAUDE.md, AGENTS.md, `.claude/skills/`, `.codex/skills/`, .codev/config.json, and .gitignore. ```bash codev init my-app # Interactive setup @@ -34,7 +34,7 @@ codev adopt -y # Skip conflict prompts ## codev update -Updates framework files (protocols, roles, skills) from the installed `@cluesmith/codev` package. **Never touches user data** (specs, plans, reviews). Creates `.codev-new` versions for customized files. +Updates framework files (protocols, roles, skills) from the installed `@cluesmith/codev` package. **Never touches user data** (specs, plans, reviews). Creates `.codev-new` versions for customized files. Missing Claude and Codex skills are added, while existing skill directories are preserved. ```bash codev update # Interactive update diff --git a/codev-skeleton/.codex/skills/afx/SKILL.md b/codev-skeleton/.codex/skills/afx/SKILL.md new file mode 100644 index 000000000..43f49ae4d --- /dev/null +++ b/codev-skeleton/.codex/skills/afx/SKILL.md @@ -0,0 +1,157 @@ +--- +name: afx +description: Agent Farm CLI — the tool for spawning builders, managing Tower, workspaces, and cron tasks. ALWAYS consult this skill BEFORE running any `afx` command to get the exact syntax. This prevents wasting time guessing flags that don't exist. Use this whenever you need to spawn a builder, check status, send messages, clean up worktrees, manage Tower, or run cron tasks. If you're about to type `afx` followed by anything, check here first. +--- + +# Agent Farm CLI + +## afx spawn + +Spawns a new builder in an isolated git worktree. + +``` +afx spawn [number] [options] +``` + +**The ONLY flags that exist:** + +| Flag | Description | +|------|-------------| +| `--protocol ` | Protocol: spir, aspir, air, bugfix, maintain, experiment. **Required for numbered spawns.** | +| `--task ` | Ad-hoc task (no issue number needed) | +| `--shell` | Bare Claude session | +| `--worktree` | Bare worktree session | +| `--files ` | Context files, comma-separated. **Requires `--task`.** | +| `--no-comment` | Skip commenting on the GitHub issue | +| `--force` | Skip dirty-worktree and collision checks | +| `--soft` | Soft mode (AI follows protocol, you verify) | +| `--strict` | Strict mode (porch orchestrates) — this is the default | +| `--resume` | Resume builder in existing worktree | +| `--no-role` | Skip loading role prompt | + +**There is NO `-t`, `--title`, `--name`, or `--branch` flag.** The branch name is auto-generated from the issue title. + +**Examples:** +```bash +afx spawn 42 --protocol spir # SPIR builder for issue #42 +afx spawn 42 --protocol aspir # ASPIR (autonomous, no human gates) +afx spawn 42 --protocol air # AIR (small features) +afx spawn 42 --protocol bugfix # Bugfix +afx spawn 42 --protocol spir --soft # Soft mode +afx spawn 42 --resume # Resume existing builder +afx spawn --task "fix the flaky test" # Ad-hoc task (no issue) +afx spawn 42 --protocol spir --force # Skip dirty-worktree check +``` + +**Pre-spawn checklist:** +1. `git status` — worktree must be clean (or use `--force`) +2. Commit specs/plans first — builders branch from HEAD and can't see uncommitted files +3. `--protocol` is required for numbered spawns + +## afx send + +Sends a message to a running builder. + +``` +afx send [builder] [message] +``` + +| Flag | Description | +|------|-------------| +| `--all` | Send to all builders | +| `--file ` | Include file content | +| `--interrupt` | Send Ctrl+C first | +| `--raw` | Skip structured formatting | +| `--no-enter` | Don't press Enter after message | + +```bash +afx send 0042 "PR approved, please merge" +afx send 0585 "check the test output" --file /tmp/test-results.txt +``` + +## afx cleanup + +Removes a builder's worktree and branch after work is done. + +``` +afx cleanup [options] +``` + +| Flag | Description | +|------|-------------| +| `-p, --project ` | Builder project ID (no leading zeros: `585` not `0585`) | +| `-i, --issue ` | Cleanup bugfix builder by issue number | +| `-t, --task ` | Cleanup task builder (e.g., `task-bEPd`) | +| `-f, --force` | Force cleanup even if branch not merged | + +```bash +afx cleanup -p 585 # Clean up project 585 +afx cleanup -p 585 -f # Force (unmerged branch) +``` + +**Note:** `afx cleanup` uses plain numbers (`585`), not zero-padded (`0585`). But `afx send` uses zero-padded IDs (`0585`). + +## afx status + +```bash +afx status # Show all builders and workspace status +``` + +No flags needed. Shows Tower status, workspace, and all active builders. + +## afx whoami + +```bash +afx whoami # Report this terminal's identity (workspace, type, name) +afx whoami --json # Same, as a single JSON object +``` + +Resolves identity from Tower/global.db's perspective: builder-worktree cwd +match first, then the Tower-injected `CODEV_ARCHITECT_NAME` env var. Exits 1 +with an explanation when identity cannot be determined — it never guesses and +never defaults to `main` (issue #1094). Builders also get an `architect:` +field naming their spawning architect when recorded. Works without Tower. + +## afx tower + +```bash +afx tower start # Start Tower on port 4100 +afx tower stop # Stop Tower +afx tower log # Tail Tower logs +afx tower status # Check daemon and cloud connection status +afx tower connect # Connect to Codev Cloud +afx tower disconnect # Disconnect from Codev Cloud +``` + +There is NO `afx tower restart` — use `afx tower stop && afx tower start`. + +## afx workspace + +```bash +afx workspace start # Start workspace for current project +afx workspace stop # Stop workspace processes +``` + +`afx dash` is a deprecated alias — use `afx workspace` instead. + +## afx cron + +```bash +afx cron list # List all cron tasks +afx cron status # Check task status +afx cron run # Run immediately +afx cron enable # Enable +afx cron disable # Disable +``` + +There is NO `afx cron add` — create YAML files in `.af-cron/` directly. + +## Other commands + +```bash +afx open # Open file in annotation viewer (NOT system open) +afx shell # Spawn utility shell +afx attach # Attach to running builder terminal +afx rename # Rename current shell session +afx architect # Start architect session in current terminal +``` diff --git a/codev-skeleton/.codex/skills/arch-init/SKILL.md b/codev-skeleton/.codex/skills/arch-init/SKILL.md new file mode 100644 index 000000000..26cacde9d --- /dev/null +++ b/codev-skeleton/.codex/skills/arch-init/SKILL.md @@ -0,0 +1,62 @@ +--- +name: arch-init +description: Adopt an architect identity and recover its state from codev/state/.md. Use when an architect terminal needs to (re)establish which architect it is — after a restart, context loss, or session handoff — or when the user says "/arch-init", "you are the X architect", or "recover your architect state". Identity resolves via `afx whoami` (an explicit name argument overrides); if neither resolves, ask the human — never guess. +argument-hint: "[name] (e.g. main; omit to auto-detect via afx whoami)" +--- + +# /arch-init — become architect `` and recover state + +You are an **architect agent** in a codev workspace. This command tells you +which architect you are and where your durable state lives, so you can resume +mid-stream. + +`$ARGUMENTS` is the architect name (e.g. `main`, or a sibling architect's +name in a multi-architect workspace). + +## What to do + +1. **Resolve your name.** + - If `$ARGUMENTS` is non-empty, that is your name — the human named you + explicitly, which removes all identity-resolution risk. **Validate it + first**: an architect name must match `[a-z][a-z0-9-]*` and be at most + 64 characters (lowercase letters, digits, hyphens; starts with a + letter). Reject anything else — slashes, `..`, uppercase, spaces — and + tell the human the rule. Never build a file path from an unvalidated + name (path-traversal guard). + - If `$ARGUMENTS` is empty, run `afx whoami` and read its output: + - `type: architect` → adopt the reported `name`. + - `type: builder` → STOP. This terminal is a builder, not an architect; + report the mismatch to the human and do not adopt an architect + identity. + - Non-zero exit (identity unknown) → STOP and ask the human which + architect you are. Do NOT guess, and do NOT default to `main` — + adopting the wrong identity and writing to another architect's state + file is the exact failure this command exists to prevent. + +2. **Read your state file: `codev/state/.md`** (relative to the + workspace root). + - If it does not exist: list the architect state files in `codev/state/` + — **excluding `*_thread.md` files**, which are builder thread logs that + share the directory — tell the human the file is missing, and ask + whether to start a fresh state file for ``. Do not fabricate + state. + - The state file is authoritative free text. It typically opens with a + role banner and may carry resume instructions; follow whatever it says. + +3. **Confirm identity + orient, then follow the state file.** In one tight + block, report: who you now are (name + one-line role from the banner, if + present), the file you read, and the current-state / open-loops summary + from the most recent dated section (or the file's leading content if it + has no dated sections). Then carry out whatever the state file says to do + on resume. Do not invent a new agenda — resume the one the state file + describes. + +## Guardrails (architect-wide; the state file may add more) + +- **Never auto-approve porch gates.** A gate notification is for the human, + not you. +- **Touch only your own builders / spawns / filings.** Sibling architects own + theirs. +- **Never `cd` into a builder worktree**; use `git -C` + absolute paths. +- **Stay on the default branch at the workspace root**; verify with + `git branch` if unsure. diff --git a/codev-skeleton/.codex/skills/codev/SKILL.md b/codev-skeleton/.codex/skills/codev/SKILL.md new file mode 100644 index 000000000..9c7a1b7e1 --- /dev/null +++ b/codev-skeleton/.codex/skills/codev/SKILL.md @@ -0,0 +1,59 @@ +--- +name: codev +description: Codev project management CLI — init, adopt, update, and doctor commands. Check this skill before running any `codev` command (except `consult`, `porch`, or `afx` which have their own skills). Use when setting up new projects, adding codev to existing repos, updating framework files, or diagnosing missing dependencies. +--- + +# codev - Project Management CLI + +## Commands + +``` +codev init [project-name] Create a new codev project directory +codev adopt Add codev to the current directory +codev update Update protocols, roles, skills from package +codev doctor Check system dependencies +``` + +## codev init + +Creates a **new directory** with the full codev structure: specs/, plans/, reviews/, protocols, CLAUDE.md, AGENTS.md, `.claude/skills/`, `.codex/skills/`, .codev/config.json, and .gitignore. + +```bash +codev init my-app # Interactive setup +codev init my-app -y # Non-interactive with defaults +``` + +## codev adopt + +Adds codev to the **current directory** (existing project). Run from the project root. If CLAUDE.md or AGENTS.md already exists, creates `.codev-new` versions and spawns Claude to merge. + +```bash +codev adopt # Interactive +codev adopt -y # Skip conflict prompts +``` + +## codev update + +Updates framework files (protocols, roles, skills) from the installed `@cluesmith/codev` package. **Never touches user data** (specs, plans, reviews). Creates `.codev-new` versions for customized files. Missing Claude and Codex skills are added, while existing skill directories are preserved. + +```bash +codev update # Interactive update +codev update --dry-run # Preview changes +codev update --force # Overwrite all framework files +codev update --agent # Agent mode: JSON stdout, no interactive merge +``` + +## codev doctor + +Checks all required dependencies: Node.js (>=18), git (>=2.5), gh (authenticated), and at least one AI CLI (Claude, Gemini, or Codex). + +```bash +codev doctor +``` + +## Common mistakes + +- There is NO `codev tower` command — use `afx tower start/stop` +- `codev init` creates a new directory — use `codev adopt` for existing projects +- Always run `codev adopt` and `codev update` from the project root +- `codev update` only updates framework files — it never touches specs/plans/reviews diff --git a/codev-skeleton/.codex/skills/consult/SKILL.md b/codev-skeleton/.codex/skills/consult/SKILL.md new file mode 100644 index 000000000..4720b0dd1 --- /dev/null +++ b/codev-skeleton/.codex/skills/consult/SKILL.md @@ -0,0 +1,96 @@ +--- +name: consult +description: AI consultation CLI — query Gemini, Codex, or Claude for reviews and analysis. ALWAYS check this skill before running any `consult` command. Use when reviewing specs, plans, implementations, or PRs with external models, running parallel 3-way reviews (cmap), or checking consultation stats. The `-m` model flag is always required except for `consult stats`. +--- + +# consult - AI Consultation CLI + +Query external AI models for reviews and analysis. Supports Gemini, Codex, and Claude. + +## Synopsis + +``` +consult -m [options] +consult stats [options] +``` + +The `-m` / `--model` flag is **always required** except for `consult stats`. + +## Models + +| Flag value | Alias | Notes | +|------------|-------|-------| +| `gemini` | `pro` | Antigravity CLI (`agy`); agentic file access (`--sandbox`), OAuth login; skips non-blockingly if unavailable | +| `codex` | `gpt` | Thorough (~200-250s), shell exploration | +| `claude` | `opus` | Agent SDK with tool use (~60-120s) | + +## All flags + +``` +-m, --model Model to use (required except stats) +--prompt Inline prompt (general mode) +--prompt-file Prompt file path (general mode) +--protocol Protocol: spir, aspir, air, bugfix, maintain +-t, --type Review type (see below) +--issue Issue number (required in architect context) +--output Save result to file +--plan-phase Scope review to a plan phase (porch use) +--context Context file with feedback (porch use) +--project-id Project ID for metrics (porch use) +--days Stats: limit to last N days (default: 30) +--project Stats: filter by project ID +--last Stats: show last N invocations +--json Stats: output as JSON +``` + +## Review types (`--type`) + +| Type | When to use | +|------|-------------| +| `spec` | Review a specification for completeness | +| `plan` | Review an implementation plan | +| `impl` | Review code implementation | +| `pr` | Review a pull request before merge | +| `phase` | Phase-scoped review (builder context only) | +| `integration` | Architect's integration review of a PR | + +## Usage patterns + +**General query:** +```bash +consult -m gemini --prompt "What's the best way to structure auth?" +consult -m codex --prompt-file review-checklist.md +``` + +**Protocol review:** +```bash +consult -m gemini --type spec --issue 42 +consult -m codex --type plan --issue 42 +consult -m claude --type integration --issue 42 +``` + +**3-way parallel review (cmap):** +Always use `--output` for background runs — without it, results go to a temp file that may be garbage-collected. + +```bash +consult -m gemini --type integration --issue 42 --output /tmp/cmap-gemini-42.md & +consult -m codex --type integration --issue 42 --output /tmp/cmap-codex-42.md & +consult -m claude --type integration --issue 42 --output /tmp/cmap-claude-42.md & +wait +``` + +**Stats:** +```bash +consult stats # 30-day summary +consult stats --days 7 --json # Last 7 days as JSON +consult stats --project 42 # Filter by project +``` + +## Rules + +- `-m` is required for all non-stats commands +- `--prompt` and `--type` are mutually exclusive (different modes) +- `--prompt` and `--prompt-file` are mutually exclusive +- `--protocol` requires `--type` +- From architect context (outside `.builders/`), `--issue` is required for protocol reviews +- From builder context (inside `.builders/`), project auto-detects from porch state diff --git a/codev-skeleton/.codex/skills/generate-image/SKILL.md b/codev-skeleton/.codex/skills/generate-image/SKILL.md new file mode 100644 index 000000000..aed0276b9 --- /dev/null +++ b/codev-skeleton/.codex/skills/generate-image/SKILL.md @@ -0,0 +1,45 @@ +--- +name: generate-image +description: AI image generation via Gemini. Use when the user wants to generate, create, or make an image, or when you need to create visual assets like logos, diagrams, or illustrations. Requires GEMINI_API_KEY or GOOGLE_API_KEY. +--- + +# generate-image - AI Image Generation + +Uses Google Gemini to generate images from text prompts. + +## Synopsis + +``` +codev generate-image "" [options] +``` + +Note: this is a `codev` subcommand, not standalone. + +## All flags + +``` +-o, --output Output file path (default: output.png) +-r, --resolution Resolution: 1K, 2K, 4K (default: 1K) +-a, --aspect Aspect ratio (default: 1:1) +--ref Reference image (repeatable, max 14) +``` + +## Aspect ratios + +`1:1` | `16:9` | `9:16` | `3:4` | `4:3` | `3:2` | `2:3` + +## Examples + +```bash +codev generate-image "A sunset over mountains" +codev generate-image "A futuristic city" -r 4K -a 16:9 -o city.png +codev generate-image "Same style but with cats" --ref style.png --ref layout.png +codev generate-image prompt.txt -o result.png # Prompt from .txt file +``` + +## Notes + +- Prompt must be quoted if it contains spaces +- Prompt can be a `.txt` file path (auto-detected by extension) +- Reference images must exist on disk +- Requires `GEMINI_API_KEY` or `GOOGLE_API_KEY` environment variable diff --git a/codev-skeleton/.codex/skills/porch/SKILL.md b/codev-skeleton/.codex/skills/porch/SKILL.md new file mode 100644 index 000000000..ef5cc17dc --- /dev/null +++ b/codev-skeleton/.codex/skills/porch/SKILL.md @@ -0,0 +1,60 @@ +--- +name: porch +description: Protocol orchestrator CLI — drives SPIR, ASPIR, AIR, and BUGFIX protocols via a state machine. ALWAYS check this skill before running any `porch` command. Use when you need to check project status, approve gates, signal phase completion, or manage protocol state. Also use when a builder asks about gate approvals or phase transitions. +--- + +# porch - Protocol Orchestrator + +Porch manages the state machine behind development protocols. It tracks phases, gates, consultations, and transitions. + +## Commands + +``` +porch status [id] Show current project state and phase +porch run [id] Run the protocol loop (strict mode) +porch next [id] Get next tasks as JSON +porch done [id] Signal current phase is complete +porch check [id] Run checks for current phase +porch gate [id] Request human approval at a gate +porch approve --a-human-explicitly-approved-this +porch rollback Rewind to an earlier phase +porch init Initialize a new project +``` + +Project ID auto-detects from worktree path when inside a builder worktree. + +## Gate approvals + +Gates are human-only approval checkpoints. The `--a-human-explicitly-approved-this` flag is **required** — it exists to prevent AI agents from auto-approving. + +| Gate | Protocol | When | +|------|----------|------| +| `spec-approval` | SPIR | After spec is written | +| `plan-approval` | SPIR | After plan is written | +| `pr` | SPIR, AIR | After PR is created | + +```bash +porch approve 42 spec-approval --a-human-explicitly-approved-this +porch approve 42 plan-approval --a-human-explicitly-approved-this +porch approve 42 pr --a-human-explicitly-approved-this +``` + +**ASPIR and BUGFIX have no spec/plan gates** — they run autonomously through those phases. + +## Checking pending gates + +```bash +porch pending # List all gates waiting for approval +``` + +## Critical rules + +- **Builders must NEVER call `porch approve`** — only humans approve gates +- **Never edit `status.yaml` directly** — porch manages all state +- Builders signal completion with `porch done`, not `porch approve` +- `porch run` is for strict mode only — soft mode builders follow the protocol document manually +- When running `porch approve` from the architect, use a subshell if you need worktree context: `(cd /path/to/worktree && porch approve ...)` + +## State storage + +Project state lives in `codev/projects/-/status.yaml`, managed automatically by porch. The status file tracks current phase, gate states, consultation results, and timestamps. diff --git a/codev-skeleton/.codex/skills/update-arch-docs/SKILL.md b/codev-skeleton/.codex/skills/update-arch-docs/SKILL.md new file mode 100644 index 000000000..9fbcfd727 --- /dev/null +++ b/codev-skeleton/.codex/skills/update-arch-docs/SKILL.md @@ -0,0 +1,132 @@ +--- +name: update-arch-docs +description: "Audit, prune, and update the project's governance docs — the COLD reference archives `codev/resources/arch.md` and `codev/resources/lessons-learned.md` AND their always-on HOT companions `codev/resources/arch-critical.md` and `codev/resources/lessons-critical.md` (Spec 987, hot/cold two-tier model). Use this skill when running MAINTAIN's arch-doc step, or when asked to update / audit / prune any of those four files. It polices the hot-tier cap (capped facts/lessons + a bounded cold-doc map), enforces displacement (demote to cold when full), keeps each hot file's map accurate, and is opinionated about what NOT to put in each tier (per-spec changelogs, exhaustive enumerations, aspirational state). Two modes: diff-mode (apply a specific change) and audit-mode (propose cuts with reasons). Edits files directly via normal file-edit tooling; no destructive shell commands." +--- + +# update-arch-docs + +This skill maintains the project's governance docs, each split into **two tiers** (Spec 987): + +- **COLD reference archives** — `codev/resources/arch.md` (architecture) and `codev/resources/lessons-learned.md` (durable wisdom). Full, on-demand; grepped/read for depth. +- **HOT always-on companions** — `codev/resources/arch-critical.md` and `codev/resources/lessons-critical.md`. Tiny, **hard-capped**, **always injected** into every porch prompt and into CLAUDE.md/AGENTS.md. Each holds capped facts/lessons **plus a bounded "consult when…" map** of its cold doc's top-level topics. + +It is invoked by the MAINTAIN protocol's documentation step, and ad-hoc whenever someone updates, audits, or prunes any of these files. The skill is opinionated about *what does not belong* in each tier and **polices the hot-tier cap, displacement, and map accuracy**. Use it whenever a doc change touches any of the four files. + +## What this skill does NOT do + +These are the patterns that have, in practice, caused arch.md and lessons-learned.md to grow without bound. Treat them as bright-line rejections during both audit-mode and diff-mode work. + +### In arch.md + +- **Per-file enumerations** that go stale the moment they're written. Document the *shape* of a directory and the handful of load-bearing files; do not list every file. `git ls-files` is authoritative; the doc is for orientation. +- **Per-spec changelog sections** ("Spec 0042 added X, Spec 0073 changed Y"). Architecture is current state, not history. The git log + the spec/review documents own the changelog framing. +- **Specs/plans tables** that mirror the contents of `codev/specs/` and `codev/plans/`. These are duplicative and rot quickly. Link to the directory; do not paginate it into the doc. +- **Aspirational state** ("we plan to…", "in the next phase we'll…"). That belongs in the relevant meta-spec or roadmap doc, not in the architecture body. arch.md describes what *is*, not what *might be*. +- **Date-stamped narrative** ("As of 2026-Q2, the system uses…"). Dates make the doc look fresh while making it harder to maintain. Use git log + commit dates for temporal context. +- **Duplication of meta-spec content**. If a subsystem has its own meta-spec under `codev/architecture/.md` or under `codev/specs/`, arch.md should carry a 1-paragraph summary plus a pointer — not a copy. +- **Retired-component graveyards**. When a component is removed, delete its section. `git log` retains history; an arch.md that describes things that no longer exist is misleading. + +### In the HOT files (`arch-critical.md`, `lessons-critical.md`) + +These are capped, always-on, and **behavior-changers only**. Bright-line rejections: + +- **Anything over the cap.** Each hot file fits in a handful of lines (≈10 single-line facts/lessons + a cold-doc map of ≈12 top-level topics, ≤35 lines). To add an entry, **demote** a weaker one into the cold doc (displacement) — never grow the hot file. +- **Spec-narrow recipes / reference detail.** Those go in the COLD archive (reference), never the hot file. +- **A full/auto table of contents** in the cold-doc map. The map lists only **top-level** cold-doc sections, each with a one-line "consult when…" — never every entry. +- **Multi-paragraph entries.** One line each. + +### In the COLD archives (`arch.md`, `lessons-learned.md`) + +These are the on-demand **reference**. They may hold spec-narrow recipes and deeper detail — the anti-accretion discipline lives in the hot cap, not here. Still avoid: + +- **Multi-paragraph lesson entries**. Split or compress to 1–3 sentences. +- **Duplicate adjacent entries**. Fold variations of the same lesson into one. +- **Spec-numbered narrative framing** ("Lesson from #0468:…"). State the general principle; link the review if needed. + +### In either file (process) + +- **Destructive shell commands**. The skill must not invoke `rm -rf`, `git rm`, or destructive `sed` scripts. All edits go through the normal Edit tool. The MAINTAIN PR diff is the human-confirmation step; that's enough. + +## arch.md vs. lessons-learned.md (two-doc framing) + +Treat the two files as siblings with different purposes. When in doubt about which file a fact belongs in, route by purpose: + +| Purpose | Goes in | +|---|---| +| Current system shape — services, transports, key mental models | `arch.md` | +| Mechanism for a unique subsystem | `arch.md` (subsystem section) — or its own meta-spec if the mechanism is large enough to warrant one, with arch.md keeping a 1-paragraph summary + pointer | +| Pointers ("see meta-spec X for details") | `arch.md` | +| A durable engineering pattern that applies across multiple specs | `lessons-learned.md` | +| A system-shape surprise verified-wrong in production ("looks like X but isn't") | `arch.md` § "Verified-Wrong Assumptions" — *not* lessons-learned.md, because it's a property of the system, not a general pattern | + +The "system-shape surprise" routing is the one most often gotten wrong. If a future reader needs to know "the system *looks* like X but actually does Y", that is system shape and lives in arch.md. If they need to know "we learned that doing X is generally a bad idea", that is engineering wisdom and lives in lessons-learned.md. + +That arch-vs-lessons routing is the **cold-tier** axis. **Orthogonal to it is the hot/cold axis**: once you know a fact is architecture (or a lesson), decide whether it is *behavior-changing enough* to earn a slot in the capped hot companion, or whether it is reference detail for the cold archive. + +## Hot tier: cap, displacement, and map accuracy + +The hot files (`arch-critical.md`, `lessons-critical.md`) are the behavior-changers, and their value depends on staying tiny. When MAINTAIN runs — or any update touches them — enforce: + +- **Cap.** Each hot file fits in a handful of lines: ≈10 single-line facts/lessons **plus** a cold-doc map of ≈12 top-level topics, ≤35 lines total. If an addition would exceed the cap, **demote** the weakest existing entry into the corresponding cold doc rather than growing the hot file. The cap is load-bearing: it is what keeps the hot tier cheap enough to inject into *every* prompt. +- **Map accuracy (bounded AND accurate).** Each hot file ends with a "Map of (consult when…)" listing only the cold doc's **top-level** sections, each with a one-line "consult when…". As cold-doc sections are added / renamed / removed, update the map to match — but keep it top-level only; never expand it into a full table of contents (that re-creates the accretion the hot tier exists to avoid). +- **Behavior-changers only.** A hot entry must be something that should change a decision up front. Reference detail, recipes, and one-offs belong in the cold archive — demote on sight. + +## Sizing by purpose, not by line count + +There is no line-count budget for either file. The right size for each section is determined by what the section needs to do. If a subsystem genuinely has unique mechanism that takes 80 lines to explain clearly, 80 lines is the right size. If a subsystem can be described in 5 lines and a pointer to a meta-spec, 5 lines is right. + +What's *wrong* is bulk that comes from any of the patterns in "What this skill does NOT do" above. Strip those, and the file's size will land where it belongs. + +When proposing a section, ask: "Could a future reader skip this section without losing anything load-bearing?" If yes, the section should not exist. The doc's purpose is orientation, not completeness. + +## Mode: diff-mode (apply a specific change) + +Use diff-mode when the request is specific: "add a section about the new caching layer," "update the Glossary entry for Tower," "remove the reference to the deleted dashboard-server." + +In diff-mode: + +1. Find the smallest section that needs to change. +2. Apply the change via the Edit tool. +3. If the change adds new content, run a quick sanity check against "What this skill does NOT do" before saving — make sure the new content is current state, not aspirational; not duplicative of a meta-spec; not a per-spec changelog entry. +4. Surface the resulting diff for review. + +Diff-mode is fast. It is the right mode for ~80% of post-MAINTAIN documentation work. + +## Mode: audit-mode (identify what to cut) + +Use audit-mode when the request is general: "the doc feels stale," "MAINTAIN says it's time to prune," "review arch.md against the principles." + +In audit-mode: + +1. Read the file end-to-end. +2. For each section in arch.md, run through the per-section pruning checklist (from the MAINTAIN protocol): + - Does it describe *current state*? If aspirational, the section moves to a meta-spec; the arch.md keeps a 1-paragraph summary + pointer (or nothing, if the meta-spec stands on its own). + - Does it duplicate a meta-spec? If yes, replace with a 1-paragraph summary + pointer. + - Is it a per-file enumeration that's gone stale? If yes, prune to the directory shape + a few key files. + - Is it a changelog/narrative section ("Spec 0042 added X")? If yes, absorb the architecturally-relevant facts and remove the spec-numbered framing. + - Is the component still alive? If retired, delete the section entirely. +3. For each entry in the COLD `lessons-learned.md`, run the per-entry checklist: + - Is it terse (1–3 sentences)? If multi-paragraph, split or compress. + - Is the topic section the right one? If filed under "Architecture (continued)" or a spec-numbered section, move it to the right topical home. + - Is it a duplicate of an adjacent entry? If yes, fold them. + - (Spec-narrow recipes are **kept** here as reference — do not cut them just for being spec-narrow. Anti-accretion now lives in the hot cap, not in the cold archive.) + + Then audit each **HOT** file (`arch-critical.md`, `lessons-critical.md`): + - **Cap**: within ≈10 entries + a ≈12-topic map, ≤35 lines? If over, **demote** the weakest entries into the cold doc. + - **Map accuracy**: does every map topic name a real top-level cold-doc section, and is any new/renamed section reflected? Fix drift; keep the map top-level only. + - **Behavior-changers only**: demote any reference detail that crept in. +4. **When in doubt, KEEP.** This rule comes from the MAINTAIN protocol and applies in audit-mode too. A confident cut is better than three speculative ones. Bias toward fewer, higher-confidence proposals with clear rationale; do not chase a maximum cut count. +5. Apply the cuts via the Edit tool — the skill does not produce a "candidate-cuts list and stop." The diff *is* the proposal. The MAINTAIN PR review is the human-confirmation step. +6. Surface a short reason alongside each cut in the run file (`codev/maintain/NNNN.md`) so PR reviewers can evaluate intent ("removed because: per-spec changelog framing"; "compressed because: duplicates the orchestrator meta-spec"). + +Audit-mode is slower than diff-mode and produces larger diffs. Reserve it for explicit audit invocations, not for routine doc updates. + +## Output contract + +The skill commits to the following: + +- It edits the four governance files — `codev/resources/arch.md` / `arch-critical.md` and `codev/resources/lessons-learned.md` / `lessons-critical.md` — directly via the Edit tool. +- It does not invoke destructive shell commands (no `rm -rf`, no `git rm`, no destructive `sed`). File deletions happen only through Edit removing the relevant content; whole-file removal would be a structural change that is out of scope for this skill. +- In audit-mode, every removal is paired with a one-line reason in the MAINTAIN run file's `## Audit Findings` section. Rationale lives there so reviewers can evaluate intent without re-deriving it from the diff. +- The skill does not modify other files (specs, plans, reviews, source code). If a fact belongs somewhere else, the skill flags it; it does not move it. +- The skill does not edit live arch.md or lessons-learned.md content as a side effect of being read or invoked — only when the user/MAINTAIN protocol explicitly asks for an update or audit. diff --git a/codev-skeleton/templates/AGENTS.md b/codev-skeleton/templates/AGENTS.md index 35d705ccf..1b166cb44 100644 --- a/codev-skeleton/templates/AGENTS.md +++ b/codev-skeleton/templates/AGENTS.md @@ -62,6 +62,8 @@ This succeeds if the protocol is registered (including via the skeleton fallback - **Plans**: `codev/plans/` - Implementation plans (HOW to build) - **Reviews**: `codev/reviews/` - Reviews and lessons learned - **Protocols**: `codev/protocols/` - Development protocols +- **Claude skills**: `.claude/skills/` - Provider-native Codev command guidance +- **Codex skills**: `.codex/skills/` - Provider-native Codev command guidance ## Working with Project Labels diff --git a/codev-skeleton/templates/CLAUDE.md b/codev-skeleton/templates/CLAUDE.md index 07e5fc4a1..a61af14d0 100644 --- a/codev-skeleton/templates/CLAUDE.md +++ b/codev-skeleton/templates/CLAUDE.md @@ -60,6 +60,8 @@ This succeeds if the protocol is registered (including via the skeleton fallback - **Plans**: `codev/plans/` - Implementation plans (HOW to build) - **Reviews**: `codev/reviews/` - Reviews and lessons learned - **Protocols**: `codev/protocols/` - Development protocols +- **Claude skills**: `.claude/skills/` - Provider-native Codev command guidance +- **Codex skills**: `.codex/skills/` - Provider-native Codev command guidance ## Working with Project Labels diff --git a/packages/codev/src/commands/adopt.ts b/packages/codev/src/commands/adopt.ts index 59738c499..5088d7946 100644 --- a/packages/codev/src/commands/adopt.ts +++ b/packages/codev/src/commands/adopt.ts @@ -131,14 +131,14 @@ export async function adopt(options: AdoptOptions = {}): Promise { // They resolve at runtime from the installed npm package via the unified file resolver. // Existing local copies in codev/ are left in place — they take precedence via the resolution chain. - // Copy .claude/skills/ - skip existing (preserve user customizations) + // Copy provider-native skills, preserving existing skill directories. const skillsResult = copySkills(targetDir, skeletonDir, { skipExisting: true }); - if (skillsResult.directoryCreated) { - console.log(chalk.green(' +'), '.claude/skills/'); + for (const directory of skillsResult.directoriesCreated) { + console.log(chalk.green(' +'), directory); fileCount++; } for (const skill of skillsResult.copied) { - console.log(chalk.green(' +'), `.claude/skills/${skill}/`); + console.log(chalk.green(' +'), skill); fileCount++; } diff --git a/packages/codev/src/commands/init.ts b/packages/codev/src/commands/init.ts index c475175b2..536ce4638 100644 --- a/packages/codev/src/commands/init.ts +++ b/packages/codev/src/commands/init.ts @@ -95,14 +95,14 @@ export async function init(projectName?: string, options: InitOptions = {}): Pro // Framework files (protocols, roles, consult-types, templates) are NOT copied. // They resolve at runtime from the installed npm package via the unified file resolver. - // Copy .claude/skills/ (Claude Code slash commands — must exist on disk) + // Copy provider-native skills (must exist on disk for tool discovery). const skillsResult = copySkills(targetDir, skeletonDir); - if (skillsResult.directoryCreated) { - console.log(chalk.green(' +'), '.claude/skills/'); + for (const directory of skillsResult.directoriesCreated) { + console.log(chalk.green(' +'), directory); fileCount++; } for (const skill of skillsResult.copied) { - console.log(chalk.green(' +'), `.claude/skills/${skill}/`); + console.log(chalk.green(' +'), skill); fileCount++; } diff --git a/packages/codev/src/commands/update.ts b/packages/codev/src/commands/update.ts index eae3ffe52..e2bfecdfb 100644 --- a/packages/codev/src/commands/update.ts +++ b/packages/codev/src/commands/update.ts @@ -1,5 +1,5 @@ /** - * codev update - Migrate config and refresh Claude-specific files. + * codev update - Migrate config and refresh AI-assistant integration files. * * Two responsibilities: * A. One-time migration (runs once, skipped on subsequent runs): @@ -8,8 +8,8 @@ * 3. Preserve user-modified files as local overrides * 4. Remove .update-hashes.json * - * B. Claude-specific file refresh (runs every time): - * 1. Update CLAUDE.md, AGENTS.md, .claude/skills/ from package + * B. AI-assistant file refresh (runs every time): + * 1. Update CLAUDE.md, AGENTS.md, and provider skill trees from package */ import * as fs from 'node:fs'; @@ -167,7 +167,7 @@ function runMigration( } /** - * Update codev — migrate config and refresh Claude-specific files. + * Update codev — migrate config and refresh AI-assistant integration files. */ export async function update(options: UpdateOptions = {}): Promise { const { dryRun = false, force = false, agent = false } = options; @@ -213,21 +213,19 @@ export async function update(options: UpdateOptions = {}): Promise log(chalk.dim(' Migration already complete.')); } - // --- B. Claude-specific file refresh (runs every time) --- + // --- B. AI-assistant file refresh (runs every time) --- log(''); - log(chalk.bold('Refreshing Claude-specific files...')); + log(chalk.bold('Refreshing AI-assistant files...')); log(''); const templatesDir = getTemplatesDir(); - // Update .claude/skills/ + // Add missing provider-native skills without replacing customizations. if (!dryRun) { - // Update skills — add new ones but preserve user-customized existing skills const skillsResult = copySkills(targetDir, templatesDir, { skipExisting: true }); for (const skill of skillsResult.copied) { - const fullPath = `.claude/skills/${skill}/`; - result.newFiles.push(fullPath); - log(chalk.green(' + (new)'), fullPath); + result.newFiles.push(skill); + log(chalk.green(' + (new)'), skill); } } diff --git a/packages/codev/src/lib/scaffold.ts b/packages/codev/src/lib/scaffold.ts index 9584ab299..4256b20d5 100644 --- a/packages/codev/src/lib/scaffold.ts +++ b/packages/codev/src/lib/scaffold.ts @@ -349,14 +349,22 @@ interface CopySkillsOptions { } interface CopySkillsResult { + /** Project-relative provider-qualified paths, including a trailing slash. */ copied: string[]; + /** Project-relative provider-qualified paths, including a trailing slash. */ skipped: string[]; - directoryCreated: boolean; + /** Project-relative provider skill roots that were created. */ + directoriesCreated: string[]; } +export const SKILL_PROVIDERS = ['claude', 'codex'] as const; + /** - * Copy .claude/skills/ from skeleton to project root. - * Skills are Claude Code slash commands that provide contextual guidance. + * Copy provider-native skill trees from the skeleton to the project root. + * + * Each skill directory is the preservation boundary: skipExisting leaves an + * existing skill completely untouched while still backfilling missing skills + * for either provider. */ export function copySkills( targetDir: string, @@ -364,41 +372,40 @@ export function copySkills( options: CopySkillsOptions = {} ): CopySkillsResult { const { skipExisting = false } = options; - const skillsDir = path.join(targetDir, '.claude', 'skills'); - const srcDir = path.join(skeletonDir, '.claude', 'skills'); const copied: string[] = []; const skipped: string[] = []; - let directoryCreated = false; + const directoriesCreated: string[] = []; - // Ensure .claude/skills directory exists - if (!fs.existsSync(skillsDir)) { - fs.mkdirSync(skillsDir, { recursive: true }); - directoryCreated = true; - } + for (const provider of SKILL_PROVIDERS) { + const relativeSkillsDir = `.${provider}/skills`; + const skillsDir = path.join(targetDir, relativeSkillsDir); + const srcDir = path.join(skeletonDir, relativeSkillsDir); - // If source directory doesn't exist, return early - if (!fs.existsSync(srcDir)) { - return { copied, skipped, directoryCreated }; - } + if (!fs.existsSync(skillsDir)) { + fs.mkdirSync(skillsDir, { recursive: true }); + directoriesCreated.push(`${relativeSkillsDir}/`); + } - // Copy each skill directory - const skills = fs.readdirSync(srcDir, { withFileTypes: true }); - for (const entry of skills) { - if (!entry.isDirectory()) continue; + // Older/corrupt skeletons may not provide every configured provider. + if (!fs.existsSync(srcDir)) continue; - const destSkillDir = path.join(skillsDir, entry.name); - const srcSkillDir = path.join(srcDir, entry.name); + for (const entry of fs.readdirSync(srcDir, { withFileTypes: true })) { + if (!entry.isDirectory()) continue; - if (skipExisting && fs.existsSync(destSkillDir)) { - skipped.push(entry.name); - continue; - } + const destSkillDir = path.join(skillsDir, entry.name); + const relativeSkillDir = `${relativeSkillsDir}/${entry.name}/`; - copyDirRecursive(srcSkillDir, destSkillDir); - copied.push(entry.name); + if (skipExisting && fs.existsSync(destSkillDir)) { + skipped.push(relativeSkillDir); + continue; + } + + copyDirRecursive(path.join(srcDir, entry.name), destSkillDir); + copied.push(relativeSkillDir); + } } - return { copied, skipped, directoryCreated }; + return { copied, skipped, directoriesCreated }; } interface CopyRolesOptions { From e854fcf24a6cf4f5cbede258df267d324a1d6729 Mon Sep 17 00:00:00 2001 From: Mohid Makhdoomi Date: Sat, 18 Jul 2026 02:41:02 -0400 Subject: [PATCH 08/23] [PIR #1196] Guard provider skill parity and preservation --- codev/state/pir-1196_thread.md | 10 ++ packages/codev/src/__tests__/adopt.test.ts | 19 ++++ packages/codev/src/__tests__/init.test.ts | 6 ++ packages/codev/src/__tests__/scaffold.test.ts | 43 ++++---- .../codev/src/__tests__/skill-parity.test.ts | 101 ++++++++++++++++++ packages/codev/src/__tests__/update.test.ts | 29 ++++- 6 files changed, 188 insertions(+), 20 deletions(-) create mode 100644 packages/codev/src/__tests__/skill-parity.test.ts diff --git a/codev/state/pir-1196_thread.md b/codev/state/pir-1196_thread.md index ae60ecbbd..0647d6808 100644 --- a/codev/state/pir-1196_thread.md +++ b/codev/state/pir-1196_thread.md @@ -10,3 +10,13 @@ centralize provider iteration, preserve existing skill directories per provider, and enforce recursive byte parity with an explicit (initially empty) provider-specific exception allowlist. +- Implemented provider-qualified skill copying for init/adopt/update, added the + seven-skill shipped Codex skeleton mirror, updated both provider docs, and + added recursive parity plus lifecycle preservation coverage. +- Verification: dependency/core/types/artifact/codev build chain passed; 58 + targeted tests passed; built-CLI smoke tests passed for fresh init, adopt, + and update. The full default suite passed 3,508 tests but has 17 unrelated + environment-sensitive failures in `tower-utils.test.ts` (Codex harness + selected where assertions assume Claude session flags) and + `findWorkspaceRoot.test.ts` (`/tmp` is detected as a workspace). No tests + were skipped or modified to mask these pre-existing failures. diff --git a/packages/codev/src/__tests__/adopt.test.ts b/packages/codev/src/__tests__/adopt.test.ts index d5423aeca..8ae80b7a5 100644 --- a/packages/codev/src/__tests__/adopt.test.ts +++ b/packages/codev/src/__tests__/adopt.test.ts @@ -89,6 +89,25 @@ describe('adopt command', () => { expect(fs.existsSync(path.join(projectDir, 'codev', 'resources', 'lessons-learned.md'))).toBe(true); }); + it('adds missing Codex skills without overwriting an existing customized skill', async () => { + const projectDir = path.join(testBaseDir, 'codex-skills-project'); + const customized = path.join(projectDir, '.codex', 'skills', 'arch-init'); + fs.mkdirSync(customized, { recursive: true }); + fs.writeFileSync(path.join(customized, 'SKILL.md'), 'user-customized codex skill'); + + process.chdir(projectDir); + + const { adopt } = await import('../commands/adopt.js'); + await adopt({ yes: true }); + + expect(fs.readFileSync(path.join(customized, 'SKILL.md'), 'utf-8')).toBe( + 'user-customized codex skill' + ); + expect( + fs.existsSync(path.join(projectDir, '.codex', 'skills', 'afx', 'SKILL.md')) + ).toBe(true); + }); + it('should throw error if codev directory already exists', async () => { const projectDir = path.join(testBaseDir, 'has-codev'); fs.mkdirSync(path.join(projectDir, 'codev'), { recursive: true }); diff --git a/packages/codev/src/__tests__/init.test.ts b/packages/codev/src/__tests__/init.test.ts index f571564a3..302ae8199 100644 --- a/packages/codev/src/__tests__/init.test.ts +++ b/packages/codev/src/__tests__/init.test.ts @@ -64,6 +64,12 @@ describe('init command', () => { expect(fs.existsSync(path.join(projectDir, 'CLAUDE.md'))).toBe(true); expect(fs.existsSync(path.join(projectDir, 'AGENTS.md'))).toBe(true); expect(fs.existsSync(path.join(projectDir, '.gitignore'))).toBe(true); + expect( + fs.existsSync(path.join(projectDir, '.claude', 'skills', 'arch-init', 'SKILL.md')) + ).toBe(true); + expect( + fs.existsSync(path.join(projectDir, '.codex', 'skills', 'arch-init', 'SKILL.md')) + ).toBe(true); // Verify user data directories (minimal structure) expect(fs.existsSync(path.join(projectDir, 'codev', 'specs'))).toBe(true); diff --git a/packages/codev/src/__tests__/scaffold.test.ts b/packages/codev/src/__tests__/scaffold.test.ts index de84ebb9b..9fc4b39dc 100644 --- a/packages/codev/src/__tests__/scaffold.test.ts +++ b/packages/codev/src/__tests__/scaffold.test.ts @@ -292,39 +292,44 @@ describe('Scaffold Utilities', () => { }); }); - // Spec 1134 — copySkills() is the install path for .claude/skills/ (used by - // init/adopt/update). First regression tests for it: the dynamic directory - // enumeration must pick up new skills (arch-init), and skipExisting must - // preserve a user's customized copy. - describe('copySkills (Spec 1134)', () => { - // The REAL repo skeleton — proves the shipped arch-init skill installs, - // not just a mock (spec 1134 test scenario 9). + describe('copySkills (Spec 1134, issue #1196)', () => { + // The real repository skeleton proves the shipped provider trees install, + // rather than only exercising a mock fixture. const realSkeletonDir = path.resolve(__dirname, '..', '..', '..', '..', 'codev-skeleton'); - it('installs arch-init/SKILL.md from the real skeleton into a fresh target', () => { + it('installs Claude and Codex skills from the real skeleton into a fresh target', () => { const result = copySkills(tempDir, realSkeletonDir); - expect(result.copied).toContain('arch-init'); + expect(result.copied).toContain('.claude/skills/arch-init/'); + expect(result.copied).toContain('.codex/skills/arch-init/'); expect( fs.existsSync(path.join(tempDir, '.claude', 'skills', 'arch-init', 'SKILL.md')) ).toBe(true); + expect( + fs.existsSync(path.join(tempDir, '.codex', 'skills', 'arch-init', 'SKILL.md')) + ).toBe(true); }); - it('enumerates skill directories dynamically (copies every skeleton skill)', () => { + it('enumerates every skill directory dynamically for both providers', () => { const result = copySkills(tempDir, realSkeletonDir); - const skeletonSkills = fs - .readdirSync(path.join(realSkeletonDir, '.claude', 'skills'), { withFileTypes: true }) - .filter((e) => e.isDirectory()) - .map((e) => e.name); - expect(result.copied.sort()).toEqual(skeletonSkills.sort()); + const expected = ['claude', 'codex'].flatMap((provider) => + fs + .readdirSync(path.join(realSkeletonDir, `.${provider}`, 'skills'), { + withFileTypes: true, + }) + .filter((entry) => entry.isDirectory()) + .map((entry) => `.${provider}/skills/${entry.name}/`) + ); + expect(result.copied.sort()).toEqual(expected.sort()); }); - it('skipExisting: true preserves an existing customized copy', () => { - const customized = path.join(tempDir, '.claude', 'skills', 'arch-init'); + it('skipExisting preserves a customized Codex skill while backfilling missing skills', () => { + const customized = path.join(tempDir, '.codex', 'skills', 'arch-init'); fs.mkdirSync(customized, { recursive: true }); fs.writeFileSync(path.join(customized, 'SKILL.md'), 'user-customized content'); const result = copySkills(tempDir, realSkeletonDir, { skipExisting: true }); - expect(result.skipped).toContain('arch-init'); + expect(result.skipped).toContain('.codex/skills/arch-init/'); + expect(result.copied).toContain('.codex/skills/afx/'); expect(fs.readFileSync(path.join(customized, 'SKILL.md'), 'utf-8')).toBe( 'user-customized content' ); @@ -336,7 +341,7 @@ describe('Scaffold Utilities', () => { fs.writeFileSync(path.join(existing, 'SKILL.md'), 'stale content'); const result = copySkills(tempDir, realSkeletonDir); - expect(result.copied).toContain('arch-init'); + expect(result.copied).toContain('.claude/skills/arch-init/'); expect(fs.readFileSync(path.join(existing, 'SKILL.md'), 'utf-8')).not.toBe('stale content'); }); }); diff --git a/packages/codev/src/__tests__/skill-parity.test.ts b/packages/codev/src/__tests__/skill-parity.test.ts new file mode 100644 index 000000000..e063fe687 --- /dev/null +++ b/packages/codev/src/__tests__/skill-parity.test.ts @@ -0,0 +1,101 @@ +/** + * Provider skill drift guard (issue #1196). + * + * Claude and Codex need physical trees at their provider-native discovery + * paths, but those copies must not become independently maintained sources. + */ +import { describe, expect, it } from 'vitest'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +const REPO_ROOT = path.resolve(__dirname, '..', '..', '..', '..'); + +/** + * Skill directories that intentionally differ between providers. + * + * An entry exempts the complete top-level skill directory in both the + * self-hosted and shipped contexts. Keep this list empty unless a reviewed, + * provider-specific implementation is genuinely required. + */ +const PROVIDER_SPECIFIC_SKILL_EXCEPTIONS = new Set(); + +const CONTEXTS = [ + { + name: 'self-hosted root', + claude: path.join(REPO_ROOT, '.claude', 'skills'), + codex: path.join(REPO_ROOT, '.codex', 'skills'), + }, + { + name: 'shipped skeleton', + claude: path.join(REPO_ROOT, 'codev-skeleton', '.claude', 'skills'), + codex: path.join(REPO_ROOT, 'codev-skeleton', '.codex', 'skills'), + }, +] as const; + +function skillNames(root: string): string[] { + return fs + .readdirSync(root, { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .map((entry) => entry.name) + .sort(); +} + +function relativeFiles(root: string): string[] { + const files: string[] = []; + + function visit(directory: string): void { + for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { + const absolute = path.join(directory, entry.name); + if (entry.isDirectory()) { + visit(absolute); + } else if (entry.isFile()) { + files.push(path.relative(root, absolute)); + } + } + } + + visit(root); + return files.sort(); +} + +describe('Claude/Codex skill parity (issue #1196)', () => { + for (const context of CONTEXTS) { + it(`${context.name}: provider skill sets and file bytes match`, () => { + const claudeSkills = skillNames(context.claude); + const codexSkills = skillNames(context.codex); + const includedClaudeSkills = claudeSkills.filter( + (skill) => !PROVIDER_SPECIFIC_SKILL_EXCEPTIONS.has(skill) + ); + const includedCodexSkills = codexSkills.filter( + (skill) => !PROVIDER_SPECIFIC_SKILL_EXCEPTIONS.has(skill) + ); + + expect(includedCodexSkills).toEqual(includedClaudeSkills); + + for (const skill of includedClaudeSkills) { + const claudeRoot = path.join(context.claude, skill); + const codexRoot = path.join(context.codex, skill); + const files = relativeFiles(claudeRoot); + + expect(relativeFiles(codexRoot), `${context.name}/${skill} file list`).toEqual(files); + for (const relativeFile of files) { + expect( + fs.readFileSync(path.join(codexRoot, relativeFile)), + `${context.name}/${skill}/${relativeFile}` + ).toEqual(fs.readFileSync(path.join(claudeRoot, relativeFile))); + } + } + }); + } + + it('every provider-specific exception names a real skill', () => { + const knownSkills = new Set(CONTEXTS.flatMap((context) => [ + ...skillNames(context.claude), + ...skillNames(context.codex), + ])); + + for (const exception of PROVIDER_SPECIFIC_SKILL_EXCEPTIONS) { + expect(knownSkills.has(exception), `stale provider exception: ${exception}`).toBe(true); + } + }); +}); diff --git a/packages/codev/src/__tests__/update.test.ts b/packages/codev/src/__tests__/update.test.ts index efe9fc8cc..8dc5356f9 100644 --- a/packages/codev/src/__tests__/update.test.ts +++ b/packages/codev/src/__tests__/update.test.ts @@ -100,6 +100,28 @@ describe('update command', () => { expect(result.rootConflicts).toHaveLength(0); }); + it('adds missing Codex skills without overwriting an existing customized skill', async () => { + const projectDir = path.join(testBaseDir, 'codex-skills'); + const customized = path.join(projectDir, '.codex', 'skills', 'arch-init'); + fs.mkdirSync(path.join(projectDir, 'codev'), { recursive: true }); + fs.mkdirSync(customized, { recursive: true }); + fs.writeFileSync(path.join(customized, 'SKILL.md'), 'user-customized codex skill'); + + process.chdir(projectDir); + + const { update } = await import('../commands/update.js'); + const result = await update({ agent: true }); + + expect(fs.readFileSync(path.join(customized, 'SKILL.md'), 'utf-8')).toBe( + 'user-customized codex skill' + ); + expect( + fs.existsSync(path.join(projectDir, '.codex', 'skills', 'afx', 'SKILL.md')) + ).toBe(true); + expect(result.newFiles).toContain('.codex/skills/afx/'); + expect(result.newFiles).not.toContain('.codex/skills/arch-init/'); + }); + it('should return UpdateResult from update()', async () => { const projectDir = path.join(testBaseDir, 'return-test'); fs.mkdirSync(path.join(projectDir, 'codev'), { recursive: true }); @@ -237,6 +259,7 @@ describe('update command', () => { for (const file of result.newFiles) { expect(file).not.toMatch(/^codev\/consult-types\//); expect(file).not.toMatch(/^\.claude\/skills\//); + expect(file).not.toMatch(/^\.codex\/skills\//); } }); @@ -264,7 +287,11 @@ describe('update command', () => { // All template files should have codev/ prefix for (const file of result.newFiles) { expect( - file.startsWith('codev/') || file.startsWith('.claude/') || file === 'CLAUDE.md' || file === 'AGENTS.md' + file.startsWith('codev/') || + file.startsWith('.claude/') || + file.startsWith('.codex/') || + file === 'CLAUDE.md' || + file === 'AGENTS.md' ).toBe(true); } for (const file of result.updated) { From 007d7f46e146b1dfbc643126e2dc89a3e3ecb3ac Mon Sep 17 00:00:00 2001 From: Mohid Makhdoomi Date: Sat, 18 Jul 2026 02:44:44 -0400 Subject: [PATCH 09/23] [PIR #1196] Record implementation verification --- codev/state/pir-1196_thread.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/codev/state/pir-1196_thread.md b/codev/state/pir-1196_thread.md index 0647d6808..7941acc4e 100644 --- a/codev/state/pir-1196_thread.md +++ b/codev/state/pir-1196_thread.md @@ -15,8 +15,7 @@ added recursive parity plus lifecycle preservation coverage. - Verification: dependency/core/types/artifact/codev build chain passed; 58 targeted tests passed; built-CLI smoke tests passed for fresh init, adopt, - and update. The full default suite passed 3,508 tests but has 17 unrelated - environment-sensitive failures in `tower-utils.test.ts` (Codex harness - selected where assertions assume Claude session flags) and - `findWorkspaceRoot.test.ts` (`/tmp` is detected as a workspace). No tests - were skipped or modified to mask these pre-existing failures. + and update. The first full-suite run exposed host contamination from the + global Codex config and `/tmp/.git`; rerunning porch checks with isolated + `HOME`/`TMPDIR` passed the build and all 3,525 executed tests (48 existing + skips). No tests were skipped or modified to mask failures. From 9627330626b4e0a06e9b0bef6d71164a0676449a Mon Sep 17 00:00:00 2001 From: Mohid Makhdoomi Date: Sat, 18 Jul 2026 02:47:00 -0400 Subject: [PATCH 10/23] chore(porch): 1196 dev-approval gate-requested --- codev/projects/1196-ship-codex-skills-in-the-skele/status.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/codev/projects/1196-ship-codex-skills-in-the-skele/status.yaml b/codev/projects/1196-ship-codex-skills-in-the-skele/status.yaml index ddbdabac2..c42a1ed1c 100644 --- a/codev/projects/1196-ship-codex-skills-in-the-skele/status.yaml +++ b/codev/projects/1196-ship-codex-skills-in-the-skele/status.yaml @@ -11,10 +11,11 @@ gates: approved_at: '2026-07-18T06:33:24.369Z' dev-approval: status: pending + requested_at: '2026-07-18T06:45:27.775Z' pr: status: pending iteration: 1 build_complete: false history: [] started_at: '2026-07-18T06:26:40.103Z' -updated_at: '2026-07-18T06:33:29.607Z' +updated_at: '2026-07-18T06:45:27.775Z' From e2e18dfbfbd5690d1dbcfc660f2c62dc43168cbb Mon Sep 17 00:00:00 2001 From: Mohid Makhdoomi Date: Sat, 18 Jul 2026 02:56:30 -0400 Subject: [PATCH 11/23] [PIR #1196] Keep skeleton instruction templates unchanged --- codev-skeleton/templates/AGENTS.md | 2 -- codev-skeleton/templates/CLAUDE.md | 2 -- codev/plans/1196-ship-codex-skills-in-the-skele.md | 14 +++++--------- codev/state/pir-1196_thread.md | 4 ++++ 4 files changed, 9 insertions(+), 13 deletions(-) diff --git a/codev-skeleton/templates/AGENTS.md b/codev-skeleton/templates/AGENTS.md index 1b166cb44..35d705ccf 100644 --- a/codev-skeleton/templates/AGENTS.md +++ b/codev-skeleton/templates/AGENTS.md @@ -62,8 +62,6 @@ This succeeds if the protocol is registered (including via the skeleton fallback - **Plans**: `codev/plans/` - Implementation plans (HOW to build) - **Reviews**: `codev/reviews/` - Reviews and lessons learned - **Protocols**: `codev/protocols/` - Development protocols -- **Claude skills**: `.claude/skills/` - Provider-native Codev command guidance -- **Codex skills**: `.codex/skills/` - Provider-native Codev command guidance ## Working with Project Labels diff --git a/codev-skeleton/templates/CLAUDE.md b/codev-skeleton/templates/CLAUDE.md index a61af14d0..07e5fc4a1 100644 --- a/codev-skeleton/templates/CLAUDE.md +++ b/codev-skeleton/templates/CLAUDE.md @@ -60,8 +60,6 @@ This succeeds if the protocol is registered (including via the skeleton fallback - **Plans**: `codev/plans/` - Implementation plans (HOW to build) - **Reviews**: `codev/reviews/` - Reviews and lessons learned - **Protocols**: `codev/protocols/` - Development protocols -- **Claude skills**: `.claude/skills/` - Provider-native Codev command guidance -- **Codex skills**: `.codex/skills/` - Provider-native Codev command guidance ## Working with Project Labels diff --git a/codev/plans/1196-ship-codex-skills-in-the-skele.md b/codev/plans/1196-ship-codex-skills-in-the-skele.md index 186d23c91..b502abde8 100644 --- a/codev/plans/1196-ship-codex-skills-in-the-skele.md +++ b/codev/plans/1196-ship-codex-skills-in-the-skele.md @@ -54,8 +54,7 @@ be expected to have repository-local framework files. provider-qualified update results/output. 6. Update lifecycle comments/output and user-facing skill/directory documentation to name both `.claude/skills/` and `.codex/skills/`. Keep - `AGENTS.md` and `CLAUDE.md` byte-identical in both the self-hosted root and - skeleton template pairs. + the self-hosted root `AGENTS.md` and `CLAUDE.md` byte-identical. ## Files to Change @@ -83,9 +82,8 @@ be expected to have repository-local framework files. - `codev-skeleton/.claude/skills/codev/SKILL.md` and `codev-skeleton/.codex/skills/codev/SKILL.md` — same documentation in the shipped skill trees. -- `AGENTS.md`, `CLAUDE.md`, `codev-skeleton/templates/AGENTS.md`, and - `codev-skeleton/templates/CLAUDE.md` — accurately show Claude and Codex skill - directories while preserving each required identical pair. +- `AGENTS.md` and `CLAUDE.md` — accurately show Claude and Codex skill + directories while preserving the required identical pair. ## Risks & Alternatives Considered @@ -128,7 +126,5 @@ be expected to have repository-local framework files. adopt/update in temporary directories; inspect both skill trees, then modify a Codex `SKILL.md`, remove a different Codex skill, rerun update, and verify the customized bytes remain while the missing skill returns. -- Documentation guard: assert/diff `AGENTS.md` against `CLAUDE.md`, the two - skeleton templates against each other, and the provider skill trees - recursively. - +- Documentation guard: assert/diff `AGENTS.md` against `CLAUDE.md` and the + provider skill trees recursively. diff --git a/codev/state/pir-1196_thread.md b/codev/state/pir-1196_thread.md index 7941acc4e..83c929c19 100644 --- a/codev/state/pir-1196_thread.md +++ b/codev/state/pir-1196_thread.md @@ -19,3 +19,7 @@ global Codex config and `/tmp/.git`; rerunning porch checks with isolated `HOME`/`TMPDIR` passed the build and all 3,525 executed tests (48 existing skips). No tests were skipped or modified to mask failures. +- Dev-gate feedback: restored `codev-skeleton/templates/{AGENTS,CLAUDE}.md` + unchanged and removed those files from the plan's implementation/test scope. + Provider-location documentation remains in lifecycle output, provider + skills, and the self-hosted root instruction pair. From 8046c421294019af7c145b5e5f47d8a53aac4676 Mon Sep 17 00:00:00 2001 From: Mohid Makhdoomi Date: Sat, 18 Jul 2026 03:15:35 -0400 Subject: [PATCH 12/23] chore(porch): 1196 review phase-transition --- .../1196-ship-codex-skills-in-the-skele/status.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/codev/projects/1196-ship-codex-skills-in-the-skele/status.yaml b/codev/projects/1196-ship-codex-skills-in-the-skele/status.yaml index c42a1ed1c..0902cbbce 100644 --- a/codev/projects/1196-ship-codex-skills-in-the-skele/status.yaml +++ b/codev/projects/1196-ship-codex-skills-in-the-skele/status.yaml @@ -1,7 +1,7 @@ id: '1196' title: ship-codex-skills-in-the-skele protocol: pir -phase: implement +phase: review plan_phases: [] current_plan_phase: null gates: @@ -10,12 +10,13 @@ gates: requested_at: '2026-07-18T06:29:49.252Z' approved_at: '2026-07-18T06:33:24.369Z' dev-approval: - status: pending + status: approved requested_at: '2026-07-18T06:45:27.775Z' + approved_at: '2026-07-18T07:08:00.955Z' pr: status: pending iteration: 1 build_complete: false history: [] started_at: '2026-07-18T06:26:40.103Z' -updated_at: '2026-07-18T06:45:27.775Z' +updated_at: '2026-07-18T07:15:35.471Z' From 0d06a2a7e6b349009678eee522162868001596cf Mon Sep 17 00:00:00 2001 From: Mohid Makhdoomi Date: Sat, 18 Jul 2026 03:19:38 -0400 Subject: [PATCH 13/23] [PIR #1196] Review + retrospective --- codev/resources/arch.md | 12 +- codev/resources/lessons-learned.md | 1 + .../1196-ship-codex-skills-in-the-skele.md | 109 ++++++++++++++++++ codev/state/pir-1196_thread.md | 3 + 4 files changed, 124 insertions(+), 1 deletion(-) create mode 100644 codev/reviews/1196-ship-codex-skills-in-the-skele.md diff --git a/codev/resources/arch.md b/codev/resources/arch.md index c85d13cda..4db67ce98 100644 --- a/codev/resources/arch.md +++ b/codev/resources/arch.md @@ -1733,7 +1733,7 @@ Messages sent via `afx send` are not injected immediately — they pass through **Location**: `commands/whoami.ts` (composes `detectCurrentBuilderId`/`detectWorkspaceRoot` from `commands/send.ts` and `lookupBuilderSpawningArchitect` from `state.ts`) -`afx whoami` reports the current terminal's agent identity (workspace, type, name) from Tower/global.db's perspective. Identity precedence is fixed: **builder-worktree cwd match** (canonical id verified against global.db — same resolution `afx send` uses, including the #1094 rule that an unverifiable worktree identity throws rather than falling through) → **`CODEV_ARCHITECT_NAME`** (the Tower-injected architect env var, read directly — NOT via `currentArchitectName()`, whose `main` default is deliberately not used here) → **unknown** (exit 1, no implicit `main`). Strictly read-only against global.db: `lookupBuilderSpawningArchitect(builderId, workspacePath?, db?)` accepts an optional connection so whoami passes its own readonly handle instead of the read-write `getDb()` singleton. Workspace display name comes from `known_workspaces` with directory-basename fallback (informational field only — fail-loud applies to type/name). `--json` emits `{workspace, type, name, architect?}`; failures emit `{"error": ...}` on stdout plus a human explanation on stderr. Works without Tower running. The shipped `/arch-init` skill (`.claude/skills/arch-init/`, mirrored in `codev-skeleton/`) builds on whoami for architect identity adoption + state recovery from `codev/state/.md`. +`afx whoami` reports the current terminal's agent identity (workspace, type, name) from Tower/global.db's perspective. Identity precedence is fixed: **builder-worktree cwd match** (canonical id verified against global.db — same resolution `afx send` uses, including the #1094 rule that an unverifiable worktree identity throws rather than falling through) → **`CODEV_ARCHITECT_NAME`** (the Tower-injected architect env var, read directly — NOT via `currentArchitectName()`, whose `main` default is deliberately not used here) → **unknown** (exit 1, no implicit `main`). Strictly read-only against global.db: `lookupBuilderSpawningArchitect(builderId, workspacePath?, db?)` accepts an optional connection so whoami passes its own readonly handle instead of the read-write `getDb()` singleton. Workspace display name comes from `known_workspaces` with directory-basename fallback (informational field only — fail-loud applies to type/name). `--json` emits `{workspace, type, name, architect?}`; failures emit `{"error": ...}` on stdout plus a human explanation on stderr. Works without Tower running. The shipped `/arch-init` skill builds on whoami for architect identity adoption + state recovery from `codev/state/.md`; it is maintained in both Claude and Codex provider skill trees. ## Installation Architecture @@ -1762,6 +1762,16 @@ Messages sent via `afx send` are not injected immediately — they pass through - Uses local skeleton (no network dependency) - Preserves existing CLAUDE.md content +Provider-native skills are the intentional exception to runtime-only framework +resolution: Claude and Codex discover skills only from root +`.claude/skills/` and `.codex/skills/`, so `codev init` materializes both from +the embedded package skeleton. `codev adopt` and `codev update` backfill +missing skills independently for each provider and treat an existing complete +skill directory as user-owned, never overwriting it. The two shipped provider +trees are physical mirrors guarded by recursive path-and-byte parity tests; +reviewed provider-specific skills must be named in the test's exception +allowlist. + ## Key Design Decisions ### 1. Context-First Philosophy diff --git a/codev/resources/lessons-learned.md b/codev/resources/lessons-learned.md index f7dea5ef1..c9dcd7bf2 100644 --- a/codev/resources/lessons-learned.md +++ b/codev/resources/lessons-learned.md @@ -371,6 +371,7 @@ Generalizable wisdom extracted from review documents, ordered by impact. Updated - [From #1012] Materialized project starter files belong in `codev-skeleton/templates/` and are copied by the scaffold, never hardcoded as TS string constants — keep the copy-from-skeleton convention uniform across every materialized file. When the desired starter differs from an existing rich template (minimal placeholder vs. the full `arch.md` reference template), add a dedicated `*.starter.md` source rather than inlining content or gutting the reference template. - [From #1012] A placeholder filled by the agent-driven review path needs an explicit "replace me" marker (mirror the hot-tier `` convention), not just an italic `_None yet._` line — otherwise the builder may append below it, leaving a self-contradictory file. The review prompts and `update-arch-docs` skill never mention the placeholder, so the file itself must signal replacement. - [From 778] In a self-hosted Codev repo the four-tier resolver means `codev/` instance copies *shadow* `codev-skeleton/`, so the two trees (and the `codev/` copies themselves) drift independently. A terminology/backend change (Gemini-CLI → `agy`) cost 3 review iterations because each round surfaced another stale copy (skeleton → `DEPENDENCIES.md` → `resources/commands/consult.md` → `codev.md` + `arch.md`). When changing any shared doc, grep BOTH trees in one pass and run `diff codev/ codev-skeleton/` for every shared file — empty diff is the consistency proof. Distinguish in-scope current docs from historical artifacts (`specs/`, `plans/`, dated analyses) which must keep their original wording. +- When provider discovery requires duplicated on-disk trees, make drift failure explicit: recursively compare skill inventories, relative file paths, and bytes in CI, with one reviewed exception allowlist for genuine provider-specific content. Apply customization preservation per provider and per complete skill directory, so backfilling one provider never overwrites or couples the user's other provider tree. ## 3-Way Reviews diff --git a/codev/reviews/1196-ship-codex-skills-in-the-skele.md b/codev/reviews/1196-ship-codex-skills-in-the-skele.md new file mode 100644 index 000000000..f5df7a41d --- /dev/null +++ b/codev/reviews/1196-ship-codex-skills-in-the-skele.md @@ -0,0 +1,109 @@ +# PIR Review: Ship Codex Skills Through the Scaffold Lifecycle + +Fixes #1196 + +## Summary + +Codev now ships a Codex-native skill tree alongside its Claude skill tree and +materializes both during init, adopt, and update. Existing skill directories +remain user-owned, while recursive parity tests prevent the two provider trees +from silently drifting. + +## Files Changed + +The branch changes 59 files (+7,712 / -74) relative to its upstream merge base. +The large root `.codex/skills/**` addition came from the contributor commit +that preceded PIR initialization. + +- `.codex/skills/**` (+6,558 / -0) — self-hosted Codex mirror. +- `codev-skeleton/.codex/skills/**` (+611 / -0) — seven shipped Codex skills. +- `.claude/skills/codev/SKILL.md` and + `codev-skeleton/.claude/skills/codev/SKILL.md` (+4 / -4) — lifecycle and + preservation documentation mirrored into the new Codex trees above. +- `packages/codev/src/lib/scaffold.ts` (+35 / -28) — provider-qualified skill + copy results and per-skill preservation. +- `packages/codev/src/commands/{init,adopt,update}.ts` (+17 / -19) — install, + backfill, and report both provider paths. +- `packages/codev/src/__tests__/{skill-parity,scaffold,init,adopt,update}.test.ts` + (+178 / -20) — fresh install, preservation, backfill, structured output, and + recursive parity coverage. +- `AGENTS.md` and `CLAUDE.md` (+8 / -2) — self-hosted directory map. +- `codev/plans/1196-ship-codex-skills-in-the-skele.md`, + `codev/state/pir-1196_thread.md`, and porch state — protocol artifacts. +- `codev/resources/arch.md` and `codev/resources/lessons-learned.md` — current + materialization architecture and durable parity/preservation guidance. +- `codev/reviews/1196-ship-codex-skills-in-the-skele.md` (+109 / -0) — this + retrospective and PR body. + +Per dev-gate feedback, `codev-skeleton/templates/AGENTS.md` and +`codev-skeleton/templates/CLAUDE.md` are unchanged from upstream. + +## Commits + +- `25e1a000` add `.codex/skills/` +- `6bb0f58a` [PIR #1196] Plan draft +- `743a30e3` [PIR #1196] Scaffold Codex skills across lifecycle +- `e854fcf2` [PIR #1196] Guard provider skill parity and preservation +- `007d7f46` [PIR #1196] Record implementation verification +- `e2e18dfb` [PIR #1196] Keep skeleton instruction templates unchanged + +Porch-generated phase and gate commits are also retained in branch history. + +## Test Results + +- Codev dependency/core/types/artifact/package build chain: ✓ pass +- Targeted lifecycle and parity tests: ✓ 58 tests +- Full default unit suite: ✓ 3,525 executed tests; 48 existing skips +- Porch implement checks: ✓ build and tests +- Built-CLI smoke test: ✓ fresh init installs both trees; adopt and update + preserve customized Codex bytes while restoring a missing Codex skill +- Human dev approval: ✓ approved after reviewing the worktree + +The full suite was run with isolated `HOME` and `TMPDIR` because the host's +global Codex configuration and `/tmp/.git` otherwise alter unrelated harness +and workspace-root tests. + +## Architecture Updates + +Updated the COLD `codev/resources/arch.md` Installation Architecture section. +It now records that provider-native skills are the deliberate exception to +runtime-only framework resolution, describes independent per-provider +materialization/preservation, and names the recursive parity guard. No HOT +change was needed because the existing hot facts already establish the +runtime-resolution and dual-tree rules. + +## Lessons Learned Updates + +Added a COLD documentation lesson to +`codev/resources/lessons-learned.md`: when provider discovery requires physical +duplicates, compare inventories, paths, and bytes in CI through one explicit +exception allowlist, and preserve customizations at the complete-skill +directory boundary for each provider independently. This is useful reference +guidance but does not displace a HOT lesson. + +## Things to Look At During PR Review + +- `copySkills()` now returns provider-qualified relative paths. Verify init, + adopt, and update all consume those paths consistently. +- `skipExisting` protects the complete skill directory independently in each + provider tree; it intentionally does not merge individual files. +- `PROVIDER_SPECIFIC_SKILL_EXCEPTIONS` starts empty and exempts a complete + top-level skill only when a future provider-specific implementation is + reviewed. +- The self-hosted root has ten skills, while the shipped skeleton has seven. + Parity is enforced between providers within each context, not between the + self-hosted and shipped inventories. + +## How to Test Locally + +- **View diff**: VSCode sidebar → right-click builder `pir-1196` → **Review + Diff**. +- **Build**: + `pnpm --filter @cluesmith/codev-types build && pnpm --filter @cluesmith/codev-core build && pnpm --filter @cluesmith/codev-artifact-canvas build && pnpm --filter @cluesmith/codev build` +- **Targeted tests**: + `pnpm --filter @cluesmith/codev exec vitest run src/__tests__/skill-parity.test.ts src/__tests__/scaffold.test.ts src/__tests__/adopt.test.ts src/__tests__/update.test.ts` +- **Verify preservation**: customize + `.codex/skills/arch-init/SKILL.md` in a temporary adopted project, remove a + different Codex skill, run the built `codev update --agent`, and confirm the + customized bytes remain while the missing skill is restored and reported in + `newFiles`. diff --git a/codev/state/pir-1196_thread.md b/codev/state/pir-1196_thread.md index 83c929c19..c54efa97c 100644 --- a/codev/state/pir-1196_thread.md +++ b/codev/state/pir-1196_thread.md @@ -23,3 +23,6 @@ unchanged and removed those files from the plan's implementation/test scope. Provider-location documentation remains in lifecycle output, provider skills, and the self-hosted root instruction pair. +- Dev approval completed. Review phase routes the current materialization model + to COLD `arch.md` and the general duplicate-tree parity/preservation pattern + to COLD `lessons-learned.md`; no HOT displacement is warranted. From 3c0a3ebd896ac36a11d9747bf46782616985cf00 Mon Sep 17 00:00:00 2001 From: Mohid Makhdoomi Date: Sat, 18 Jul 2026 03:19:59 -0400 Subject: [PATCH 14/23] chore(porch): 1196 record PR #1197 --- .../1196-ship-codex-skills-in-the-skele/status.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/codev/projects/1196-ship-codex-skills-in-the-skele/status.yaml b/codev/projects/1196-ship-codex-skills-in-the-skele/status.yaml index 0902cbbce..07949f059 100644 --- a/codev/projects/1196-ship-codex-skills-in-the-skele/status.yaml +++ b/codev/projects/1196-ship-codex-skills-in-the-skele/status.yaml @@ -19,4 +19,9 @@ iteration: 1 build_complete: false history: [] started_at: '2026-07-18T06:26:40.103Z' -updated_at: '2026-07-18T07:15:35.471Z' +updated_at: '2026-07-18T07:19:59.767Z' +pr_history: + - phase: review + pr_number: 1197 + branch: builder/pir-1196 + created_at: '2026-07-18T07:19:59.767Z' From 56de3fabdd6632962e3c739e65fe697b1d930bf9 Mon Sep 17 00:00:00 2001 From: Mohid Makhdoomi Date: Sat, 18 Jul 2026 03:20:16 -0400 Subject: [PATCH 15/23] chore(porch): 1196 review build-complete --- .../projects/1196-ship-codex-skills-in-the-skele/status.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/codev/projects/1196-ship-codex-skills-in-the-skele/status.yaml b/codev/projects/1196-ship-codex-skills-in-the-skele/status.yaml index 07949f059..c693c7ccb 100644 --- a/codev/projects/1196-ship-codex-skills-in-the-skele/status.yaml +++ b/codev/projects/1196-ship-codex-skills-in-the-skele/status.yaml @@ -16,10 +16,10 @@ gates: pr: status: pending iteration: 1 -build_complete: false +build_complete: true history: [] started_at: '2026-07-18T06:26:40.103Z' -updated_at: '2026-07-18T07:19:59.767Z' +updated_at: '2026-07-18T07:20:16.548Z' pr_history: - phase: review pr_number: 1197 From 6c1008feee2ab72e575807e28203e4848e30a1d7 Mon Sep 17 00:00:00 2001 From: Mohid Makhdoomi Date: Sat, 18 Jul 2026 03:24:10 -0400 Subject: [PATCH 16/23] [PIR #1196] Address consultation feedback --- .../1196-review-iter1-rebuttals.md | 37 +++++++++++++++++++ .../1196-ship-codex-skills-in-the-skele.md | 12 ++++-- codev/state/pir-1196_thread.md | 5 +++ packages/codev/src/commands/adopt.ts | 5 ++- packages/codev/src/commands/init.ts | 5 ++- 5 files changed, 57 insertions(+), 7 deletions(-) create mode 100644 codev/projects/1196-ship-codex-skills-in-the-skele/1196-review-iter1-rebuttals.md diff --git a/codev/projects/1196-ship-codex-skills-in-the-skele/1196-review-iter1-rebuttals.md b/codev/projects/1196-ship-codex-skills-in-the-skele/1196-review-iter1-rebuttals.md new file mode 100644 index 000000000..faeed088a --- /dev/null +++ b/codev/projects/1196-ship-codex-skills-in-the-skele/1196-review-iter1-rebuttals.md @@ -0,0 +1,37 @@ +# PIR #1196 Review Iteration 1 Rebuttals + +## Codex: review artifact scope + +**Disposition: rebutted, wording clarified.** + +The review's 59-file count and inclusion of the root `.codex/skills/**` tree +match GitHub's canonical upstream PR: + +```text +gh pr view 1197 --repo cluesmith/codev --json files --jq '.files | length' +59 +``` + +The consultation prompt listed 27 files because its local comparison baseline +already includes contributor commit `25e1a000` (`add .codex/skills/`). Upstream +`cluesmith/codev:main` does not contain that commit, so GitHub correctly +includes the root mirror in PR #1197. The retrospective now avoids a volatile +line-total and explicitly explains why the contributor commit is in the +upstream PR; its **Things to Look At** section also flags the baseline +difference for the human reviewer. + +## Codex: stale init/adopt module headers + +**Disposition: accepted and fixed.** + +Updated `packages/codev/src/commands/init.ts` and +`packages/codev/src/commands/adopt.ts` to describe the actual current +lifecycle: minimal user-owned structure plus materialized provider +skills/root instructions/governance starters, while framework files continue +to resolve from the package at runtime. This is a documentation-only +correction; the already-covered lifecycle behavior is unchanged, so no new +regression test is warranted. + +## Gemini + +Gemini approved with no requested changes. diff --git a/codev/reviews/1196-ship-codex-skills-in-the-skele.md b/codev/reviews/1196-ship-codex-skills-in-the-skele.md index f5df7a41d..b96d433b6 100644 --- a/codev/reviews/1196-ship-codex-skills-in-the-skele.md +++ b/codev/reviews/1196-ship-codex-skills-in-the-skele.md @@ -11,9 +11,10 @@ from silently drifting. ## Files Changed -The branch changes 59 files (+7,712 / -74) relative to its upstream merge base. -The large root `.codex/skills/**` addition came from the contributor commit -that preceded PIR initialization. +GitHub's canonical PR scope contains 59 files. The large root +`.codex/skills/**` addition came from the contributor commit that preceded PIR +initialization; it is part of the upstream PR because that commit is not on +`cluesmith/codev:main`. - `.codex/skills/**` (+6,558 / -0) — self-hosted Codex mirror. - `codev-skeleton/.codex/skills/**` (+611 / -0) — seven shipped Codex skills. @@ -85,6 +86,8 @@ guidance but does not displace a HOT lesson. - `copySkills()` now returns provider-qualified relative paths. Verify init, adopt, and update all consume those paths consistently. +- The init/adopt module headers now distinguish materialized integration and + governance files from framework files that continue to resolve at runtime. - `skipExisting` protects the complete skill directory independently in each provider tree; it intentionally does not merge individual files. - `PROVIDER_SPECIFIC_SKILL_EXCEPTIONS` starts empty and exempts a complete @@ -93,6 +96,9 @@ guidance but does not displace a HOT lesson. - The self-hosted root has ten skills, while the shipped skeleton has seven. Parity is enforced between providers within each context, not between the self-hosted and shipped inventories. +- The Codex consultation reported a 27-file scope because its local-main + baseline already contained contributor commit `25e1a000`; GitHub's actual + upstream PR file list contains 59 files, including that root Codex mirror. ## How to Test Locally diff --git a/codev/state/pir-1196_thread.md b/codev/state/pir-1196_thread.md index c54efa97c..8d68ab5e9 100644 --- a/codev/state/pir-1196_thread.md +++ b/codev/state/pir-1196_thread.md @@ -26,3 +26,8 @@ - Dev approval completed. Review phase routes the current materialization model to COLD `arch.md` and the general duplicate-tree parity/preservation pattern to COLD `lessons-learned.md`; no HOT displacement is warranted. +- Single-pass consultation: Gemini APPROVE; Codex REQUEST_CHANGES. Accepted the + stale init/adopt module-header finding and corrected both comments. Rebutted + the claimed review scope error: GitHub reports 59 PR files, while consult + reported 27 because local `main` already contains contributor commit + `25e1a000`; the review now explains that baseline difference explicitly. diff --git a/packages/codev/src/commands/adopt.ts b/packages/codev/src/commands/adopt.ts index 5088d7946..3e915a36f 100644 --- a/packages/codev/src/commands/adopt.ts +++ b/packages/codev/src/commands/adopt.ts @@ -1,8 +1,9 @@ /** * codev adopt - Add codev to an existing project * - * Creates a codev structure with protocols, roles, consult-types, and - * resource templates copied from the embedded skeleton. + * Creates the minimal user-owned Codev structure and materializes missing + * provider skills, root instructions, and governance starters from the + * embedded skeleton. Framework files resolve from the package at runtime. */ import * as fs from 'node:fs'; diff --git a/packages/codev/src/commands/init.ts b/packages/codev/src/commands/init.ts index 536ce4638..4dccd5e16 100644 --- a/packages/codev/src/commands/init.ts +++ b/packages/codev/src/commands/init.ts @@ -1,8 +1,9 @@ /** * codev init - Create a new codev project * - * Creates a codev structure with protocols, roles, consult-types, and - * resource templates copied from the embedded skeleton. + * Creates the minimal user-owned Codev structure and materializes provider + * skills, root instructions, and governance starters from the embedded + * skeleton. Framework files resolve from the package at runtime. */ import * as fs from 'node:fs'; From 0278d5e23fdf8cd204e27cff4bc44ea33d68e4d1 Mon Sep 17 00:00:00 2001 From: Mohid Makhdoomi Date: Sat, 18 Jul 2026 03:24:18 -0400 Subject: [PATCH 17/23] chore(porch): 1196 pr gate-requested --- .../projects/1196-ship-codex-skills-in-the-skele/status.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/codev/projects/1196-ship-codex-skills-in-the-skele/status.yaml b/codev/projects/1196-ship-codex-skills-in-the-skele/status.yaml index c693c7ccb..591a2ec5e 100644 --- a/codev/projects/1196-ship-codex-skills-in-the-skele/status.yaml +++ b/codev/projects/1196-ship-codex-skills-in-the-skele/status.yaml @@ -15,13 +15,15 @@ gates: approved_at: '2026-07-18T07:08:00.955Z' pr: status: pending + requested_at: '2026-07-18T07:24:18.724Z' iteration: 1 build_complete: true history: [] started_at: '2026-07-18T06:26:40.103Z' -updated_at: '2026-07-18T07:20:16.548Z' +updated_at: '2026-07-18T07:24:18.724Z' pr_history: - phase: review pr_number: 1197 branch: builder/pir-1196 created_at: '2026-07-18T07:19:59.767Z' +pr_ready_for_human: true From bd6027e3a19a2105353abc83f7b58d018cdd52c8 Mon Sep 17 00:00:00 2001 From: Mohid Makhdoomi Date: Sat, 18 Jul 2026 03:44:07 -0400 Subject: [PATCH 18/23] [PIR #1196] Add CI-active init skill coverage --- codev/reviews/1196-ship-codex-skills-in-the-skele.md | 8 ++++++-- codev/state/pir-1196_thread.md | 6 ++++++ packages/codev/src/__tests__/cli/init.e2e.test.ts | 12 ++++++++++++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/codev/reviews/1196-ship-codex-skills-in-the-skele.md b/codev/reviews/1196-ship-codex-skills-in-the-skele.md index b96d433b6..304cf6549 100644 --- a/codev/reviews/1196-ship-codex-skills-in-the-skele.md +++ b/codev/reviews/1196-ship-codex-skills-in-the-skele.md @@ -11,7 +11,7 @@ from silently drifting. ## Files Changed -GitHub's canonical PR scope contains 59 files. The large root +GitHub's canonical PR scope includes the large root `.codex/skills/**` addition came from the contributor commit that preceded PIR initialization; it is part of the upstream PR because that commit is not on `cluesmith/codev:main`. @@ -54,6 +54,8 @@ Porch-generated phase and gate commits are also retained in branch history. - Codev dependency/core/types/artifact/package build chain: ✓ pass - Targeted lifecycle and parity tests: ✓ 58 tests +- Built-CLI integration suite: ✓ 87 tests, including CI-active fresh-init + assertions for matching packaged Claude and Codex skills - Full default unit suite: ✓ 3,525 executed tests; 48 existing skips - Porch implement checks: ✓ build and tests - Built-CLI smoke test: ✓ fresh init installs both trees; adopt and update @@ -98,7 +100,7 @@ guidance but does not displace a HOT lesson. self-hosted and shipped inventories. - The Codex consultation reported a 27-file scope because its local-main baseline already contained contributor commit `25e1a000`; GitHub's actual - upstream PR file list contains 59 files, including that root Codex mirror. + upstream PR scope includes that root Codex mirror. ## How to Test Locally @@ -108,6 +110,8 @@ guidance but does not displace a HOT lesson. `pnpm --filter @cluesmith/codev-types build && pnpm --filter @cluesmith/codev-core build && pnpm --filter @cluesmith/codev-artifact-canvas build && pnpm --filter @cluesmith/codev build` - **Targeted tests**: `pnpm --filter @cluesmith/codev exec vitest run src/__tests__/skill-parity.test.ts src/__tests__/scaffold.test.ts src/__tests__/adopt.test.ts src/__tests__/update.test.ts` +- **Built-CLI integration suite**: + `pnpm --filter @cluesmith/codev test:e2e:cli` - **Verify preservation**: customize `.codex/skills/arch-init/SKILL.md` in a temporary adopted project, remove a different Codex skill, run the built `codev update --agent`, and confirm the diff --git a/codev/state/pir-1196_thread.md b/codev/state/pir-1196_thread.md index 8d68ab5e9..8e9a69de3 100644 --- a/codev/state/pir-1196_thread.md +++ b/codev/state/pir-1196_thread.md @@ -31,3 +31,9 @@ the claimed review scope error: GitHub reports 59 PR files, while consult reported 27 because local `main` already contains contributor commit `25e1a000`; the review now explains that baseline difference explicitly. +- Architect PR review requested CI-active fresh-init coverage and non-volatile + PR-scope wording (the initially requested skill-wording change was explicitly + withdrawn). Follow-up adds provider skill assertions to the built-CLI init + suite while leaving every root/skeleton provider `SKILL.md` byte unchanged. +- Follow-up verification passed: 58 targeted parity/lifecycle tests and all 87 + built-CLI integration tests. diff --git a/packages/codev/src/__tests__/cli/init.e2e.test.ts b/packages/codev/src/__tests__/cli/init.e2e.test.ts index 2a0b2aad0..542a0f007 100644 --- a/packages/codev/src/__tests__/cli/init.e2e.test.ts +++ b/packages/codev/src/__tests__/cli/init.e2e.test.ts @@ -49,6 +49,18 @@ describe('codev init (CLI)', () => { expect(existsSync(join(env.dir, 'my-project/AGENTS.md'))).toBe(true); }); + it('creates Claude and Codex skills from the packaged skeleton', () => { + const result = runCodev(['init', 'my-project', '--yes'], env.dir, env.env); + const base = join(env.dir, 'my-project'); + const claudeSkill = join(base, '.claude/skills/codev/SKILL.md'); + const codexSkill = join(base, '.codex/skills/codev/SKILL.md'); + + expect(result.status).toBe(0); + expect(existsSync(claudeSkill)).toBe(true); + expect(existsSync(codexSkill)).toBe(true); + expect(readFileSync(codexSkill, 'utf-8')).toBe(readFileSync(claudeSkill, 'utf-8')); + }); + it('creates .gitignore', () => { runCodev(['init', 'my-project', '--yes'], env.dir, env.env); expect(existsSync(join(env.dir, 'my-project/.gitignore'))).toBe(true); From f630257cde7a860b650afaf0913d3e7450bcd1e6 Mon Sep 17 00:00:00 2001 From: Mohid Makhdoomi Date: Sat, 18 Jul 2026 03:45:32 -0400 Subject: [PATCH 19/23] [PIR #1196] Fix review scope wording --- codev/reviews/1196-ship-codex-skills-in-the-skele.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/codev/reviews/1196-ship-codex-skills-in-the-skele.md b/codev/reviews/1196-ship-codex-skills-in-the-skele.md index 304cf6549..9b7abfb15 100644 --- a/codev/reviews/1196-ship-codex-skills-in-the-skele.md +++ b/codev/reviews/1196-ship-codex-skills-in-the-skele.md @@ -11,10 +11,9 @@ from silently drifting. ## Files Changed -GitHub's canonical PR scope includes the large root -`.codex/skills/**` addition came from the contributor commit that preceded PIR -initialization; it is part of the upstream PR because that commit is not on -`cluesmith/codev:main`. +GitHub's canonical PR scope includes the large root `.codex/skills/**` addition +from the contributor commit that preceded PIR initialization; it is part of +the upstream PR because that commit is not on `cluesmith/codev:main`. - `.codex/skills/**` (+6,558 / -0) — self-hosted Codex mirror. - `codev-skeleton/.codex/skills/**` (+611 / -0) — seven shipped Codex skills. From 756c4f7131c5e2a8debc8f4c249d0647659ff347 Mon Sep 17 00:00:00 2001 From: Mohid Makhdoomi Date: Sat, 18 Jul 2026 03:51:10 -0400 Subject: [PATCH 20/23] chore(porch): 1196 pr gate-approved --- .../1196-ship-codex-skills-in-the-skele/status.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/codev/projects/1196-ship-codex-skills-in-the-skele/status.yaml b/codev/projects/1196-ship-codex-skills-in-the-skele/status.yaml index 591a2ec5e..57754396a 100644 --- a/codev/projects/1196-ship-codex-skills-in-the-skele/status.yaml +++ b/codev/projects/1196-ship-codex-skills-in-the-skele/status.yaml @@ -14,16 +14,17 @@ gates: requested_at: '2026-07-18T06:45:27.775Z' approved_at: '2026-07-18T07:08:00.955Z' pr: - status: pending + status: approved requested_at: '2026-07-18T07:24:18.724Z' + approved_at: '2026-07-18T07:51:10.277Z' iteration: 1 build_complete: true history: [] started_at: '2026-07-18T06:26:40.103Z' -updated_at: '2026-07-18T07:24:18.724Z' +updated_at: '2026-07-18T07:51:10.278Z' pr_history: - phase: review pr_number: 1197 branch: builder/pir-1196 created_at: '2026-07-18T07:19:59.767Z' -pr_ready_for_human: true +pr_ready_for_human: false From 824053743754e1588974e83ac42f5c48726b19fb Mon Sep 17 00:00:00 2001 From: Mohid Makhdoomi Date: Sat, 18 Jul 2026 03:51:30 -0400 Subject: [PATCH 21/23] chore(porch): 1196 protocol complete --- .../projects/1196-ship-codex-skills-in-the-skele/status.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/codev/projects/1196-ship-codex-skills-in-the-skele/status.yaml b/codev/projects/1196-ship-codex-skills-in-the-skele/status.yaml index 57754396a..fb85d31a6 100644 --- a/codev/projects/1196-ship-codex-skills-in-the-skele/status.yaml +++ b/codev/projects/1196-ship-codex-skills-in-the-skele/status.yaml @@ -1,7 +1,7 @@ id: '1196' title: ship-codex-skills-in-the-skele protocol: pir -phase: review +phase: verified plan_phases: [] current_plan_phase: null gates: @@ -21,7 +21,7 @@ iteration: 1 build_complete: true history: [] started_at: '2026-07-18T06:26:40.103Z' -updated_at: '2026-07-18T07:51:10.278Z' +updated_at: '2026-07-18T07:51:30.422Z' pr_history: - phase: review pr_number: 1197 From bfae767702caa67fb6f62784072d17a62cc7d408 Mon Sep 17 00:00:00 2001 From: Mohid Makhdoomi Date: Wed, 22 Jul 2026 11:00:17 -0400 Subject: [PATCH 22/23] resolve conflict with PR #1213 --- codev/resources/arch.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codev/resources/arch.md b/codev/resources/arch.md index 4db67ce98..64017f2f0 100644 --- a/codev/resources/arch.md +++ b/codev/resources/arch.md @@ -1733,7 +1733,7 @@ Messages sent via `afx send` are not injected immediately — they pass through **Location**: `commands/whoami.ts` (composes `detectCurrentBuilderId`/`detectWorkspaceRoot` from `commands/send.ts` and `lookupBuilderSpawningArchitect` from `state.ts`) -`afx whoami` reports the current terminal's agent identity (workspace, type, name) from Tower/global.db's perspective. Identity precedence is fixed: **builder-worktree cwd match** (canonical id verified against global.db — same resolution `afx send` uses, including the #1094 rule that an unverifiable worktree identity throws rather than falling through) → **`CODEV_ARCHITECT_NAME`** (the Tower-injected architect env var, read directly — NOT via `currentArchitectName()`, whose `main` default is deliberately not used here) → **unknown** (exit 1, no implicit `main`). Strictly read-only against global.db: `lookupBuilderSpawningArchitect(builderId, workspacePath?, db?)` accepts an optional connection so whoami passes its own readonly handle instead of the read-write `getDb()` singleton. Workspace display name comes from `known_workspaces` with directory-basename fallback (informational field only — fail-loud applies to type/name). `--json` emits `{workspace, type, name, architect?}`; failures emit `{"error": ...}` on stdout plus a human explanation on stderr. Works without Tower running. The shipped `/arch-init` skill builds on whoami for architect identity adoption + state recovery from `codev/state/.md`; it is maintained in both Claude and Codex provider skill trees. +`afx whoami` reports the current terminal's agent identity (workspace, type, name) from Tower/global.db's perspective. Identity precedence is fixed: **builder-worktree cwd match** (canonical id verified against global.db — same resolution `afx send` uses, including the #1094 rule that an unverifiable worktree identity throws rather than falling through) → **`CODEV_ARCHITECT_NAME`** (the Tower-injected architect env var, read directly — NOT via `currentArchitectName()`, whose `main` default is deliberately not used here) → **unknown** (exit 1, no implicit `main`). Strictly read-only against global.db: `lookupBuilderSpawningArchitect(builderId, workspacePath?, db?)` accepts an optional connection so whoami passes its own readonly handle instead of the read-write `getDb()` singleton. Workspace display name comes from `known_workspaces` with directory-basename fallback (informational field only — fail-loud applies to type/name). `--json` emits `{workspace, type, name, architect?}`; failures emit `{"error": ...}` on stdout plus a human explanation on stderr. Works without Tower running. The shipped `/arch-init` skill builds on whoami for architect identity adoption + state recovery from `codev/state/.md`; it is maintained in both Claude and Codex provider skill trees. Architect state files are per-person (every team member has their own `main`, so a committed one collides across the team) and gitignored via `codev/state/*.md`; builder thread files (`codev/state/_thread.md`) have the opposite lifecycle and are versioned via a `!codev/state/*_thread.md` negation, since they ship with each builder's PR (Issue #1192). `codev init`/`adopt`/`update` all source this rule pair from the single `CODEV_GITIGNORE_ENTRIES` constant (`packages/codev/src/lib/gitignore.ts`); `codev doctor` audits the split via `auditStateFileIgnore()`. ## Installation Architecture From 6b74a98a3feb5fa6102b9e0e0b507eb5a3fc31bc Mon Sep 17 00:00:00 2001 From: Mohid Makhdoomi Date: Wed, 22 Jul 2026 12:02:37 -0400 Subject: [PATCH 23/23] [Spec 1196] fix: sync .codex arch-init mirror after merging updated .claude skill The merge from main updated .claude/skills/arch-init/SKILL.md (new state-saving guidance from Specs 1192/1134) but left the .codex provider mirrors stale, breaking the skill-parity invariant in both the self-hosted root and the shipped skeleton. Sync both .codex copies to their .claude source; skill-parity.test.ts passes 3/3. Co-Authored-By: Claude Opus 4.8 (1M context) --- .codex/skills/arch-init/SKILL.md | 48 +++++++++++++++++++ .../.codex/skills/arch-init/SKILL.md | 48 +++++++++++++++++++ 2 files changed, 96 insertions(+) diff --git a/.codex/skills/arch-init/SKILL.md b/.codex/skills/arch-init/SKILL.md index 26cacde9d..e032a0f5f 100644 --- a/.codex/skills/arch-init/SKILL.md +++ b/.codex/skills/arch-init/SKILL.md @@ -42,6 +42,9 @@ name in a multi-architect workspace). state. - The state file is authoritative free text. It typically opens with a role banner and may carry resume instructions; follow whatever it says. + - Architect state files are per-person and gitignored + (`codev/state/*.md`); never commit them. Builder `*_thread.md` files + are the opposite: versioned, shipping with each builder PR. 3. **Confirm identity + orient, then follow the state file.** In one tight block, report: who you now are (name + one-line role from the banner, if @@ -51,6 +54,51 @@ name in a multi-architect workspace). on resume. Do not invent a new agenda — resume the one the state file describes. +## Saving your state (and knowing when to `/clear`) + +Recovery is only half the loop. `/arch-init` reads state; **you** write it. The +state file is not crash insurance — it is your deliberate memory-management +mechanism. Auto-compaction happens at an arbitrary moment with content you did +not choose; a state save happens at a boundary **you** pick, with a summary +**you** curate. That is strictly better, so use it: + +``` +/arch-init (recover) → work → save at a checkpoint → suggest /clear → human /clears → /arch-init → … +``` + +**When to save.** Save at a *resumable boundary* — a point a fresh session +could pick up cleanly from. Good moments, judged by you: a gate approval, a PR +merge, a completed investigation, the end of a long tool-heavy stretch. +**Never save mid-task.** The state file must describe a point you can resume +*from*, not a half-finished action; a mid-task snapshot resumes into confusion. + +**How to save (write format = read format).** Recovery reads *the role banner +plus the most recent dated section*, so a save must leave exactly that behind: + +1. **Rewrite the current-state / open-loops section in place** — overwrite it + with where things actually stand now (current focus + open loops + how to + resume). Do not accumulate stale "current state" blocks. +2. **Append one short dated log entry** capturing what changed this stretch. +3. **Keep it to one screen (compaction discipline).** The state file is a + summary, not a transcript. When you append, prune stale dated sections so + the file stays readable at a glance. + +**Content guardrails.** No secrets (tokens, keys, credentials). No transcript +dumps or raw tool output. Include only: current focus, open loops, and the +instructions a fresh session needs to resume. + +**Then — and only then — suggest `/clear`.** Save first, *then* tell the human +it is a good time to clear. You cannot clear your own context and must never +decide unilaterally to lose it; keeping the irreversible step behind a human +keystroke means accepting the suggestion can never lose anything, because the +save already happened. Make the suggestion **advisory, never nagging**, and +only right after a save — e.g.: + +> State saved to `codev/state/.md` — good time to `/clear` if this +> session is feeling heavy. + +Do not repeat it, and do not prompt to `/clear` at any other time. + ## Guardrails (architect-wide; the state file may add more) - **Never auto-approve porch gates.** A gate notification is for the human, diff --git a/codev-skeleton/.codex/skills/arch-init/SKILL.md b/codev-skeleton/.codex/skills/arch-init/SKILL.md index 26cacde9d..e032a0f5f 100644 --- a/codev-skeleton/.codex/skills/arch-init/SKILL.md +++ b/codev-skeleton/.codex/skills/arch-init/SKILL.md @@ -42,6 +42,9 @@ name in a multi-architect workspace). state. - The state file is authoritative free text. It typically opens with a role banner and may carry resume instructions; follow whatever it says. + - Architect state files are per-person and gitignored + (`codev/state/*.md`); never commit them. Builder `*_thread.md` files + are the opposite: versioned, shipping with each builder PR. 3. **Confirm identity + orient, then follow the state file.** In one tight block, report: who you now are (name + one-line role from the banner, if @@ -51,6 +54,51 @@ name in a multi-architect workspace). on resume. Do not invent a new agenda — resume the one the state file describes. +## Saving your state (and knowing when to `/clear`) + +Recovery is only half the loop. `/arch-init` reads state; **you** write it. The +state file is not crash insurance — it is your deliberate memory-management +mechanism. Auto-compaction happens at an arbitrary moment with content you did +not choose; a state save happens at a boundary **you** pick, with a summary +**you** curate. That is strictly better, so use it: + +``` +/arch-init (recover) → work → save at a checkpoint → suggest /clear → human /clears → /arch-init → … +``` + +**When to save.** Save at a *resumable boundary* — a point a fresh session +could pick up cleanly from. Good moments, judged by you: a gate approval, a PR +merge, a completed investigation, the end of a long tool-heavy stretch. +**Never save mid-task.** The state file must describe a point you can resume +*from*, not a half-finished action; a mid-task snapshot resumes into confusion. + +**How to save (write format = read format).** Recovery reads *the role banner +plus the most recent dated section*, so a save must leave exactly that behind: + +1. **Rewrite the current-state / open-loops section in place** — overwrite it + with where things actually stand now (current focus + open loops + how to + resume). Do not accumulate stale "current state" blocks. +2. **Append one short dated log entry** capturing what changed this stretch. +3. **Keep it to one screen (compaction discipline).** The state file is a + summary, not a transcript. When you append, prune stale dated sections so + the file stays readable at a glance. + +**Content guardrails.** No secrets (tokens, keys, credentials). No transcript +dumps or raw tool output. Include only: current focus, open loops, and the +instructions a fresh session needs to resume. + +**Then — and only then — suggest `/clear`.** Save first, *then* tell the human +it is a good time to clear. You cannot clear your own context and must never +decide unilaterally to lose it; keeping the irreversible step behind a human +keystroke means accepting the suggestion can never lose anything, because the +save already happened. Make the suggestion **advisory, never nagging**, and +only right after a save — e.g.: + +> State saved to `codev/state/.md` — good time to `/clear` if this +> session is feeling heavy. + +Do not repeat it, and do not prompt to `/clear` at any other time. + ## Guardrails (architect-wide; the state file may add more) - **Never auto-approve porch gates.** A gate notification is for the human,