diff --git a/.env.example b/.env.example index 4ee4c5d..e257b9a 100644 --- a/.env.example +++ b/.env.example @@ -2,8 +2,9 @@ # # cp .env.example .env # -# Then set GEMINI_API_KEY below and run `docker compose up`. Everything else -# here has a working default. See SELFHOSTING.md for the full walkthrough. +# Then set GEMINI_API_KEY below and run `docker compose up` (or set +# TRAILHEAD_LLM=offline to try it with no key). Everything else here has a +# working default. See SELFHOSTING.md for the full walkthrough. # # Trailhead is self-hosted: there is no hosted API. Every client (browser-ext, # vscode-ext, mcp-server, dashboard) defaults to http://localhost:3000, which @@ -15,10 +16,17 @@ # --- Gemini ------------------------------------------------------------------ # The only value you must supply. Powers /score, /coach, /improve, /diff and -# the rich wiki bootstrap. The API refuses to start without it. -# Get one at https://aistudio.google.com/apikey +# the rich wiki bootstrap. The API refuses to start without it, unless +# TRAILHEAD_LLM=offline. Get one at https://aistudio.google.com/apikey GEMINI_API_KEY= +# gemini (default) | offline. Offline runs with NO model: prompts are scored by +# the rule-based scorer (packages/scoring/src/heuristic-score.mjs), coaching +# text comes from static templates, the rich wiki bootstrap is disabled, and +# every /score and /coach response says so. For trying the stack without a key; +# it never switches on by itself. +# TRAILHEAD_LLM=offline + # ============================================================================= # OPTIONAL — every value below has a working default # ============================================================================= @@ -60,6 +68,11 @@ TRAILHEAD_AUTO_CREATE_TEAMS=false # X-Admin-Token (`init --admin-token`). Leave empty for open registration while # the API is bound to 127.0.0.1; set it before exposing the API on a network. TRAILHEAD_ADMIN_TOKEN= +# The seeded demo team (public secret trailhead_demo_acme_2026). on | off. +# Default: on, but off whenever TRAILHEAD_ADMIN_TOKEN is set, because anyone +# who can reach the API can write to the demo team and spend your Gemini quota +# through it. Set `on` to keep it on a networked deploy anyway. +# TRAILHEAD_DEMO_TEAM= # How /coach prompts join the team library. Both modes require the exact # average >= 7.0, no dimension < 5, and an independent re-score that agrees. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8fb2ca4..c21276c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,7 +86,7 @@ jobs: run: npm audit --audit-level=high docker: - name: docker image builds + name: docker image + compose smoke test runs-on: ubuntu-latest timeout-minutes: 15 steps: @@ -97,6 +97,31 @@ jobs: run: GEMINI_API_KEY=ci-placeholder docker compose config --quiet - name: build api image run: docker build -f apps/api/Dockerfile -t trailhead-api:ci . + # Run the quick start the way a stranger would, with no key (offline + # mode), and check the stack actually works: schema applied, API up, + # registration, a wiki write, a rule-based score, and the demo seed. + - name: compose smoke test (offline mode, demo seed) + env: + TRAILHEAD_LLM: offline + run: | + set -euo pipefail + docker compose -p ci up -d --build --wait --wait-timeout 180 + docker compose -p ci --profile demo run --rm seed + api=http://127.0.0.1:3000 + curl -fsS $api/ | jq -e '.status == "ok" and .llm == "offline"' + secret=$(curl -fsS -X POST $api/teams -H 'content-type: application/json' \ + -d '{"team_id":"team_ci_smoke","name":"ci"}' | jq -er .secret) + curl -fsS -X POST $api/wiki/propose -H "X-Team-Token: $secret" -H 'content-type: application/json' \ + -d '{"node_path":"src/","insight":"Errors return JSON {error, detail}.","user_id":"ci"}' | jq -e '.action == "created"' + curl -fsS -X POST $api/score -H "X-Team-Token: $secret" -H 'content-type: application/json' \ + -d '{"prompt":"fix it","user_id":"ci"}' | jq -e '.scorer == "heuristic" and .overall <= 2' + curl -fsS $api/team/metrics -H 'X-Team-Token: trailhead_demo_acme_2026' | jq -e '.total_obs >= 700 and .active_users == 3' + - name: compose logs + if: failure() + run: docker compose -p ci --profile demo logs + - name: compose down + if: always() + run: docker compose -p ci --profile demo down -v integration: name: api integration tests (Postgres) @@ -131,3 +156,29 @@ jobs: env: TRAILHEAD_IT_DATABASE_URL: postgresql://trailhead:trailhead@localhost:5432/trailhead_it run: npm --workspace=apps/api run test:integration + + package: + name: package extensions + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '22' + cache: npm + - name: install + run: npm ci + # Production builds of both extensions, packed the way a release ships + # them (release.yml attaches the same files to a GitHub release). + - name: browser extension (.zip) + run: npm --workspace=@trailhead/browser-ext run package + - name: VS Code extension (.vsix) + run: npm --workspace=apps/vscode-ext run package + - uses: actions/upload-artifact@v4 + with: + name: extensions + path: | + apps/browser-ext/learnloop-browser-ext-*.zip + apps/vscode-ext/*.vsix + if-no-files-found: error diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..25962da --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,55 @@ +name: release + +# Push a tag vX.Y.Z (from main) and this attaches both extension packages to +# the GitHub release for that tag, creating it as a draft if it doesn't exist +# yet. Publishing the release, and any store listing, stays a human step. +# +# The tag must match the version in apps/browser-ext/manifest.json and +# apps/vscode-ext/package.json, or the job fails before building anything. + +on: + push: + tags: ['v*'] + +permissions: + contents: write + +jobs: + extensions: + name: package and attach extensions + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - name: versions match the tag + run: | + want="${GITHUB_REF_NAME#v}" + for f in apps/browser-ext/manifest.json apps/browser-ext/package.json apps/vscode-ext/package.json; do + have=$(jq -r .version "$f") + if [ "$have" != "$want" ]; then + echo "::error file=$f::version is $have but the tag is $GITHUB_REF_NAME" + exit 1 + fi + done + - uses: actions/setup-node@v4 + with: + node-version: '22' + cache: npm + - name: install + run: npm ci + - name: test + run: npm test + - name: package + run: | + npm --workspace=@trailhead/browser-ext run package + npm --workspace=apps/vscode-ext run package + - name: attach to the release + env: + GH_TOKEN: ${{ github.token }} + run: | + if ! gh release view "$GITHUB_REF_NAME" >/dev/null 2>&1; then + gh release create "$GITHUB_REF_NAME" --draft --verify-tag --generate-notes --title "LearnLoop $GITHUB_REF_NAME" + fi + gh release upload "$GITHUB_REF_NAME" --clobber \ + apps/browser-ext/learnloop-browser-ext-*.zip \ + apps/vscode-ext/*.vsix diff --git a/.gitignore b/.gitignore index 6f709ad..bda116c 100644 --- a/.gitignore +++ b/.gitignore @@ -47,4 +47,5 @@ apps/browser-ext/build/ # VS Code extension build artifacts apps/vscode-ext/*.vsix +apps/browser-ext/*.zip apps/api/eval/results/ diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..d5197b4 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,30 @@ +# Changelog + +## 0.1.0 — unreleased (to be tagged `v0.1.0`) + +The first versioned release: the browser and VS Code extensions are packaged +and attached to the GitHub release. Changes since the PoliHack build, by pull +request: + +- **#19 (2026-08-17) Audit fixes.** Revived the API (every LLM call was in an + infinite recursion), closed the `GET /teams` token leak, made self-hosting + the default with `docker compose`, added wiki export. +- **#20 (2026-09-30) Hardening.** Server-minted, hashed team secrets; + tenant-isolation integration tests; the extension calls the API from its + service worker (Chrome Local Network Access); rate limits; fencing of + team-written text; the scoring eval harness. +- **#21 Landing page.** The waitlist, which only wrote to the visitor's own + `localStorage`, is replaced by links to the code, the self-hosting guide and + the demo; feature claims match the code. +- **#22 API structure.** `app.ts` split into route modules; one Gemini client + for every call; dead learning-extraction code removed. +- **#23 Keyless demo.** `TRAILHEAD_LLM=offline` runs the stack with no model, + using a rule-based scorer that is measured on the golden set and a held-out + set in CI; seeded demo data (`docker compose --profile demo up`); a switch + for the public demo team; a `docker compose` smoke test in CI. +- **#24 Distribution.** Extension icons and packages, a `package` CI job and a + tag-triggered release workflow; MCP configs without machine-specific secret + paths; a coaching directive that adds to the agent's tools instead of + overriding them. +- **#25 Try it and docs.** The landing page scores prompts in the browser; the + README gains an architecture diagram and loses its stale claims. diff --git a/README.md b/README.md index f7ac45f..50f0294 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,25 @@ # LearnLoop -A team-wide prompting coach. Every prompt sent through Claude.ai, VS Code, -Claude Code, or Copilot Chat is scored on five dimensions in real time. Weak -prompts trigger a short teaching loop. Strong prompts feed a team wiki, which -gets injected back into the next person's context — so a team's "way of -prompting" compounds without anyone writing docs. - -> Trailhead is the engineering codename inside the repo; **LearnLoop** is the -> product name on the marketing site. - -- **Landing page / waitlist:** -- **Demo video (3 min walkthrough):** - - - -### Screenshots +A prompting coach for teams. Every prompt sent through Claude.ai, Claude Code, +VS Code or Copilot Chat is scored on five dimensions before it goes out. Weak +prompts get a short teaching loop; strong ones join the team's prompt library, +and the conventions people state are collected into a team wiki that is fed +back into the next person's context. A team's way of prompting compounds +without anyone writing docs. + +- **Try the scorer in your browser:** +- **3-minute walkthrough:** +- **Run the whole stack, no API key needed:** [Try it](#try-it) below + +Built at PoliHack v19 (April 2026, BMW track "Applications that encourage AI +adoption") by [@Bogzx](https://github.com/Bogzx), +[@KunMihai1](https://github.com/KunMihai1), +[@bbeatricecretu](https://github.com/bbeatricecretu) and +[@CosovanuGabi912](https://github.com/CosovanuGabi912). Open source (MIT) and +self-hosted: there is no hosted service to sign up for. + +> *Trailhead* is the codename used inside the code (package names, settings, +> env vars); *LearnLoop* is the product name. @@ -33,385 +38,200 @@ prompting" compounds without anyone writing docs. --- -## The 5-dimension rubric - -Every prompt is scored 0–10 on: - -1. `goal_clarity` — what outcome is being asked for -2. `specificity` — concrete files, functions, errors named -3. `context_loading` — relevant code/docs/examples attached -4. `constraint_articulation` — what *must not* change, perf/style limits -5. `output_specification` — desired shape of the response - -`overall = round(mean of the five dims)`. Below 7 triggers coaching; ≥ 7 lands -silently. Joining the team's prompt library is stricter: the unrounded mean -must be ≥ 7.0, no dimension below 5, and an independent re-score must agree -(optionally plus a teammate's review — see SELFHOSTING.md → Security model). -The rubric is concrete enough that a human reviewer could apply it — the LLM -is the implementation, not the product. Scores come from an LLM and vary run -to run; `apps/api/eval/` measures how much. - ---- +## How it works + +```mermaid +flowchart LR + subgraph clients["Where people prompt"] + BX["Chrome extension
on claude.ai"] + VS["VS Code extension
sidebar"] + MCP["MCP server
Claude Code · Copilot Chat"] + end + DASH["Dashboard
Next.js"] + subgraph api["apps/api · Hono"] + ROUTES["/score · /coach · /improve · /diff
/wiki/* · /prompts/* · /onboard/*"] + LLM["llm.ts"] + end + PG[("Postgres
wiki · library · score history")] + GEM["Gemini
gemini-3-flash-preview"] + RULES["rule-based scorer
TRAILHEAD_LLM=offline"] + LF["Langfuse
optional"] + + BX -- "X-Team-Token" --> ROUTES + VS -- "X-Team-Token" --> ROUTES + MCP -- "X-Team-Token" --> ROUTES + DASH -- "read-only proxy" --> ROUTES + ROUTES --> PG + ROUTES --> LLM + LLM --> GEM + LLM -.-> RULES + GEM -. traces .-> LF +``` -## What's actually implemented - -The repo is a working npm-workspaces monorepo. Six surfaces, all wired to one -backend, all sharing the same TypeScript contract. - -### `apps/api` — Hono backend (TypeScript, Node 22, Postgres) - -Single source of truth. Multi-tenant: each team has a public team id and a -server-minted secret (only its SHA-256 is stored), sent as `X-Team-Token`. -Pre-2026-09-30 tokens derived from the git remote still work behind -`TRAILHEAD_ACCEPT_LEGACY_TOKENS` (deprecated). Routes are in -`apps/api/src/app.ts` (`index.ts` just serves them): - -| Method + Path | What it does | -|---|---| -| `GET /` | Health + endpoint catalog (unauth) | -| `POST /teams` | Register a team (unauth; gated by `TRAILHEAD_ADMIN_TOKEN` when set). Returns `{ team_id, name, secret }` once; `409` if the id is taken — the join flow | -| `POST /teams/rotate-secret` | New secret for the caller's team; the old credential stops working. Upgrades a legacy team | -| `GET /teams` | Resolves the caller's own team. Never returns the secret — `{ name, id, legacy, team_id? }` (`id` is an opaque digest; `team_id` only for non-legacy teams) | -| `POST /score` | 5-dimension Gemini score; writes `skill_observation` rows with a 30 s per-dimension dedup window | -| `POST /coach` | Stateless teach→reveal coaching loop, capped at 5 rounds | -| `POST /capture` | Stores a `(prompt, response, outcome)` capture from any surface | -| `POST /wiki/propose` | Normalize + dedup an insight on `(node_id, body_normalized)`, increment `reinforcement_count`, promote `draft → durable` at ≥ 3 | -| `GET /context?path=` | Ancestor walk: returns every wiki node whose path is a prefix of the file path, plus its durable learnings | -| `GET /examples?path=` | Top graduated prompts for an ancestor of a file path | -| `GET /prompts/proven` | The team's graduated prompts, filterable by score, path, topic | -| `GET /prompts/pending` | Library candidates awaiting review (`TRAILHEAD_PROMOTION_MODE=review`) | -| `POST /prompts/:id/review` | `{ approve: true }` graduates a pending prompt, `false` discards it | -| `GET /search?q=&scope=` | Substring search over rules, durable learnings and graduated prompts | -| `GET /wiki/recent?since=ISO` | Polling endpoint for the VS Code wiki-toast surface | -| `POST /diff` | Picks the closest graduated team prompt by topic + ancestry, scores both prompts, asks Gemini to narrate the difference | -| `POST /improve` | Multi-turn Gemini-driven prompt rewrite, capped at 5 user replies | -| `GET /skill-arc` | Time-series of per-dimension scores (powers the dashboard hero chart) | -| `GET /team/metrics` | Snapshot: avg overall, reuse rate, durable count, draft count, active users | -| `GET /wiki/tree` | Full node + learnings tree | -| `GET /wiki/export` | The whole team wiki as one markdown document (`?drafts=true`, `?format=json`) | -| `POST /onboard/repo` | Bulk-upsert one node per path, idempotent, optional `initial_rules[path]` for seeding `body_md` | -| `POST /onboard/repo/full` | Async rich bootstrap: accepts a folder + file bundle (capped at 16 MB / 2 000 files / 32 KB per file), enqueues a `wiki_jobs` row, three-pass Gemini fan-out via `setImmediate` | -| `GET /onboard/jobs/:id` | Per-path progress for a rich-bootstrap job | -| `DELETE /team/data` | Wipes the requesting team's data; demo team is protected unless `TRAILHEAD_ALLOW_DEMO_RESET=true` | - -LLM work runs through `apps/api/src/gemini.ts`. Model assignments live in -`packages/scoring/src/models.mjs`: `gemini-3-flash-preview` for scoring -(JSON-schema mode), topic extraction and diff narration; `gemma-4-31b-it` for -async learning extraction, where latency is tolerable. - -Every Gemini call is instrumented with **Langfuse** when -`LANGFUSE_PUBLIC_KEY` / `LANGFUSE_SECRET_KEY` are set — one trace per HTTP -request, one nested generation per LLM call, with token usage and latency. -Tracing silently no-ops when keys are missing. - -### `apps/browser-ext` — Chrome MV3 extension for Claude.ai - -Vanilla TypeScript + esbuild. Manifest declares `https://claude.ai/*` as the -content-script host and allowlists `http://localhost/*` for a self-hosted API. -The popup's **API server** row shows and edits that URL. - -Implemented widgets (`src/widgets/`): - -- **Score card** under the textarea — scores on send (not on keystroke), - per-dimension bars, missing-dimension hints -- **Score badge** on each user bubble -- **Prompt diff panel** — "Compare to team" expands a `/diff` view inline -- **Outcome rating** chips on each assistant bubble (👍 / 🤷 / 👎 → `/capture`) -- **Wiki toast** — drops in when `/wiki/recent` polling sees a new learning -- **Improve chat** — multi-turn rewrite using `/improve` -- **Context pill + popup** — pick a wiki node to bias scoring -- **Send-intercept** — on send: `≥ 7` lets the native send fire; `< 7` keeps the - score card up with *Improve* / *Send as-is* / *Edit* and waits for the user. - There is no timer and nothing is ever sent automatically. Fail-open on every - API error. - -Kill-switch: `chrome.storage.local.set({ 'trailhead.disabled': true })` halts -the extension on next page load. - -### `apps/vscode-ext` — VS Code extension - -Sidebar webview registered under the `trailhead` activity bar. Settings expose -`trailhead.apiUrl`, `trailhead.teamToken`, `trailhead.userId`. Same 5-dimension -score-card render as the browser extension, plus a wiki-diff polling loop that -toasts when `/wiki/recent` reports a new insight. - -### `apps/mcp-server` — MCP server for Claude Code + Copilot Chat - -STDIO MCP server. Five hero tools, deliberately collapsed from a previous -seven-tool surface so Copilot's tool selector picks reliably: - -| Tool | Routes to | -|---|---| -| `coach` | `POST /coach` — server-side teach→reveal cycle returns `proceed: true/false` and a rendered `text` block; the directive is a thin "relay text, follow `proceed`" loop | -| `wiki_lookup` | `GET /context` + `GET /examples` (file-path based) and/or `GET /search` (query) | -| `wiki_save` | `POST /wiki/propose` with server-side dedup | -| `wiki_bootstrap` | `POST /onboard/repo` (skeleton) or `POST /onboard/repo/full` (rich, LLM-populated) | -| `wiki_proven_prompts` | `GET /prompts/proven` — the team's graduated prompts, filterable by score, path and topic | - -Plus a `ping` for health checks. - -> **Not published to npm.** The package is `private: true` and neither -> `trailhead-mcp` nor `@trailhead/mcp-server` exists on the registry, so -> `npx trailhead-mcp` does not work. Run it from a clone — see -> [SELFHOSTING.md](SELFHOSTING.md). - -CLI subcommands (`bin/cli.mjs`): - -- `trailhead-mcp init` — per-repo install. Sets up the repo's team, then writes - `.mcp.json` + `CLAUDE.md` for Claude Code and `.vscode/mcp.json` + - `.github/copilot-instructions.md` for Copilot. Idempotent. Credential order: - `--team-token` → `TRAILHEAD_TEAM_TOKEN` → `.trailhead-team` → otherwise - register `team_` via `POST /teams` and save - the returned secret in `.trailhead-team` (gitignored). If the team is already - registered, `init` explains how to join (get the secret from a teammate, - `--team-token`). `--upgrade-legacy` moves a pre-2026-09-30 team to a secret. - The MCP configs reference `.trailhead-team` (`TRAILHEAD_TEAM_FILE`) instead - of embedding the secret. -- `trailhead-mcp bootstrap` — walks the cwd, bundles source files, posts to - `/onboard/repo/full`. Default rich mode shows a live progress bar. Flags: - `--minimal`, `--paths`, `--force`, `--dry-run`, `--yes`. -- `trailhead-mcp reset` — wipes the team's wiki/captures/observations. - -### `apps/dashboard` — Next.js 16 dashboard (Vercel) - -App router, server components for the team view, SWR for the live charts. -Shows one team — the one whose secret is in the server-side -`TRAILHEAD_TEAM_TOKEN`; the browser never sees the secret (client charts go -through a read-only proxy route, `/api/trailhead/*`). Pages (`src/app/`): - -- `/` — team view -- `/skill-arc` — per-dimension team chart driven by `/skill-arc`, - polls every 2 s during the demo -- `/team` — L1→L2 metric cards from `/team/metrics` -- `/wiki` — node tree + durable learnings from `/wiki/tree` -- `/onboarding` — the wiki as an onboarding guide - -### `apps/landing-page` — LearnLoop marketing site - -Single static `index.html` + JSX components loaded at runtime via Babel -standalone. Tailwind via CDN. Sections: hero, problem, solution, features, -demo, footer. Deployed at . - -### Packages - -- `packages/shared` — TypeScript types for every API request/response. Every - surface imports from here so wire shapes can't drift. -- `packages/scoring` — Locked Gemini prompt templates (score, augment, teach, - topic, extract) and pure helpers (`buildAugmentation`, `normalize`, - `normalizePath`, `ancestorPaths`, the teach/skip/success reveal renderers). -- `packages/score-card` — Pure-DOM render function for the 5-dimension card. - Used by the browser extension and the VS Code webview. -- `packages/db` — `schema.sql` (idempotent, every `CREATE` uses `IF NOT - EXISTS`), `migrate.mjs`, `check.mjs`, and `seed.mjs` for the Acme Fintech - demo data. - -### Database (Postgres on Neon) - -Eight tables in `packages/db/schema.sql`: - -- `teams` — tenancy -- `nodes` — one row per folder or file path; carries `body_md` -- `learnings` — accumulated insights with normalize-based dedup, counter, - and `draft | durable` status -- `prompts` — graduated prompt templates with `topic` and `reuse_count` -- `captures` — stored conversations with outcome -- `skill_observations` — per-dimension score writes; `prompt_hash` backs the - 30 s dedup window -- `wiki_jobs` + `wiki_job_paths` — async rich-bootstrap state - -The demo team (`Acme Fintech`, public secret `trailhead_demo_acme_2026`) is hardcoded -into the schema with a fixed UUID so every surface can reference it without a -lookup. +The loop, end to end: + +1. **Score.** A prompt is sent (Chrome extension) or a code task starts + (Claude Code / Copilot Chat via the MCP `coach` tool). The API scores it on + the five dimensions below and records the scores for the team's skill arc. +2. **Coach.** At 7/10 or more it goes through untouched. Below 7, the + extension holds the send and offers *Improve* / *Send as-is* / *Edit*; + nothing is sent automatically. In Claude Code the `coach` tool runs a short + teach → revise loop (at most 5 rounds) on the weakest dimension, using a + teammate's proven prompt as the example when there is one. +3. **Promote.** A prompt that clears a strict bar (unrounded mean ≥ 7, no + dimension below 5, and an independent re-score that agrees) joins the + team's prompt library, optionally after a teammate's review. +4. **Remember.** When someone states a team convention, `wiki_save` records + it; repeated mentions are deduplicated and become *durable* after three. +5. **Reuse.** The next prompt about the same part of the codebase is scored + and coached with that wiki node's conventions and library prompts in + context, so the rubric reads against how this team works. --- -## Repository layout +## The rubric, and how the score is measured -``` -apps/ - api/ Hono + TypeScript backend (Railway) - browser-ext/ Chrome MV3 extension for Claude.ai - vscode-ext/ VS Code IDE extension - mcp-server/ MCP server (Claude Code + Copilot Chat) + CLI - dashboard/ Next.js 16 dashboard (Vercel) - landing-page/ Static marketing site (LearnLoop) -packages/ - shared/ TypeScript types — single source of truth for API shapes - scoring/ Locked Gemini prompt templates + pure helpers - score-card/ Pure-DOM render of the 5-dimension card - db/ Postgres schema, migrations, seed data -docs/ - superpowers/specs/ Design specs - roadmaps/ Per-surface 24h build roadmaps -``` - ---- +Every prompt is scored 0–10 on: -## Quick start +1. `goal_clarity` — the desired outcome is stated unambiguously +2. `specificity` — the change itself is named (files, functions, errors) +3. `context_loading` — the relevant code, docs or examples are referenced +4. `constraint_articulation` — what must not change; limits and invariants +5. `output_specification` — the shape the answer should take -Trailhead is self-hosted. There is no hosted backend to sign up for — you run -the API, and every client points at it. +`overall` is the rounded mean. The rubric text lives in +`packages/scoring/src/score-prompt.mjs`, and the score comes from one Gemini +call, so it varies from run to run. `apps/api/eval/` measures how much on a +golden set of 30 prompts plus a held-out set of 16, reporting run-to-run +spread, how often the coaching and library gates flip, band hits, and whether +better prompts are ranked above worse ones. -### The short way: Docker +What has been measured so far is the **rule-based scorer** +(`packages/scoring/src/heuristic-score.mjs`), the floor the model has to +beat. CI checks it on every change: -Everything you need is Docker and a Gemini API key from -. +| Prompt set | Overall in expected band | Pairs ranked the right way round | +|---|---|---| +| golden (30, visible while its rules were written) | 29/30 | 152/152 | +| held-out (16, never tuned on) | 14/16 | 38/38 | -```bash -cp .env.example .env # then put your Gemini key in it -docker compose up -# → API on http://localhost:3000, Postgres schema applied automatically -``` +The bands were written in this repo, not by independent reviewers, and the +Gemini scorer has not been run on either set yet (it needs a key; one command +in [`apps/api/eval/README.md`](apps/api/eval/README.md)). Known misses and +the full method are documented there. -That is the whole setup. See [SELFHOSTING.md](SELFHOSTING.md) for pointing the -browser extension, VS Code extension, MCP server and dashboard at it, and for -running against an external database instead. +--- -### The long way: local Node + your own Postgres +## Try it -Prerequisites: +**In your browser, 10 seconds:** the [Try it section of the landing +page](https://learnloop-gules.vercel.app/#try) runs the rule-based scorer on +whatever you type. Nothing leaves the page. -- Node `>= 22.6` -- A Postgres database (Neon — `DATABASE_URL` must include `sslmode=require`) -- A Gemini API key from +**The whole stack, no key, about 2 minutes** (Docker with Compose v2): ```bash -# 1. Install workspace dependencies -npm install - -# 2. Configure environment -cp .env.example .env -# Edit .env — fill in DATABASE_URL and GEMINI_API_KEY - -# 3. Apply the schema (idempotent, safe to re-run) -psql "$DATABASE_URL" -f packages/db/schema.sql - -# 4. (optional) Seed the Acme Fintech demo data -node packages/db/seed.mjs - -# 5. Run the API -npm run dev -# → http://localhost:3000 +git clone https://github.com/Bogzx/LearnLoop && cd LearnLoop +TRAILHEAD_LLM=offline docker compose --profile demo up --build +# API on http://localhost:3000, Postgres schema applied, demo team seeded +curl -s -X POST localhost:3000/score -H 'content-type: application/json' \ + -H 'X-Team-Token: trailhead_demo_acme_2026' \ + -d '{"prompt":"fix the retry","user_id":"me"}' ``` -The API refuses to boot without `DATABASE_URL` and `GEMINI_API_KEY`. +Offline mode has no model: scoring uses the rule-based scorer, coaching uses +the static templates, and every response says so. `--profile demo` loads a +fictional team ("Acme Fintech") with a small wiki, four library prompts and +six days of synthetic activity, so the dashboard has something to show. -Before exposing the API beyond `localhost`, read -[SELFHOSTING.md → Security model](SELFHOSTING.md#security-model): set -`TRAILHEAD_ADMIN_TOKEN`, and turn legacy tokens off once your teams have -upgraded. +**With Gemini:** `cp .env.example .env`, put a key from + in it, and `docker compose up`. -### Run individual surfaces +Then point the clients at it, all described in +[SELFHOSTING.md](SELFHOSTING.md): ```bash -# Dashboard (Next.js, port 3001) -NEXT_PUBLIC_API_URL=http://localhost:3000 \ -NEXT_PUBLIC_TEAM_TOKEN=trailhead_demo_acme_2026 \ +# Dashboard on http://localhost:3001 (server-side env; the secret never reaches the browser) +TRAILHEAD_API_URL=http://localhost:3000 TRAILHEAD_TEAM_TOKEN=trailhead_demo_acme_2026 \ npm --workspace=apps/dashboard run dev -# Browser extension — build, then load apps/browser-ext/dist as unpacked -npm --workspace=@trailhead/browser-ext run build -# chrome://extensions → Developer mode → Load unpacked → apps/browser-ext/dist/ - -# VS Code extension — build, then F5 with apps/vscode-ext as the workspace -npm --workspace=apps/vscode-ext run build - -# MCP server — install into a target repo. The package is unpublished -# (private: true), so `npx trailhead-mcp` does NOT work — invoke the CLI by -# path from this clone. It operates on the cwd, so cd into the target first. +# Claude Code / Copilot Chat in one of your repos (the MCP package is not on npm, +# so the CLI runs from this clone; `init` registers the repo's team) cd /path/to/your/repo node /path/to/LearnLoop/apps/mcp-server/bin/cli.mjs init -node /path/to/LearnLoop/apps/mcp-server/bin/cli.mjs bootstrap ``` -### Workspace scripts +Packages of the Chrome extension (`.zip`, load unpacked) and the VS Code +extension (`.vsix`) are built by every CI run (the `extensions` artifact) and +attached to [GitHub releases](https://github.com/Bogzx/LearnLoop/releases) +from v0.1.0; neither is on a store. -```bash -npm run typecheck # tsc --noEmit across all workspaces -npm run lint # ESLint (flat config: eslint.config.mjs) over the whole repo -npm run test # run all workspace tests -npm run build # build all workspaces that expose a build script -``` +Before exposing the API beyond `localhost`, read +[SELFHOSTING.md → Security model](SELFHOSTING.md#security-model). --- -## Environment variables +## What's in the box -Single root `.env.example` — every surface reads from the same set. - -| Var | Used by | Notes | +| Surface | What it does | Code | |---|---|---| -| `DATABASE_URL` | api | Postgres connection string, `sslmode=require` | -| `GEMINI_API_KEY` | api | `gemini-3-flash-preview` + `gemma-4-31b-it` | -| `LANGFUSE_PUBLIC_KEY` | api | Optional. Hosted Langfuse public key (`pk-lf-…`) | -| `LANGFUSE_SECRET_KEY` | api | Optional. Hosted Langfuse secret key (`sk-lf-…`) | -| `LANGFUSE_BASEURL` | api | Defaults to `https://cloud.langfuse.com` (EU). Use `https://us.cloud.langfuse.com` for US | -| `TEAM_TOKEN` | — | Documentation only: the public demo team's token. The API does not read it; clients hardcode the same value as their fallback | -| `PORT` | api | Defaults to 3000; Railway injects automatically | -| `TRAILHEAD_ADMIN_TOKEN` | api | When set, `POST /teams` (registration) requires it as `X-Admin-Token` | -| `TRAILHEAD_ACCEPT_LEGACY_TOKENS` | api | Default `true`. Accept pre-2026-09-30 remote-derived tokens for teams without a secret (deprecated) | -| `TRAILHEAD_AUTO_CREATE_TEAMS` | api | Default `false`. Legacy only: unknown tokens create legacy teams | -| `TRAILHEAD_SCORE_TEMPERATURE` / `TRAILHEAD_SCORE_THINKING_BUDGET` | api | Scorer sampling (defaults `0.2` / `-1` = dynamic). Measure before changing: `apps/api/eval/` | -| `TRAILHEAD_RL_REGISTER_PER_IP` / `TRAILHEAD_RL_LLM_PER_TEAM` / `TRAILHEAD_RL_LLM_PER_IP` / `TRAILHEAD_RL_BOOTSTRAP_PER_TEAM` | api | Rate limits as `N/W` (defaults `10/1h`, `120/1m`, `120/1m`, `6/1h`), or `off`. In-process, so per replica — see SELFHOSTING.md | -| `TRAILHEAD_RATE_LIMIT` | api | `off` disables every rate limit | -| `TRAILHEAD_TRUST_PROXY` | api | `true` behind your own (single-hop) reverse proxy: per-IP limits key on the last `X-Forwarded-For` entry, the one the proxy appended | -| `TRAILHEAD_EXPOSE_ERRORS` | api | `true` to include the raw error message in 500 responses (local debugging). Default: only a `request_id` that matches the server log | -| `TRAILHEAD_PROMOTION_MODE` | api | `auto` (default): gated auto-promotion into the library. `review`: promoted prompts wait for a teammate's approval | -| `TRAILHEAD_ALLOW_DEMO_RESET` | api | `true` to allow `DELETE /team/data` on the demo team | -| `TRAILHEAD_API_URL` | dashboard | Server-side, runtime. Where the dashboard fetches (fallback: legacy `NEXT_PUBLIC_API_URL`) | -| `TRAILHEAD_TEAM_TOKEN` | dashboard | Server-side, runtime. The team secret; never sent to the browser (fallback: legacy `NEXT_PUBLIC_TEAM_TOKEN`) | -| `trailhead.apiUrl` / `.teamToken` / `.userId` / `.shareUserId` | vscode-ext | VS Code settings. `userId` empty = random per-install id; `shareUserId: false` sends `anonymous` | -| `TRAILHEAD_USER_ID` / `TRAILHEAD_SHARE_USER_ID` | mcp-server | Override the per-machine anonymous id, or `false` to send `anonymous` (see SELFHOSTING.md → Security model) | -| `TRAILHEAD_API_URL` / `TRAILHEAD_TEAM_FILE` / `TRAILHEAD_TEAM_TOKEN` | mcp-server | Per-repo MCP config. `init` writes `TEAM_FILE` (path to `.trailhead-team`); `TEAM_TOKEN` overrides it | +| API | Hono + TypeScript on Node 22, raw `pg`. Multi-tenant: each team has a public id and a server-minted secret (stored hashed). Scoring, coaching, library, wiki, rich bootstrap, rate limits, Langfuse tracing. | [`apps/api`](apps/api) — routes in [its README](apps/api/README.md) | +| Chrome extension | MV3, on claude.ai. Score card under the composer, score badges on each message, *Compare to team* diff, *Improve* chat, outcome chips, wiki toasts, context picker. Every API call goes through the service worker. | [`apps/browser-ext`](apps/browser-ext) | +| VS Code extension | Sidebar with the same score card, team examples for the open file, wiki-update toasts. | [`apps/vscode-ext`](apps/vscode-ext) | +| MCP server | STDIO server for Claude Code and Copilot Chat: `coach`, `wiki_lookup`, `wiki_save`, `wiki_bootstrap`, `wiki_proven_prompts` (+ `ping`), and a CLI: `init`, `bootstrap`, `reset`. | [`apps/mcp-server`](apps/mcp-server) | +| Dashboard | Next.js 16: team skill arc, metrics, wiki tree, onboarding view; reads through a server-side proxy so the team secret stays on the server. | [`apps/dashboard`](apps/dashboard) | +| Landing page | Static page with the in-browser scorer. | [`apps/landing-page`](apps/landing-page) | +| Shared packages | `shared` (every request/response type), `scoring` (prompts, rubric, rule-based scorer, renderers), `score-card` (DOM score card), `db` (schema, migrations, demo seed). | [`packages/`](packages) | + +Postgres holds eight tables (`packages/db/schema.sql`): teams, wiki nodes, +learnings, library prompts, captures, skill observations, and the two +rich-bootstrap job tables. --- -## Deployment +## Development -- **API** → Railway. `railway.json` declares `npm --workspace=apps/api start` - with healthcheck on `/`. -- **Dashboard** → Vercel. Set `TRAILHEAD_API_URL` and `TRAILHEAD_TEAM_TOKEN` - (server-side env), then `vercel --prod` from `apps/dashboard/`. Anyone who - can open it can read that team's data (read-only), so restrict access. -- **Landing page** → Vercel — already live at - . -- **Browser extension** → loaded unpacked from `apps/browser-ext/dist/`. -- **VS Code extension** → `vsce package` from `apps/vscode-ext/`. -- **MCP server** → not published to npm (`private: true`). Wired into a repo by - running `apps/mcp-server/bin/cli.mjs init` from a clone (per-repo wiring, - multi-tenant token derivation from the git remote). +Node ≥ 22.6, npm workspaces. ---- +```bash +npm ci +npm run typecheck # tsc --noEmit in every workspace +npm run lint # ESLint flat config +npm test # every workspace's tests, including the scorer baseline +npm run build # every workspace with a build script + +# API integration tests against a throwaway Postgres (they wipe it): +docker run -d --rm --name trailhead-it -p 55432:5432 -e POSTGRES_USER=trailhead \ + -e POSTGRES_PASSWORD=trailhead -e POSTGRES_DB=trailhead_it postgres:16-alpine +TRAILHEAD_IT_DATABASE_URL=postgresql://trailhead:trailhead@127.0.0.1:55432/trailhead_it \ + npm --workspace=apps/api run test:integration +``` -## How the pieces fit - -1. Engineer types a prompt and hits send. The extension intercepts the send and - calls `/score`. The card mounts under the textarea with five per-dimension - bars and missing-dimension hints. -2. `≥ 7` sends straight through. Below 7 the card stays up with *Improve* / - *Send as-is* / *Edit* and waits for an explicit choice — no timer, no - auto-send. Each `/score` writes 5 `skill_observation` rows; the dashboard's - `/skill-arc` chart polls every 2 s, so the rightmost bucket climbs as the - user prompts. -3. In Claude Code or Copilot Chat, the MCP server's `coach` tool is called - first. Server returns `proceed: false` plus a teach-block when the score - is low; the host LLM relays the block, gathers a reply, calls back. Five - rounds max, then a reveal block shows the score arc and prompt diff. -4. When the user states a teamwide convention, `wiki_save` calls - `POST /wiki/propose`. Server-side normalize + dedup means repeated calls - reinforce the same draft instead of duplicating; `reinforcement_count >= 3` - promotes `draft → durable`. The VS Code extension polls `/wiki/recent` and - toasts the update — visible proof of the autonomous loop. -5. Next prompt the same engineer (or a teammate) types in the same path - triggers `/score` again, but now `context_path` pulls the team's HCL - bundle into Gemini's system prompt, so the rubric is calibrated against - the team's own conventions. +CI runs all of that, plus `npm audit`, a Docker build with a +`docker compose up` smoke test in offline mode, and packaging of both +extensions. A `v*` tag attaches the extension packages to a draft release +(`.github/workflows/release.yml`). ---- +Without Docker, the API runs against any Postgres: +`psql "$DATABASE_URL" -f packages/db/schema.sql`, optionally +`node packages/db/seed.mjs`, then `npm run dev` (reads `.env`). + +## Deployment + +- **API:** self-hosted. `docker compose` as above, or any Node 22 host + (`railway.json` is kept for Railway). Every environment variable is in + [SELFHOSTING.md → All variables](SELFHOSTING.md#all-variables). +- **Dashboard:** Vercel or any Next.js host. Set `TRAILHEAD_API_URL` and + `TRAILHEAD_TEAM_TOKEN` server-side; anyone who can open it can read that + team's data, so restrict access. +- **Landing page:** static files on Vercel. +- **Extensions:** from the release packages, loaded unpacked / installed from + the `.vsix`. ## Specs -The project was specced before it was built. Source of truth for *why*: +The project was specced before it was built; the specs are the record of +*why*: - `docs/superpowers/specs/2026-04-25-trailhead-design.md` — master spec - `docs/superpowers/specs/2026-04-25-mcp-plugin-ux-design.md` — MCP install @@ -428,23 +248,8 @@ The project was specced before it was built. Source of truth for *why*: improve widget - `docs/roadmaps/` — per-surface 24-hour build plans -Each app and package also has its own `README.md` covering surface-specific -contracts, builds, and tests. +Each app and package has its own README with surface-specific details. ---- +## License -## Tech stack - -- **Backend:** Hono, TypeScript, Node 22, `@hono/node-server`, raw `pg` -- **DB:** Postgres on Neon, no ORM -- **LLMs:** `gemini-3-flash-preview` (scoring, JSON-schema mode), `gemma-4-31b-it` - (diff narration, rich bootstrap) -- **Observability:** Langfuse (hosted) — one trace per request, one - generation per LLM call -- **Frontend:** Next.js 16 + Tailwind + Recharts + SWR (dashboard); vanilla - TS + esbuild (extensions); React via CDN (landing page) -- **MCP:** `@modelcontextprotocol/sdk`, STDIO transport -- **Build:** npm workspaces; per-package `tsc` / `esbuild` -- **Hosts:** self-hosted API (see SELFHOSTING.md; `railway.json` remains for - anyone who wants a Railway deploy), Vercel (dashboard + landing page), per-repo - MCP wired from a clone via `apps/mcp-server/bin/cli.mjs init` (unpublished) +MIT — see [LICENSE](LICENSE). diff --git a/SELFHOSTING.md b/SELFHOSTING.md index 5cc7866..09eb0bd 100644 --- a/SELFHOSTING.md +++ b/SELFHOSTING.md @@ -2,7 +2,7 @@ Trailhead is self-hosted. There is no hosted API and no account to sign up for — you run the backend, and it is yours. The only external dependency is a Gemini -API key. +API key, and even that is optional while you try it out (offline mode, below). Everything below assumes you are at the repo root. @@ -13,7 +13,7 @@ Everything below assumes you are at the repo root. | | | |---|---| | **Docker** | Docker Desktop, OrbStack, or Docker Engine with the Compose v2 plugin (`docker compose version`). | -| **A Gemini API key** | Free at . | +| **A Gemini API key** | Free at . Optional for a first look: see [Try it without a key](#try-it-without-a-key-offline-mode). | | **Node 22.6+** | Only needed to run the *clients* (browser extension, VS Code extension, MCP server, dashboard). The API itself runs entirely inside Docker. | --- @@ -41,8 +41,38 @@ curl http://localhost:3000 # {"name":"trailhead-api","status":"ok",...} ``` -If `GEMINI_API_KEY` is missing, `docker compose up` stops immediately and tells -you so — it will not start a stack that cannot score a prompt. +If `GEMINI_API_KEY` is missing (and `TRAILHEAD_LLM` is not `offline`), the API +refuses to start and logs what to set; Compose keeps restarting it until you +fix `.env`. It never quietly falls back to offline mode. + +### Try it without a key (offline mode) + +```bash +TRAILHEAD_LLM=offline docker compose --profile demo up --build +``` + +Same stack, no model, nothing sent to Google. `--profile demo` also runs the +`seed` service once, which fills the public demo team (`trailhead_demo_acme_2026`) +with synthetic data, so the dashboard has something to show: a small wiki, +four library prompts, and six days of prompt activity from three made-up users +(`packages/db/seed.mjs`; re-run it to move the activity window up to now). +Drop `--profile demo` for an empty stack. + + +- `/score` and `/coach` use the rule-based scorer in + `packages/scoring/src/heuristic-score.mjs`. It reads surface features (file + paths, identifiers, constraint and output phrasing), so it is a reasonable + floor, not a substitute for the model; `apps/api/eval/README.md` has its + numbers and known misses. Every response carries `"scorer": "heuristic"`, + and coaching text ends with a note saying no model was involved. +- Coaching uses the static per-dimension templates (plus your team's library + prompts when there are any). `/improve` asks the template question for each + weak dimension and appends your answers; `/diff` names the biggest gap. +- The rich wiki bootstrap is unavailable (`503`); `bootstrap --minimal` works. +- `GET /` reports `"llm": "offline"`, and the API logs a warning at startup. +- Library promotion still applies its gate, but the confirming re-score is the + same deterministic rules, so it confirms nothing. Set + `TRAILHEAD_PROMOTION_MODE=review` if a team will rely on the library. Run it in the background with `docker compose up -d`, and stop it with `docker compose down`. @@ -55,14 +85,48 @@ ones you are most likely to touch: | Variable | Default | Why you'd change it | |---|---|---| -| `GEMINI_API_KEY` | *(required)* | — | +| `GEMINI_API_KEY` | *(required unless offline)* | — | +| `TRAILHEAD_LLM` | `gemini` | `offline` to run with no model (see above). | | `PORT` | `3000` | Something else already owns port 3000. Changing this means updating each client's API URL too. | | `POSTGRES_PORT` | `5432` | You already run Postgres locally. | | `DATABASE_URL` | *(the bundled Postgres)* | Use an external database (Neon, RDS) instead of the container. | | `TRAILHEAD_ADMIN_TOKEN` | *(empty)* | Set it to restrict team registration to people you give it to. Do this before exposing the API. | +| `TRAILHEAD_DEMO_TEAM` | `on` (`off` when an admin token is set) | Turn the public demo team on or off explicitly. | | `TRAILHEAD_ACCEPT_LEGACY_TOKENS` | `true` | Set `false` once every team has upgraded from a pre-2026-09-30 token. | | `TRAILHEAD_AUTO_CREATE_TEAMS` | `false` | Legacy only: let unknown tokens create teams on the fly. Throwaway demos only. | +### All variables + +Single root `.env` (template: `.env.example`); every surface reads from the +same set. + +| Var | Used by | Notes | +|---|---|---| +| `DATABASE_URL` | api | Postgres connection string. Unset with compose: the bundled Postgres. Hosted databases usually need `?sslmode=require` | +| `GEMINI_API_KEY` | api | Used for every call (`gemini-3-flash-preview`) | +| `TRAILHEAD_LLM` | api | `gemini` (default) or `offline`: no model; rule-based scoring and template coaching (SELFHOSTING.md) | +| `LANGFUSE_PUBLIC_KEY` | api | Optional. Hosted Langfuse public key (`pk-lf-…`) | +| `LANGFUSE_SECRET_KEY` | api | Optional. Hosted Langfuse secret key (`sk-lf-…`) | +| `LANGFUSE_BASEURL` | api | Defaults to `https://cloud.langfuse.com` (EU). Use `https://us.cloud.langfuse.com` for US | +| `TEAM_TOKEN` | — | Documentation only: the public demo team's token. The API does not read it; clients hardcode the same value as their fallback | +| `PORT` | api | Defaults to 3000 (with compose, the host port); platforms like Railway inject it | +| `TRAILHEAD_ADMIN_TOKEN` | api | When set, `POST /teams` (registration) requires it as `X-Admin-Token`, and the public demo team is turned off | +| `TRAILHEAD_DEMO_TEAM` | api | `on` / `off`: the public demo team (default on, off when `TRAILHEAD_ADMIN_TOKEN` is set) | +| `TRAILHEAD_ACCEPT_LEGACY_TOKENS` | api | Default `true`. Accept pre-2026-09-30 remote-derived tokens for teams without a secret (deprecated) | +| `TRAILHEAD_AUTO_CREATE_TEAMS` | api | Default `false`. Legacy only: unknown tokens create legacy teams | +| `TRAILHEAD_SCORE_TEMPERATURE` / `TRAILHEAD_SCORE_THINKING_BUDGET` | api | Scorer sampling (defaults `0.2` / `-1` = dynamic). Measure before changing: `apps/api/eval/` | +| `TRAILHEAD_RL_REGISTER_PER_IP` / `TRAILHEAD_RL_LLM_PER_TEAM` / `TRAILHEAD_RL_LLM_PER_IP` / `TRAILHEAD_RL_BOOTSTRAP_PER_TEAM` | api | Rate limits as `N/W` (defaults `10/1h`, `120/1m`, `120/1m`, `6/1h`), or `off`. In-process, so per replica — see SELFHOSTING.md | +| `TRAILHEAD_RATE_LIMIT` | api | `off` disables every rate limit | +| `TRAILHEAD_TRUST_PROXY` | api | `true` behind your own (single-hop) reverse proxy: per-IP limits key on the last `X-Forwarded-For` entry, the one the proxy appended | +| `TRAILHEAD_EXPOSE_ERRORS` | api | `true` to include the raw error message in 500 responses (local debugging). Default: only a `request_id` that matches the server log | +| `TRAILHEAD_PROMOTION_MODE` | api | `auto` (default): gated auto-promotion into the library. `review`: promoted prompts wait for a teammate's approval | +| `TRAILHEAD_ALLOW_DEMO_RESET` | api | `true` to allow `DELETE /team/data` on the demo team | +| `TRAILHEAD_API_URL` | dashboard | Server-side, runtime. Where the dashboard fetches (fallback: legacy `NEXT_PUBLIC_API_URL`) | +| `TRAILHEAD_TEAM_TOKEN` | dashboard | Server-side, runtime. The team secret; never sent to the browser (fallback: legacy `NEXT_PUBLIC_TEAM_TOKEN`) | +| `trailhead.apiUrl` / `.teamToken` / `.userId` / `.shareUserId` | vscode-ext | VS Code settings. `userId` empty = random per-install id; `shareUserId: false` sends `anonymous` | +| `TRAILHEAD_USER_ID` / `TRAILHEAD_SHARE_USER_ID` | mcp-server | Override the per-machine anonymous id, or `false` to send `anonymous` (see SELFHOSTING.md → Security model) | +| `TRAILHEAD_API_URL` / `TRAILHEAD_TEAM_FILE` / `TRAILHEAD_TEAM_TOKEN` | mcp-server | Per-repo MCP config. `init` writes `TEAM_FILE` (path to `.trailhead-team`); `TEAM_TOKEN` overrides it | + ### Data management ```bash @@ -212,6 +276,7 @@ The defaults are safe for a local setup because both ports are bound to can reach the port can create teams and spend your Gemini quota, and can *squat* a repo's derived team id before the real team registers it. With it set, pass it to `init --admin-token` (or hand people pre-made secrets). + Setting it also turns the public demo team off (below). - **Turn legacy tokens off** (`TRAILHEAD_ACCEPT_LEGACY_TOKENS=false`) once every team has run `init --upgrade-legacy`. A legacy token is `repo_` + SHA-256 of the raw remote URL: anyone who knows or guesses the URL @@ -232,7 +297,10 @@ The defaults are safe for a local setup because both ports are bound to metrics through it. Put it behind your SSO/VPN if that matters. - **The demo team's secret `trailhead_demo_acme_2026` is public** (it is in this repo). The demo team is protected from `DELETE /team/data` and from secret - rotation, but not from writes. + rotation, but not from writes or from spending your quota. It is **off by + default once `TRAILHEAD_ADMIN_TOKEN` is set**: its secret then gets + `401 demo_team_disabled`. `TRAILHEAD_DEMO_TEAM=on|off` overrides that either + way, and `GET /` reports the current setting as `demo_team`. **Rate limits.** The API limits the calls that cost you something, with in-process token buckets (a limit of `N/W` allows a burst of N and refills at N @@ -307,7 +375,8 @@ sent the same `demo` id.) Where prompts go: every scored prompt, any wiki context attached to it, and — for the default rich `bootstrap` — the first 8,000 characters of up to 500 source files (5 levels deep) are sent to Google's Gemini API -using your `GEMINI_API_KEY`. When `LANGFUSE_*` keys are set, the same model +using your `GEMINI_API_KEY` (in offline mode nothing is sent to Google). When +`LANGFUSE_*` keys are set, the same model inputs and outputs are also sent to Langfuse. The browser extension's 👍/🤷/👎 chips store the prompt *and Claude's full reply* in the `captures` table. `bootstrap` picks files by extension (so `.env` and key files are never read) @@ -320,9 +389,10 @@ uploaded. ## Troubleshooting -**`docker compose up` exits with "required variable GEMINI_API_KEY is missing"** +**The API restarts in a loop and logs "GEMINI_API_KEY not set"** You skipped `cp .env.example .env`, or left the key blank. `.env` must be at the -repo root, next to `docker-compose.yml`. +repo root, next to `docker-compose.yml`. To try the stack without a key, set +`TRAILHEAD_LLM=offline` instead. **API restarts in a loop / `DATABASE_URL not set`** `DATABASE_URL` is set in your `.env` but points somewhere unreachable. Comment diff --git a/apps/api/README.md b/apps/api/README.md index 9734d27..32bbee0 100644 --- a/apps/api/README.md +++ b/apps/api/README.md @@ -6,18 +6,61 @@ artifact talks to. Single source of truth. **Tech:** Hono + TypeScript, run via `docker compose up` from the repo root (see `SELFHOSTING.md`). Any Postgres (bundled container, Neon, RDS). -**Endpoints (spec §3 — partial; the full, current table is in the root README):** -- `POST /score` — 5-dim Gemini score; writes skill_observation inline -- `POST /capture` — store conversation + outcome -- `GET /context` — HCL bundle (path-walked, spec §8) -- `GET /examples` — team-anchored prompts for a path -- `POST /diff` — Prompt Diff synthesis (Gemini) -- `POST /wiki/propose` — autonomous wiki update with normalize + dedup + counter - -**Imports:** `packages/shared` (types), `packages/scoring` (Gemini -prompts), `packages/db` (schema). - -**Spec refs:** §3, §4, §5, §10 +**Layout** (`src/`): + +| File | Role | +|---|---| +| `index.ts` | Entry point: env checks, startup migrations, listener, graceful shutdown | +| `app.ts` | The Hono app: CORS, body caps, tracing, auth and rate-limit middleware, `GET /`, route mounting, error handler | +| `http.ts` | Shared request plumbing: env type, tenant policy, rate limiting, prompt cap, skill-observation writes | +| `routes/score.ts` | `POST /score`, `POST /capture` | +| `routes/coach.ts` | `POST /coach` (teach → reveal loop) | +| `routes/assist.ts` | `POST /diff`, `POST /improve` | +| `routes/wiki.ts` | `/wiki/propose`, `/context`, `/examples`, `/search`, `/wiki/recent`, `/wiki/tree`, `/wiki/export` | +| `routes/prompts.ts` | `/prompts/proven`, `/prompts/pending`, `/prompts/:id/review` | +| `routes/metrics.ts` | `/skill-arc`, `/team/metrics` | +| `routes/teams.ts` | `/teams` (register, probe, rotate), `DELETE /team/data` | +| `routes/onboard.ts` | `/onboard/repo`, `/onboard/repo/full`, `/onboard/jobs/:id` | +| `gemini.ts` | The one Gemini client: retries, timeouts, Langfuse tracing, every LLM call | +| `wiki-bootstrap-job.ts` | The async three-pass rich bootstrap behind `/onboard/repo/full` | +| `db.ts` | `pg` pool, team resolution and registration | + +## Routes + +Every route except `GET /` and `POST /teams` needs `X-Team-Token` (the team +secret). `GET /` lists them all, and the integration suite fails if one listed +there has no test. + +| Method + Path | What it does | +|---|---| +| `GET /` | Health + endpoint catalog (unauth) | +| `POST /teams` | Register a team (unauth; gated by `TRAILHEAD_ADMIN_TOKEN` when set). Returns `{ team_id, name, secret }` once; `409` if the id is taken — the join flow | +| `POST /teams/rotate-secret` | New secret for the caller's team; the old credential stops working. Upgrades a legacy team | +| `GET /teams` | Resolves the caller's own team. Never returns the secret — `{ name, id, legacy, team_id? }` (`id` is an opaque digest; `team_id` only for non-legacy teams) | +| `POST /score` | 5-dimension Gemini score; writes `skill_observation` rows with a 30 s per-dimension dedup window | +| `POST /coach` | Stateless teach→reveal coaching loop, capped at 5 rounds | +| `POST /capture` | Stores a `(prompt, response, outcome)` capture from any surface | +| `POST /wiki/propose` | Normalize + dedup an insight on `(node_id, body_normalized)`, increment `reinforcement_count`, promote `draft → durable` at ≥ 3 | +| `GET /context?path=` | Ancestor walk: returns every wiki node whose path is a prefix of the file path, plus its durable learnings | +| `GET /examples?path=` | Top graduated prompts for an ancestor of a file path | +| `GET /prompts/proven` | The team's graduated prompts, filterable by score, path, topic | +| `GET /prompts/pending` | Library candidates awaiting review (`TRAILHEAD_PROMOTION_MODE=review`) | +| `POST /prompts/:id/review` | `{ approve: true }` graduates a pending prompt, `false` discards it | +| `GET /search?q=&scope=` | Substring search over rules, durable learnings and graduated prompts | +| `GET /wiki/recent?since=ISO` | Polling endpoint for the VS Code wiki-toast surface | +| `POST /diff` | Picks the closest graduated team prompt by topic + ancestry, scores both prompts, asks Gemini to narrate the difference | +| `POST /improve` | Multi-turn Gemini-driven prompt rewrite, capped at 5 user replies | +| `GET /skill-arc` | Time-series of per-dimension scores (powers the dashboard hero chart) | +| `GET /team/metrics` | Snapshot: avg overall, reuse rate, durable count, draft count, active users | +| `GET /wiki/tree` | Full node + learnings tree | +| `GET /wiki/export` | The whole team wiki as one markdown document (`?drafts=true`, `?format=json`) | +| `POST /onboard/repo` | Bulk-upsert one node per path, idempotent, optional `initial_rules[path]` for seeding `body_md` | +| `POST /onboard/repo/full` | Async rich bootstrap: accepts a folder + file bundle (capped at 16 MB / 2 000 files / 32 KB per file), enqueues a `wiki_jobs` row, three-pass Gemini fan-out via `setImmediate` | +| `GET /onboard/jobs/:id` | Per-path progress for a rich-bootstrap job | +| `DELETE /team/data` | Wipes the requesting team's data; demo team is protected unless `TRAILHEAD_ALLOW_DEMO_RESET=true` | + +**Imports:** `packages/shared` (types), `packages/scoring` (prompts and pure +helpers), `packages/db` (schema). ## Tests diff --git a/apps/api/eval/README.md b/apps/api/eval/README.md index e87c4a9..2516588 100644 --- a/apps/api/eval/README.md +++ b/apps/api/eval/README.md @@ -6,8 +6,8 @@ The 5-dimension score comes from one Gemini call (`scorePrompt` in much, on a fixed golden set, so decisions about sampling settings — or about trusting a single score for coaching and library promotion — rest on data. -Nothing here runs in CI, and nothing runs against the paid API unless you pass -`--yes`. +Nothing runs against the paid API unless you pass `--yes`. CI runs only the +free parts: the harness's own tests and the rule-based baseline below. ## Run it @@ -15,6 +15,10 @@ Nothing here runs in CI, and nothing runs against the paid API unless you pass # Free, offline: a fake scorer exercises the whole pipeline (numbers are meaningless). npm --workspace=apps/api run eval -- --dry-run +# Free, offline: the rule-based scorer (deterministic, so one run is enough). +npm --workspace=apps/api run eval -- --scorer heuristic --runs 1 +npm --workspace=apps/api run eval -- --scorer heuristic --runs 1 --golden eval/holdout.json + # Real: 30 prompts × 5 runs = 150 Gemini calls on your key. GEMINI_API_KEY=... npm --workspace=apps/api run eval -- --yes --runs 5 @@ -24,7 +28,8 @@ GEMINI_API_KEY=... npm --workspace=apps/api run eval -- --yes --temperature 1 GEMINI_API_KEY=... npm --workspace=apps/api run eval -- --yes --thinking-budget 512 --label b512 ``` -Flags: `--runs N` (1–50, default 5), `--only id1,id2`, `--temperature T`, +Flags: `--scorer gemini|heuristic` (default gemini), `--runs N` (1–50, +default 5), `--only id1,id2`, `--temperature T`, `--thinking-budget B` (-1 dynamic, 0 off, else tokens), `--label L`, `--out DIR` (default `eval/results/`, gitignored), `--golden FILE`, `--delay-ms MS` (pause between calls, for free-tier rate limits). @@ -45,6 +50,10 @@ Written as `-