diff --git a/.env.example b/.env.example index 4ee4c5d..e257b9a 100644 --- a/.env.example +++ b/.env.example @@ -2,8 +2,9 @@ # # cp .env.example .env # -# Then set GEMINI_API_KEY below and run `docker compose up`. Everything else -# here has a working default. See SELFHOSTING.md for the full walkthrough. +# Then set GEMINI_API_KEY below and run `docker compose up` (or set +# TRAILHEAD_LLM=offline to try it with no key). Everything else here has a +# working default. See SELFHOSTING.md for the full walkthrough. # # Trailhead is self-hosted: there is no hosted API. Every client (browser-ext, # vscode-ext, mcp-server, dashboard) defaults to http://localhost:3000, which @@ -15,10 +16,17 @@ # --- Gemini ------------------------------------------------------------------ # The only value you must supply. Powers /score, /coach, /improve, /diff and -# the rich wiki bootstrap. The API refuses to start without it. -# Get one at https://aistudio.google.com/apikey +# the rich wiki bootstrap. The API refuses to start without it, unless +# TRAILHEAD_LLM=offline. Get one at https://aistudio.google.com/apikey GEMINI_API_KEY= +# gemini (default) | offline. Offline runs with NO model: prompts are scored by +# the rule-based scorer (packages/scoring/src/heuristic-score.mjs), coaching +# text comes from static templates, the rich wiki bootstrap is disabled, and +# every /score and /coach response says so. For trying the stack without a key; +# it never switches on by itself. +# TRAILHEAD_LLM=offline + # ============================================================================= # OPTIONAL — every value below has a working default # ============================================================================= @@ -60,6 +68,11 @@ TRAILHEAD_AUTO_CREATE_TEAMS=false # X-Admin-Token (`init --admin-token`). Leave empty for open registration while # the API is bound to 127.0.0.1; set it before exposing the API on a network. TRAILHEAD_ADMIN_TOKEN= +# The seeded demo team (public secret trailhead_demo_acme_2026). on | off. +# Default: on, but off whenever TRAILHEAD_ADMIN_TOKEN is set, because anyone +# who can reach the API can write to the demo team and spend your Gemini quota +# through it. Set `on` to keep it on a networked deploy anyway. +# TRAILHEAD_DEMO_TEAM= # How /coach prompts join the team library. Both modes require the exact # average >= 7.0, no dimension < 5, and an independent re-score that agrees. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8fb2ca4..c6872ab 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,7 +86,7 @@ jobs: run: npm audit --audit-level=high docker: - name: docker image builds + name: docker image + compose smoke test runs-on: ubuntu-latest timeout-minutes: 15 steps: @@ -97,6 +97,31 @@ jobs: run: GEMINI_API_KEY=ci-placeholder docker compose config --quiet - name: build api image run: docker build -f apps/api/Dockerfile -t trailhead-api:ci . + # Run the quick start the way a stranger would, with no key (offline + # mode), and check the stack actually works: schema applied, API up, + # registration, a wiki write, a rule-based score, and the demo seed. + - name: compose smoke test (offline mode, demo seed) + env: + TRAILHEAD_LLM: offline + run: | + set -euo pipefail + docker compose -p ci up -d --build --wait --wait-timeout 180 + docker compose -p ci --profile demo run --rm seed + api=http://127.0.0.1:3000 + curl -fsS $api/ | jq -e '.status == "ok" and .llm == "offline"' + secret=$(curl -fsS -X POST $api/teams -H 'content-type: application/json' \ + -d '{"team_id":"team_ci_smoke","name":"ci"}' | jq -er .secret) + curl -fsS -X POST $api/wiki/propose -H "X-Team-Token: $secret" -H 'content-type: application/json' \ + -d '{"node_path":"src/","insight":"Errors return JSON {error, detail}.","user_id":"ci"}' | jq -e '.action == "created"' + curl -fsS -X POST $api/score -H "X-Team-Token: $secret" -H 'content-type: application/json' \ + -d '{"prompt":"fix it","user_id":"ci"}' | jq -e '.scorer == "heuristic" and .overall <= 2' + curl -fsS $api/team/metrics -H 'X-Team-Token: trailhead_demo_acme_2026' | jq -e '.total_obs >= 700 and .active_users == 3' + - name: compose logs + if: failure() + run: docker compose -p ci --profile demo logs + - name: compose down + if: always() + run: docker compose -p ci --profile demo down -v integration: name: api integration tests (Postgres) diff --git a/README.md b/README.md index f7ac45f..48f18e1 100644 --- a/README.md +++ b/README.md @@ -63,8 +63,9 @@ backend, all sharing the same TypeScript contract. Single source of truth. Multi-tenant: each team has a public team id and a server-minted secret (only its SHA-256 is stored), sent as `X-Team-Token`. Pre-2026-09-30 tokens derived from the git remote still work behind -`TRAILHEAD_ACCEPT_LEGACY_TOKENS` (deprecated). Routes are in -`apps/api/src/app.ts` (`index.ts` just serves them): +`TRAILHEAD_ACCEPT_LEGACY_TOKENS` (deprecated). Middleware is in +`apps/api/src/app.ts`, routes in `apps/api/src/routes/` (`index.ts` just serves +them): | Method + Path | What it does | |---|---| @@ -94,10 +95,11 @@ Pre-2026-09-30 tokens derived from the git remote still work behind | `GET /onboard/jobs/:id` | Per-path progress for a rich-bootstrap job | | `DELETE /team/data` | Wipes the requesting team's data; demo team is protected unless `TRAILHEAD_ALLOW_DEMO_RESET=true` | -LLM work runs through `apps/api/src/gemini.ts`. Model assignments live in -`packages/scoring/src/models.mjs`: `gemini-3-flash-preview` for scoring -(JSON-schema mode), topic extraction and diff narration; `gemma-4-31b-it` for -async learning extraction, where latency is tolerable. +Every LLM call runs through `apps/api/src/gemini.ts` (one client, one retry +policy, Langfuse tracing). Model assignments live in +`packages/scoring/src/models.mjs`; today every call uses +`gemini-3-flash-preview` (JSON-schema mode for scoring, topic extraction, +coaching rewrites and the rich wiki bootstrap). Every Gemini call is instrumented with **Langfuse** when `LANGFUSE_PUBLIC_KEY` / `LANGFUSE_SECRET_KEY` are set — one trace per HTTP @@ -263,6 +265,11 @@ docker compose up # → API on http://localhost:3000, Postgres schema applied automatically ``` +No key yet? `TRAILHEAD_LLM=offline docker compose up` runs the same stack with +no model: prompts are scored by a rule-based scorer and coached with static +templates, and every response says so +([SELFHOSTING.md → offline mode](SELFHOSTING.md#try-it-without-a-key-offline-mode)). + That is the whole setup. See [SELFHOSTING.md](SELFHOSTING.md) for pointing the browser extension, VS Code extension, MCP server and dashboard at it, and for running against an external database instead. @@ -294,7 +301,8 @@ npm run dev # → http://localhost:3000 ``` -The API refuses to boot without `DATABASE_URL` and `GEMINI_API_KEY`. +The API refuses to boot without `DATABASE_URL`, and without `GEMINI_API_KEY` +unless `TRAILHEAD_LLM=offline`. Before exposing the API beyond `localhost`, read [SELFHOSTING.md → Security model](SELFHOSTING.md#security-model): set @@ -342,13 +350,15 @@ Single root `.env.example` — every surface reads from the same set. | Var | Used by | Notes | |---|---|---| | `DATABASE_URL` | api | Postgres connection string, `sslmode=require` | -| `GEMINI_API_KEY` | api | `gemini-3-flash-preview` + `gemma-4-31b-it` | +| `GEMINI_API_KEY` | api | Used for every call (`gemini-3-flash-preview`) | +| `TRAILHEAD_LLM` | api | `gemini` (default) or `offline`: no model; rule-based scoring and template coaching (SELFHOSTING.md) | | `LANGFUSE_PUBLIC_KEY` | api | Optional. Hosted Langfuse public key (`pk-lf-…`) | | `LANGFUSE_SECRET_KEY` | api | Optional. Hosted Langfuse secret key (`sk-lf-…`) | | `LANGFUSE_BASEURL` | api | Defaults to `https://cloud.langfuse.com` (EU). Use `https://us.cloud.langfuse.com` for US | | `TEAM_TOKEN` | — | Documentation only: the public demo team's token. The API does not read it; clients hardcode the same value as their fallback | | `PORT` | api | Defaults to 3000; Railway injects automatically | -| `TRAILHEAD_ADMIN_TOKEN` | api | When set, `POST /teams` (registration) requires it as `X-Admin-Token` | +| `TRAILHEAD_ADMIN_TOKEN` | api | When set, `POST /teams` (registration) requires it as `X-Admin-Token`, and the public demo team is turned off | +| `TRAILHEAD_DEMO_TEAM` | api | `on` / `off`: the public demo team (default on, off when `TRAILHEAD_ADMIN_TOKEN` is set) | | `TRAILHEAD_ACCEPT_LEGACY_TOKENS` | api | Default `true`. Accept pre-2026-09-30 remote-derived tokens for teams without a secret (deprecated) | | `TRAILHEAD_AUTO_CREATE_TEAMS` | api | Default `false`. Legacy only: unknown tokens create legacy teams | | `TRAILHEAD_SCORE_TEMPERATURE` / `TRAILHEAD_SCORE_THINKING_BUDGET` | api | Scorer sampling (defaults `0.2` / `-1` = dynamic). Measure before changing: `apps/api/eval/` | @@ -437,8 +447,8 @@ contracts, builds, and tests. - **Backend:** Hono, TypeScript, Node 22, `@hono/node-server`, raw `pg` - **DB:** Postgres on Neon, no ORM -- **LLMs:** `gemini-3-flash-preview` (scoring, JSON-schema mode), `gemma-4-31b-it` - (diff narration, rich bootstrap) +- **LLMs:** `gemini-3-flash-preview` for every call (scoring in JSON-schema + mode, coaching, diff narration, rich bootstrap) - **Observability:** Langfuse (hosted) — one trace per request, one generation per LLM call - **Frontend:** Next.js 16 + Tailwind + Recharts + SWR (dashboard); vanilla diff --git a/SELFHOSTING.md b/SELFHOSTING.md index 5cc7866..46cae58 100644 --- a/SELFHOSTING.md +++ b/SELFHOSTING.md @@ -2,7 +2,7 @@ Trailhead is self-hosted. There is no hosted API and no account to sign up for — you run the backend, and it is yours. The only external dependency is a Gemini -API key. +API key, and even that is optional while you try it out (offline mode, below). Everything below assumes you are at the repo root. @@ -13,7 +13,7 @@ Everything below assumes you are at the repo root. | | | |---|---| | **Docker** | Docker Desktop, OrbStack, or Docker Engine with the Compose v2 plugin (`docker compose version`). | -| **A Gemini API key** | Free at . | +| **A Gemini API key** | Free at . Optional for a first look: see [Try it without a key](#try-it-without-a-key-offline-mode). | | **Node 22.6+** | Only needed to run the *clients* (browser extension, VS Code extension, MCP server, dashboard). The API itself runs entirely inside Docker. | --- @@ -41,8 +41,38 @@ curl http://localhost:3000 # {"name":"trailhead-api","status":"ok",...} ``` -If `GEMINI_API_KEY` is missing, `docker compose up` stops immediately and tells -you so — it will not start a stack that cannot score a prompt. +If `GEMINI_API_KEY` is missing (and `TRAILHEAD_LLM` is not `offline`), the API +refuses to start and logs what to set; Compose keeps restarting it until you +fix `.env`. It never quietly falls back to offline mode. + +### Try it without a key (offline mode) + +```bash +TRAILHEAD_LLM=offline docker compose --profile demo up --build +``` + +Same stack, no model, nothing sent to Google. `--profile demo` also runs the +`seed` service once, which fills the public demo team (`trailhead_demo_acme_2026`) +with synthetic data, so the dashboard has something to show: a small wiki, +four library prompts, and six days of prompt activity from three made-up users +(`packages/db/seed.mjs`; re-run it to move the activity window up to now). +Drop `--profile demo` for an empty stack. + + +- `/score` and `/coach` use the rule-based scorer in + `packages/scoring/src/heuristic-score.mjs`. It reads surface features (file + paths, identifiers, constraint and output phrasing), so it is a reasonable + floor, not a substitute for the model; `apps/api/eval/README.md` has its + numbers and known misses. Every response carries `"scorer": "heuristic"`, + and coaching text ends with a note saying no model was involved. +- Coaching uses the static per-dimension templates (plus your team's library + prompts when there are any). `/improve` asks the template question for each + weak dimension and appends your answers; `/diff` names the biggest gap. +- The rich wiki bootstrap is unavailable (`503`); `bootstrap --minimal` works. +- `GET /` reports `"llm": "offline"`, and the API logs a warning at startup. +- Library promotion still applies its gate, but the confirming re-score is the + same deterministic rules, so it confirms nothing. Set + `TRAILHEAD_PROMOTION_MODE=review` if a team will rely on the library. Run it in the background with `docker compose up -d`, and stop it with `docker compose down`. @@ -55,11 +85,13 @@ ones you are most likely to touch: | Variable | Default | Why you'd change it | |---|---|---| -| `GEMINI_API_KEY` | *(required)* | — | +| `GEMINI_API_KEY` | *(required unless offline)* | — | +| `TRAILHEAD_LLM` | `gemini` | `offline` to run with no model (see above). | | `PORT` | `3000` | Something else already owns port 3000. Changing this means updating each client's API URL too. | | `POSTGRES_PORT` | `5432` | You already run Postgres locally. | | `DATABASE_URL` | *(the bundled Postgres)* | Use an external database (Neon, RDS) instead of the container. | | `TRAILHEAD_ADMIN_TOKEN` | *(empty)* | Set it to restrict team registration to people you give it to. Do this before exposing the API. | +| `TRAILHEAD_DEMO_TEAM` | `on` (`off` when an admin token is set) | Turn the public demo team on or off explicitly. | | `TRAILHEAD_ACCEPT_LEGACY_TOKENS` | `true` | Set `false` once every team has upgraded from a pre-2026-09-30 token. | | `TRAILHEAD_AUTO_CREATE_TEAMS` | `false` | Legacy only: let unknown tokens create teams on the fly. Throwaway demos only. | @@ -212,6 +244,7 @@ The defaults are safe for a local setup because both ports are bound to can reach the port can create teams and spend your Gemini quota, and can *squat* a repo's derived team id before the real team registers it. With it set, pass it to `init --admin-token` (or hand people pre-made secrets). + Setting it also turns the public demo team off (below). - **Turn legacy tokens off** (`TRAILHEAD_ACCEPT_LEGACY_TOKENS=false`) once every team has run `init --upgrade-legacy`. A legacy token is `repo_` + SHA-256 of the raw remote URL: anyone who knows or guesses the URL @@ -232,7 +265,10 @@ The defaults are safe for a local setup because both ports are bound to metrics through it. Put it behind your SSO/VPN if that matters. - **The demo team's secret `trailhead_demo_acme_2026` is public** (it is in this repo). The demo team is protected from `DELETE /team/data` and from secret - rotation, but not from writes. + rotation, but not from writes or from spending your quota. It is **off by + default once `TRAILHEAD_ADMIN_TOKEN` is set**: its secret then gets + `401 demo_team_disabled`. `TRAILHEAD_DEMO_TEAM=on|off` overrides that either + way, and `GET /` reports the current setting as `demo_team`. **Rate limits.** The API limits the calls that cost you something, with in-process token buckets (a limit of `N/W` allows a burst of N and refills at N @@ -307,7 +343,8 @@ sent the same `demo` id.) Where prompts go: every scored prompt, any wiki context attached to it, and — for the default rich `bootstrap` — the first 8,000 characters of up to 500 source files (5 levels deep) are sent to Google's Gemini API -using your `GEMINI_API_KEY`. When `LANGFUSE_*` keys are set, the same model +using your `GEMINI_API_KEY` (in offline mode nothing is sent to Google). When +`LANGFUSE_*` keys are set, the same model inputs and outputs are also sent to Langfuse. The browser extension's 👍/🤷/👎 chips store the prompt *and Claude's full reply* in the `captures` table. `bootstrap` picks files by extension (so `.env` and key files are never read) @@ -320,9 +357,10 @@ uploaded. ## Troubleshooting -**`docker compose up` exits with "required variable GEMINI_API_KEY is missing"** +**The API restarts in a loop and logs "GEMINI_API_KEY not set"** You skipped `cp .env.example .env`, or left the key blank. `.env` must be at the -repo root, next to `docker-compose.yml`. +repo root, next to `docker-compose.yml`. To try the stack without a key, set +`TRAILHEAD_LLM=offline` instead. **API restarts in a loop / `DATABASE_URL not set`** `DATABASE_URL` is set in your `.env` but points somewhere unreachable. Comment diff --git a/apps/api/README.md b/apps/api/README.md index 9734d27..f5e978a 100644 --- a/apps/api/README.md +++ b/apps/api/README.md @@ -6,18 +6,29 @@ artifact talks to. Single source of truth. **Tech:** Hono + TypeScript, run via `docker compose up` from the repo root (see `SELFHOSTING.md`). Any Postgres (bundled container, Neon, RDS). -**Endpoints (spec §3 — partial; the full, current table is in the root README):** -- `POST /score` — 5-dim Gemini score; writes skill_observation inline -- `POST /capture` — store conversation + outcome -- `GET /context` — HCL bundle (path-walked, spec §8) -- `GET /examples` — team-anchored prompts for a path -- `POST /diff` — Prompt Diff synthesis (Gemini) -- `POST /wiki/propose` — autonomous wiki update with normalize + dedup + counter - -**Imports:** `packages/shared` (types), `packages/scoring` (Gemini -prompts), `packages/db` (schema). - -**Spec refs:** §3, §4, §5, §10 +**Layout** (`src/`): + +| File | Role | +|---|---| +| `index.ts` | Entry point: env checks, startup migrations, listener, graceful shutdown | +| `app.ts` | The Hono app: CORS, body caps, tracing, auth and rate-limit middleware, `GET /`, route mounting, error handler | +| `http.ts` | Shared request plumbing: env type, tenant policy, rate limiting, prompt cap, skill-observation writes | +| `routes/score.ts` | `POST /score`, `POST /capture` | +| `routes/coach.ts` | `POST /coach` (teach → reveal loop) | +| `routes/assist.ts` | `POST /diff`, `POST /improve` | +| `routes/wiki.ts` | `/wiki/propose`, `/context`, `/examples`, `/search`, `/wiki/recent`, `/wiki/tree`, `/wiki/export` | +| `routes/prompts.ts` | `/prompts/proven`, `/prompts/pending`, `/prompts/:id/review` | +| `routes/metrics.ts` | `/skill-arc`, `/team/metrics` | +| `routes/teams.ts` | `/teams` (register, probe, rotate), `DELETE /team/data` | +| `routes/onboard.ts` | `/onboard/repo`, `/onboard/repo/full`, `/onboard/jobs/:id` | +| `gemini.ts` | The one Gemini client: retries, timeouts, Langfuse tracing, every LLM call | +| `wiki-bootstrap-job.ts` | The async three-pass rich bootstrap behind `/onboard/repo/full` | +| `db.ts` | `pg` pool, team resolution and registration | + +The full route table is in the root README. + +**Imports:** `packages/shared` (types), `packages/scoring` (prompts and pure +helpers), `packages/db` (schema). ## Tests diff --git a/apps/api/eval/README.md b/apps/api/eval/README.md index e87c4a9..2516588 100644 --- a/apps/api/eval/README.md +++ b/apps/api/eval/README.md @@ -6,8 +6,8 @@ The 5-dimension score comes from one Gemini call (`scorePrompt` in much, on a fixed golden set, so decisions about sampling settings — or about trusting a single score for coaching and library promotion — rest on data. -Nothing here runs in CI, and nothing runs against the paid API unless you pass -`--yes`. +Nothing runs against the paid API unless you pass `--yes`. CI runs only the +free parts: the harness's own tests and the rule-based baseline below. ## Run it @@ -15,6 +15,10 @@ Nothing here runs in CI, and nothing runs against the paid API unless you pass # Free, offline: a fake scorer exercises the whole pipeline (numbers are meaningless). npm --workspace=apps/api run eval -- --dry-run +# Free, offline: the rule-based scorer (deterministic, so one run is enough). +npm --workspace=apps/api run eval -- --scorer heuristic --runs 1 +npm --workspace=apps/api run eval -- --scorer heuristic --runs 1 --golden eval/holdout.json + # Real: 30 prompts × 5 runs = 150 Gemini calls on your key. GEMINI_API_KEY=... npm --workspace=apps/api run eval -- --yes --runs 5 @@ -24,7 +28,8 @@ GEMINI_API_KEY=... npm --workspace=apps/api run eval -- --yes --temperature 1 GEMINI_API_KEY=... npm --workspace=apps/api run eval -- --yes --thinking-budget 512 --label b512 ``` -Flags: `--runs N` (1–50, default 5), `--only id1,id2`, `--temperature T`, +Flags: `--scorer gemini|heuristic` (default gemini), `--runs N` (1–50, +default 5), `--only id1,id2`, `--temperature T`, `--thinking-budget B` (-1 dynamic, 0 off, else tokens), `--label L`, `--out DIR` (default `eval/results/`, gitignored), `--golden FILE`, `--delay-ms MS` (pause between calls, for free-tier rate limits). @@ -45,6 +50,10 @@ Written as `-