From 63b18db53ef4cbc6ba42f818d37cb3d4c565f7ef Mon Sep 17 00:00:00 2001 From: Bogdan Truta Date: Thu, 1 Oct 2026 12:26:01 +0000 Subject: [PATCH 01/14] api: split the 2,277-line app.ts into route modules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit app.ts held every route, its helpers and the middleware in one 87 KB file. It is now the app shell (CORS, body caps, tracing, auth, LLM rate limits, GET /, route mounting, error handler: 243 lines), with: - http.ts: what the middleware and the routes share (env type, tenant policy, rate limiting, the prompt cap, skill-observation writes) - routes/{score,coach,assist,wiki,prompts,metrics,teams,onboard}.ts: one Hono sub-app per area, mounted after the middleware so every route still gets auth, body caps, tracing and rate limits Code moved verbatim (apart from `app.` → `.` and `export` on the shared helpers), so behaviour is unchanged: the 38 Postgres integration tests, which drive every route and cross-check the GET / catalog, pass unchanged. apps/api/README.md documents the layout. Co-Authored-By: Claude Opus 5.5 --- apps/api/README.md | 35 +- apps/api/src/app.ts | 2087 +------------------------------- apps/api/src/http.ts | 158 +++ apps/api/src/routes/assist.ts | 183 +++ apps/api/src/routes/coach.ts | 642 ++++++++++ apps/api/src/routes/metrics.ts | 111 ++ apps/api/src/routes/onboard.ts | 288 +++++ apps/api/src/routes/prompts.ts | 122 ++ apps/api/src/routes/score.ts | 96 ++ apps/api/src/routes/teams.ts | 143 +++ apps/api/src/routes/wiki.ts | 388 ++++++ 11 files changed, 2180 insertions(+), 2073 deletions(-) create mode 100644 apps/api/src/http.ts create mode 100644 apps/api/src/routes/assist.ts create mode 100644 apps/api/src/routes/coach.ts create mode 100644 apps/api/src/routes/metrics.ts create mode 100644 apps/api/src/routes/onboard.ts create mode 100644 apps/api/src/routes/prompts.ts create mode 100644 apps/api/src/routes/score.ts create mode 100644 apps/api/src/routes/teams.ts create mode 100644 apps/api/src/routes/wiki.ts diff --git a/apps/api/README.md b/apps/api/README.md index 9734d27..f5e978a 100644 --- a/apps/api/README.md +++ b/apps/api/README.md @@ -6,18 +6,29 @@ artifact talks to. Single source of truth. **Tech:** Hono + TypeScript, run via `docker compose up` from the repo root (see `SELFHOSTING.md`). Any Postgres (bundled container, Neon, RDS). -**Endpoints (spec §3 — partial; the full, current table is in the root README):** -- `POST /score` — 5-dim Gemini score; writes skill_observation inline -- `POST /capture` — store conversation + outcome -- `GET /context` — HCL bundle (path-walked, spec §8) -- `GET /examples` — team-anchored prompts for a path -- `POST /diff` — Prompt Diff synthesis (Gemini) -- `POST /wiki/propose` — autonomous wiki update with normalize + dedup + counter - -**Imports:** `packages/shared` (types), `packages/scoring` (Gemini -prompts), `packages/db` (schema). - -**Spec refs:** §3, §4, §5, §10 +**Layout** (`src/`): + +| File | Role | +|---|---| +| `index.ts` | Entry point: env checks, startup migrations, listener, graceful shutdown | +| `app.ts` | The Hono app: CORS, body caps, tracing, auth and rate-limit middleware, `GET /`, route mounting, error handler | +| `http.ts` | Shared request plumbing: env type, tenant policy, rate limiting, prompt cap, skill-observation writes | +| `routes/score.ts` | `POST /score`, `POST /capture` | +| `routes/coach.ts` | `POST /coach` (teach → reveal loop) | +| `routes/assist.ts` | `POST /diff`, `POST /improve` | +| `routes/wiki.ts` | `/wiki/propose`, `/context`, `/examples`, `/search`, `/wiki/recent`, `/wiki/tree`, `/wiki/export` | +| `routes/prompts.ts` | `/prompts/proven`, `/prompts/pending`, `/prompts/:id/review` | +| `routes/metrics.ts` | `/skill-arc`, `/team/metrics` | +| `routes/teams.ts` | `/teams` (register, probe, rotate), `DELETE /team/data` | +| `routes/onboard.ts` | `/onboard/repo`, `/onboard/repo/full`, `/onboard/jobs/:id` | +| `gemini.ts` | The one Gemini client: retries, timeouts, Langfuse tracing, every LLM call | +| `wiki-bootstrap-job.ts` | The async three-pass rich bootstrap behind `/onboard/repo/full` | +| `db.ts` | `pg` pool, team resolution and registration | + +The full route table is in the root README. + +**Imports:** `packages/shared` (types), `packages/scoring` (prompts and pure +helpers), `packages/db` (schema). ## Tests diff --git a/apps/api/src/app.ts b/apps/api/src/app.ts index a4c255b..5a2feab 100644 --- a/apps/api/src/app.ts +++ b/apps/api/src/app.ts @@ -1,130 +1,26 @@ -// The Hono app: every route and middleware, no listener. Imported by index.ts -// (which serves it) and by the integration tests (which call app.request() -// directly against a real Postgres). Importing this module opens the pg pool -// and requires DATABASE_URL and GEMINI_API_KEY to be set. +// The Hono app: middleware, the GET / catalog and the route modules (routes/), +// no listener. Imported by index.ts (which serves it) and by the integration +// tests (which call app.request() directly against a real Postgres). Importing +// this module opens the pg pool and requires DATABASE_URL and GEMINI_API_KEY. import './env.ts'; import { Hono, type Context } from 'hono'; -import { getConnInfo } from '@hono/node-server/conninfo'; import { bodyLimit } from 'hono/body-limit'; import { cors } from 'hono/cors'; import { logger } from 'hono/logger'; -import type { - CaptureRequest, - CaptureResponse, - CoachMode, - CoachNextRoundInputs, - CoachRequest, - CoachResponse, - ContextNode, - ContextResponse, - DiffRequest, - DiffResponse, - Dimension, - DimensionScores, - ExamplesItem, - ExamplesResponse, - ImproveRequest, - ImproveResponse, - ImproveTurn, - OnboardRepoFullRequest, - OnboardRepoFullResponse, - OnboardRepoRequest, - OnboardRepoResponse, - ProvenPromptItem, - ProvenPromptsResponse, - ScoreRequest, - ScoreResponse, - SearchResponse, - SkillArcObservation, - SkillArcResponse, - TeamMetricsResponse, - TeamSummary, - TeamsListResponse, - WikiJobPathKind, - WikiJobPathStatus, - WikiJobStatusResponse, - WikiProposeRequest, - WikiProposeResponse, - WikiRecentItem, - WikiRecentResponse, - WikiTreeResponse, -} from '@trailhead/shared'; -import { DIMENSIONS } from '@trailhead/shared'; -import { - ancestorPaths, - buildAugmentation, - normalize, - normalizePath, - renderSkipReveal, - renderSuccessReveal, - renderTeachBlock, -} from '@trailhead/scoring'; -import { - applyTeamNameIfPlaceholder, - DEMO_TEAM_SECRET_HASH, - DEMO_TEAM_TOKEN, - isSecretTeamId, - q, - registerTeam, - resolveTeam, - rotateTeamSecret, - upsertNode, - wipeTeamData, -} from './db.ts'; -import { isValidTeamId, randomTeamId, safeEqual } from './team-auth.ts'; -import { ipRateKey, limiterFor, type LimitName, type TokenBucketLimiter } from './rate-limit.ts'; -import { createHash, randomUUID } from 'node:crypto'; -import { degradedCoachResponse, isUnparseableScore } from './coach-degraded.ts'; -import { intParam, isUuid } from './request-params.ts'; -import { loadWikiTree } from './wiki-tree.ts'; -import { exportFilename, renderWikiMarkdown } from './wiki-export.ts'; -import { - acknowledgeProgress, - extractTopic, - improveCoach, - overallScore, - rewriteForDims, - scorePrompt, - summarizeCoaching, - synthesizeDiff, -} from './gemini.ts'; +import { DEMO_TEAM_SECRET_HASH, DEMO_TEAM_TOKEN, isSecretTeamId, q, resolveTeam } from './db.ts'; +import type { LimitName } from './rate-limit.ts'; +import { randomUUID } from 'node:crypto'; import { langfuse, withTrace } from './langfuse.ts'; -import { tryPromotePrompt } from './prompt-promotion.ts'; -import { - passesPromotionGate, - promotionMode, - renderLibraryBanner, - renderNotPromotedNote, -} from './promotion-gate.ts'; -import { invalidateTeamContext, renderTeamContext } from './team-context.ts'; -import { bundleFromRequest, runJob } from './wiki-bootstrap-job.ts'; +import { authPolicy, clientIp, opaqueTeamId, rateLimit, teamIdFromHeader, type AppEnv } from './http.ts'; +import { scoreRoutes } from './routes/score.ts'; +import { coachRoutes } from './routes/coach.ts'; +import { wikiRoutes } from './routes/wiki.ts'; +import { promptRoutes } from './routes/prompts.ts'; +import { assistRoutes } from './routes/assist.ts'; +import { metricsRoutes } from './routes/metrics.ts'; +import { teamRoutes } from './routes/teams.ts'; +import { onboardRoutes } from './routes/onboard.ts'; - -// Tenant policy, read per request so tests (and operators flipping env in a -// long-lived process manager) see the current value. -// -// TRAILHEAD_ACCEPT_LEGACY_TOKENS (default true) — accept pre-2026-09-30 -// id-as-credential tokens (repo_…, custom strings) for teams that have not -// been given a secret yet. Deprecated; set false once every team has run -// `init --upgrade-legacy` (or POST /teams/rotate-secret). -// TRAILHEAD_AUTO_CREATE_TEAMS (default false) — with legacy tokens accepted, -// an unknown X-Team-Token spawns a legacy team. Unauthenticated tenant -// creation; only for throwaway demo deploys. Has no effect when legacy -// tokens are off. New teams should use POST /teams instead. -// TRAILHEAD_ADMIN_TOKEN (default unset) — when set, POST /teams requires it -// in X-Admin-Token, so only the operator can register teams. Unset means -// open registration, which is fine while the API is bound to 127.0.0.1. -function authPolicy() { - return { - acceptLegacy: process.env.TRAILHEAD_ACCEPT_LEGACY_TOKENS !== 'false', - autoCreate: process.env.TRAILHEAD_AUTO_CREATE_TEAMS === 'true', - adminToken: process.env.TRAILHEAD_ADMIN_TOKEN || null, - }; -} - -// Hono context typing — the auth middleware sets `team_token` (the team id; -// the column kept its historical name) and `legacy_auth` for every handler. -type AppEnv = { Variables: { team_token: string; legacy_auth: boolean } }; export const app = new Hono(); app.use('*', logger()); @@ -154,54 +50,6 @@ app.use('*', (c, next) => (c.req.path === '/onboard/repo/full' ? bootstrapBodyLimit : defaultBodyLimit)(c, next), ); -// The caller's rate-limit key: the socket's remote address, or — only when -// TRAILHEAD_TRUST_PROXY=true, i.e. behind one reverse proxy you control — the -// LAST X-Forwarded-For entry, which is the one that proxy appended. Earlier -// entries come from the client (nginx's $proxy_add_x_forwarded_for, Caddy and -// Traefik all append to whatever the client sent), so reading the first one -// would let every request pick its own key. Trusting the header without a -// proxy would do the same. IPv6 is keyed per /64 (ipRateKey). -function clientIp(c: Context): string { - if (process.env.TRAILHEAD_TRUST_PROXY === 'true') { - const fwd = c.req.header('x-forwarded-for')?.split(',').at(-1)?.trim(); - if (fwd) return ipRateKey(fwd); - } - try { - return ipRateKey(getConnInfo(c).remote.address ?? 'unknown'); - } catch { - return 'unknown'; // no Node socket (app.request() in tests) - } -} - -// Take a token from each named limiter, all or none: every bucket is checked -// first, and only when all of them allow the request is a token spent in -// each. So a request refused by the per-team limit doesn't also use up the -// caller's per-IP allowance. On refusal, returns the 429 for the first -// refusing limit. Retry-After is in whole seconds. Clients treat 429 like any -// other failure: coaching fails open. (Synchronous, so no request can slip in -// between the checks and the takes.) -function rateLimit(c: Context, checks: Array<[LimitName, string]>): Response | null { - const active = checks - .map(([name, key]) => ({ name, key, limiter: limiterFor(name) })) - .filter((x): x is { name: LimitName; key: string; limiter: TokenBucketLimiter } => x.limiter !== null); - for (const { name, key, limiter } of active) { - const r = limiter.peek(key); - if (r.ok) continue; - c.header('Retry-After', String(r.retryAfterSec)); - return c.json( - { - error: 'rate_limited', - limit: name, - retry_after: r.retryAfterSec, - detail: `Too many requests (${name.replace(/_/g, ' ')}: ${limiter.spec.text}). Retry in ${r.retryAfterSec}s.`, - }, - 429, - ); - } - for (const { key, limiter } of active) limiter.take(key); - return null; -} - // Langfuse: one trace per HTTP request, stored in AsyncLocalStorage so // gemini.ts can hang generations off it without us having to thread the // trace handle through every function signature. @@ -302,19 +150,6 @@ app.use('*', async (c, next) => { await next(); }); -// A stable, opaque handle for a team that is safe to hand to a client. -// -// SHA-256 of the token, truncated to 16 hex chars. Not reversible, not -// replayable as an X-Team-Token, and stable across requests so it works as a -// React key or a client-side lookup handle. -function opaqueTeamId(token: string): string { - return createHash('sha256').update(token).digest('hex').slice(0, 16); -} - -function teamIdFromHeader(token: string | undefined): string | null { - return token ? opaqueTeamId(token) : null; -} - app.get('/', (c) => c.json({ name: 'trailhead-api', @@ -352,1887 +187,17 @@ app.get('/', (c) => }), ); -// ----- POST /score ----------------------------------------------------------- -// Live 5-dim Gemini score; writes 5 skill_observation rows (one per dimension) -// with a 30s dedup window per (team, user, dimension, prompt-hash) per spec -// §19 risk register. Fails closed (returns 500 on Gemini error) — the browser -// extension fails open on its side so the user is never blocked. - -function simpleHash(s: string): string { - // Tiny non-crypto hash for the skill_observation dedup key. Collisions - // are harmless here — they'd just suppress one extra row. - let h = 0; - for (let i = 0; i < s.length; i++) h = ((h << 5) - h + s.charCodeAt(i)) | 0; - return h.toString(16); -} - -// Bulk insert one skill_observation row per dimension with the per-(user, -// dim, prompt-hash) 30s dedup window from spec §19. Shared between /score -// and /coach so both write to the same rubric stream — the dashboard and -// skill arc don't care which endpoint produced the row. -async function writeSkillObservations( - teamToken: string, - userId: string, - prompt: string, - dimensions: DimensionScores, -): Promise { - const promptHash = simpleHash(prompt); - await q( - `INSERT INTO skill_observations (team_token, user_id, dimension, score, prompt_hash) - SELECT i.team_token, i.user_id, i.dimension, i.score, i.prompt_hash - FROM ( VALUES - ${DIMENSIONS.map((_, i) => - `($1::text, $2::text, $${3 + i * 2}::text, $${4 + i * 2}::int, $13::text)` - ).join(',\n ')} - ) AS i(team_token, user_id, dimension, score, prompt_hash) - WHERE NOT EXISTS ( - SELECT 1 FROM skill_observations s - WHERE s.team_token = i.team_token - AND s.user_id = i.user_id - AND s.dimension = i.dimension - AND s.prompt_hash = i.prompt_hash - AND s.ts > NOW() - INTERVAL '30 seconds' - )`, - [ - teamToken, - userId, - ...DIMENSIONS.flatMap((d) => [d, dimensions[d]]), - promptHash, - ], - ); -} - -// Upper bound on any prompt text handed to Gemini. Every scored character is -// billed to the operator's GEMINI_API_KEY, and without a cap one request can -// carry an arbitrarily large body. 64K chars (~16K tokens) is far above a real -// chat prompt, pasted code included; clients already fail open on non-2xx, so -// an over-cap prompt is simply sent uncoached. -const MAX_PROMPT_CHARS = 64_000; - -function promptTooLong(...texts: (string | undefined)[]): boolean { - return texts.some((t) => typeof t === 'string' && t.length > MAX_PROMPT_CHARS); -} - -const PROMPT_TOO_LONG = { - error: 'prompt_too_long', - detail: `max ${MAX_PROMPT_CHARS} characters`, -} as const; - -app.post('/score', async (c) => { - const body = await c.req.json().catch(() => null); - if (!body || typeof body.prompt !== 'string' || typeof body.user_id !== 'string') { - return c.json({ error: 'bad_request' }, 400); - } - if (promptTooLong(body.prompt)) return c.json(PROMPT_TOO_LONG, 413); - - // Optional sticky wiki context from the popup picker. Bundle is rendered - // out-of-band and prepended to Gemini's system instruction so the rubric - // is calibrated against the team's conventions without inflating the - // prompt being scored. - const teamContext = body.context_path - ? await renderTeamContext(c.get('team_token'), body.context_path) - : null; - - const result = await scorePrompt({ - prompt: body.prompt, - file_path: body.file_path, - team_context: teamContext ?? undefined, - }); - // Parse failure comes back as all-zero + no hints rather than a throw (see - // isUnparseableScore). Report it as an upstream failure and write nothing: - // persisting it would record five fake 0/10 observations for this user. - if (isUnparseableScore(result.dimensions, result.missing)) { - console.error('[api] /score scorePrompt returned unparseable output (zero+empty fingerprint)'); - return c.json({ error: 'score_unparseable' }, 502); - } - const overall = overallScore(result.dimensions); - - await writeSkillObservations( - c.get('team_token'), - body.user_id, - body.prompt, - result.dimensions, - ); - - const res: ScoreResponse = { - overall, - dimensions: result.dimensions, - missing: result.missing, - }; - return c.json(res); -}); - -// ----- POST /coach ----------------------------------------------------------- -// Educational coaching loop. Drives the teach→reveal cycle described in -// docs/superpowers/specs/2026-04-26-trailhead-educational-loop-design.md. -// -// Stateless: caller (the MCP server) carries round state explicitly. The -// `proceed` flag is the directive's only decision input — when false the -// caller relays `text`, gathers a user reply, and calls /coach again with -// next_round_inputs echoed back. Server enforces the round cap and bails -// on no-progress. - -const COACH_MAX_ROUNDS = 5; - -function clampRound(n: number | undefined): number { - if (typeof n !== 'number' || !Number.isFinite(n)) return 1; - return Math.max(1, Math.min(COACH_MAX_ROUNDS, Math.floor(n))); -} - -// Derive a wiki context_path from a file_path so coach scoring is grounded -// in the team's subtree without the caller having to know wiki internals. -// 'src/api/webhooks/handler.ts' → 'src/api/webhooks/' (parent folder). -// 'src/api/webhooks/' → 'src/api/webhooks/' (already a folder). -// 'README.md' → '' (no parent → no scope). -// Empty string is treated as "no scope" by renderTeamContext. -function deriveContextPath(filePath: string): string { - const idx = filePath.lastIndexOf('/'); - return idx < 0 ? '' : filePath.slice(0, idx + 1); -} - -// Pick the lowest-scoring dimension under the threshold (default 7). Stable -// against tied scores by walking DIMENSIONS in declaration order — same -// prompt always teaches the same dim. -function lowestDimBelow(dims: DimensionScores, threshold: number = 7): Dimension | null { - let pick: Dimension | null = null; - let pickScore = Infinity; - for (const d of DIMENSIONS) { - const s = dims[d]; - if (s < threshold && s < pickScore) { - pick = d; - pickScore = s; - } - } - return pick; -} - -// Wiki-first lookup: top graduated prompt in the file_path's ancestor nodes. -// Prefers prompts authored by SOMEONE OTHER than `userId` so the user isn't -// shown their own prompt back as the strong example. Self-authored rows are -// still returned when no other-authored alternative exists — keeps the -// single-user demo posture working. -async function fetchTopGraduatedForPath( - teamToken: string, - filePath: string, - userId: string, -): Promise { - const ancestors = ancestorPaths(filePath); - if (ancestors.length === 0) return null; - const rows = await q<{ template: string }>( - `SELECT p.template - FROM prompts p - JOIN nodes n ON n.id = p.node_id - WHERE n.team_token = $1 - AND n.path = ANY($2::text[]) - AND p.status = 'graduated' - ORDER BY (p.author_user_id IS NULL OR p.author_user_id <> $3) DESC, - p.reuse_count DESC, - length(n.path) DESC - LIMIT 1`, - [teamToken, ancestors, userId], - ); - return rows.length ? rows[0]!.template : null; -} - -// Wiki-first lookup: top graduated prompt across the team. Used when no -// file_path is available. Same self-author preference as the path variant. -async function fetchTopGraduatedForTeam(teamToken: string, userId: string): Promise { - const rows = await q<{ template: string }>( - `SELECT p.template - FROM prompts p - JOIN nodes n ON n.id = p.node_id - WHERE n.team_token = $1 - AND p.status = 'graduated' - ORDER BY (p.author_user_id IS NULL OR p.author_user_id <> $2) DESC, - p.reuse_count DESC - LIMIT 1`, - [teamToken, userId], - ); - return rows.length ? rows[0]!.template : null; -} - -// Wiki-first → Gemini fallback. Hackathon simplification: we don't re-score -// graduated prompts to filter for `target_dims`; the assumption is that a -// graduated team prompt is already strong on most dims and seeing it -// teaches the user something either way. If the wiki has nothing, fall -// back to a Gemini rewrite that explicitly targets the named dimensions. -// -// Returns the example string AND the optional tip — the wiki path has no -// tip (we just have the template), the Gemini fallback emits one alongside -// the rewrite. Callers showing a single-dim teach block render the tip; -// multi-dim callers (skip / no-progress) ignore it because one tip can't -// honestly summarize several principles at once. -async function getStrongExample(args: { - teamToken: string; - userId: string; - prompt: string; - file_path?: string; - target_dims: Dimension[]; - team_context: string | null; -}): Promise<{ example: string; tip: string; fromTeam: boolean }> { - const wiki = args.file_path - ? await fetchTopGraduatedForPath(args.teamToken, args.file_path, args.userId) - : await fetchTopGraduatedForTeam(args.teamToken, args.userId); - // fromTeam: a teammate's text — the renderers fence it as untrusted. - if (wiki) return { example: wiki, tip: '', fromTeam: true }; - - const fallback = await rewriteForDims({ - prompt: args.prompt, - target_dims: args.target_dims, - file_path: args.file_path, - team_context: args.team_context ?? undefined, - }); - // Empty strings on Gemini failure — render block falls back accordingly. - return { example: fallback.rewritten_prompt, tip: fallback.tip, fromTeam: false }; -} - -// Library promotion for a /coach prompt that needs no (more) coaching. -// Checks the gate (promotion-gate.ts: unrounded mean >= 7, no dimension < 5) -// synchronously and, if it passes, schedules the background promotion, which -// re-scores independently before inserting. Returns the line for `text` — -// it ships in `text` so a forgetful host LLM can't drop it, and it never -// claims more than has happened. -function promoteIfEligible(args: { - teamToken: string; - userId: string; - prompt: string; - filePath: string | null; - dimensions: DimensionScores; - overall: number; -}): string { - const gate = passesPromotionGate(args.dimensions); - if (!gate.ok) return renderNotPromotedNote(args.overall, gate); - const mode = promotionMode(); - // Fire-and-forget: off the response path, fail-open inside - // tryPromotePrompt. MCP-only by call site (only /coach promotes). - setImmediate(() => { - void tryPromotePrompt({ ...args, mode }); - }); - return renderLibraryBanner(args.overall, mode); -} - -// Round-state token. Compresses the four `next_round_inputs` fields into a -// single opaque base64url JSON blob. Round 2+ callers can echo only the -// token instead of all four fields — fewer slots for an LLM to drop. Both -// shapes are accepted on input; the token wins when both are present. -type RoundState = { - original_prompt: string; - original_dimensions: DimensionScores; - previous_dimensions: DimensionScores; - round: number; -}; -function encodeRoundToken(state: RoundState): string { - return Buffer.from(JSON.stringify(state), 'utf8').toString('base64url'); -} -function decodeRoundToken(token: string): RoundState | null { - try { - const json = Buffer.from(token, 'base64url').toString('utf8'); - const p = JSON.parse(json) as Partial; - if ( - typeof p.original_prompt === 'string' && - typeof p.round === 'number' && - p.original_dimensions && typeof p.original_dimensions === 'object' && - p.previous_dimensions && typeof p.previous_dimensions === 'object' - ) { - return p as RoundState; - } - return null; - } catch { - return null; - } -} - -app.post('/coach', async (c) => { - const body = await c.req.json().catch(() => null); - if (!body || typeof body.prompt !== 'string' || typeof body.user_id !== 'string') { - return c.json({ error: 'bad_request' }, 400); - } - if (promptTooLong(body.prompt, body.original_prompt)) return c.json(PROMPT_TOO_LONG, 413); - - // Round-token shorthand. When present, decode and use as authoritative - // round state — overrides any individual field the caller also sent. - // Invalid tokens fall through to the four-field path with a warning. - if (body.round_token) { - const decoded = decodeRoundToken(body.round_token); - if (decoded) { - body.original_prompt = decoded.original_prompt; - body.original_dimensions = decoded.original_dimensions; - body.previous_dimensions = decoded.previous_dimensions; - body.round = decoded.round; - } else { - console.warn('[coach] received invalid round_token; falling back to explicit fields'); - } - } - - const mode: CoachMode = body.mode === 'augment' || body.mode === 'skip_reveal' - ? body.mode - : 'score'; - - // Same team-context pattern as /score and /improve. When the caller does - // not pass an explicit context_path, derive one from file_path so the - // teach prompts and Gemini's scoring see the team's subtree automatically. - // The MCP coach tool only forwards file_path, so this is what makes coach - // wiki-aware in practice. - const teamToken = c.get('team_token'); - const contextPath = - body.context_path ?? (body.file_path ? deriveContextPath(body.file_path) : ''); - const teamContext = contextPath - ? await renderTeamContext(teamToken, contextPath) - : null; - - // 1. Score (always). Failure is fail-open: hand the LLM a "no coaching - // this turn" signal and let it produce its answer with the original - // prompt. Spec §7. - let scoreResult: { dimensions: DimensionScores; missing: Record }; - try { - const result = await scorePrompt({ - prompt: body.prompt, - file_path: body.file_path, - team_context: teamContext ?? undefined, - }); - scoreResult = { dimensions: result.dimensions, missing: result.missing as Record }; - } catch (err) { - // Fail-open on `proceed`, but NEVER fail silent. A coaching outage must - // not block the user's real work, so proceed stays true — but the caller - // is told plainly that this turn was not coached, and why. Returning - // text:'' here (the old behaviour) made an outage look identical to a - // perfect prompt, so the MCP tool ran indefinitely without ever coaching. - console.error('[api] /coach scorePrompt failed', err); - const zeros = Object.fromEntries(DIMENSIONS.map((d) => [d, 0])) as DimensionScores; - return c.json(degradedCoachResponse(mode, zeros, 'score_failed', err)); - } - const overall = overallScore(scoreResult.dimensions); - - // Fail-open: scorePrompt does NOT throw on Gemini parse failures — it - // silently returns zeros + empty missing (see gemini.ts coerceScore). - // That signal is indistinguishable from a real all-zero score except by - // the empty `missing` object: a real-zero score from Gemini populates - // hints for the dims < 5. When we detect the zero+empty fingerprint, - // treat it as "Gemini failed, no coaching this turn" rather than - // pretending the user wrote a perfectly empty prompt. Spec §7. - if (isUnparseableScore(scoreResult.dimensions, scoreResult.missing)) { - console.error('[api] /coach scorePrompt returned unparseable output (zero+empty fingerprint)'); - return c.json( - degradedCoachResponse(mode, scoreResult.dimensions, 'score_unparseable'), - ); - } - - // 2. Skill_observation writes (same dedup as /score). - await writeSkillObservations(teamToken, body.user_id, body.prompt, scoreResult.dimensions); - - // 3. Branch on mode. - // ---- Augment mode (legacy passthrough) ----------------------------------- - if (mode === 'augment') { - const augmented = buildAugmentation({ - original: body.prompt, - missing: scoreResult.missing, - }); - const res: CoachResponse = { - proceed: true, - mode: 'augment', - overall, - dimensions: scoreResult.dimensions, - missing: scoreResult.missing, - text: '', - augmented_prompt: augmented, - missing_dims: Object.keys(scoreResult.missing), - }; - return c.json(res); - } - - // ---- Skip reveal mode ---------------------------------------------------- - if (mode === 'skip_reveal') { - const originalDims = body.original_dimensions ?? scoreResult.dimensions; - // Dimensions we'd want to lift on the rewrite. Spec §5 picks dims that - // scored below 5; if the original is already above that bar, fall back - // to dims below 7 so the rewrite still has direction. - let dimsToImprove = DIMENSIONS.filter((d) => originalDims[d] < 5); - if (dimsToImprove.length === 0) { - dimsToImprove = DIMENSIONS.filter((d) => originalDims[d] < 7); - } - const originalPrompt = body.original_prompt ?? body.prompt; - // Run the rewrite and the closing summary in parallel — both are - // independent Gemini calls and the user is already waiting on the - // skip-reveal text. Each fails open to '' so a partial outage still - // produces a useful (if shorter) reveal. - const [strong, summary] = await Promise.all([ - getStrongExample({ - teamToken, - userId: body.user_id, - prompt: originalPrompt, - file_path: body.file_path, - target_dims: dimsToImprove.length ? dimsToImprove : ['specificity'], - team_context: teamContext, - }), - summarizeCoaching({ - original_prompt: originalPrompt, - final_prompt: body.prompt, - original_dimensions: originalDims, - final_dimensions: scoreResult.dimensions, - reason: 'skip', - }), - ]); - const text = strong.example - ? renderSkipReveal({ - strongRewrite: strong.example, - strongRewriteFromTeam: strong.fromTeam, - originalDimensions: originalDims, - reason: 'skip', - summary, - overall: overallScore(originalDims), - }) - : ''; - const res: CoachResponse = { - proceed: true, - mode: 'skip_reveal', - overall, - dimensions: scoreResult.dimensions, - missing: scoreResult.missing, - text, - }; - return c.json(res); - } - - // ---- Score mode (the main loop) ------------------------------------------ - const round = clampRound(body.round); - const isRound1 = round === 1 || !body.original_prompt; - const lowest = lowestDimBelow(scoreResult.dimensions, 7); - - // Round 1, score >=7 → silent fast path. Power users see no friction. - if (isRound1 && overall >= 7) { - const res: CoachResponse = { - proceed: true, - mode: 'score', - overall, - dimensions: scoreResult.dimensions, - missing: scoreResult.missing, - text: promoteIfEligible({ - teamToken, - userId: body.user_id, - prompt: body.prompt, - filePath: body.file_path ?? null, - dimensions: scoreResult.dimensions, - overall, - }), - }; - return c.json(res); - } - - // Round 1, score <7 → first teach block. - if (isRound1 && lowest) { - const strong = await getStrongExample({ - teamToken, - userId: body.user_id, - prompt: body.prompt, - file_path: body.file_path, - target_dims: [lowest], - team_context: teamContext, - }); - const text = renderTeachBlock({ - targetDim: lowest, - targetScore: scoreResult.dimensions[lowest], - strongExample: strong.example, - strongExampleFromTeam: strong.fromTeam, - tip: strong.tip, - dimensions: scoreResult.dimensions, - overall, - }); - const nextState: RoundState = { - original_prompt: body.prompt, - original_dimensions: scoreResult.dimensions, - previous_dimensions: scoreResult.dimensions, - round: 2, - }; - const next: CoachNextRoundInputs = { - ...nextState, - round_token: encodeRoundToken(nextState), - }; - const res: CoachResponse = { - proceed: false, - mode: 'score', - overall, - dimensions: scoreResult.dimensions, - missing: scoreResult.missing, - text, - next_round_inputs: next, - }; - return c.json(res); - } - - // Round >=2 paths. Need original_prompt (we treated round 1 already). - const originalPrompt = body.original_prompt!; - const originalDims = body.original_dimensions ?? scoreResult.dimensions; - const previousDims = body.previous_dimensions ?? originalDims; - const previousOverall = overallScore(previousDims); - const previousLowest = lowestDimBelow(previousDims, 7); - const originalOverall = overallScore(originalDims); - - // Score crossed 7 → success reveal. - if (overall >= 7) { - const summary = await summarizeCoaching({ - original_prompt: originalPrompt, - final_prompt: body.prompt, - original_dimensions: originalDims, - final_dimensions: scoreResult.dimensions, - reason: 'success', - }); - const text = - renderSuccessReveal({ - originalPrompt, - finalPrompt: body.prompt, - originalOverall, - finalOverall: overall, - originalDimensions: originalDims, - finalDimensions: scoreResult.dimensions, - summary, - }) + - // The user iterated through coaching and landed a >=7 prompt — the - // final form is the promotion candidate. - `\n\n${promoteIfEligible({ - teamToken, - userId: body.user_id, - prompt: body.prompt, - filePath: body.file_path ?? null, - dimensions: scoreResult.dimensions, - overall, - })}`; - const res: CoachResponse = { - proceed: true, - mode: 'score', - overall, - dimensions: scoreResult.dimensions, - missing: scoreResult.missing, - text, - }; - return c.json(res); - } - - // No-progress detection: the dim we were teaching about (= last round's - // lowest) did NOT improve, AND overall did not improve. We test the - // previously-targeted dim directly rather than checking `lowest` equality, - // because Gemini's tiebreakers can shuffle which 0-scored dim is "lowest" - // between rounds even when nothing material changed (this was the original - // failing case from the design spec — "fix the retry. it needs to be more - // accurate" leaves context_loading at 0 but the lowest tiebreaker drifts). - const targetDimDidNotImprove = !!( - previousLowest && - scoreResult.dimensions[previousLowest] <= previousDims[previousLowest] - ); - if (targetDimDidNotImprove && overall <= previousOverall) { - let dimsToImprove = DIMENSIONS.filter((d) => originalDims[d] < 5); - if (dimsToImprove.length === 0) { - dimsToImprove = DIMENSIONS.filter((d) => originalDims[d] < 7); - } - // Parallel: rewrite + closing recap. Same fail-open posture as the - // skip-reveal branch — both helpers return '' on Gemini failure and the - // render fallback handles each independently. - const [strong, summary] = await Promise.all([ - getStrongExample({ - teamToken, - userId: body.user_id, - prompt: originalPrompt, - file_path: body.file_path, - target_dims: dimsToImprove.length ? dimsToImprove : [lowest!], - team_context: teamContext, - }), - summarizeCoaching({ - original_prompt: originalPrompt, - final_prompt: body.prompt, - original_dimensions: originalDims, - final_dimensions: scoreResult.dimensions, - reason: 'no_progress', - }), - ]); - const text = strong.example - ? renderSkipReveal({ - strongRewrite: strong.example, - strongRewriteFromTeam: strong.fromTeam, - originalDimensions: originalDims, - reason: 'no_progress', - noProgressDim: previousLowest ?? undefined, - summary, - overall: overallScore(originalDims), - }) - : ''; - const res: CoachResponse = { - proceed: true, - mode: 'score', - overall, - dimensions: scoreResult.dimensions, - missing: scoreResult.missing, - text, - }; - return c.json(res); - } - - // Forced exit at COACH_MAX_ROUNDS (still <7, made progress, but rounds exhausted). - if (round >= COACH_MAX_ROUNDS) { - const summary = await summarizeCoaching({ - original_prompt: originalPrompt, - final_prompt: body.prompt, - original_dimensions: originalDims, - final_dimensions: scoreResult.dimensions, - reason: 'max_rounds', - }); - const text = renderSuccessReveal({ - originalPrompt, - finalPrompt: body.prompt, - originalOverall, - finalOverall: overall, - originalDimensions: originalDims, - finalDimensions: scoreResult.dimensions, - maxRoundsHit: true, - summary, - }); - const res: CoachResponse = { - proceed: true, - mode: 'score', - overall, - dimensions: scoreResult.dimensions, - missing: scoreResult.missing, - text, - }; - return c.json(res); - } - - // Else: score still <7, made progress, more rounds remain. Keep teaching. - if (lowest) { - // Parallel: pull a fresh strong example AND ask Gemini to acknowledge - // what the user just added. Both feed renderTeachBlock; both fail-open - // to '' so the static template still produces a usable block. - const [strong, acknowledgment] = await Promise.all([ - getStrongExample({ - teamToken, - userId: body.user_id, - prompt: body.prompt, - file_path: body.file_path, - target_dims: [lowest], - team_context: teamContext, - }), - acknowledgeProgress({ - previous_prompt: originalPrompt, - current_prompt: body.prompt, - previous_dimensions: previousDims, - current_dimensions: scoreResult.dimensions, - }), - ]); - const text = renderTeachBlock({ - targetDim: lowest, - targetScore: scoreResult.dimensions[lowest], - strongExample: strong.example, - strongExampleFromTeam: strong.fromTeam, - previousLowestDim: - previousLowest && previousLowest !== lowest ? previousLowest : undefined, - acknowledgment, - tip: strong.tip, - dimensions: scoreResult.dimensions, - overall, - }); - const nextState: RoundState = { - original_prompt: originalPrompt, - original_dimensions: originalDims, - previous_dimensions: scoreResult.dimensions, - round: round + 1, - }; - const next: CoachNextRoundInputs = { - ...nextState, - round_token: encodeRoundToken(nextState), - }; - const res: CoachResponse = { - proceed: false, - mode: 'score', - overall, - dimensions: scoreResult.dimensions, - missing: scoreResult.missing, - text, - next_round_inputs: next, - }; - return c.json(res); - } - - // Defensive fallback — shouldn't be reachable (overall < 7 implies a - // lowest dim exists). If we land here anyway, exit silently rather than - // 500ing the loop. - const res: CoachResponse = { - proceed: true, - mode: 'score', - overall, - dimensions: scoreResult.dimensions, - missing: scoreResult.missing, - text: '', - }; - return c.json(res); -}); - -// ----- POST /capture --------------------------------------------------------- -app.post('/capture', async (c) => { - const body = await c.req.json().catch(() => null); - if (!body || typeof body.user_prompt !== 'string' || typeof body.user_id !== 'string') { - return c.json({ error: 'bad_request' }, 400); - } - const surface = body.surface; - if (surface !== 'browser' && surface !== 'vscode' && surface !== 'mcp') { - return c.json({ error: 'bad_surface' }, 400); - } - - const rows = await q<{ id: string }>( - `INSERT INTO captures - (team_token, surface, user_prompt, ai_response, file_path, outcome, scored_dimensions) - VALUES ($1, $2, $3, $4, $5, $6, $7) - RETURNING id`, - [ - c.get('team_token'), - surface, - body.user_prompt, - body.ai_response ?? null, - body.file_path ?? null, - body.outcome ?? null, - body.scored_dimensions ? JSON.stringify(body.scored_dimensions) : null, - ], - ); - const res: CaptureResponse = { id: rows[0]!.id }; - return c.json(res); -}); - -// ----- POST /wiki/propose ---------------------------------------------------- -// Normalize → dedup on (node_id, body_normalized) → increment count → promote -// to durable at >= 3. Idempotent: repeated calls for the same insight only -// reinforce the existing draft. - -// Bigram-Jaccard similarity over normalized strings. Used as a paraphrase -// fallback when exact body_normalized match misses — catches "go through" / -// "flow through" style edits that the lowercase+strip-punct normalize can't -// collapse. Bigrams (vs unigrams) are deliberate: a polarity flip ("never" -// inserted into an otherwise identical sentence) drops the bigram score -// well below the threshold, so opposite-meaning insights stay distinct. -const PARAPHRASE_THRESHOLD = 0.7; - -// Headroom under Postgres's ~2704-byte btree tuple limit for -// idx_learnings_node_normalized (the key also carries node_id + tuple header). -const MAX_INSIGHT_BYTES = 2048; - -function bigramSet(normalized: string): Set { - const tokens = normalized.split(' ').filter(Boolean); - const out = new Set(); - for (let i = 0; i < tokens.length - 1; i++) { - out.add(`${tokens[i]} ${tokens[i + 1]}`); - } - return out; -} - -function bigramJaccard(a: string, b: string): number { - const aBg = bigramSet(a); - const bBg = bigramSet(b); - if (aBg.size === 0 || bBg.size === 0) return 0; - let intersection = 0; - for (const bg of aBg) if (bBg.has(bg)) intersection++; - const union = aBg.size + bBg.size - intersection; - return union === 0 ? 0 : intersection / union; -} - -app.post('/wiki/propose', async (c) => { - const body = await c.req.json().catch(() => null); - if (!body || typeof body.node_path !== 'string' || typeof body.insight !== 'string') { - return c.json({ error: 'bad_request' }, 400); - } - if (!body.insight.trim()) return c.json({ error: 'empty_insight' }, 400); - - // body_normalized is a btree index key, and Postgres rejects index rows over - // ~2.7 KB — a longer insight failed the INSERT with a raw 500. Insights are - // meant to be one-sentence conventions, so reject oversize ones up front. - const bodyNormalized = normalize(body.insight); - if (Buffer.byteLength(bodyNormalized, 'utf8') > MAX_INSIGHT_BYTES) { - return c.json( - { error: 'insight_too_long', detail: `max ${MAX_INSIGHT_BYTES} bytes after normalization` }, - 400, - ); - } - - const path = normalizePath(body.node_path); - const nodeId = await upsertNode(c.get('team_token'), path); - - // Step 1: exact match on body_normalized — the cheap fast path. Hits when - // the user (or LLM) sent the same insight verbatim or with only - // punctuation/whitespace/case differences. - const exact = await q<{ - id: string; reinforcement_count: number; status: 'draft' | 'durable'; - }>( - `SELECT id, reinforcement_count, status - FROM learnings - WHERE node_id = $1 AND body_normalized = $2 - LIMIT 1`, - [nodeId, bodyNormalized], - ); - - let matchId: string | null = null; - let matchPriorStatus: 'draft' | 'durable' | null = null; - if (exact.length) { - matchId = exact[0]!.id; - matchPriorStatus = exact[0]!.status; - } else { - // Step 2: paraphrase fallback. Pull this node's existing learnings and - // compute bigram-Jaccard against each. Keeps the wiki from accumulating - // near-duplicate drafts that never hit the 3× durability threshold. - const candidates = await q<{ - id: string; body_normalized: string; status: 'draft' | 'durable'; - }>( - `SELECT id, body_normalized, status - FROM learnings - WHERE node_id = $1`, - [nodeId], - ); - let best: { id: string; status: 'draft' | 'durable'; sim: number } | null = null; - for (const cand of candidates) { - const sim = bigramJaccard(bodyNormalized, cand.body_normalized); - if (sim >= PARAPHRASE_THRESHOLD && (!best || sim > best.sim)) { - best = { id: cand.id, status: cand.status, sim }; - } - } - if (best) { - matchId = best.id; - matchPriorStatus = best.status; - } - } - - let action: WikiProposeResponse['action']; - let currentCount: number; - let promotedToDurable = false; - - if (matchId === null) { - const inserted = await q<{ reinforcement_count: number }>( - `INSERT INTO learnings (node_id, body, body_normalized) - VALUES ($1, $2, $3) - RETURNING reinforcement_count`, - [nodeId, body.insight.trim(), bodyNormalized], - ); - action = 'created'; - currentCount = inserted[0]!.reinforcement_count; - } else { - const updated = await q<{ reinforcement_count: number; status: 'draft' | 'durable' }>( - `UPDATE learnings - SET reinforcement_count = reinforcement_count + 1, - last_seen_at = NOW(), - status = CASE WHEN reinforcement_count + 1 >= 3 THEN 'durable' ELSE status END - WHERE id = $1 - RETURNING reinforcement_count, status`, - [matchId], - ); - const after = updated[0]!; - currentCount = after.reinforcement_count; - promotedToDurable = matchPriorStatus === 'draft' && after.status === 'durable'; - action = promotedToDurable ? 'promoted' : 'reinforced'; - } - - invalidateTeamContext(c.get('team_token')); - const res: WikiProposeResponse = { - action, - current_count: currentCount, - ...(promotedToDurable ? { promoted_to_durable: true } : {}), - }; - return c.json(res); -}); - -// ----- GET /context?path= ---------------------------------------------------- -// HCL ancestor walk: every node whose path is a prefix of the file path, -// shallow → deep, plus its top durable learnings. - -app.get('/context', async (c) => { - const filePath = c.req.query('path') ?? ''; - if (!filePath) return c.json({ error: 'missing_path' }, 400); - - const ancestors = ancestorPaths(filePath); - if (ancestors.length === 0) { - const res: ContextResponse = { nodes: [] }; - return c.json(res); - } - - const rows = await q<{ - path: string; body_md: string; learning_body: string | null; reinforcement_count: number | null; - }>( - `SELECT n.path, n.body_md, l.body AS learning_body, l.reinforcement_count - FROM nodes n - LEFT JOIN learnings l - ON l.node_id = n.id - AND l.status = 'durable' - WHERE n.team_token = $1 - AND n.path = ANY($2::text[]) - ORDER BY length(n.path) ASC, n.path ASC, - COALESCE(l.reinforcement_count, 0) DESC`, - [c.get('team_token'), ancestors], - ); - - const byPath = new Map(); - for (const r of rows) { - let node = byPath.get(r.path); - if (!node) { - node = { path: r.path, body_md: r.body_md, durable_learnings: [] }; - byPath.set(r.path, node); - } - if (r.learning_body) { - node.durable_learnings.push({ - body: r.learning_body, - reinforcement_count: r.reinforcement_count ?? 0, - }); - } - } - - const res: ContextResponse = { - nodes: ancestors - .map((p) => byPath.get(p)) - .filter((n): n is ContextNode => Boolean(n)), - }; - return c.json(res); -}); - -// ----- GET /examples?path= --------------------------------------------------- -app.get('/examples', async (c) => { - const filePath = c.req.query('path') ?? ''; - if (!filePath) return c.json({ error: 'missing_path' }, 400); - const limit = intParam(c.req.query('limit'), 3, 1, 10); - const ancestors = ancestorPaths(filePath); - - const rows = await q<{ - template: string; topic: string | null; reuse_count: number; node_path: string; - }>( - `SELECT p.template, p.topic, p.reuse_count, n.path AS node_path - FROM prompts p - JOIN nodes n ON n.id = p.node_id - WHERE n.team_token = $1 - AND n.path = ANY($2::text[]) - AND p.status = 'graduated' - ORDER BY p.reuse_count DESC, length(n.path) DESC - LIMIT $3`, - [c.get('team_token'), ancestors, limit], - ); - - const res: ExamplesResponse = { items: rows.map((r): ExamplesItem => ({ - template: r.template, - topic: r.topic, - reuse_count: r.reuse_count, - node_path: r.node_path, - })) }; - return c.json(res); -}); - -// ----- GET /prompts/proven --------------------------------------------------- -// All graduated prompts for the team, optionally filtered by min score, an -// ancestor path, or topic. Powers the wiki_proven_prompts MCP tool. -// -// "Proven" == status='graduated'. Today every graduated prompt is by -// definition overall>=7 (the gate in /coach), and the actual score is now -// stored on graduated_overall_score so callers can filter ≥8 / ≥9 too. -// -// Ranking: score DESC, reuse_count DESC, created_at DESC. Score is the -// primary signal because reuse_count starts at 0 and grows over time — -// without the score tiebreaker, brand-new 10/10 prompts would rank below -// older 7/10 prompts that happened to be re-graduated once or twice. -app.get('/prompts/proven', async (c) => { - const minScore = intParam(c.req.query('min_score'), 7, 0, 10); - const limit = intParam(c.req.query('limit'), 20, 1, 100); - const pathScope = c.req.query('path'); - const topic = c.req.query('topic'); - const ancestors = pathScope ? ancestorPaths(pathScope) : null; - - const rows = await q<{ - id: string; - template: string; - topic: string | null; - reuse_count: number; - graduated_overall_score: number; - author_user_id: string | null; - node_path: string; - created_at: Date; - }>( - `SELECT p.id, p.template, p.topic, p.reuse_count, - p.graduated_overall_score, p.author_user_id, - n.path AS node_path, p.created_at - FROM prompts p - JOIN nodes n ON n.id = p.node_id - WHERE n.team_token = $1 - AND p.status = 'graduated' - AND p.graduated_overall_score >= $2 - AND ($3::text[] IS NULL OR n.path = ANY($3::text[])) - AND ($4::text IS NULL OR p.topic = $4) - ORDER BY p.graduated_overall_score DESC, - p.reuse_count DESC, - p.created_at DESC - LIMIT $5`, - [c.get('team_token'), minScore, ancestors, topic ?? null, limit], - ); - - const res: ProvenPromptsResponse = { - items: rows.map((r): ProvenPromptItem => ({ - id: r.id, - template: r.template, - topic: r.topic, - reuse_count: r.reuse_count, - graduated_overall_score: r.graduated_overall_score, - author_user_id: r.author_user_id, - node_path: r.node_path, - created_at: r.created_at.toISOString(), - })), - }; - return c.json(res); -}); - -// ----- GET /prompts/pending + POST /prompts/:id/review ------------------------ -// Review queue for TRAILHEAD_PROMOTION_MODE=review (promotion-gate.ts): /coach -// promotions land as 'pending_review' and only join the library when approved. -// Anyone holding the team secret can review — user ids are self-asserted, so -// "a teammate other than the author" is a team convention, not enforced. -app.get('/prompts/pending', async (c) => { - const rows = await q<{ - id: string; template: string; topic: string | null; graduated_overall_score: number; - author_user_id: string | null; node_path: string; created_at: Date; - }>( - `SELECT p.id, p.template, p.topic, p.graduated_overall_score, p.author_user_id, - n.path AS node_path, p.created_at - FROM prompts p - JOIN nodes n ON n.id = p.node_id - WHERE n.team_token = $1 AND p.status = 'pending_review' - ORDER BY p.created_at ASC - LIMIT 200`, - [c.get('team_token')], - ); - return c.json({ - items: rows.map((r) => ({ ...r, created_at: r.created_at.toISOString() })), - }); -}); - -app.post('/prompts/:id/review', async (c) => { - const id = c.req.param('id'); - if (!isUuid(id)) return c.json({ error: 'bad_request', detail: 'invalid prompt id' }, 400); - const body = await c.req.json<{ approve?: unknown }>().catch(() => null); - if (!body || typeof body.approve !== 'boolean') { - return c.json({ error: 'bad_request', detail: 'body must be { "approve": true | false }' }, 400); - } - // Scoped through nodes.team_token: another team's prompt id is a 404. - const rows = body.approve - ? await q<{ id: string }>( - `UPDATE prompts p SET status = 'graduated' - FROM nodes n - WHERE p.id = $1 AND p.node_id = n.id AND n.team_token = $2 AND p.status = 'pending_review' - RETURNING p.id`, - [id, c.get('team_token')], - ) - : await q<{ id: string }>( - `DELETE FROM prompts p - USING nodes n - WHERE p.id = $1 AND p.node_id = n.id AND n.team_token = $2 AND p.status = 'pending_review' - RETURNING p.id`, - [id, c.get('team_token')], - ); - if (!rows.length) return c.json({ error: 'not_found' }, 404); - return c.json({ id, status: body.approve ? 'graduated' : 'rejected' }); -}); - -// ----- GET /search?q=&scope= ------------------------------------------------- -// Free-text substring search across the team's wiki: rules (nodes.body_md), -// durable learnings (learnings.body), and graduated prompts (prompts.template). -// Optional `scope` constrains results to the ancestor paths of a file/folder -// (same shape as /context). Used by the wiki_lookup MCP tool when the caller -// passes only `query`, or `query` + `file_path` for a path-scoped search. -app.get('/search', async (c) => { - const query = (c.req.query('q') ?? '').trim(); - if (!query) return c.json({ error: 'missing_q' }, 400); - const limit = intParam(c.req.query('limit'), 50, 1, 100); - const scope = c.req.query('scope'); - // ILIKE wildcards from user input shouldn't bleed into the pattern. Escape - // %, _, and the escape char itself so a search for "100%" matches the - // literal substring rather than "100". - const escaped = query.replace(/[\\%_]/g, (ch) => `\\${ch}`); - const pattern = `%${escaped}%`; - const teamToken = c.get('team_token'); - const ancestors = scope ? ancestorPaths(scope) : null; - - // Rule branch matches on body_md OR the node path itself — so a query like - // "scoring" surfaces packages/scoring/ even when body_md doesn't repeat the - // folder name. Path-only matches return a placeholder body so the renderer - // doesn't dump the whole node narrative when the match was structural. - const rows = await q<{ kind: 'rule' | 'learning' | 'prompt'; body: string; node_path: string }>( - `SELECT 'rule'::text AS kind, - CASE - WHEN n.body_md ILIKE $2 THEN n.body_md - ELSE '(matched on path: ' || n.path || ')' - END AS body, - n.path AS node_path - FROM nodes n - WHERE n.team_token = $1 - AND (n.body_md ILIKE $2 OR n.path ILIKE $2) - AND ($3::text[] IS NULL OR n.path = ANY($3::text[])) - UNION ALL - SELECT 'learning'::text AS kind, l.body AS body, n.path AS node_path - FROM learnings l - JOIN nodes n ON n.id = l.node_id - WHERE n.team_token = $1 - AND l.status = 'durable' - AND l.body ILIKE $2 - AND ($3::text[] IS NULL OR n.path = ANY($3::text[])) - UNION ALL - SELECT 'prompt'::text AS kind, p.template AS body, n.path AS node_path - FROM prompts p - JOIN nodes n ON n.id = p.node_id - WHERE n.team_token = $1 - AND p.status = 'graduated' - AND p.template ILIKE $2 - AND ($3::text[] IS NULL OR n.path = ANY($3::text[])) - LIMIT $4`, - [teamToken, pattern, ancestors, limit], - ); - - const res: SearchResponse = { items: rows }; - return c.json(res); -}); - -// ----- GET /wiki/recent?since=ISO -------------------------------------------- -app.get('/wiki/recent', async (c) => { - const sinceParam = c.req.query('since'); - const since = sinceParam ? new Date(sinceParam) : new Date(Date.now() - 60 * 60 * 1000); - if (Number.isNaN(since.getTime())) return c.json({ error: 'bad_since' }, 400); - const limit = intParam(c.req.query('limit'), 50, 1, 200); - - const rows = await q<{ - id: string; - node_path: string; - body: string; - status: 'draft' | 'durable'; - reinforcement_count: number; - last_seen_at: Date; - created_at: Date; - }>( - `SELECT l.id, n.path AS node_path, l.body, l.status, - l.reinforcement_count, l.last_seen_at, l.created_at - FROM learnings l - JOIN nodes n ON n.id = l.node_id - WHERE n.team_token = $1 - AND l.last_seen_at > $2 - ORDER BY l.last_seen_at DESC - LIMIT $3`, - [c.get('team_token'), since.toISOString(), limit], - ); - - const res: WikiRecentResponse = { - items: rows.map((r): WikiRecentItem => ({ - id: r.id, - node_path: r.node_path, - body: r.body, - status: r.status, - reinforcement_count: r.reinforcement_count, - last_seen_at: r.last_seen_at.toISOString(), - created_at: r.created_at.toISOString(), - })), - }; - return c.json(res); -}); - -// ----- POST /diff ------------------------------------------------------------ -// Find the closest graduated prompt in the same path/topic ancestry, score -// both, and have Gemma narrate the differences. Spec §10. - -app.post('/diff', async (c) => { - const body = await c.req.json().catch(() => null); - if (!body || typeof body.user_prompt !== 'string' || typeof body.user_id !== 'string') { - return c.json({ error: 'bad_request' }, 400); - } - if (promptTooLong(body.user_prompt)) return c.json(PROMPT_TOO_LONG, 413); - - const ancestors = body.file_path ? ancestorPaths(body.file_path) : ['']; - const topic = await extractTopic(body.user_prompt); - - // Prefer same-topic + same-ancestry. Fall back to any topic in ancestry. - // Final fallback: any graduated prompt in this team. - let candidate = ( - await q<{ template: string; topic: string | null; node_path: string }>( - `SELECT p.template, p.topic, n.path AS node_path - FROM prompts p - JOIN nodes n ON n.id = p.node_id - WHERE n.team_token = $1 - AND p.status = 'graduated' - AND p.topic = $2 - AND n.path = ANY($3::text[]) - ORDER BY p.reuse_count DESC, length(n.path) DESC - LIMIT 1`, - [c.get('team_token'), topic, ancestors], - ) - )[0]; - if (!candidate) { - candidate = ( - await q<{ template: string; topic: string | null; node_path: string }>( - `SELECT p.template, p.topic, n.path AS node_path - FROM prompts p - JOIN nodes n ON n.id = p.node_id - WHERE n.team_token = $1 - AND p.status = 'graduated' - AND n.path = ANY($2::text[]) - ORDER BY p.reuse_count DESC, length(n.path) DESC - LIMIT 1`, - [c.get('team_token'), ancestors], - ) - )[0]; - } - if (!candidate) { - candidate = ( - await q<{ template: string; topic: string | null; node_path: string }>( - `SELECT p.template, p.topic, n.path AS node_path - FROM prompts p - JOIN nodes n ON n.id = p.node_id - WHERE n.team_token = $1 AND p.status = 'graduated' - ORDER BY p.reuse_count DESC - LIMIT 1`, - [c.get('team_token')], - ) - )[0]; - } - if (!candidate) { - return c.json({ error: 'no_team_prompts_available' }, 404); - } - - // Sequential, not parallel: Gemini's free tier serialises requests per - // API key in practice — concurrent Flash calls slow each other to a - // crawl. Sequential keeps total /diff latency below 10s. - const userScore = await scorePrompt({ prompt: body.user_prompt, file_path: body.file_path }); - const teamScore = await scorePrompt({ prompt: candidate.template, file_path: candidate.node_path }); - const narrative = await synthesizeDiff({ - user_prompt: body.user_prompt, - user_scores: userScore.dimensions as DimensionScores, - team_prompt: candidate.template, - team_scores: teamScore.dimensions as DimensionScores, - }); - - const res: DiffResponse = { - user: { - prompt: body.user_prompt, - overall: overallScore(userScore.dimensions), - dimensions: userScore.dimensions, - }, - team: { - prompt: candidate.template, - overall: overallScore(teamScore.dimensions), - dimensions: teamScore.dimensions, - node_path: candidate.node_path, - topic: candidate.topic, - }, - narrative, - }; - return c.json(res); -}); - -// ----- GET /skill-arc?user_id=&since=ISO ------------------------------------- -// Time-series of per-dimension scores for the dashboard hero chart. Drives -// the §13 close beat (live tick during demo). Defaults: any user, last 7 -// days. Capped at 5000 rows to keep the chart responsive. - -app.get('/skill-arc', async (c) => { - const userIdParam = c.req.query('user_id'); - const sinceParam = c.req.query('since'); - const since = sinceParam ? new Date(sinceParam) : new Date(Date.now() - 7 * 24 * 60 * 60 * 1000); - if (Number.isNaN(since.getTime())) return c.json({ error: 'bad_since' }, 400); - const limit = intParam(c.req.query('limit'), 1000, 1, 5000); - - const rows = userIdParam - ? await q<{ dimension: Dimension; score: number; ts: Date }>( - `SELECT dimension, score, ts - FROM skill_observations - WHERE team_token = $1 AND user_id = $2 AND ts > $3 - ORDER BY ts ASC - LIMIT $4`, - [c.get('team_token'), userIdParam, since.toISOString(), limit], - ) - : await q<{ dimension: Dimension; score: number; ts: Date }>( - `SELECT dimension, score, ts - FROM skill_observations - WHERE team_token = $1 AND ts > $2 - ORDER BY ts ASC - LIMIT $3`, - [c.get('team_token'), since.toISOString(), limit], - ); - - const res: SkillArcResponse = { - observations: rows.map((r): SkillArcObservation => ({ - dimension: r.dimension, - score: r.score, - ts: r.ts.toISOString(), - })), - }; - return c.json(res); -}); - -// ----- GET /team/metrics ----------------------------------------------------- -// Snapshot for the dashboard /team page. All cheap aggregate counts; no -// time-series. Reuse rate is captures with outcome='helpful' over total -// captures (proxy for "team's prompts work" until we have the real -// graduated-prompt-match metric). - -app.get('/team/metrics', async (c) => { - const sevenDaysAgo = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000).toISOString(); - - const [obs, learnings, captures, users] = await Promise.all([ - q<{ avg_overall: number | null; total_obs: number }>( - `SELECT AVG(score)::float AS avg_overall, COUNT(*)::int AS total_obs - FROM skill_observations - WHERE team_token = $1 AND ts > $2`, - [c.get('team_token'), sevenDaysAgo], - ), - q<{ durable_count: number; draft_count: number }>( - `SELECT - COUNT(*) FILTER (WHERE l.status = 'durable')::int AS durable_count, - COUNT(*) FILTER (WHERE l.status = 'draft')::int AS draft_count - FROM learnings l - JOIN nodes n ON n.id = l.node_id - WHERE n.team_token = $1`, - [c.get('team_token')], - ), - q<{ total: number; helpful: number }>( - `SELECT - COUNT(*)::int AS total, - COUNT(*) FILTER (WHERE outcome = 'helpful')::int AS helpful - FROM captures - WHERE team_token = $1 AND created_at > $2`, - [c.get('team_token'), sevenDaysAgo], - ), - q<{ active_users: number }>( - `SELECT COUNT(DISTINCT user_id)::int AS active_users - FROM skill_observations - WHERE team_token = $1 AND ts > $2`, - [c.get('team_token'), sevenDaysAgo], - ), - ]); - - const obsRow = obs[0]!; - const learningsRow = learnings[0]!; - const capturesRow = captures[0]!; - const usersRow = users[0]!; - - const res: TeamMetricsResponse = { - avg_overall: obsRow.avg_overall ? Math.round(obsRow.avg_overall * 10) / 10 : 0, - reuse_rate: capturesRow.total > 0 ? capturesRow.helpful / capturesRow.total : 0, - durable_count: learningsRow.durable_count, - draft_count: learningsRow.draft_count, - total_obs: obsRow.total_obs, - active_users: usersRow.active_users, - }; - return c.json(res); -}); - -// ----- GET /wiki/tree -------------------------------------------------------- -// Full node list for the dashboard /wiki page. One row per node with its -// learnings split into durable vs draft. Sort by path (prefix-friendly). - -app.get('/wiki/tree', async (c) => { - const nodes = await loadWikiTree(c.get('team_token')); - const res: WikiTreeResponse = { nodes }; - return c.json(res); -}); - -// ----- GET /wiki/export ----------------------------------------------------- -// Markdown export of the whole team wiki. Teams will not pour knowledge into -// a store they cannot get it back out of, so this is a trust signal as much -// as a backup story. -// -// GET /wiki/export -> text/markdown, as a download -// GET /wiki/export?drafts=true -> include draft learnings too -// GET /wiki/export?format=json -> { filename, markdown } for browser clients -app.get('/wiki/export', async (c) => { - const teamToken = c.get('team_token'); - const [nodes, teamRows] = await Promise.all([ - loadWikiTree(teamToken), - q<{ name: string }>('SELECT name FROM teams WHERE token = $1', [teamToken]), - ]); - const teamName = teamRows[0]?.name; - const now = new Date(); - const markdown = renderWikiMarkdown(nodes, { - teamName, - generatedAt: now, - includeDrafts: c.req.query('drafts') === 'true', - }); - const filename = exportFilename(teamName, now); - - if (c.req.query('format') === 'json') { - return c.json({ filename, markdown }); - } - return new Response(markdown, { - status: 200, - headers: { - 'content-type': 'text/markdown; charset=utf-8', - 'content-disposition': `attachment; filename="${filename}"`, - }, - }); -}); - -// ----- GET /teams ------------------------------------------------------------ -// Resolves the CALLER's team. Authenticated, and it never returns a token. -// -// This used to be unauthenticated and return every team on the server together -// with its token — with CORS `*`, so any web page could read it. The team token -// is the only credential in this system: it grants read on the wiki (which -// summarises private source code) and write on everything. A single unauth GET -// therefore compromised every tenant at once. The browser popup's convenience -// of pre-populating a team dropdown before any token was configured is what -// paid for that, and it is nowhere near worth the price. -// -// The response is a list of one so the TeamsListResponse shape (and every -// caller that maps over `teams`) keeps working. -app.get('/teams', async (c) => { - const teamToken = c.get('team_token'); - const legacy = c.get('legacy_auth'); - const rows = await q<{ name: string; token: string }>( - 'SELECT name, token FROM teams WHERE token = $1', - [teamToken], - ); - const teams: TeamSummary[] = rows.map((r) => ({ - name: r.name, - id: opaqueTeamId(r.token), - legacy, - // A secret team's id is public by design. A legacy team's id IS its - // credential, so it is never echoed. - ...(legacy ? {} : { team_id: r.token }), - })); - const res: TeamsListResponse = { teams }; - return c.json(res); -}); - -// ----- POST /teams ----------------------------------------------------------- -// Register a team. Returns its secret exactly once; only the hash is stored. -// -// body: { team_id?: string, name?: string } -// 201 { team_id, name, secret } -// 409 team_exists — someone already registered this id. That is the join -// flow: ask a teammate for the secret (or pick another team_id). -// 403 admin_token_required — TRAILHEAD_ADMIN_TOKEN is set and X-Admin-Token -// did not match. -// -// `init` derives team_id from the normalised git remote URL, so every clone of -// a repo proposes the same id and the second one lands on the 409. -app.post('/teams', async (c) => { - // Before the admin check, so the limit also slows admin-token guessing. - const limited = rateLimit(c, [['register_per_ip', clientIp(c)]]); - if (limited) return limited; - const { adminToken } = authPolicy(); - if (adminToken !== null && !safeEqual(c.req.header('x-admin-token') ?? '', adminToken)) { - return c.json( - { - error: 'admin_token_required', - detail: 'This server restricts team registration. Ask its operator for a team secret, or for the admin token.', - }, - 403, - ); - } - const body = (await c.req.json<{ team_id?: unknown; name?: unknown }>().catch(() => null)) ?? {}; - if (body.team_id !== undefined && !isValidTeamId(body.team_id)) { - return c.json( - { error: 'bad_team_id', detail: '3-100 chars of letters, digits, _ . - starting with a letter or digit' }, - 400, - ); - } - const teamId = (body.team_id as string | undefined) ?? randomTeamId(); - const rawName = typeof body.name === 'string' ? body.name.trim().slice(0, 200) : ''; - const created = await registerTeam(teamId, rawName || `team:${teamId.slice(0, 16)}`); - if (!created) { - return c.json( - { - error: 'team_exists', - team_id: teamId, - detail: - 'A team with this id already exists. To join it, get the team secret from a teammate ' + - '(it is in their .trailhead-team file) and run `init --team-token `.', - }, - 409, - ); - } - return c.json({ team_id: created.teamId, name: created.name, secret: created.secret }, 201); -}); - -// ----- POST /teams/rotate-secret --------------------------------------------- -// Mint a new secret for the caller's team; the old credential stops working -// immediately. For a legacy team this is the upgrade to the secret model: -// afterwards its id is no longer accepted as a credential. Share the new -// secret with teammates. -app.post('/teams/rotate-secret', async (c) => { - const teamId = c.get('team_token'); - if (teamId === DEMO_TEAM_TOKEN) { - return c.json({ error: 'demo_team_protected', detail: 'The demo team keeps its public secret.' }, 403); - } - const secret = await rotateTeamSecret(teamId); - const rows = await q<{ name: string }>('SELECT name FROM teams WHERE token = $1', [teamId]); - return c.json({ team_id: teamId, name: rows[0]?.name ?? '', secret }); -}); - -// ----- POST /improve --------------------------------------------------------- -// Gemini-driven multi-turn prompt coach. Stateless — caller carries the full -// conversation each turn. Spec: 2026-04-26-improve-widget-design.md -const IMPROVE_TURN_CAP = 5; // user replies; history.length cap is 2 * cap - -app.post('/improve', async (c) => { - const body = await c.req.json().catch(() => null); - if ( - !body || - typeof body.original_prompt !== 'string' || - typeof body.user_id !== 'string' || - !Array.isArray(body.history) || - (body.command !== 'next' && body.command !== 'finalize') - ) { - return c.json({ error: 'bad_request' }, 400); - } - - // Validate every history entry; reject anything malformed so we never - // hand garbage to Gemini. - for (const t of body.history as ImproveTurn[]) { - if ( - !t || - (t.role !== 'assistant' && t.role !== 'user') || - typeof t.text !== 'string' - ) { - return c.json({ error: 'bad_request' }, 400); - } - } - if (promptTooLong(body.original_prompt, ...body.history.map((t) => t.text))) { - return c.json(PROMPT_TOO_LONG, 413); - } - - // Server-side cap: if the user has already replied IMPROVE_TURN_CAP times, - // force finalize regardless of the client-supplied command. The client - // also enforces this; the server check is a safety net. - const userReplies = body.history.filter((t) => t.role === 'user').length; - const command = userReplies >= IMPROVE_TURN_CAP ? 'finalize' : body.command; - - // Same context-injection pattern as /score — give Gemini the team's wiki - // subtree as system context so the coach's clarifying questions and the - // polished prompt land in the team's idiom. - const teamContext = body.context_path - ? await renderTeamContext(c.get('team_token'), body.context_path) - : null; - - try { - const out = await improveCoach({ - original_prompt: body.original_prompt, - missing: body.missing ?? {}, - history: body.history, - command, - team_context: teamContext ?? undefined, - }); - if (out.kind === 'question') { - const res: ImproveResponse = { - kind: 'question', - text: out.text, - turn: userReplies + 1, - }; - return c.json(res); - } - const res: ImproveResponse = { - kind: 'final', - polished: out.polished, - rationale: out.rationale, - }; - return c.json(res); - } catch (err) { - console.warn('[api] /improve failed', err); - return c.json({ error: 'improve_failed' }, 502); - } -}); - -// ----- DELETE /team/data ----------------------------------------------------- -// Wipe every nodes / learnings / prompts / captures / skill_observations row -// for the requesting team. The teams row itself is preserved so re-running -// the same token continues to land in the same id (matters for the -// trailhead-mcp reset CLI which talks to localhost first then prod). -// -// The demo team is protected against accidental nukes — wiping it would -// erase the seeded data the dashboard demo relies on. Override with -// TRAILHEAD_ALLOW_DEMO_RESET=true if you really need to reseed. - -app.delete('/team/data', async (c) => { - const body = await c.req.json<{ confirm?: boolean }>().catch(() => null); - if (!body || body.confirm !== true) { - return c.json( - { error: 'confirm_required', detail: 'POST { "confirm": true } to wipe.' }, - 400, - ); - } - const teamToken = c.get('team_token'); - const isDemo = teamToken === DEMO_TEAM_TOKEN; - if (isDemo && process.env.TRAILHEAD_ALLOW_DEMO_RESET !== 'true') { - return c.json( - { - error: 'demo_team_protected', - detail: - 'Refusing to wipe the demo team. Set TRAILHEAD_ALLOW_DEMO_RESET=true on the API to override.', - }, - 403, - ); - } - const deleted = await wipeTeamData(teamToken); - invalidateTeamContext(teamToken); - return c.json({ team_token: teamToken, deleted }); -}); - -// ----- POST /onboard/repo ---------------------------------------------------- -// Bootstrap a team wiki by upserting one node per path. Idempotent: re-running -// with the same paths is a no-op (the existing node row is left untouched). -// `initial_rules[path]` lets the caller seed `body_md` for any/all of the -// supplied paths — useful when the caller has, say, scanned a repo's existing -// CLAUDE.md or copied conventions from another tool. -// -// Spec ref: -// docs/superpowers/specs/2026-04-25-demo-completion-design.md §C.1 -// docs/superpowers/specs/2026-04-25-mcp-plugin-ux-design.md (bootstrap UX -// follow-up — wired through wiki_bootstrap MCP tool + `trailhead-mcp -// bootstrap` CLI subcommand). - -const ONBOARD_MAX_PATHS = 200; - -app.post('/onboard/repo', async (c) => { - const body = await c.req.json().catch(() => null); - if (!body || !Array.isArray(body.paths)) { - return c.json({ error: 'bad_request', detail: 'paths: string[] required' }, 400); - } - if (body.paths.length === 0) { - return c.json({ error: 'bad_request', detail: 'paths must not be empty' }, 400); - } - if (body.paths.length > ONBOARD_MAX_PATHS) { - return c.json( - { error: 'too_many_paths', detail: `max ${ONBOARD_MAX_PATHS} paths per request` }, - 400, - ); - } - const initialRules = - body.initial_rules && typeof body.initial_rules === 'object' && !Array.isArray(body.initial_rules) - ? body.initial_rules - : {}; - - // Normalize + dedupe paths so we don't issue duplicate inserts inside one - // request (the UNIQUE constraint would catch it but the per-row upsert - // round-trip is wasted). - const seen = new Set(); - const normalized: { raw: string; path: string }[] = []; - for (const raw of body.paths) { - if (typeof raw !== 'string') continue; - const path = normalizePath(raw); - if (!path) continue; // empty string after normalization — skip - if (seen.has(path)) continue; - seen.add(path); - normalized.push({ raw, path }); - } - - if (normalized.length === 0) { - return c.json({ error: 'bad_request', detail: 'no valid paths after normalization' }, 400); - } - - if (typeof body.team_name === 'string' && body.team_name.trim()) { - await applyTeamNameIfPlaceholder(c.get('team_token'), body.team_name); - } - - const nodes: { path: string; id: string }[] = []; - let nodes_created = 0; - - for (const { raw, path } of normalized) { - // body_md from initial_rules — match against either the normalized form - // or the caller's raw string so callers don't need to pre-normalize keys. - const seedBody = - typeof initialRules[path] === 'string' - ? initialRules[path] - : typeof initialRules[raw] === 'string' - ? initialRules[raw] - : ''; - - // xmax = 0 in the RETURNING row means the tuple was newly inserted (PG - // marks it 0 on fresh inserts; ON CONFLICT updates set xmax to the - // current xid). Lets us count creates without a second query. - const rows = await q<{ id: string; inserted: boolean }>( - `INSERT INTO nodes (team_token, path, body_md) - VALUES ($1, $2, $3) - ON CONFLICT (team_token, path) DO UPDATE - SET body_md = CASE - WHEN $3 <> '' AND nodes.body_md = '' THEN $3 - ELSE nodes.body_md - END, - updated_at = NOW() - RETURNING id, (xmax = 0) AS inserted`, - [c.get('team_token'), path, seedBody], - ); - const row = rows[0]!; - if (row.inserted) nodes_created += 1; - nodes.push({ path, id: row.id }); - } - - invalidateTeamContext(c.get('team_token')); - const res: OnboardRepoResponse = { nodes_created, nodes }; - return c.json(res); -}); - -// ----- POST /onboard/repo/full ----------------------------------------------- -// Rich (LLM-generated) bootstrap. Accepts the discovered folder paths plus -// the file contents (already capped client-side) plus optional manifest -// snippets and CLAUDE.md seed text. Creates a wiki_jobs row + one -// wiki_job_paths row per node and kicks off the worker via setImmediate. -// Returns the job_id immediately; the worker fills body_md asynchronously. -// -// Spec: docs/superpowers/specs/2026-04-26-wiki-bootstrap-rich-design.md §9 - -// Server-side hard ceilings. Independent of the client's CLI flags so a -// rogue client can't blow the API host's RAM. Conservative — these are -// "abuse cap" not "expected size". -const ONBOARD_FULL_MAX_FOLDERS = 1_000; -const ONBOARD_FULL_MAX_FILES = 2_000; -const ONBOARD_FULL_MAX_FILE_CHARS = 32_000; // per file -const ONBOARD_FULL_MAX_BUNDLE_BYTES = 16 * 1024 * 1024; // 16 MB - -app.post('/onboard/repo/full', async (c) => { - const body = await c.req.json().catch(() => null); - if (!body || !Array.isArray(body.folders) || !Array.isArray(body.files)) { - return c.json( - { error: 'bad_request', detail: 'folders: string[] and files: {path,content}[] required' }, - 400, - ); - } - if (body.folders.length > ONBOARD_FULL_MAX_FOLDERS) { - return c.json({ error: 'too_many_folders', detail: `max ${ONBOARD_FULL_MAX_FOLDERS}` }, 400); - } - if (body.files.length > ONBOARD_FULL_MAX_FILES) { - return c.json({ error: 'too_many_files', detail: `max ${ONBOARD_FULL_MAX_FILES}` }, 400); - } - let bundleBytes = 0; - for (const f of body.files) { - if (typeof f?.path !== 'string' || typeof f?.content !== 'string') { - return c.json({ error: 'bad_request', detail: 'each file requires path:string and content:string' }, 400); - } - if (f.content.length > ONBOARD_FULL_MAX_FILE_CHARS) { - return c.json( - { error: 'file_too_large', detail: `${f.path}: max ${ONBOARD_FULL_MAX_FILE_CHARS} chars per file` }, - 400, - ); - } - bundleBytes += Buffer.byteLength(f.content, 'utf8'); - if (bundleBytes > ONBOARD_FULL_MAX_BUNDLE_BYTES) { - return c.json( - { error: 'bundle_too_large', detail: `max ${ONBOARD_FULL_MAX_BUNDLE_BYTES / (1024 * 1024)} MB` }, - 400, - ); - } - } - - const teamToken = c.get('team_token'); - - if (typeof body.team_name === 'string' && body.team_name.trim()) { - await applyTeamNameIfPlaceholder(teamToken, body.team_name); - } - - // Normalize folder paths (trailing slash) and dedupe. - const folderSet = new Set(); - for (const raw of body.folders) { - const p = normalizePath(raw); - if (p) folderSet.add(p); - } - const folders = [...folderSet]; - // De-dupe files on path; preserve first occurrence. - const seenFiles = new Set(); - const files = body.files.filter((f) => { - const p = String(f.path).trim(); - if (!p || p.endsWith('/') || seenFiles.has(p)) return false; - seenFiles.add(p); - return true; - }); - - // paths_total = folders + files + 1 root pass. - const pathsTotal = folders.length + files.length + 1; - - // Insert job header and per-path rows in one transaction so a partial - // failure doesn't leave a job with no work items. - const jobRows = await q<{ id: string }>( - `INSERT INTO wiki_jobs (team_token, paths_total) VALUES ($1, $2) RETURNING id`, - [teamToken, pathsTotal], - ); - const jobId = jobRows[0]!.id; - - // Build wiki_job_paths rows. Use a single multi-row insert for speed. - const pathRows: Array<[string, string, WikiJobPathKind]> = [ - [jobId, '', 'root'], - ...folders.map((p): [string, string, WikiJobPathKind] => [jobId, p, 'folder']), - ...files.map((f): [string, string, WikiJobPathKind] => [jobId, f.path, 'file']), - ]; - // Pg parameter array unrolling — keep it simple with one INSERT per row; - // the volume is low enough (typically 100-700 rows) that batching isn't - // critical, and the simpler code is harder to get wrong. - for (const [job, p, kind] of pathRows) { - await q( - `INSERT INTO wiki_job_paths (job_id, path, kind) VALUES ($1, $2, $3) - ON CONFLICT (job_id, path) DO NOTHING`, - [job, p, kind], - ); - } - - // Kick off the worker. setImmediate keeps it strictly fire-and-forget — - // the response returns now; runJob handles its own errors and never - // throws to here. - const bundle = bundleFromRequest({ ...body, folders, files }); - setImmediate(() => { - runJob(jobId, teamToken, bundle).catch((e) => { - console.error(`[wiki-job ${jobId}] uncaught:`, e); - }); - }); - - const res: OnboardRepoFullResponse = { job_id: jobId, paths_total: pathsTotal }; - return c.json(res); -}); - -// ----- GET /onboard/jobs/:id ------------------------------------------------- -// Status snapshot for a rich-bootstrap job. Clients (CLI, MCP tool) poll -// this every 2s. Returns the job header counters plus per-path rows so the -// UI can render which path is processing / which failed. -// -// Cross-team safety: the auth middleware sets team_token from the X-Team-Token -// header; the WHERE clause filters on it. A team can only see its own jobs -// (otherwise a leaked job_id would be a tenancy break). - -app.get('/onboard/jobs/:id', async (c) => { - const id = c.req.param('id'); - if (!id || !isUuid(id)) { - return c.json({ error: 'bad_request', detail: 'invalid job id' }, 400); - } - const teamToken = c.get('team_token'); - - const headers = await q<{ - id: string; - status: 'pending' | 'running' | 'done' | 'failed'; - paths_total: number; - paths_done: number; - paths_failed: number; - started_at: Date | null; - finished_at: Date | null; - error: string | null; - }>( - `SELECT id, status, paths_total, paths_done, paths_failed, started_at, finished_at, error - FROM wiki_jobs WHERE team_token = $1 AND id = $2`, - [teamToken, id], - ); - if (headers.length === 0) return c.json({ error: 'not_found' }, 404); - const h = headers[0]!; - - const pathRows = await q<{ - path: string; kind: WikiJobPathKind; status: WikiJobPathStatus['status']; error: string | null; - }>( - `SELECT path, kind, status, error FROM wiki_job_paths WHERE job_id = $1 ORDER BY kind, path`, - [id], - ); - - const res: WikiJobStatusResponse = { - job_id: h.id, - status: h.status, - paths_total: h.paths_total, - paths_done: h.paths_done, - paths_failed: h.paths_failed, - started_at: h.started_at ? h.started_at.toISOString() : null, - finished_at: h.finished_at ? h.finished_at.toISOString() : null, - error: h.error, - paths: pathRows.map((r) => ({ - path: r.path, - kind: r.kind, - status: r.status, - ...(r.error ? { error: r.error } : {}), - })), - }; - return c.json(res); -}); +// Routes, one module per area (src/routes/). Mounted after the middleware above, +// so every route gets auth, body caps, tracing and rate limits. +app.route('/', scoreRoutes); +app.route('/', coachRoutes); +app.route('/', wikiRoutes); +app.route('/', promptRoutes); +app.route('/', assistRoutes); +app.route('/', metricsRoutes); +app.route('/', teamRoutes); +app.route('/', onboardRoutes); // Surface unhandled errors as 500 with a one-line shape clients can show. // Unhandled errors → 500 with a short request id that also appears in the diff --git a/apps/api/src/http.ts b/apps/api/src/http.ts new file mode 100644 index 0000000..ac06372 --- /dev/null +++ b/apps/api/src/http.ts @@ -0,0 +1,158 @@ +// Request plumbing shared by app.ts (middleware) and the route modules in +// routes/: the Hono env type, tenant policy, rate limiting, and the +// prompt-size cap and skill-observation writer that /score and /coach share. +import type { Context } from 'hono'; +import { getConnInfo } from '@hono/node-server/conninfo'; +import type { DimensionScores } from '@trailhead/shared'; +import { DIMENSIONS } from '@trailhead/shared'; +import { q } from './db.ts'; +import { ipRateKey, limiterFor, type LimitName, type TokenBucketLimiter } from './rate-limit.ts'; +import { createHash } from 'node:crypto'; + +// Tenant policy, read per request so tests (and operators flipping env in a +// long-lived process manager) see the current value. +// +// TRAILHEAD_ACCEPT_LEGACY_TOKENS (default true) — accept pre-2026-09-30 +// id-as-credential tokens (repo_…, custom strings) for teams that have not +// been given a secret yet. Deprecated; set false once every team has run +// `init --upgrade-legacy` (or POST /teams/rotate-secret). +// TRAILHEAD_AUTO_CREATE_TEAMS (default false) — with legacy tokens accepted, +// an unknown X-Team-Token spawns a legacy team. Unauthenticated tenant +// creation; only for throwaway demo deploys. Has no effect when legacy +// tokens are off. New teams should use POST /teams instead. +// TRAILHEAD_ADMIN_TOKEN (default unset) — when set, POST /teams requires it +// in X-Admin-Token, so only the operator can register teams. Unset means +// open registration, which is fine while the API is bound to 127.0.0.1. +export function authPolicy() { + return { + acceptLegacy: process.env.TRAILHEAD_ACCEPT_LEGACY_TOKENS !== 'false', + autoCreate: process.env.TRAILHEAD_AUTO_CREATE_TEAMS === 'true', + adminToken: process.env.TRAILHEAD_ADMIN_TOKEN || null, + }; +} + + +// Hono context typing — the auth middleware sets `team_token` (the team id; +// the column kept its historical name) and `legacy_auth` for every handler. +export type AppEnv = { Variables: { team_token: string; legacy_auth: boolean } }; + +// The caller's rate-limit key: the socket's remote address, or — only when +// TRAILHEAD_TRUST_PROXY=true, i.e. behind one reverse proxy you control — the +// LAST X-Forwarded-For entry, which is the one that proxy appended. Earlier +// entries come from the client (nginx's $proxy_add_x_forwarded_for, Caddy and +// Traefik all append to whatever the client sent), so reading the first one +// would let every request pick its own key. Trusting the header without a +// proxy would do the same. IPv6 is keyed per /64 (ipRateKey). +export function clientIp(c: Context): string { + if (process.env.TRAILHEAD_TRUST_PROXY === 'true') { + const fwd = c.req.header('x-forwarded-for')?.split(',').at(-1)?.trim(); + if (fwd) return ipRateKey(fwd); + } + try { + return ipRateKey(getConnInfo(c).remote.address ?? 'unknown'); + } catch { + return 'unknown'; // no Node socket (app.request() in tests) + } +} + +// Take a token from each named limiter, all or none: every bucket is checked +// first, and only when all of them allow the request is a token spent in +// each. So a request refused by the per-team limit doesn't also use up the +// caller's per-IP allowance. On refusal, returns the 429 for the first +// refusing limit. Retry-After is in whole seconds. Clients treat 429 like any +// other failure: coaching fails open. (Synchronous, so no request can slip in +// between the checks and the takes.) +export function rateLimit(c: Context, checks: Array<[LimitName, string]>): Response | null { + const active = checks + .map(([name, key]) => ({ name, key, limiter: limiterFor(name) })) + .filter((x): x is { name: LimitName; key: string; limiter: TokenBucketLimiter } => x.limiter !== null); + for (const { name, key, limiter } of active) { + const r = limiter.peek(key); + if (r.ok) continue; + c.header('Retry-After', String(r.retryAfterSec)); + return c.json( + { + error: 'rate_limited', + limit: name, + retry_after: r.retryAfterSec, + detail: `Too many requests (${name.replace(/_/g, ' ')}: ${limiter.spec.text}). Retry in ${r.retryAfterSec}s.`, + }, + 429, + ); + } + for (const { key, limiter } of active) limiter.take(key); + return null; +} + +// A stable, opaque handle for a team that is safe to hand to a client. +// +// SHA-256 of the token, truncated to 16 hex chars. Not reversible, not +// replayable as an X-Team-Token, and stable across requests so it works as a +// React key or a client-side lookup handle. +export function opaqueTeamId(token: string): string { + return createHash('sha256').update(token).digest('hex').slice(0, 16); +} + +export function teamIdFromHeader(token: string | undefined): string | null { + return token ? opaqueTeamId(token) : null; +} + +export function simpleHash(s: string): string { + // Tiny non-crypto hash for the skill_observation dedup key. Collisions + // are harmless here — they'd just suppress one extra row. + let h = 0; + for (let i = 0; i < s.length; i++) h = ((h << 5) - h + s.charCodeAt(i)) | 0; + return h.toString(16); +} + +// Bulk insert one skill_observation row per dimension with the per-(user, +// dim, prompt-hash) 30s dedup window from spec §19. Shared between /score +// and /coach so both write to the same rubric stream — the dashboard and +// skill arc don't care which endpoint produced the row. +export async function writeSkillObservations( + teamToken: string, + userId: string, + prompt: string, + dimensions: DimensionScores, +): Promise { + const promptHash = simpleHash(prompt); + await q( + `INSERT INTO skill_observations (team_token, user_id, dimension, score, prompt_hash) + SELECT i.team_token, i.user_id, i.dimension, i.score, i.prompt_hash + FROM ( VALUES + ${DIMENSIONS.map((_, i) => + `($1::text, $2::text, $${3 + i * 2}::text, $${4 + i * 2}::int, $13::text)` + ).join(',\n ')} + ) AS i(team_token, user_id, dimension, score, prompt_hash) + WHERE NOT EXISTS ( + SELECT 1 FROM skill_observations s + WHERE s.team_token = i.team_token + AND s.user_id = i.user_id + AND s.dimension = i.dimension + AND s.prompt_hash = i.prompt_hash + AND s.ts > NOW() - INTERVAL '30 seconds' + )`, + [ + teamToken, + userId, + ...DIMENSIONS.flatMap((d) => [d, dimensions[d]]), + promptHash, + ], + ); +} + +// Upper bound on any prompt text handed to Gemini. Every scored character is +// billed to the operator's GEMINI_API_KEY, and without a cap one request can +// carry an arbitrarily large body. 64K chars (~16K tokens) is far above a real +// chat prompt, pasted code included; clients already fail open on non-2xx, so +// an over-cap prompt is simply sent uncoached. +export const MAX_PROMPT_CHARS = 64_000; + +export function promptTooLong(...texts: (string | undefined)[]): boolean { + return texts.some((t) => typeof t === 'string' && t.length > MAX_PROMPT_CHARS); +} + +export const PROMPT_TOO_LONG = { + error: 'prompt_too_long', + detail: `max ${MAX_PROMPT_CHARS} characters`, +} as const; diff --git a/apps/api/src/routes/assist.ts b/apps/api/src/routes/assist.ts new file mode 100644 index 0000000..063221a --- /dev/null +++ b/apps/api/src/routes/assist.ts @@ -0,0 +1,183 @@ +// LLM-assisted rewrites: POST /diff (compare to a team prompt) and POST /improve. +// Mounted by app.ts, which applies auth, body caps and rate limits first. +import { Hono } from 'hono'; +import type { + DiffRequest, + DiffResponse, + DimensionScores, + ImproveRequest, + ImproveResponse, + ImproveTurn, +} from '@trailhead/shared'; +import { ancestorPaths } from '@trailhead/scoring'; +import { q } from '../db.ts'; +import { extractTopic, improveCoach, overallScore, scorePrompt, synthesizeDiff } from '../gemini.ts'; +import { renderTeamContext } from '../team-context.ts'; +import { PROMPT_TOO_LONG, promptTooLong, type AppEnv } from '../http.ts'; + +export const assistRoutes = new Hono(); + +// ----- POST /diff ------------------------------------------------------------ +// Find the closest graduated prompt in the same path/topic ancestry, score +// both, and have Gemma narrate the differences. Spec §10. + +assistRoutes.post('/diff', async (c) => { + const body = await c.req.json().catch(() => null); + if (!body || typeof body.user_prompt !== 'string' || typeof body.user_id !== 'string') { + return c.json({ error: 'bad_request' }, 400); + } + if (promptTooLong(body.user_prompt)) return c.json(PROMPT_TOO_LONG, 413); + + const ancestors = body.file_path ? ancestorPaths(body.file_path) : ['']; + const topic = await extractTopic(body.user_prompt); + + // Prefer same-topic + same-ancestry. Fall back to any topic in ancestry. + // Final fallback: any graduated prompt in this team. + let candidate = ( + await q<{ template: string; topic: string | null; node_path: string }>( + `SELECT p.template, p.topic, n.path AS node_path + FROM prompts p + JOIN nodes n ON n.id = p.node_id + WHERE n.team_token = $1 + AND p.status = 'graduated' + AND p.topic = $2 + AND n.path = ANY($3::text[]) + ORDER BY p.reuse_count DESC, length(n.path) DESC + LIMIT 1`, + [c.get('team_token'), topic, ancestors], + ) + )[0]; + if (!candidate) { + candidate = ( + await q<{ template: string; topic: string | null; node_path: string }>( + `SELECT p.template, p.topic, n.path AS node_path + FROM prompts p + JOIN nodes n ON n.id = p.node_id + WHERE n.team_token = $1 + AND p.status = 'graduated' + AND n.path = ANY($2::text[]) + ORDER BY p.reuse_count DESC, length(n.path) DESC + LIMIT 1`, + [c.get('team_token'), ancestors], + ) + )[0]; + } + if (!candidate) { + candidate = ( + await q<{ template: string; topic: string | null; node_path: string }>( + `SELECT p.template, p.topic, n.path AS node_path + FROM prompts p + JOIN nodes n ON n.id = p.node_id + WHERE n.team_token = $1 AND p.status = 'graduated' + ORDER BY p.reuse_count DESC + LIMIT 1`, + [c.get('team_token')], + ) + )[0]; + } + if (!candidate) { + return c.json({ error: 'no_team_prompts_available' }, 404); + } + + // Sequential, not parallel: Gemini's free tier serialises requests per + // API key in practice — concurrent Flash calls slow each other to a + // crawl. Sequential keeps total /diff latency below 10s. + const userScore = await scorePrompt({ prompt: body.user_prompt, file_path: body.file_path }); + const teamScore = await scorePrompt({ prompt: candidate.template, file_path: candidate.node_path }); + const narrative = await synthesizeDiff({ + user_prompt: body.user_prompt, + user_scores: userScore.dimensions as DimensionScores, + team_prompt: candidate.template, + team_scores: teamScore.dimensions as DimensionScores, + }); + + const res: DiffResponse = { + user: { + prompt: body.user_prompt, + overall: overallScore(userScore.dimensions), + dimensions: userScore.dimensions, + }, + team: { + prompt: candidate.template, + overall: overallScore(teamScore.dimensions), + dimensions: teamScore.dimensions, + node_path: candidate.node_path, + topic: candidate.topic, + }, + narrative, + }; + return c.json(res); +}); + +// ----- POST /improve --------------------------------------------------------- +// Gemini-driven multi-turn prompt coach. Stateless — caller carries the full +// conversation each turn. Spec: 2026-04-26-improve-widget-design.md +const IMPROVE_TURN_CAP = 5; // user replies; history.length cap is 2 * cap + +assistRoutes.post('/improve', async (c) => { + const body = await c.req.json().catch(() => null); + if ( + !body || + typeof body.original_prompt !== 'string' || + typeof body.user_id !== 'string' || + !Array.isArray(body.history) || + (body.command !== 'next' && body.command !== 'finalize') + ) { + return c.json({ error: 'bad_request' }, 400); + } + + // Validate every history entry; reject anything malformed so we never + // hand garbage to Gemini. + for (const t of body.history as ImproveTurn[]) { + if ( + !t || + (t.role !== 'assistant' && t.role !== 'user') || + typeof t.text !== 'string' + ) { + return c.json({ error: 'bad_request' }, 400); + } + } + if (promptTooLong(body.original_prompt, ...body.history.map((t) => t.text))) { + return c.json(PROMPT_TOO_LONG, 413); + } + + // Server-side cap: if the user has already replied IMPROVE_TURN_CAP times, + // force finalize regardless of the client-supplied command. The client + // also enforces this; the server check is a safety net. + const userReplies = body.history.filter((t) => t.role === 'user').length; + const command = userReplies >= IMPROVE_TURN_CAP ? 'finalize' : body.command; + + // Same context-injection pattern as /score — give Gemini the team's wiki + // subtree as system context so the coach's clarifying questions and the + // polished prompt land in the team's idiom. + const teamContext = body.context_path + ? await renderTeamContext(c.get('team_token'), body.context_path) + : null; + + try { + const out = await improveCoach({ + original_prompt: body.original_prompt, + missing: body.missing ?? {}, + history: body.history, + command, + team_context: teamContext ?? undefined, + }); + if (out.kind === 'question') { + const res: ImproveResponse = { + kind: 'question', + text: out.text, + turn: userReplies + 1, + }; + return c.json(res); + } + const res: ImproveResponse = { + kind: 'final', + polished: out.polished, + rationale: out.rationale, + }; + return c.json(res); + } catch (err) { + console.warn('[api] /improve failed', err); + return c.json({ error: 'improve_failed' }, 502); + } +}); diff --git a/apps/api/src/routes/coach.ts b/apps/api/src/routes/coach.ts new file mode 100644 index 0000000..0ef489c --- /dev/null +++ b/apps/api/src/routes/coach.ts @@ -0,0 +1,642 @@ +// POST /coach: the server-side teach → reveal coaching loop. +// Mounted by app.ts, which applies auth, body caps and rate limits first. +import { Hono } from 'hono'; +import type { + CoachMode, + CoachNextRoundInputs, + CoachRequest, + CoachResponse, + Dimension, + DimensionScores, +} from '@trailhead/shared'; +import { DIMENSIONS } from '@trailhead/shared'; +import { + ancestorPaths, + buildAugmentation, + renderSkipReveal, + renderSuccessReveal, + renderTeachBlock, +} from '@trailhead/scoring'; +import { q } from '../db.ts'; +import { degradedCoachResponse, isUnparseableScore } from '../coach-degraded.ts'; +import { + acknowledgeProgress, + overallScore, + rewriteForDims, + scorePrompt, + summarizeCoaching, +} from '../gemini.ts'; +import { tryPromotePrompt } from '../prompt-promotion.ts'; +import { + passesPromotionGate, + promotionMode, + renderLibraryBanner, + renderNotPromotedNote, +} from '../promotion-gate.ts'; +import { renderTeamContext } from '../team-context.ts'; +import { PROMPT_TOO_LONG, promptTooLong, writeSkillObservations, type AppEnv } from '../http.ts'; + +export const coachRoutes = new Hono(); + +// ----- POST /coach ----------------------------------------------------------- +// Educational coaching loop. Drives the teach→reveal cycle described in +// docs/superpowers/specs/2026-04-26-trailhead-educational-loop-design.md. +// +// Stateless: caller (the MCP server) carries round state explicitly. The +// `proceed` flag is the directive's only decision input — when false the +// caller relays `text`, gathers a user reply, and calls /coach again with +// next_round_inputs echoed back. Server enforces the round cap and bails +// on no-progress. + +const COACH_MAX_ROUNDS = 5; + +function clampRound(n: number | undefined): number { + if (typeof n !== 'number' || !Number.isFinite(n)) return 1; + return Math.max(1, Math.min(COACH_MAX_ROUNDS, Math.floor(n))); +} + +// Derive a wiki context_path from a file_path so coach scoring is grounded +// in the team's subtree without the caller having to know wiki internals. +// 'src/api/webhooks/handler.ts' → 'src/api/webhooks/' (parent folder). +// 'src/api/webhooks/' → 'src/api/webhooks/' (already a folder). +// 'README.md' → '' (no parent → no scope). +// Empty string is treated as "no scope" by renderTeamContext. +function deriveContextPath(filePath: string): string { + const idx = filePath.lastIndexOf('/'); + return idx < 0 ? '' : filePath.slice(0, idx + 1); +} + +// Pick the lowest-scoring dimension under the threshold (default 7). Stable +// against tied scores by walking DIMENSIONS in declaration order — same +// prompt always teaches the same dim. +function lowestDimBelow(dims: DimensionScores, threshold: number = 7): Dimension | null { + let pick: Dimension | null = null; + let pickScore = Infinity; + for (const d of DIMENSIONS) { + const s = dims[d]; + if (s < threshold && s < pickScore) { + pick = d; + pickScore = s; + } + } + return pick; +} + +// Wiki-first lookup: top graduated prompt in the file_path's ancestor nodes. +// Prefers prompts authored by SOMEONE OTHER than `userId` so the user isn't +// shown their own prompt back as the strong example. Self-authored rows are +// still returned when no other-authored alternative exists — keeps the +// single-user demo posture working. +async function fetchTopGraduatedForPath( + teamToken: string, + filePath: string, + userId: string, +): Promise { + const ancestors = ancestorPaths(filePath); + if (ancestors.length === 0) return null; + const rows = await q<{ template: string }>( + `SELECT p.template + FROM prompts p + JOIN nodes n ON n.id = p.node_id + WHERE n.team_token = $1 + AND n.path = ANY($2::text[]) + AND p.status = 'graduated' + ORDER BY (p.author_user_id IS NULL OR p.author_user_id <> $3) DESC, + p.reuse_count DESC, + length(n.path) DESC + LIMIT 1`, + [teamToken, ancestors, userId], + ); + return rows.length ? rows[0]!.template : null; +} + +// Wiki-first lookup: top graduated prompt across the team. Used when no +// file_path is available. Same self-author preference as the path variant. +async function fetchTopGraduatedForTeam(teamToken: string, userId: string): Promise { + const rows = await q<{ template: string }>( + `SELECT p.template + FROM prompts p + JOIN nodes n ON n.id = p.node_id + WHERE n.team_token = $1 + AND p.status = 'graduated' + ORDER BY (p.author_user_id IS NULL OR p.author_user_id <> $2) DESC, + p.reuse_count DESC + LIMIT 1`, + [teamToken, userId], + ); + return rows.length ? rows[0]!.template : null; +} + +// Wiki-first → Gemini fallback. Hackathon simplification: we don't re-score +// graduated prompts to filter for `target_dims`; the assumption is that a +// graduated team prompt is already strong on most dims and seeing it +// teaches the user something either way. If the wiki has nothing, fall +// back to a Gemini rewrite that explicitly targets the named dimensions. +// +// Returns the example string AND the optional tip — the wiki path has no +// tip (we just have the template), the Gemini fallback emits one alongside +// the rewrite. Callers showing a single-dim teach block render the tip; +// multi-dim callers (skip / no-progress) ignore it because one tip can't +// honestly summarize several principles at once. +async function getStrongExample(args: { + teamToken: string; + userId: string; + prompt: string; + file_path?: string; + target_dims: Dimension[]; + team_context: string | null; +}): Promise<{ example: string; tip: string; fromTeam: boolean }> { + const wiki = args.file_path + ? await fetchTopGraduatedForPath(args.teamToken, args.file_path, args.userId) + : await fetchTopGraduatedForTeam(args.teamToken, args.userId); + // fromTeam: a teammate's text — the renderers fence it as untrusted. + if (wiki) return { example: wiki, tip: '', fromTeam: true }; + + const fallback = await rewriteForDims({ + prompt: args.prompt, + target_dims: args.target_dims, + file_path: args.file_path, + team_context: args.team_context ?? undefined, + }); + // Empty strings on Gemini failure — render block falls back accordingly. + return { example: fallback.rewritten_prompt, tip: fallback.tip, fromTeam: false }; +} + +// Library promotion for a /coach prompt that needs no (more) coaching. +// Checks the gate (promotion-gate.ts: unrounded mean >= 7, no dimension < 5) +// synchronously and, if it passes, schedules the background promotion, which +// re-scores independently before inserting. Returns the line for `text` — +// it ships in `text` so a forgetful host LLM can't drop it, and it never +// claims more than has happened. +function promoteIfEligible(args: { + teamToken: string; + userId: string; + prompt: string; + filePath: string | null; + dimensions: DimensionScores; + overall: number; +}): string { + const gate = passesPromotionGate(args.dimensions); + if (!gate.ok) return renderNotPromotedNote(args.overall, gate); + const mode = promotionMode(); + // Fire-and-forget: off the response path, fail-open inside + // tryPromotePrompt. MCP-only by call site (only /coach promotes). + setImmediate(() => { + void tryPromotePrompt({ ...args, mode }); + }); + return renderLibraryBanner(args.overall, mode); +} + +// Round-state token. Compresses the four `next_round_inputs` fields into a +// single opaque base64url JSON blob. Round 2+ callers can echo only the +// token instead of all four fields — fewer slots for an LLM to drop. Both +// shapes are accepted on input; the token wins when both are present. +type RoundState = { + original_prompt: string; + original_dimensions: DimensionScores; + previous_dimensions: DimensionScores; + round: number; +}; +function encodeRoundToken(state: RoundState): string { + return Buffer.from(JSON.stringify(state), 'utf8').toString('base64url'); +} +function decodeRoundToken(token: string): RoundState | null { + try { + const json = Buffer.from(token, 'base64url').toString('utf8'); + const p = JSON.parse(json) as Partial; + if ( + typeof p.original_prompt === 'string' && + typeof p.round === 'number' && + p.original_dimensions && typeof p.original_dimensions === 'object' && + p.previous_dimensions && typeof p.previous_dimensions === 'object' + ) { + return p as RoundState; + } + return null; + } catch { + return null; + } +} + +coachRoutes.post('/coach', async (c) => { + const body = await c.req.json().catch(() => null); + if (!body || typeof body.prompt !== 'string' || typeof body.user_id !== 'string') { + return c.json({ error: 'bad_request' }, 400); + } + if (promptTooLong(body.prompt, body.original_prompt)) return c.json(PROMPT_TOO_LONG, 413); + + // Round-token shorthand. When present, decode and use as authoritative + // round state — overrides any individual field the caller also sent. + // Invalid tokens fall through to the four-field path with a warning. + if (body.round_token) { + const decoded = decodeRoundToken(body.round_token); + if (decoded) { + body.original_prompt = decoded.original_prompt; + body.original_dimensions = decoded.original_dimensions; + body.previous_dimensions = decoded.previous_dimensions; + body.round = decoded.round; + } else { + console.warn('[coach] received invalid round_token; falling back to explicit fields'); + } + } + + const mode: CoachMode = body.mode === 'augment' || body.mode === 'skip_reveal' + ? body.mode + : 'score'; + + // Same team-context pattern as /score and /improve. When the caller does + // not pass an explicit context_path, derive one from file_path so the + // teach prompts and Gemini's scoring see the team's subtree automatically. + // The MCP coach tool only forwards file_path, so this is what makes coach + // wiki-aware in practice. + const teamToken = c.get('team_token'); + const contextPath = + body.context_path ?? (body.file_path ? deriveContextPath(body.file_path) : ''); + const teamContext = contextPath + ? await renderTeamContext(teamToken, contextPath) + : null; + + // 1. Score (always). Failure is fail-open: hand the LLM a "no coaching + // this turn" signal and let it produce its answer with the original + // prompt. Spec §7. + let scoreResult: { dimensions: DimensionScores; missing: Record }; + try { + const result = await scorePrompt({ + prompt: body.prompt, + file_path: body.file_path, + team_context: teamContext ?? undefined, + }); + scoreResult = { dimensions: result.dimensions, missing: result.missing as Record }; + } catch (err) { + // Fail-open on `proceed`, but NEVER fail silent. A coaching outage must + // not block the user's real work, so proceed stays true — but the caller + // is told plainly that this turn was not coached, and why. Returning + // text:'' here (the old behaviour) made an outage look identical to a + // perfect prompt, so the MCP tool ran indefinitely without ever coaching. + console.error('[api] /coach scorePrompt failed', err); + const zeros = Object.fromEntries(DIMENSIONS.map((d) => [d, 0])) as DimensionScores; + return c.json(degradedCoachResponse(mode, zeros, 'score_failed', err)); + } + const overall = overallScore(scoreResult.dimensions); + + // Fail-open: scorePrompt does NOT throw on Gemini parse failures — it + // silently returns zeros + empty missing (see gemini.ts coerceScore). + // That signal is indistinguishable from a real all-zero score except by + // the empty `missing` object: a real-zero score from Gemini populates + // hints for the dims < 5. When we detect the zero+empty fingerprint, + // treat it as "Gemini failed, no coaching this turn" rather than + // pretending the user wrote a perfectly empty prompt. Spec §7. + if (isUnparseableScore(scoreResult.dimensions, scoreResult.missing)) { + console.error('[api] /coach scorePrompt returned unparseable output (zero+empty fingerprint)'); + return c.json( + degradedCoachResponse(mode, scoreResult.dimensions, 'score_unparseable'), + ); + } + + // 2. Skill_observation writes (same dedup as /score). + await writeSkillObservations(teamToken, body.user_id, body.prompt, scoreResult.dimensions); + + // 3. Branch on mode. + + // ---- Augment mode (legacy passthrough) ----------------------------------- + if (mode === 'augment') { + const augmented = buildAugmentation({ + original: body.prompt, + missing: scoreResult.missing, + }); + const res: CoachResponse = { + proceed: true, + mode: 'augment', + overall, + dimensions: scoreResult.dimensions, + missing: scoreResult.missing, + text: '', + augmented_prompt: augmented, + missing_dims: Object.keys(scoreResult.missing), + }; + return c.json(res); + } + + // ---- Skip reveal mode ---------------------------------------------------- + if (mode === 'skip_reveal') { + const originalDims = body.original_dimensions ?? scoreResult.dimensions; + // Dimensions we'd want to lift on the rewrite. Spec §5 picks dims that + // scored below 5; if the original is already above that bar, fall back + // to dims below 7 so the rewrite still has direction. + let dimsToImprove = DIMENSIONS.filter((d) => originalDims[d] < 5); + if (dimsToImprove.length === 0) { + dimsToImprove = DIMENSIONS.filter((d) => originalDims[d] < 7); + } + const originalPrompt = body.original_prompt ?? body.prompt; + // Run the rewrite and the closing summary in parallel — both are + // independent Gemini calls and the user is already waiting on the + // skip-reveal text. Each fails open to '' so a partial outage still + // produces a useful (if shorter) reveal. + const [strong, summary] = await Promise.all([ + getStrongExample({ + teamToken, + userId: body.user_id, + prompt: originalPrompt, + file_path: body.file_path, + target_dims: dimsToImprove.length ? dimsToImprove : ['specificity'], + team_context: teamContext, + }), + summarizeCoaching({ + original_prompt: originalPrompt, + final_prompt: body.prompt, + original_dimensions: originalDims, + final_dimensions: scoreResult.dimensions, + reason: 'skip', + }), + ]); + const text = strong.example + ? renderSkipReveal({ + strongRewrite: strong.example, + strongRewriteFromTeam: strong.fromTeam, + originalDimensions: originalDims, + reason: 'skip', + summary, + overall: overallScore(originalDims), + }) + : ''; + const res: CoachResponse = { + proceed: true, + mode: 'skip_reveal', + overall, + dimensions: scoreResult.dimensions, + missing: scoreResult.missing, + text, + }; + return c.json(res); + } + + // ---- Score mode (the main loop) ------------------------------------------ + const round = clampRound(body.round); + const isRound1 = round === 1 || !body.original_prompt; + const lowest = lowestDimBelow(scoreResult.dimensions, 7); + + // Round 1, score >=7 → silent fast path. Power users see no friction. + if (isRound1 && overall >= 7) { + const res: CoachResponse = { + proceed: true, + mode: 'score', + overall, + dimensions: scoreResult.dimensions, + missing: scoreResult.missing, + text: promoteIfEligible({ + teamToken, + userId: body.user_id, + prompt: body.prompt, + filePath: body.file_path ?? null, + dimensions: scoreResult.dimensions, + overall, + }), + }; + return c.json(res); + } + + // Round 1, score <7 → first teach block. + if (isRound1 && lowest) { + const strong = await getStrongExample({ + teamToken, + userId: body.user_id, + prompt: body.prompt, + file_path: body.file_path, + target_dims: [lowest], + team_context: teamContext, + }); + const text = renderTeachBlock({ + targetDim: lowest, + targetScore: scoreResult.dimensions[lowest], + strongExample: strong.example, + strongExampleFromTeam: strong.fromTeam, + tip: strong.tip, + dimensions: scoreResult.dimensions, + overall, + }); + const nextState: RoundState = { + original_prompt: body.prompt, + original_dimensions: scoreResult.dimensions, + previous_dimensions: scoreResult.dimensions, + round: 2, + }; + const next: CoachNextRoundInputs = { + ...nextState, + round_token: encodeRoundToken(nextState), + }; + const res: CoachResponse = { + proceed: false, + mode: 'score', + overall, + dimensions: scoreResult.dimensions, + missing: scoreResult.missing, + text, + next_round_inputs: next, + }; + return c.json(res); + } + + // Round >=2 paths. Need original_prompt (we treated round 1 already). + const originalPrompt = body.original_prompt!; + const originalDims = body.original_dimensions ?? scoreResult.dimensions; + const previousDims = body.previous_dimensions ?? originalDims; + const previousOverall = overallScore(previousDims); + const previousLowest = lowestDimBelow(previousDims, 7); + const originalOverall = overallScore(originalDims); + + // Score crossed 7 → success reveal. + if (overall >= 7) { + const summary = await summarizeCoaching({ + original_prompt: originalPrompt, + final_prompt: body.prompt, + original_dimensions: originalDims, + final_dimensions: scoreResult.dimensions, + reason: 'success', + }); + const text = + renderSuccessReveal({ + originalPrompt, + finalPrompt: body.prompt, + originalOverall, + finalOverall: overall, + originalDimensions: originalDims, + finalDimensions: scoreResult.dimensions, + summary, + }) + + // The user iterated through coaching and landed a >=7 prompt — the + // final form is the promotion candidate. + `\n\n${promoteIfEligible({ + teamToken, + userId: body.user_id, + prompt: body.prompt, + filePath: body.file_path ?? null, + dimensions: scoreResult.dimensions, + overall, + })}`; + const res: CoachResponse = { + proceed: true, + mode: 'score', + overall, + dimensions: scoreResult.dimensions, + missing: scoreResult.missing, + text, + }; + return c.json(res); + } + + // No-progress detection: the dim we were teaching about (= last round's + // lowest) did NOT improve, AND overall did not improve. We test the + // previously-targeted dim directly rather than checking `lowest` equality, + // because Gemini's tiebreakers can shuffle which 0-scored dim is "lowest" + // between rounds even when nothing material changed (this was the original + // failing case from the design spec — "fix the retry. it needs to be more + // accurate" leaves context_loading at 0 but the lowest tiebreaker drifts). + const targetDimDidNotImprove = !!( + previousLowest && + scoreResult.dimensions[previousLowest] <= previousDims[previousLowest] + ); + if (targetDimDidNotImprove && overall <= previousOverall) { + let dimsToImprove = DIMENSIONS.filter((d) => originalDims[d] < 5); + if (dimsToImprove.length === 0) { + dimsToImprove = DIMENSIONS.filter((d) => originalDims[d] < 7); + } + // Parallel: rewrite + closing recap. Same fail-open posture as the + // skip-reveal branch — both helpers return '' on Gemini failure and the + // render fallback handles each independently. + const [strong, summary] = await Promise.all([ + getStrongExample({ + teamToken, + userId: body.user_id, + prompt: originalPrompt, + file_path: body.file_path, + target_dims: dimsToImprove.length ? dimsToImprove : [lowest!], + team_context: teamContext, + }), + summarizeCoaching({ + original_prompt: originalPrompt, + final_prompt: body.prompt, + original_dimensions: originalDims, + final_dimensions: scoreResult.dimensions, + reason: 'no_progress', + }), + ]); + const text = strong.example + ? renderSkipReveal({ + strongRewrite: strong.example, + strongRewriteFromTeam: strong.fromTeam, + originalDimensions: originalDims, + reason: 'no_progress', + noProgressDim: previousLowest ?? undefined, + summary, + overall: overallScore(originalDims), + }) + : ''; + const res: CoachResponse = { + proceed: true, + mode: 'score', + overall, + dimensions: scoreResult.dimensions, + missing: scoreResult.missing, + text, + }; + return c.json(res); + } + + // Forced exit at COACH_MAX_ROUNDS (still <7, made progress, but rounds exhausted). + if (round >= COACH_MAX_ROUNDS) { + const summary = await summarizeCoaching({ + original_prompt: originalPrompt, + final_prompt: body.prompt, + original_dimensions: originalDims, + final_dimensions: scoreResult.dimensions, + reason: 'max_rounds', + }); + const text = renderSuccessReveal({ + originalPrompt, + finalPrompt: body.prompt, + originalOverall, + finalOverall: overall, + originalDimensions: originalDims, + finalDimensions: scoreResult.dimensions, + maxRoundsHit: true, + summary, + }); + const res: CoachResponse = { + proceed: true, + mode: 'score', + overall, + dimensions: scoreResult.dimensions, + missing: scoreResult.missing, + text, + }; + return c.json(res); + } + + // Else: score still <7, made progress, more rounds remain. Keep teaching. + if (lowest) { + // Parallel: pull a fresh strong example AND ask Gemini to acknowledge + // what the user just added. Both feed renderTeachBlock; both fail-open + // to '' so the static template still produces a usable block. + const [strong, acknowledgment] = await Promise.all([ + getStrongExample({ + teamToken, + userId: body.user_id, + prompt: body.prompt, + file_path: body.file_path, + target_dims: [lowest], + team_context: teamContext, + }), + acknowledgeProgress({ + previous_prompt: originalPrompt, + current_prompt: body.prompt, + previous_dimensions: previousDims, + current_dimensions: scoreResult.dimensions, + }), + ]); + const text = renderTeachBlock({ + targetDim: lowest, + targetScore: scoreResult.dimensions[lowest], + strongExample: strong.example, + strongExampleFromTeam: strong.fromTeam, + previousLowestDim: + previousLowest && previousLowest !== lowest ? previousLowest : undefined, + acknowledgment, + tip: strong.tip, + dimensions: scoreResult.dimensions, + overall, + }); + const nextState: RoundState = { + original_prompt: originalPrompt, + original_dimensions: originalDims, + previous_dimensions: scoreResult.dimensions, + round: round + 1, + }; + const next: CoachNextRoundInputs = { + ...nextState, + round_token: encodeRoundToken(nextState), + }; + const res: CoachResponse = { + proceed: false, + mode: 'score', + overall, + dimensions: scoreResult.dimensions, + missing: scoreResult.missing, + text, + next_round_inputs: next, + }; + return c.json(res); + } + + // Defensive fallback — shouldn't be reachable (overall < 7 implies a + // lowest dim exists). If we land here anyway, exit silently rather than + // 500ing the loop. + const res: CoachResponse = { + proceed: true, + mode: 'score', + overall, + dimensions: scoreResult.dimensions, + missing: scoreResult.missing, + text: '', + }; + return c.json(res); +}); diff --git a/apps/api/src/routes/metrics.ts b/apps/api/src/routes/metrics.ts new file mode 100644 index 0000000..2cfebef --- /dev/null +++ b/apps/api/src/routes/metrics.ts @@ -0,0 +1,111 @@ +// Read-only dashboards: GET /skill-arc and GET /team/metrics. +// Mounted by app.ts, which applies auth, body caps and rate limits first. +import { Hono } from 'hono'; +import type { + Dimension, + SkillArcObservation, + SkillArcResponse, + TeamMetricsResponse, +} from '@trailhead/shared'; +import { q } from '../db.ts'; +import { intParam } from '../request-params.ts'; +import type { AppEnv } from '../http.ts'; + +export const metricsRoutes = new Hono(); + +// ----- GET /skill-arc?user_id=&since=ISO ------------------------------------- +// Time-series of per-dimension scores for the dashboard hero chart. Drives +// the §13 close beat (live tick during demo). Defaults: any user, last 7 +// days. Capped at 5000 rows to keep the chart responsive. + +metricsRoutes.get('/skill-arc', async (c) => { + const userIdParam = c.req.query('user_id'); + const sinceParam = c.req.query('since'); + const since = sinceParam ? new Date(sinceParam) : new Date(Date.now() - 7 * 24 * 60 * 60 * 1000); + if (Number.isNaN(since.getTime())) return c.json({ error: 'bad_since' }, 400); + const limit = intParam(c.req.query('limit'), 1000, 1, 5000); + + const rows = userIdParam + ? await q<{ dimension: Dimension; score: number; ts: Date }>( + `SELECT dimension, score, ts + FROM skill_observations + WHERE team_token = $1 AND user_id = $2 AND ts > $3 + ORDER BY ts ASC + LIMIT $4`, + [c.get('team_token'), userIdParam, since.toISOString(), limit], + ) + : await q<{ dimension: Dimension; score: number; ts: Date }>( + `SELECT dimension, score, ts + FROM skill_observations + WHERE team_token = $1 AND ts > $2 + ORDER BY ts ASC + LIMIT $3`, + [c.get('team_token'), since.toISOString(), limit], + ); + + const res: SkillArcResponse = { + observations: rows.map((r): SkillArcObservation => ({ + dimension: r.dimension, + score: r.score, + ts: r.ts.toISOString(), + })), + }; + return c.json(res); +}); + +// ----- GET /team/metrics ----------------------------------------------------- +// Snapshot for the dashboard /team page. All cheap aggregate counts; no +// time-series. Reuse rate is captures with outcome='helpful' over total +// captures (proxy for "team's prompts work" until we have the real +// graduated-prompt-match metric). + +metricsRoutes.get('/team/metrics', async (c) => { + const sevenDaysAgo = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000).toISOString(); + + const [obs, learnings, captures, users] = await Promise.all([ + q<{ avg_overall: number | null; total_obs: number }>( + `SELECT AVG(score)::float AS avg_overall, COUNT(*)::int AS total_obs + FROM skill_observations + WHERE team_token = $1 AND ts > $2`, + [c.get('team_token'), sevenDaysAgo], + ), + q<{ durable_count: number; draft_count: number }>( + `SELECT + COUNT(*) FILTER (WHERE l.status = 'durable')::int AS durable_count, + COUNT(*) FILTER (WHERE l.status = 'draft')::int AS draft_count + FROM learnings l + JOIN nodes n ON n.id = l.node_id + WHERE n.team_token = $1`, + [c.get('team_token')], + ), + q<{ total: number; helpful: number }>( + `SELECT + COUNT(*)::int AS total, + COUNT(*) FILTER (WHERE outcome = 'helpful')::int AS helpful + FROM captures + WHERE team_token = $1 AND created_at > $2`, + [c.get('team_token'), sevenDaysAgo], + ), + q<{ active_users: number }>( + `SELECT COUNT(DISTINCT user_id)::int AS active_users + FROM skill_observations + WHERE team_token = $1 AND ts > $2`, + [c.get('team_token'), sevenDaysAgo], + ), + ]); + + const obsRow = obs[0]!; + const learningsRow = learnings[0]!; + const capturesRow = captures[0]!; + const usersRow = users[0]!; + + const res: TeamMetricsResponse = { + avg_overall: obsRow.avg_overall ? Math.round(obsRow.avg_overall * 10) / 10 : 0, + reuse_rate: capturesRow.total > 0 ? capturesRow.helpful / capturesRow.total : 0, + durable_count: learningsRow.durable_count, + draft_count: learningsRow.draft_count, + total_obs: obsRow.total_obs, + active_users: usersRow.active_users, + }; + return c.json(res); +}); diff --git a/apps/api/src/routes/onboard.ts b/apps/api/src/routes/onboard.ts new file mode 100644 index 0000000..36c5731 --- /dev/null +++ b/apps/api/src/routes/onboard.ts @@ -0,0 +1,288 @@ +// Wiki bootstrap: POST /onboard/repo (skeleton), /onboard/repo/full (async rich job) and job status. +// Mounted by app.ts, which applies auth, body caps and rate limits first. +import { Hono } from 'hono'; +import type { + OnboardRepoFullRequest, + OnboardRepoFullResponse, + OnboardRepoRequest, + OnboardRepoResponse, + WikiJobPathKind, + WikiJobPathStatus, + WikiJobStatusResponse, +} from '@trailhead/shared'; +import { normalizePath } from '@trailhead/scoring'; +import { applyTeamNameIfPlaceholder, q } from '../db.ts'; +import { isUuid } from '../request-params.ts'; +import { invalidateTeamContext } from '../team-context.ts'; +import { bundleFromRequest, runJob } from '../wiki-bootstrap-job.ts'; +import type { AppEnv } from '../http.ts'; + +export const onboardRoutes = new Hono(); + +// ----- POST /onboard/repo ---------------------------------------------------- +// Bootstrap a team wiki by upserting one node per path. Idempotent: re-running +// with the same paths is a no-op (the existing node row is left untouched). +// `initial_rules[path]` lets the caller seed `body_md` for any/all of the +// supplied paths — useful when the caller has, say, scanned a repo's existing +// CLAUDE.md or copied conventions from another tool. +// +// Spec ref: +// docs/superpowers/specs/2026-04-25-demo-completion-design.md §C.1 +// docs/superpowers/specs/2026-04-25-mcp-plugin-ux-design.md (bootstrap UX +// follow-up — wired through wiki_bootstrap MCP tool + `trailhead-mcp +// bootstrap` CLI subcommand). + +const ONBOARD_MAX_PATHS = 200; + +onboardRoutes.post('/onboard/repo', async (c) => { + const body = await c.req.json().catch(() => null); + if (!body || !Array.isArray(body.paths)) { + return c.json({ error: 'bad_request', detail: 'paths: string[] required' }, 400); + } + if (body.paths.length === 0) { + return c.json({ error: 'bad_request', detail: 'paths must not be empty' }, 400); + } + if (body.paths.length > ONBOARD_MAX_PATHS) { + return c.json( + { error: 'too_many_paths', detail: `max ${ONBOARD_MAX_PATHS} paths per request` }, + 400, + ); + } + const initialRules = + body.initial_rules && typeof body.initial_rules === 'object' && !Array.isArray(body.initial_rules) + ? body.initial_rules + : {}; + + // Normalize + dedupe paths so we don't issue duplicate inserts inside one + // request (the UNIQUE constraint would catch it but the per-row upsert + // round-trip is wasted). + const seen = new Set(); + const normalized: { raw: string; path: string }[] = []; + for (const raw of body.paths) { + if (typeof raw !== 'string') continue; + const path = normalizePath(raw); + if (!path) continue; // empty string after normalization — skip + if (seen.has(path)) continue; + seen.add(path); + normalized.push({ raw, path }); + } + + if (normalized.length === 0) { + return c.json({ error: 'bad_request', detail: 'no valid paths after normalization' }, 400); + } + + if (typeof body.team_name === 'string' && body.team_name.trim()) { + await applyTeamNameIfPlaceholder(c.get('team_token'), body.team_name); + } + + const nodes: { path: string; id: string }[] = []; + let nodes_created = 0; + + for (const { raw, path } of normalized) { + // body_md from initial_rules — match against either the normalized form + // or the caller's raw string so callers don't need to pre-normalize keys. + const seedBody = + typeof initialRules[path] === 'string' + ? initialRules[path] + : typeof initialRules[raw] === 'string' + ? initialRules[raw] + : ''; + + // xmax = 0 in the RETURNING row means the tuple was newly inserted (PG + // marks it 0 on fresh inserts; ON CONFLICT updates set xmax to the + // current xid). Lets us count creates without a second query. + const rows = await q<{ id: string; inserted: boolean }>( + `INSERT INTO nodes (team_token, path, body_md) + VALUES ($1, $2, $3) + ON CONFLICT (team_token, path) DO UPDATE + SET body_md = CASE + WHEN $3 <> '' AND nodes.body_md = '' THEN $3 + ELSE nodes.body_md + END, + updated_at = NOW() + RETURNING id, (xmax = 0) AS inserted`, + [c.get('team_token'), path, seedBody], + ); + const row = rows[0]!; + if (row.inserted) nodes_created += 1; + nodes.push({ path, id: row.id }); + } + + invalidateTeamContext(c.get('team_token')); + const res: OnboardRepoResponse = { nodes_created, nodes }; + return c.json(res); +}); + +// ----- POST /onboard/repo/full ----------------------------------------------- +// Rich (LLM-generated) bootstrap. Accepts the discovered folder paths plus +// the file contents (already capped client-side) plus optional manifest +// snippets and CLAUDE.md seed text. Creates a wiki_jobs row + one +// wiki_job_paths row per node and kicks off the worker via setImmediate. +// Returns the job_id immediately; the worker fills body_md asynchronously. +// +// Spec: docs/superpowers/specs/2026-04-26-wiki-bootstrap-rich-design.md §9 + +// Server-side hard ceilings. Independent of the client's CLI flags so a +// rogue client can't blow the API host's RAM. Conservative — these are +// "abuse cap" not "expected size". +const ONBOARD_FULL_MAX_FOLDERS = 1_000; +const ONBOARD_FULL_MAX_FILES = 2_000; +const ONBOARD_FULL_MAX_FILE_CHARS = 32_000; // per file +const ONBOARD_FULL_MAX_BUNDLE_BYTES = 16 * 1024 * 1024; // 16 MB + +onboardRoutes.post('/onboard/repo/full', async (c) => { + const body = await c.req.json().catch(() => null); + if (!body || !Array.isArray(body.folders) || !Array.isArray(body.files)) { + return c.json( + { error: 'bad_request', detail: 'folders: string[] and files: {path,content}[] required' }, + 400, + ); + } + if (body.folders.length > ONBOARD_FULL_MAX_FOLDERS) { + return c.json({ error: 'too_many_folders', detail: `max ${ONBOARD_FULL_MAX_FOLDERS}` }, 400); + } + if (body.files.length > ONBOARD_FULL_MAX_FILES) { + return c.json({ error: 'too_many_files', detail: `max ${ONBOARD_FULL_MAX_FILES}` }, 400); + } + let bundleBytes = 0; + for (const f of body.files) { + if (typeof f?.path !== 'string' || typeof f?.content !== 'string') { + return c.json({ error: 'bad_request', detail: 'each file requires path:string and content:string' }, 400); + } + if (f.content.length > ONBOARD_FULL_MAX_FILE_CHARS) { + return c.json( + { error: 'file_too_large', detail: `${f.path}: max ${ONBOARD_FULL_MAX_FILE_CHARS} chars per file` }, + 400, + ); + } + bundleBytes += Buffer.byteLength(f.content, 'utf8'); + if (bundleBytes > ONBOARD_FULL_MAX_BUNDLE_BYTES) { + return c.json( + { error: 'bundle_too_large', detail: `max ${ONBOARD_FULL_MAX_BUNDLE_BYTES / (1024 * 1024)} MB` }, + 400, + ); + } + } + + const teamToken = c.get('team_token'); + + if (typeof body.team_name === 'string' && body.team_name.trim()) { + await applyTeamNameIfPlaceholder(teamToken, body.team_name); + } + + // Normalize folder paths (trailing slash) and dedupe. + const folderSet = new Set(); + for (const raw of body.folders) { + const p = normalizePath(raw); + if (p) folderSet.add(p); + } + const folders = [...folderSet]; + // De-dupe files on path; preserve first occurrence. + const seenFiles = new Set(); + const files = body.files.filter((f) => { + const p = String(f.path).trim(); + if (!p || p.endsWith('/') || seenFiles.has(p)) return false; + seenFiles.add(p); + return true; + }); + + // paths_total = folders + files + 1 root pass. + const pathsTotal = folders.length + files.length + 1; + + // Insert job header and per-path rows in one transaction so a partial + // failure doesn't leave a job with no work items. + const jobRows = await q<{ id: string }>( + `INSERT INTO wiki_jobs (team_token, paths_total) VALUES ($1, $2) RETURNING id`, + [teamToken, pathsTotal], + ); + const jobId = jobRows[0]!.id; + + // Build wiki_job_paths rows. Use a single multi-row insert for speed. + const pathRows: Array<[string, string, WikiJobPathKind]> = [ + [jobId, '', 'root'], + ...folders.map((p): [string, string, WikiJobPathKind] => [jobId, p, 'folder']), + ...files.map((f): [string, string, WikiJobPathKind] => [jobId, f.path, 'file']), + ]; + // Pg parameter array unrolling — keep it simple with one INSERT per row; + // the volume is low enough (typically 100-700 rows) that batching isn't + // critical, and the simpler code is harder to get wrong. + for (const [job, p, kind] of pathRows) { + await q( + `INSERT INTO wiki_job_paths (job_id, path, kind) VALUES ($1, $2, $3) + ON CONFLICT (job_id, path) DO NOTHING`, + [job, p, kind], + ); + } + + // Kick off the worker. setImmediate keeps it strictly fire-and-forget — + // the response returns now; runJob handles its own errors and never + // throws to here. + const bundle = bundleFromRequest({ ...body, folders, files }); + setImmediate(() => { + runJob(jobId, teamToken, bundle).catch((e) => { + console.error(`[wiki-job ${jobId}] uncaught:`, e); + }); + }); + + const res: OnboardRepoFullResponse = { job_id: jobId, paths_total: pathsTotal }; + return c.json(res); +}); + +// ----- GET /onboard/jobs/:id ------------------------------------------------- +// Status snapshot for a rich-bootstrap job. Clients (CLI, MCP tool) poll +// this every 2s. Returns the job header counters plus per-path rows so the +// UI can render which path is processing / which failed. +// +// Cross-team safety: the auth middleware sets team_token from the X-Team-Token +// header; the WHERE clause filters on it. A team can only see its own jobs +// (otherwise a leaked job_id would be a tenancy break). + +onboardRoutes.get('/onboard/jobs/:id', async (c) => { + const id = c.req.param('id'); + if (!id || !isUuid(id)) { + return c.json({ error: 'bad_request', detail: 'invalid job id' }, 400); + } + const teamToken = c.get('team_token'); + + const headers = await q<{ + id: string; + status: 'pending' | 'running' | 'done' | 'failed'; + paths_total: number; + paths_done: number; + paths_failed: number; + started_at: Date | null; + finished_at: Date | null; + error: string | null; + }>( + `SELECT id, status, paths_total, paths_done, paths_failed, started_at, finished_at, error + FROM wiki_jobs WHERE team_token = $1 AND id = $2`, + [teamToken, id], + ); + if (headers.length === 0) return c.json({ error: 'not_found' }, 404); + const h = headers[0]!; + + const pathRows = await q<{ + path: string; kind: WikiJobPathKind; status: WikiJobPathStatus['status']; error: string | null; + }>( + `SELECT path, kind, status, error FROM wiki_job_paths WHERE job_id = $1 ORDER BY kind, path`, + [id], + ); + + const res: WikiJobStatusResponse = { + job_id: h.id, + status: h.status, + paths_total: h.paths_total, + paths_done: h.paths_done, + paths_failed: h.paths_failed, + started_at: h.started_at ? h.started_at.toISOString() : null, + finished_at: h.finished_at ? h.finished_at.toISOString() : null, + error: h.error, + paths: pathRows.map((r) => ({ + path: r.path, + kind: r.kind, + status: r.status, + ...(r.error ? { error: r.error } : {}), + })), + }; + return c.json(res); +}); diff --git a/apps/api/src/routes/prompts.ts b/apps/api/src/routes/prompts.ts new file mode 100644 index 0000000..ea6e86a --- /dev/null +++ b/apps/api/src/routes/prompts.ts @@ -0,0 +1,122 @@ +// The team's prompt library: graduated prompts and the review queue. +// Mounted by app.ts, which applies auth, body caps and rate limits first. +import { Hono } from 'hono'; +import type { ProvenPromptItem, ProvenPromptsResponse } from '@trailhead/shared'; +import { ancestorPaths } from '@trailhead/scoring'; +import { q } from '../db.ts'; +import { intParam, isUuid } from '../request-params.ts'; +import type { AppEnv } from '../http.ts'; + +export const promptRoutes = new Hono(); + +// ----- GET /prompts/proven --------------------------------------------------- +// All graduated prompts for the team, optionally filtered by min score, an +// ancestor path, or topic. Powers the wiki_proven_prompts MCP tool. +// +// "Proven" == status='graduated'. Today every graduated prompt is by +// definition overall>=7 (the gate in /coach), and the actual score is now +// stored on graduated_overall_score so callers can filter ≥8 / ≥9 too. +// +// Ranking: score DESC, reuse_count DESC, created_at DESC. Score is the +// primary signal because reuse_count starts at 0 and grows over time — +// without the score tiebreaker, brand-new 10/10 prompts would rank below +// older 7/10 prompts that happened to be re-graduated once or twice. +promptRoutes.get('/prompts/proven', async (c) => { + const minScore = intParam(c.req.query('min_score'), 7, 0, 10); + const limit = intParam(c.req.query('limit'), 20, 1, 100); + const pathScope = c.req.query('path'); + const topic = c.req.query('topic'); + const ancestors = pathScope ? ancestorPaths(pathScope) : null; + + const rows = await q<{ + id: string; + template: string; + topic: string | null; + reuse_count: number; + graduated_overall_score: number; + author_user_id: string | null; + node_path: string; + created_at: Date; + }>( + `SELECT p.id, p.template, p.topic, p.reuse_count, + p.graduated_overall_score, p.author_user_id, + n.path AS node_path, p.created_at + FROM prompts p + JOIN nodes n ON n.id = p.node_id + WHERE n.team_token = $1 + AND p.status = 'graduated' + AND p.graduated_overall_score >= $2 + AND ($3::text[] IS NULL OR n.path = ANY($3::text[])) + AND ($4::text IS NULL OR p.topic = $4) + ORDER BY p.graduated_overall_score DESC, + p.reuse_count DESC, + p.created_at DESC + LIMIT $5`, + [c.get('team_token'), minScore, ancestors, topic ?? null, limit], + ); + + const res: ProvenPromptsResponse = { + items: rows.map((r): ProvenPromptItem => ({ + id: r.id, + template: r.template, + topic: r.topic, + reuse_count: r.reuse_count, + graduated_overall_score: r.graduated_overall_score, + author_user_id: r.author_user_id, + node_path: r.node_path, + created_at: r.created_at.toISOString(), + })), + }; + return c.json(res); +}); + +// ----- GET /prompts/pending + POST /prompts/:id/review ------------------------ +// Review queue for TRAILHEAD_PROMOTION_MODE=review (promotion-gate.ts): /coach +// promotions land as 'pending_review' and only join the library when approved. +// Anyone holding the team secret can review — user ids are self-asserted, so +// "a teammate other than the author" is a team convention, not enforced. +promptRoutes.get('/prompts/pending', async (c) => { + const rows = await q<{ + id: string; template: string; topic: string | null; graduated_overall_score: number; + author_user_id: string | null; node_path: string; created_at: Date; + }>( + `SELECT p.id, p.template, p.topic, p.graduated_overall_score, p.author_user_id, + n.path AS node_path, p.created_at + FROM prompts p + JOIN nodes n ON n.id = p.node_id + WHERE n.team_token = $1 AND p.status = 'pending_review' + ORDER BY p.created_at ASC + LIMIT 200`, + [c.get('team_token')], + ); + return c.json({ + items: rows.map((r) => ({ ...r, created_at: r.created_at.toISOString() })), + }); +}); + +promptRoutes.post('/prompts/:id/review', async (c) => { + const id = c.req.param('id'); + if (!isUuid(id)) return c.json({ error: 'bad_request', detail: 'invalid prompt id' }, 400); + const body = await c.req.json<{ approve?: unknown }>().catch(() => null); + if (!body || typeof body.approve !== 'boolean') { + return c.json({ error: 'bad_request', detail: 'body must be { "approve": true | false }' }, 400); + } + // Scoped through nodes.team_token: another team's prompt id is a 404. + const rows = body.approve + ? await q<{ id: string }>( + `UPDATE prompts p SET status = 'graduated' + FROM nodes n + WHERE p.id = $1 AND p.node_id = n.id AND n.team_token = $2 AND p.status = 'pending_review' + RETURNING p.id`, + [id, c.get('team_token')], + ) + : await q<{ id: string }>( + `DELETE FROM prompts p + USING nodes n + WHERE p.id = $1 AND p.node_id = n.id AND n.team_token = $2 AND p.status = 'pending_review' + RETURNING p.id`, + [id, c.get('team_token')], + ); + if (!rows.length) return c.json({ error: 'not_found' }, 404); + return c.json({ id, status: body.approve ? 'graduated' : 'rejected' }); +}); diff --git a/apps/api/src/routes/score.ts b/apps/api/src/routes/score.ts new file mode 100644 index 0000000..d125d3c --- /dev/null +++ b/apps/api/src/routes/score.ts @@ -0,0 +1,96 @@ +// POST /score (5-dimension score, recorded as skill observations) and POST /capture. +// Mounted by app.ts, which applies auth, body caps and rate limits first. +import { Hono } from 'hono'; +import type { + CaptureRequest, + CaptureResponse, + ScoreRequest, + ScoreResponse, +} from '@trailhead/shared'; +import { q } from '../db.ts'; +import { isUnparseableScore } from '../coach-degraded.ts'; +import { overallScore, scorePrompt } from '../gemini.ts'; +import { renderTeamContext } from '../team-context.ts'; +import { PROMPT_TOO_LONG, promptTooLong, writeSkillObservations, type AppEnv } from '../http.ts'; + +export const scoreRoutes = new Hono(); + +// ----- POST /score ----------------------------------------------------------- +// Live 5-dim Gemini score; writes 5 skill_observation rows (one per dimension) +// with a 30s dedup window per (team, user, dimension, prompt-hash) per spec +// §19 risk register. Fails closed (returns 500 on Gemini error) — the browser +// extension fails open on its side so the user is never blocked. + +scoreRoutes.post('/score', async (c) => { + const body = await c.req.json().catch(() => null); + if (!body || typeof body.prompt !== 'string' || typeof body.user_id !== 'string') { + return c.json({ error: 'bad_request' }, 400); + } + if (promptTooLong(body.prompt)) return c.json(PROMPT_TOO_LONG, 413); + + // Optional sticky wiki context from the popup picker. Bundle is rendered + // out-of-band and prepended to Gemini's system instruction so the rubric + // is calibrated against the team's conventions without inflating the + // prompt being scored. + const teamContext = body.context_path + ? await renderTeamContext(c.get('team_token'), body.context_path) + : null; + + const result = await scorePrompt({ + prompt: body.prompt, + file_path: body.file_path, + team_context: teamContext ?? undefined, + }); + // Parse failure comes back as all-zero + no hints rather than a throw (see + // isUnparseableScore). Report it as an upstream failure and write nothing: + // persisting it would record five fake 0/10 observations for this user. + if (isUnparseableScore(result.dimensions, result.missing)) { + console.error('[api] /score scorePrompt returned unparseable output (zero+empty fingerprint)'); + return c.json({ error: 'score_unparseable' }, 502); + } + const overall = overallScore(result.dimensions); + + await writeSkillObservations( + c.get('team_token'), + body.user_id, + body.prompt, + result.dimensions, + ); + + const res: ScoreResponse = { + overall, + dimensions: result.dimensions, + missing: result.missing, + }; + return c.json(res); +}); + +// ----- POST /capture --------------------------------------------------------- +scoreRoutes.post('/capture', async (c) => { + const body = await c.req.json().catch(() => null); + if (!body || typeof body.user_prompt !== 'string' || typeof body.user_id !== 'string') { + return c.json({ error: 'bad_request' }, 400); + } + const surface = body.surface; + if (surface !== 'browser' && surface !== 'vscode' && surface !== 'mcp') { + return c.json({ error: 'bad_surface' }, 400); + } + + const rows = await q<{ id: string }>( + `INSERT INTO captures + (team_token, surface, user_prompt, ai_response, file_path, outcome, scored_dimensions) + VALUES ($1, $2, $3, $4, $5, $6, $7) + RETURNING id`, + [ + c.get('team_token'), + surface, + body.user_prompt, + body.ai_response ?? null, + body.file_path ?? null, + body.outcome ?? null, + body.scored_dimensions ? JSON.stringify(body.scored_dimensions) : null, + ], + ); + const res: CaptureResponse = { id: rows[0]!.id }; + return c.json(res); +}); diff --git a/apps/api/src/routes/teams.ts b/apps/api/src/routes/teams.ts new file mode 100644 index 0000000..520bc6b --- /dev/null +++ b/apps/api/src/routes/teams.ts @@ -0,0 +1,143 @@ +// Team registration, credential probe and rotation, and DELETE /team/data. +// Mounted by app.ts, which applies auth, body caps and rate limits first. +import { Hono } from 'hono'; +import type { TeamSummary, TeamsListResponse } from '@trailhead/shared'; +import { DEMO_TEAM_TOKEN, q, registerTeam, rotateTeamSecret, wipeTeamData } from '../db.ts'; +import { isValidTeamId, randomTeamId, safeEqual } from '../team-auth.ts'; +import { invalidateTeamContext } from '../team-context.ts'; +import { authPolicy, clientIp, opaqueTeamId, rateLimit, type AppEnv } from '../http.ts'; + +export const teamRoutes = new Hono(); + +// ----- GET /teams ------------------------------------------------------------ +// Resolves the CALLER's team. Authenticated, and it never returns a token. +// +// This used to be unauthenticated and return every team on the server together +// with its token — with CORS `*`, so any web page could read it. The team token +// is the only credential in this system: it grants read on the wiki (which +// summarises private source code) and write on everything. A single unauth GET +// therefore compromised every tenant at once. The browser popup's convenience +// of pre-populating a team dropdown before any token was configured is what +// paid for that, and it is nowhere near worth the price. +// +// The response is a list of one so the TeamsListResponse shape (and every +// caller that maps over `teams`) keeps working. +teamRoutes.get('/teams', async (c) => { + const teamToken = c.get('team_token'); + const legacy = c.get('legacy_auth'); + const rows = await q<{ name: string; token: string }>( + 'SELECT name, token FROM teams WHERE token = $1', + [teamToken], + ); + const teams: TeamSummary[] = rows.map((r) => ({ + name: r.name, + id: opaqueTeamId(r.token), + legacy, + // A secret team's id is public by design. A legacy team's id IS its + // credential, so it is never echoed. + ...(legacy ? {} : { team_id: r.token }), + })); + const res: TeamsListResponse = { teams }; + return c.json(res); +}); + +// ----- POST /teams ----------------------------------------------------------- +// Register a team. Returns its secret exactly once; only the hash is stored. +// +// body: { team_id?: string, name?: string } +// 201 { team_id, name, secret } +// 409 team_exists — someone already registered this id. That is the join +// flow: ask a teammate for the secret (or pick another team_id). +// 403 admin_token_required — TRAILHEAD_ADMIN_TOKEN is set and X-Admin-Token +// did not match. +// +// `init` derives team_id from the normalised git remote URL, so every clone of +// a repo proposes the same id and the second one lands on the 409. +teamRoutes.post('/teams', async (c) => { + // Before the admin check, so the limit also slows admin-token guessing. + const limited = rateLimit(c, [['register_per_ip', clientIp(c)]]); + if (limited) return limited; + const { adminToken } = authPolicy(); + if (adminToken !== null && !safeEqual(c.req.header('x-admin-token') ?? '', adminToken)) { + return c.json( + { + error: 'admin_token_required', + detail: 'This server restricts team registration. Ask its operator for a team secret, or for the admin token.', + }, + 403, + ); + } + const body = (await c.req.json<{ team_id?: unknown; name?: unknown }>().catch(() => null)) ?? {}; + if (body.team_id !== undefined && !isValidTeamId(body.team_id)) { + return c.json( + { error: 'bad_team_id', detail: '3-100 chars of letters, digits, _ . - starting with a letter or digit' }, + 400, + ); + } + const teamId = (body.team_id as string | undefined) ?? randomTeamId(); + const rawName = typeof body.name === 'string' ? body.name.trim().slice(0, 200) : ''; + const created = await registerTeam(teamId, rawName || `team:${teamId.slice(0, 16)}`); + if (!created) { + return c.json( + { + error: 'team_exists', + team_id: teamId, + detail: + 'A team with this id already exists. To join it, get the team secret from a teammate ' + + '(it is in their .trailhead-team file) and run `init --team-token `.', + }, + 409, + ); + } + return c.json({ team_id: created.teamId, name: created.name, secret: created.secret }, 201); +}); + +// ----- POST /teams/rotate-secret --------------------------------------------- +// Mint a new secret for the caller's team; the old credential stops working +// immediately. For a legacy team this is the upgrade to the secret model: +// afterwards its id is no longer accepted as a credential. Share the new +// secret with teammates. +teamRoutes.post('/teams/rotate-secret', async (c) => { + const teamId = c.get('team_token'); + if (teamId === DEMO_TEAM_TOKEN) { + return c.json({ error: 'demo_team_protected', detail: 'The demo team keeps its public secret.' }, 403); + } + const secret = await rotateTeamSecret(teamId); + const rows = await q<{ name: string }>('SELECT name FROM teams WHERE token = $1', [teamId]); + return c.json({ team_id: teamId, name: rows[0]?.name ?? '', secret }); +}); + +// ----- DELETE /team/data ----------------------------------------------------- +// Wipe every nodes / learnings / prompts / captures / skill_observations row +// for the requesting team. The teams row itself is preserved so re-running +// the same token continues to land in the same id (matters for the +// trailhead-mcp reset CLI which talks to localhost first then prod). +// +// The demo team is protected against accidental nukes — wiping it would +// erase the seeded data the dashboard demo relies on. Override with +// TRAILHEAD_ALLOW_DEMO_RESET=true if you really need to reseed. + +teamRoutes.delete('/team/data', async (c) => { + const body = await c.req.json<{ confirm?: boolean }>().catch(() => null); + if (!body || body.confirm !== true) { + return c.json( + { error: 'confirm_required', detail: 'POST { "confirm": true } to wipe.' }, + 400, + ); + } + const teamToken = c.get('team_token'); + const isDemo = teamToken === DEMO_TEAM_TOKEN; + if (isDemo && process.env.TRAILHEAD_ALLOW_DEMO_RESET !== 'true') { + return c.json( + { + error: 'demo_team_protected', + detail: + 'Refusing to wipe the demo team. Set TRAILHEAD_ALLOW_DEMO_RESET=true on the API to override.', + }, + 403, + ); + } + const deleted = await wipeTeamData(teamToken); + invalidateTeamContext(teamToken); + return c.json({ team_token: teamToken, deleted }); +}); diff --git a/apps/api/src/routes/wiki.ts b/apps/api/src/routes/wiki.ts new file mode 100644 index 0000000..548893c --- /dev/null +++ b/apps/api/src/routes/wiki.ts @@ -0,0 +1,388 @@ +// The team wiki: propose, context/examples lookups, search, recent changes, tree and export. +// Mounted by app.ts, which applies auth, body caps and rate limits first. +import { Hono } from 'hono'; +import type { + ContextNode, + ContextResponse, + ExamplesItem, + ExamplesResponse, + SearchResponse, + WikiProposeRequest, + WikiProposeResponse, + WikiRecentItem, + WikiRecentResponse, + WikiTreeResponse, +} from '@trailhead/shared'; +import { ancestorPaths, normalize, normalizePath } from '@trailhead/scoring'; +import { q, upsertNode } from '../db.ts'; +import { intParam } from '../request-params.ts'; +import { loadWikiTree } from '../wiki-tree.ts'; +import { exportFilename, renderWikiMarkdown } from '../wiki-export.ts'; +import { invalidateTeamContext } from '../team-context.ts'; +import type { AppEnv } from '../http.ts'; + +export const wikiRoutes = new Hono(); + +// ----- POST /wiki/propose ---------------------------------------------------- +// Normalize → dedup on (node_id, body_normalized) → increment count → promote +// to durable at >= 3. Idempotent: repeated calls for the same insight only +// reinforce the existing draft. + +// Bigram-Jaccard similarity over normalized strings. Used as a paraphrase +// fallback when exact body_normalized match misses — catches "go through" / +// "flow through" style edits that the lowercase+strip-punct normalize can't +// collapse. Bigrams (vs unigrams) are deliberate: a polarity flip ("never" +// inserted into an otherwise identical sentence) drops the bigram score +// well below the threshold, so opposite-meaning insights stay distinct. +const PARAPHRASE_THRESHOLD = 0.7; + +// Headroom under Postgres's ~2704-byte btree tuple limit for +// idx_learnings_node_normalized (the key also carries node_id + tuple header). +const MAX_INSIGHT_BYTES = 2048; + +function bigramSet(normalized: string): Set { + const tokens = normalized.split(' ').filter(Boolean); + const out = new Set(); + for (let i = 0; i < tokens.length - 1; i++) { + out.add(`${tokens[i]} ${tokens[i + 1]}`); + } + return out; +} + +function bigramJaccard(a: string, b: string): number { + const aBg = bigramSet(a); + const bBg = bigramSet(b); + if (aBg.size === 0 || bBg.size === 0) return 0; + let intersection = 0; + for (const bg of aBg) if (bBg.has(bg)) intersection++; + const union = aBg.size + bBg.size - intersection; + return union === 0 ? 0 : intersection / union; +} + +wikiRoutes.post('/wiki/propose', async (c) => { + const body = await c.req.json().catch(() => null); + if (!body || typeof body.node_path !== 'string' || typeof body.insight !== 'string') { + return c.json({ error: 'bad_request' }, 400); + } + if (!body.insight.trim()) return c.json({ error: 'empty_insight' }, 400); + + // body_normalized is a btree index key, and Postgres rejects index rows over + // ~2.7 KB — a longer insight failed the INSERT with a raw 500. Insights are + // meant to be one-sentence conventions, so reject oversize ones up front. + const bodyNormalized = normalize(body.insight); + if (Buffer.byteLength(bodyNormalized, 'utf8') > MAX_INSIGHT_BYTES) { + return c.json( + { error: 'insight_too_long', detail: `max ${MAX_INSIGHT_BYTES} bytes after normalization` }, + 400, + ); + } + + const path = normalizePath(body.node_path); + const nodeId = await upsertNode(c.get('team_token'), path); + + // Step 1: exact match on body_normalized — the cheap fast path. Hits when + // the user (or LLM) sent the same insight verbatim or with only + // punctuation/whitespace/case differences. + const exact = await q<{ + id: string; reinforcement_count: number; status: 'draft' | 'durable'; + }>( + `SELECT id, reinforcement_count, status + FROM learnings + WHERE node_id = $1 AND body_normalized = $2 + LIMIT 1`, + [nodeId, bodyNormalized], + ); + + let matchId: string | null = null; + let matchPriorStatus: 'draft' | 'durable' | null = null; + if (exact.length) { + matchId = exact[0]!.id; + matchPriorStatus = exact[0]!.status; + } else { + // Step 2: paraphrase fallback. Pull this node's existing learnings and + // compute bigram-Jaccard against each. Keeps the wiki from accumulating + // near-duplicate drafts that never hit the 3× durability threshold. + const candidates = await q<{ + id: string; body_normalized: string; status: 'draft' | 'durable'; + }>( + `SELECT id, body_normalized, status + FROM learnings + WHERE node_id = $1`, + [nodeId], + ); + let best: { id: string; status: 'draft' | 'durable'; sim: number } | null = null; + for (const cand of candidates) { + const sim = bigramJaccard(bodyNormalized, cand.body_normalized); + if (sim >= PARAPHRASE_THRESHOLD && (!best || sim > best.sim)) { + best = { id: cand.id, status: cand.status, sim }; + } + } + if (best) { + matchId = best.id; + matchPriorStatus = best.status; + } + } + + let action: WikiProposeResponse['action']; + let currentCount: number; + let promotedToDurable = false; + + if (matchId === null) { + const inserted = await q<{ reinforcement_count: number }>( + `INSERT INTO learnings (node_id, body, body_normalized) + VALUES ($1, $2, $3) + RETURNING reinforcement_count`, + [nodeId, body.insight.trim(), bodyNormalized], + ); + action = 'created'; + currentCount = inserted[0]!.reinforcement_count; + } else { + const updated = await q<{ reinforcement_count: number; status: 'draft' | 'durable' }>( + `UPDATE learnings + SET reinforcement_count = reinforcement_count + 1, + last_seen_at = NOW(), + status = CASE WHEN reinforcement_count + 1 >= 3 THEN 'durable' ELSE status END + WHERE id = $1 + RETURNING reinforcement_count, status`, + [matchId], + ); + const after = updated[0]!; + currentCount = after.reinforcement_count; + promotedToDurable = matchPriorStatus === 'draft' && after.status === 'durable'; + action = promotedToDurable ? 'promoted' : 'reinforced'; + } + + invalidateTeamContext(c.get('team_token')); + const res: WikiProposeResponse = { + action, + current_count: currentCount, + ...(promotedToDurable ? { promoted_to_durable: true } : {}), + }; + return c.json(res); +}); + +// ----- GET /context?path= ---------------------------------------------------- +// HCL ancestor walk: every node whose path is a prefix of the file path, +// shallow → deep, plus its top durable learnings. + +wikiRoutes.get('/context', async (c) => { + const filePath = c.req.query('path') ?? ''; + if (!filePath) return c.json({ error: 'missing_path' }, 400); + + const ancestors = ancestorPaths(filePath); + if (ancestors.length === 0) { + const res: ContextResponse = { nodes: [] }; + return c.json(res); + } + + const rows = await q<{ + path: string; body_md: string; learning_body: string | null; reinforcement_count: number | null; + }>( + `SELECT n.path, n.body_md, l.body AS learning_body, l.reinforcement_count + FROM nodes n + LEFT JOIN learnings l + ON l.node_id = n.id + AND l.status = 'durable' + WHERE n.team_token = $1 + AND n.path = ANY($2::text[]) + ORDER BY length(n.path) ASC, n.path ASC, + COALESCE(l.reinforcement_count, 0) DESC`, + [c.get('team_token'), ancestors], + ); + + const byPath = new Map(); + for (const r of rows) { + let node = byPath.get(r.path); + if (!node) { + node = { path: r.path, body_md: r.body_md, durable_learnings: [] }; + byPath.set(r.path, node); + } + if (r.learning_body) { + node.durable_learnings.push({ + body: r.learning_body, + reinforcement_count: r.reinforcement_count ?? 0, + }); + } + } + + const res: ContextResponse = { + nodes: ancestors + .map((p) => byPath.get(p)) + .filter((n): n is ContextNode => Boolean(n)), + }; + return c.json(res); +}); + +// ----- GET /examples?path= --------------------------------------------------- +wikiRoutes.get('/examples', async (c) => { + const filePath = c.req.query('path') ?? ''; + if (!filePath) return c.json({ error: 'missing_path' }, 400); + const limit = intParam(c.req.query('limit'), 3, 1, 10); + const ancestors = ancestorPaths(filePath); + + const rows = await q<{ + template: string; topic: string | null; reuse_count: number; node_path: string; + }>( + `SELECT p.template, p.topic, p.reuse_count, n.path AS node_path + FROM prompts p + JOIN nodes n ON n.id = p.node_id + WHERE n.team_token = $1 + AND n.path = ANY($2::text[]) + AND p.status = 'graduated' + ORDER BY p.reuse_count DESC, length(n.path) DESC + LIMIT $3`, + [c.get('team_token'), ancestors, limit], + ); + + const res: ExamplesResponse = { items: rows.map((r): ExamplesItem => ({ + template: r.template, + topic: r.topic, + reuse_count: r.reuse_count, + node_path: r.node_path, + })) }; + return c.json(res); +}); + +// ----- GET /search?q=&scope= ------------------------------------------------- +// Free-text substring search across the team's wiki: rules (nodes.body_md), +// durable learnings (learnings.body), and graduated prompts (prompts.template). +// Optional `scope` constrains results to the ancestor paths of a file/folder +// (same shape as /context). Used by the wiki_lookup MCP tool when the caller +// passes only `query`, or `query` + `file_path` for a path-scoped search. +wikiRoutes.get('/search', async (c) => { + const query = (c.req.query('q') ?? '').trim(); + if (!query) return c.json({ error: 'missing_q' }, 400); + const limit = intParam(c.req.query('limit'), 50, 1, 100); + const scope = c.req.query('scope'); + // ILIKE wildcards from user input shouldn't bleed into the pattern. Escape + // %, _, and the escape char itself so a search for "100%" matches the + // literal substring rather than "100". + const escaped = query.replace(/[\\%_]/g, (ch) => `\\${ch}`); + const pattern = `%${escaped}%`; + const teamToken = c.get('team_token'); + const ancestors = scope ? ancestorPaths(scope) : null; + + // Rule branch matches on body_md OR the node path itself — so a query like + // "scoring" surfaces packages/scoring/ even when body_md doesn't repeat the + // folder name. Path-only matches return a placeholder body so the renderer + // doesn't dump the whole node narrative when the match was structural. + const rows = await q<{ kind: 'rule' | 'learning' | 'prompt'; body: string; node_path: string }>( + `SELECT 'rule'::text AS kind, + CASE + WHEN n.body_md ILIKE $2 THEN n.body_md + ELSE '(matched on path: ' || n.path || ')' + END AS body, + n.path AS node_path + FROM nodes n + WHERE n.team_token = $1 + AND (n.body_md ILIKE $2 OR n.path ILIKE $2) + AND ($3::text[] IS NULL OR n.path = ANY($3::text[])) + UNION ALL + SELECT 'learning'::text AS kind, l.body AS body, n.path AS node_path + FROM learnings l + JOIN nodes n ON n.id = l.node_id + WHERE n.team_token = $1 + AND l.status = 'durable' + AND l.body ILIKE $2 + AND ($3::text[] IS NULL OR n.path = ANY($3::text[])) + UNION ALL + SELECT 'prompt'::text AS kind, p.template AS body, n.path AS node_path + FROM prompts p + JOIN nodes n ON n.id = p.node_id + WHERE n.team_token = $1 + AND p.status = 'graduated' + AND p.template ILIKE $2 + AND ($3::text[] IS NULL OR n.path = ANY($3::text[])) + LIMIT $4`, + [teamToken, pattern, ancestors, limit], + ); + + const res: SearchResponse = { items: rows }; + return c.json(res); +}); + +// ----- GET /wiki/recent?since=ISO -------------------------------------------- +wikiRoutes.get('/wiki/recent', async (c) => { + const sinceParam = c.req.query('since'); + const since = sinceParam ? new Date(sinceParam) : new Date(Date.now() - 60 * 60 * 1000); + if (Number.isNaN(since.getTime())) return c.json({ error: 'bad_since' }, 400); + const limit = intParam(c.req.query('limit'), 50, 1, 200); + + const rows = await q<{ + id: string; + node_path: string; + body: string; + status: 'draft' | 'durable'; + reinforcement_count: number; + last_seen_at: Date; + created_at: Date; + }>( + `SELECT l.id, n.path AS node_path, l.body, l.status, + l.reinforcement_count, l.last_seen_at, l.created_at + FROM learnings l + JOIN nodes n ON n.id = l.node_id + WHERE n.team_token = $1 + AND l.last_seen_at > $2 + ORDER BY l.last_seen_at DESC + LIMIT $3`, + [c.get('team_token'), since.toISOString(), limit], + ); + + const res: WikiRecentResponse = { + items: rows.map((r): WikiRecentItem => ({ + id: r.id, + node_path: r.node_path, + body: r.body, + status: r.status, + reinforcement_count: r.reinforcement_count, + last_seen_at: r.last_seen_at.toISOString(), + created_at: r.created_at.toISOString(), + })), + }; + return c.json(res); +}); + +// ----- GET /wiki/tree -------------------------------------------------------- +// Full node list for the dashboard /wiki page. One row per node with its +// learnings split into durable vs draft. Sort by path (prefix-friendly). + +wikiRoutes.get('/wiki/tree', async (c) => { + const nodes = await loadWikiTree(c.get('team_token')); + const res: WikiTreeResponse = { nodes }; + return c.json(res); +}); + +// ----- GET /wiki/export ----------------------------------------------------- +// Markdown export of the whole team wiki. Teams will not pour knowledge into +// a store they cannot get it back out of, so this is a trust signal as much +// as a backup story. +// +// GET /wiki/export -> text/markdown, as a download +// GET /wiki/export?drafts=true -> include draft learnings too +// GET /wiki/export?format=json -> { filename, markdown } for browser clients +wikiRoutes.get('/wiki/export', async (c) => { + const teamToken = c.get('team_token'); + const [nodes, teamRows] = await Promise.all([ + loadWikiTree(teamToken), + q<{ name: string }>('SELECT name FROM teams WHERE token = $1', [teamToken]), + ]); + const teamName = teamRows[0]?.name; + const now = new Date(); + const markdown = renderWikiMarkdown(nodes, { + teamName, + generatedAt: now, + includeDrafts: c.req.query('drafts') === 'true', + }); + const filename = exportFilename(teamName, now); + + if (c.req.query('format') === 'json') { + return c.json({ filename, markdown }); + } + return new Response(markdown, { + status: 200, + headers: { + 'content-type': 'text/markdown; charset=utf-8', + 'content-disposition': `attachment; filename="${filename}"`, + }, + }); +}); From f897485cc2460689f14a4095602650d11d37ce05 Mon Sep 17 00:00:00 2001 From: Bogdan Truta Date: Thu, 1 Oct 2026 12:26:01 +0000 Subject: [PATCH 02/14] api: one Gemini client for every call; drop the dead learning extractor wiki-bootstrap-job.ts built its own GoogleGenAI client, so the rich bootstrap (the biggest fan-out of LLM calls in the API) bypassed the retry policy and never appeared in Langfuse traces. It now calls gemini.generateText, which goes through the same tracedGenerate/withRetry path as the request handlers: 429/5xx are retried with backoff, timeouts are still never retried (90 s per call, as before). generateText does not apply the 4 000-char cap meant for short structured answers, because folder narratives are several thousand words; a new test pins both properties. Removed as dead code: extractLearning (gemini.ts), EXTRACT_MODEL (gemma-4-31b-it) and the extract-prompt module in packages/scoring with its tests. They served the Stop hook that was removed in PR #5; nothing has imported them since. The README's "Gemma for async learning extraction" claim goes with them; every call uses gemini-3-flash-preview. Co-Authored-By: Claude Opus 5.5 --- apps/api/src/gemini.test.ts | 19 ++++- apps/api/src/gemini.ts | 72 ++++++---------- apps/api/src/wiki-bootstrap-job.ts | 82 ++++++------------- packages/scoring/README.md | 34 ++++---- packages/scoring/package.json | 3 +- .../scoring/src/declarations-match.test.mjs | 2 +- packages/scoring/src/extract-prompt.d.mts | 12 --- packages/scoring/src/extract-prompt.mjs | 58 ------------- packages/scoring/src/extract-prompt.test.mjs | 65 --------------- packages/scoring/src/index.ts | 13 +-- packages/scoring/src/models.d.mts | 1 - packages/scoring/src/models.mjs | 13 +-- 12 files changed, 98 insertions(+), 276 deletions(-) delete mode 100644 packages/scoring/src/extract-prompt.d.mts delete mode 100644 packages/scoring/src/extract-prompt.mjs delete mode 100644 packages/scoring/src/extract-prompt.test.mjs diff --git a/apps/api/src/gemini.test.ts b/apps/api/src/gemini.test.ts index 934dcdd..2c7e786 100644 --- a/apps/api/src/gemini.test.ts +++ b/apps/api/src/gemini.test.ts @@ -23,7 +23,7 @@ import { DIMENSIONS } from '@trailhead/shared'; // request in this file is intercepted by the fetch stub, so it is never // sent anywhere and is not a credential. process.env.GEMINI_API_KEY ??= 'test-key-not-a-real-credential'; -const { extractTopic, overallScore, scorePrompt, scoreSamplingConfig } = await import('./gemini.ts'); +const { extractTopic, generateText, overallScore, scorePrompt, scoreSamplingConfig } = await import('./gemini.ts'); interface FetchCall { url: string; @@ -250,3 +250,20 @@ test('scorePrompt sends the configured temperature and thinking budget', async ( delete process.env.TRAILHEAD_SCORE_THINKING_BUDGET; } }); + +// The rich wiki bootstrap (wiki-bootstrap-job.ts) used to build its own +// GoogleGenAI client, so its calls skipped the retry policy and never showed +// up in Langfuse. It now goes through generateText. +test('generateText sends one request through the shared client and does not cap long-form answers', async () => { + const narrative = 'x'.repeat(12_000); // a folder narrative is several thousand words + const stub = withFetchStub(() => geminiTextResponse(JSON.stringify({ narrative_md: narrative }))); + try { + const text = await generateText({ model: 'gemini-3-flash-preview', contents: 'document src/' }, 'folder(src/)'); + assert.equal(stub.calls.length, 1); + assert.match(stub.calls[0]!.url, /generativelanguage\.googleapis\.com/); + // The short structured answers are capped at 4 000 chars; this must not be. + assert.equal(JSON.parse(text).narrative_md.length, 12_000); + } finally { + stub.restore(); + } +}); diff --git a/apps/api/src/gemini.ts b/apps/api/src/gemini.ts index f798010..f83207c 100644 --- a/apps/api/src/gemini.ts +++ b/apps/api/src/gemini.ts @@ -1,13 +1,13 @@ import './env.ts'; -// Thin Gemini wrapper. One client, one place to encode model quirks. +// Thin Gemini wrapper. One client, one place to encode model quirks. Every +// Gemini call in the API goes through it (wiki-bootstrap-job.ts via +// generateText). // -// gemini-3-flash-preview supports responseSchema enforcement and thinkingBudget=0 -// for fast structured output. We use it for /score and topic extraction. -// -// gemma-4-31b-it is a thinking model with no schema enforcement and no -// thinking-budget control. It returns chain-of-thought as `thought:true` -// parts followed by the answer in plain `text`. We use it for /diff and -// the learning-extractor helper below, where latency is tolerable. +// Model ids live in packages/scoring/src/models.mjs; today every call uses +// gemini-3-flash-preview (responseSchema enforcement, thinkingConfig). The +// non-Flash branches below handle a Gemma-family model, which has no schema +// enforcement and returns chain-of-thought as `thought:true` parts, in case +// one is swapped in there. import { GoogleGenAI, Type } from '@google/genai'; import { currentTrace } from './langfuse.ts'; @@ -15,8 +15,6 @@ import type { Dimension, DimensionScores, MissingHints } from '@trailhead/shared import { DIMENSIONS } from '@trailhead/shared'; import { DIFF_MODEL, - EXTRACT_MODEL, - EXTRACT_SYSTEM_PROMPT, SCORE_MODEL, SCORE_SYSTEM_PROMPT, TEACH_SYSTEM_PROMPT, @@ -149,21 +147,34 @@ type GenResp = { candidates?: { content?: { parts?: { text?: string; thought?: b // fed to JSON.parse (which scales O(n) on input length). const MAX_EXTRACT_CHARS = 4000; -function extractAnswer(resp: GenResp): string { +function extractAnswer(resp: GenResp, maxChars: number = MAX_EXTRACT_CHARS): string { const parts = resp.candidates?.[0]?.content?.parts ?? []; const joined = parts .filter((p) => p.thought !== true && typeof p.text === 'string') .map((p) => p.text) .join('') .trim(); - return joined.length > MAX_EXTRACT_CHARS - ? joined.slice(0, MAX_EXTRACT_CHARS) - : joined; + return joined.length > maxChars ? joined.slice(0, maxChars) : joined; +} + +// One Gemini call with this module's retry/timeout policy and Langfuse +// tracing, returning the answer text (thought parts dropped). For callers +// outside this file that build their own prompts and schemas — the rich wiki +// bootstrap (wiki-bootstrap-job.ts) — so every Gemini call in the API goes +// through one client. Long-form output (folder narratives run to several +// thousand words) is not capped like the short structured answers above. +export async function generateText( + params: GenContentParams, + label: string, + perCallTimeoutMs?: number, +): Promise { + const resp = await withRetry(() => tracedGenerate(params), label, perCallTimeoutMs); + return extractAnswer(resp as GenResp, Number.POSITIVE_INFINITY); } // Strip ``` fences if a model wrapped its JSON in markdown despite being // told not to. Returns `null` if nothing parses. -function tryParseJson(raw: string): T | null { +export function tryParseJson(raw: string): T | null { let s = raw.trim(); if (s.startsWith('```')) { s = s.replace(/^```(?:json)?\s*/i, '').replace(/```\s*$/i, '').trim(); @@ -754,7 +765,7 @@ export async function extractPathAndTopic( } } -// ----- /diff narrative synthesis (Gemma 4 31B; quality > latency) ----------- +// ----- /diff narrative synthesis (DIFF_MODEL) -------------------------------- export async function synthesizeDiff(args: { user_prompt: string; user_scores: DimensionScores; @@ -895,33 +906,4 @@ export async function improveCoach(input: ImproveCoachInput): Promise { - const resp = await withRetry( - () => tracedGenerate({ - model: EXTRACT_MODEL, - contents: - `${EXTRACT_SYSTEM_PROMPT}\n\n` + - `USER PROMPT:\n${args.user_prompt}\n\n` + - `ASSISTANT RESPONSE:\n${args.assistant_response}\n\n` + - `Return ONLY the JSON object.`, - config: { temperature: 0 }, - }), - 'extract', - ); - const parsed = tryParseJson(extractAnswer(resp)); - if (!parsed) return { node_path: null, insight: null }; - return { - node_path: typeof parsed.node_path === 'string' ? parsed.node_path : null, - insight: typeof parsed.insight === 'string' ? parsed.insight : null, - }; -} - export type { Dimension, DimensionScores, MissingHints }; diff --git a/apps/api/src/wiki-bootstrap-job.ts b/apps/api/src/wiki-bootstrap-job.ts index db1dd8b..accfba7 100644 --- a/apps/api/src/wiki-bootstrap-job.ts +++ b/apps/api/src/wiki-bootstrap-job.ts @@ -19,17 +19,17 @@ // generated `body_md` and `learnings.body` text reaches the DB. // // Spec ref: docs/superpowers/specs/2026-04-26-wiki-bootstrap-rich-design.md -import { GoogleGenAI, Type } from '@google/genai'; +import { Type } from '@google/genai'; import { normalize as normalizeBody } from '@trailhead/scoring'; import type { OnboardRepoFullFile, OnboardRepoFullRequest, } from '@trailhead/shared'; import { q, upsertNode } from './db.ts'; - -// Reuse the same client GEMINI_API_KEY env var — the API server's index.ts -// asserts this is set on startup so we can lean on it being present. -const ai = new GoogleGenAI({ apiKey: process.env.GEMINI_API_KEY }); +// Every call goes through gemini.ts: the same client, retry policy (429/5xx +// retried with backoff, timeouts never retried) and Langfuse tracing as the +// request-path calls. +import { generateText, tryParseJson } from './gemini.ts'; // Flash for all three passes (spec §6 reasoning). Sourced from the shared // model registry so a single rename in packages/scoring/src/models.mjs @@ -46,9 +46,10 @@ const MAX_OUTPUT_TOKENS_FOLDER = 4_000; const MAX_OUTPUT_TOKENS_FILE = 700; const MAX_OUTPUT_TOKENS_ROOT = 5_000; -// Per-call timeout. Same posture as gemini.ts withRetry: timeouts are -// non-retryable to avoid the repetition-loop class of failures (2026-04-25 -// incident). One long Gemini call hanging shouldn't take down the job. +// Per-call timeout (gemini.ts withRetry; timeouts are never retried, to avoid +// the repetition-loop class of failures from the 2026-04-25 incident). Long: +// a folder narrative is several thousand tokens. One hanging call shouldn't +// take down the job; the 10-minute job budget is the overall safety net. const PER_CALL_TIMEOUT_MS = 90_000; // Cap on raw source we embed in a file node's body_md. The team-context @@ -235,8 +236,8 @@ async function callFolderPass(folderPath: string, files: OnboardRepoFullFile[]): `Do not invent behavior — if a file's purpose is unclear, say so.${truncationNote}\n\n` + `=== Files in \`${folderPath || '/'}\` ===\n\n${filesBlock}`; - const resp = await withTimeout( - () => ai.models.generateContent({ + const text = await generateText( + { model: MODEL, contents: userPrompt, config: { @@ -254,10 +255,11 @@ async function callFolderPass(folderPath: string, files: OnboardRepoFullFile[]): }, }, }, - }), + }, `folder(${folderPath})`, + PER_CALL_TIMEOUT_MS, ); - return parseStructured(resp, ['narrative_md']); + return parseStructured(text, ['narrative_md']); } interface FilePassOutput { @@ -284,8 +286,8 @@ async function callFilePass( `Conventions visible in this file (e.g. "throws on invalid input", "logs to stderr only") MUST be returned in the conventions[] array — short imperative phrases, one rule each.${folderHint}\n\n` + `=== \`${file.path}\` ===\n\`\`\`\n${file.content}\n\`\`\``; - const resp = await withTimeout( - () => ai.models.generateContent({ + const text = await generateText( + { model: MODEL, contents: userPrompt, config: { @@ -302,10 +304,11 @@ async function callFilePass( }, }, }, - }), + }, `file(${file.path})`, + PER_CALL_TIMEOUT_MS, ); - return parseStructured(resp, ['narrative_md']); + return parseStructured(text, ['narrative_md']); } interface RootPassOutput { @@ -343,8 +346,8 @@ async function callRootPass(args: { `=== Per-folder summaries ===\n\n${folderBlock}\n\n` + `=== Manifests ===\n\n${manifestBlock}${seedBlock}`; - const resp = await withTimeout( - () => ai.models.generateContent({ + const text = await generateText( + { model: MODEL, contents: userPrompt, config: { @@ -360,10 +363,11 @@ async function callRootPass(args: { }, }, }, - }), + }, 'root', + PER_CALL_TIMEOUT_MS, ); - return parseStructured(resp, ['narrative_md']); + return parseStructured(text, ['narrative_md']); } // ============================================================================ @@ -592,43 +596,7 @@ async function runWithConcurrency(items: T[], cap: number, fn: (item: T) => P await Promise.all(workers); } -// Hard timeout. Mirrors gemini.ts withRetry's posture but without retries — -// repetition-loop risk is real on long-form prose and the time budget at -// the job level (10 min) is the safety net. -function withTimeout(fn: () => Promise, label: string): Promise { - let timeoutId: ReturnType | undefined; - const timeoutPromise = new Promise((_, reject) => { - timeoutId = setTimeout(() => reject(new Error(`__trailhead_timeout__:${label}`)), PER_CALL_TIMEOUT_MS); - }); - return Promise.race([fn(), timeoutPromise]).finally(() => { - if (timeoutId !== undefined) clearTimeout(timeoutId); - }) as Promise; -} - -type GenResp = { candidates?: { content?: { parts?: { text?: string; thought?: boolean }[] } }[] }; - -function extractAnswer(resp: GenResp): string { - const parts = resp.candidates?.[0]?.content?.parts ?? []; - return parts - .filter((p) => p.thought !== true && typeof p.text === 'string') - .map((p) => p.text) - .join('') - .trim(); -} - -function tryParseJson(raw: string): T | null { - let s = raw.trim(); - if (s.startsWith('```')) { - s = s.replace(/^```(?:json)?\s*/i, '').replace(/```\s*$/i, '').trim(); - } - const start = s.indexOf('{'); - const end = s.lastIndexOf('}'); - if (start !== -1 && end > start) s = s.slice(start, end + 1); - try { return JSON.parse(s) as T; } catch { return null; } -} - -function parseStructured(resp: GenResp, requiredStrings: (keyof T & string)[]): T { - const text = extractAnswer(resp); +function parseStructured(text: string, requiredStrings: (keyof T & string)[]): T { const parsed = tryParseJson(text); if (!parsed) throw new Error('LLM response not parseable as JSON'); for (const k of requiredStrings) { diff --git a/packages/scoring/README.md b/packages/scoring/README.md index 88fc099..76c93c5 100644 --- a/packages/scoring/README.md +++ b/packages/scoring/README.md @@ -1,19 +1,23 @@ -# scoring — Gemini prompt templates +# scoring — prompt templates and pure helpers -Locked prompts that score, augment, and extract. Versioned in one place -so `apps/api` and `apps/mcp-server` use the same wording. Prompt-cache -friendly — system prompts stay byte-stable across calls so cache hit -rate is high. +Locked prompts and the pure functions around them, versioned in one place so +`apps/api`, `apps/mcp-server` and the browser extension use the same wording. +System prompts stay byte-stable across calls so the prompt cache hits. -**Templates (spec §18):** -- `score-prompt.ts` — 5-dim Gemini scorer (lock by hour 4, §18 #1) -- `augment-prompt.ts` — "Have Claude clarify" template (§6, §18 #2) -- `extract-prompt.ts` — learning-extraction prompt (§18 #3) -- `topic-prompt.ts` — Prompt Diff topic extraction (§18 #4) +Runtime code is plain ESM (`.mjs`, no build step) with a `.d.mts` declaration +next to each file; `declarations-match.test.mjs` fails if the two drift. -**Why a separate package:** `apps/api` uses score, augment, topic. -Sharing prevents the "three slightly different scoring prompts" failure -mode and keeps cost-per-call predictable. +| File | What it holds | +|---|---| +| `score-prompt.mjs` | The 5-dimension scorer's system prompt (the rubric) | +| `score-helpers.mjs` | `buildScoreUserPrompt`, `buildAugmentation` ("have Claude clarify") | +| `teach-prompt.mjs` | System prompt for the coaching rewrite (`/coach`) | +| `teach-templates.mjs` | Per-dimension teach text: title, definition, why, question | +| `reveal-render.mjs` | Renderers for the teach / success / skip blocks `/coach` returns | +| `topic-prompt.mjs` | Topic classifier prompt (`/diff` finds a team prompt on the same topic) | +| `fence.mjs` | `fenceUntrusted`: wraps team-authored text before it reaches an LLM | +| `normalize.mjs` | Text normalisation behind wiki dedup | +| `path-helpers.mjs` | `normalizePath`, `ancestorPaths` | +| `models.mjs` | Model id per call site (single source of truth) | -**Spec refs:** §5 (scoring prompt template), §18 (open implementation -questions — which prompts must be locked when) +**Spec refs:** §5 (scoring prompt template), §18 (which prompts are locked). diff --git a/packages/scoring/package.json b/packages/scoring/package.json index e79128f..f82f850 100644 --- a/packages/scoring/package.json +++ b/packages/scoring/package.json @@ -12,7 +12,6 @@ "default": "./src/index.ts" }, "./normalize": "./src/normalize.mjs", - "./extract-prompt": "./src/extract-prompt.mjs", "./score-prompt": "./src/score-prompt.mjs", "./topic-prompt": "./src/topic-prompt.mjs", "./score-helpers": "./src/score-helpers.mjs", @@ -22,7 +21,7 @@ }, "scripts": { "typecheck": "tsc --noEmit", - "test": "node --test src/normalize.test.mjs src/extract-prompt.test.mjs src/dedup-invariant.test.mjs src/declarations-match.test.mjs src/fence.test.mjs src/reveal-render.test.mjs" + "test": "node --test src/normalize.test.mjs src/dedup-invariant.test.mjs src/declarations-match.test.mjs src/fence.test.mjs src/reveal-render.test.mjs" }, "devDependencies": { "@types/node": "^22.10.0", diff --git a/packages/scoring/src/declarations-match.test.mjs b/packages/scoring/src/declarations-match.test.mjs index d898989..a0be57d 100644 --- a/packages/scoring/src/declarations-match.test.mjs +++ b/packages/scoring/src/declarations-match.test.mjs @@ -86,7 +86,7 @@ test('models.mjs exports the model ids the API and README both cite', async () = // documents them. A silent rename here is a production incident. /** @type {Record} */ const models = await import('./models.mjs'); - for (const k of ['SCORE_MODEL', 'TOPIC_MODEL', 'DIFF_MODEL', 'EXTRACT_MODEL']) { + for (const k of ['SCORE_MODEL', 'TOPIC_MODEL', 'DIFF_MODEL']) { const v = models[k]; assert.equal(typeof v, 'string', `${k} must be a string`); assert.ok(/** @type {string} */ (v).length > 0, `${k} must not be empty`); diff --git a/packages/scoring/src/extract-prompt.d.mts b/packages/scoring/src/extract-prompt.d.mts deleted file mode 100644 index 9ea9701..0000000 --- a/packages/scoring/src/extract-prompt.d.mts +++ /dev/null @@ -1,12 +0,0 @@ -export declare const EXTRACT_SYSTEM_PROMPT: string; - -export interface ExtractedLearning { - node_path: string; - insight: string; -} - -export interface ExtractResult { - learning: ExtractedLearning | null; -} - -export declare function parseExtractResponse(raw: unknown): ExtractResult; diff --git a/packages/scoring/src/extract-prompt.mjs b/packages/scoring/src/extract-prompt.mjs deleted file mode 100644 index 07f4411..0000000 --- a/packages/scoring/src/extract-prompt.mjs +++ /dev/null @@ -1,58 +0,0 @@ -// Stop hook learning-extractor prompt — spec §18 #3. -// -// Locked wording so the system prompt cache key is stable across hook calls. -// Tuning rules: -// - Bias toward "no learning" — false positives spam the wiki. The hook fires -// on EVERY assistant turn, so the default outcome must be silence. -// - A learning is a teamwide convention: "we always do X here". A one-off -// answer to a one-off question is not a learning. -// - The model must return strict JSON with one of two shapes (see below). -// Anything else is dropped silently by the hook. -export const EXTRACT_SYSTEM_PROMPT = `You read a single Claude Code turn (the user's prompt and the assistant's response) and decide whether it contains a teamwide engineering convention worth saving to the team wiki. - -A learning is worth saving ONLY if ALL of the following hold: -- It is a convention or rule the team consistently follows ("we always X", "our convention is Y", "the standard pattern here is Z"). -- It is concrete and code-relevant — not vague advice, not motivation, not coaching. -- It is not already implied by language standards or framework defaults. -- The user's prompt or the assistant's response explicitly states the convention. - -If no such learning is present, return: -{"learning": null} - -If a learning is present, return: -{"learning": {"node_path": "", "insight": ""}} - -Rules for the JSON: -- "node_path" must be a folder path with a trailing slash, e.g. "src/api/webhooks/". If unsure, use the most specific folder mentioned in the turn, or "src/" as a fallback. Never invent a path that wasn't referenced. -- "insight" must be a single declarative sentence in the team's voice ("Use exponential backoff with jitter for webhook retries."). -- No code fences, no prose, no explanation outside the JSON. - -Default to {"learning": null}. Wiki spam is worse than a missed learning.`; - -// Robust parser: the model occasionally wraps JSON in markdown fences or adds -// a leading "Here is the JSON:". Strip anything before the first '{' and after -// the last matching '}'. Returns { learning: null } on any malformed shape. -export function parseExtractResponse(raw) { - if (typeof raw !== 'string') return { learning: null }; - const start = raw.indexOf('{'); - const end = raw.lastIndexOf('}'); - if (start === -1 || end === -1 || end <= start) return { learning: null }; - let parsed; - try { - parsed = JSON.parse(raw.slice(start, end + 1)); - } catch { - return { learning: null }; - } - if (!parsed || typeof parsed !== 'object') return { learning: null }; - const l = parsed.learning; - if (!l || typeof l !== 'object') return { learning: null }; - if (typeof l.node_path !== 'string' || typeof l.insight !== 'string') { - return { learning: null }; - } - const node_path = l.node_path.trim(); - const insight = l.insight.trim(); - if (!node_path || !insight) return { learning: null }; - // Enforce trailing slash convention (roadmap §3) — caller can trust it. - const normalizedPath = node_path.endsWith('/') ? node_path : `${node_path}/`; - return { learning: { node_path: normalizedPath, insight } }; -} diff --git a/packages/scoring/src/extract-prompt.test.mjs b/packages/scoring/src/extract-prompt.test.mjs deleted file mode 100644 index 00e5ba5..0000000 --- a/packages/scoring/src/extract-prompt.test.mjs +++ /dev/null @@ -1,65 +0,0 @@ -import test from 'node:test'; -import assert from 'node:assert/strict'; -import { parseExtractResponse } from './extract-prompt.mjs'; - -test('parses the no-learning shape', () => { - const r = parseExtractResponse('{"learning": null}'); - assert.deepEqual(r, { learning: null }); -}); - -test('parses a real learning shape', () => { - const r = parseExtractResponse( - '{"learning":{"node_path":"src/api/webhooks/","insight":"Use exponential backoff with jitter."}}', - ); - assert.deepEqual(r, { - learning: { - node_path: 'src/api/webhooks/', - insight: 'Use exponential backoff with jitter.', - }, - }); -}); - -test('strips markdown fences around the JSON', () => { - const raw = '```json\n{"learning": null}\n```'; - assert.deepEqual(parseExtractResponse(raw), { learning: null }); -}); - -test('handles a leading "Here is the JSON:" preamble', () => { - const raw = 'Here is the JSON:\n{"learning": null}'; - assert.deepEqual(parseExtractResponse(raw), { learning: null }); -}); - -test('appends trailing slash to node_path if missing', () => { - const r = parseExtractResponse( - '{"learning":{"node_path":"src/api","insight":"x"}}', - ); - assert.equal(r.learning?.node_path, 'src/api/'); -}); - -test('drops malformed JSON (returns no-learning)', () => { - assert.deepEqual(parseExtractResponse('not json'), { learning: null }); - assert.deepEqual(parseExtractResponse(''), { learning: null }); - assert.deepEqual(parseExtractResponse('{broken'), { learning: null }); -}); - -test('drops shapes missing required fields', () => { - assert.deepEqual( - parseExtractResponse('{"learning":{"node_path":"src/"}}'), - { learning: null }, - ); - assert.deepEqual( - parseExtractResponse('{"learning":{"insight":"x"}}'), - { learning: null }, - ); - assert.deepEqual( - parseExtractResponse('{"learning":{"node_path":"","insight":"x"}}'), - { learning: null }, - ); -}); - -test('drops non-string types in fields', () => { - assert.deepEqual( - parseExtractResponse('{"learning":{"node_path":42,"insight":"x"}}'), - { learning: null }, - ); -}); diff --git a/packages/scoring/src/index.ts b/packages/scoring/src/index.ts index 4f0c6cc..e009411 100644 --- a/packages/scoring/src/index.ts +++ b/packages/scoring/src/index.ts @@ -1,13 +1,7 @@ -// Re-export the runtime helpers (defined in `.mjs` so any consumer — the -// MCP server (TS) or the Stop hook (plain ESM) — can import them with no -// build step) plus the type-only declarations. +// Re-export the runtime helpers (defined in `.mjs` so any consumer, TypeScript +// or plain ESM, can import them with no build step) plus the type-only +// declarations. export { normalize } from './normalize.mjs'; -export { - EXTRACT_SYSTEM_PROMPT, - parseExtractResponse, - type ExtractedLearning, - type ExtractResult, -} from './extract-prompt.mjs'; export { SCORE_SYSTEM_PROMPT } from './score-prompt.mjs'; export { TEACH_SYSTEM_PROMPT } from './teach-prompt.mjs'; export { TOPIC_SYSTEM_PROMPT } from './topic-prompt.mjs'; @@ -35,5 +29,4 @@ export { SCORE_MODEL, TOPIC_MODEL, DIFF_MODEL, - EXTRACT_MODEL, } from './models.mjs'; diff --git a/packages/scoring/src/models.d.mts b/packages/scoring/src/models.d.mts index 8897b1b..e177595 100644 --- a/packages/scoring/src/models.d.mts +++ b/packages/scoring/src/models.d.mts @@ -1,4 +1,3 @@ export declare const SCORE_MODEL: string; export declare const TOPIC_MODEL: string; export declare const DIFF_MODEL: string; -export declare const EXTRACT_MODEL: string; diff --git a/packages/scoring/src/models.mjs b/packages/scoring/src/models.mjs index e28a07e..03c2734 100644 --- a/packages/scoring/src/models.mjs +++ b/packages/scoring/src/models.mjs @@ -7,16 +7,11 @@ // thinkingConfig still supported. Free tier // available in AI Studio. Migrated 2026-04-26 // (was gemini-2.5-flash). -// gemma-4-31b-it : ~9-16s, thinking model (cannot disable -// thinking), no responseSchema enforcement -// — JSON parsed defensively. Reserved for -// batch / async learning extraction where -// latency is tolerable. // -// /diff was originally Gemma 4 31B for richer narrative; flipped to Flash -// because compounding 4 LLM calls put /diff over budget on free-tier -// quotas. Swap back here when paid quota lands. +// /diff was originally Gemma 4 31B (gemma-4-31b-it) for richer narrative; +// flipped to Flash because compounding 4 LLM calls put /diff over budget on +// free-tier quotas. gemini.ts still handles a Gemma-family model (no schema +// enforcement, chain-of-thought parts) if one is swapped back in here. export const SCORE_MODEL = 'gemini-3-flash-preview'; export const TOPIC_MODEL = 'gemini-3-flash-preview'; export const DIFF_MODEL = 'gemini-3-flash-preview'; -export const EXTRACT_MODEL = 'gemma-4-31b-it'; From 93e92814f426a8ca4570807f5e793b76cb68a334 Mon Sep 17 00:00:00 2001 From: Bogdan Truta Date: Thu, 1 Oct 2026 12:26:01 +0000 Subject: [PATCH 03/14] chore: stop next dev writing AGENTS.md/CLAUDE.md; mark the MCP CLI executable Next 16.3 generates AGENTS.md and CLAUDE.md in apps/dashboard on every `next dev` unless `agentRules: false` is set; they showed up as untracked files for anyone running the dashboard. apps/mcp-server/bin/cli.mjs has a shebang and is the package's `bin`, but was committed as 100644 (npm fixed the mode on install, leaving a dirty tree). Co-Authored-By: Claude Opus 5.5 --- apps/dashboard/next.config.mjs | 3 +++ apps/mcp-server/bin/cli.mjs | 0 2 files changed, 3 insertions(+) mode change 100644 => 100755 apps/mcp-server/bin/cli.mjs diff --git a/apps/dashboard/next.config.mjs b/apps/dashboard/next.config.mjs index 68ac78e..bbcc21b 100644 --- a/apps/dashboard/next.config.mjs +++ b/apps/dashboard/next.config.mjs @@ -4,6 +4,9 @@ const nextConfig = { // The shared types live as raw .ts in packages/shared. Tell Next/webpack // to transpile them rather than expecting a pre-built dist. transpilePackages: ['@trailhead/shared'], + // Next 16.3 writes AGENTS.md and CLAUDE.md into the app directory on every + // `next dev` unless this is off; they are not part of the repo. + agentRules: false, }; export default nextConfig; diff --git a/apps/mcp-server/bin/cli.mjs b/apps/mcp-server/bin/cli.mjs old mode 100644 new mode 100755 From 0918f3f759aaf3fa78ed7b9f75bcb428d040e0b6 Mon Sep 17 00:00:00 2001 From: Bogdan Truta Date: Thu, 1 Oct 2026 12:26:02 +0000 Subject: [PATCH 04/14] readme: every call uses gemini-3-flash-preview; routes live in src/routes/ The README said gemma-4-31b-it handled "async learning extraction" and "diff narration, rich bootstrap". Learning extraction was never wired up (removed in the previous commits) and /diff and the bootstrap use gemini-3-flash-preview. Also points the route table at the new layout. Co-Authored-By: Claude Opus 5.5 --- README.md | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index f7ac45f..89feaaf 100644 --- a/README.md +++ b/README.md @@ -63,8 +63,9 @@ backend, all sharing the same TypeScript contract. Single source of truth. Multi-tenant: each team has a public team id and a server-minted secret (only its SHA-256 is stored), sent as `X-Team-Token`. Pre-2026-09-30 tokens derived from the git remote still work behind -`TRAILHEAD_ACCEPT_LEGACY_TOKENS` (deprecated). Routes are in -`apps/api/src/app.ts` (`index.ts` just serves them): +`TRAILHEAD_ACCEPT_LEGACY_TOKENS` (deprecated). Middleware is in +`apps/api/src/app.ts`, routes in `apps/api/src/routes/` (`index.ts` just serves +them): | Method + Path | What it does | |---|---| @@ -94,10 +95,11 @@ Pre-2026-09-30 tokens derived from the git remote still work behind | `GET /onboard/jobs/:id` | Per-path progress for a rich-bootstrap job | | `DELETE /team/data` | Wipes the requesting team's data; demo team is protected unless `TRAILHEAD_ALLOW_DEMO_RESET=true` | -LLM work runs through `apps/api/src/gemini.ts`. Model assignments live in -`packages/scoring/src/models.mjs`: `gemini-3-flash-preview` for scoring -(JSON-schema mode), topic extraction and diff narration; `gemma-4-31b-it` for -async learning extraction, where latency is tolerable. +Every LLM call runs through `apps/api/src/gemini.ts` (one client, one retry +policy, Langfuse tracing). Model assignments live in +`packages/scoring/src/models.mjs`; today every call uses +`gemini-3-flash-preview` (JSON-schema mode for scoring, topic extraction, +coaching rewrites and the rich wiki bootstrap). Every Gemini call is instrumented with **Langfuse** when `LANGFUSE_PUBLIC_KEY` / `LANGFUSE_SECRET_KEY` are set — one trace per HTTP @@ -342,7 +344,7 @@ Single root `.env.example` — every surface reads from the same set. | Var | Used by | Notes | |---|---|---| | `DATABASE_URL` | api | Postgres connection string, `sslmode=require` | -| `GEMINI_API_KEY` | api | `gemini-3-flash-preview` + `gemma-4-31b-it` | +| `GEMINI_API_KEY` | api | Used for every call (`gemini-3-flash-preview`) | | `LANGFUSE_PUBLIC_KEY` | api | Optional. Hosted Langfuse public key (`pk-lf-…`) | | `LANGFUSE_SECRET_KEY` | api | Optional. Hosted Langfuse secret key (`sk-lf-…`) | | `LANGFUSE_BASEURL` | api | Defaults to `https://cloud.langfuse.com` (EU). Use `https://us.cloud.langfuse.com` for US | @@ -437,8 +439,8 @@ contracts, builds, and tests. - **Backend:** Hono, TypeScript, Node 22, `@hono/node-server`, raw `pg` - **DB:** Postgres on Neon, no ORM -- **LLMs:** `gemini-3-flash-preview` (scoring, JSON-schema mode), `gemma-4-31b-it` - (diff narration, rich bootstrap) +- **LLMs:** `gemini-3-flash-preview` for every call (scoring in JSON-schema + mode, coaching, diff narration, rich bootstrap) - **Observability:** Langfuse (hosted) — one trace per request, one generation per LLM call - **Frontend:** Next.js 16 + Tailwind + Recharts + SWR (dashboard); vanilla From e3c7e2c89d38c21c1af8060ae813484893744d1e Mon Sep 17 00:00:00 2001 From: Bogdan Truta Date: Thu, 1 Oct 2026 12:34:18 +0000 Subject: [PATCH 05/14] scoring: a rule-based baseline scorer, an ordering metric, and an eval that runs in CI The scorer at the centre of the product (one Gemini call) had never been measured: the eval harness existed but needs a key, so nothing ran. This adds the part that can run anywhere. packages/scoring/src/heuristic-score.mjs scores a prompt on the five rubric dimensions with transparent rules over surface features (file paths, identifiers, numbers with units, constraint and output phrasing, meta instructions aimed at the scorer). It is deterministic, has no imports and no Node APIs, so the API's offline mode and the landing page can both use it. It is a floor for the model, not a substitute. The eval gains: - an ordering metric: for every pair of prompts whose expected bands don't overlap, does the scorer rank the better one higher (ties count as wrong); - `--scorer heuristic` (free, no key, no --yes); - eval/holdout.json: 16 prompts with bands, written before the rule set was frozen and never used to tune it (golden.json was visible while writing); - baseline.test.ts: runs the rule-based scorer over both sets in `npm test` and fails if it drops below what it measured when frozen. Measured (heuristic-v1): golden (seen): 29/30 overall in band, 47/49 dims, 152/152 pairs ordered holdout (unseen): 14/16 overall in band, 18/22 dims, 38/38 pairs ordered The known misses are listed in eval/README.md. The Gemini scorer still has not been run on either set; that needs a key. Co-Authored-By: Claude Opus 5.5 --- apps/api/eval/README.md | 48 +++- apps/api/eval/baseline.test.ts | 50 ++++ apps/api/eval/holdout.json | 25 ++ apps/api/eval/run.ts | 15 +- apps/api/eval/stats.test.ts | 30 ++- apps/api/eval/stats.ts | 38 +++ apps/api/package.json | 2 +- packages/scoring/package.json | 3 +- packages/scoring/src/heuristic-score.d.mts | 14 + packages/scoring/src/heuristic-score.mjs | 252 ++++++++++++++++++ packages/scoring/src/heuristic-score.test.mjs | 83 ++++++ packages/scoring/src/index.ts | 1 + 12 files changed, 553 insertions(+), 8 deletions(-) create mode 100644 apps/api/eval/baseline.test.ts create mode 100644 apps/api/eval/holdout.json create mode 100644 packages/scoring/src/heuristic-score.d.mts create mode 100644 packages/scoring/src/heuristic-score.mjs create mode 100644 packages/scoring/src/heuristic-score.test.mjs diff --git a/apps/api/eval/README.md b/apps/api/eval/README.md index e87c4a9..2516588 100644 --- a/apps/api/eval/README.md +++ b/apps/api/eval/README.md @@ -6,8 +6,8 @@ The 5-dimension score comes from one Gemini call (`scorePrompt` in much, on a fixed golden set, so decisions about sampling settings — or about trusting a single score for coaching and library promotion — rest on data. -Nothing here runs in CI, and nothing runs against the paid API unless you pass -`--yes`. +Nothing runs against the paid API unless you pass `--yes`. CI runs only the +free parts: the harness's own tests and the rule-based baseline below. ## Run it @@ -15,6 +15,10 @@ Nothing here runs in CI, and nothing runs against the paid API unless you pass # Free, offline: a fake scorer exercises the whole pipeline (numbers are meaningless). npm --workspace=apps/api run eval -- --dry-run +# Free, offline: the rule-based scorer (deterministic, so one run is enough). +npm --workspace=apps/api run eval -- --scorer heuristic --runs 1 +npm --workspace=apps/api run eval -- --scorer heuristic --runs 1 --golden eval/holdout.json + # Real: 30 prompts × 5 runs = 150 Gemini calls on your key. GEMINI_API_KEY=... npm --workspace=apps/api run eval -- --yes --runs 5 @@ -24,7 +28,8 @@ GEMINI_API_KEY=... npm --workspace=apps/api run eval -- --yes --temperature 1 GEMINI_API_KEY=... npm --workspace=apps/api run eval -- --yes --thinking-budget 512 --label b512 ``` -Flags: `--runs N` (1–50, default 5), `--only id1,id2`, `--temperature T`, +Flags: `--scorer gemini|heuristic` (default gemini), `--runs N` (1–50, +default 5), `--only id1,id2`, `--temperature T`, `--thinking-budget B` (-1 dynamic, 0 off, else tokens), `--label L`, `--out DIR` (default `eval/results/`, gitignored), `--golden FILE`, `--delay-ms MS` (pause between calls, for free-tier rate limits). @@ -45,6 +50,10 @@ Written as `-