Review follow-ups: redact bootstrap source from traces, checkable claims, stricter config, least-privilege release

.env.example

@@ -71,7 +71,8 @@ TRAILHEAD_ADMIN_TOKEN=
 # The seeded demo team (public secret trailhead_demo_acme_2026). on | off.
 # Default: on, but off whenever TRAILHEAD_ADMIN_TOKEN is set, because anyone
 # who can reach the API can write to the demo team and spend your Gemini quota
-# through it. Set `on` to keep it on a networked deploy anyway.
+# through it. Set `on` to keep it on a networked deploy anyway. Any value other
+# than on/off (or true/false) stops the API at startup.
 # TRAILHEAD_DEMO_TEAM=
 
 # How /coach prompts join the team library. Both modes require the exact

.github/workflows/release.yml

@@ -5,22 +5,28 @@ name: release
 # yet. Publishing the release, and any store listing, stays a human step.
 #
 # The tag must match the version in apps/browser-ext/manifest.json and
-# apps/vscode-ext/package.json, or the job fails before building anything.
+# apps/vscode-ext/package.json, or the build fails before packaging anything.
+# Assets are only ever (re)placed on a draft: once a release is published, a
+# re-run fails instead of swapping the files people may already have.
 
 on:
   push:
     tags: ['v*']
 
-permissions:
-  contents: write
+# Nothing by default; each job asks for what it needs.
+permissions: {}
 
 jobs:
-  extensions:
-    name: package and attach extensions
+  build:
+    name: test and package the extensions
     runs-on: ubuntu-latest
     timeout-minutes: 15
+    permissions:
+      contents: read
     steps:
       - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
       - name: versions match the tag
         run: |
           want="${GITHUB_REF_NAME#v}"
@@ -43,13 +49,41 @@ jobs:
         run: |
           npm --workspace=@trailhead/browser-ext run package
           npm --workspace=apps/vscode-ext run package
-      - name: attach to the release
+      - uses: actions/upload-artifact@v4
+        with:
+          name: release-packages
+          path: |
+            apps/browser-ext/learnloop-browser-ext-*.zip
+            apps/vscode-ext/*.vsix
+          if-no-files-found: error
+
+  release:
+    name: attach to the draft release
+    needs: build
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    # The only job that can write to the repository, and it runs no repo code.
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: release-packages
+          path: packages
+      - name: attach
         env:
           GH_TOKEN: ${{ github.token }}
+          GH_REPO: ${{ github.repository }}
         run: |
-          if ! gh release view "$GITHUB_REF_NAME" >/dev/null 2>&1; then
-            gh release create "$GITHUB_REF_NAME" --draft --verify-tag --generate-notes --title "LearnLoop $GITHUB_REF_NAME"
+          tag="$GITHUB_REF_NAME"
+          if draft=$(gh release view "$tag" --json isDraft --jq .isDraft 2>/dev/null); then
+            if [ "$draft" != "true" ]; then
+              echo "::error::release $tag is already published; not replacing its assets. Delete the assets or tag a new version."
+              exit 1
+            fi
+          else
+            gh release create "$tag" --draft --verify-tag --generate-notes --title "LearnLoop $tag"
           fi
-          gh release upload "$GITHUB_REF_NAME" --clobber \
-            apps/browser-ext/learnloop-browser-ext-*.zip \
-            apps/vscode-ext/*.vsix
+          # Still a draft here, so replacing a file from an earlier run is safe.
+          find packages -type f \( -name '*.zip' -o -name '*.vsix' \) -print0 \
+            | xargs -0 gh release upload "$tag" --clobber

README.md

@@ -8,7 +8,7 @@ back into the next person's context. A team's way of prompting compounds
 without anyone writing docs.
 
 - **Try the scorer in your browser:** <https://learnloop-gules.vercel.app/#try>
-- **3-minute walkthrough:** <https://www.youtube.com/watch?v=kD6nnJAmRK8>
+- **Video walkthrough:** <https://www.youtube.com/watch?v=kD6nnJAmRK8>
 - **Run the whole stack, no API key needed:** [Try it](#try-it) below
 
 Built at PoliHack v19 (April 2026, BMW track "Applications that encourage AI
@@ -107,13 +107,14 @@ spread, how often the coaching and library gates flip, band hits, and whether
 better prompts are ranked above worse ones.
 
 What has been measured so far is the **rule-based scorer**
-(`packages/scoring/src/heuristic-score.mjs`), the floor the model has to
-beat. CI checks it on every change:
+(`packages/scoring/src/heuristic-score.mjs`), a transparent baseline that a
+model scorer should beat; the two have not been compared yet. CI checks it on
+every change:
 
 | Prompt set | Overall in expected band | Pairs ranked the right way round |
 |---|---|---|
 | golden (30, visible while its rules were written) | 29/30 | 152/152 |
-| held-out (16, never tuned on) | 14/16 | 38/38 |
+| held-out (16; per its author, written before the rules were frozen and not tuned on, but added in the same commit, so this can't be checked from history) | 14/16 | 38/38 |
 
 The bands were written in this repo, not by independent reviewers, and the
 Gemini scorer has not been run on either set yet (it needs a key; one command

SELFHOSTING.md

@@ -41,9 +41,10 @@ curl http://localhost:3000
 # {"name":"trailhead-api","status":"ok",...}
 ```
 
-If `GEMINI_API_KEY` is missing (and `TRAILHEAD_LLM` is not `offline`), the API
-refuses to start and logs what to set; Compose keeps restarting it until you
-fix `.env`. It never quietly falls back to offline mode.
+If `GEMINI_API_KEY` is missing (and `TRAILHEAD_LLM` is not `offline`),
+`docker compose up` stops before the API starts: the one-shot `config-check`
+service prints what to set and exits 1. The API checks the same thing at
+startup. It never quietly falls back to offline mode.
 
 ### Try it without a key (offline mode)
 
@@ -61,8 +62,8 @@ Drop `--profile demo` for an empty stack.
 
 - `/score` and `/coach` use the rule-based scorer in
   `packages/scoring/src/heuristic-score.mjs`. It reads surface features (file
-  paths, identifiers, constraint and output phrasing), so it is a reasonable
-  floor, not a substitute for the model; `apps/api/eval/README.md` has its
+  paths, identifiers, constraint and output phrasing), so it is a transparent
+  baseline, not a substitute for the model; `apps/api/eval/README.md` has its
   numbers and known misses. Every response carries `"scorer": "heuristic"`,
   and coaching text ends with a note saying no model was involved.
 - Coaching uses the static per-dimension templates (plus your team's library
@@ -300,7 +301,8 @@ The defaults are safe for a local setup because both ports are bound to
   rotation, but not from writes or from spending your quota. It is **off by
   default once `TRAILHEAD_ADMIN_TOKEN` is set**: its secret then gets
   `401 demo_team_disabled`. `TRAILHEAD_DEMO_TEAM=on|off` overrides that either
-  way, and `GET /` reports the current setting as `demo_team`.
+  way (any other value stops the API at startup), and `GET /` reports the
+  current setting as `demo_team`.
 
 **Rate limits.** The API limits the calls that cost you something, with
 in-process token buckets (a limit of `N/W` allows a burst of N and refills at N
@@ -376,8 +378,11 @@ Where prompts go: every scored prompt, any wiki context attached to it, and —
 for the default rich `bootstrap` — the first 8,000 characters of up to 500
 source files (5 levels deep) are sent to Google's Gemini API
 using your `GEMINI_API_KEY` (in offline mode nothing is sent to Google). When
-`LANGFUSE_*` keys are set, the same model
-inputs and outputs are also sent to Langfuse. The browser extension's 👍/🤷/👎
+`LANGFUSE_*` keys are set, every model call is also traced in Langfuse: for
+scoring, coaching, `/improve` and `/diff` that includes the prompt, any team
+context and the model's answer. Rich-bootstrap calls are the exception: their
+traces record only sizes, model settings and token usage, never the source
+files or the summaries generated from them. The browser extension's 👍/🤷/👎
 chips store the prompt *and Claude's full reply* in the `captures` table.
 `bootstrap` picks files by extension (so `.env` and key files are never read)
 and skips anything git ignores (`git check-ignore`: nested `.gitignore`s,
@@ -389,9 +394,10 @@ uploaded.
 
 ## Troubleshooting
 
-**The API restarts in a loop and logs "GEMINI_API_KEY not set"**
-You skipped `cp .env.example .env`, or left the key blank. `.env` must be at the
-repo root, next to `docker-compose.yml`. To try the stack without a key, set
+**`docker compose up` stops with `service "config-check" didn't complete successfully`**
+Its log says `GEMINI_API_KEY is not set`: you skipped `cp .env.example .env`,
+or left the key blank. `.env` must be at the repo root, next to
+`docker-compose.yml`. To try the stack without a key, set
 `TRAILHEAD_LLM=offline` instead.
 
 **API restarts in a loop / `DATABASE_URL not set`**

apps/api/eval/README.md

@@ -91,7 +91,8 @@ repetition loops `src/gemini.ts` already guards against. Suggested procedure:
 `packages/scoring/src/heuristic-score.mjs` implements the rubric as transparent
 rules over surface features: file paths, identifiers, numbers with units,
 constraint and output phrasing. It is what `TRAILHEAD_LLM=offline` and the
-landing-page demo use, and it is the floor a model scorer should beat.
+landing-page demo use, and a baseline a model scorer should beat (the Gemini
+scorer has not been run against it yet).
 `baseline.test.ts` runs it over both prompt sets in CI and fails if it gets
 worse.
 
@@ -100,9 +101,12 @@ Measured when its rules were frozen (2026-10-01, `heuristic-v1`):
 | Set | Overall in band | Dimensions in band | Ranked the right way round |
 |---|---|---|---|
 | `golden.json` (30 prompts, **visible while the rules were written**) | 29/30 | 47/49 | 152/152 pairs |
-| `holdout.json` (16 prompts, written before the rules were frozen, **never tuned on**) | 14/16 | 18/22 | 38/38 pairs |
+| `holdout.json` (16 prompts, **author-attested** as written before the rules were frozen and not tuned on) | 14/16 | 18/22 | 38/38 pairs |
 
-The holdout is the honest number. Both sets, bands included, were written in
+The holdout is the more honest number, with a caveat: `holdout.json` and
+`heuristic-score.mjs` were added in the same commit (e3c7e2c), so the
+separation rests on the author's word; git history can't show it. A later
+change to the rules is checkable: the set must not be edited alongside it. Both sets, bands included, were written in
 this repo rather than labelled by independent reviewers, so treat them as a
 regression check, not a measure of agreement with people. Known misses, all
 consistent with a scorer that only reads surface features:

apps/api/eval/baseline.test.ts

@@ -2,11 +2,12 @@
 // golden set and the held-out set, in CI, with no key and no network.
 //
 // It is the scorer behind TRAILHEAD_LLM=offline and the landing-page demo, and
-// the floor a model scorer has to beat. The thresholds sit just under what it
+// a baseline a model scorer should beat. The thresholds sit just under what it
 // measured when its rules were frozen (see eval/README.md → Baseline), so a
 // rule change that makes it worse fails here. The golden set was visible while
-// the rules were written; the held-out set was not, which is why its
-// thresholds are lower and why it is the more honest of the two numbers.
+// the rules were written; the held-out set, by its author's account, was not
+// (author-attested: both landed in one commit), which is why its thresholds
+// are lower.
 
 import test from 'node:test';
 import assert from 'node:assert/strict';
@@ -34,7 +35,7 @@ test('rule-based scorer on the golden set (seen while writing the rules)', () =>
   assert.ok(t.ordering.rate >= 0.97, `ordering ${t.ordering.agree}/${t.ordering.pairs}`);
 });
 
-test('rule-based scorer on the held-out set (written before the rules were frozen, never tuned on)', () => {
+test('rule-based scorer on the held-out set (author-attested: not used to tune the rules)', () => {
   const t = evaluate('holdout.json');
   assert.ok(t.overallBandHit >= 0.8, `overall band hit ${t.overallBandHit}`);
   assert.ok(t.dimBandHit >= 0.75, `dimension band hit ${t.dimBandHit}`);

apps/api/eval/holdout.json

@@ -1,5 +1,5 @@
 {
-  "_about": "Held-out prompts for the rule-based scorer (packages/scoring/src/heuristic-score.mjs). Written, with their bands, before that scorer's rules were frozen, and never used to tune them; golden.json was visible while the rules were written. Same format and band semantics as golden.json, so the eval harness can run either (`--golden eval/holdout.json`).",
+  "_about": "Held-out prompts for the rule-based scorer (packages/scoring/src/heuristic-score.mjs). Per its author, written with their bands before that scorer's rules were frozen and not used to tune them (author-attested: this file and the scorer were added in the same commit, e3c7e2c). golden.json was visible while the rules were written. Same format and band semantics as golden.json, so the eval harness can run either (`--golden eval/holdout.json`).",
   "prompts": [
     { "id": "ho-vague-slow", "prompt": "the app is slow, can you do something about it", "overall": [0, 2] },
     { "id": "ho-vague-refactor", "prompt": "refactor this", "overall": [0, 2] },

apps/api/package.json

@@ -9,7 +9,7 @@
     "dev": "tsx watch src/index.ts",
     "start": "tsx src/index.ts",
     "typecheck": "tsc --noEmit",
-    "test": "tsx --test src/gemini.test.ts src/coach-degraded.test.ts src/wiki-export.test.ts src/request-params.test.ts src/team-auth.test.ts src/promotion-gate.test.ts src/rate-limit.test.ts src/llm-mode.test.ts src/offline-llm.test.ts eval/stats.test.ts eval/baseline.test.ts",
+    "test": "tsx --test src/gemini.test.ts src/coach-degraded.test.ts src/wiki-export.test.ts src/request-params.test.ts src/team-auth.test.ts src/promotion-gate.test.ts src/rate-limit.test.ts src/llm-mode.test.ts src/demo-team.test.ts src/offline-llm.test.ts eval/stats.test.ts eval/baseline.test.ts",
     "test:integration": "tsx --test test/integration.test.ts",
     "eval": "tsx eval/run.ts"
   },

apps/api/src/demo-team.test.ts

@@ -0,0 +1,22 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { demoTeamConfigError, demoTeamEnabled } from './demo-team.ts';
+
+test('on by default, off once an admin token is set, an explicit value wins', () => {
+  assert.equal(demoTeamEnabled({}), true);
+  assert.equal(demoTeamEnabled({ TRAILHEAD_ADMIN_TOKEN: 'x' }), false);
+  assert.equal(demoTeamEnabled({ TRAILHEAD_ADMIN_TOKEN: 'x', TRAILHEAD_DEMO_TEAM: 'on' }), true);
+  assert.equal(demoTeamEnabled({ TRAILHEAD_DEMO_TEAM: 'off' }), false);
+  assert.equal(demoTeamEnabled({ TRAILHEAD_DEMO_TEAM: 'true' }), true);
+  assert.equal(demoTeamEnabled({ TRAILHEAD_DEMO_TEAM: 'false' }), false);
+});
+
+test('unknown values are a startup error, and fail closed if they slip through', () => {
+  for (const v of ['0', '1', 'no', 'yes', 'OFF', 'disabled']) {
+    assert.match(demoTeamConfigError({ TRAILHEAD_DEMO_TEAM: v })!, /must be "on" or "off"/, v);
+    assert.equal(demoTeamEnabled({ TRAILHEAD_DEMO_TEAM: v }), false, v);
+  }
+  for (const v of [undefined, '', 'on', 'off', 'true', 'false']) {
+    assert.equal(demoTeamConfigError({ TRAILHEAD_DEMO_TEAM: v }), null, String(v));
+  }
+});

apps/api/src/demo-team.ts

@@ -0,0 +1,26 @@
+// TRAILHEAD_DEMO_TEAM: whether the seeded public demo team can authenticate.
+//
+//   on | true     always
+//   off | false   never (its public secret gets 401 demo_team_disabled)
+//   unset / ''    on, unless TRAILHEAD_ADMIN_TOKEN is set — an admin token is
+//                 the sign of a networked deploy, where anyone could write to
+//                 the demo team and spend the operator's Gemini quota
+//
+// Anything else is a configuration error: index.ts refuses to start, rather
+// than guessing what "0" or "no" meant. Read per call, so tests can flip it.
+
+const ON = new Set(['on', 'true']);
+const OFF = new Set(['off', 'false']);
+
+export function demoTeamConfigError(env: NodeJS.ProcessEnv = process.env): string | null {
+  const v = env.TRAILHEAD_DEMO_TEAM;
+  if (v === undefined || v === '' || ON.has(v) || OFF.has(v)) return null;
+  return `TRAILHEAD_DEMO_TEAM must be "on" or "off" (or unset for the default); got "${v}"`;
+}
+
+export function demoTeamEnabled(env: NodeJS.ProcessEnv = process.env): boolean {
+  const v = env.TRAILHEAD_DEMO_TEAM;
+  if (v === undefined || v === '') return !env.TRAILHEAD_ADMIN_TOKEN;
+  // An unknown value is rejected at startup; should one appear later, fail closed.
+  return ON.has(v);
+}

apps/api/src/gemini.test.ts

@@ -266,3 +266,44 @@ test('generateText sends one request through the shared client and does not cap
     stub.restore();
   }
 });
+
+// Rich-bootstrap prompts carry the team's source files. With Langfuse on they
+// must not be copied into traces: only sizes and usage.
+test('generateText keeps the prompt and answer out of the Langfuse trace', async () => {
+  const { withTrace } = await import('./langfuse.ts');
+  const generations: { start: Record<string, unknown>; end?: Record<string, unknown> }[] = [];
+  const fakeTrace = {
+    generation(start: Record<string, unknown>) {
+      const g: { start: Record<string, unknown>; end?: Record<string, unknown> } = { start };
+      generations.push(g);
+      return { end: (e: Record<string, unknown>) => { g.end = e; } };
+    },
+  };
+  const SECRET_SOURCE = 'const apiKey = "sk-source-line-that-must-not-be-traced";';
+  const stub = withFetchStub(() => geminiTextResponse(JSON.stringify({ narrative_md: `quotes ${SECRET_SOURCE}` })));
+  try {
+    // withTrace takes a LangfuseTraceClient; only generation() is used here.
+    await withTrace(fakeTrace as never, () =>
+      generateText({ model: 'gemini-3-flash-preview', contents: `=== src/a.ts ===\n${SECRET_SOURCE}` }, 'file(src/a.ts)'),
+    );
+    assert.equal(generations.length, 1);
+    const traced = JSON.stringify(generations[0]);
+    assert.ok(!traced.includes('sk-source-line'), traced);
+    assert.match(traced, /"chars":\d+/);
+  } finally {
+    stub.restore();
+  }
+});
+
+test('request-path calls still trace their prompt (only bootstrap is redacted)', async () => {
+  const { withTrace } = await import('./langfuse.ts');
+  const starts: Record<string, unknown>[] = [];
+  const fakeTrace = { generation(start: Record<string, unknown>) { starts.push(start); return { end() {} }; } };
+  const stub = withFetchStub(() => geminiTextResponse(WELL_FORMED_SCORE));
+  try {
+    await withTrace(fakeTrace as never, () => scorePrompt({ prompt: 'fix the retry logic' }));
+    assert.match(JSON.stringify(starts[0]), /fix the retry logic/);
+  } finally {
+    stub.restore();
+  }
+});