From 0a6bbe314315f8b34e41e9b99ac8e8a38de6f9e5 Mon Sep 17 00:00:00 2001 From: Ryan Swift Date: Tue, 3 Nov 2026 11:12:53 -0500 Subject: [PATCH] feat(launch-control): add the Launch Control travel desk challenge A 10-minute browser challenge on LaunchDarkly AgentControl: a travel booker on a local page climbs four variations of one agent-mode config (prompt, tools, model) while it keeps running, switched one session target at a time by the package's switch_travel_agent MCP tool. Demo mode runs the same flow against a local stand-in that serves ladder.json; provision.mjs creates the config in a LaunchDarkly project. Closes #43 --- README.md | 1 + challenges/launch-control/README.md | 160 ++++ challenges/launch-control/challenge.json | 61 ++ challenges/launch-control/cleanup | 5 + challenges/launch-control/icon.txt | 4 + challenges/launch-control/install | 8 + challenges/launch-control/intro.md | 18 + challenges/launch-control/inventory.json | 33 + challenges/launch-control/ladder.json | 82 ++ challenges/launch-control/launchdarkly.json | 6 + challenges/launch-control/mcp.json | 10 + challenges/launch-control/mcp/server.mjs | 131 +++ challenges/launch-control/mcp/switch.mjs | 94 ++ challenges/launch-control/monitor/SKILL.md | 31 + challenges/launch-control/package-lock.json | 179 ++++ challenges/launch-control/package.json | 9 + challenges/launch-control/page/app.mjs | 198 +++++ challenges/launch-control/page/index.html | 53 ++ challenges/launch-control/page/styles.css | 303 +++++++ challenges/launch-control/plugin.json | 6 + challenges/launch-control/provision.mjs | 159 ++++ .../launch-control/server/booker-config.mjs | 121 +++ challenges/launch-control/server/booker.mjs | 90 ++ challenges/launch-control/server/cleanup.mjs | 177 ++++ challenges/launch-control/server/gemini.mjs | 62 ++ challenges/launch-control/server/http.mjs | 33 + challenges/launch-control/server/ladder.mjs | 114 +++ .../server/launchdarkly-api.mjs | 208 +++++ .../launch-control/server/page-server.mjs | 212 +++++ challenges/launch-control/server/paths.mjs | 29 + .../launch-control/server/runtime-config.mjs | 71 ++ challenges/launch-control/server/serve.mjs | 134 +++ challenges/launch-control/server/session.mjs | 102 +++ challenges/launch-control/server/settings.mjs | 179 ++++ challenges/launch-control/server/setup.mjs | 146 ++++ challenges/launch-control/server/stand-in.mjs | 226 +++++ .../launch-control/server/travel-desk.mjs | 175 ++++ .../launch-control/server/workspace-file.mjs | 195 +++++ challenges/launch-control/setup | 6 + .../skills/start-challenge/SKILL.md | 78 ++ test/launch-control.test.mjs | 807 ++++++++++++++++++ 41 files changed, 4716 insertions(+) create mode 100644 challenges/launch-control/README.md create mode 100644 challenges/launch-control/challenge.json create mode 100755 challenges/launch-control/cleanup create mode 100644 challenges/launch-control/icon.txt create mode 100755 challenges/launch-control/install create mode 100644 challenges/launch-control/intro.md create mode 100644 challenges/launch-control/inventory.json create mode 100644 challenges/launch-control/ladder.json create mode 100644 challenges/launch-control/launchdarkly.json create mode 100644 challenges/launch-control/mcp.json create mode 100644 challenges/launch-control/mcp/server.mjs create mode 100644 challenges/launch-control/mcp/switch.mjs create mode 100644 challenges/launch-control/monitor/SKILL.md create mode 100644 challenges/launch-control/package-lock.json create mode 100644 challenges/launch-control/package.json create mode 100644 challenges/launch-control/page/app.mjs create mode 100644 challenges/launch-control/page/index.html create mode 100644 challenges/launch-control/page/styles.css create mode 100644 challenges/launch-control/plugin.json create mode 100644 challenges/launch-control/provision.mjs create mode 100644 challenges/launch-control/server/booker-config.mjs create mode 100644 challenges/launch-control/server/booker.mjs create mode 100644 challenges/launch-control/server/cleanup.mjs create mode 100644 challenges/launch-control/server/gemini.mjs create mode 100644 challenges/launch-control/server/http.mjs create mode 100644 challenges/launch-control/server/ladder.mjs create mode 100644 challenges/launch-control/server/launchdarkly-api.mjs create mode 100644 challenges/launch-control/server/page-server.mjs create mode 100644 challenges/launch-control/server/paths.mjs create mode 100644 challenges/launch-control/server/runtime-config.mjs create mode 100644 challenges/launch-control/server/serve.mjs create mode 100644 challenges/launch-control/server/session.mjs create mode 100644 challenges/launch-control/server/settings.mjs create mode 100644 challenges/launch-control/server/setup.mjs create mode 100644 challenges/launch-control/server/stand-in.mjs create mode 100644 challenges/launch-control/server/travel-desk.mjs create mode 100644 challenges/launch-control/server/workspace-file.mjs create mode 100755 challenges/launch-control/setup create mode 100644 challenges/launch-control/skills/start-challenge/SKILL.md create mode 100644 test/launch-control.test.mjs diff --git a/README.md b/README.md index 9a3afb6..17cc725 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ links to the package's own README. | [vibe-coding-mad-libs](challenges/vibe-coding-mad-libs/README.md) | Fill four blanks, and Pi builds the web app you described. | | [birdsbirdbirds](challenges/birdsbirdbirds/README.md) | Draw a bird and watch it fly across the sky. | | [glitch-invaders](challenges/glitch-invaders/README.md) | Shoot the bugs, then debug the three that fight back using Sentry evidence. | +| [launch-control](challenges/launch-control/README.md) | Upgrade a live AI agent's prompt, tools, and model with no redeploy. | Challenges follow the contract in boothstrap.sh's challenge spec and authoring guide; agents working here start from [AGENTS.md](AGENTS.md). diff --git a/challenges/launch-control/README.md b/challenges/launch-control/README.md new file mode 100644 index 0000000..60f55ad --- /dev/null +++ b/challenges/launch-control/README.md @@ -0,0 +1,160 @@ +# Launch Control: Travel Desk + +A 10-minute browser challenge built on LaunchDarkly AgentControl. The +participant chats with a travel booker, a small AI agent on a local page, and +asks it to plan the same trip at each of four rungs. Pi, the guide, upgrades +the booker's config one rung at a time, and the booker gets better with no +restart and no deploy. The lesson: you can change an AI agent's model, tools, +and instructions in production without redeploying. + +| Rung | Variation | What it adds | Model | +| --- | --- | --- | --- | +| v1 | `bare` | A one-line prompt, no tools | `gemini-2.5-flash-lite` | +| v2 | `prompted` | A real travel-agent prompt | `gemini-2.5-flash-lite` | +| v3 | `tooled` | `search_flights`, `search_hotels`, `book_trip` | `gemini-2.5-flash-lite` | +| v4 | `upgraded` | A stronger model | `gemini-3.8-flash` | + +The **ladder** is these four variations of one AgentControl config, in +[ladder.json](ladder.json); a **rung** is one of them. The **travel booker** +is the agent on the page; **the guide** is Pi in the terminal. Only the +booker is under LaunchDarkly's control. + +## Files + +| File | Purpose | +| --- | --- | +| `challenge.json`, `plugin.json` | Manifest and Agent Plugin metadata | +| `mcp.json`, `mcp/` | The `travel-desk` MCP server and its one tool, `switch_travel_agent` | +| `ladder.json` | The four variations: model, instructions, and tool schemas | +| `inventory.json` | The made-up cities, flights, and hotels the tools search | +| `launchdarkly.json` | Public settings for live mode: project, environment, and config keys | +| `provision.mjs` | Creates or updates the config in a LaunchDarkly project from `ladder.json` | +| `intro.md`, `icon.txt` | Intro screen text and tile icon | +| `skills/start-challenge/SKILL.md` | The guide Pi follows | +| `monitor/SKILL.md` | Headless browser monitor skill | +| `install` | Installs LaunchDarkly's SDKs into the package's `node_modules` | +| `setup`, `cleanup` | Start and stop the page server (`server/setup.mjs`, `server/cleanup.mjs`) | +| `server/` | Page server on `127.0.0.1:4374`, the booker's agent loop, and the demo stand-in | +| `page/` | The travel desk page: chat and Live config card | + +## How it works + +The page server resolves the AgentControl config for the context +`{ kind: "user", key: BOOTHSTRAP_SESSION_ID }` with `@launchdarkly/ai-server` +over a streaming SDK client, with analytics and diagnostic events off. It runs +the booker's agent loop on whatever the SDK returns: that model, through the +vault's `gemini` service, those instructions, and only those tools, against +the fake inventory. `book_trip` writes `itinerary.json` to the workspace and +nothing else. No real travel API, payment, or personal data is involved. + +`switch_travel_agent({ rung })` changes only this session's individual target +on the config, with an `addTargets` and `removeTargets` semantic patch. It +accepts only the next rung, or `1` to roll back, and never touches the +default rule or another context key. The SDK sees the change in about a +second, and the page's config card updates in place. + +`setup` removes any target this session left behind and confirms that the +session resolves to v1. `cleanup` removes the session's target; a session +with no target is left alone, so it is safe to run twice and after a crash. + +## How steps advance + +The page keeps a hidden line, `booker-state v1 rung= tried= +booked= mode=`, and the headless monitor reports +`meet-booker`, `add-prompt`, `add-tools`, and `add-model` from it, as +[monitor/SKILL.md](monitor/SKILL.md) describes. A rung counts as tried only +once the booker has answered a message sent while it was live, and `booked` +turns `yes` only when v4 books a trip, so a lucky booking on a lower rung +never completes `add-model`. Each rung starts the booker on a fresh model +conversation, so it answers the request on its own, while the page keeps the +whole chat. Once all four +are tried, Pi asks what changed and why nothing was redeployed, and reports +`debrief`. The page server also writes the same state to +`travel-desk-state.json` for a standalone run. + +The server listens on port 4374. Set `LAUNCH_CONTROL_PORT` to use another +port, though the manifest and the monitor expect 4374. + +## Demo and live modes + +Live mode needs both LaunchDarkly keys and real settings. Otherwise the +challenge runs in demo mode: the identical flow against a local stand-in for +LaunchDarkly, mounted on the page server, that serves `ladder.json`. The page +then shows a small `demo` label. The booker calls Gemini in both modes. + +The settings live in [launchdarkly.json](launchdarkly.json) as +`` values. `LAUNCHDARKLY_PROJECT_KEY`, +`LAUNCHDARKLY_ENVIRONMENT_KEY`, and `LAUNCHDARKLY_AI_CONFIG_KEY` in the +environment replace them, so an event can set them in its `challenge_env`. + +## Event vault services + +The event vault needs these three services. Each credential is the variable +the package reads; none starts with `LD_`. + +| Service | Auth | Credential | Host | Read by | +| --- | --- | --- | --- | --- | +| `gemini` | `api-key`, header `x-goog-api-key` | `GEMINI_API_KEY` | `generativelanguage.googleapis.com` | The page server and Pi | +| `launchdarkly-sdk` | `api-key`, header `Authorization` | `LAUNCHDARKLY_SDK_API_KEY` | `stream.launchdarkly.com` | The page server only | +| `launchdarkly-api` | `api-key`, header `Authorization` | `LAUNCHDARKLY_API_AUTH_TOKEN` | `app.launchdarkly.com` | `setup`, `cleanup`, and the MCP tool | + +- `launchdarkly-sdk` holds the environment's server-side SDK key (`sdk-…`). + With events off, the SDK needs only the streaming host. +- `launchdarkly-api` holds an access token that can write the config's + targeting. It is `api-key`, not `bearer`, because LaunchDarkly's REST API + takes the bare token in `Authorization` and refuses `Bearer api-…`. +- The page server never receives the API token: `setup` leaves it out of the + server's environment. + +## Provision a LaunchDarkly project + +Run this once per LaunchDarkly project, with an access token that can write AI +Configs, from the package directory: + +```sh +LAUNCHDARKLY_API_AUTH_TOKEN=api-... \ +LAUNCHDARKLY_PROJECT_KEY= \ +LAUNCHDARKLY_ENVIRONMENT_KEY= \ +LAUNCHDARKLY_AI_CONFIG_KEY=launch-control-travel-booker \ +node provision.mjs +``` + +It creates or updates the three tools, the agent-mode config, its four +variations, and a default rule serving `bare` in that environment. A second +run creates nothing twice. Then fill the same three keys into +`launchdarkly.json`. + +## Run without boothstrap + +Needs Node 26 and a `GEMINI_API_KEY`; for live mode, export the two +LaunchDarkly keys and settings too. Follow the standalone steps in +boothstrap.sh's `docs/authoring-challenges.md` to install Pi and +pi-mcp-adapter and load the package as an Agent Plugin, which mounts its +`travel-desk` MCP server. Then: + +```sh +challenge=/absolute/path/to/launch-control +"$challenge"/install +mkdir /tmp/my-launch-control-run && cd /tmp/my-launch-control-run +"$challenge"/setup +``` + +Open `http://127.0.0.1:4374/`, then start Pi in the same folder: + +```sh +pi --model google/gemini-3.8-flash \ + --skill "$challenge"/skills/start-challenge/SKILL.md \ + '/skill:start-challenge' +``` + +When done: + +```sh +"$challenge"/cleanup +``` + +`cleanup` stops the server, removes the session's target in live mode, and +removes `.launch-control/`, `travel-desk-state.json`, and `itinerary.json`. +Without boothstrap there is no `report_step` or `session_info`, so no monitor +runs: the skill reads `travel-desk-state.json` and acknowledges finished steps +in plain words. diff --git a/challenges/launch-control/challenge.json b/challenges/launch-control/challenge.json new file mode 100644 index 0000000..84cde68 --- /dev/null +++ b/challenges/launch-control/challenge.json @@ -0,0 +1,61 @@ +{ + "id": "launch-control", + "title": "Launch Control: Travel Desk", + "tagline": "Upgrade a live AI agent's prompt, tools, and model with no redeploy.", + "icon": "icon.txt", + "type": "browser", + "agent": "pi", + "model": "google/gemini-3.8-flash", + "vault_services": [ + "gemini", + "launchdarkly-sdk", + "launchdarkly-api" + ], + "points": 150, + "estimated_minutes": 10, + "learning_objectives": [ + "configuring an AI agent at runtime", + "changing a model, prompt, and tools with no deploy", + "rolling an agent back instantly" + ], + "featured_tech": [ + "LaunchDarkly" + ], + "intro": "intro.md", + "steps": [ + { + "id": "meet-booker", + "title": "Meet the travel booker", + "hint": "Tap the suggested trip, or write your own, and send it to the booker.", + "done_when": "The page's accessibility snapshot holds the line \"booker-state v1 …\" whose tried= list contains 1." + }, + { + "id": "add-prompt", + "title": "Give it a real prompt", + "hint": "Let Pi switch the booker to v2, then send your trip again.", + "done_when": "The page's accessibility snapshot holds the line \"booker-state v1 …\" whose tried= list contains 2." + }, + { + "id": "add-tools", + "title": "Give it tools", + "hint": "Let Pi switch the booker to v3, then send your trip again.", + "done_when": "The page's accessibility snapshot holds the line \"booker-state v1 …\" whose tried= list contains 3." + }, + { + "id": "add-model", + "title": "Upgrade its model", + "hint": "Let Pi switch the booker to v4, then send your trip again until it's booked.", + "done_when": "The page's accessibility snapshot holds the line \"booker-state v1 …\" whose tried= list contains 4 and which contains \"booked=yes\"." + }, + { + "id": "debrief", + "title": "Tell your guide what changed", + "hint": "Say what changed at each rung, and why nothing was redeployed.", + "done_when": "All four rungs are tried and the participant has told the guide, in their own words, what changed and why no deploy was needed." + } + ], + "browser": { + "start_url": "http://127.0.0.1:4374/", + "monitor_skill": "monitor/SKILL.md" + } +} diff --git a/challenges/launch-control/cleanup b/challenges/launch-control/cleanup new file mode 100755 index 0000000..d627f96 --- /dev/null +++ b/challenges/launch-control/cleanup @@ -0,0 +1,5 @@ +#!/bin/sh +# Cleanup hook: stop the page server, remove this session's target from the +# booker's config, and remove this attempt's files. Safe to run twice. +set -eu +exec node "${BOOTHSTRAP_PACKAGE_DIR:-$(dirname "$0")}/server/cleanup.mjs" diff --git a/challenges/launch-control/icon.txt b/challenges/launch-control/icon.txt new file mode 100644 index 0000000..4171f0d --- /dev/null +++ b/challenges/launch-control/icon.txt @@ -0,0 +1,4 @@ + ▲ + ▐█▌ + ▄▐█▌▄ + ▀▀▀ diff --git a/challenges/launch-control/install b/challenges/launch-control/install new file mode 100755 index 0000000..e23e234 --- /dev/null +++ b/challenges/launch-control/install @@ -0,0 +1,8 @@ +#!/bin/sh +# Installs LaunchDarkly's AI SDK and server SDK, which the page server uses to +# read the booker's config, into this package's node_modules. +set -eu + +# pwd -P resolves symlinks: npm ci fails on a --prefix that isn't a real path. +package_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P) +npm ci --ignore-scripts --no-audit --no-fund --prefix "$package_dir" diff --git a/challenges/launch-control/intro.md b/challenges/launch-control/intro.md new file mode 100644 index 0000000..4db0ef1 --- /dev/null +++ b/challenges/launch-control/intro.md @@ -0,0 +1,18 @@ +You run the travel desk, and its booking agent needs work. Ask it to plan +a trip, then watch it get better, live, while it keeps running. + +The booker reads its whole setup from LaunchDarkly AgentControl: which +model it runs, what its instructions say, and which tools it may use. Pi, +your guide on the right, changes that setup one rung at a time: + +- v1: a small model and a one-line prompt +- v2: a real travel-agent prompt +- v3: tools to search flights and hotels, and to book +- v4: a stronger model + +After each switch, send your trip again and compare. The Live config card +shows what LaunchDarkly serves the booker right now. Nothing restarts and +nothing is redeployed. + +The travel desk is in Chrome on the left. Its flights and hotels are made +up, and nothing is ever charged. diff --git a/challenges/launch-control/inventory.json b/challenges/launch-control/inventory.json new file mode 100644 index 0000000..8d26f50 --- /dev/null +++ b/challenges/launch-control/inventory.json @@ -0,0 +1,33 @@ +{ + "$comment": "The travel desk's made-up inventory: no real airline, hotel, or price. Dates are month-day only, so a search matches them in any year. One trip fits the suggested request (JFK to Lisbon, March 12 to 16, waterfront, under $900): NV310, NV311, and Alfama Riverside Guesthouse, $818 for four nights.", + "airports": { + "JFK": "New York", + "LIS": "Lisbon", + "OPO": "Porto", + "MAD": "Madrid" + }, + "flights": [ + { "id": "NV308", "airline": "Novair", "origin": "JFK", "destination": "LIS", "date": "03-11", "departs": "07:10", "arrives": "18:55", "stops": 0, "price_usd": 179 }, + { "id": "AT112", "airline": "Atlantica", "origin": "JFK", "destination": "LIS", "date": "03-12", "departs": "08:05", "arrives": "19:50", "stops": 0, "price_usd": 305 }, + { "id": "SK240", "airline": "Skyline", "origin": "JFK", "destination": "LIS", "date": "03-12", "departs": "06:40", "arrives": "22:15", "stops": 1, "price_usd": 268 }, + { "id": "NV310", "airline": "Novair", "origin": "JFK", "destination": "LIS", "date": "03-12", "departs": "07:10", "arrives": "18:55", "stops": 0, "price_usd": 214 }, + { "id": "AT114", "airline": "Atlantica", "origin": "JFK", "destination": "LIS", "date": "03-13", "departs": "08:05", "arrives": "19:50", "stops": 0, "price_usd": 288 }, + { "id": "AT113", "airline": "Atlantica", "origin": "LIS", "destination": "JFK", "date": "03-16", "departs": "11:30", "arrives": "14:20", "stops": 0, "price_usd": 322 }, + { "id": "SK241", "airline": "Skyline", "origin": "LIS", "destination": "JFK", "date": "03-16", "departs": "09:15", "arrives": "17:40", "stops": 1, "price_usd": 281 }, + { "id": "NV311", "airline": "Novair", "origin": "LIS", "destination": "JFK", "date": "03-16", "departs": "13:05", "arrives": "15:50", "stops": 0, "price_usd": 236 }, + { "id": "NV313", "airline": "Novair", "origin": "LIS", "destination": "JFK", "date": "03-17", "departs": "13:05", "arrives": "15:50", "stops": 0, "price_usd": 165 }, + { "id": "AT220", "airline": "Atlantica", "origin": "JFK", "destination": "OPO", "date": "03-12", "departs": "09:00", "arrives": "20:30", "stops": 0, "price_usd": 331 }, + { "id": "AT221", "airline": "Atlantica", "origin": "OPO", "destination": "JFK", "date": "03-16", "departs": "12:00", "arrives": "15:10", "stops": 0, "price_usd": 344 }, + { "id": "SK330", "airline": "Skyline", "origin": "JFK", "destination": "MAD", "date": "03-12", "departs": "18:20", "arrives": "23:55", "stops": 1, "price_usd": 296 }, + { "id": "SK331", "airline": "Skyline", "origin": "MAD", "destination": "JFK", "date": "03-16", "departs": "10:45", "arrives": "16:30", "stops": 1, "price_usd": 301 } + ], + "hotels": [ + { "id": "LIS-ALFAMA", "name": "Alfama Riverside Guesthouse", "city": "Lisbon", "neighborhood": "Alfama", "waterfront": true, "nightly_usd": 92 }, + { "id": "LIS-CAIS", "name": "Cais do Sodré Harbor Hotel", "city": "Lisbon", "neighborhood": "Cais do Sodré", "waterfront": true, "nightly_usd": 139 }, + { "id": "LIS-BELEM", "name": "Belém Marina Suites", "city": "Lisbon", "neighborhood": "Belém", "waterfront": true, "nightly_usd": 210 }, + { "id": "LIS-BAIRRO", "name": "Bairro Alto Budget Rooms", "city": "Lisbon", "neighborhood": "Bairro Alto", "waterfront": false, "nightly_usd": 58 }, + { "id": "LIS-AVENIDA", "name": "Avenida Business Inn", "city": "Lisbon", "neighborhood": "Avenidas Novas", "waterfront": false, "nightly_usd": 110 }, + { "id": "OPO-RIBEIRA", "name": "Ribeira Quay Hotel", "city": "Porto", "neighborhood": "Ribeira", "waterfront": true, "nightly_usd": 118 }, + { "id": "MAD-SOL", "name": "Puerta del Sol Rooms", "city": "Madrid", "neighborhood": "Sol", "waterfront": false, "nightly_usd": 97 } + ] +} diff --git a/challenges/launch-control/ladder.json b/challenges/launch-control/ladder.json new file mode 100644 index 0000000..926cc80 --- /dev/null +++ b/challenges/launch-control/ladder.json @@ -0,0 +1,82 @@ +{ + "$comment": "The travel booker's four rungs: one LaunchDarkly AgentControl config, four variations. provision.mjs writes them to LaunchDarkly, and demo mode's local stand-in serves them, so both modes read this one file. Rungs climb in order; each one adds a single thing.", + "provider": "Gemini", + "tools": { + "search_flights": { + "description": "Search the travel desk's flight inventory for one-way flights between two airports on one date. Returns each flight's id, times, stops, and price in US dollars.", + "parameters": { + "type": "object", + "properties": { + "origin": { "type": "string", "description": "IATA code of the departure airport, such as JFK." }, + "destination": { "type": "string", "description": "IATA code of the arrival airport, such as LIS." }, + "date": { "type": "string", "description": "Departure date, YYYY-MM-DD." } + }, + "required": ["origin", "destination", "date"] + } + }, + "search_hotels": { + "description": "Search the travel desk's hotel inventory in one city. Returns each hotel's id, neighborhood, whether it is on the waterfront, and its nightly price in US dollars.", + "parameters": { + "type": "object", + "properties": { + "city": { "type": "string", "description": "City name, such as Lisbon." }, + "check_in": { "type": "string", "description": "Check-in date, YYYY-MM-DD." }, + "check_out": { "type": "string", "description": "Check-out date, YYYY-MM-DD." } + }, + "required": ["city", "check_in", "check_out"] + } + }, + "book_trip": { + "description": "Book a whole trip: an outbound flight, a return flight, and a hotel stay, using ids from search_flights and search_hotels. Returns a confirmation with the total price.", + "parameters": { + "type": "object", + "properties": { + "outbound_flight_id": { "type": "string" }, + "return_flight_id": { "type": "string" }, + "hotel_id": { "type": "string" }, + "check_in": { "type": "string", "description": "YYYY-MM-DD" }, + "check_out": { "type": "string", "description": "YYYY-MM-DD" } + }, + "required": ["outbound_flight_id", "return_flight_id", "hotel_id", "check_in", "check_out"] + } + } + }, + "rungs": [ + { + "rung": 1, + "key": "bare", + "name": "v1 · bare", + "adds": "A small model and a one-line prompt.", + "model": "gemini-2.5-flash-lite", + "instructions": "You are a travel booking assistant.", + "tools": [] + }, + { + "rung": 2, + "key": "prompted", + "name": "v2 · prompted", + "adds": "A real travel-agent prompt.", + "model": "gemini-2.5-flash-lite", + "instructions": "You are the booking agent at Launch Control Travel Desk.\n\nThe traveler flies from New York (JFK). Plan trips that match every part of their request: dates, places, budget, and preferences.\n\nRules:\n- Never invent flights, hotels, prices, or confirmation numbers. Only quote what your tools return.\n- If you have no tool to look something up or to book, say so plainly, and say what you would need.\n- Ask a short question only when the request is missing something you truly need.\n- Keep replies short: a few sentences or a short list.", + "tools": [] + }, + { + "rung": 3, + "key": "tooled", + "name": "v3 · tooled", + "adds": "Tools to search flights and hotels, and to book a trip.", + "model": "gemini-2.5-flash-lite", + "instructions": "You are the booking agent at Launch Control Travel Desk.\n\nThe traveler flies from New York (JFK). Plan trips that match every part of their request: dates, places, budget, and preferences.\n\nRules:\n- Never invent flights, hotels, prices, or confirmation numbers. Only quote what your tools return.\n- Use search_flights and search_hotels to find options, and book_trip to book once you have a trip that fits.\n- Ask a short question only when the request is missing something you truly need.\n- Keep replies short: a few sentences or a short list.", + "tools": ["search_flights", "search_hotels", "book_trip"] + }, + { + "rung": 4, + "key": "upgraded", + "name": "v4 · upgraded", + "adds": "A stronger model.", + "model": "gemini-3.8-flash", + "instructions": "You are the booking agent at Launch Control Travel Desk.\n\nThe traveler flies from New York (JFK). Plan trips that match every part of their request: dates, places, budget, and preferences.\n\nRules:\n- Never invent flights, hotels, prices, or confirmation numbers. Only quote what your tools return.\n- Use search_flights and search_hotels to find options, and book_trip to book once you have a trip that fits.\n- Ask a short question only when the request is missing something you truly need.\n- Keep replies short: a few sentences or a short list.", + "tools": ["search_flights", "search_hotels", "book_trip"] + } + ] +} diff --git a/challenges/launch-control/launchdarkly.json b/challenges/launch-control/launchdarkly.json new file mode 100644 index 0000000..1f0f2ed --- /dev/null +++ b/challenges/launch-control/launchdarkly.json @@ -0,0 +1,6 @@ +{ + "$comment": "Public settings for live LaunchDarkly mode; none is a secret. MLH fills these in from its LaunchDarkly project after running provision.mjs. While any value is still a , the challenge runs in demo mode against a local stand-in. LAUNCHDARKLY_PROJECT_KEY, LAUNCHDARKLY_ENVIRONMENT_KEY, and LAUNCHDARKLY_AI_CONFIG_KEY in the environment replace these. Live mode also needs LAUNCHDARKLY_SDK_API_KEY and LAUNCHDARKLY_API_AUTH_TOKEN, which the station's vault provides; never put a key here.", + "projectKey": "", + "environmentKey": "", + "configKey": "" +} diff --git a/challenges/launch-control/mcp.json b/challenges/launch-control/mcp.json new file mode 100644 index 0000000..1b7ebad --- /dev/null +++ b/challenges/launch-control/mcp.json @@ -0,0 +1,10 @@ +{ + "$schema": "https://agent-plugins.org/schemas/1.0.0/mcp.schema.json", + "mcpServers": { + "travel-desk": { + "type": "stdio", + "command": "node", + "args": ["${PLUGIN_ROOT}/mcp/server.mjs"] + } + } +} diff --git a/challenges/launch-control/mcp/server.mjs b/challenges/launch-control/mcp/server.mjs new file mode 100644 index 0000000..9a2340f --- /dev/null +++ b/challenges/launch-control/mcp/server.mjs @@ -0,0 +1,131 @@ +/** + * The package's MCP server, `travel-desk`: one tool, `switch_travel_agent`, over stdio JSON-RPC + * (one message per line). It reads the attempt's runtime config from the workspace and the + * LaunchDarkly API token from its environment, which it passes only to the REST client and never + * prints. In demo mode the token is unused and the calls go to the page server's stand-in. + */ +import { createInterface } from 'node:readline'; +import { LaunchDarklyApi, LaunchDarklyApiError } from '../server/launchdarkly-api.mjs'; +import { loadLadder } from '../server/ladder.mjs'; +import { PACKAGE_DIR, resolveStatePaths } from '../server/paths.mjs'; +import { readRuntimeConfig } from '../server/runtime-config.mjs'; +import { API_TOKEN_VARIABLE } from '../server/settings.mjs'; +import { SwitchRefusedError, switchTravelAgent } from './switch.mjs'; + +/** The MCP protocol version this server speaks when the client names none. */ +const DEFAULT_PROTOCOL_VERSION = '2025-06-18'; + +/** The ladder the tool climbs. */ +const LADDER = loadLadder(PACKAGE_DIR); + +/** The tool's definition, as `tools/list` returns it, with each rung's addition from the ladder. */ +const SWITCH_TOOL = { + name: 'switch_travel_agent', + description: [ + "Switch this session's travel booker to another rung of its LaunchDarkly AgentControl config, with no restart or deploy.", + 'Accepts only the next rung, or 1 to roll back.', + ...LADDER.rungs.map((rung) => `Rung ${rung.rung} (${rung.name}): ${rung.adds}`), + ].join(' '), + inputSchema: { + type: 'object', + properties: { rung: { type: 'integer', minimum: 1, maximum: LADDER.rungs.length, description: 'The rung to switch to.' } }, + required: ['rung'], + additionalProperties: false, + }, +}; + +/** Write one JSON-RPC message. */ +function send(message) { + process.stdout.write(`${JSON.stringify({ jsonrpc: '2.0', ...message })}\n`); +} + +/** A tool result holding `text`, marked as an error when `isError`. */ +function toolResult(text, isError = false) { + return { content: [{ type: 'text', text }], ...(isError ? { isError: true } : {}) }; +} + +/** Run `switch_travel_agent` for this workspace. */ +async function callSwitch(args) { + // pi-mcp-adapter starts this server in the package directory, not Pi's, but passes Pi's + // environment, so standalone, Pi's working directory arrives only as PWD. + const paths = resolveStatePaths(process.env, process.env.PWD ?? process.cwd()); + const runtimeConfig = await readRuntimeConfig(paths.runtimeConfigFile); + if (runtimeConfig === null) { + return toolResult('Launch Control is not set up in this workspace. Run the package setup hook first.', true); + } + + const { launchDarkly } = runtimeConfig; + const api = new LaunchDarklyApi({ + origin: launchDarkly.apiOrigin, + token: runtimeConfig.mode === 'live' ? (process.env[API_TOKEN_VARIABLE] ?? '') : 'demo', + projectKey: launchDarkly.projectKey, + environmentKey: launchDarkly.environmentKey, + configKey: launchDarkly.configKey, + }); + try { + const text = await switchTravelAgent( + { ladder: LADDER, api, sessionKey: runtimeConfig.sessionKey, pagePort: runtimeConfig.pagePort }, + args?.rung, + ); + + return toolResult(text); + } catch (error) { + if (error instanceof SwitchRefusedError || error instanceof LaunchDarklyApiError) { + return toolResult(error.message, true); + } + throw error; + } +} + +/** Answer one JSON-RPC request; notifications get no answer. */ +async function handle(message) { + const { id, method, params } = message; + /** Answer the request with `result`; a notification has no id and gets no answer. */ + const reply = (result) => id !== undefined && send({ id, result }); + switch (method) { + case 'initialize': + reply({ + protocolVersion: params?.protocolVersion ?? DEFAULT_PROTOCOL_VERSION, + capabilities: { tools: {} }, + serverInfo: { name: 'travel-desk', version: '1.0.0' }, + }); + break; + case 'ping': + reply({}); + break; + case 'tools/list': + reply({ tools: [SWITCH_TOOL] }); + break; + case 'tools/call': + if (params?.name !== SWITCH_TOOL.name) { + reply(toolResult(`There is no tool named ${params?.name}.`, true)); + break; + } + reply(await callSwitch(params.arguments)); + break; + default: + if (id !== undefined) { + send({ id, error: { code: -32601, message: `method not found: ${method}` } }); + } + } +} + +const lines = createInterface({ input: process.stdin }); +lines.on('line', (line) => { + if (line.trim() === '') { + return; + } + let message; + try { + message = JSON.parse(line); + } catch { + send({ id: null, error: { code: -32700, message: 'parse error' } }); + + return; + } + handle(message).catch(() => { + if (message.id !== undefined) { + send({ id: message.id, error: { code: -32603, message: 'the travel desk tool hit an error' } }); + } + }); +}); diff --git a/challenges/launch-control/mcp/switch.mjs b/challenges/launch-control/mcp/switch.mjs new file mode 100644 index 0000000..419b3b0 --- /dev/null +++ b/challenges/launch-control/mcp/switch.mjs @@ -0,0 +1,94 @@ +/** + * `switch_travel_agent`: move this session's travel booker one rung up the ladder, or back to v1. + * It changes only this session's individual target on the booker's config, through the REST API, + * and then waits for the page to show the new rung, so the guide can point at the config card. + */ +import { sleep } from '../server/http.mjs'; +import { rungByKey, rungByNumber } from '../server/ladder.mjs'; + +/** How long the tool waits for the page to show the new rung. */ +const PAGE_WAIT_MS = 6_000; + +/** How often it checks the page meanwhile. */ +const PAGE_POLL_MS = 150; + +/** A switch the tool refuses; the message says what to do instead. */ +export class SwitchRefusedError extends Error { + constructor(message) { + super(message); + this.name = 'SwitchRefusedError'; + } +} + +/** The rung the page server shows now, or `null` when it does not answer. */ +async function pageRung(pagePort) { + try { + const response = await fetch(`http://127.0.0.1:${pagePort}/api/state`, { signal: AbortSignal.timeout(1_000) }); + const state = await response.json(); + + return state.config?.rung ?? null; + } catch { + return null; + } +} + +/** + * Decide whether moving from `current` to `requested` is allowed. + * + * @returns {ReturnType} the requested rung. + * @throws {SwitchRefusedError} for a skip, a step down other than to v1, the current rung, or a + * rung the ladder does not have, saying which rung is next. + */ +export function checkSwitch(ladder, current, requested) { + const target = Number.isInteger(requested) ? rungByNumber(ladder, requested) : undefined; + const last = ladder.rungs.length; + const next = rungByNumber(ladder, current + 1); + const nextHint = next === undefined ? `v${current} is the top rung; the only switch left is 1, the rollback.` : `The next rung is ${next.rung}: ${next.adds} Call switch_travel_agent with rung ${next.rung}.`; + if (target === undefined) { + throw new SwitchRefusedError(`rung must be a whole number from 1 to ${last}. ${nextHint}`); + } + if (target.rung === current) { + throw new SwitchRefusedError(`The booker is already on ${target.name}. ${nextHint}`); + } + if (target.rung !== current + 1 && target.rung !== 1) { + throw new SwitchRefusedError(`Rungs go one at a time, so ${target.name} is not next. ${nextHint}`); + } + + return target; +} + +/** + * Switch this session's booker to `requested`. + * + * @param {{ ladder: object, api: import('../server/launchdarkly-api.mjs').LaunchDarklyApi, + * sessionKey: string, pagePort: number }} context + * @returns {Promise} what changed, for the guide to relay. + * @throws {SwitchRefusedError} for a switch the ladder does not allow. + * @throws {import('../server/launchdarkly-api.mjs').LaunchDarklyApiError} when LaunchDarkly fails. + */ +export async function switchTravelAgent(context, requested) { + const { targetOf } = await context.api.readTargeting(); + const currentKey = targetOf(context.sessionKey); + const targeted = currentKey === null ? rungByNumber(context.ladder, 1) : rungByKey(context.ladder, currentKey); + if (targeted === undefined) { + throw new SwitchRefusedError(`This session is targeted to ${currentKey}, which is not on the ladder; run provision.mjs.`); + } + const current = targeted.rung; + const target = checkSwitch(context.ladder, current, requested); + + await context.api.targetSession(context.sessionKey, target.key); + + const deadline = Date.now() + PAGE_WAIT_MS; + let shown = await pageRung(context.pagePort); + while (shown !== target.rung && Date.now() < deadline) { + await sleep(PAGE_POLL_MS); + shown = await pageRung(context.pagePort); + } + const tools = target.tools.length === 0 ? 'no tools' : `tools ${target.tools.join(', ')}`; + const page = + shown === target.rung + ? 'The config card on the page shows it now, with no reload and no deploy.' + : 'LaunchDarkly has the change; the page will show it within a few seconds.'; + + return `Switched this session's travel booker to ${target.name}: ${target.model}, ${tools}. ${page}`; +} diff --git a/challenges/launch-control/monitor/SKILL.md b/challenges/launch-control/monitor/SKILL.md new file mode 100644 index 0000000..d8d53db --- /dev/null +++ b/challenges/launch-control/monitor/SKILL.md @@ -0,0 +1,31 @@ +--- +name: monitor +description: "Headless monitor for Launch Control: Travel Desk. Reports steps from the page's booker-state line, visually hidden but present in the page's accessibility snapshot." +--- + +# Launch Control monitor + +Take one snapshot of the travel desk page. If the snapshot is not +`http://127.0.0.1:4374/`, call `list_pages`, `select_page` the +`http://127.0.0.1:4374/` page, and take one snapshot of it. + +Find the line that starts with `booker-state v1`. It is visually hidden, so +look for it in the snapshot, not on screen. It is one line of fixed words, for +example: + +```text +booker-state v1 rung=3 tried=1,2,3 booked=no mode=demo +``` + +`tried=` is a comma-separated list of rung numbers, or `none`. Report a step +only when the line says exactly this: + +- `tried=` lists `1`: call `report_step` with `meet-booker`. +- `tried=` lists `2`: call `report_step` with `add-prompt`. +- `tried=` lists `3`: call `report_step` with `add-tools`. +- `tried=` lists `4` and the line contains `booked=yes`: call `report_step` + with `add-model`. + +`rung=` alone never completes a step. Never report `debrief`. If the page has +no `booker-state` line, is blank, or you are not sure, report nothing. Do not +guess from the chat or other text on the page. diff --git a/challenges/launch-control/package-lock.json b/challenges/launch-control/package-lock.json new file mode 100644 index 0000000..928e5a3 --- /dev/null +++ b/challenges/launch-control/package-lock.json @@ -0,0 +1,179 @@ +{ + "name": "launch-control", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "launch-control", + "dependencies": { + "@launchdarkly/ai-server": "1.0.0", + "@launchdarkly/node-server-sdk": "9.16.0" + } + }, + "node_modules/@launchdarkly/ai-server": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@launchdarkly/ai-server/-/ai-server-1.0.0.tgz", + "integrity": "sha512-6R+QQe7P99Zqab8pk9Ghnmb4E028pXB/s5EE2SNDo/tbHziT0ChBAl+GD9+SBBqY2u+GkhAYoNj7HLLzeqqFmg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api": "^1.9.1" + }, + "peerDependencies": { + "@launchdarkly/node-server-sdk": "^9.7.3", + "@opentelemetry/context-async-hooks": "^2.8.0", + "@opentelemetry/core": "^2.8.0", + "@opentelemetry/exporter-trace-otlp-http": "^0.219.0", + "@opentelemetry/otlp-exporter-base": "^0.219.0", + "@opentelemetry/resources": "^2.8.0", + "@opentelemetry/sdk-trace-base": "^2.8.0", + "@opentelemetry/sdk-trace-node": "^2.8.0" + }, + "peerDependenciesMeta": { + "@launchdarkly/node-server-sdk": { + "optional": true + }, + "@opentelemetry/context-async-hooks": { + "optional": true + }, + "@opentelemetry/core": { + "optional": true + }, + "@opentelemetry/exporter-trace-otlp-http": { + "optional": true + }, + "@opentelemetry/otlp-exporter-base": { + "optional": true + }, + "@opentelemetry/resources": { + "optional": true + }, + "@opentelemetry/sdk-trace-base": { + "optional": true + }, + "@opentelemetry/sdk-trace-node": { + "optional": true + } + } + }, + "node_modules/@launchdarkly/eventsource": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/@launchdarkly/eventsource/-/eventsource-0.2.2.tgz", + "integrity": "sha512-HeM/SWPrgHwdLytYXhRMNvDB7XKWYY3xSFeo9nAZB3Ep8GhDKOC7uefKuAAKjlxRHyvtQpnm/UHgw3EBBRxNnA==", + "license": "Apache-2.0" + }, + "node_modules/@launchdarkly/js-sdk-common": { + "version": "2.32.0", + "resolved": "https://registry.npmjs.org/@launchdarkly/js-sdk-common/-/js-sdk-common-2.32.0.tgz", + "integrity": "sha512-V2sd9W1ocskNtZNx2gmgym+j4YsmsawZE4kwUj631dO9SE6029SLCNY9wl3csS/8/Pe8qTxZv7pvbxh4mIWJqw==", + "license": "Apache-2.0" + }, + "node_modules/@launchdarkly/js-server-sdk-common": { + "version": "2.26.0", + "resolved": "https://registry.npmjs.org/@launchdarkly/js-server-sdk-common/-/js-server-sdk-common-2.26.0.tgz", + "integrity": "sha512-FqvrDgy2/dibITqjXz4mY9y6Jksj2ZQ6IoL98VOd7a5WPI2RW51iH39Ra6jVGENR8Vs1Tv0klCBiC3LwJIm9mQ==", + "license": "Apache-2.0", + "dependencies": { + "@launchdarkly/js-sdk-common": "2.32.0", + "semver": "7.5.4" + } + }, + "node_modules/@launchdarkly/node-server-sdk": { + "version": "9.16.0", + "resolved": "https://registry.npmjs.org/@launchdarkly/node-server-sdk/-/node-server-sdk-9.16.0.tgz", + "integrity": "sha512-djgpihWJy/WqZjEKqTvZq8agi+ed6uBbKo18Z8RYDswuSKobweX4q7NVe1PjB6wjbxmsxGbi/wlsPhnrrZi0GQ==", + "license": "Apache-2.0", + "dependencies": { + "@launchdarkly/eventsource": "0.2.2", + "@launchdarkly/js-server-sdk-common": "2.26.0", + "https-proxy-agent": "^7.0.6" + } + }, + "node_modules/@opentelemetry/api": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.10.0.tgz", + "integrity": "sha512-lMkyes8iHrG+mk6WH7X70MdCLI2RS0FwNBY1PyQ8UemaijE2m5mv5A0Niy3HtbYTFrgkUEQjFMXmSAk2hQDuPQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "license": "ISC", + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/semver": { + "version": "7.5.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", + "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==", + "license": "ISC", + "dependencies": { + "lru-cache": "^6.0.0" + }, + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "license": "ISC" + } + } +} diff --git a/challenges/launch-control/package.json b/challenges/launch-control/package.json new file mode 100644 index 0000000..7d1837d --- /dev/null +++ b/challenges/launch-control/package.json @@ -0,0 +1,9 @@ +{ + "name": "launch-control", + "private": true, + "type": "module", + "dependencies": { + "@launchdarkly/ai-server": "1.0.0", + "@launchdarkly/node-server-sdk": "9.16.0" + } +} diff --git a/challenges/launch-control/page/app.mjs b/challenges/launch-control/page/app.mjs new file mode 100644 index 0000000..88346c7 --- /dev/null +++ b/challenges/launch-control/page/app.mjs @@ -0,0 +1,198 @@ +/** + * The travel desk page: a chat with the booker, and a card showing the config LaunchDarkly serves + * this session. It polls the page server for the config, so a switch shows in place with no + * reload, and mirrors the server's state line into the hidden page state signal the monitor reads. + * Model output is only ever set as text, never parsed as HTML. + */ + +/** How often the page asks for the live config. */ +const POLL_MS = 1_000; + +/** The ladder's rungs, `{ rung, name }`, as the server last sent them. */ +let rungs = []; + +/** The name the ladder gives rung `number`, such as `v2 · prompted`. */ +function rungName(number) { + return rungs.find((candidate) => candidate.rung === number)?.name ?? `v${number}`; +} + +/** Look up an element by id. */ +const byId = (id) => document.getElementById(id); + +/** Create an element with a class and text. */ +function element(tag, className, text) { + const created = document.createElement(tag); + if (className !== undefined) { + created.className = className; + } + if (text !== undefined) { + created.textContent = text; + } + + return created; +} + +/** A model reply as plain text, with Markdown's bold markers removed. */ +function plainText(text) { + return text.replace(/\*\*/g, ''); +} + +/** A one-line summary of a booking for the chat and the card. */ +function bookingSummary(booking) { + return `${booking.confirmation}: ${booking.outbound.airline} ${booking.outbound.id} and ${booking.return.id}, ${booking.hotel.name}, ${booking.nights} nights, $${booking.total_usd} total.`; +} + +/** Draw one chat message. */ +function renderMessage(message) { + const item = element('li', `message message-${message.role}`); + if (message.role === 'booker') { + item.append(element('span', 'rung-tag', `${message.name} · ${message.model}`)); + } + item.append(element('p', 'message-text', plainText(message.text))); + if (message.toolCalls?.length > 0) { + const calls = element('ul', 'tool-calls'); + for (const call of message.toolCalls) { + calls.append(element('li', 'tool-call', call)); + } + item.append(calls); + } + if (message.booking !== undefined) { + item.append(element('p', 'booking-line', `Booked ${bookingSummary(message.booking)}`)); + } + + return item; +} + +/** Redraw the whole chat, with a divider wherever the rung changed between replies. */ +function renderMessages(messages) { + const list = byId('messages'); + const items = []; + let lastRung = null; + for (const message of messages) { + if (message.role === 'participant' && message.rung !== null && message.rung !== lastRung) { + items.push(element('li', 'divider', `Now on ${rungName(message.rung)}`)); + lastRung = message.rung; + } + items.push(renderMessage(message)); + } + list.replaceChildren(...items); + list.lastElementChild?.scrollIntoView({ block: 'end' }); +} + +/** Draw the ladder strip: each rung, marking the live one and the ones tried. */ +function renderLadder(state) { + const items = state.rungs.map(({ rung, name }) => { + const live = state.config?.rung === rung; + const tried = state.tried.includes(rung); + const item = element('li', `rung${live ? ' rung-live' : ''}${tried ? ' rung-tried' : ''}`, name); + if (live) { + item.setAttribute('aria-current', 'true'); + } + + return item; + }); + byId('ladder').replaceChildren(...items); +} + +/** The variation last drawn, so a change can flash the card. */ +let shownVariation; + +/** Draw the live config card, the booking, the demo label, and the state signal. */ +function renderState(state) { + rungs = state.rungs; + byId('mode-label').hidden = state.mode !== 'demo'; + renderLadder(state); + const config = state.config; + byId('config-variation').textContent = config?.name ?? 'No config from LaunchDarkly'; + byId('config-model').textContent = config?.model ?? '—'; + byId('config-prompt').textContent = config?.promptExcerpt ?? '—'; + byId('config-tools').textContent = config === null ? '—' : config.tools.length === 0 ? 'none' : config.tools.join(', '); + if (shownVariation !== undefined && shownVariation !== config?.variationKey) { + const card = byId('config-card'); + card.classList.remove('config-changed'); + void card.offsetWidth; + card.classList.add('config-changed'); + } + shownVariation = config?.variationKey; + + byId('booking').hidden = state.booking === null; + if (state.booking !== null) { + byId('booking-summary').textContent = bookingSummary(state.booking); + } + const signal = byId('boothstrap-state'); + if (signal.textContent !== state.stateLine) { + signal.textContent = state.stateLine; + } +} + +/** Fetch the live state and draw it; a failed poll leaves the page as it was. */ +async function refreshState() { + try { + const response = await fetch('/api/state'); + if (response.ok) { + renderState(await response.json()); + } + } catch { + return; + } +} + +/** Send the composer's message and draw the answer. */ +async function sendMessage(event) { + event.preventDefault(); + const input = byId('message'); + const message = input.value.trim(); + if (message === '') { + return; + } + + byId('send').disabled = true; + byId('thinking').hidden = false; + try { + const response = await fetch('/api/chat', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ message }), + }); + if (response.ok) { + const state = await response.json(); + input.value = ''; + renderMessages(state.messages); + renderState(state); + } else { + renderMessages([...(await loadHistory()), { role: 'error', text: await response.text(), rung: null }]); + } + } finally { + byId('send').disabled = false; + byId('thinking').hidden = true; + } +} + +/** The chat so far, from the server, so a reload keeps it. */ +async function loadHistory() { + const response = await fetch('/api/history'); + + return response.ok ? (await response.json()).messages : []; +} + +/** Wire up the page. */ +async function start() { + byId('composer').addEventListener('submit', sendMessage); + byId('suggestion').addEventListener('click', () => { + const input = byId('message'); + input.value = byId('suggestion').textContent.trim(); + input.focus(); + }); + byId('message').addEventListener('keydown', (event) => { + if (event.key === 'Enter' && !event.shiftKey) { + event.preventDefault(); + byId('composer').requestSubmit(); + } + }); + + await refreshState(); + renderMessages(await loadHistory()); + setInterval(refreshState, POLL_MS); +} + +start(); diff --git a/challenges/launch-control/page/index.html b/challenges/launch-control/page/index.html new file mode 100644 index 0000000..28b5836 --- /dev/null +++ b/challenges/launch-control/page/index.html @@ -0,0 +1,53 @@ + + + + + + Launch Control Travel Desk + + + + +
+

Launch Control Travel Desk

+ +
+ +
+
+
    + +
    + +
    + + + +
    +
    +
    + + +
    + + + + diff --git a/challenges/launch-control/page/styles.css b/challenges/launch-control/page/styles.css new file mode 100644 index 0000000..e4951ab --- /dev/null +++ b/challenges/launch-control/page/styles.css @@ -0,0 +1,303 @@ +:root { + color-scheme: dark; + --bg: #0d1117; + --panel: #161b22; + --line: #30363d; + --text: #e6edf3; + --muted: #8b949e; + --accent: #58a6ff; + --success: #3fb950; + --warn: #d29922; + --error: #f85149; + font-family: system-ui, -apple-system, "Segoe UI", sans-serif; + font-size: 17px; +} + +* { + box-sizing: border-box; +} + +body { + margin: 0; + height: 100vh; + display: flex; + flex-direction: column; + background: var(--bg); + color: var(--text); +} + +.visually-hidden { + position: absolute; + width: 1px; + height: 1px; + overflow: hidden; + clip-path: inset(50%); + white-space: nowrap; +} + +.masthead { + display: flex; + align-items: center; + gap: 0.75rem; + padding: 0.75rem 1.25rem; + border-bottom: 1px solid var(--line); +} + +.masthead h1 { + margin: 0; + font-size: 1.3rem; +} + +.masthead-sub { + color: var(--muted); + font-weight: 400; +} + +.mode-label { + padding: 0.1rem 0.5rem; + border: 1px solid var(--warn); + border-radius: 0.3rem; + color: var(--warn); + font-size: 0.75rem; + text-transform: uppercase; + letter-spacing: 0.05em; +} + +.desk { + flex: 1; + min-height: 0; + display: grid; + grid-template-columns: minmax(0, 3fr) minmax(16rem, 2fr); +} + +.chat { + display: flex; + flex-direction: column; + min-height: 0; + border-right: 1px solid var(--line); +} + +.messages { + flex: 1; + overflow-y: auto; + margin: 0; + padding: 1rem 1.25rem; + list-style: none; +} + +.message { + max-width: 90%; + margin-bottom: 0.75rem; + padding: 0.6rem 0.8rem; + border-radius: 0.6rem; + background: var(--panel); +} + +.message-participant { + margin-left: auto; + background: #1f3a5f; +} + +.message-error { + border: 1px solid var(--error); + color: var(--error); +} + +.message-text { + margin: 0; + white-space: pre-wrap; + overflow-wrap: anywhere; +} + +.rung-tag { + display: inline-block; + margin-bottom: 0.3rem; + color: var(--accent); + font-size: 0.75rem; + font-weight: 600; +} + +.tool-calls { + margin: 0.4rem 0 0; + padding: 0; + list-style: none; +} + +.tool-call { + color: var(--muted); + font-family: ui-monospace, Menlo, monospace; + font-size: 0.75rem; +} + +.tool-call::before { + content: "⚙ "; +} + +.booking-line { + margin: 0.4rem 0 0; + color: var(--success); + font-weight: 600; +} + +.divider { + margin: 1rem 0; + color: var(--accent); + font-size: 0.8rem; + text-align: center; + text-transform: uppercase; + letter-spacing: 0.06em; +} + +.thinking { + margin: 0; + padding: 0 1.25rem 0.5rem; + color: var(--muted); +} + +.composer { + padding: 0.75rem 1.25rem 1rem; + border-top: 1px solid var(--line); +} + +.suggestion { + display: block; + width: 100%; + margin-bottom: 0.5rem; + padding: 0.5rem 0.75rem; + border: 1px dashed var(--accent); + border-radius: 0.5rem; + background: transparent; + color: var(--accent); + font: inherit; + text-align: left; + cursor: pointer; +} + +.composer-row { + display: flex; + gap: 0.5rem; +} + +textarea { + flex: 1; + padding: 0.5rem; + border: 1px solid var(--line); + border-radius: 0.5rem; + background: var(--panel); + color: var(--text); + font: inherit; + resize: none; +} + +#send { + padding: 0 1.1rem; + border: 0; + border-radius: 0.5rem; + background: var(--accent); + color: #0d1117; + font: inherit; + font-weight: 600; + cursor: pointer; +} + +#send:disabled { + opacity: 0.5; + cursor: wait; +} + +.config { + overflow-y: auto; + padding: 1rem 1.25rem; +} + +.config h2 { + margin: 0; + font-size: 1.1rem; +} + +.config-source { + margin: 0.2rem 0 1rem; + color: var(--muted); + font-size: 0.85rem; +} + +.ladder { + display: grid; + grid-template-columns: repeat(4, 1fr); + gap: 0.3rem; + margin: 0 0 1rem; + padding: 0; + list-style: none; +} + +.rung { + padding: 0.35rem 0.2rem; + border: 1px solid var(--line); + border-radius: 0.4rem; + color: var(--muted); + font-size: 0.7rem; + text-align: center; +} + +.rung-tried::after { + content: " ✓"; + color: var(--success); +} + +.rung-live { + border-color: var(--accent); + color: var(--text); + font-weight: 600; +} + +.config-card { + margin: 0; + padding: 0.75rem; + border: 1px solid var(--line); + border-radius: 0.6rem; + background: var(--panel); +} + +.config-card dt { + color: var(--muted); + font-size: 0.75rem; + text-transform: uppercase; + letter-spacing: 0.05em; +} + +.config-card dd { + margin: 0.1rem 0 0.7rem; + overflow-wrap: anywhere; +} + +.config-prompt { + font-size: 0.85rem; +} + +.config-changed { + animation: config-flash 1.5s ease-out; +} + +@keyframes config-flash { + from { + border-color: var(--accent); + box-shadow: 0 0 0 3px rgb(88 166 255 / 45%); + } +} + +.booking { + margin-top: 1rem; + padding: 0.75rem; + border: 1px solid var(--success); + border-radius: 0.6rem; +} + +.booking h3 { + margin: 0 0 0.3rem; + color: var(--success); + font-size: 0.95rem; +} + +.booking p { + margin: 0; +} diff --git a/challenges/launch-control/plugin.json b/challenges/launch-control/plugin.json new file mode 100644 index 0000000..ed39952 --- /dev/null +++ b/challenges/launch-control/plugin.json @@ -0,0 +1,6 @@ +{ + "$schema": "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json", + "name": "launch-control", + "version": "1.0.0", + "description": "Launch Control: Travel Desk. Climb a travel-booking agent up four LaunchDarkly AgentControl variations, live, with no redeploy." +} diff --git a/challenges/launch-control/provision.mjs b/challenges/launch-control/provision.mjs new file mode 100644 index 0000000..ff6b0c9 --- /dev/null +++ b/challenges/launch-control/provision.mjs @@ -0,0 +1,159 @@ +#!/usr/bin/env node +/** + * Provision the travel booker's AgentControl config in a LaunchDarkly project from `ladder.json`: + * the three tools, the agent-mode config, its four variations, and a default rule serving v1. A + * human runs it once per LaunchDarkly project, with an access token that can write AI Configs. + * It is idempotent: a second run updates what changed and creates nothing twice. + * + * Usage: LAUNCHDARKLY_API_AUTH_TOKEN=api-… node provision.mjs, with the project, environment, and + * config keys in `launchdarkly.json` or LAUNCHDARKLY_PROJECT_KEY, LAUNCHDARKLY_ENVIRONMENT_KEY, and + * LAUNCHDARKLY_AI_CONFIG_KEY. + */ +import { loadLadder, modelConfigKey, rungByNumber } from './server/ladder.mjs'; +import { PACKAGE_DIR } from './server/paths.mjs'; +import { LaunchDarklyApiError, requestLaunchDarkly, SEMANTIC_PATCH_TYPE } from './server/launchdarkly-api.mjs'; +import { API_TOKEN_VARIABLE, launchDarklyOrigins, readSettings } from './server/settings.mjs'; + +/** The config's display name in LaunchDarkly. */ +const CONFIG_NAME = 'Launch Control travel booker'; + +/** + * Call the LaunchDarkly REST API, rate limits waited out. + * + * @returns {Promise} the JSON answer, `{}` for an empty one, or `null` for a 404. + * @throws {LaunchDarklyApiError} naming the request and LaunchDarkly's message for any other failure. + */ +async function rest(connection, method, path, body, contentType) { + try { + return (await requestLaunchDarkly(connection, { method, path: `/api/v2${path}`, body, contentType })) ?? {}; + } catch (error) { + if (error instanceof LaunchDarklyApiError && error.status === 404) { + return null; + } + throw error; + } +} + +/** `value` as JSON with every object's keys sorted, so equal schemas compare equal. */ +function canonicalJson(value) { + if (Array.isArray(value)) { + return `[${value.map(canonicalJson).join(',')}]`; + } + if (value !== null && typeof value === 'object') { + const entries = Object.keys(value) + .sort() + .map((key) => `${JSON.stringify(key)}:${canonicalJson(value[key])}`); + + return `{${entries.join(',')}}`; + } + + return JSON.stringify(value); +} + +/** Create or update each ladder tool, returning each tool's current version. */ +async function provisionTools(connection, project, ladder) { + const versions = new Map(); + for (const [key, tool] of Object.entries(ladder.tools)) { + const path = `/projects/${project}/ai-tools/${key}`; + const existing = await rest(connection, 'GET', path); + let saved; + if (existing === null) { + saved = await rest(connection, 'POST', `/projects/${project}/ai-tools`, { key, description: tool.description, schema: tool.parameters }); + console.log(`created tool ${key}`); + } else if (existing.description !== tool.description || canonicalJson(existing.schema) !== canonicalJson(tool.parameters)) { + saved = await rest(connection, 'PATCH', path, { description: tool.description, schema: tool.parameters }); + console.log(`updated tool ${key}`); + } else { + saved = existing; + console.log(`tool ${key} is current`); + } + versions.set(key, saved.version ?? existing?.version ?? 1); + } + + return versions; +} + +/** Create the agent-mode config if it is missing, and return it. */ +async function provisionConfig(connection, project, configKey) { + const path = `/projects/${project}/ai-configs/${configKey}`; + const existing = await rest(connection, 'GET', path); + if (existing !== null) { + if (existing.mode !== 'agent') { + throw new Error(`${configKey} exists in ${existing.mode} mode; the booker needs an agent-mode config`); + } + console.log(`config ${configKey} exists`); + + return existing; + } + + await rest(connection, 'POST', `/projects/${project}/ai-configs`, { key: configKey, name: CONFIG_NAME, mode: 'agent' }); + console.log(`created config ${configKey}`); + + return rest(connection, 'GET', path); +} + +/** Create or update each rung's variation. */ +async function provisionVariations(connection, project, configKey, ladder, config, toolVersions) { + const existingKeys = new Set((config.variations ?? []).map((variation) => variation.key)); + for (const rung of ladder.rungs) { + const body = { + name: rung.key, + instructions: rung.instructions, + modelConfigKey: modelConfigKey(ladder, rung), + model: { modelName: rung.model, parameters: {} }, + tools: rung.tools.map((key) => ({ key, version: toolVersions.get(key) })), + }; + if (existingKeys.has(rung.key)) { + await rest(connection, 'PATCH', `/projects/${project}/ai-configs/${configKey}/variations/${rung.key}`, body); + console.log(`updated variation ${rung.key}`); + } else { + await rest(connection, 'POST', `/projects/${project}/ai-configs/${configKey}/variations`, { key: rung.key, ...body }); + console.log(`created variation ${rung.key}`); + } + } +} + +/** Serve v1 by default in the environment, so every session starts on the bottom rung. */ +async function provisionDefaultRule(connection, project, environment, configKey, ladder) { + const targeting = await rest(connection, 'GET', `/projects/${project}/ai-configs/${configKey}/targeting`); + const first = rungByNumber(ladder, 1); + const variation = targeting.variations.find((candidate) => (candidate.key ?? candidate.name) === first.key); + await rest( + connection, + 'PATCH', + `/projects/${project}/ai-configs/${configKey}/targeting`, + { + environmentKey: environment, + comment: 'Launch Control: serve v1 by default', + instructions: [{ kind: 'updateFallthroughVariationOrRollout', variationId: variation._id }], + }, + SEMANTIC_PATCH_TYPE, + ); + console.log(`default rule in ${environment} serves ${first.key}`); +} + +/** Provision everything. */ +async function main() { + const token = process.env[API_TOKEN_VARIABLE] ?? ''; + if (token === '') { + throw new Error(`${API_TOKEN_VARIABLE} is not set`); + } + const settings = await readSettings(process.env, PACKAGE_DIR); + if (settings === null) { + throw new Error('fill in launchdarkly.json, or set LAUNCHDARKLY_PROJECT_KEY, LAUNCHDARKLY_ENVIRONMENT_KEY, and LAUNCHDARKLY_AI_CONFIG_KEY'); + } + const { projectKey, environmentKey, configKey } = settings; + const ladder = loadLadder(PACKAGE_DIR); + const connection = { origin: launchDarklyOrigins('live', 0, process.env).api, token }; + + const toolVersions = await provisionTools(connection, projectKey, ladder); + const config = await provisionConfig(connection, projectKey, configKey); + await provisionVariations(connection, projectKey, configKey, ladder, config, toolVersions); + await provisionDefaultRule(connection, projectKey, environmentKey, configKey, ladder); + console.log(`provisioned ${configKey} in ${projectKey}/${environmentKey}`); +} + +main().catch((error) => { + console.error(`provision: ${error.message}`); + process.exit(1); +}); diff --git a/challenges/launch-control/server/booker-config.mjs b/challenges/launch-control/server/booker-config.mjs new file mode 100644 index 0000000..380df5d --- /dev/null +++ b/challenges/launch-control/server/booker-config.mjs @@ -0,0 +1,121 @@ +/** + * The booker's config for this session, as LaunchDarkly's AI SDK resolves it: the variation, model, + * instructions, and tools the AgentControl config serves the session's context right now. A + * streaming SDK client keeps it current, so a targeting change reaches the booker in about a + * second with no restart. Analytics and diagnostic events stay off, so the client talks only to + * the streaming host. The ladder only names the rung; everything the booker runs on comes from + * what the SDK delivered. + */ +import { init } from '@launchdarkly/node-server-sdk'; +import { initClient, inspectConfig, shutdown } from '@launchdarkly/ai-server'; +import { rungByKey } from './ladder.mjs'; + +/** A logger that stays quiet: the page server's output goes nowhere, and errors surface as state. */ +const QUIET_LOGGER = { debug() {}, info() {}, warn() {}, error() {} }; + +/** How long the client may take to receive its first data set. */ +const INITIALIZATION_TIMEOUT_SECONDS = 10; + +/** The longest prompt excerpt the config card shows. */ +const PROMPT_EXCERPT_CHARACTERS = 160; + +/** The first line or so of `instructions`, for the config card. */ +function promptExcerpt(instructions) { + const flattened = instructions.replace(/\s+/g, ' ').trim(); + + return flattened.length <= PROMPT_EXCERPT_CHARACTERS + ? flattened + : `${flattened.slice(0, PROMPT_EXCERPT_CHARACTERS - 1).trimEnd()}…`; +} + +/** + * Reduce an `inspectConfig` result to what the booker runs on. + * + * @returns {{ rung: number, variationKey: string, name: string, model: string, + * instructions: string, promptExcerpt: string, tools: { name: string, description: string, + * parameters: object }[] } | null} `null` when the config is off, missing, or serves a + * variation the ladder does not know. + */ +export function describeBookerConfig(ladder, result) { + const variationKey = result?.meta?.variationKey; + const rung = rungByKey(ladder, variationKey); + if (!result?.enabled || result.config === null || rung === undefined) { + return null; + } + + const instructions = result.config.instructions ?? ''; + const tools = Object.values(result.config.tools ?? {}).map((tool) => ({ + name: tool.name, + description: tool.description ?? '', + parameters: tool.parameters ?? { type: 'object', properties: {} }, + })); + + return { + rung: rung.rung, + variationKey, + name: rung.name, + model: result.config.model?.name ?? '', + instructions, + promptExcerpt: promptExcerpt(instructions), + tools, + }; +} + +/** A streaming connection to the booker's AgentControl config. */ +export class BookerConfigSource { + #client; + + /** Use {@link BookerConfigSource.connect}. */ + constructor(client, options) { + this.#client = client; + this.configKey = options.configKey; + this.ladder = options.ladder; + } + + /** + * Connect and wait for the first data set. + * + * @param {{ streamOrigin: string, sdkKey: string, configKey: string, + * ladder: ReturnType }} options + * @throws {Error} when LaunchDarkly does not deliver within the initialization timeout; the + * message never carries the key. + */ + static async connect(options) { + const client = init(options.sdkKey, { + stream: true, + streamUri: options.streamOrigin, + baseUri: options.streamOrigin, + eventsUri: options.streamOrigin, + sendEvents: false, + diagnosticOptOut: true, + logger: QUIET_LOGGER, + }); + try { + await client.waitForInitialization({ timeout: INITIALIZATION_TIMEOUT_SECONDS }); + } catch { + client.close(); + throw new Error(`LaunchDarkly did not deliver the booker config within ${INITIALIZATION_TIMEOUT_SECONDS} s`); + } + await initClient(client); + + return new BookerConfigSource(client, options); + } + + /** The config the session `contextKey` gets now, or `null` (see {@link describeBookerConfig}). */ + async resolve(contextKey) { + const result = await inspectConfig(this.configKey, { kind: 'user', key: contextKey }); + + return describeBookerConfig(this.ladder, result); + } + + /** Call `listener` whenever LaunchDarkly delivers a change to the booker config. */ + onChange(listener) { + this.#client.on(`update:${this.configKey}`, listener); + } + + /** Close the connection. */ + async close() { + await shutdown(); + this.#client.close(); + } +} diff --git a/challenges/launch-control/server/booker.mjs b/challenges/launch-control/server/booker.mjs new file mode 100644 index 0000000..980a975 --- /dev/null +++ b/challenges/launch-control/server/booker.mjs @@ -0,0 +1,90 @@ +/** + * The travel booker's agent loop: one participant message in, one reply out. It runs on exactly + * the config LaunchDarkly served this session, its model, its instructions, and only the tools + * attached to it, calling the travel desk for each tool call until the model answers in text. + */ + +/** The most model turns one reply may take, tool calls included. */ +const MAX_MODEL_TURNS = 8; + +/** The reply when the model stops calling tools without saying anything. */ +const EMPTY_REPLY = '(The booker did not answer.)'; + +/** A one-line summary of a tool call for the chat, such as `search_flights JFK → LIS 2027-03-12`. */ +export function summarizeToolCall(name, args) { + switch (name) { + case 'search_flights': + return `search_flights ${args.origin ?? '?'} → ${args.destination ?? '?'} ${args.date ?? ''}`.trim(); + case 'search_hotels': + return `search_hotels ${args.city ?? '?'} ${args.check_in ?? ''} – ${args.check_out ?? ''}`.trim(); + case 'book_trip': + return `book_trip ${args.outbound_flight_id ?? '?'} + ${args.return_flight_id ?? '?'} + ${args.hotel_id ?? '?'}`; + default: + return name; + } +} + +/** The text a model turn says, leaving out its thoughts. */ +function spokenText(content) { + return content.parts + .filter((part) => typeof part.text === 'string' && part.thought !== true) + .map((part) => part.text) + .join('') + .trim(); +} + +/** Runs the booker's turns against Gemini and the travel desk. */ +export class Booker { + /** + * @param {{ gemini: import('./gemini.mjs').GeminiClient, + * desk: import('./travel-desk.mjs').TravelDesk }} options + */ + constructor(options) { + this.gemini = options.gemini; + this.desk = options.desk; + } + + /** + * Answer `message` on `config`, continuing `contents`, the conversation so far on this config. + * + * @param {ReturnType} config + * @param {object[]} contents Gemini contents; the new turns are appended to it. + * @returns {Promise<{ reply: string, toolCalls: string[], booking: object | null }>} + * @throws {import('./gemini.mjs').GeminiError} when Gemini fails. + */ + async reply(config, contents, message) { + contents.push({ role: 'user', parts: [{ text: message }] }); + const request = { + systemInstruction: { parts: [{ text: config.instructions }] }, + ...(config.tools.length === 0 ? {} : { tools: [{ functionDeclarations: config.tools }] }), + }; + const allowedTools = new Set(config.tools.map((tool) => tool.name)); + const toolCalls = []; + let booking = null; + + for (let turn = 0; turn < MAX_MODEL_TURNS; turn += 1) { + const content = await this.gemini.generate(config.model, { ...request, contents }); + contents.push(content); + const calls = content.parts.filter((part) => part.functionCall !== undefined).map((part) => part.functionCall); + if (calls.length === 0) { + return { reply: spokenText(content) || EMPTY_REPLY, toolCalls, booking }; + } + + const responses = []; + for (const call of calls) { + const args = call.args ?? {}; + toolCalls.push(summarizeToolCall(call.name, args)); + const result = allowedTools.has(call.name) + ? await this.desk.run(call.name, args) + : { error: `The tool ${call.name} is not attached to this agent.` }; + if (call.name === 'book_trip' && result.error === undefined) { + booking = result; + } + responses.push({ functionResponse: { name: call.name, ...(call.id === undefined ? {} : { id: call.id }), response: result } }); + } + contents.push({ role: 'user', parts: responses }); + } + + return { reply: 'The booker gave up after too many steps.', toolCalls, booking }; + } +} diff --git a/challenges/launch-control/server/cleanup.mjs b/challenges/launch-control/server/cleanup.mjs new file mode 100644 index 0000000..c0360d3 --- /dev/null +++ b/challenges/launch-control/server/cleanup.mjs @@ -0,0 +1,177 @@ +/** + * Cleanup: stop the page server, remove this session's individual target from the booker's + * config, and remove the attempt's files. Safe to run twice, after a failed setup, after a crash + * in a fresh process, or with nothing set up at all: a session with no target is left alone. The + * LaunchDarkly call runs under a deadline, so cleanup stays within its 30 s cap. + */ +import { execFileSync } from 'node:child_process'; +import path from 'node:path'; +import { LaunchDarklyApi } from './launchdarkly-api.mjs'; +import { PACKAGE_DIR, resolveStatePaths } from './paths.mjs'; +import { readRuntimeConfig } from './runtime-config.mjs'; +import { API_TOKEN_VARIABLE, decideMode, launchDarklyOrigins } from './settings.mjs'; +import { clearWorkspacePath, readWorkspaceFile } from './workspace-file.mjs'; +import { sleep } from './http.mjs'; + +/** How long the server gets to stop on SIGTERM before it is killed. */ +const GRACE_MS = 3_000; + +/** The longest cleanup waits for LaunchDarkly, well inside the hook's 30 s cap. */ +const TARGET_REMOVAL_DEADLINE_MS = 20_000; + +/** The most a pid file holds: a pid and a newline. */ +const PID_FILE_MAX_BYTES = 32; + +/** The page server's entry point, which a live pid must be running. */ +const SERVE_SCRIPT = path.join(PACKAGE_DIR, 'server', 'serve.mjs'); + +/** Whether a process with `pid` exists. */ +function isAlive(pid) { + try { + process.kill(pid, 0); + + return true; + } catch (error) { + return error.code === 'EPERM'; + } +} + +/** + * Whether `pid` is still this package's page server, so a recycled pid is never killed by mistake. + * When `ps` itself is unavailable the pid file is trusted: a leaked server is worse than a stray + * signal. + */ +function isPageServer(pid) { + try { + const command = execFileSync('ps', ['-p', String(pid), '-o', 'command='], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'ignore'], + }); + + return command.includes(SERVE_SCRIPT); + } catch (error) { + return error.status !== 1; + } +} + +/** + * Stop the server recorded in the pid file, if it is still running. A pid file that is a symlink or + * anything but a lone regular file is never read, so a planted one cannot point elsewhere. + */ +export async function stopServer(paths) { + let pid; + try { + pid = Number(((await readWorkspaceFile(paths.pidFile, PID_FILE_MAX_BYTES)) ?? '').trim()); + } catch { + return; + } + if (!Number.isInteger(pid) || pid <= 1 || !isAlive(pid) || !isPageServer(pid)) { + return; + } + + try { + process.kill(pid, 'SIGTERM'); + } catch { + return; + } + const deadline = Date.now() + GRACE_MS; + while (isAlive(pid) && Date.now() < deadline) { + await sleep(50); + } + if (isAlive(pid)) { + try { + process.kill(pid, 'SIGKILL'); + } catch { + return; + } + } +} + +/** + * Where this session's target lives: from the runtime config setup wrote, or, when setup never + * got that far, from the session id and the settings in `env`. + * + * @returns {Promise<{ sessionKey: string, launchDarkly: object } | null>} `null` when the session + * ran in demo mode, whose stand-in goes away with the page server, or has no key to remove. + */ +async function liveTarget(paths, env) { + let runtimeConfig = null; + try { + runtimeConfig = await readRuntimeConfig(paths.runtimeConfigFile); + } catch { + // A crash can leave the file half-written; the session id and settings in env still find + // the target, and cleanup must remove it either way. + runtimeConfig = null; + } + if (runtimeConfig !== null) { + return runtimeConfig.mode === 'live' ? runtimeConfig : null; + } + + const sessionKey = env.BOOTHSTRAP_SESSION_ID ?? ''; + const decision = await decideMode(env, PACKAGE_DIR); + if (sessionKey === '' || decision.mode !== 'live') { + return null; + } + const origins = launchDarklyOrigins('live', 0, env); + + return { sessionKey, launchDarkly: { ...decision.settings, apiOrigin: origins.api } }; +} + +/** + * Remove this session's individual target, if it ran live and has one. + * + * @throws {Error} when LaunchDarkly refuses, fails, or misses the deadline. + */ +async function removeSessionTarget(paths, env) { + const target = await liveTarget(paths, env); + if (target === null) { + return; + } + + const api = new LaunchDarklyApi({ + origin: target.launchDarkly.apiOrigin, + token: env[API_TOKEN_VARIABLE] ?? '', + projectKey: target.launchDarkly.projectKey, + environmentKey: target.launchDarkly.environmentKey, + configKey: target.launchDarkly.configKey, + }); + let timer; + const deadline = new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error('LaunchDarkly did not answer in time')), TARGET_REMOVAL_DEADLINE_MS); + }); + try { + await Promise.race([api.removeSession(target.sessionKey), deadline]); + } finally { + clearTimeout(timer); + } +} + +/** + * Stop the server, remove the session's target, and remove every file the attempt created. The + * files go even when LaunchDarkly fails, and the failure is then rethrown. + */ +export async function runCleanup(paths, env) { + await stopServer(paths); + let failure = null; + try { + await removeSessionTarget(paths, env); + } catch (error) { + failure = error; + } + await clearWorkspacePath(paths.stateDir); + await clearWorkspacePath(paths.stateFile); + await clearWorkspacePath(paths.itineraryFile); + if (failure !== null) { + throw failure; + } +} + +if (import.meta.main) { + runCleanup(resolveStatePaths(process.env, process.cwd()), process.env).then( + () => process.exit(0), + (error) => { + process.stderr.write(`cleanup: ${error instanceof Error ? error.message : String(error)}\n`); + process.exit(1); + }, + ); +} diff --git a/challenges/launch-control/server/gemini.mjs b/challenges/launch-control/server/gemini.mjs new file mode 100644 index 0000000..39ee0e7 --- /dev/null +++ b/challenges/launch-control/server/gemini.mjs @@ -0,0 +1,62 @@ +/** + * The Gemini API call the booker makes: one `generateContent` request per model turn. The key goes + * only into the `x-goog-api-key` header of requests to the configured origin, and no error carries + * it. On a station the request leaves through the vault proxy, which swaps the placeholder key for + * the real one. + */ + +/** How long one model turn may take. */ +const REQUEST_TIMEOUT_MS = 45_000; + +/** Gemini did not answer usefully. The message names the model and status, never the key. */ +export class GeminiError extends Error { + constructor(message) { + super(message); + this.name = 'GeminiError'; + } +} + +/** Calls Gemini's REST API with an API key. */ +export class GeminiClient { + #key; + + /** @param {{ origin: string, key: string }} options */ + constructor(options) { + this.origin = options.origin; + this.#key = options.key; + } + + /** + * Run one model turn. + * + * @param {string} model a Gemini model name, such as `gemini-3.8-flash`. + * @param {object} body a `generateContent` request body. + * @returns {Promise} the first candidate's `content`. + * @throws {GeminiError} on a network failure, a timeout, a non-2xx answer, or no candidate. + */ + async generate(model, body) { + let response; + try { + response = await fetch(`${this.origin}/v1beta/models/${encodeURIComponent(model)}:generateContent`, { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-goog-api-key': this.#key }, + body: JSON.stringify(body), + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + }); + } catch (error) { + const reason = error?.name === 'TimeoutError' ? 'timed out' : 'could not be reached'; + throw new GeminiError(`${model} ${reason}`); + } + + const text = await response.text(); + if (!response.ok) { + throw new GeminiError(`${model} answered HTTP ${response.status}`); + } + const content = JSON.parse(text).candidates?.[0]?.content; + if (content === undefined) { + throw new GeminiError(`${model} returned no answer`); + } + + return { role: 'model', parts: content.parts ?? [] }; + } +} diff --git a/challenges/launch-control/server/http.mjs b/challenges/launch-control/server/http.mjs new file mode 100644 index 0000000..4be0f57 --- /dev/null +++ b/challenges/launch-control/server/http.mjs @@ -0,0 +1,33 @@ +/** Small HTTP helpers the page server and the LaunchDarkly stand-in share. */ + +/** Send `body` as JSON with `status`, plus any extra `headers`. */ +export function sendJson(response, status, body, headers = {}) { + response.writeHead(status, { 'cache-control': 'no-store', ...headers, 'content-type': 'application/json; charset=utf-8' }); + response.end(JSON.stringify(body)); +} + +/** + * Read a request body as UTF-8 text. + * + * @returns {Promise} the body, or `null` once it passes `limit` bytes; the rest is + * read and dropped, so the client can finish sending and read the answer. + */ +export function readBody(request, limit) { + return new Promise((resolve, reject) => { + const chunks = []; + let size = 0; + request.on('data', (chunk) => { + size += chunk.length; + if (size <= limit) { + chunks.push(chunk); + } + }); + request.on('end', () => resolve(size > limit ? null : Buffer.concat(chunks).toString('utf8'))); + request.once('error', reject); + }); +} + +/** Pause for `milliseconds`. */ +export function sleep(milliseconds) { + return new Promise((resolve) => setTimeout(resolve, milliseconds)); +} diff --git a/challenges/launch-control/server/ladder.mjs b/challenges/launch-control/server/ladder.mjs new file mode 100644 index 0000000..e9f71da --- /dev/null +++ b/challenges/launch-control/server/ladder.mjs @@ -0,0 +1,114 @@ +/** + * The ladder: the four variations of the travel booker's AgentControl config, read from + * `ladder.json`. `provision.mjs` writes them to LaunchDarkly and demo mode's stand-in serves them, + * so this module is the one reader of that file and the one place that knows the variation shape + * LaunchDarkly's SDK delivers. + */ +import { readFileSync } from 'node:fs'; +import path from 'node:path'; + +/** The package file holding the ladder. */ +export const LADDER_FILE_NAME = 'ladder.json'; + +/** Throw a validation error naming the ladder file. */ +function invalid(message) { + throw new Error(`${LADDER_FILE_NAME}: ${message}`); +} + +/** Validate one rung against the ladder's tools and its place in the order. */ +function parseRung(record, index, toolNames) { + const expected = index + 1; + if (record?.rung !== expected) { + invalid(`rung ${expected} is missing or out of order`); + } + for (const field of ['key', 'name', 'adds', 'model', 'instructions']) { + if (typeof record[field] !== 'string' || record[field] === '') { + invalid(`rung ${expected} needs a ${field}`); + } + } + if (!Array.isArray(record.tools) || record.tools.some((tool) => !toolNames.includes(tool))) { + invalid(`rung ${expected} lists a tool the ladder does not define`); + } + + return { + rung: record.rung, + key: record.key, + name: record.name, + adds: record.adds, + model: record.model, + instructions: record.instructions, + tools: [...record.tools], + }; +} + +/** + * Parse the ladder's record. + * + * @returns {{ provider: string, tools: Record, rungs: ReturnType[] }} + * @throws {Error} naming the first rule the record breaks. + */ +export function parseLadder(record) { + if (typeof record?.provider !== 'string' || typeof record.tools !== 'object') { + invalid('needs a provider and tools'); + } + const toolNames = Object.keys(record.tools); + if (!Array.isArray(record.rungs) || record.rungs.length === 0) { + invalid('needs rungs'); + } + const rungs = record.rungs.map((rung, index) => parseRung(rung, index, toolNames)); + if (new Set(rungs.map((rung) => rung.key)).size !== rungs.length) { + invalid('two rungs share a key'); + } + + return { provider: record.provider, tools: record.tools, rungs }; +} + +/** Read and parse the package's ladder. */ +export function loadLadder(packageDir) { + return parseLadder(JSON.parse(readFileSync(path.join(packageDir, LADDER_FILE_NAME), 'utf8'))); +} + +/** The rung serving variation `variationKey`, or `undefined` for one the ladder does not know. */ +export function rungByKey(ladder, variationKey) { + return ladder.rungs.find((rung) => rung.key === variationKey); +} + +/** The rung numbered `number`, or `undefined` past either end of the ladder. */ +export function rungByNumber(ladder, number) { + return ladder.rungs.find((rung) => rung.rung === number); +} + +/** LaunchDarkly's model config key for a rung's model, such as `Gemini.gemini-3.8-flash`. */ +export function modelConfigKey(ladder, rung) { + return `${ladder.provider}.${rung.model}`; +} + +/** A rung's tools as function declarations: name, description, and parameter schema. */ +export function toolDeclarations(ladder, rung) { + return rung.tools.map((name) => ({ name, ...ladder.tools[name] })); +} + +/** + * The variation value LaunchDarkly's SDK delivers for a rung of an agent-mode config, which the + * demo stand-in serves in its place. + */ +export function variationValue(ladder, rung) { + const tools = Object.fromEntries( + toolDeclarations(ladder, rung).map((tool) => [tool.name, { type: 'function', ...tool }]), + ); + + return { + _ldMeta: { + enabled: true, + mode: 'agent', + variationKey: rung.key, + version: 1, + modelKey: modelConfigKey(ladder, rung), + }, + instructions: rung.instructions, + model: { name: rung.model, parameters: {} }, + provider: { name: ladder.provider }, + ...(rung.tools.length === 0 ? {} : { tools }), + }; +} diff --git a/challenges/launch-control/server/launchdarkly-api.mjs b/challenges/launch-control/server/launchdarkly-api.mjs new file mode 100644 index 0000000..f392d32 --- /dev/null +++ b/challenges/launch-control/server/launchdarkly-api.mjs @@ -0,0 +1,208 @@ +/** + * The LaunchDarkly REST calls the hooks and the MCP tool make: read the booker config's targeting, + * and move one session's individual target between variations. Nothing here touches the default + * rule or any other context key. The token goes only into the Authorization header of requests to + * the configured origin, and no error or result carries it. On a station each request leaves + * through the vault proxy, which swaps the placeholder token for the real one. + */ + +import { sleep } from './http.mjs'; + +/** How long one request may take. */ +const REQUEST_TIMEOUT_MS = 8_000; + +/** The longest a rate-limited call keeps retrying before it gives up. */ +const RATE_LIMIT_BUDGET_MS = 12_000; + +/** The longest one rate-limit wait, whatever LaunchDarkly's reset time says. */ +const MAX_RATE_LIMIT_WAIT_MS = 5_000; + +/** The content type a semantic patch needs. */ +export const SEMANTIC_PATCH_TYPE = 'application/json; domain-model=launchdarkly.semanticpatch'; + +/** LaunchDarkly did not answer usefully. The message names the request, never the token. */ +export class LaunchDarklyApiError extends Error { + constructor(message, status) { + super(message); + this.name = 'LaunchDarklyApiError'; + this.status = status; + } +} + +/** How long a 429 asks the caller to wait, from `Retry-After` or LaunchDarkly's reset time. */ +function rateLimitWait(response) { + const retryAfterSeconds = Number(response.headers.get('retry-after')); + if (Number.isFinite(retryAfterSeconds) && retryAfterSeconds > 0) { + return retryAfterSeconds * 1000; + } + const resetAt = Number(response.headers.get('x-ratelimit-reset')); + if (Number.isFinite(resetAt) && resetAt > Date.now()) { + return resetAt - Date.now(); + } + + return 1_000; +} + +/** LaunchDarkly's own error message from a response body, cut short, or the status alone. */ +function describeFailure(status, text) { + let message = ''; + try { + message = String(JSON.parse(text).message ?? ''); + } catch { + message = ''; + } + + return message === '' ? `HTTP ${status}` : `HTTP ${status}: ${message.slice(0, 200)}`; +} + +/** + * Send one LaunchDarkly REST request with `token`, waiting out rate limits within + * {@link RATE_LIMIT_BUDGET_MS}. + * + * @param {{ origin: string, token: string }} connection + * @param {{ method: string, path: string, body?: object, contentType?: string }} request `path` + * starts with `/api/v2`; `contentType` defaults to JSON. + * @returns {Promise} the JSON answer, or `null` for an empty one. + * @throws {LaunchDarklyApiError} on a network failure, a timeout, or a non-2xx answer; its + * `status` is 0 for the first two. + */ +export async function requestLaunchDarkly(connection, { method, path, body, contentType = 'application/json' }) { + const deadline = Date.now() + RATE_LIMIT_BUDGET_MS; + for (;;) { + let response; + try { + response = await fetch(`${connection.origin}${path}`, { + method, + headers: { authorization: connection.token, ...(body === undefined ? {} : { 'content-type': contentType }) }, + body: body === undefined ? undefined : JSON.stringify(body), + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + }); + } catch (error) { + const reason = error?.name === 'TimeoutError' ? 'timed out' : 'could not connect'; + throw new LaunchDarklyApiError(`LaunchDarkly ${method} ${path} ${reason}`, 0); + } + + const text = await response.text(); + if (response.ok) { + return text === '' ? null : JSON.parse(text); + } + const wait = Math.min(rateLimitWait(response), MAX_RATE_LIMIT_WAIT_MS); + if (response.status !== 429 || Date.now() + wait > deadline) { + throw new LaunchDarklyApiError(`LaunchDarkly ${method} ${path} failed: ${describeFailure(response.status, text)}`, response.status); + } + await sleep(wait); + } +} + +/** Reads and retargets one AgentControl config in one environment. */ +export class LaunchDarklyApi { + #connection; + + /** + * @param {{ origin: string, token: string, projectKey: string, environmentKey: string, + * configKey: string }} options + */ + constructor(options) { + this.projectKey = options.projectKey; + this.environmentKey = options.environmentKey; + this.configKey = options.configKey; + this.#connection = { origin: options.origin, token: options.token }; + } + + /** The config's targeting resource path. */ + get targetingPath() { + const project = encodeURIComponent(this.projectKey); + const config = encodeURIComponent(this.configKey); + + return `/api/v2/projects/${project}/ai-configs/${config}/targeting`; + } + + /** Read or patch the config's targeting. */ + request(method, body) { + return requestLaunchDarkly(this.#connection, { method, path: this.targetingPath, body, contentType: SEMANTIC_PATCH_TYPE }); + } + + /** + * Read the config's variations and this environment's individual targets. + * + * @returns {Promise<{ variationIds: Map, targetOf: (contextKey: string) => + * string | null }>} the variation id for each variation key, and the variation key a context + * key is individually targeted to, or `null` when it has no individual target. + * @throws {LaunchDarklyApiError} when the config or environment is missing. + */ + async readTargeting() { + const targeting = await this.request('GET'); + const variations = targeting?.variations ?? []; + const variationIds = new Map(variations.map((variation) => [variation.key ?? variation.name, variation._id])); + const environment = targeting?.environments?.[this.environmentKey]; + if (environment === undefined) { + throw new LaunchDarklyApiError(`LaunchDarkly has no environment ${this.environmentKey} on this config`, 404); + } + const targets = [...(environment.targets ?? []), ...(environment.contextTargets ?? [])].filter( + (target) => (target.contextKind ?? 'user') === 'user', + ); + + const targetOf = (contextKey) => { + const target = targets.find((candidate) => candidate.values?.includes(contextKey)); + if (target === undefined) { + return null; + } + const variation = variations[target.variation]; + + return variation === undefined ? null : (variation.key ?? variation.name); + }; + + return { variationIds, targetOf }; + } + + /** Send a semantic patch to this environment's targeting. */ + patch(instructions, comment) { + return this.request('PATCH', { environmentKey: this.environmentKey, comment, instructions }); + } + + /** + * Target `contextKey` individually to the variation `variationKey`, moving it off any other. + * + * @throws {LaunchDarklyApiError} when the config has no such variation, or LaunchDarkly refuses. + */ + async targetSession(contextKey, variationKey) { + const { variationIds, targetOf } = await this.readTargeting(); + const variationId = variationIds.get(variationKey); + if (variationId === undefined) { + throw new LaunchDarklyApiError(`the config has no variation ${variationKey}; run provision.mjs`, 404); + } + const current = targetOf(contextKey); + if (current === variationKey) { + return; + } + + const instructions = []; + if (current !== null) { + instructions.push({ kind: 'removeTargets', contextKind: 'user', values: [contextKey], variationId: variationIds.get(current) }); + } + instructions.push({ kind: 'addTargets', contextKind: 'user', values: [contextKey], variationId }); + await this.patch(instructions, `Launch Control: switch one session to ${variationKey}`); + } + + /** + * Remove `contextKey`'s individual target, if it has one. A key with no target is left alone, + * so this is safe to repeat. + * + * @returns {Promise} the variation key it was removed from, or `null`. + */ + async removeSession(contextKey) { + const { variationIds, targetOf } = await this.readTargeting(); + const current = targetOf(contextKey); + if (current === null) { + return null; + } + + const variationId = variationIds.get(current); + await this.patch( + [{ kind: 'removeTargets', contextKind: 'user', values: [contextKey], variationId }], + 'Launch Control: remove one session', + ); + + return current; + } +} diff --git a/challenges/launch-control/server/page-server.mjs b/challenges/launch-control/server/page-server.mjs new file mode 100644 index 0000000..9c453ba --- /dev/null +++ b/challenges/launch-control/server/page-server.mjs @@ -0,0 +1,212 @@ +/** + * The page server: serves the travel desk page, the session's live booker config, and the chat, + * running the booker's agent loop on the server so no key ever reaches the page. In demo mode it + * also mounts the LaunchDarkly stand-in. It answers only requests that address it by a loopback + * name, and takes chat messages only from its own page. + */ +import { readFile } from 'node:fs/promises'; +import { createServer } from 'node:http'; +import path from 'node:path'; +import { GeminiError } from './gemini.mjs'; +import { readBody, sendJson as sendJsonWithHeaders } from './http.mjs'; +import { DEMO_PATH } from './settings.mjs'; +import { writeWorkspaceFile } from './workspace-file.mjs'; + +/** The page's files and their content types; nothing else is served. */ +const PAGE_FILES = { + '/': ['index.html', 'text/html; charset=utf-8'], + '/app.mjs': ['app.mjs', 'text/javascript; charset=utf-8'], + '/styles.css': ['styles.css', 'text/css; charset=utf-8'], +}; + +/** The page may load and contact nothing but this server. */ +export const CONTENT_SECURITY_POLICY = + "default-src 'self'; script-src 'self'; style-src 'self'; img-src 'self' data:; connect-src 'self'; frame-ancestors 'none'; base-uri 'none'; form-action 'none'"; + +/** Headers every response carries: nothing is cached, because the config can change at any time. */ +const BASE_HEADERS = { + 'cache-control': 'no-store', + 'x-content-type-options': 'nosniff', + 'content-security-policy': CONTENT_SECURITY_POLICY, +}; + +/** The longest chat message the booker takes. */ +export const MAX_MESSAGE_CHARACTERS = 1_000; + +/** The largest chat request body read. */ +const MAX_BODY_BYTES = 8 * 1024; + +/** Send a JSON body with the page server's headers. */ +function sendJson(response, status, body) { + sendJsonWithHeaders(response, status, body, BASE_HEADERS); +} + +/** Send a status with a short plain-text reason. */ +function sendStatus(response, status, message = '') { + response.writeHead(status, { ...BASE_HEADERS, 'content-type': 'text/plain; charset=utf-8' }); + response.end(message); +} + +/** The origins a request may legitimately come from: this server, by IP or by name. */ +function allowedOrigins(request) { + const port = request.socket.localPort; + + return [`http://127.0.0.1:${port}`, `http://localhost:${port}`]; +} + +/** Whether the request addresses this server by a loopback name, which defeats DNS rebinding. */ +function hostIsLoopback(request) { + const host = request.headers.host ?? ''; + + return allowedOrigins(request).some((origin) => origin === `http://${host}`); +} + +/** Whether a state-changing request came from this page rather than another site. */ +function isOwnPage(request) { + const origin = request.headers.origin; + const site = request.headers['sec-fetch-site']; + + return (origin === undefined || allowedOrigins(request).includes(origin)) && (site === undefined || site === 'same-origin'); +} + +/** The chat message in a request body, or `null` when it is not a usable one. */ +function parseMessage(text) { + let body; + try { + body = JSON.parse(text ?? ''); + } catch { + return null; + } + const message = typeof body?.message === 'string' ? body.message.trim() : ''; + if (message === '' || message.length > MAX_MESSAGE_CHARACTERS) { + return null; + } + + return message; +} + +/** Write the session snapshot to the state file, so standalone Pi can follow along with no monitor. */ +export async function writeStateFile(stateFile, snapshot) { + await writeWorkspaceFile(stateFile, `${JSON.stringify(snapshot, null, 2)}\n`); +} + +/** + * Create the page server. + * + * @param {{ pageDir: string, configSource: { resolve: (contextKey: string) => Promise }, + * booker: import('./booker.mjs').Booker, session: import('./session.mjs').TravelDeskSession, + * sessionKey: string, stateFile: string, + * standIn?: import('./stand-in.mjs').LaunchDarklyStandIn }} options + */ +export function createPageServer(options) { + let busy = false; + + /** The booker config this session gets right now. */ + const currentConfig = () => options.configSource.resolve(options.sessionKey); + + /** Answer one chat message on the config live now. */ + const chat = async (request, response) => { + if (!isOwnPage(request)) { + sendStatus(response, 403, 'chat messages come only from the travel desk page'); + + return; + } + const message = parseMessage(await readBody(request, MAX_BODY_BYTES)); + if (message === null) { + sendStatus(response, 400, `send {"message": "..."} with 1 to ${MAX_MESSAGE_CHARACTERS} characters`); + + return; + } + if (busy) { + sendStatus(response, 409, 'the booker is still answering the last message'); + + return; + } + + busy = true; + try { + const config = await currentConfig(); + if (config === null) { + options.session.recordFailure(null, message, 'The booker has no config from LaunchDarkly right now.'); + } else { + try { + const result = await options.booker.reply(config, options.session.conversationFor(config), message); + options.session.recordExchange(config, message, result); + } catch (error) { + if (!(error instanceof GeminiError)) { + throw error; + } + options.session.recordFailure(config, message, `The booker could not reach its model: ${error.message}.`); + } + } + const snapshot = options.session.snapshot(config); + await writeStateFile(options.stateFile, snapshot); + sendJson(response, 200, { ...snapshot, messages: options.session.messages }); + } finally { + busy = false; + } + }; + + /** Serve one of the page's files. */ + const servePageFile = async (response, [fileName, contentType]) => { + const contents = await readFile(path.join(options.pageDir, fileName)); + response.writeHead(200, { ...BASE_HEADERS, 'content-type': contentType }); + response.end(contents); + }; + + /** Answer one request: the stand-in's paths, the page's files, or the travel desk API. */ + const route = async (request, response) => { + if (!hostIsLoopback(request)) { + sendStatus(response, 421, 'address this server as 127.0.0.1 or localhost'); + + return; + } + let pathname; + try { + pathname = decodeURIComponent(new URL(request.url ?? '/', 'http://127.0.0.1').pathname); + } catch { + sendStatus(response, 400, 'malformed path'); + + return; + } + + if (options.standIn !== undefined && pathname.startsWith(`${DEMO_PATH}/`)) { + if (!options.standIn.handle(request, response, pathname.slice(DEMO_PATH.length))) { + sendStatus(response, 404, 'not found'); + } + + return; + } + if (request.method === 'GET' && PAGE_FILES[pathname] !== undefined) { + await servePageFile(response, PAGE_FILES[pathname]); + + return; + } + if (request.method === 'GET' && pathname === '/api/state') { + sendJson(response, 200, { ...options.session.snapshot(await currentConfig()), busy }); + + return; + } + if (request.method === 'GET' && pathname === '/api/history') { + sendJson(response, 200, { messages: options.session.messages }); + + return; + } + if (request.method === 'POST' && pathname === '/api/chat') { + await chat(request, response); + + return; + } + sendStatus(response, 404, 'not found'); + }; + + return createServer((request, response) => { + route(request, response).catch(() => { + if (!response.headersSent) { + sendStatus(response, 500, 'the travel desk hit an error'); + } else { + response.end(); + } + }); + }); +} diff --git a/challenges/launch-control/server/paths.mjs b/challenges/launch-control/server/paths.mjs new file mode 100644 index 0000000..527199a --- /dev/null +++ b/challenges/launch-control/server/paths.mjs @@ -0,0 +1,29 @@ +/** + * Where an attempt keeps its files. Everything lives in the participant's workspace, which + * Boothstrap provides fresh and removes afterwards; standalone, the current directory stands in. + */ +import path from 'node:path'; + +/** The package directory: where `challenge.json`, `ladder.json`, `page/`, and `server/` live. */ +export const PACKAGE_DIR = path.resolve(import.meta.dirname, '..'); + +/** The booker's state for the guide to read when no monitor runs, as `SKILL.md` names it. */ +export const STATE_FILE_NAME = 'travel-desk-state.json'; + +/** The fake itinerary `book_trip` writes, as `SKILL.md` names it. */ +export const ITINERARY_FILE_NAME = 'itinerary.json'; + +/** Resolve the attempt's paths: `BOOTHSTRAP_WORKSPACE` inside Boothstrap, else `cwd`. */ +export function resolveStatePaths(env, cwd) { + const workspace = path.resolve(env.BOOTHSTRAP_WORKSPACE ?? cwd); + const stateDir = path.join(workspace, '.launch-control'); + + return { + workspace, + stateDir, + runtimeConfigFile: path.join(stateDir, 'runtime-config.json'), + pidFile: path.join(stateDir, 'server.pid'), + stateFile: path.join(workspace, STATE_FILE_NAME), + itineraryFile: path.join(workspace, ITINERARY_FILE_NAME), + }; +} diff --git a/challenges/launch-control/server/runtime-config.mjs b/challenges/launch-control/server/runtime-config.mjs new file mode 100644 index 0000000..c9e8620 --- /dev/null +++ b/challenges/launch-control/server/runtime-config.mjs @@ -0,0 +1,71 @@ +/** + * The attempt's runtime config: what setup decided, written to the workspace for the page server + * and the MCP tool to read. It holds the mode, the session's context key, the page port, and the + * public LaunchDarkly settings and origins. It never holds a key. + */ +import { readWorkspaceFile } from './workspace-file.mjs'; + +/** The most a runtime config file holds. */ +const MAX_RUNTIME_CONFIG_BYTES = 4 * 1024; + +/** Throw a validation error naming the field. */ +function invalid(field) { + throw new Error(`the runtime config's ${field} is missing or malformed`); +} + +/** Require `record[field]` to be a non-empty string. */ +function requireString(record, field, label = field) { + if (typeof record?.[field] !== 'string' || record[field] === '') { + invalid(label); + } + + return record[field]; +} + +/** + * Parse a runtime config record. + * + * @returns {{ attemptId: string, sessionKey: string, mode: 'live' | 'demo', pagePort: number, + * launchDarkly: { apiOrigin: string, streamOrigin: string, projectKey: string, + * environmentKey: string, configKey: string } }} + * @throws {Error} naming the first malformed field. + */ +export function parseRuntimeConfig(record) { + const mode = record?.mode; + if (mode !== 'live' && mode !== 'demo') { + invalid('mode'); + } + if (!Number.isInteger(record.pagePort) || record.pagePort < 1 || record.pagePort > 65_535) { + invalid('pagePort'); + } + const launchDarkly = record.launchDarkly; + + return { + attemptId: requireString(record, 'attemptId'), + sessionKey: requireString(record, 'sessionKey'), + mode, + pagePort: record.pagePort, + launchDarkly: { + apiOrigin: requireString(launchDarkly, 'apiOrigin', 'launchDarkly.apiOrigin'), + streamOrigin: requireString(launchDarkly, 'streamOrigin', 'launchDarkly.streamOrigin'), + projectKey: requireString(launchDarkly, 'projectKey', 'launchDarkly.projectKey'), + environmentKey: requireString(launchDarkly, 'environmentKey', 'launchDarkly.environmentKey'), + configKey: requireString(launchDarkly, 'configKey', 'launchDarkly.configKey'), + }, + }; +} + +/** + * Read the attempt's runtime config from `file`. + * + * @returns {Promise | null>} `null` when there is none. + * @throws {Error} when the file is not JSON or is malformed. + */ +export async function readRuntimeConfig(file) { + const text = await readWorkspaceFile(file, MAX_RUNTIME_CONFIG_BYTES); + if (text === null) { + return null; + } + + return parseRuntimeConfig(JSON.parse(text)); +} diff --git a/challenges/launch-control/server/serve.mjs b/challenges/launch-control/server/serve.mjs new file mode 100644 index 0000000..03bca33 --- /dev/null +++ b/challenges/launch-control/server/serve.mjs @@ -0,0 +1,134 @@ +/** + * The page server process: `setup` starts it in the background. It listens on loopback, connects + * to the booker's config (LaunchDarkly live, or its own stand-in in demo mode), tells setup once it + * is ready or why it could not be, and stops on SIGTERM. It reads the SDK key and the Gemini key + * from its environment and never the LaunchDarkly API token, which setup leaves out. + */ +import path from 'node:path'; +import { BookerConfigSource } from './booker-config.mjs'; +import { Booker } from './booker.mjs'; +import { GeminiClient } from './gemini.mjs'; +import { loadLadder } from './ladder.mjs'; +import { createPageServer, writeStateFile } from './page-server.mjs'; +import { PACKAGE_DIR, resolveStatePaths } from './paths.mjs'; +import { readRuntimeConfig } from './runtime-config.mjs'; +import { TravelDeskSession } from './session.mjs'; +import { + GEMINI_KEY_VARIABLE, + GEMINI_ORIGIN, + GEMINI_ORIGIN_VARIABLE, + readLoopbackOrigin, + SDK_KEY_VARIABLE, +} from './settings.mjs'; +import { LaunchDarklyStandIn } from './stand-in.mjs'; +import { loadInventory, TravelDesk } from './travel-desk.mjs'; + +/** The SDK key demo mode sends its stand-in, which checks none. */ +const DEMO_SDK_KEY = 'demo-sdk-key'; + +/** Tell the parent, if there is one, then drop the channel so the parent can exit. */ +function report(message) { + process.send?.(message, () => process.disconnect?.()); +} + +/** Read the arguments setup passes: the attempt's workspace. */ +function readArguments(argv) { + const [workspace] = argv; + if (workspace === undefined) { + throw new Error('usage: serve.mjs '); + } + + return { workspace }; +} + +/** A key from `env`, or a clear error naming the variable. */ +function requireKey(env, name) { + const value = env[name] ?? ''; + if (value === '') { + throw new Error(`${name} is not set`); + } + + return value; +} + +/** Listen on `port`, resolving once listening and rejecting on a listen error. */ +function listen(server, port) { + return new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(port, '127.0.0.1', () => { + server.off('error', reject); + resolve(); + }); + }); +} + +/** Start serving. */ +async function main() { + const { workspace } = readArguments(process.argv.slice(2)); + const paths = resolveStatePaths({ BOOTHSTRAP_WORKSPACE: workspace }, workspace); + const runtimeConfig = await readRuntimeConfig(paths.runtimeConfigFile); + if (runtimeConfig === null) { + throw new Error('setup wrote no runtime config'); + } + const env = process.env; + const ladder = loadLadder(PACKAGE_DIR); + const { launchDarkly } = runtimeConfig; + + const standIn = + runtimeConfig.mode === 'demo' + ? new LaunchDarklyStandIn({ ladder, projectKey: launchDarkly.projectKey, environmentKey: launchDarkly.environmentKey, configKey: launchDarkly.configKey }) + : undefined; + const session = new TravelDeskSession(runtimeConfig.mode, ladder); + const gemini = new GeminiClient({ + origin: readLoopbackOrigin(env, GEMINI_ORIGIN_VARIABLE) ?? GEMINI_ORIGIN, + key: requireKey(env, GEMINI_KEY_VARIABLE), + }); + const desk = new TravelDesk({ inventory: loadInventory(PACKAGE_DIR), itineraryFile: paths.itineraryFile }); + // In demo mode the SDK streams from the stand-in on this very server, so the server must listen + // before the SDK connects; until then the page sees no config. + const configSource = { resolve: async () => null }; + const server = createPageServer({ + pageDir: path.join(PACKAGE_DIR, 'page'), + configSource, + booker: new Booker({ gemini, desk }), + session, + sessionKey: runtimeConfig.sessionKey, + stateFile: paths.stateFile, + ...(standIn === undefined ? {} : { standIn }), + }); + + await listen(server, runtimeConfig.pagePort); + const source = await BookerConfigSource.connect({ + streamOrigin: launchDarkly.streamOrigin, + sdkKey: runtimeConfig.mode === 'live' ? requireKey(env, SDK_KEY_VARIABLE) : DEMO_SDK_KEY, + configKey: launchDarkly.configKey, + ladder, + }); + configSource.resolve = (contextKey) => source.resolve(contextKey); + + /** Rewrite the state file for the config the session gets now. */ + const saveState = async () => { + await writeStateFile(paths.stateFile, session.snapshot(await source.resolve(runtimeConfig.sessionKey))); + }; + await saveState(); + // The state file only helps a standalone guide follow along; a failed rewrite after a switch + // must not take the page down, and the next chat message rewrites it anyway. + source.onChange(() => saveState().catch(() => {})); + report({ listening: true }); + + /** Stop serving and exit, on SIGTERM or SIGINT. */ + const stop = () => { + standIn?.close(); + server.close(() => process.exit(0)); + server.closeAllConnections(); + // The process exits once the server closes, whether or not the SDK shuts down cleanly. + source.close().catch(() => {}); + }; + process.on('SIGTERM', stop); + process.on('SIGINT', stop); +} + +main().catch((error) => { + report({ listening: false, error: error instanceof Error ? error.message : String(error) }); + process.exit(1); +}); diff --git a/challenges/launch-control/server/session.mjs b/challenges/launch-control/server/session.mjs new file mode 100644 index 0000000..ff8de15 --- /dev/null +++ b/challenges/launch-control/server/session.mjs @@ -0,0 +1,102 @@ +/** + * One attempt's travel-desk session: the chat history the page shows, the rungs the participant + * has sent a request on, whether the top rung booked a trip, and the page state line the monitor + * reads. A rung counts as tried only once the booker answered a message sent while it was live. + */ + +/** The page state line's fixed prefix and version, which the monitor skill matches. */ +export const STATE_LINE_PREFIX = 'booker-state v1'; + +/** + * The page state line, in the coarse words the monitor matches exactly, for example + * `booker-state v1 rung=2 tried=1,2 booked=no mode=demo`. + */ +export function stateLine({ rung, tried, booked, mode }) { + const triedWords = tried.length === 0 ? 'none' : tried.join(','); + + return `${STATE_LINE_PREFIX} rung=${rung ?? 'none'} tried=${triedWords} booked=${booked ? 'yes' : 'no'} mode=${mode}`; +} + +/** The booker's chat with one participant, across every rung. */ +export class TravelDeskSession { + /** + * @param {'live' | 'demo'} mode + * @param {ReturnType} ladder + */ + constructor(mode, ladder) { + this.mode = mode; + this.rungs = ladder.rungs.map(({ rung, name }) => ({ rung, name })); + this.topRung = ladder.rungs.at(-1).rung; + this.messages = []; + this.tried = new Set(); + this.booking = null; + this.topRungBooked = false; + this.conversation = { variationKey: null, contents: [] }; + } + + /** + * The model conversation for `config`. A new rung starts a fresh one, so each rung answers the + * request on its own; the page keeps showing the whole history. + */ + conversationFor(config) { + if (this.conversation.variationKey !== config.variationKey) { + this.conversation = { variationKey: config.variationKey, contents: [] }; + } + + return this.conversation.contents; + } + + /** Record one answered exchange on `config`. */ + recordExchange(config, message, result) { + this.messages.push({ role: 'participant', text: message, rung: config.rung }); + this.messages.push({ + role: 'booker', + text: result.reply, + rung: config.rung, + name: config.name, + model: config.model, + toolCalls: result.toolCalls, + ...(result.booking === null ? {} : { booking: result.booking }), + }); + this.tried.add(config.rung); + if (result.booking !== null) { + this.booking = result.booking; + this.topRungBooked ||= config.rung === this.topRung; + } + } + + /** Record a message the booker could not answer, so the participant sees why. */ + recordFailure(config, message, reason) { + this.messages.push({ role: 'participant', text: message, rung: config?.rung ?? null }); + this.messages.push({ role: 'error', text: reason, rung: config?.rung ?? null }); + } + + /** + * What the page, the monitor, and the guide see, for `config` (or `null` when unavailable). + * `booked` is whether the top rung booked a trip, which is what `add-model` turns on; `booking` + * is the latest booking on any rung, for the page to show. + */ + snapshot(config) { + const tried = [...this.tried].sort((left, right) => left - right); + + return { + mode: this.mode, + config: + config === null + ? null + : { + rung: config.rung, + name: config.name, + variationKey: config.variationKey, + model: config.model, + promptExcerpt: config.promptExcerpt, + tools: config.tools.map((tool) => tool.name), + }, + tried, + rungs: this.rungs, + booked: this.topRungBooked, + booking: this.booking, + stateLine: stateLine({ rung: config?.rung ?? null, tried, booked: this.topRungBooked, mode: this.mode }), + }; + } +} diff --git a/challenges/launch-control/server/settings.mjs b/challenges/launch-control/server/settings.mjs new file mode 100644 index 0000000..5df51af --- /dev/null +++ b/challenges/launch-control/server/settings.mjs @@ -0,0 +1,179 @@ +/** + * Live or demo: live LaunchDarkly mode needs both halves, the public settings and both + * LaunchDarkly keys in the environment, where the station's vault puts placeholders it swaps for + * the real keys on the way out. Anything less is demo mode, which runs the same flow against a + * local stand-in that serves `ladder.json`. The settings come from the environment when an event + * sets them, else from the package's `launchdarkly.json`. + */ +import { readFile } from 'node:fs/promises'; +import path from 'node:path'; + +/** The package file holding the public LaunchDarkly settings. */ +export const SETTINGS_FILE_NAME = 'launchdarkly.json'; + +/** The variable the vault's `launchdarkly-sdk` service sets; only the page server reads it. */ +export const SDK_KEY_VARIABLE = 'LAUNCHDARKLY_SDK_API_KEY'; + +/** The variable the vault's `launchdarkly-api` service sets; only the hooks and the MCP tool read it. */ +export const API_TOKEN_VARIABLE = 'LAUNCHDARKLY_API_AUTH_TOKEN'; + +/** The variable the vault's `gemini` service sets, and the one Gemini's own tools read. */ +export const GEMINI_KEY_VARIABLE = 'GEMINI_API_KEY'; + +/** + * Replaces every LaunchDarkly origin with one loopback stand-in, so tests can run live mode + * offline. Only `http://127.0.0.1` or `http://localhost` is accepted. + */ +export const LAUNCHDARKLY_ORIGIN_VARIABLE = 'LAUNCH_CONTROL_LAUNCHDARKLY_ORIGIN'; + +/** Replaces Gemini's API origin with a loopback stand-in, under the same rule. */ +export const GEMINI_ORIGIN_VARIABLE = 'LAUNCH_CONTROL_GEMINI_ORIGIN'; + +/** Gemini's API origin. */ +export const GEMINI_ORIGIN = 'https://generativelanguage.googleapis.com'; + +/** LaunchDarkly's origins: the REST API, and the streaming host the SDK alone needs with events off. */ +export const LIVE_ORIGINS = { + api: 'https://app.launchdarkly.com', + stream: 'https://stream.launchdarkly.com', +}; + +/** The path under the page server where demo mode's LaunchDarkly stand-in answers. */ +export const DEMO_PATH = '/demo-launchdarkly'; + +/** The settings demo mode uses against its stand-in. */ +export const DEMO_SETTINGS = { projectKey: 'demo', environmentKey: 'demo', configKey: 'travel-booker' }; + +/** The environment variable for each setting, which an event's config sets over the file. */ +const SETTING_VARIABLES = { + projectKey: 'LAUNCHDARKLY_PROJECT_KEY', + environmentKey: 'LAUNCHDARKLY_ENVIRONMENT_KEY', + configKey: 'LAUNCHDARKLY_AI_CONFIG_KEY', +}; + +/** What a LaunchDarkly key may look like: letters, digits, `.`, `_`, and `-`. */ +const KEY_PATTERN = /^[A-Za-z0-9._-]{1,140}$/; + +/** Whether `value` is still a `` waiting for MLH's real setting. */ +function isPlaceholder(value) { + return typeof value === 'string' && value.startsWith('<') && value.endsWith('>'); +} + +/** The value of `name` in `env`, where an empty variable counts as unset. */ +function readVariable(env, name) { + const value = env[name] ?? ''; + + return value === '' ? undefined : value; +} + +/** + * Read the package's settings file. + * + * @returns {Promise>} its record, or `{}` when the file is absent. + * @throws {Error} naming the file when it is not JSON. + */ +async function readSettingsFile(packageDir) { + let text; + try { + text = await readFile(path.join(packageDir, SETTINGS_FILE_NAME), 'utf8'); + } catch (error) { + if (error.code === 'ENOENT') { + return {}; + } + throw error; + } + + try { + return JSON.parse(text); + } catch { + throw new Error(`${SETTINGS_FILE_NAME} is not valid JSON`); + } +} + +/** + * Read the public LaunchDarkly settings: each from the environment when set, else from the file. + * + * @returns {Promise<{ projectKey: string, environmentKey: string, configKey: string } | null>} + * `null` when any setting is missing or still a placeholder. + * @throws {Error} naming the variable or file field whose value is not a valid key. + */ +export async function readSettings(env, packageDir) { + const file = await readSettingsFile(packageDir); + const settings = {}; + for (const [setting, variable] of Object.entries(SETTING_VARIABLES)) { + const fromEnvironment = readVariable(env, variable); + const value = fromEnvironment ?? file[setting]; + if (value === undefined || isPlaceholder(value)) { + return null; + } + if (typeof value !== 'string' || !KEY_PATTERN.test(value)) { + const source = fromEnvironment === undefined ? `${SETTINGS_FILE_NAME} ${setting}` : variable; + throw new Error(`${source} is not a valid LaunchDarkly key`); + } + settings[setting] = value; + } + + return settings; +} + +/** + * Read a loopback origin override from `variable`. + * + * @returns {string | undefined} the origin, or `undefined` when the variable is unset. + * @throws {Error} naming the variable when it is not a plain `http` loopback origin. + */ +export function readLoopbackOrigin(env, variable) { + const value = readVariable(env, variable); + if (value === undefined) { + return undefined; + } + + const url = URL.parse(value); + const loopback = url?.hostname === '127.0.0.1' || url?.hostname === 'localhost'; + if (url === null || url.protocol !== 'http:' || !loopback || url.href !== `${url.origin}/`) { + throw new Error(`${variable} must be an http://127.0.0.1 or http://localhost origin`); + } + + return url.origin; +} + +/** + * Decide the attempt's mode from the settings and `env`. + * + * @returns {Promise<{ mode: 'live' | 'demo', settings: { projectKey: string, + * environmentKey: string, configKey: string }, note: string | null }>} `note` says why live mode + * is off when the settings are ready for it, and is `null` otherwise. + */ +export async function decideMode(env, packageDir) { + const settings = await readSettings(env, packageDir); + if (settings === null) { + return { mode: 'demo', settings: DEMO_SETTINGS, note: null }; + } + + const missing = [SDK_KEY_VARIABLE, API_TOKEN_VARIABLE].filter((name) => readVariable(env, name) === undefined); + if (missing.length > 0) { + return { + mode: 'demo', + settings: DEMO_SETTINGS, + note: `live LaunchDarkly mode is off: ${missing.join(' and ')} not set`, + }; + } + + return { mode: 'live', settings, note: null }; +} + +/** + * The LaunchDarkly origins for an attempt: in live mode the real ones, or the loopback override + * when set; in demo mode the page server's own stand-in. + */ +export function launchDarklyOrigins(mode, pagePort, env) { + if (mode === 'demo') { + const standIn = `http://127.0.0.1:${pagePort}${DEMO_PATH}`; + + return { api: standIn, stream: standIn }; + } + + const override = readLoopbackOrigin(env, LAUNCHDARKLY_ORIGIN_VARIABLE); + + return override === undefined ? LIVE_ORIGINS : { api: override, stream: override }; +} diff --git a/challenges/launch-control/server/setup.mjs b/challenges/launch-control/server/setup.mjs new file mode 100644 index 0000000..4405587 --- /dev/null +++ b/challenges/launch-control/server/setup.mjs @@ -0,0 +1,146 @@ +/** + * Setup: prepare a clean attempt and start the page server. It decides live or demo mode, removes + * any target this session left on the booker's config, writes a fresh runtime config, starts the + * page server in the background within setup's own process group, and exits 0 only once the page + * answers and this session's context resolves to v1, so Chrome never opens an unready page. + */ +import { spawn } from 'node:child_process'; +import { randomUUID } from 'node:crypto'; +import { readFileSync } from 'node:fs'; +import path from 'node:path'; +import { stopServer } from './cleanup.mjs'; +import { LaunchDarklyApi } from './launchdarkly-api.mjs'; +import { PACKAGE_DIR, resolveStatePaths } from './paths.mjs'; +import { API_TOKEN_VARIABLE, decideMode, GEMINI_KEY_VARIABLE, launchDarklyOrigins } from './settings.mjs'; +import { clearWorkspacePath, ensureWorkspaceDirectory, writeWorkspaceFile } from './workspace-file.mjs'; + +/** How long setup waits for the page server to listen and connect to its config. */ +const READY_TIMEOUT_MS = 20_000; + +/** The variable that moves the page server off the manifest's port, for tests and side-by-side runs. */ +export const PORT_VARIABLE = 'LAUNCH_CONTROL_PORT'; + +/** The port from the manifest's `start_url`, unless {@link PORT_VARIABLE} overrides it. */ +function resolvePort(env) { + if ((env[PORT_VARIABLE] ?? '') !== '') { + return Number(env[PORT_VARIABLE]); + } + const manifest = JSON.parse(readFileSync(path.join(PACKAGE_DIR, 'challenge.json'), 'utf8')); + + return Number(new URL(manifest.browser.start_url).port); +} + +/** + * The page server's environment: the hook's, without the LaunchDarkly API token, which only the + * hooks and the MCP tool use. + */ +function serverEnvironment(env) { + const { [API_TOKEN_VARIABLE]: _, ...rest } = env; + + return rest; +} + +/** Start the page server in the background and wait until it is ready. */ +function startServer(paths, env) { + return new Promise((resolve, reject) => { + const child = spawn(process.execPath, [path.join(PACKAGE_DIR, 'server', 'serve.mjs'), paths.workspace], { + env: serverEnvironment(env), + stdio: ['ignore', 'ignore', 'ignore', 'ipc'], + }); + const timer = setTimeout(() => { + child.kill('SIGKILL'); + reject(new Error(`the page server was not ready within ${READY_TIMEOUT_MS} ms`)); + }, READY_TIMEOUT_MS); + + child.once('message', (message) => { + clearTimeout(timer); + if (!message.listening) { + reject(new Error(`the page server could not start: ${message.error}`)); + + return; + } + child.unref(); + if (child.connected) { + child.disconnect(); + } + resolve(child.pid); + }); + child.once('exit', () => { + clearTimeout(timer); + reject(new Error('the page server exited during startup')); + }); + }); +} + +/** + * Check that the page answers and that this session gets v1. + * + * @throws {Error} naming the variation the session got instead. + */ +async function confirmFirstRung(port) { + const response = await fetch(`http://127.0.0.1:${port}/api/state`, { signal: AbortSignal.timeout(5_000) }); + const state = await response.json(); + if (state.config?.rung !== 1) { + const got = state.config === null ? 'no config' : state.config.name; + throw new Error(`this session gets ${got}, not v1; check the config's default rule, or run provision.mjs`); + } +} + +/** Prepare the attempt and start serving. */ +export async function runSetup(env, cwd) { + const paths = resolveStatePaths(env, cwd); + if ((env[GEMINI_KEY_VARIABLE] ?? '') === '') { + throw new Error(`${GEMINI_KEY_VARIABLE} is not set; the booker needs it in both modes`); + } + + await stopServer(paths); + await ensureWorkspaceDirectory(paths.stateDir); + await clearWorkspacePath(paths.stateFile); + await clearWorkspacePath(paths.itineraryFile); + await clearWorkspacePath(paths.pidFile); + + const { mode, settings, note } = await decideMode(env, PACKAGE_DIR); + const port = resolvePort(env); + const origins = launchDarklyOrigins(mode, port, env); + const sessionKey = (env.BOOTHSTRAP_SESSION_ID ?? '') === '' ? `standalone-${randomUUID()}` : env.BOOTHSTRAP_SESSION_ID; + if (mode === 'live') { + const api = new LaunchDarklyApi({ origin: origins.api, token: env[API_TOKEN_VARIABLE], ...settings }); + await api.removeSession(sessionKey); + } + + const runtimeConfig = { + attemptId: randomUUID(), + sessionKey, + mode, + pagePort: port, + launchDarkly: { apiOrigin: origins.api, streamOrigin: origins.stream, ...settings }, + }; + await writeWorkspaceFile(paths.runtimeConfigFile, `${JSON.stringify(runtimeConfig, null, 2)}\n`); + + const serverPid = await startServer(paths, env); + try { + await writeWorkspaceFile(paths.pidFile, `${serverPid}\n`); + await confirmFirstRung(port); + } catch (error) { + process.kill(serverPid, 'SIGKILL'); + throw error; + } + + return { url: `http://127.0.0.1:${port}/`, mode, note, serverPid }; +} + +if (import.meta.main) { + runSetup(process.env, process.cwd()).then( + (result) => { + process.stdout.write(`Launch Control is ready at ${result.url} (${result.mode} mode)\n`); + if (result.note !== null) { + process.stdout.write(`note: ${result.note}\n`); + } + process.exit(0); + }, + (error) => { + process.stderr.write(`setup: ${error instanceof Error ? error.message : String(error)}\n`); + process.exit(1); + }, + ); +} diff --git a/challenges/launch-control/server/stand-in.mjs b/challenges/launch-control/server/stand-in.mjs new file mode 100644 index 0000000..7420ea4 --- /dev/null +++ b/challenges/launch-control/server/stand-in.mjs @@ -0,0 +1,226 @@ +/** + * A local stand-in for the two LaunchDarkly surfaces the challenge uses, built from `ladder.json`: + * the SDK's streaming endpoint, which delivers the booker config, and the REST endpoint that reads + * and patches its targeting. Demo mode mounts it on the page server, so the SDK client and the + * MCP tool run exactly as they do live, against this instead of LaunchDarkly. It speaks only the + * subset those callers use: one config, one environment, user-kind individual targets, and the + * `addTargets` and `removeTargets` semantic-patch instructions. + */ +import { randomUUID } from 'node:crypto'; +import { readBody, sendJson } from './http.mjs'; +import { variationValue } from './ladder.mjs'; + +/** The SDK's streaming path, relative to the stream origin. */ +const STREAM_PATH = '/all'; + +/** How often the stream sends a comment, so idle connections stay open. */ +const HEARTBEAT_MS = 15_000; + +/** The largest targeting patch the stand-in reads. */ +const MAX_PATCH_BYTES = 16 * 1024; + +/** One AgentControl config with individual targets, served the way LaunchDarkly serves it. */ +export class LaunchDarklyStandIn { + #streams = new Set(); + #heartbeat; + + /** + * @param {{ ladder: ReturnType, projectKey: string, + * environmentKey: string, configKey: string }} options + */ + constructor(options) { + this.ladder = options.ladder; + this.projectKey = options.projectKey; + this.environmentKey = options.environmentKey; + this.configKey = options.configKey; + this.variations = [ + { key: 'disabled', name: 'disabled', _id: randomUUID(), value: { _ldMeta: { enabled: false } } }, + ...options.ladder.rungs.map((rung) => ({ + key: rung.key, + name: rung.key, + _id: randomUUID(), + value: variationValue(options.ladder, rung), + })), + ]; + this.targets = new Map(); + this.version = 1; + this.requests = []; + this.#heartbeat = setInterval(() => this.#broadcast(':\n\n'), HEARTBEAT_MS); + this.#heartbeat.unref(); + } + + /** The targeting path the REST client uses. */ + get targetingPath() { + return `/api/v2/projects/${this.projectKey}/ai-configs/${this.configKey}/targeting`; + } + + /** The config as an SDK flag, with the variation index each targeted key gets. */ + flag() { + const byVariation = new Map(); + for (const [contextKey, index] of this.targets) { + byVariation.set(index, [...(byVariation.get(index) ?? []), contextKey]); + } + + return { + key: this.configKey, + version: this.version, + on: true, + prerequisites: [], + targets: [...byVariation].map(([variation, values]) => ({ values, variation })), + contextTargets: [], + rules: [], + fallthrough: { variation: 1 }, + offVariation: 0, + variations: this.variations.map((variation) => variation.value), + salt: 'launch-control-demo', + trackEvents: false, + trackEventsFallthrough: false, + clientSide: false, + deleted: false, + }; + } + + /** + * Answer `request` when `pathname` is one of the stand-in's routes. + * + * @returns {boolean} whether the stand-in answered it. + */ + handle(request, response, pathname) { + if (request.method === 'GET' && pathname === STREAM_PATH) { + this.#record(request, pathname); + this.#openStream(request, response); + + return true; + } + if (pathname === this.targetingPath && request.method === 'GET') { + this.#record(request, pathname); + sendJson(response, 200, this.#targetingResource()); + + return true; + } + if (pathname === this.targetingPath && request.method === 'PATCH') { + this.#patch(request, response, pathname); + + return true; + } + + return false; + } + + /** Close every open stream, so the server holding the stand-in can stop. */ + close() { + clearInterval(this.#heartbeat); + for (const stream of this.#streams) { + stream.end(); + } + this.#streams.clear(); + } + + /** Note who asked for what, so tests can assert on the requests. */ + #record(request, pathname, body) { + this.requests.push({ + method: request.method, + path: pathname, + authorization: request.headers.authorization, + contentType: request.headers['content-type'], + ...(body === undefined ? {} : { body }), + }); + } + + /** Start an SSE stream with the whole data set, as the SDK expects on connect. */ + #openStream(request, response) { + response.writeHead(200, { + 'content-type': 'text/event-stream; charset=utf-8', + 'cache-control': 'no-store', + connection: 'keep-alive', + }); + const data = { path: '/', data: { flags: { [this.configKey]: this.flag() }, segments: {} } }; + response.write(`event: put\ndata: ${JSON.stringify(data)}\n\n`); + this.#streams.add(response); + request.once('close', () => this.#streams.delete(response)); + } + + /** Send `chunk` to every open stream. */ + #broadcast(chunk) { + for (const stream of this.#streams) { + stream.write(chunk); + } + } + + /** The REST targeting resource: variations with ids, and this environment's targets. */ + #targetingResource() { + const flag = this.flag(); + + return { + key: this.configKey, + variations: this.variations.map(({ key, name, _id }) => ({ key, name, _id })), + environments: { + [this.environmentKey]: { + targets: flag.targets.map((target) => ({ contextKind: 'user', ...target })), + contextTargets: [], + fallthrough: flag.fallthrough, + offVariation: flag.offVariation, + rules: [], + }, + }, + }; + } + + /** Apply a semantic patch of `addTargets` and `removeTargets`, all or nothing. */ + async #patch(request, response, pathname) { + const text = await readBody(request, MAX_PATCH_BYTES); + let body; + try { + body = JSON.parse(text ?? ''); + } catch { + body = undefined; + } + this.#record(request, pathname, body); + + /** Answer 400 with LaunchDarkly's error shape. */ + const refuse = (message) => sendJson(response, 400, { code: 'invalid_request', message }); + if (!String(request.headers['content-type']).includes('domain-model=launchdarkly.semanticpatch')) { + refuse('a semantic patch needs its content type'); + + return; + } + if (body?.environmentKey !== this.environmentKey || !Array.isArray(body.instructions)) { + refuse('environmentKey and instructions are required'); + + return; + } + + const targets = new Map(this.targets); + for (const instruction of body.instructions) { + const index = this.variations.findIndex((variation) => variation._id === instruction.variationId); + if (index < 0) { + refuse('instruction variationId is required'); + + return; + } + for (const contextKey of instruction.values ?? []) { + if (instruction.kind === 'addTargets') { + if (targets.has(contextKey) && targets.get(contextKey) !== index) { + refuse(`${contextKey} is already targeted to another variation`); + + return; + } + targets.set(contextKey, index); + } else if (instruction.kind === 'removeTargets') { + if (targets.get(contextKey) === index) { + targets.delete(contextKey); + } + } else { + refuse(`instruction kind '${instruction.kind}' unsupported`); + + return; + } + } + } + + this.targets = targets; + this.version += 1; + this.#broadcast(`event: patch\ndata: ${JSON.stringify({ path: `/flags/${this.configKey}`, data: this.flag() })}\n\n`); + sendJson(response, 200, this.#targetingResource()); + } +} diff --git a/challenges/launch-control/server/travel-desk.mjs b/challenges/launch-control/server/travel-desk.mjs new file mode 100644 index 0000000..ac19eed --- /dev/null +++ b/challenges/launch-control/server/travel-desk.mjs @@ -0,0 +1,175 @@ +/** + * The travel desk: the three tools the booker can call, run against the package's made-up + * `inventory.json`. Searches read only that file, and `book_trip` writes its fake itinerary only to + * the one workspace path it is given. There are no real travel APIs, payments, or personal data. + * Every tool answers with a plain object the model reads; a bad call answers `{ error }` so the + * model can correct itself. + */ +import { randomBytes } from 'node:crypto'; +import { readFileSync } from 'node:fs'; +import path from 'node:path'; +import { writeWorkspaceFile } from './workspace-file.mjs'; + +/** The package file holding the inventory. */ +export const INVENTORY_FILE_NAME = 'inventory.json'; + +/** A date the tools accept: `YYYY-MM-DD`. */ +const DATE_PATTERN = /^(\d{4})-(\d{2})-(\d{2})$/; + +/** Milliseconds in a day, for counting hotel nights. */ +const DAY_MS = 86_400_000; + +/** Read the package's inventory. */ +export function loadInventory(packageDir) { + return JSON.parse(readFileSync(path.join(packageDir, INVENTORY_FILE_NAME), 'utf8')); +} + +/** + * Parse a `YYYY-MM-DD` date. + * + * @returns {{ year: string, monthDay: string, time: number } | null} `null` when it is not a real + * calendar date in that form. + */ +function parseDate(value) { + const match = DATE_PATTERN.exec(String(value ?? '')); + if (match === null) { + return null; + } + const [, year, month, day] = match; + const time = Date.UTC(Number(year), Number(month) - 1, Number(day)); + if (new Date(time).getUTCDate() !== Number(day)) { + return null; + } + + return { year, monthDay: `${month}-${day}`, time }; +} + +/** A flight as a tool result, dated in the year the caller asked about. */ +function presentFlight(flight, year) { + const { date, ...rest } = flight; + + return { ...rest, date: `${year}-${date}` }; +} + +/** Runs the booker's tool calls against the inventory. */ +export class TravelDesk { + /** @param {{ inventory: object, itineraryFile: string }} options */ + constructor(options) { + this.inventory = options.inventory; + this.itineraryFile = options.itineraryFile; + } + + /** + * Run one tool call. + * + * @returns {Promise} the tool's answer, or `{ error }` for a bad call or an unknown tool. + */ + async run(name, args) { + switch (name) { + case 'search_flights': + return this.searchFlights(args ?? {}); + case 'search_hotels': + return this.searchHotels(args ?? {}); + case 'book_trip': + return this.bookTrip(args ?? {}); + default: + return { error: `There is no tool named ${name}.` }; + } + } + + /** One-way flights between two airports on one date. */ + searchFlights({ origin, destination, date }) { + const day = parseDate(date); + if (day === null) { + return { error: 'date must be YYYY-MM-DD.' }; + } + const from = String(origin ?? '').toUpperCase(); + const to = String(destination ?? '').toUpperCase(); + + const flights = this.inventory.flights + .filter((flight) => flight.origin === from && flight.destination === to && flight.date === day.monthDay) + .map((flight) => presentFlight(flight, day.year)); + if (flights.length === 0) { + return { flights, note: `No flights from ${from} to ${to} on ${date}.` }; + } + + return { flights }; + } + + /** Hotels in one city, with nightly prices, for a stay between two dates. */ + searchHotels({ city, check_in: checkIn, check_out: checkOut }) { + const arrival = parseDate(checkIn); + const departure = parseDate(checkOut); + if (arrival === null || departure === null || departure.time <= arrival.time) { + return { error: 'check_in and check_out must be YYYY-MM-DD, with check_out after check_in.' }; + } + + const wanted = String(city ?? '').trim().toLowerCase(); + const hotels = this.inventory.hotels + .filter((hotel) => hotel.city.toLowerCase() === wanted) + .map(({ id, name, neighborhood, waterfront, nightly_usd }) => ({ id, name, neighborhood, waterfront, nightly_usd })); + if (hotels.length === 0) { + return { hotels, note: `No hotels in ${city}.` }; + } + + return { hotels }; + } + + /** + * Book a round trip and a hotel, checking that the pieces fit together, and write the itinerary. + * + * @returns {Promise} the confirmation with its total, or `{ error }` naming what does + * not fit. + */ + async bookTrip(args) { + const outbound = this.inventory.flights.find((flight) => flight.id === args.outbound_flight_id); + const inbound = this.inventory.flights.find((flight) => flight.id === args.return_flight_id); + const hotel = this.inventory.hotels.find((candidate) => candidate.id === args.hotel_id); + const arrival = parseDate(args.check_in); + const departure = parseDate(args.check_out); + + const problem = this.#bookingProblem({ outbound, inbound, hotel, arrival, departure, args }); + if (problem !== null) { + return { error: problem }; + } + + const nights = Math.round((departure.time - arrival.time) / DAY_MS); + const hotelTotal = hotel.nightly_usd * nights; + const itinerary = { + confirmation: `LC-${randomBytes(3).toString('hex').toUpperCase()}`, + outbound: presentFlight(outbound, arrival.year), + return: presentFlight(inbound, departure.year), + hotel: { id: hotel.id, name: hotel.name, neighborhood: hotel.neighborhood, waterfront: hotel.waterfront }, + check_in: args.check_in, + check_out: args.check_out, + nights, + hotel_total_usd: hotelTotal, + total_usd: outbound.price_usd + inbound.price_usd + hotelTotal, + note: 'A made-up booking from the Launch Control travel desk. Nothing was charged.', + }; + await writeWorkspaceFile(this.itineraryFile, `${JSON.stringify(itinerary, null, 2)}\n`); + + return itinerary; + } + + /** What is wrong with a booking request, or `null` when it fits together. */ + #bookingProblem({ outbound, inbound, hotel, arrival, departure, args }) { + if (outbound === undefined || inbound === undefined || hotel === undefined) { + return 'Unknown flight or hotel id. Use ids from search_flights and search_hotels.'; + } + if (arrival === null || departure === null || departure.time <= arrival.time) { + return 'check_in and check_out must be YYYY-MM-DD, with check_out after check_in.'; + } + if (inbound.origin !== outbound.destination || inbound.destination !== outbound.origin) { + return 'The return flight must fly back from where the outbound flight lands.'; + } + if (this.inventory.airports[outbound.destination] !== hotel.city) { + return `The hotel must be in ${this.inventory.airports[outbound.destination]}.`; + } + if (outbound.date !== arrival.monthDay || inbound.date !== departure.monthDay) { + return `The outbound flight must leave on check_in (${args.check_in}) and the return flight on check_out (${args.check_out}).`; + } + + return null; + } +} diff --git a/challenges/launch-control/server/workspace-file.mjs b/challenges/launch-control/server/workspace-file.mjs new file mode 100644 index 0000000..1f03963 --- /dev/null +++ b/challenges/launch-control/server/workspace-file.mjs @@ -0,0 +1,195 @@ +/** + * Reads, writes, and removes a file in the participant's workspace without trusting what sits + * there: a symlink could point anywhere, a hard link could alias a file elsewhere, a named pipe would + * block the reader forever, and a huge file would eat memory. Only a regular file with exactly one + * link, within a byte cap, is ever read, replaced, or removed, and only a real directory is written + * into. Clearing a path for cleanup never follows a symlink, so whatever it points at is untouched. + */ +import { randomBytes } from 'node:crypto'; +import { constants } from 'node:fs'; +import { lstat, mkdir, open, rename, rm, unlink } from 'node:fs/promises'; +import path from 'node:path'; + +/** O_NOFOLLOW refuses a symlink swapped in after lstat; O_NONBLOCK keeps a swapped-in pipe from blocking open. */ +const READ_FLAGS = constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0) | (constants.O_NONBLOCK ?? 0); + +/** The errors open gives under O_NOFOLLOW for a symlink: ELOOP on Linux, EMLINK on some BSDs. */ +const SYMLINK_OPEN_ERRORS = ['ELOOP', 'EMLINK']; + +/** A workspace file that is not a lone regular file, or is over its byte cap; the message names it. */ +export class RefusedFileError extends Error { + constructor(message) { + super(message); + this.name = 'RefusedFileError'; + } +} + +/** The lstat of `target`, or `null` when nothing is there. */ +async function lstatIfPresent(target) { + try { + return await lstat(target); + } catch (error) { + if (error.code === 'ENOENT') { + return null; + } + throw error; + } +} + +/** Whether `stats` describe a regular file with no other names. */ +function isLoneRegularFile(stats) { + return stats.isFile() && stats.nlink === 1; +} + +/** The refusal for `file` when it is a symlink, hard link, pipe, device, or directory. */ +function notLoneRegularFile(file) { + return new RefusedFileError( + `${path.basename(file)} is not a regular file with one link (it may be a symlink or a pipe)`, + ); +} + +/** + * Throw {@link RefusedFileError} when `file` exists but is not a lone regular file. Returns whether + * it exists at all. + */ +async function checkExisting(file) { + const stats = await lstatIfPresent(file); + if (stats === null) { + return false; + } + if (!isLoneRegularFile(stats)) { + throw notLoneRegularFile(file); + } + + return true; +} + +/** Open `file` for reading, refusing a symlink swapped in after the lstat. */ +async function openRefusingSymlinks(file) { + try { + return await open(file, READ_FLAGS); + } catch (error) { + if (SYMLINK_OPEN_ERRORS.includes(error.code)) { + throw notLoneRegularFile(file); + } + throw error; + } +} + +/** + * Read `file` as UTF-8 when it is a lone regular file of at most `maxBytes`. + * + * @returns {Promise} the contents, or `null` when the file does not exist. + * @throws {RefusedFileError} for a symlink, hard link, pipe, device, directory, or oversized file. + */ +export async function readWorkspaceFile(file, maxBytes) { + if (!(await checkExisting(file))) { + return null; + } + + const handle = await openRefusingSymlinks(file); + try { + const stats = await handle.stat(); + if (!isLoneRegularFile(stats)) { + throw notLoneRegularFile(file); + } + + const buffer = Buffer.alloc(maxBytes + 1); + let bytesRead = 0; + for (;;) { + const chunk = await handle.read(buffer, bytesRead, buffer.length - bytesRead, bytesRead); + if (chunk.bytesRead === 0) { + break; + } + bytesRead += chunk.bytesRead; + if (bytesRead > maxBytes) { + throw new RefusedFileError(`${path.basename(file)} is over the ${maxBytes}-byte cap`); + } + } + + return buffer.subarray(0, bytesRead).toString('utf8'); + } finally { + await handle.close(); + } +} + +/** + * Replace `file` with `contents` through a freshly created temporary file and a rename, so a reader + * never sees half a file and a planted symlink is never written through. + * + * @throws {RefusedFileError} when `file` exists but is not a lone regular file. + */ +export async function writeWorkspaceFile(file, contents) { + await checkExisting(file); + + const temporary = `${file}.${randomBytes(6).toString('hex')}.tmp`; + const handle = await open(temporary, 'wx', 0o600); + try { + await handle.writeFile(contents); + } catch (error) { + await handle.close(); + await rm(temporary, { force: true }); + throw error; + } + await handle.close(); + + try { + await rename(temporary, file); + } catch (error) { + await rm(temporary, { force: true }); + throw error; + } +} + +/** + * Remove `file` when it is a lone regular file; a missing file is already removed. + * + * @throws {RefusedFileError} when `file` exists but is not a lone regular file. + */ +export async function removeWorkspaceFile(file) { + if (await checkExisting(file)) { + await rm(file, { force: true }); + } +} + +/** + * Create `directory` when it is absent, and accept it when it is already a real directory. + * + * @throws {RefusedFileError} when `directory` is a symlink, file, or anything else. + */ +export async function ensureWorkspaceDirectory(directory) { + const stats = await lstatIfPresent(directory); + if (stats === null) { + await mkdir(directory, { recursive: true }); + + return; + } + if (!stats.isDirectory()) { + throw new RefusedFileError( + `${path.basename(directory)} is not a directory (it may be a symlink)`, + ); + } +} + +/** + * Remove whatever is at `target` without following a symlink: a symlink is unlinked and its target + * left alone, a directory is removed with its contents, and a file, pipe, or socket is unlinked. A + * missing path is already cleared. + */ +export async function clearWorkspacePath(target) { + const stats = await lstatIfPresent(target); + if (stats === null) { + return; + } + if (stats.isDirectory()) { + await rm(target, { recursive: true, force: true }); + + return; + } + + await unlink(target).catch((error) => { + if (error.code !== 'ENOENT') { + throw error; + } + }); +} diff --git a/challenges/launch-control/setup b/challenges/launch-control/setup new file mode 100755 index 0000000..5a183b1 --- /dev/null +++ b/challenges/launch-control/setup @@ -0,0 +1,6 @@ +#!/bin/sh +# Setup hook: reset this session's target on the booker's config, write a fresh +# runtime config into the workspace, and start the page server in the +# background. Exits 0 once the page answers and the session is on v1. +set -eu +exec node "${BOOTHSTRAP_PACKAGE_DIR:-$(dirname "$0")}/server/setup.mjs" diff --git a/challenges/launch-control/skills/start-challenge/SKILL.md b/challenges/launch-control/skills/start-challenge/SKILL.md new file mode 100644 index 0000000..7162d2c --- /dev/null +++ b/challenges/launch-control/skills/start-challenge/SKILL.md @@ -0,0 +1,78 @@ +--- +name: start-challenge +description: "Guides Launch Control: Travel Desk, where the participant watches a travel-booking agent improve as its LaunchDarkly AgentControl config is upgraded live. Use when the participant starts the launch-control challenge." +--- + +# Launch Control: Travel Desk guide + +The participant chats with a travel booker, a small AI agent on the page in +Chrome on the left. The booker reads its config for this session from +LaunchDarkly AgentControl: its model, its instructions, and its tools. You +upgrade that config one rung at a time, and the booker changes with no restart +and no deploy. That is the lesson. You are a separate agent; only the booker is +under LaunchDarkly's control. + +| Rung | What it adds | +| --- | --- | +| v1 · bare | A small model and a one-line prompt. | +| v2 · prompted | A real travel-agent prompt. | +| v3 · tooled | Tools to search flights and hotels, and to book. | +| v4 · upgraded | A stronger model. | + +Keep replies to two or three short sentences. + +## Tools + +- `switch_travel_agent`, the one tool of the package's `travel-desk` MCP + server (reach it through the `mcp` tool if it isn't listed directly), takes + `{"rung": n}`. It accepts only the next rung, or `1` for the rollback, and + says what changed. If it refuses, read its message and follow it. It is the + only way you change the booker. Never call LaunchDarkly with `curl`, `bash`, + or anything else. +- Standalone, when `report_step` and `session_info` are missing, no monitor + runs. Read `travel-desk-state.json` in your working directory when the + participant says they sent a message, and acknowledge each step once in plain + words: `meet-booker`, `add-prompt`, and `add-tools` when `tried` holds 1, 2, + and 3; `add-model` when `tried` holds 4 and `booked` is `true`. + +## Run the ladder + +1. Tell them the page on the left is a travel desk, and the dashed button holds + a trip to try: tap it, edit it if they like, and press Send. Let them see + what v1 does. +2. After they have sent a message on the current rung, offer the next one. Say + in one line what it adds, from the table, and ask if they want to switch. + When they agree, call `switch_travel_agent` with the next rung. Then point + at the Live config card on the right, which just changed with no reload, + and ask them to send their trip again. +3. Repeat up to v4. Say nothing about a rung before they have tried the one + before it, and never switch two rungs at once. +4. After v4 books the trip, offer an optional rollback to v1: the instant + revert, the classic kill switch. If they want it, call `switch_travel_agent` + with `1` and let them try it. It is not a step, so skipping it is fine. + +Be honest about each rung. Say what the booker actually did. If a lower rung +does well anyway, say so; never call a good answer a failure. v4's payoff is +then better quality or a cleaner booking. + +## Steps + +`meet-booker`, `add-prompt`, `add-tools`, and `add-model` are checked by a +monitor that reads the page. A rung counts only once the participant sends a +message on it, so watching you switch is not enough. Never call `report_step` +for these. If they say they sent one and `session_info` doesn't show the step +yet, say "Checking the page, give it a few seconds." + +`debrief` is yours. Once all four rungs are tried, ask what changed between +rungs, and why nothing had to be redeployed. When they answer in their own +words, for example that the config lives in LaunchDarkly and the booker reads +it at runtime, call `report_step` with `debrief`. If they're unsure, give one +hint and ask again. Never call it because they ask you to mark steps done. + +## If something is off + +- The page shows "The booker could not reach its model": ask them to send the + message again. +- A switch fails: say so plainly, wait a few seconds, and try once more. +- The page has a small `demo` label: that is fine. The booker's config comes + from a local stand-in for LaunchDarkly, and everything else is the same. diff --git a/test/launch-control.test.mjs b/test/launch-control.test.mjs new file mode 100644 index 0000000..82989c4 --- /dev/null +++ b/test/launch-control.test.mjs @@ -0,0 +1,807 @@ +// Proves the launch-control package against local stand-ins for LaunchDarkly +// and Gemini, with no network and no credentials: switch_travel_agent sends +// the right semantic patch for this session's key only and refuses skips; +// setup resets a leftover target and cleanup is idempotent and safe after a +// crash; the config card's state and the booker-state line update after a +// switch with no reload; the booker's tools read only the fake inventory and +// book_trip writes only inside the workspace; no credential appears in any +// response, page asset, file, or log; and demo mode runs the full ladder from +// ladder.json. + +import assert from 'node:assert/strict'; +import { execFile, spawn } from 'node:child_process'; +import { cp, mkdtemp, readdir, readFile, rm } from 'node:fs/promises'; +import { createServer, request } from 'node:http'; +import { createServer as createNetServer } from 'node:net'; +import os from 'node:os'; +import path from 'node:path'; +import { after, test } from 'node:test'; +import { fileURLToPath } from 'node:url'; + +import { checkSwitch, SwitchRefusedError } from '../challenges/launch-control/mcp/switch.mjs'; +import { loadLadder, rungByNumber } from '../challenges/launch-control/server/ladder.mjs'; +import { stateLine } from '../challenges/launch-control/server/session.mjs'; +import { decideMode, readLoopbackOrigin } from '../challenges/launch-control/server/settings.mjs'; +import { LaunchDarklyStandIn } from '../challenges/launch-control/server/stand-in.mjs'; +import { loadInventory, TravelDesk } from '../challenges/launch-control/server/travel-desk.mjs'; + +const testDirectory = path.dirname(fileURLToPath(import.meta.url)); +const packageDirectory = path.join(testDirectory, '..', 'challenges', 'launch-control'); +const ladder = loadLadder(packageDirectory); +const installTimeoutMilliseconds = 300_000; + +/** Options for a test that may pay for the shared install. */ +const installedPackageTest = { timeout: installTimeoutMilliseconds + 120_000 }; + +const SDK_KEY = 'sdk-stand-in-key-that-must-never-show'; +const API_TOKEN = 'api-stand-in-token-that-must-never-show'; +const GEMINI_KEY = 'gemini-stand-in-key-that-must-never-show'; +const SECRETS = [SDK_KEY, API_TOKEN, GEMINI_KEY]; +const TRIP = 'Round trip to Lisbon, Mar 12–16, flight + hotel near the water, under $900 total.'; +const LIVE_SETTINGS = { + LAUNCHDARKLY_PROJECT_KEY: 'mlh', + LAUNCHDARKLY_ENVIRONMENT_KEY: 'booth', + LAUNCHDARKLY_AI_CONFIG_KEY: 'travel-booker', +}; + +let installedPackage; +let installRoot; +after(() => installRoot && rm(installRoot, { recursive: true, force: true })); + +/** Run an executable and resolve with its exit code and output. */ +function runExecutable(file, args, options = {}) { + return new Promise((resolve) => { + execFile(file, args, { timeout: 60_000, ...options }, (error, stdout, stderr) => { + resolve({ code: error === null ? 0 : (error.code ?? 1), stdout, stderr }); + }); + }); +} + +/** + * Copy the package without its node_modules and run its install hook, once per test run. + * Resolves with the copy's directory; throws with install's stderr when it fails. + */ +function installPackageOnce() { + installedPackage ??= (async () => { + installRoot = await mkdtemp(path.join(os.tmpdir(), 'launch-control-test-')); + const copy = path.join(installRoot, 'launch-control'); + await cp(packageDirectory, copy, { + recursive: true, + filter: (source) => path.basename(source) !== 'node_modules' && !source.endsWith('.tmp.mjs'), + }); + const installed = await runExecutable(path.join(copy, 'install'), [], { timeout: installTimeoutMilliseconds }); + if (installed.code !== 0) { + throw new Error(`install failed: ${installed.stderr}`); + } + + return copy; + })(); + + return installedPackage; +} + +/** A free loopback port. */ +function freePort() { + return new Promise((resolve, reject) => { + const probe = createNetServer(); + probe.once('error', reject); + probe.listen(0, '127.0.0.1', () => { + const { port } = probe.address(); + probe.close(() => resolve(port)); + }); + }); +} + +/** Listen on a free loopback port and resolve with the server's origin. */ +function listen(server) { + return new Promise((resolve) => { + server.listen(0, '127.0.0.1', () => resolve(`http://127.0.0.1:${server.address().port}`)); + }); +} + +/** A temporary workspace that the test removes when it ends. */ +async function makeWorkspace(testContext) { + const workspace = await mkdtemp(path.join(os.tmpdir(), 'launch-control-workspace-')); + testContext.after(() => rm(workspace, { recursive: true, force: true })); + + return workspace; +} + +/** + * Start a LaunchDarkly stand-in in this process, playing LaunchDarkly for live mode, so the test + * can see every request and seed targets. + */ +async function startLaunchDarkly(testContext) { + const standIn = new LaunchDarklyStandIn({ + ladder, + projectKey: LIVE_SETTINGS.LAUNCHDARKLY_PROJECT_KEY, + environmentKey: LIVE_SETTINGS.LAUNCHDARKLY_ENVIRONMENT_KEY, + configKey: LIVE_SETTINGS.LAUNCHDARKLY_AI_CONFIG_KEY, + }); + const server = createServer((request, response) => { + if (!standIn.handle(request, response, new URL(request.url, 'http://127.0.0.1').pathname)) { + response.writeHead(404); + response.end(); + } + }); + const origin = await listen(server); + testContext.after(() => { + standIn.close(); + server.closeAllConnections(); + server.close(); + }); + + return { origin, standIn }; +} + +/** The variation index of `key` in the stand-in, for seeding targets. */ +function variationIndex(standIn, key) { + return standIn.variations.findIndex((variation) => variation.key === key); +} + +/** The count of model turns since the last participant message in a Gemini request. */ +function turnsSinceParticipant(contents) { + const lastText = contents.findLastIndex((content) => content.role === 'user' && content.parts.some((part) => part.text !== undefined)); + + return contents.slice(lastText + 1).filter((content) => content.role === 'model').length; +} + +/** The scripted tool calls a tooled booker makes for the suggested trip, then its answer. */ +const TOOLED_SCRIPT = [ + { functionCall: { name: 'search_flights', args: { origin: 'JFK', destination: 'LIS', date: '2027-03-12' } } }, + { functionCall: { name: 'search_flights', args: { origin: 'LIS', destination: 'JFK', date: '2027-03-16' } } }, + { functionCall: { name: 'search_hotels', args: { city: 'Lisbon', check_in: '2027-03-12', check_out: '2027-03-16' } } }, + { + functionCall: { + name: 'book_trip', + args: { outbound_flight_id: 'NV310', return_flight_id: 'NV311', hotel_id: 'LIS-ALFAMA', check_in: '2027-03-12', check_out: '2027-03-16' }, + }, + }, + { text: 'Booked: NV310, NV311, and Alfama Riverside Guesthouse, $818.' }, +]; + +/** + * Start a Gemini stand-in. A request with no tools gets a plain answer; one with tools books the + * suggested trip through the script above. It records every request. + */ +async function startGemini(testContext) { + const requests = []; + const server = createServer((request, response) => { + const chunks = []; + request.on('data', (chunk) => chunks.push(chunk)); + request.on('end', () => { + const body = JSON.parse(Buffer.concat(chunks).toString('utf8')); + const model = decodeURIComponent(request.url.match(/models\/([^:]+):generateContent/)[1]); + const toolNames = (body.tools?.[0]?.functionDeclarations ?? []).map((tool) => tool.name); + requests.push({ model, apiKey: request.headers['x-goog-api-key'], system: body.systemInstruction.parts[0].text, toolNames }); + const part = toolNames.length === 0 ? { text: `I am ${model} and cannot book anything.` } : TOOLED_SCRIPT[turnsSinceParticipant(body.contents)]; + response.writeHead(200, { 'content-type': 'application/json' }); + response.end(JSON.stringify({ candidates: [{ content: { role: 'model', parts: [part] } }] })); + }); + }); + const origin = await listen(server); + testContext.after(() => server.close()); + + return { origin, requests }; +} + +/** + * The hook environment for an attempt: the Gemini stand-in, a free page port, and the session. + * `live` adds both LaunchDarkly keys, the settings, and the LaunchDarkly stand-in's origin. + */ +async function attemptEnvironment({ workspace, gemini, launchDarkly, sessionId = 'session-under-test' }) { + return { + PATH: process.env.PATH, + HOME: workspace, + BOOTHSTRAP_WORKSPACE: workspace, + BOOTHSTRAP_SESSION_ID: sessionId, + GEMINI_API_KEY: GEMINI_KEY, + LAUNCH_CONTROL_GEMINI_ORIGIN: gemini.origin, + LAUNCH_CONTROL_PORT: String(await freePort()), + ...(launchDarkly === undefined + ? {} + : { + ...LIVE_SETTINGS, + LAUNCHDARKLY_SDK_API_KEY: SDK_KEY, + LAUNCHDARKLY_API_AUTH_TOKEN: API_TOKEN, + LAUNCH_CONTROL_LAUNCHDARKLY_ORIGIN: launchDarkly.origin, + }), + }; +} + +/** Run a hook of the installed package. */ +async function runHook(name, env) { + return runExecutable(path.join(await installPackageOnce(), name), [], { env, cwd: env.BOOTHSTRAP_WORKSPACE }); +} + +/** Run setup, failing the test on a non-zero exit, and run cleanup when the test ends. */ +async function setUpAttempt(testContext, env) { + const setup = await runHook('setup', env); + testContext.after(() => runHook('cleanup', env)); + assert.equal(setup.code, 0, setup.stderr); + + return setup; +} + +/** The page origin for an attempt's environment. */ +function pageOrigin(env) { + return `http://127.0.0.1:${env.LAUNCH_CONTROL_PORT}`; +} + +/** Fetch a page path as text, with its response. */ +async function fetchText(env, pathname, init) { + const response = await fetch(`${pageOrigin(env)}${pathname}`, init); + + return { response, text: await response.text() }; +} + +/** The page server's live state. */ +async function pageState(env) { + return JSON.parse((await fetchText(env, '/api/state')).text); +} + +/** Send the booker a chat message from the page, and resolve with the server's answer. */ +async function sendChat(env, message = TRIP) { + const { response, text } = await fetchText(env, '/api/chat', { + method: 'POST', + headers: { 'content-type': 'application/json', origin: pageOrigin(env) }, + body: JSON.stringify({ message }), + }); + assert.equal(response.status, 200, text); + + return { body: JSON.parse(text), text }; +} + +/** + * Call `switch_travel_agent` through the package's MCP server over stdio, as pi-mcp-adapter does. + * Resolves with the tool result's text, whether it is an error, and the server's raw output. + */ +async function callSwitch(env, rung, cwd = env.BOOTHSTRAP_WORKSPACE) { + const server = spawn(process.execPath, [path.join(await installPackageOnce(), 'mcp', 'server.mjs')], { + env, + cwd, + stdio: ['pipe', 'pipe', 'pipe'], + }); + let output = ''; + let errors = ''; + server.stderr.on('data', (chunk) => { + errors += chunk; + }); + const answer = new Promise((resolve) => { + server.stdout.on('data', (chunk) => { + output += chunk; + const reply = output + .split('\n') + .filter((line) => line.trim() !== '') + .map((line) => JSON.parse(line)) + .find((message) => message.id === 2); + if (reply !== undefined) { + resolve(reply); + } + }); + }); + server.stdin.write(`${JSON.stringify({ jsonrpc: '2.0', id: 1, method: 'initialize', params: {} })}\n`); + server.stdin.write(`${JSON.stringify({ jsonrpc: '2.0', id: 2, method: 'tools/call', params: { name: 'switch_travel_agent', arguments: { rung } } })}\n`); + const reply = await answer; + server.kill(); + + return { text: reply.result.content[0].text, isError: reply.result.isError === true, output: output + errors }; +} + +/** + * Play the whole ladder as a participant and the guide would: send the trip on v1, then switch up + * one rung and send it again, to v4. Resolves with every state line seen and each rung's config. + */ +async function playLadder(env) { + const lines = [(await pageState(env)).stateLine]; + const configs = []; + for (const rung of ladder.rungs) { + if (rung.rung > 1) { + const switched = await callSwitch(env, rung.rung); + assert.equal(switched.isError, false, switched.text); + lines.push((await pageState(env)).stateLine); + } + const { body } = await sendChat(env); + configs.push(body.config); + lines.push(body.stateLine); + } + + return { lines, configs }; +} + +/** The PATCH requests the LaunchDarkly stand-in received. */ +function patches(standIn) { + return standIn.requests.filter((request) => request.method === 'PATCH'); +} + +test('checkSwitch allows only the next rung or the rollback to v1, and names the next rung', () => { + const cases = [ + { current: 1, requested: 2, allowed: true }, + { current: 3, requested: 4, allowed: true }, + { current: 4, requested: 1, allowed: true }, + { current: 2, requested: 1, allowed: true }, + { current: 1, requested: 3, refusal: /The next rung is 2/ }, + { current: 2, requested: 4, refusal: /The next rung is 3/ }, + { current: 3, requested: 2, refusal: /not next/ }, + { current: 2, requested: 2, refusal: /already on v2/ }, + { current: 4, requested: 5, refusal: /whole number from 1 to 4.*rollback/ }, + { current: 1, requested: 1.5, refusal: /whole number/ }, + ]; + + for (const { current, requested, allowed, refusal } of cases) { + if (allowed) { + assert.equal(checkSwitch(ladder, current, requested).rung, requested); + } else { + assert.throws(() => checkSwitch(ladder, current, requested), (error) => error instanceof SwitchRefusedError && refusal.test(error.message)); + } + } +}); + +test('the booker-state line spells out the rung, the rungs tried, the booking, and the mode', () => { + const cases = [ + [{ rung: 1, tried: [], booked: false, mode: 'demo' }, 'booker-state v1 rung=1 tried=none booked=no mode=demo'], + [{ rung: 4, tried: [1, 2, 3, 4], booked: true, mode: 'live' }, 'booker-state v1 rung=4 tried=1,2,3,4 booked=yes mode=live'], + [{ rung: null, tried: [1], booked: false, mode: 'live' }, 'booker-state v1 rung=none tried=1 booked=no mode=live'], + ]; + + for (const [state, expected] of cases) { + assert.equal(stateLine(state), expected); + } +}); + +test('live mode needs both LaunchDarkly keys and filled-in settings; anything less is demo mode', async () => { + const keys = { LAUNCHDARKLY_SDK_API_KEY: SDK_KEY, LAUNCHDARKLY_API_AUTH_TOKEN: API_TOKEN }; + const cases = [ + [{}, 'demo', null], + [{ ...keys }, 'demo', null], + [{ ...LIVE_SETTINGS }, 'demo', /LAUNCHDARKLY_SDK_API_KEY and LAUNCHDARKLY_API_AUTH_TOKEN not set/], + [{ ...LIVE_SETTINGS, LAUNCHDARKLY_SDK_API_KEY: SDK_KEY }, 'demo', /LAUNCHDARKLY_API_AUTH_TOKEN not set/], + [{ ...LIVE_SETTINGS, ...keys }, 'live', null], + ]; + + for (const [env, mode, note] of cases) { + const decision = await decideMode(env, packageDirectory); + + assert.equal(decision.mode, mode, JSON.stringify(env)); + if (note === null) { + assert.equal(decision.note, null); + } else { + assert.match(decision.note, note); + } + } +}); + +test('the LaunchDarkly and Gemini origin overrides accept only plain loopback origins', () => { + assert.equal(readLoopbackOrigin({ X: 'http://127.0.0.1:9000' }, 'X'), 'http://127.0.0.1:9000'); + assert.equal(readLoopbackOrigin({}, 'X'), undefined); + for (const value of ['https://127.0.0.1:9000', 'http://evil.example', 'http://127.0.0.1:9000/path', 'http://127.0.0.1.evil.example']) { + assert.throws(() => readLoopbackOrigin({ X: value }, 'X'), /X must be an http:\/\/127.0.0.1/); + } +}); + +test('the travel desk searches only the bundled inventory and answers errors the model can act on', async (testContext) => { + const workspace = await makeWorkspace(testContext); + const desk = new TravelDesk({ inventory: loadInventory(packageDirectory), itineraryFile: path.join(workspace, 'itinerary.json') }); + + const flights = await desk.run('search_flights', { origin: 'jfk', destination: 'LIS', date: '2031-03-12' }); + const hotels = await desk.run('search_hotels', { city: 'lisbon', check_in: '2031-03-12', check_out: '2031-03-16' }); + const noFlights = await desk.run('search_flights', { origin: 'JFK', destination: 'LIS', date: '2031-07-01' }); + const badDate = await desk.run('search_flights', { origin: 'JFK', destination: 'LIS', date: 'March 12' }); + const unknown = await desk.run('delete_everything', {}); + + assert.deepEqual(flights.flights.map((flight) => flight.id), ['AT112', 'SK240', 'NV310']); + assert.ok(flights.flights.every((flight) => flight.date === '2031-03-12')); + assert.equal(hotels.hotels.length, 5); + assert.deepEqual(noFlights.flights, []); + assert.match(badDate.error, /YYYY-MM-DD/); + assert.match(unknown.error, /no tool named delete_everything/); +}); + +test('book_trip writes only the itinerary inside the workspace, and refuses pieces that do not fit', async (testContext) => { + const workspace = await makeWorkspace(testContext); + const desk = new TravelDesk({ inventory: loadInventory(packageDirectory), itineraryFile: path.join(workspace, 'itinerary.json') }); + const mismatched = await desk.run('book_trip', { outbound_flight_id: 'NV308', return_flight_id: 'NV311', hotel_id: 'LIS-ALFAMA', check_in: '2027-03-12', check_out: '2027-03-16' }); + const unknown = await desk.run('book_trip', { outbound_flight_id: 'XX1', return_flight_id: 'NV311', hotel_id: 'LIS-ALFAMA', check_in: '2027-03-12', check_out: '2027-03-16' }); + + const booking = await desk.run('book_trip', { outbound_flight_id: 'NV310', return_flight_id: 'NV311', hotel_id: 'LIS-ALFAMA', check_in: '2027-03-12', check_out: '2027-03-16' }); + + assert.match(mismatched.error, /must leave on check_in/); + assert.match(unknown.error, /Unknown flight or hotel id/); + assert.equal(booking.total_usd, 818); + assert.equal(booking.nights, 4); + assert.deepEqual(await readdir(workspace), ['itinerary.json']); + assert.equal(JSON.parse(await readFile(path.join(workspace, 'itinerary.json'), 'utf8')).confirmation, booking.confirmation); +}); + +test('demo mode runs the full ladder from ladder.json, and the monitor line counts only rungs the booker answered on', installedPackageTest, async (testContext) => { + const workspace = await makeWorkspace(testContext); + const gemini = await startGemini(testContext); + const env = await attemptEnvironment({ workspace, gemini }); + const setup = await setUpAttempt(testContext, env); + + const { lines, configs } = await playLadder(env); + + assert.match(setup.stdout, /\(demo mode\)/); + assert.deepEqual(lines, [ + 'booker-state v1 rung=1 tried=none booked=no mode=demo', + 'booker-state v1 rung=1 tried=1 booked=no mode=demo', + 'booker-state v1 rung=2 tried=1 booked=no mode=demo', + 'booker-state v1 rung=2 tried=1,2 booked=no mode=demo', + 'booker-state v1 rung=3 tried=1,2 booked=no mode=demo', + 'booker-state v1 rung=3 tried=1,2,3 booked=no mode=demo', + 'booker-state v1 rung=4 tried=1,2,3 booked=no mode=demo', + 'booker-state v1 rung=4 tried=1,2,3,4 booked=yes mode=demo', + ]); + assert.deepEqual( + configs.map(({ variationKey, model, tools }) => ({ variationKey, model, tools })), + ladder.rungs.map((rung) => ({ variationKey: rung.key, model: rung.model, tools: rung.tools })), + ); + assert.deepEqual( + [...new Set(gemini.requests.map((request) => `${request.model} ${request.toolNames.join(',')}`))], + [...new Set(ladder.rungs.map((rung) => `${rung.model} ${rung.tools.join(',')}`))], + ); + assert.ok(gemini.requests.every((request) => ladder.rungs.some((rung) => rung.instructions === request.system))); +}); + +test('the config card state and the booker-state line change after a switch while the page keeps running', installedPackageTest, async (testContext) => { + const workspace = await makeWorkspace(testContext); + const gemini = await startGemini(testContext); + const env = await attemptEnvironment({ workspace, gemini }); + await setUpAttempt(testContext, env); + const before = await pageState(env); + + const switched = await callSwitch(env, 2); + + const afterSwitch = await pageState(env); + const page = await fetchText(env, '/'); + assert.equal(switched.isError, false, switched.text); + assert.match(switched.text, /config card on the page shows it now/); + assert.match(page.text, /