diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cc3ed67a..10da8265 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -315,3 +315,175 @@ jobs: OPENCODE_E2E: "1" OPENCODE_API_KEY: ${{ secrets.OPENCODE_API_KEY }} OPENCODE_GO_API_KEY: ${{ secrets.OPENCODE_GO_API_KEY }} + + # The published host binary with this checkout linked into a throwaway + # profile: the only run that boots `dsh` itself rather than the host packages + # the unit suite assembles. `test/e2e/host/probe-plugin.mjs` measures the + # registry, the listings, validation, where each turn is sent and the save of + # a catalog-unknown id, and `test/e2e/host-binary.e2e.ts` asserts today's + # rows (`S0_TODAY`, shared with the in-process characterization). + # + # Two cells. `pinned` installs the host with npm `overrides` forcing pi, + # cordis and the loader to the versions `test/support/host-versions.ts` + # names (the published host declares the last two as tilde ranges, so a + # fresh install could otherwise take a newer patch) and applies the + # harness's own patch to pi; `floating` takes whatever pi npm resolves for the + # published host (newer than the harness pins), so a pi release that moves + # the ground shows up nightly. Pull requests run `pinned` only, and only when + # they touch the host seam; dispatch and the nightly run both cells. + # + # The scope is decided by a step rather than by `on.pull_request.paths`: a + # workflow-level path filter would silence every other job in this file. + # Offline: the probe answers every gateway request itself, and the keys it + # stores are dummies. No secret is passed. + e2e-host: + name: e2e · host binary (${{ matrix.cell }}) + if: github.event_name != 'push' + runs-on: ubuntu-latest + timeout-minutes: 25 + strategy: + fail-fast: false + matrix: + cell: ${{ github.event_name == 'pull_request' && fromJSON('["pinned"]') || fromJSON('["pinned","floating"]') }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + - name: does this change touch the host seam? + id: scope + env: + EVENT: ${{ github.event_name }} + BASE_REF: ${{ github.base_ref }} + run: | + set -euo pipefail + if [ "$EVENT" != "pull_request" ]; then + echo "run=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + # Its own command, so a diff that cannot be computed (a base ref + # that was not fetched, a deleted base branch) fails the step rather + # than reading as "nothing changed" and skipping the job green. + changed="$(git diff --name-only "origin/${BASE_REF}...HEAD")" + # The job's own definition and runner config are in scope: a pull + # request that breaks the install, the override, the patch step or + # the e2e config must run the job it breaks. That includes + # vite.config.ts: it holds the pack config that builds lib/index.mjs + # (the only test that loads that bundle into a real host is this + # one, and its externals keep cordis and schemastery the host's + # copies) and the test settings vitest.e2e.config.ts inherits. + # package.json is not (every dependency bump touches it), so a + # change to its `test:e2e:host` script alone is exercised only by + # the nightly run. + # Until P-R1 moves the host seam under src/host/, the modules that + # install the host patches and mounts whose behaviour depends on the + # real loader and apply order (the hidden-route mount, the listing + # patch and adapter decoration, the Google bridge, the fetch patch, + # the plane bases and the stream hook) are in scope by name. Every + # other row producer (key-capture.ts, models-catalog.ts, config.ts, + # the generated catalog) is covered on each pull request by the + # in-process characterization of the same S0_TODAY rows, and here by + # the nightly run. + if printf '%s\n' "$changed" \ + | grep -Eq '^(src/host/|src/lifecycle\.ts$|src/identity\.ts$|src/responses-provider\.ts$|src/responses-routes\.ts$|src/models-discovery\.ts$|src/google-bridge\.ts$|src/fetch-patch\.ts$|src/pi-provider\.ts$|src/stream-hook\.ts$|cordis\.patch\.yml$|test/e2e/host/|test/e2e/host-binary\.e2e\.ts$|test/support/host\.ts$|test/support/host-versions\.ts$|patches/|\.github/workflows/ci\.yml$|vitest\.e2e\.config\.ts$|vite\.config\.ts$)'; then + echo "run=true" >> "$GITHUB_OUTPUT" + else + echo "run=false" >> "$GITHUB_OUTPUT" + echo "no host-seam path changed; the host binary e2e does not run for this pull request" + fi + - if: steps.scope.outputs.run == 'true' + uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 + - if: steps.scope.outputs.run == 'true' + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: 26 + cache: pnpm + - if: steps.scope.outputs.run == 'true' + run: pnpm install --frozen-lockfile + - name: the host versions this repository pins + id: versions + if: steps.scope.outputs.run == 'true' + run: | + node --experimental-strip-types --input-type=module -e "import { HOST_VERSIONS } from './test/support/host-versions.ts'; console.log('dsh=' + HOST_VERSIONS.dsh); console.log('pi=' + HOST_VERSIONS.pi); console.log('cordis=' + HOST_VERSIONS.cordis); console.log('loader=' + HOST_VERSIONS.loader)" >> "$GITHUB_OUTPUT" + echo "PREFIX=$RUNNER_TEMP/dsh-host" >> "$GITHUB_ENV" + # The pinned install forces every host version HOST_VERSIONS names, so it + # is cached by those versions and the patch (the rest of the published + # host's tree is whatever its ranges resolved on the miss that filled + # the cache); the floating one is the point of re-resolving, so never. + - name: cache the pinned host install + id: host-cache + if: steps.scope.outputs.run == 'true' && matrix.cell == 'pinned' + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ runner.temp }}/dsh-host + key: dsh-host-${{ matrix.cell }}-${{ steps.versions.outputs.dsh }}-${{ steps.versions.outputs.pi }}-${{ steps.versions.outputs.cordis }}-${{ steps.versions.outputs.loader }}-${{ hashFiles('patches/*.patch') }} + - name: install the published host into a throwaway prefix + if: steps.scope.outputs.run == 'true' && steps.host-cache.outputs.cache-hit != 'true' + env: + CELL: ${{ matrix.cell }} + DSH: ${{ steps.versions.outputs.dsh }} + PI: ${{ steps.versions.outputs.pi }} + CORDIS: ${{ steps.versions.outputs.cordis }} + LOADER: ${{ steps.versions.outputs.loader }} + run: | + mkdir -p "$PREFIX" + # The overrides must be in the prefix's own package.json BEFORE the + # install, or npm resolves the wrapper's ^range to the newest pi. + if [ "$CELL" = pinned ]; then + printf '{ "private": true, "overrides": { "@earendil-works/pi-ai": "%s", "@deepseek-ai/cordis": "%s", "@deepseek-ai/cordis-plugin-loader": "%s" } }\n' "$PI" "$CORDIS" "$LOADER" > "$PREFIX/package.json" + else + printf '{ "private": true }\n' > "$PREFIX/package.json" + fi + npm install --prefix "$PREFIX" --no-audit --no-fund "@deepseek-ai/dsh@$DSH" + # Only on a fresh install: re-applying to a cached, patched copy fails. + if [ "$CELL" = pinned ]; then + patch -p1 -d "$PREFIX/node_modules/@earendil-works/pi-ai" < "patches/@earendil-works__pi-ai@$PI.patch" + fi + - name: assert the pi, cordis and loader the host resolved + if: steps.scope.outputs.run == 'true' + env: + CELL: ${{ matrix.cell }} + PI: ${{ steps.versions.outputs.pi }} + CORDIS: ${{ steps.versions.outputs.cordis }} + LOADER: ${{ steps.versions.outputs.loader }} + run: | + resolved="$(node -p "require(process.env.PREFIX + '/node_modules/@earendil-works/pi-ai/package.json').version")" + echo "cell $CELL: pi $resolved" + copies="$(find "$PREFIX/node_modules" -path '*/@earendil-works/pi-ai/package.json' | wc -l)" + if [ "$copies" -ne 1 ]; then + echo "::error::expected one copy of pi under the host prefix, found $copies" + exit 1 + fi + if [ "$CELL" = pinned ] && [ "$resolved" != "$PI" ]; then + echo "::error::the pinned cell resolved pi $resolved, not $PI" + exit 1 + fi + # The check test/host-versions.test.ts runs for the unit install, + # here for the host the binary e2e boots (a cached prefix included). + for pair in "cordis:$CORDIS" "cordis-plugin-loader:$LOADER"; do + name="@deepseek-ai/${pair%%:*}" + want="${pair#*:}" + got="$(node -p "require(process.env.PREFIX + '/node_modules/$name/package.json').version")" + echo "cell $CELL: $name $got" + if [ "$CELL" = pinned ] && [ "$got" != "$want" ]; then + echo "::error::the pinned cell resolved $name $got, not $want" + exit 1 + fi + done + - if: steps.scope.outputs.run == 'true' + run: pnpm run build + - name: boot the host binary with the probe + if: steps.scope.outputs.run == 'true' + run: pnpm run test:e2e:host --run + env: + DSH_HOST_E2E: "1" + DSH_HOST_PREFIX: ${{ runner.temp }}/dsh-host + DSH_HOST_PI_CELL: ${{ matrix.cell }} + DSH_HOST_PROBE_COPY: ${{ runner.temp }}/probe-${{ matrix.cell }}.json + - name: upload the probe's measurement + if: always() && steps.scope.outputs.run == 'true' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: host-probe-${{ matrix.cell }} + path: ${{ runner.temp }}/probe-${{ matrix.cell }}.json + if-no-files-found: ignore + retention-days: 7 diff --git a/AGENTS.md b/AGENTS.md index b5e36a83..dccec4e7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -43,6 +43,9 @@ Host bundle (`lib/index.mjs`), a thin `apply` barrel over small modules: - `src/index.ts` · `src/identity.ts` · `src/lifecycle.ts` — the public barrel, the identity constants, and the `apply()` composition (one installer per side effect, in load-bearing order). The barrel is the host's contract and nothing more: `apply`, `Config`, `inject`, `name` and the config types. Everything else is internal, and `test/public-api.test.ts` pins the list, so widening it is a reviewed change. - `test/support/src.ts` — the one door tests use into `src`: each name re-exported from the module that DEFINES it, never through the barrel, so a module move edits one file. `test/architecture.test.ts` enforces both halves — every name is declared in its target, and no test outside the named exemptions (`vi.mock` targets, fresh re-imports after `vi.resetModules()`, the architecture and public-API tests, the jsdom client tests, the `reset-process-state.ts` setup file) imports a `src` path, and no jsdom test loads the door. - `test/support/host-versions.ts` — the host the test install runs (pi, the `@deepseek-ai` cluster, cordis and its loader), named once; `test/host-versions.test.ts` checks the installed tree against it (with `pnpm-lock.yaml` as the record of what pnpm resolved, and the published peer ranges) and `test/harness-channel.test.ts` checks the declarations and the lockfile, including that `@deepseek-ai/schemastery` resolves to one copy inside the cluster's range. +- `test/support/host.ts` — the real host, in-process, in two tiers: **A** (`mountTierA`: a bare cordis `Context`, `LlmRuntime` and `dsh-llm-pi-ai` mounted with `{ providers }`) for the `internal/config` waterfall and pi-ai's credential paths, and **B** (`bootTierB`: dsh-app-boot's `boot()` with a temporary HOME and a test bundle) for the config editor, `settings.describe`, the volatile commit and the characterization of the plugin as it ships. Its fakes (`fakeCredentials`, `fakeLaunchEnvironment`, `recordingFetch`) keep every case offline and its credentials out of the environment (a boot blanks the two keys the plugin reads directly, should the shell export them, and restores them on dispose); `S0_PROFILE` is the contract fixture, `CHARACTERIZED_PROFILE` (S0 plus a declared catalog-unknown id) the characterization's, and `S0_TODAY` the measured per-model rows, tagged per facet, that the binary e2e shares (`todayRowsFor` picks them by the apply order the probe infers from whether the plugin's decoration wrapped pi-ai's plane adapter). Tier A has no loader, so this plugin mounted there cannot find pi-ai and mounts no hidden route and no Google bridge (whose adapter class comes from the same loader entry, `loadPiAiAdapterHost`); `test/host-harness.test.ts` exercises the builder options no contract or characterization case passes, and tier B's two failure paths (an invalid first row, an exported key). Tier B's rows carry PACKAGE names through a stub module loader, never `cordis:` builtins, because the plugin finds pi-ai's entry by `options.name` (`piAiEntry` in `src/responses-provider.ts`) and a builtin-named row would drop the hidden-route mount from the characterization. It also fixes which of pi-ai and the plugin applies first (`order`), because the loader starts rows in parallel and the published binary applies the plugin first; the held row is released when the first one applies or its fiber settles, so an invalid first row fails instead of hanging the boot. +- `test/support/keyless-matrix.ts` — the keyless 401-vs-404 probe matrix's cell table and classifier. Pure, though it lives in `test/support/` rather than `src/utils/`: a function of its arguments, with no code path that can set a credential header; `test/e2e/keyless-matrix.e2e.ts` sends the cells. +- `test/e2e/host/` — the measurement bundle (`probe-plugin.mjs`, its manifest and patch) the `e2e-host` job links into a throwaway profile under a temporary HOME, through a relative `link:` path. `test/e2e/host-binary.e2e.ts` is its reader, gated on `DSH_HOST_E2E=1`; it installs no host itself and needs `DSH_HOST_PREFIX`, a prefix holding the installed one. - `patches/` — the harness's own pi patch, copied verbatim and applied by `patchedDependencies` in `pnpm-workspace.yaml`. Its hash is pinned by `test/host-versions.test.ts` and recorded in `pnpm-lock.yaml`, so never hand-edit it: re-copy it from the harness and re-run `pnpm install`. - `src/config.ts` · `src/config-values.ts` · `src/utils/guards.ts` — the schema and its defaults; dependency-free readers and type guards. The client bundle may import these, and it does — along with the contract and presentation modules (`usage-contract.ts`, `login-contract.ts`, `settings-copy.ts`, `settings-fields.ts`, `usage-ui.ts`, `go-limits.ts`). The rule is not a list of names: **the client may import a module that reaches nothing host-only** — no cordis context, no credential seam, no node built-in — and the bundle gate is what enforces it, by failing the build rather than by anyone remembering. An earlier version of this line named two files as the only ones allowed, which the tree had already outgrown. @@ -109,6 +112,7 @@ Web client bundle (`lib/client.js`): `src/settings-page.tsx` (entry, wiring only vp test run # unit suite (refuses to start below engines.node) pnpm run test:shuffle # the unit suite in random order; CI runs it on Node 24 pnpm run test:mock # gateway mock suite: offline, no key, runs in CI +pnpm run test:e2e:host # opt-in: the published host binary with the probe (CONTRIBUTING, "The e2e-host job") vp test run --coverage # with the coverage ratchet (95/90/93/95) vp check # format, lint, types vp fmt --write # rewrite formatting diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1d7185e9..8f37a65a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -19,6 +19,7 @@ pnpm run test:shuffle # the same run in a random order; reproduce a failure w pnpm run test:mock # gateway mock suite: offline, loopback only, runs in CI pnpm run build # vp pack + client rename -> lib/index.mjs, lib/index.d.mts, lib/client.js pnpm run test:e2e # opt-in end-to-end suite; talks to the real OpenCode gateway +pnpm run test:e2e:host # opt-in: the published host binary with the probe (see "The e2e-host job") pnpm run catalog:shim # is src/catalog-data.ts current? (-- --write regenerates it from models.dev) pnpm run limits:shim # is src/go-limits-data.ts current? (-- --write regenerates it) pnpm run cli:version # is the pinned OpenCode CLI version the published one? @@ -36,13 +37,62 @@ OPENCODE_E2E=1 OPENCODE_API_KEY=… OPENCODE_GO_API_KEY=… pnpm run test:e2e - `OPENCODE_E2E=1` is required; without it every case skips, so a bare `pnpm run test:e2e` is a safe no-op. - `OPENCODE_API_KEY` (Zen) arms the live `/models` enrichment case and the **paid** protocol-routing cases; `OPENCODE_GO_API_KEY` (Go plan) arms the live `/usage` case. Each block skips when its key is absent, which is why the CI `e2e` job stays green on fork PRs (secrets are not exposed to them) while still checking the endpoints answer. -- The **free-tier and unmetered** routing cases run keyless on purpose: measured 2026-10-06, a paid model answers `401 AuthError` on every endpoint without a credential, so keyless probing cannot tell them apart at all. Keyless discrimination is real only for the free classes. -- `OPENCODE_ZEN_BASE_URL` / `OPENCODE_GO_BASE_URL` retarget the suite at a mirror. +- The **free-tier and unmetered** routing cases run keyless on purpose: measured 2026-10-06, a paid model answers `401 AuthError` on every endpoint without a credential, so keyless probing cannot tell the right PROTOCOL endpoint from a wrong one. Keyless protocol discrimination is real only for the free classes. What a paid model's keyless answer does still tell is whether a PATH exists at all — a removed path answers 404 — which is the one thing the keyless matrix below reads. +- `OPENCODE_ZEN_BASE_URL` / `OPENCODE_GO_BASE_URL` retarget `opencode-live` and `protocol-routing` at a mirror. The keyless matrix measures the vendor's own paths, which a mirror's base cannot name, so it skips while either is set. -Two files, two jobs: +What the main files are for: - `opencode-live.e2e.ts` — the only place the plugin meets the real API. It catches what a stub cannot: **the vendor changing the payload**. The meter parses `/zen/go/v1/usage` and the picker consumes an enriched `/models` listing, so both shapes are asserted against the live service. The unkeyed cases assert reachability (never a 404) and that our header set does not change the gateway's verdict on a request — compared against the same call without it, rather than hardcoding a status the vendor may tighten. - `patched-fetch-headers.e2e.ts` — a local `node:http` listener, so the _outgoing_ header set crosses a real socket. The live gateway cannot echo a request back; everywhere else in the suite the injected headers are asserted against a captured `fetch`. +- `keyless-matrix.e2e.ts` — the keyless 401-vs-404 matrix: two planes × four protocols × the legacy (`/zen(/go)/v1`) and modern (`/inference(/go)/…`) endpoint families, one PAID model per cell from the live catalog — never a free one, which the gateway serves keyless with a real completion. It **sends no key and reads none**, so the `e2e` job runs it on every pull request, and it retries a cell once on a 429, a 5xx or a transport error before reading it. A 404 means the path is gone, a 401/403 that it exists, a 410 or a 400 "Model is unavailable" that the model is. It asserts that all sixteen cells have a model and were sent, that every answer is readable, and that no cell the plugin dispatches to today with the default endpoint (`auto`, an alias of `modern`; `TODAY_DISPATCH`: completions on the LEGACY family, Responses, Messages and Google on the modern one) is gone — nor the modern completions cells P-R1 moves completions to (`NEXT_DISPATCH`, a labelled forward guard). The other legacy verdicts are recorded for P-R2 — among them the cells an `endpoint: legacy` user dispatches to (Responses and Messages on both planes, and Zen's Google), which are recorded, not asserted; Go's Google stays on `/inference/go/google/v1beta` under every family (`GO_GOOGLE_BASE_URL`), the modern cell `TODAY_DISPATCH` already asserts. The cell table, both sets and the classifier are a pure module (`test/support/keyless-matrix.ts`), unit-tested offline in `test/keyless-matrix.test.ts`, which also derives `TODAY_DISPATCH` from the URLs `S0_TODAY` measured. +- `protocol-routing.e2e.ts` — asks the gateway which endpoint serves each model's wire format: keyless for the free classes, keyed (and skipped without a secret) for paid ones. +- `dispatch-levels.e2e.ts` — every thinking level the picker offers is one pi's own rule accepts, swept over the whole shipped catalog; no network. +- `refusal-classification.e2e.ts` — the refusal classifier's verdicts on the gateway's bytes over a real `node:http` socket. +- `host-binary.e2e.ts` — the published host binary; see [The real-host harness](#the-real-host-harness). It skips unless `DSH_HOST_E2E=1`. + +## The real-host harness + +`test/support/host.ts` builds the host from the packages this repository installs, at the versions `test/support/host-versions.ts` names (pi 1.1.0 with the harness patch, the `@deepseek-ai` cluster at the published host's tag). The routing work is judged against it: the contract tests pin the host facts the design rests on that the host can answer in-process (`docs/improvement-plan.md`: F1, F2, F4, F5, F8, F10–F16, F18 and F20, each by number; F6 and F19's auth half (constructing a `PiAiAdapter`) are first exercised by P-R1; F19's export half is pinned in `host-shapes` (`recordKeyFor`); F17 is honoured, not pinned — tier B fixes the apply order and the binary e2e infers it; F9 is a fact about opencode's own client, outside this host), and the characterization rows record what today's plugin does. + +Two tiers, both in-process: + +- **Tier A — `mountTierA`.** A bare cordis `Context`, `LlmRuntime`, and `dsh-llm-pi-ai` mounted as a plugin with `{ providers }` (and optionally this plugin). No loader and no files. Use it for the `internal/config` waterfall and pi-ai's credential paths — anything that needs a fiber and the registry and nothing that reads a profile (`host-waterfall`, `host-shapes`, `host-credentials`). Mounted here, this plugin is a **degraded** one: it finds pi-ai through the loader's entries, so with no loader it patches fetch but mounts no hidden route and no Google bridge — the bridge's adapter class comes from the loader's pi-ai entry too (`host-harness` pins both: the stand-down line, and `loadPiAiAdapterHost` answering `undefined`). Characterize the real plugin in tier B. +- **Tier B — `bootTierB`.** dsh-app-boot's `boot()` with a temporary HOME, a test bundle and a profile patch, the way the harness's own composition tests boot one. Use it for the config editor, `settings.describe`, the loader's volatile commit and the characterization of the plugin as it ships (`host-editor-contracts`, `host-characterization`). Its rows carry **package names**, resolved by a stub module loader, not `cordis:` builtins: the plugin finds pi-ai's entry by `options.name === "@deepseek-ai/dsh-llm-pi-ai"` (`src/responses-provider.ts`), so a builtin-named row would silently drop today's hidden-route mount and the Google bridge. The loader starts rows in parallel, so tier B makes the order explicit: `order: "plugin-first"` (the default, and what the published binary does) or `"pi-ai-first"`. + +What every case does: + +- **Install the recording fetch before mounting** (`vi.stubGlobal("fetch", recordingFetch().fetch)`) and restore it with `vi.unstubAllGlobals()` after disposal: the plugin's fetch patch wraps whatever `fetch` is current when it mounts. The recording fetch answers `opencode.ai` and `models.opencode.ai` with a 401, refuses every other host, and keeps credential headers by length only. +- **Dispose in `afterEach`** (or a `finally`), and boot per case: a key the fetch patch captured in one case would change the next one's listing. +- **Credentials come from the fakes, not the environment.** The fakes (`fakeCredentials`, `fakeLaunchEnvironment`) are provided on the root, which is where pi-ai looks first. The plugin reads `OPENCODE_API_KEY`/`OPENCODE_GO_API_KEY` from the environment directly, so `bootTierB` blanks either one your shell exports for the life of the boot and restores it on dispose: the suite measures the keyless deployment whatever the shell holds (`host-harness` pins it). A case never sets them itself. +- **Give tier-B cases 30 s**, and wait with a deadline (`waitForRoutes`, 10 s by default, so a route that never registers fails by name inside the hook's budget), never a fixed sleep: the plugin mounts its hidden routes asynchronously. +- **A first row that fails does not hang the boot.** Tier B holds the second of pi-ai and the plugin until the first applies OR its fiber settles in any state, so a row whose config the schema rejects boots with that fiber FAILED (`fiber.await()` rethrows the real error) and the other row applied. + +**Tags.** Every characterization row is titled `[BASELINE]` (must survive every phase), `[KNOWN_DEFECT]` (wrong today; a phase fixes it) or `[INTENDED_CHANGE]` (right today; a phase changes it on purpose). A phase that flips a row names it in its PR body; nothing else may move, except that P-R2's measured endpoint-family flip, if it happens, may move the family part of a `[BASELINE]` wire facet, re-tagging each one it moves and naming it in its PR body. The per-model rows live in `S0_TODAY` in `host.ts`, which the binary e2e reads too, and each model's three facets — model info, the call-config verdict, the wire — carry their own tag there: `host-characterization` generates one test per model and facet titled with that tag, in each apply order (`todayRowsFor(order)` carries the pi-ai-first table's own tags, where Go Gemini's call config is already the behaviour P-R1 keeps), so the tags in `S0_TODAY` are what govern which entries a phase may flip. The one facet that depends on the apply order today (Go `gemini-4-argon-contributor`'s call config) is pinned by its own `[KNOWN_DEFECT]` case, outside the booted blocks; the phase that flips that row also drops `todayRowsFor`'s pi-ai-first override and turns that case into a `[BASELINE]` "no facet depends on the apply order" check, in the same diff. The fixture is S0 plus one declared id no catalog knows (`CHARACTERIZED_PROFILE`); the contract tests keep plain S0. + +**A catalog refresh can touch the rows.** `S0_TODAY`'s context windows and effort lists are copied from the generated `src/catalog-data.ts`, which the nightly refresh rewrites. A fixture-sanity case compares each row's context window and effort ids to the bundled catalog's derivation and names the model that moved; when it fails on a refresh pull request, update the row in `S0_TODAY`. A phase that moves a non-`BASELINE` `info` facet off the catalog on purpose (the user's context window, a toggle model's Off/On) adds that row's key to the case's `INFO_NOT_FROM_CATALOG` set in the same diff; the case refuses to exempt a `BASELINE` facet. The `e2e-host` job's scope includes `test/support/host.ts`, so the binary e2e runs on that pull request too and checks the edited row against the published binary. + +**The apply order is a race on the real host.** Tier B fixes it; the published binary does not. The rows depend on whether the plugin's adapter decoration wrapped pi-ai's plane adapter, so the probe records that and the order it implies (`inferredOrder` — inferred, not observed: a pi-ai restart after the plugin applied would read as plugin-first), and the binary e2e picks its rows with `todayRowsFor(inferredOrder)` instead of pinning one. + +### The `e2e-host` job + +`.github/workflows/ci.yml`'s `e2e-host` job installs the published `@deepseek-ai/dsh` (at `HOST_VERSIONS.dsh`) into a throwaway prefix, builds this checkout, and runs `test/e2e/host-binary.e2e.ts`. That file creates a temporary HOME whose profile links this checkout and the probe bundle (`test/e2e/host/`) through relative `link:` paths, stores dummy keys in the temporary home's credentials document (`$DSH_HOME/.credentials.yaml`, shared by every profile under that home), boots `dsh --profile e2e --no-open --port 0`, and reads what `test/e2e/host/probe-plugin.mjs` measured. It asserts today's rows; P-R1 flips them. + +- **Cells.** `pinned` forces pi, cordis and the loader to `HOST_VERSIONS` with npm `overrides` (the published host declares cordis and the loader as tilde ranges), applies the harness patch to pi with `patch -p1`, asserts all three installed versions (a cached prefix included), then asserts every row. `floating` takes the pi npm resolves for the published host. It skips the rows that depend on which ids pi describes (the keyed listing and the per-model rows) and still asserts the save rows. Those assume the floating pi lacks at least one of `ZEN_UNDESCRIBED` and still describes every other declared Zen id, so a pi that learns all of them or retires a declared id turns them red by design: triage a red floating run as a pi change first. Its stored-gap save row asserts only that SOME undescribed id blocks the save, not which one; its probe JSON is the record. +- **Triggers.** Dispatch and the nightly run take both cells. A pull request takes `pinned`, and only when it touches `src/host/**`, `src/lifecycle.ts`, `src/identity.ts`, the modules that install the host patches and mounts whose behaviour depends on the real loader and apply order, until P-R1 moves them under `src/host/` (`src/responses-provider.ts`, `src/responses-routes.ts`, `src/models-discovery.ts`, `src/google-bridge.ts`, `src/fetch-patch.ts`, `src/pi-provider.ts`, `src/stream-hook.ts`; every other row producer, such as `src/key-capture.ts`, `src/models-catalog.ts` or `src/config.ts`, is covered on each pull request by the in-process characterization of the same `S0_TODAY` rows, and by the nightly run), `cordis.patch.yml`, `test/e2e/host/**`, `test/e2e/host-binary.e2e.ts`, `test/support/host.ts` (the rows the binary e2e asserts), `test/support/host-versions.ts` or `patches/` (what the job installs and patches), `.github/workflows/ci.yml`, `vitest.e2e.config.ts` or `vite.config.ts` (the job's own steps and runner config; `vite.config.ts` also holds the pack config that builds `lib/index.mjs`, the bundle only this job loads into a real host, and the test settings the e2e config inherits) (scoped by a step: a workflow-level path filter would silence every other job; a diff the step cannot compute fails it rather than skipping the job). `package.json` is left out because every dependency bump touches it, so a change to its `test:e2e:host` script alone is exercised only by the nightly run. It needs no secret. + +To run it locally, install the host into a prefix outside the checkout, then point the suite at it (placeholder paths; `pnpm` must be on `PATH`). The first line reads the versions from `test/support/host-versions.ts`, the same way the CI job does, so the snippet never names one itself: + +```sh +eval "$(node --experimental-strip-types --input-type=module -e "import { HOST_VERSIONS as v } from './test/support/host-versions.ts'; console.log('DSH=' + v.dsh + ' PI=' + v.pi + ' CORDIS=' + v.cordis + ' LOADER=' + v.loader)")" +mkdir -p /home/you/dsh-host +printf '{ "private": true, "overrides": { "@earendil-works/pi-ai": "%s", "@deepseek-ai/cordis": "%s", "@deepseek-ai/cordis-plugin-loader": "%s" } }\n' "$PI" "$CORDIS" "$LOADER" > /home/you/dsh-host/package.json +npm install --prefix /home/you/dsh-host "@deepseek-ai/dsh@$DSH" +patch -p1 -d /home/you/dsh-host/node_modules/@earendil-works/pi-ai < "patches/@earendil-works__pi-ai@$PI.patch" +pnpm run build +DSH_HOST_E2E=1 DSH_HOST_PREFIX=/home/you/dsh-host pnpm run test:e2e:host --run +``` + +`DSH_HOST_PI_CELL=floating` runs the floating cell against an install without the override and the patch. ## Live profile loop diff --git a/docs/improvement-plan.md b/docs/improvement-plan.md index 74f6937f..b323e4a8 100644 --- a/docs/improvement-plan.md +++ b/docs/improvement-plan.md @@ -23,7 +23,7 @@ Phases land as **stacked pull requests**. Each one is based on the previous phas | P1b | `chore:` hygiene | in review | | P2 | `feat:` a smaller public API | in review | | P-T1a | `build:` pin the host | in review | -| P-T1b | `test:` a harness that is the real host | planned | +| P-T1b | `test:` a harness that is the real host | in review | | P-H | `refactor:` one way to patch a host method | planned | | P-R1 | `feat:` the plugin owns the plane routes | planned | | P-R2 | `fix:` the endpoint family, measured | planned | @@ -123,7 +123,7 @@ Phases land as **stacked pull requests**. Each one is based on the previous phas **The problem, in the owner's words.** A provider defines one backend. Supporting every protocol and the newer catalog meant un-registering the original `opencode`/`opencode-go` provider and registering the plugin's own. Deleting another plugin's registration is unsupported, so the plugin mounted extra routes per protocol instead, merged their models back into the user's provider, and redirected dispatch. This section shows that the host **does** allow the original intent, through supported paths (F11–F13): keep pi-ai from claiming the two ids, and register the plugin's provider under them. A spike on the real host confirmed it (run S0). -**Facts from the host's own code and data.** F1–F5 and F10 are measured: the data files were counted and the resolver read. F6–F9 and F11–F13 are read from the code, and run S0 exercised F11–F13 end to end. P-T1's real-host harness pins them all, and a fact that fails there sends the design back to this section. +**Facts from the host's own code and data.** F1–F5 and F10 are measured: the data files were counted and the resolver read. F6–F9 and F11–F13 are read from the code, and run S0 exercised F11–F13 end to end. P-T1's real-host harness pins the ones the host can answer in-process: P-T1b pins F1, F2, F4, F5, F8, F10–F16, F18 and F20 by number (F3 through its F4 consequence, F7 through F13's register and DUPLICATE halves); F6 (dispatch through a non-pi adapter) and F19's auth half (constructing a `PiAiAdapter`) are first exercised by P-R1, F6 through its mock gateway and the real `LlmRuntime`; F19's export half is pinned in `host-shapes` (`recordKeyFor`); F17 is honoured, not pinned — tier B fixes the apply order and the binary e2e infers it; F9 is a fact about opencode's own client, outside this host. A fact that fails where it is pinned sends the design back to this section. | # | Fact | Evidence | | :-- | :-- | :-- | @@ -321,15 +321,16 @@ What stays private is pinned by contract tests against the real host packages: - **Pinning phases open with a green pin**, which the PR shows unchanged: P-T1a/b, P-H, P5, P7, P8, P9, P10a and P-T2. - **P-T3 and P12 are exempt.** -**Characterization rows are tagged when they are recorded**, so a later phase flips named rows: +**Characterization rows are tagged when they are recorded**, so a later phase flips named rows. The authoritative per-facet table is `S0_TODAY` in `test/support/host.ts` (each model's info, call-config verdict and wire carry their own tag); a phase's PR body cites the rows it flips from there. The three tags: -- `KNOWN_DEFECT`: - - Go Gemini answers `INVALID_CONFIG`; +- `BASELINE`: right today and must survive every phase. No phase may move a `BASELINE` facet, except that P-R2's measured family flip, if it happens, may move the endpoint-family part of a wire facet; it re-tags each one it moves and names it in its PR body. +- `KNOWN_DEFECT`, wrong today and a phase fixes it, for example: + - Go Gemini answers `INVALID_CONFIG` — in the published host's apply order (`plugin-first`); with pi-ai applying first it validates, so the row is order-dependent (`todayRowsFor`). P-R1 flips it and, in the same diff, drops `todayRowsFor`'s pi-ai-first override and turns the `[KNOWN_DEFECT]` apply-order case into a `[BASELINE]` "no facet depends on the apply order" check; - an unknown id is rejected; - toggle models show Minimal/Xhigh/Max. -- `INTENDED_CHANGE`: +- `INTENDED_CHANGE`, right today and a phase changes it on purpose, for example: - completions models move from `/zen(/go)/v1` to `/inference(/go)/openai/v1`; - - `decoratePlaneAdapters`' baseUrl repair never reaches the wire. + - `decoratePlaneAdapters`' baseUrl repair never reaches the wire — a row of the `pi-ai-first` block only: in `plugin-first` the decoration never runs, and that row is a `KNOWN_DEFECT`. 1. **P1a `fix:` shipped breakage.** - The wrapper's broken `export { default }`. @@ -357,15 +358,24 @@ What stays private is pinned by contract tests against the real host packages: - The `@deepseek-ai/*` packages move to rc.2, and the devDependencies gain `dsh-llm`, `dsh-app-boot`, `dsh-config-editor`, `dsh-settings` and `cordis-plugin-loader`. - Add `test/support/host-versions.ts`, with a version test: `loadHostPi().version` is 1.1.0, and the root pi is the same copy. - Rewrite `mount-protocols.test.ts` as "every catalog SDK maps to a protocol that pi's `api/*` exports implement", keeping the row that pi 1.1.0's Go provider lacks Google. -5. **P-T1b `test:` the real-host harness.** It is additive only. `test/support/host.ts` provides two tiers that run inside the test process: - - **Tier A:** a cordis `Context`, `LlmRuntime`, and pi-ai mounted with `{ providers }`. - - **Tier B:** dsh-app-boot's `boot()` with a temporary HOME, and a test bundle that names the loader builtins `llm`, `llm-pi-ai`, `ConfigEditor`, `Settings` and the plugin. It follows the harness's `experimental/speech-to-text/tests/configuration.spec.ts:7-33`, with a fake `credentials` service and a launch-environment snapshot. +5. **P-T1b `test:` the real-host harness.** Additive only; in review. It landed `test/support/host.ts` (tier A: `Context` + `LlmRuntime` + pi-ai; tier B: app-boot `boot()` with a temporary HOME), the contract tests (`host-waterfall`, `host-shapes`, `host-credentials`, `host-editor-contracts`), the tagged characterization (`host-characterization`, with `S0_TODAY` shared by the binary e2e), the keyless matrix, the probe bundle and the `e2e-host` job. Two departures from the phase as first planned, both deliberate: tier B's rows carry package names through a stub module loader rather than `cordis:` builtins, because the plugin finds pi-ai by `options.name` (`piAiEntry` in `src/responses-provider.ts`); and tier B fixes the apply order (`plugin-first`, the binary's, or `pi-ai-first`), because the loader starts rows in parallel. Two narrower ones: the `e2e-host` PR scope is wider than first named (the job's own inputs, and until P-R1 the `src` modules that install the host patches and mounts whose behaviour depends on the real loader and apply order), and the floating cell skips the rows that depend on which ids pi describes. - This phase lands: - - the contract tests; - - the tagged characterization rows; - - the S0 assertions, including `settings.describe`, `hasProviderApiKey` and the Models store's join; - - the CI additions below. + The `hasProviderApiKey` baseline (`api/session-controller/src/catalog.ts:97-116`) is a **restatement**: no installed copy reaches the function (the published `@deepseek-ai/dsh-api-session-controller` entry exports `SessionController`, `buildModelCatalog` and a few classes but not it, its tarball ships `lib/` and no `src/`, and the package is not a dependency here in any case), so `test/support/host.ts` restates it (`providerWithApiKey`) from the real parts tier B holds — `settings.describe({ redactSecrets: true })`, `llm.listConfigurableProviders()` and `credentials.describe` — keeping upstream's `credentialRef` grammar check as a `TypeError` (a host-editor-contracts case pins it), answering the provider id where upstream answers `true`, and omitting the `RemoteError` for a missing service tier B always has; both `host-editor-contracts` and `host-characterization` use it. The cases in `host-editor-contracts` boot WITHOUT the plugin, so they are host contracts that nothing in `src/` can move: with only the plane keys stored it answers true (the Zen plane); under a test-local strip listener it answers false, because the planes' `describe().value` is gone (F20), which upstream makes `initializeDefaultModel` replace the user's default model. That false is the host fact the D18 wrapper exists to counter, not a row any phase flips. The guard on the plugin is the `[BASELINE]` row in `host-characterization`: today's plugin as it ships, with only the plane keys stored, in both apply orders, answers the Zen plane, and pi-ai's `settings.describe` value lists both planes. P-R1 must keep that row green while its own strip is active. + + **What remains:** the Models store's join. It lives in the client package `ui-settings-models`, which is not installed here and has no server-side seam to restate, so P-R1 owns it (its contract list below, unconditionally), measured on the binary or with the package added at the pinned cluster tag. + + The tier-A F11 smoke (`listProviders()` equals pi-ai's resolved keys) lives in `host-waterfall`, beside the other F-numbered contracts; `host-harness` holds the builder-seam cases. + + **Corrections the harness measured** (P-R1 must design for these): + - **F8/F12, the save check.** An outermost listener that strips the planes and returns a copy does NOT shield pi-ai's own listener: pi-ai validates the value `next()` returns, which is the raw value the inner chain saw (`llm-pi-ai/src/index.ts:177-186`). Against its stripped previous config every plane entry reads as changed, so the strict check runs on the planes: an editor save is refused while a plane stores any gap id. The claim listener must either keep pi-ai's listener from seeing the planes (run the nested waterfall on the copy, with a re-entrancy guard) or veto the chain for pi-ai's fiber and validate in its place. + - **F16, `!!js` in a provider.** pi-ai's listener runs `Config(raw)` on that same un-interpolated value, so a `!!js` node inside `providers` boots (evaluated) but makes every later save of the row fail with `expected string but got [object Object]`. The F16 pin is therefore taken on the plugin's own row. + - **`update()` runs the waterfall twice**, both as commits (`raw === _config`): once while ACTIVE with pi-ai's listener present (`fiber.ts:747`), once on the restart without it (`fiber.ts:655`). A first load and a `restart()` run it once, without pi-ai's listener. + - **F15 holds** for a plane-only edit under the strip: no `loader/volatile-update`, no `llm/adapters-updated`, no restart. A sibling's edit emits one volatile update and two registry events (route replace and directory replace, `llm/src/index.ts:475,536`). + - **The apply order changes today's rows.** On the published binary the plugin applies about 70 ms before pi-ai registers, so `decoratePlaneAdapters` (which wraps only what is registered when its effect runs) never wraps pi-ai's plane adapter. In that order Go `gemini-4-argon-contributor` answers `INVALID_CONFIG` — the plan's `KNOWN_DEFECT` holds — while the turn is still served on the Go Google base through the bridge. With pi-ai first it validates, and the Go plane's repaired base is Zen's (both planes share one adapter instance, wrapped twice). + - **Zen `gpt-5.3-codex-spark`** is refused by `resolveCallConfig` (`INVALID_CONFIG`) yet dispatched to `/inference/openai/v1/responses` through the hidden route, and the user's `name`/`contextWindow` never reach `resolveModelInfo` (the catalog's 128000 does). + - **Listings.** Before a key is captured the listing patch hides every paid model on both planes (`[]`); after one, only pi-resolved ids are listed and the gap ids are not merged back. + - **A stored gap id blocks every edit of its plane**, `displayName` included: the strict check names the stored id first, ahead of the one being saved. + - **Shapes.** A resolved profile also carries the profile's pass-through fields (`compat`, `defaultContextWindow`, `defaultInput`, `defaultMaxTokens`, `headers`, `modelOverrides`, `thinkingBudgets`), and a built pi provider exposes `getModels()` but no `api` map, so F10 is pinned by behaviour (pi's own "no API implementation" error) rather than by its keys. 6. **P-H `refactor:` one way to patch a host method.** `host/patch-method.ts` and `test/reload.test.ts` (apply A → apply B → dispose A → B intact → dispose B → originals). 7. **P-R1 `feat:` the plugin owns the plane routes.** @@ -385,7 +395,7 @@ What stays private is pinned by contract tests against the real host packages: - **The PR body** names every tagged row it flips. 8. **P-R2 `fix:` the endpoint family, measured.** - Probe 3 URL shapes × 4 protocols × 2 planes, keyless, plus keyed probes per family with the CI keys. - - Set the default family and pin it with a test. If a `modern` cell fails, flip the default; `endpoint: legacy` still maps every protocol. + - Set the default family and pin it with a test. If a `modern` cell fails, flip the default; `endpoint: legacy` still maps every protocol except Go's Google, which stays on `/inference/go/google/v1beta` under every family (`GO_GOOGLE_BASE_URL`). Decide here whether `endpoint: legacy` is a guarded configuration: the keyless matrix records its Responses and Messages cells and Zen's Google cell but does not assert them. 9. **P3 `feat:` Settings complements pi's list.** - The `discoverModels` wrapper answers servable, non-deprecated models, deduped and whitelisted. - Delete `installModelDiscovery` and `toDiscovered`. @@ -461,15 +471,16 @@ P0 → P1a → P1b → P2 → P-T1a → P-T1b → P-H → P-R1 → P-R2 → P3 - the listener fires for another plugin's fiber, and `prepend` puts it outermost; - `raw === _config` holds for `_reload`, `update` and `_commitVolatile`, and not for the editor's candidate (F14); - a deep-frozen input is never mutated, and the fiber keeps the return value (F12); - - the profile shape, the `Config` defaults, the credential record mapping and `recordKeyFor`; + - the profile shape, the `Config` defaults, the credential record mapping (each of `toPiCredential`'s three branches: an api-key record with a key, one carrying `env` too — observable on a plane only in pi's `Models.getAuth` answer, since no plane request reads it — and a non-api-key payload handed through) and `recordKeyFor`; - no secret-role nodes; - the loader's `interpolate`; - pi's public exports. - **Real host, tier B:** - `configEditor.edit` writes the raw `next` to the YAML; - `_commitVolatile` re-resolves through a prepended listener; - - baselines for `settings.describe` and `hasProviderApiKey`. -- **Characterization rows**, tagged, on pi 1.1.0. + - the `settings.describe` and `hasProviderApiKey` host contracts, the latter restated from the real parts (above), with the with-plugin `[BASELINE]` row in the characterization; the Models store's join moves to P-R1. +- **Characterization rows**, tagged, on pi 1.1.0, in both apply orders. +- **Binary e2e and the keyless matrix**, as below. **P-R1** @@ -550,23 +561,23 @@ P0 → P1a → P1b → P2 → P-T1a → P-T1b → P-H → P-R1 → P-R2 → P3 ### The `e2e-host` job (`ci.yml`) -**When it runs:** on dispatch, nightly, and on PRs that touch `src/host/**`, `src/lifecycle.ts`, `src/identity.ts` or `cordis.patch.yml`. +**When it runs:** on dispatch, nightly, and on PRs that touch `src/host/**`, `src/lifecycle.ts`, `src/identity.ts`, the modules that install the host patches and mounts whose behaviour depends on the real loader and apply order, until P-R1 moves them under `src/host/` (`src/responses-provider.ts`, `src/responses-routes.ts`, `src/models-discovery.ts`, `src/google-bridge.ts`, `src/fetch-patch.ts`, `src/pi-provider.ts`, `src/stream-hook.ts`; every other row producer, such as `src/key-capture.ts`, `src/models-catalog.ts` or `src/config.ts`, is covered on each PR by the in-process characterization of the same `S0_TODAY` rows, and by the nightly run), `cordis.patch.yml`, `test/e2e/host/**`, `test/e2e/host-binary.e2e.ts`, `test/support/host.ts`, `test/support/host-versions.ts`, `patches/**`, `.github/workflows/ci.yml`, `vitest.e2e.config.ts` or `vite.config.ts` (the pack config that builds `lib/index.mjs`, which only this job loads into a real host, and the test settings the e2e config inherits) (scoped by a step, since a workflow-level path filter would silence every job). **What it does:** -1. Installs `@deepseek-ai/dsh@$DSH_VERSION` into a throwaway prefix, cached by host version, with npm `overrides` forcing pi. -2. Asserts the pi version that resolved. +1. Installs `@deepseek-ai/dsh@$DSH_VERSION` into a throwaway prefix, cached by host version, with npm `overrides` forcing pi (and, in the pinned cell, cordis and the loader, which the published host declares as tilde ranges). +2. Asserts the pi version that resolved, and in the pinned cell the cordis and loader versions against `HOST_VERSIONS`. 3. Builds, and boots a temporary HOME whose profile links the checkout through a relative `link:`. 4. Runs `test/e2e/host/probe-plugin.mjs` together with `test/e2e/host-binary.e2e.ts`, which runs only when `DSH_HOST_E2E=1`. **Cells:** -- (a) pi 1.1.0, pinned and patched, on PRs; -- (b) whatever npm installs, nightly and on dispatch. +- (a) pi 1.1.0, pinned and patched, on PRs, nightly and on dispatch; it asserts every row; +- (b) whatever npm installs, nightly and on dispatch. It skips the rows that depend on which ids pi describes (the keyed listing and the per-model rows) and still asserts the save rows. Those assume the floating pi lacks at least one of `ZEN_UNDESCRIBED` and still describes every other declared Zen id, so a pi that learns all of them or retires a declared id turns them red by design. Its probe JSON is the record. ### The live gateway job (`OPENCODE_API_KEY` / `OPENCODE_GO_API_KEY`) -- **Every PR:** the keyless matrix. A 404 means the path is gone; a 400 "Model is unavailable" or a 410 means the model is gone. +- **Every PR:** the keyless matrix. A 404 means the path is gone; a 400 "Model is unavailable" or a 410 means the model is gone. It asserts that every cell has a model and was sent, and that no cell today's plugin dispatches to with the default endpoint (`auto`, an alias of `modern`: legacy completions; modern Responses, Messages and Google) nor P-R1's modern completions target answers 404; the other legacy verdicts are P-R2's record. `endpoint: legacy` users dispatch Responses and Messages on both planes, and Zen's Google, to the legacy cells, which are recorded for P-R2, not asserted; Go's Google stays on `/inference/go/google/v1beta` under every family (`GO_GOOGLE_BASE_URL`), the modern cell already asserted. - **PRs that touch `src/host/**`, and nightly:** minimal keyed probes, one per plane × protocol, with a small `max_tokens`. - **Nightly and on dispatch:** the full keyed suites. diff --git a/package.json b/package.json index 898077a6..3d7b5005 100644 --- a/package.json +++ b/package.json @@ -76,6 +76,7 @@ "test": "vp test", "test:coverage": "vp test run --coverage", "test:e2e": "vp test --config vitest.e2e.config.ts", + "test:e2e:host": "vp test --config vitest.e2e.config.ts test/e2e/host-binary.e2e.ts", "test:mock": "vp test --config vitest.mock.config.ts", "test:shuffle": "vp test run --sequence.shuffle", "typecheck": "tsc --noEmit", diff --git a/test/e2e/host-binary.e2e.ts b/test/e2e/host-binary.e2e.ts new file mode 100644 index 00000000..e88041d8 --- /dev/null +++ b/test/e2e/host-binary.e2e.ts @@ -0,0 +1,500 @@ +/** + * The published host binary, booted with this plugin and the measurement probe. + * + * The in-process tiers (`test/support/host.ts`) assemble the host from the + * packages this repository installs; this file boots the real thing — the + * `@deepseek-ai/dsh` the `e2e-host` job installs into a throwaway prefix — the + * way a user's profile does: a temporary HOME, a profile whose `package.json` + * links this checkout and the probe bundle (`test/e2e/host/`) through RELATIVE + * `link:` paths, `dsh plugin … install`, then `dsh --profile e2e --no-open`. + * The probe records what the host serves and exits the process; this file reads + * its JSON. + * + * The assertions are TODAY's behaviour, read from the same `S0_TODAY` table the + * in-process characterization pins, so the binary and the harness cannot drift + * apart unnoticed. P-R1 flips the rows its PR names. The loader starts rows in + * parallel, so which of pi-ai and the plugin applies first is a race. The rows + * depend on whether the plugin's adapter decoration wrapped pi-ai's plane + * adapter, so the probe records that and the order it implies + * (`inferredOrder`), and the per-model rows follow it (`todayRowsFor`) rather + * than pinning a 70 ms margin. + * + * Gated: it runs only with `DSH_HOST_E2E=1`, and needs `DSH_HOST_PREFIX`, a + * directory whose `node_modules/@deepseek-ai/dsh` is the installed host. + * `DSH_HOST_PI_CELL=pinned` (the default) also requires the pi + * `host-versions.ts` names, with the harness patch, and asserts every row; + * `floating` accepts whatever pi the host resolved. It skips the rows that + * depend on which ids pi describes (the keyed listing and the per-model rows) + * and still asserts the save rows. Those assume the floating pi lacks at least + * one of `ZEN_UNDESCRIBED` and still describes every other declared Zen id, so + * a pi that learns all of them or retires a declared id turns them red by + * design — triage a red floating run as a pi change first. + * + * `DSH_HOST_PROBE_COPY` names a file to keep the probe's JSON in, for the CI + * artifact; it is written whenever the probe wrote one, a failed run included. + * `pnpm` must be on `PATH`, and `pnpm run build` must have run. + * The keys the profile names are DUMMY values written to the temporary + * credentials store, so every turn reaches the probe's 401 and nothing real is + * ever sent. + * + * @module test/e2e/host-binary.e2e + */ +import { type ChildProcess, spawn, spawnSync } from "node:child_process"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join, relative, sep } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { afterAll, beforeAll, describe, expect, it } from "vitest"; + +import { HOST_VERSIONS } from "../support/host-versions.ts"; +import { + type ApplyOrder, + CHARACTERIZED_MODELS, + CHARACTERIZED_PROFILE, + PI_AI_PACKAGE, + S0_TODAY, + SAVE_BLOCKED_BY_ANY_UNDESCRIBED, + SAVE_CANDIDATE, + todayRowsFor, + ZEN_UNDESCRIBED, +} from "../support/host.ts"; +import { INTERNAL_ROUTES } from "../support/src.ts"; + +const ENABLED = process.env.DSH_HOST_E2E === "1"; +const PREFIX = process.env.DSH_HOST_PREFIX ?? ""; +const CELL = process.env.DSH_HOST_PI_CELL ?? "pinned"; +const PINNED = CELL === "pinned"; +const BOOT_DEADLINE_MS = 180_000; +/** Each `dsh plugin install` (a blocking `spawnSync`). */ +const INSTALL_DEADLINE_MS = 120_000; +/** + * The hook's budget, derived from the deadlines it actually runs — two installs + * and a boot — so every failure path reaches its own message and the probe + * copy before vitest's hook timeout can fire. `spawnSync` blocks the worker, so + * the hook timer cannot interrupt an install; it can only fire during the boot + * wait, which is the window where the child is live and the artifact matters. + */ +const HOOK_BUDGET_MS = 2 * INSTALL_DEADLINE_MS + BOOT_DEADLINE_MS + 30_000; +const CHECKOUT = fileURLToPath(new URL("../..", import.meta.url)); +const PROBE_BUNDLE = join(CHECKOUT, "test", "e2e", "host"); + +/** What the probe writes (test/e2e/host/probe-plugin.mjs). */ +interface ProbeOutput { + readonly fatal?: string; + readonly fetchInnermost: boolean; + readonly routesSettled: boolean; + readonly planeAdapterDecorated: boolean; + /** Inferred from `planeAdapterDecorated`, not observed: see the probe. */ + readonly inferredOrder: ApplyOrder; + readonly providers: string[]; + readonly registryRoutes: string[]; + readonly owners: Record; + readonly configurable: { provider: string; settingsNs: string }[]; + readonly listModels: Record; + readonly listModelsAfter: Record; + readonly modelInfo: Record< + string, + { context: number; efforts: string[] } | { error: string } + >; + readonly callConfig: Record; + readonly dispatch: Record; + readonly describe: { user: string[]; value: string[] } | undefined; + readonly saveBlocked: { id: string; rejected?: string; accepted?: true }; + readonly saveCleared?: { + dropped: string[]; + rejected?: string; + accepted?: true; + }; + readonly save: { id: string; rejected?: string; accepted?: true }; + readonly afterSave?: unknown; +} + +/** The entries of a directory, or none when it cannot be read. */ +const entriesOf = (dir: string): string[] => { + try { + return readdirSync(dir).filter((entry) => !entry.startsWith(".")); + } catch { + return []; + } +}; + +/** + * Every `package.json` of pi under a `node_modules` tree, at any depth: the + * same set as CI's `find -path '*\/@earendil-works/pi-ai/package.json'`. + * Every package's own `node_modules` is walked, scoped ones included — pi-ai's + * and its `@earendil-works` siblings' too, where a nested copy would hide. + */ +const piCopies = (root: string): string[] => { + const found: string[] = []; + const walkPackage = (dir: string, scope: string, name: string): void => { + const manifest = join(dir, "package.json"); + if ( + scope === "@earendil-works" && + name === "pi-ai" && + existsSync(manifest) + ) { + found.push(manifest); + } + walk(join(dir, "node_modules")); + }; + const walk = (dir: string): void => { + for (const entry of entriesOf(dir)) { + const full = join(dir, entry); + if (entry.startsWith("@")) { + for (const scoped of entriesOf(full)) { + walkPackage(join(full, scoped), entry, scoped); + } + } else { + walkPackage(full, "", entry); + } + } + }; + walk(root); + return found; +}; + +/** + * How the host process ended. `timedOut` is set only by this file's own + * deadline timer, so a signal the test did not send (a V8 abort, the runner's + * OOM killer) never reads as a timeout. + */ +interface RunEnd { + code: number | null; + signal: NodeJS.Signals | null; + timedOut: boolean; +} + +/** + * Why the run failed, from how it ended and what it left behind, for the + * failure message: the real cause first, so a timeout never reads as an exit + * code, a crash never reads as a timeout, and a complete result never reads + * as an unreadable one. + * + * @param end - how the process ended. + * @param written - the probe file's text, if it was written. + * @param parsed - that text parsed, if it parsed. + * @returns one sentence naming the cause. + */ +const whyRunFailed = ( + end: RunEnd, + written: string | undefined, + parsed: ProbeOutput | undefined +): string => { + let how = `exited ${String(end.code)}`; + if (end.timedOut) { + how = `timed out after ${BOOT_DEADLINE_MS} ms and was killed`; + } else if (end.signal !== null) { + how = `was killed by ${end.signal}`; + } + if (parsed?.fatal !== undefined) { + return `the host ${how}, with the probe's fatal error: ${parsed.fatal}`; + } + let left = "after writing a complete probe result (see the artifact)"; + if (written === undefined) { + left = "without a probe result"; + } else if (parsed === undefined) { + left = "with an unreadable probe result"; + } + return `the host ${how}, ${left}`; +}; + +/** POSIX separators, so a `link:` spec reads the same on every runner. */ +const posix = (path: string): string => path.split(sep).join("/"); + +let home = ""; +let out: ProbeOutput; +/** The booted host, kept so `afterAll` can stop it before removing HOME. */ +let child: ChildProcess | undefined; + +describe.skipIf(!ENABLED)("e2e · the published host binary (dsh)", () => { + beforeAll(async () => { + if ( + PREFIX.length === 0 || + !existsSync(join(PREFIX, "node_modules", "@deepseek-ai", "dsh")) + ) { + throw new Error( + "DSH_HOST_PREFIX must name a directory whose node_modules/@deepseek-ai/dsh is the installed host" + ); + } + const bin = join( + PREFIX, + "node_modules", + "@deepseek-ai", + "dsh", + "lib", + "bin.js" + ); + const piRoot = join(PREFIX, "node_modules", "@earendil-works", "pi-ai"); + const piVersion = ( + JSON.parse(readFileSync(join(piRoot, "package.json"), "utf8")) as { + version: string; + } + ).version; + console.info(`[e2e-host] cell ${CELL}: pi ${piVersion}`); + expect(piCopies(join(PREFIX, "node_modules"))).toHaveLength(1); + if (PINNED) { + expect(piVersion).toBe(HOST_VERSIONS.pi); + // The harness patch is applied (the check test/host-versions.test.ts runs). + expect( + readFileSync(join(piRoot, "dist", "api", "pi-messages.js"), "utf8") + ).not.toContain("parseStreamingJson(json)"); + } + expect(existsSync(join(CHECKOUT, "lib", "index.mjs"))).toBe(true); + + home = mkdtempSync(join(tmpdir(), "dsh-host-e2e-")); + const dshHome = join(home, ".dsh"); + const profileDir = join(dshHome, "profiles", "e2e"); + // No inherited npm_config_* and no OPENCODE_* key: the run sees only what + // this file gives it. + const env = { + CI: "1", + DSH_HOME: dshHome, + DSH_TELEMETRY_DISABLED: "1", + HOME: home, + PATH: process.env.PATH ?? "", + }; + const dsh = (args: string[]) => { + const run = spawnSync(process.execPath, [bin, ...args], { + cwd: home, + encoding: "utf8", + env, + timeout: INSTALL_DEADLINE_MS, + }); + if (run.status !== 0) { + // A spawn error, a timeout and a signal all leave `status` null; name + // the one that happened rather than "exited null". + let how = `exited ${String(run.status)}`; + if (run.error !== undefined) { + how = `failed: ${run.error.message}`; + } else if (run.signal !== null) { + how = `was killed by ${run.signal}`; + } + throw new Error( + `dsh ${args.join(" ")} ${how}:\n${(run.stderr ?? "").slice(-2000)}` + ); + } + }; + // Initializes the profile, then installs the two linked bundles. + dsh(["plugin", "--profile", "e2e", "install"]); + writeFileSync( + join(profileDir, "package.json"), + `${JSON.stringify( + { + dependencies: { + "dsh-host-probe": `link:${posix(relative(profileDir, PROBE_BUNDLE))}`, + "dsh-opencode-patch": `link:${posix(relative(profileDir, CHECKOUT))}`, + }, + dsh: { + profile: { + bundles: [ + "@deepseek-ai/dsh-base", + "@deepseek-ai/dsh-web-app", + "dsh-opencode-patch", + "dsh-host-probe", + ], + }, + }, + name: "dsh-profile-e2e", + private: true, + }, + null, + 2 + )}\n` + ); + dsh(["plugin", "--profile", "e2e", "install"]); + // The characterization profile, as JSON text (valid YAML). + writeFileSync( + join(profileDir, "cordis.patch.yml"), + `${JSON.stringify([{ config: { providers: CHARACTERIZED_PROFILE }, id: "llm-pi-ai" }], null, 2)}\n` + ); + // Dummy keys in the credentials store, never the environment: the plugin + // reads the environment directly, and the in-process rows were measured + // with the keys in the credentials service only. + mkdirSync(dshHome, { recursive: true }); + writeFileSync( + join(dshHome, ".credentials.yaml"), + "version: 1\nrefs:\n OPENCODE_API_KEY: probe-zen-not-a-key\n OPENCODE_GO_API_KEY: probe-go-not-a-key\n", + { mode: 0o600 } + ); + + const probeOut = join(home, "probe.json"); + const host = spawn( + process.execPath, + [bin, "--profile", "e2e", "--no-open", "--port", "0"], + { + cwd: home, + env: { + ...env, + PROBE_MODELS: JSON.stringify(CHARACTERIZED_MODELS), + PROBE_OUT: probeOut, + PROBE_PI_PACKAGE: PI_AI_PACKAGE, + PROBE_ROUTES: JSON.stringify(INTERNAL_ROUTES), + PROBE_SAVE: SAVE_CANDIDATE, + PROBE_SAVE_DROP: JSON.stringify(ZEN_UNDESCRIBED), + }, + stdio: ["ignore", "pipe", "pipe"], + } + ); + child = host; + let stderr = ""; + host.stderr.on("data", (chunk: Buffer) => { + stderr = `${stderr}${chunk.toString("utf8")}`.slice(-4000); + }); + host.stdout.resume(); + const end = await new Promise((resolve) => { + const timer = setTimeout(() => { + host.kill("SIGKILL"); + resolve({ code: null, signal: "SIGKILL", timedOut: true }); + }, BOOT_DEADLINE_MS); + host.on("exit", (code, signal) => { + clearTimeout(timer); + resolve({ code, signal, timedOut: false }); + }); + }); + // Whatever the probe wrote is kept BEFORE the exit code is judged: a + // failed run is the one whose artifact matters. The probe writes URLs, + // model ids, verdicts and host error messages (cut to 500 characters) — + // never a header or a key value. + const written = existsSync(probeOut) + ? readFileSync(probeOut, "utf8") + : undefined; + const copy = process.env.DSH_HOST_PROBE_COPY ?? ""; + if (written !== undefined && copy.length > 0) { + writeFileSync(copy, written); + } + let parsed: ProbeOutput | undefined; + try { + parsed = + written === undefined + ? undefined + : (JSON.parse(written) as ProbeOutput); + } catch { + parsed = undefined; + } + if (end.code !== 0 || parsed === undefined || parsed.fatal !== undefined) { + throw new Error( + `${whyRunFailed(end, written, parsed)}\n--- stderr (tail) ---\n${stderr.slice(-2000)}` + ); + } + out = parsed; + console.info(`[e2e-host] probe output:\n${JSON.stringify(out, null, 2)}`); + }, HOOK_BUDGET_MS); + + afterAll(() => { + // A hook that failed before the boot settled leaves the host running; + // stop it before its HOME goes. + if ( + child !== undefined && + child.exitCode === null && + child.signalCode === null + ) { + child.kill("SIGKILL"); + } + if (home.length > 0) { + rmSync(home, { force: true, recursive: true }); + } + }); + + it("measured with the probe innermost and the hidden routes settled", () => { + expect(out.fatal).toBeUndefined(); + expect(out.fetchInnermost).toBe(true); + expect(out.routesSettled).toBe(true); + }); + + it("[INTENDED_CHANGE] registry: the planes are pi-ai's; the hidden routes are registered and not listed (P-R1 claims the planes)", () => { + expect(out.providers).toEqual( + expect.arrayContaining(["opencode", "opencode-go"]) + ); + for (const route of INTERNAL_ROUTES) { + expect(out.providers).not.toContain(route); + expect(out.registryRoutes).toContain(route); + } + expect(out.owners.opencode).toBe("PiAiAdapter"); + expect(out.owners["opencode-go"]).toBe("PiAiAdapter"); + for (const plane of ["opencode", "opencode-go"]) { + expect(out.configurable).toContainEqual( + expect.objectContaining({ provider: plane, settingsNs: "llm-pi-ai" }) + ); + } + }); + + it("[INTENDED_CHANGE] keyless listing: every paid model is hidden before a key is captured", () => { + expect(out.listModels).toEqual(S0_TODAY.keylessListing); + }); + + // These two depend on which ids pi describes, so they are pinned for the + // pi this repository pins; the floating cell skips them and its probe JSON + // is the record. + describe("rows that depend on which ids pi describes", () => { + it.skipIf(!PINNED)( + "[KNOWN_DEFECT] keyed listing: the gap ids are missing (pinned pi)", + () => { + expect(out.listModelsAfter).toEqual(S0_TODAY.keyedListing); + } + ); + + it.skipIf(!PINNED)( + "resolveModelInfo, resolveCallConfig and the wire match the in-process rows for the decoration state the probe recorded (pinned pi)", + () => { + // The loader starts rows in parallel (F17), so whether the plugin's + // decoration wrapped pi-ai's plane adapter is a race: measured on the + // published binary it did not (the plugin applied about 70 ms first), + // which is no margin to pin. So the rows follow the recorded state, and + // `todayRowsFor` throws on a missing or unknown value. + console.info( + `[e2e-host] plane adapter decorated: ${String(out.planeAdapterDecorated)} (inferred order ${out.inferredOrder})` + ); + for (const [key, row] of Object.entries( + todayRowsFor(out.inferredOrder) + )) { + expect(out.modelInfo[key], key).toEqual(row.info); + expect(out.callConfig[key], key).toBe(row.callConfig); + expect(out.dispatch[key], key).toEqual({ + finish: row.finish, + sent: row.sent, + }); + } + } + ); + }); + + it("[BASELINE] the settings page reads every plane the profile declares", () => { + expect(out.describe?.value.toSorted()).toEqual([ + "deepseek", + "opencode", + "opencode-go", + ]); + }); + + it("[KNOWN_DEFECT] a stored gap id blocks saving any new id on its plane", () => { + // pi-ai's strict check names the FIRST stored id the installed pi cannot + // describe. The pinned pi lacks gpt-5.3-codex-spark, so it is named; a + // floating pi may have learned it, and then names the next undescribed id + // — so the floating cell asserts only that one of them blocks the save. + expect(out.saveBlocked.rejected).toMatch( + PINNED ? S0_TODAY.saveBlockedByStoredGap : SAVE_BLOCKED_BY_ANY_UNDESCRIBED + ); + }); + + it("[KNOWN_DEFECT] a catalog-unknown id cannot be saved, once the plane stores no gap id", () => { + // Clearing only ZEN_UNDESCRIBED assumes the installed pi still describes + // the other opencode ids the profile declares. The floating cell asserts + // it too: a pi that retires one of them would fail here, and that is a + // retirement worth a red nightly rather than a row to skip. + expect(out.saveCleared).toEqual({ + accepted: true, + dropped: [...ZEN_UNDESCRIBED], + }); + expect(out.save.rejected).toMatch(S0_TODAY.saveRejectedUnknown); + expect(out.afterSave).toBeUndefined(); + }); +}); diff --git a/test/e2e/host/cordis.patch.yml b/test/e2e/host/cordis.patch.yml new file mode 100644 index 00000000..e6df604d --- /dev/null +++ b/test/e2e/host/cordis.patch.yml @@ -0,0 +1,5 @@ +# The measurement bundle the e2e-host job links into a throwaway profile: one +# row, the probe. See test/e2e/host-binary.e2e.ts. +- insert: + - id: dsh-host-probe + name: dsh-host-probe diff --git a/test/e2e/host/package.json b/test/e2e/host/package.json new file mode 100644 index 00000000..b20a1677 --- /dev/null +++ b/test/e2e/host/package.json @@ -0,0 +1,12 @@ +{ + "name": "dsh-host-probe", + "version": "0.0.0", + "private": true, + "type": "module", + "main": "probe-plugin.mjs", + "dsh": { + "bundle": { + "patch": "./cordis.patch.yml" + } + } +} diff --git a/test/e2e/host/probe-plugin.mjs b/test/e2e/host/probe-plugin.mjs new file mode 100644 index 00000000..163a07e3 --- /dev/null +++ b/test/e2e/host/probe-plugin.mjs @@ -0,0 +1,347 @@ +/** + * The measurement probe the `e2e-host` job boots inside the real host. + * + * A bundle plugin with one job: once the published `dsh` binary has booted a + * profile holding this plugin and `dsh-opencode-patch`, ask the host what it + * now serves — the registry, the listings, the validation verdicts, where one + * turn per model is sent, what the settings page reads, and whether a + * catalog-unknown id can be saved — write that as JSON to `PROBE_OUT`, and exit + * the process. `test/e2e/host-binary.e2e.ts` reads the file. + * + * It never reaches the network: `fetch` is replaced at module load, so every + * request to `opencode.ai` or `models.opencode.ai` is recorded (URL and model + * only — never a header, so never a key) and answered with a 401, and every + * other host passes through untouched. What it writes is URLs, model ids, + * verdicts and host error messages — never a header or a key value — and + * every error message is cut to {@link MESSAGE_LIMIT} characters, since one + * could name a path on the runner. + * + * Inputs, all from the environment the e2e sets: + * + * - `PROBE_OUT`: where to write the JSON; + * - `PROBE_MODELS`: `{ "": ["", …] }`, the ids to ask about; + * - `PROBE_SAVE`: an id to append to `opencode`'s models through the config + * editor, the way the Settings page saves one. It is tried three times over: + * onto the plane as booted (`saveBlocked`), then — after the ids in + * `PROBE_SAVE_DROP` are removed from the plane (`saveCleared`) — once more + * (`save`), so the last answer is about the id itself and not about a gap id + * the plane already stores; + * - `PROBE_SAVE_DROP`: `["", …]`, the ids to remove first; + * - `PROBE_ROUTES`: `["", …]`, the hidden routes to wait for, which the + * e2e passes from `INTERNAL_ROUTES` so the probe holds no copy of them; + * - `PROBE_PI_PACKAGE`: the package name pi-ai's loader entry carries; + * - `PROBE_SETTLE_MS`: how long to wait for the plugin's hidden routes + * (default 30000). + * + * @module test/e2e/host/probe-plugin + */ +import { writeFileSync } from "node:fs"; + +export const name = "dsh-host-probe"; + +const OUT = process.env.PROBE_OUT; +const SETTLE_MS = Number(process.env.PROBE_SETTLE_MS ?? "30000"); +const GATEWAY_HOSTS = new Set(["opencode.ai", "models.opencode.ai"]); +/** The routes today's plugin mounts behind the planes. */ +const HIDDEN_ROUTES = JSON.parse(process.env.PROBE_ROUTES ?? "[]"); +const PI_AI_PACKAGE = process.env.PROBE_PI_PACKAGE ?? ""; +/** The longest error message written: enough to triage, too short for a dump. */ +const MESSAGE_LIMIT = 500; + +/** An error's message, cut to {@link MESSAGE_LIMIT}. */ +const messageOf = (error) => + String(error?.message ?? error).slice(0, MESSAGE_LIMIT); + +/** A service the measurement needs, or a named failure where it is missing. */ +const required = (ctx, service) => { + const value = ctx.get(service); + if (value === undefined) { + throw new Error(`probe: no ${service} service in this host`); + } + return value; +}; + +const requests = []; +const passThrough = globalThis.fetch; + +const urlOf = (input) => { + if (typeof input === "string") return input; + return input instanceof URL ? input.href : input.url; +}; + +const hostOf = (url) => { + try { + return new URL(url).hostname; + } catch { + return ""; + } +}; + +const modelOf = async (input, init) => { + let body = init?.body; + if (body === undefined && input instanceof Request) { + body = await input.clone().text(); + } + try { + const { model } = JSON.parse(String(body)); + return typeof model === "string" ? model : null; + } catch { + return null; + } +}; + +const probeFetch = async (input, init) => { + const url = urlOf(input); + if (!GATEWAY_HOSTS.has(hostOf(url))) { + return passThrough(input, init); + } + requests.push({ model: await modelOf(input, init), url }); + return Response.json( + { error: { message: "probe: offline", type: "AuthError" } }, + { status: 401 } + ); +}; +// Installed at module load, which the host runs before any plugin's `apply` +// that needs a service: the plugin's fetch patch then wraps THIS function, so +// the probe sees the request the patch actually sends. +globalThis.fetch = probeFetch; + +/** A call's answer, or its failure as `{ code, message }`. */ +const settle = async (run) => { + try { + return { ok: await run() }; + } catch (error) { + return { + error: { + code: typeof error?.code === "string" ? error.code : undefined, + message: messageOf(error), + }, + }; + } +}; + +const sleep = (ms) => + new Promise((resolve) => { + setTimeout(resolve, ms); + }); + +/** Drain one turn through the host's own stream path. */ +const turn = async (llm, provider, model) => { + const before = requests.length; + let finish; + const drained = await settle(async () => { + for await (const chunk of llm.stream({ + messages: [{ content: [{ text: "hi", type: "text" }], role: "user" }], + model, + provider, + signal: AbortSignal.timeout(15_000), + })) { + if (chunk.type === "finish") { + finish = chunk.reason?.failure?.code ?? chunk.reason?.kind; + } + } + }); + return { + finish, + sent: requests + .slice(before) + .map((request) => request.url) + .filter((url) => hostOf(url) === "opencode.ai"), + ...(drained.error === undefined ? {} : { error: drained.error }), + }; +}; + +/** Every id in `PROBE_MODELS` asked three questions: listing, info, verdict. */ +const askAbout = async (llm, ids) => { + const listModels = {}; + const modelInfo = {}; + const callConfig = {}; + for (const [route, models] of Object.entries(ids)) { + const listed = await settle(() => llm.listModels(route)); + listModels[route] = listed.ok?.map((model) => model.id) ?? listed; + for (const model of models) { + const info = await settle(() => llm.resolveModelInfo(route, model)); + modelInfo[`${route}/${model}`] = info.ok + ? { + context: info.ok.context?.contextWindow, + efforts: + info.ok.reasoning?.efforts?.map((effort) => effort.id) ?? [], + } + : { error: info.error.code }; + const call = await settle(() => + llm.resolveCallConfig({ model, provider: route }) + ); + callConfig[`${route}/${model}`] = call.ok ? "ok" : call.error.code; + } + } + return { callConfig, listModels, modelInfo }; +}; + +/** One config-editor edit of `opencode`, as the Settings page makes it. */ +const editZen = async (editor, entry, mutate) => { + const saved = await settle(() => + editor.edit(entry, (current) => { + const providers = { ...current.providers }; + providers.opencode = mutate(providers.opencode ?? {}); + return { ...current, providers }; + }) + ); + return saved.error === undefined + ? { accepted: true } + : { rejected: saved.error.message }; +}; + +/** + * Save `saveId` into `opencode`'s models: onto the plane as booted, then onto + * the plane without the `drop` ids, so the last verdict is about `saveId`. + */ +const saveSequence = async (llm, editor, saveId, drop) => { + const entry = editor + .entries() + .find((candidate) => candidate.options?.name === PI_AI_PACKAGE); + const append = (plane) => ({ + ...plane, + models: [...(plane.models ?? []), { id: saveId }], + }); + const blocked = await editZen(editor, entry, append); + let out = { saveBlocked: { id: saveId, ...blocked } }; + let saved = blocked; + if (blocked.rejected !== undefined) { + const cleared = await editZen(editor, entry, (plane) => ({ + ...plane, + models: (plane.models ?? []).filter((model) => !drop.includes(model.id)), + })); + saved = await editZen(editor, entry, append); + out = { ...out, saveCleared: { dropped: drop, ...cleared } }; + } + out = { ...out, save: { id: saveId, ...saved } }; + if (saved.rejected !== undefined) { + return out; + } + const listed = await settle(() => llm.listModels("opencode")); + const call = await settle(() => + llm.resolveCallConfig({ model: saveId, provider: "opencode" }) + ); + return { + ...out, + afterSave: { + callConfig: call.error?.code ?? "ok", + dispatch: await turn(llm, "opencode", saveId), + listed: listed.ok?.map((model) => model.id) ?? listed, + }, + }; +}; + +const measure = async (ctx) => { + if (HIDDEN_ROUTES.length === 0 || PI_AI_PACKAGE.length === 0) { + throw new Error("probe: PROBE_ROUTES and PROBE_PI_PACKAGE must be set"); + } + await ctx.get("loader")?.await?.(); + const llm = required(ctx, "llm"); + const registry = () => llm.adapters; + const until = Date.now() + SETTLE_MS; + while ( + HIDDEN_ROUTES.some((route) => !registry().has(route)) && + Date.now() < until + ) { + await sleep(100); + } + const ids = JSON.parse(process.env.PROBE_MODELS ?? "{}"); + const registered = { + configurable: llm + .listConfigurableProviders() + .filter((entry) => /opencode/u.test(entry.provider)) + .map((entry) => ({ + provider: entry.provider, + settingsNs: entry.settingsNs, + settingsPath: entry.settingsPath, + })), + fetchInnermost: globalThis.fetch !== probeFetch, + owners: Object.fromEntries( + [...registry()].map(([route, row]) => [ + route, + row.adapter.constructor.name, + ]) + ), + // Whether the plugin's adapter decoration (decoratePlaneAdapters in + // src/models-discovery.ts) wrapped pi-ai's plane adapter. It wraps only + // the adapters registered when its effect ran, so the apply order is + // INFERRED from it (`inferredOrder`) — not observed: a pi-ai restart after + // the plugin applied would read as plugin-first. The per-model rows depend + // on the decoration, not on the order as such, so this is what they need. + planeAdapterDecorated: Object.hasOwn( + registry().get("opencode")?.adapter ?? {}, + "resolveModel" + ), + providers: llm.listProviders().map((provider) => provider.id), + registryRoutes: [...registry().keys()], + routesSettled: HIDDEN_ROUTES.every((route) => registry().has(route)), + }; + registered.inferredOrder = registered.planeAdapterDecorated + ? "pi-ai-first" + : "plugin-first"; + const asked = await askAbout(llm, ids); + const zenNs = registered.configurable.find( + (entry) => entry.provider === "opencode" + )?.settingsNs; + const discovered = await settle(() => + llm.discoverModels(zenNs, { provider: "opencode" }) + ); + const dispatch = {}; + for (const [route, models] of Object.entries(ids)) { + for (const model of models) { + dispatch[`${route}/${model}`] = await turn(llm, route, model); + } + } + const listModelsAfter = {}; + for (const route of Object.keys(ids)) { + const listed = await settle(() => llm.listModels(route)); + listModelsAfter[route] = listed.ok?.map((model) => model.id) ?? listed; + } + const piDescriptor = required(ctx, "settings") + .describe() + .find((descriptor) => descriptor.ns === "llm-pi-ai"); + const editor = required(ctx, "configEditor"); + const saveId = process.env.PROBE_SAVE ?? ""; + const drop = JSON.parse(process.env.PROBE_SAVE_DROP ?? "[]"); + const saving = + saveId.length === 0 ? {} : await saveSequence(llm, editor, saveId, drop); + return { + ...registered, + ...asked, + ...saving, + describe: + piDescriptor === undefined + ? undefined + : { + user: Object.keys(piDescriptor.user?.providers ?? {}), + value: Object.keys(piDescriptor.value?.providers ?? {}), + }, + discovery: discovered.ok ? { count: discovered.ok.length } : discovered, + dispatch, + listModelsAfter, + }; +}; + +export const apply = (ctx) => { + let stopped = false; + ctx.effect?.(() => () => { + stopped = true; + }); + // Not awaited: `apply` returns at once, and the measurement runs once the + // loader has settled every row. + void (async () => { + try { + const out = await measure(ctx); + if (stopped) return; + writeFileSync(OUT, `${JSON.stringify(out, null, 2)}\n`); + setTimeout(() => process.exit(0), 200); + } catch (error) { + const fatal = messageOf(error); + // On stderr too: the e2e shows the tail when the host exits non-zero. + process.stderr.write(`[dsh-host-probe] fatal: ${fatal}\n`); + writeFileSync(OUT, `${JSON.stringify({ fatal })}\n`); + setTimeout(() => process.exit(1), 200); + } + })(); +}; diff --git a/test/e2e/keyless-matrix.e2e.ts b/test/e2e/keyless-matrix.e2e.ts new file mode 100644 index 00000000..c803deef --- /dev/null +++ b/test/e2e/keyless-matrix.e2e.ts @@ -0,0 +1,185 @@ +/** + * The keyless 401-vs-404 matrix against the live gateway. + * + * Sixteen POSTs, none carrying a key: two planes × four protocols × the + * legacy (`/zen(/go)/v1`) and modern (`/inference(/go)/…`) endpoint families, + * each with one PAID model from the live catalog slice (a free model answers + * keyless with a real completion, so the matrix never picks one). The gateway + * authenticates before it routes, so the answer says whether the PATH exists + * without spending anything. It runs on every pull request, because it needs + * no secret; it never reads `OPENCODE_API_KEY` or `OPENCODE_GO_API_KEY`. + * + * What it asserts: + * + * - every one of the sixteen cells has a model and was SENT, and every answer + * is one the matrix can read — an empty or reshaped catalog fails here + * rather than measuring nothing; + * - no cell today's plugin dispatches to with the default endpoint (`auto`, + * an alias of `modern`; `TODAY_DISPATCH`: completions on the legacy family, + * the hidden routes' Responses, Messages and Google on the modern one) + * answers "path gone"; + * - nor do the modern completions cells (`NEXT_DISPATCH`), where P-R1 moves + * completions: a forward guard, labelled as such when it fails. + * + * The other legacy verdicts are recorded, not asserted: P-R2 decides the + * family from them. They include the cells an `endpoint: legacy` user + * dispatches to — Responses and Messages on both planes, and Zen's Google — + * so this file does not guard that configuration; P-R2 decides whether it + * should. Go's Google stays on `/inference/go/google/v1beta` under every + * family (`GO_GOOGLE_BASE_URL`), the modern cell `TODAY_DISPATCH` asserts. + * The cell table, both sets and the classifier are unit-tested offline in + * `test/keyless-matrix.test.ts`. + * + * It always measures the vendor's own gateway: the cells are the vendor's + * paths, which a mirror's `/zen(/go)/v1` base cannot name. So it SKIPS when + * `OPENCODE_ZEN_BASE_URL` or `OPENCODE_GO_BASE_URL` points the rest of the + * suite at a mirror. + * + * Opt-in: `OPENCODE_E2E=1 pnpm run test:e2e`. + * + * @module test/e2e/keyless-matrix.e2e + */ +import { describe, expect, it } from "vitest"; + +import { + cellKey, + classifyKeylessProbe, + isTransientProbeAnswer, + type KeylessVerdict, + keylessMatrix, + type MatrixCell, + NEXT_DISPATCH, + TODAY_DISPATCH, +} from "../support/keyless-matrix.ts"; +import { MODELS_OPENCODE_URL, parseModelsDevCatalog } from "../support/src.ts"; + +const LIVE = process.env.OPENCODE_E2E === "1"; +/** A mirror the rest of the suite is pointed at, which this file cannot follow. */ +const MIRRORED = ["OPENCODE_ZEN_BASE_URL", "OPENCODE_GO_BASE_URL"].some( + (name) => (process.env[name] ?? "").length > 0 +); +if (LIVE && MIRRORED) { + console.info( + "[keyless-matrix] skipped: a mirror base URL is set, and the matrix measures only the vendor's own paths" + ); +} + +const PROBE_TIMEOUT_MS = 30_000; +/** The pause before the one retry of a transient answer. */ +const RETRY_AFTER_MS = 2000; + +/** One probe's outcome, with no request or response header in it. */ +interface ProbeRow { + readonly cell: string; + readonly model: string | undefined; + readonly status: number | undefined; + readonly verdict: KeylessVerdict | "transport-error"; + /** The start of the body, kept only for a verdict the matrix cannot read. */ + readonly body?: string; +} + +/** One keyless request: its status and body, or `undefined` on a transport error. */ +const send = async ( + cell: MatrixCell +): Promise<{ status: number; bodyText: string } | undefined> => { + try { + const response = await fetch(cell.url, { + body: JSON.stringify(cell.body), + headers: cell.headers, + method: cell.method, + signal: AbortSignal.timeout(PROBE_TIMEOUT_MS), + }); + return { bodyText: await response.text(), status: response.status }; + } catch { + return undefined; + } +}; + +const probe = async (cell: MatrixCell): Promise => { + const name = cellKey(cell); + if (cell.body === undefined) { + return { + cell: name, + model: undefined, + status: undefined, + verdict: "no-model", + }; + } + let answer = await send(cell); + // A rate limit, a server error or a dropped connection is retried once + // before it is read: one transient answer must not fail every pull request. + if (isTransientProbeAnswer(answer?.status)) { + await new Promise((resolve) => { + setTimeout(resolve, RETRY_AFTER_MS); + }); + answer = await send(cell); + } + if (answer === undefined) { + return { + cell: name, + model: cell.model, + status: undefined, + verdict: "transport-error", + }; + } + const verdict = classifyKeylessProbe(answer); + return { + cell: name, + model: cell.model, + status: answer.status, + verdict, + ...(verdict === "other" ? { body: answer.bodyText.slice(0, 200) } : {}), + }; +}; + +describe.skipIf(!LIVE || MIRRORED)( + "live gateway · the keyless path matrix", + () => { + it( + "sends every cell, reads every answer, and no cell the plugin dispatches to (or P-R1 will) answers path-gone", + async () => { + const listing = await fetch(MODELS_OPENCODE_URL, { + signal: AbortSignal.timeout(PROBE_TIMEOUT_MS), + }); + expect(listing.status).toBe(200); + const parsed = parseModelsDevCatalog(await listing.json()); + // The parsed slices have already dropped their deprecated rows, so no + // `isDeprecated` predicate is passed: it could never exclude an id here. + const cells = keylessMatrix({ go: parsed.go, zen: parsed.zen }); + expect(cells).toHaveLength(16); + const rows: ProbeRow[] = []; + for (const cell of cells) { + rows.push(await probe(cell)); + } + console.table(rows); + // A model-less cell sends nothing. On the bundled catalogs every cell + // has one, so a live slice without one is a catalog change to triage — + // a reshaped listing would otherwise pass after sending no probe at all. + const unsent = rows.filter((row) => row.verdict === "no-model"); + expect(unsent, JSON.stringify(unsent)).toEqual([]); + // A 200 stays unreadable on purpose: it means a free model slipped past + // the matrix's filter and a completion was spent. + const unread = rows.filter( + (row) => + !["path-exists", "model-gone", "path-gone"].includes(row.verdict) + ); + expect(unread, JSON.stringify(unread)).toEqual([]); + const goneToday = rows.filter( + (row) => TODAY_DISPATCH.has(row.cell) && row.verdict === "path-gone" + ); + expect( + goneToday, + `a path today's plugin dispatches to is gone: ${JSON.stringify(goneToday)}` + ).toEqual([]); + const goneNext = rows.filter( + (row) => NEXT_DISPATCH.has(row.cell) && row.verdict === "path-gone" + ); + expect( + goneNext, + `forward guard: a path P-R1 moves completions to is gone: ${JSON.stringify(goneNext)}` + ).toEqual([]); + }, + 16 * 2 * (PROBE_TIMEOUT_MS + RETRY_AFTER_MS) + PROBE_TIMEOUT_MS + ); + } +); diff --git a/test/host-characterization.test.ts b/test/host-characterization.test.ts new file mode 100644 index 00000000..b878df58 --- /dev/null +++ b/test/host-characterization.test.ts @@ -0,0 +1,768 @@ +/** + * TODAY's plugin on the real host, characterized (tier B, pi 1.1.0). + * + * The plugin is mounted as it ships — its row carries no config, so the + * shipped defaults apply: the hidden ids in `providers`, `enrichModels` on, + * `endpoint: auto` — beside pi-ai holding the characterization profile (S0 + * plus a declared catalog-unknown id on each plane) with a key stored for each + * plane. Every row asserts what was MEASURED, and is tagged so a later phase + * flips named rows (docs/improvement-plan.md, "Phases"): + * + * - `[KNOWN_DEFECT]` is wrong today, and a phase fixes it; + * - `[INTENDED_CHANGE]` is right today, and a phase changes it on purpose; + * - `[BASELINE]` must survive every phase. + * + * The per-model expectations live in `S0_TODAY` (`support/host.ts`), which + * the binary e2e reads too, so the in-process host and the published binary + * cannot drift apart unnoticed. Each model's three facets (model info, the + * call-config verdict, the wire) carry their own tag there, and this file + * generates one test per model and facet titled with that tag, in each apply + * order (`todayRowsFor(order)` carries that order's tags), so the tag is what + * a reader and a phase's diff go by. Where a measurement contradicted the + * plan's expectation the row keeps the measurement and says so. + * + * Every case boots its own host, so a key the fetch patch captured in one case + * cannot change another's listing. The main block boots in the published + * host's apply order (`plugin-first`: the plugin applies before pi-ai + * registers, measured on the binary); a second block boots `pi-ai-first` and + * records the rows that change with the order — the adapter decoration only + * wraps what was registered when the plugin applied. + * + * @module test/host-characterization.test + */ +import { readFileSync } from "node:fs"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { + adaptersOf, + type ApplyOrder, + APPLY_ORDERS, + bootTierB, + CHARACTERIZED_MODELS, + characterizedProviders, + drainStream, + fakeCredentials, + PLANES, + piKnows, + providerWithApiKey, + recordingFetch, + type RecordingFetch, + type RowFacet, + S0_TODAY, + s0Providers, + SAVE_BLOCKED_BY_ANY_UNDESCRIBED, + SAVE_CANDIDATE, + type TierB, + todayRowsFor, + type TodayRow, + UNKNOWN_ID, + waitForRoutes, + ZEN_UNDESCRIBED, +} from "./support/host.ts"; +import { + INTERNAL_ROUTES, + OPENCODE_GO_CATALOG, + OPENCODE_ZEN_CATALOG, + PI_THINKING_LEVELS, + thinkingLevelMapFor, +} from "./support/src.ts"; + +type RowKey = keyof typeof S0_TODAY.rows; + +let tier: TierB | undefined; +let wire: RecordingFetch; + +/** Boot the characterization profile with a key stored for each plane, the plugin as it ships. */ +const bootS0 = async (order: ApplyOrder): Promise => { + wire = recordingFetch(); + vi.stubGlobal("fetch", wire.fetch); + tier = await bootTierB({ + credentials: fakeCredentials({ + refs: { + OPENCODE_API_KEY: "zen-test-key", + OPENCODE_GO_API_KEY: "go-test-key", + }, + }), + order, + providers: characterizedProviders(), + }); + // The plugin mounts its hidden routes asynchronously (mountInternalRoutes). + await waitForRoutes(tier.llm, INTERNAL_ROUTES); +}; + +afterEach(async () => { + await tier?.dispose(); + tier = undefined; + vi.unstubAllGlobals(); +}); + +const booted = (): TierB => { + if (tier === undefined) { + throw new Error("no host booted"); + } + return tier; +}; + +/** One turn's gateway URLs, the catalog read excluded. */ +const sentBy = async (route: string, model: string) => { + const before = wire.requests.length; + const drained = await drainStream(booted().llm, route, model); + return { + drained, + requests: wire.requests + .slice(before) + .filter( + (request) => !request.url.startsWith("https://models.opencode.ai/") + ), + }; +}; + +/** pi-ai's plane adapter, private methods included. */ +const planeAdapter = () => + adaptersOf(booted().llm).get("opencode")?.adapter as unknown as { + prepareCall: ( + route: string, + model: string + ) => Promise<{ baseUrl?: string; api?: string }>; + }; + +/** `resolveCallConfig`'s verdict: "ok", or the code it rejected with. */ +const callConfigOf = async (route: string, model: string): Promise => { + try { + await booted().llm.resolveCallConfig({ model, provider: route } as never); + return "ok"; + } catch (error) { + return String((error as { code?: unknown }).code); + } +}; + +const ROW_KEYS = Object.keys(S0_TODAY.rows) as RowKey[]; +const split = (key: RowKey): [string, string] => { + const [route = "", ...rest] = key.split("/"); + return [route, rest.join("/")]; +}; + +/** One facet of one row, measured on the booted host. */ +const measureFacet = async (facet: RowFacet, key: RowKey): Promise => { + const [route, model] = split(key); + if (facet === "info") { + try { + const info = await booted().llm.resolveModelInfo(route, model); + return { + context: info.context?.contextWindow, + efforts: info.reasoning?.efforts?.map((effort) => effort.id) ?? [], + }; + } catch (error) { + return { error: String((error as { code?: unknown }).code) }; + } + } + if (facet === "callConfig") { + return callConfigOf(route, model); + } + const { drained, requests } = await sentBy(route, model); + // Every body-addressed request carries the model it was for. + for (const request of requests) { + if (!request.url.includes(":streamGenerateContent")) { + expect(request.model).toBe(model); + } + } + return { + finish: drained.finish?.code, + sent: requests.map((request) => request.url), + }; +}; + +/** What `S0_TODAY` (or another order's table) records for one facet of one row. */ +const expectedFacet = ( + facet: RowFacet, + key: RowKey, + rows: Readonly> = S0_TODAY.rows +): unknown => { + const row = rows[key]; + if (facet === "info") { + return row.info; + } + if (facet === "callConfig") { + return row.callConfig; + } + return { finish: row.finish, sent: row.sent }; +}; + +/** A config-editor mutation appending `id` to a plane's models. */ +const appendTo = + (plane: string, id: string) => + (current: Record): Record => { + const providers = current.providers as Record< + string, + { models?: { id: string }[] } + >; + return { + ...current, + providers: { + ...providers, + [plane]: { + ...providers[plane], + models: [...(providers[plane]?.models ?? []), { id }], + }, + }, + }; + }; + +/** A config-editor mutation dropping `ids` from a plane's models. */ +const dropFrom = + (plane: string, ids: readonly string[]) => + (current: Record): Record => { + const providers = current.providers as Record< + string, + { models?: { id: string }[] } + >; + return { + ...current, + providers: { + ...providers, + [plane]: { + ...providers[plane], + models: (providers[plane]?.models ?? []).filter( + (model) => !ids.includes(model.id) + ), + }, + }, + }; + }; + +const FACETS = ["info", "callConfig", "wire"] as const; + +/** + * One generated case per row and facet, titled with the facet's own tag in + * that order's table, so a phase's diff names every facet it moves in both + * orders. + */ +const facetCasesFor = (order: ApplyOrder) => + ROW_KEYS.flatMap((key) => + FACETS.map((facet) => ({ + facet, + key, + tag: todayRowsFor(order)[key].tags[facet], + })) + ); + +/** The plane a row key belongs to, and the bundled catalog for it. */ +const catalogFor = (key: RowKey) => + key.startsWith("opencode-go/") ? OPENCODE_GO_CATALOG : OPENCODE_ZEN_CATALOG; + +/** The provider keys one side of a settings descriptor holds. */ +const providerKeys = (side: unknown): string[] => + Object.keys((side as { providers?: object } | undefined)?.providers ?? {}); + +/** + * The `[BASELINE]` session-controller reads after a Sign-in, on today's plugin + * with only the two plane keys stored (what `bootS0` stores): the Zen plane + * answers `hasProviderApiKey`, and pi-ai's `settings.describe` value lists + * both planes. P-R1 strips the planes from pi-ai's resolved config, which on + * the host alone turns this false (F20, the plugin-less contract in + * `test/host-editor-contracts.test.ts`); its D18 wrapper is what must keep + * this case green. + */ +const expectSignInReadsTheZenPlane = async (): Promise => { + const host = booted(); + const from = host.credentials.calls.length; + expect(await providerWithApiKey(host)).toBe("opencode"); + expect( + host.credentials.calls + .slice(from) + .filter(([method]) => method === "describe") + .map(([, ref]) => ref) + ).toContain("OPENCODE_API_KEY"); + const descriptor = host.ctx.settings + .describe() + .find((entry) => entry.ns === "llm-pi-ai"); + expect(providerKeys(descriptor?.value).toSorted()).toEqual([ + "deepseek", + ...PLANES, + ]); +}; + +describe( + "characterization · today's plugin on pi 1.1.0, the published host's order (tier B)", + { timeout: 30_000 }, + () => { + beforeEach(() => bootS0("plugin-first"), 30_000); + + it("fixture: the gap ids are gaps on pi 1.1.0, the unknown id is unknown everywhere, and the known ids are known", () => { + expect(piKnows("opencode", "gpt-5.3-codex-spark")).toBeUndefined(); + expect(piKnows("opencode-go", "gemini-4-argon")).toBeUndefined(); + expect(piKnows("opencode-go", "minimax-m3.1-flash")).toBeUndefined(); + expect(piKnows("opencode", "deepseek-v4-pro")).toBe("openai-completions"); + expect(piKnows("opencode", "gemini-3-flash")).toBe( + "google-generative-ai" + ); + expect(piKnows("opencode-go", "minimax-m3")).toBe("anthropic-messages"); + // The gap ids are ones the plugin's catalog still describes; the + // declared unknown id is in neither catalog (G1's two classes). + expect( + OPENCODE_ZEN_CATALOG.some((model) => model.id === "gpt-5.3-codex-spark") + ).toBe(true); + for (const plane of PLANES) { + expect(piKnows(plane, UNKNOWN_ID)).toBeUndefined(); + expect(CHARACTERIZED_MODELS[plane].at(-1)).toBe(UNKNOWN_ID); + } + for (const catalog of [OPENCODE_ZEN_CATALOG, OPENCODE_GO_CATALOG]) { + expect(catalog.some((model) => model.id === UNKNOWN_ID)).toBe(false); + expect(catalog.some((model) => model.id === SAVE_CANDIDATE)).toBe( + false + ); + } + expect(ROW_KEYS).toEqual( + PLANES.flatMap((plane) => + CHARACTERIZED_MODELS[plane].map((id) => `${plane}/${id}`) + ) + ); + expect(INTERNAL_ROUTES).toHaveLength(6); + }); + + it("fixture: each row's context and efforts are the bundled catalog's, so a catalog refresh that moves a fixture model fails here by name", () => { + // S0_TODAY copies context windows AND effort lists from the generated + // catalog, which the nightly refresh rewrites: decorateModelInfo makes + // the catalog authoritative for both (src/models-discovery.ts, the + // `thinkingLevelMapFor` → effortsFromLevelMap derivation, re-stated + // here from its two exported halves). A refresh that changes one of + // these ids must update S0_TODAY (and with it the binary e2e) — and must + // fail HERE, by name, not as a baseline facet that seems to regress. + // + // Two rows' `info` facets are KNOWN_DEFECT today and still copy the + // catalog (Zen gpt-5.3-codex-spark's context, Go minimax-m3's efforts); + // the plan moves them OFF the catalog on purpose (D17: the user's row + // wins; D1: a toggle model offers Off/On). The phase that flips such a + // facet adds its key to INFO_NOT_FROM_CATALOG in the same diff — and only + // a non-BASELINE `info` facet may be exempted, since a BASELINE one is + // meant to stay the catalog's across phases. + const INFO_NOT_FROM_CATALOG = new Set(); + for (const key of INFO_NOT_FROM_CATALOG) { + expect(ROW_KEYS, `${key} is not a characterized row`).toContain(key); + expect( + S0_TODAY.rows[key].tags.info, + `${key}: a BASELINE info facet cannot be exempted from the catalog` + ).not.toBe("BASELINE"); + } + let compared = 0; + for (const key of ROW_KEYS) { + if (INFO_NOT_FROM_CATALOG.has(key)) { + continue; + } + const [, id] = split(key); + const spec = catalogFor(key).find((model) => model.id === id); + if (spec === undefined) { + expect(id, key).toBe(UNKNOWN_ID); + continue; + } + const stale = + `S0_TODAY's info for ${key} no longer matches the bundled catalog: ` + + "either a catalog refresh moved it (update the row in " + + "test/support/host.ts), or a phase flipped this facet off the " + + "catalog (add the key to INFO_NOT_FROM_CATALOG here)"; + const info = S0_TODAY.rows[key].info as { + context?: number; + efforts?: readonly string[]; + }; + expect(spec.context_window, stale).toEqual(info.context); + if (info.efforts === undefined) { + continue; + } + const map = thinkingLevelMapFor(spec); + expect( + map === undefined + ? [] + : PI_THINKING_LEVELS.filter((level) => map[level] !== null), + stale + ).toEqual(info.efforts); + compared += 1; + } + // The case must have compared something: an `info` shape change that + // drops `efforts` would otherwise pass vacuously. + expect(compared).toBeGreaterThan(0); + }); + + it("fixture: the binary e2e's cell-aware helpers — the floating save-block pattern names any undescribed id and only those, todayRowsFor refuses an order it does not know, and a facet its pi-ai-first table moves is retagged with it, from KNOWN_DEFECT to BASELINE", () => { + // The pinned message is one the floating pattern accepts, so the two + // cells cannot disagree on a pinned run. + const pinned = + 'provider "opencode" model "gpt-5.3-codex-spark" needs an api'; + expect(pinned).toMatch(S0_TODAY.saveBlockedByStoredGap); + expect(pinned).toMatch(SAVE_BLOCKED_BY_ANY_UNDESCRIBED); + // A floating pi that learned gpt-5.3-codex-spark names the next one. + expect(`provider "opencode" model "${UNKNOWN_ID}" needs an api`).toMatch( + SAVE_BLOCKED_BY_ANY_UNDESCRIBED + ); + // A described id, or the dot read as a wildcard, is not a match. + for (const other of ["deepseek-v4-pro", "gpt-5x3-codex-spark"]) { + expect(`provider "opencode" model "${other}" needs an api`).not.toMatch( + SAVE_BLOCKED_BY_ANY_UNDESCRIBED + ); + } + for (const order of APPLY_ORDERS) { + expect(Object.keys(todayRowsFor(order))).toEqual(ROW_KEYS); + } + // Structural, so no phase has to edit it: a facet whose VALUE differs + // between the orders is exactly a facet whose TAG differs, and each one + // is a defect in the published order whose pi-ai-first value is already + // what P-R1 keeps. Which facets those are today is pinned by the + // [KNOWN_DEFECT] apply-order case below; once P-R1 collapses the + // override, both sets are empty and this still holds. + const piAiFirst = todayRowsFor("pi-ai-first"); + const moved = ROW_KEYS.flatMap((key) => + FACETS.filter( + (facet) => + JSON.stringify(expectedFacet(facet, key, piAiFirst)) !== + JSON.stringify(expectedFacet(facet, key)) + ).map((facet) => `${key} · ${facet}`) + ); + const retagged = ROW_KEYS.flatMap((key) => + FACETS.filter( + (facet) => + piAiFirst[key].tags[facet] !== S0_TODAY.rows[key].tags[facet] + ).map((facet) => `${key} · ${facet}`) + ); + expect(retagged).toEqual(moved); + for (const key of ROW_KEYS) { + for (const facet of FACETS) { + if (moved.includes(`${key} · ${facet}`)) { + expect(S0_TODAY.rows[key].tags[facet], `${key} · ${facet}`).toBe( + "KNOWN_DEFECT" + ); + expect(piAiFirst[key].tags[facet], `${key} · ${facet}`).toBe( + "BASELINE" + ); + } + } + } + // A probe JSON missing the field reads `undefined`; it must not select + // the pi-ai-first table. + expect(() => todayRowsFor(undefined as unknown as ApplyOrder)).toThrow( + /unknown apply order/u + ); + }); + + it("[BASELINE] hasProviderApiKey answers the Zen plane with only the plane keys stored, and llm-pi-ai's settings.describe value lists both planes (F20, D18)", async () => { + await expectSignInReadsTheZenPlane(); + }); + + it("[INTENDED_CHANGE] registry: both planes are pi-ai's, and six hidden routes are registered and hidden from listProviders (P-R1 claims the planes)", () => { + const { llm } = booted(); + const registry = adaptersOf(llm); + expect([...registry.keys()].toSorted()).toEqual( + ["deepseek", "opencode", "opencode-go", ...INTERNAL_ROUTES].toSorted() + ); + expect( + llm + .listProviders() + .map((provider) => provider.id) + .toSorted() + ).toEqual(["deepseek", "opencode", "opencode-go"]); + for (const route of [...PLANES, ...INTERNAL_ROUTES]) { + expect(registry.get(route)?.adapter.constructor.name).toBe( + "PiAiAdapter" + ); + } + // pi-ai's own adapter serves the planes; the hidden routes are not it. + const piAdapter = registry.get("opencode")?.adapter; + expect(registry.get("opencode-go")?.adapter).toBe(piAdapter); + for (const route of INTERNAL_ROUTES) { + expect(registry.get(route)?.adapter).not.toBe(piAdapter); + } + const directory = llm.listConfigurableProviders(); + for (const plane of PLANES) { + expect( + directory.find((entry) => entry.provider === plane) + ).toMatchObject({ + settingsNs: "llm-pi-ai", + settingsPath: ["providers", plane], + }); + } + // The directory reports each plane's first stored gap id. + expect( + directory.find((entry) => entry.provider === "opencode")?.error + ).toMatch(/model "gpt-5\.3-codex-spark" needs an api/u); + expect( + directory.find((entry) => entry.provider === "opencode-go")?.error + ).toMatch(/model "gemini-4-argon" needs an api/u); + }); + + it("[INTENDED_CHANGE] keyless listing: before a key is captured, the listing patch hides every paid model on both planes (D21)", async () => { + const { llm } = booted(); + for (const plane of PLANES) { + expect(await llm.listModels(plane)).toEqual( + S0_TODAY.keylessListing[plane] + ); + } + }); + + it("[KNOWN_DEFECT] keyed listing: once a turn captured a key, only pi-resolved ids are listed — every gap id and the declared unknown id are missing (G1)", async () => { + const { llm } = booted(); + await sentBy("opencode", "deepseek-v4-pro"); + await sentBy("opencode-go", "deepseek-v4-pro"); + const listed: Record = {}; + for (const plane of PLANES) { + const models = await llm.listModels(plane); + listed[plane] = models.map((model) => model.id); + } + expect(listed).toEqual(S0_TODAY.keyedListing); + expect(listed.opencode).not.toContain("gpt-5.3-codex-spark"); + expect(listed["opencode-go"]).not.toContain("gemini-4-argon"); + expect(listed["opencode-go"]).not.toContain("minimax-m3.1-flash"); + for (const plane of PLANES) { + expect(listed[plane]).not.toContain(UNKNOWN_ID); + } + }); + + it.each(facetCasesFor("plugin-first"))( + "[$tag] $key · $facet", + async ({ facet, key }) => { + expect(await measureFacet(facet, key)).toEqual( + expectedFacet(facet, key) + ); + } + ); + + it("[KNOWN_DEFECT] the toggle-only model is offered Minimal/Xhigh/Max (D1)", async () => { + // Chosen from the bundled catalog, not by name: on/off reasoning, no levels. + const spec = OPENCODE_GO_CATALOG.find( + (model) => model.id === "minimax-m3" + ); + expect(spec?.reasoning_options).toEqual([{ type: "toggle" }]); + const info = await booted().llm.resolveModelInfo( + "opencode-go", + "minimax-m3" + ); + const efforts = info.reasoning?.efforts?.map((effort) => effort.id) ?? []; + expect(efforts).toEqual( + S0_TODAY.rows["opencode-go/minimax-m3"].info.efforts + ); + expect(efforts).toEqual( + expect.arrayContaining(["minimal", "xhigh", "max"]) + ); + expect(efforts).not.toContain("off"); + }); + + it("[INTENDED_CHANGE] completions models dispatch to pi 1.1.0's legacy bases", async () => { + for (const [route, wireUrl] of [ + ["opencode", "https://opencode.ai/zen/v1/chat/completions"], + ["opencode-go", "https://opencode.ai/zen/go/v1/chat/completions"], + ] as const) { + const { drained, requests } = await sentBy(route, "deepseek-v4-pro"); + expect(requests.map((request) => request.url)).toEqual([wireUrl]); + expect(drained.finish).toEqual({ code: "AUTH", kind: "error" }); + } + }); + + it("[KNOWN_DEFECT] in this order the adapter decoration never runs: pi-ai's plane adapter is not wrapped, so its prepared call carries no repaired base (decoratePlaneAdapters in src/models-discovery.ts)", async () => { + // decoratePlaneAdapters wraps the adapters the registry holds when the + // plugin's effect runs, and here pi-ai has not registered yet. + const adapter = planeAdapter(); + expect(Object.hasOwn(adapter, "prepareCall")).toBe(false); + expect(Object.hasOwn(adapter, "resolveModel")).toBe(false); + const prepared = await adapter.prepareCall("opencode", "deepseek-v4-pro"); + expect(prepared.baseUrl).toBeUndefined(); + }); + + it("[KNOWN_DEFECT] Zen gpt-5.3-codex-spark: the user's row is ignored and resolveCallConfig refuses what dispatch serves through the hidden Responses route", async () => { + const { llm } = booted(); + const info = await llm.resolveModelInfo( + "opencode", + "gpt-5.3-codex-spark" + ); + // The row declares name "My Spark" and contextWindow 400000. + expect(info.context?.contextWindow).toBe(128_000); + expect(info.name).not.toBe("My Spark"); + await expect( + llm.resolveCallConfig({ + model: "gpt-5.3-codex-spark", + provider: "opencode", + } as never) + ).rejects.toMatchObject({ code: "INVALID_CONFIG" }); + const { requests } = await sentBy("opencode", "gpt-5.3-codex-spark"); + expect(requests.map((request) => request.url)).toEqual([ + "https://opencode.ai/inference/openai/v1/responses", + ]); + }); + + it("[KNOWN_DEFECT] Go minimax-m3.1-flash answers INVALID_CONFIG 'needs an api' and sends nothing (F4)", async () => { + const { llm } = booted(); + await expect( + llm.resolveCallConfig({ + model: "minimax-m3.1-flash", + provider: "opencode-go", + } as never) + ).rejects.toMatchObject({ + code: "INVALID_CONFIG", + message: expect.stringMatching(/"minimax-m3\.1-flash" needs an api/u), + }); + const { drained, requests } = await sentBy( + "opencode-go", + "minimax-m3.1-flash" + ); + expect(requests).toEqual([]); + expect(drained.finish).toEqual({ code: "INVALID_CONFIG", kind: "error" }); + }); + + it("[KNOWN_DEFECT] Go gemini-4-argon: resolveCallConfig refuses it (INVALID_CONFIG) while the turn is served on the Go Google base through the bridge", async () => { + expect(await callConfigOf("opencode-go", "gemini-4-argon")).toBe( + "INVALID_CONFIG" + ); + const { drained, requests } = await sentBy( + "opencode-go", + "gemini-4-argon" + ); + expect(requests.map((request) => request.url)).toEqual( + S0_TODAY.rows["opencode-go/gemini-4-argon"].sent + ); + expect(requests[0]?.headers["x-goog-api-key"]).toBe("go-test-key".length); + // pi maps a Google 401 to its own code, not AUTH (as with pi alone). + expect(drained.finish).toEqual({ code: "PI_AI_ERROR", kind: "error" }); + }); + + it("[KNOWN_DEFECT] a stored gap id blocks every save of its plane: appending a new id and a displayName change are both refused naming the STORED id, and the file is unchanged (F4; llm-pi-ai config.ts:364-368, catalog.ts:917-921)", async () => { + const { ctx, piEntry, profile } = booted(); + const before = readFileSync(profile.patchPath); + await expect( + ctx.configEditor.edit(piEntry, appendTo("opencode", SAVE_CANDIDATE)) + ).rejects.toThrow(S0_TODAY.saveBlockedByStoredGap); + await expect( + ctx.configEditor.edit(piEntry, (current) => { + const providers = current.providers as Record; + return { + ...current, + providers: { + ...providers, + opencode: { ...providers.opencode, displayName: "Zen (edited)" }, + }, + }; + }) + ).rejects.toThrow(S0_TODAY.saveBlockedByStoredGap); + expect(readFileSync(profile.patchPath)).toEqual(before); + }); + + it("[KNOWN_DEFECT] the probe's save sequence: dropping the plane's undescribed ids is accepted, then the catalog-unknown id itself is what the check names", async () => { + // The binary e2e's probe runs exactly this, so its rows isolate the + // unknown id although every plane of the profile stores a gap id. + const { ctx, piEntry, profile } = booted(); + await ctx.configEditor.edit( + piEntry, + dropFrom("opencode", ZEN_UNDESCRIBED) + ); + const cleared = readFileSync(profile.patchPath, "utf8"); + expect(cleared).not.toContain("gpt-5.3-codex-spark"); + expect(cleared).toContain("gemini-4-argon"); + await expect( + ctx.configEditor.edit(piEntry, appendTo("opencode", SAVE_CANDIDATE)) + ).rejects.toThrow(S0_TODAY.saveRejectedUnknown); + expect(readFileSync(profile.patchPath, "utf8")).toBe(cleared); + }); + } +); + +describe( + "characterization · pi-ai applying first (tier B)", + { timeout: 30_000 }, + () => { + beforeEach(() => bootS0("pi-ai-first"), 30_000); + + it("[BASELINE] hasProviderApiKey answers the Zen plane with only the plane keys stored, and llm-pi-ai's settings.describe value lists both planes (F20, D18)", async () => { + await expectSignInReadsTheZenPlane(); + }); + + it("[INTENDED_CHANGE] the adapter decoration runs, and the base it repairs never reaches the wire", async () => { + const adapter = planeAdapter(); + expect(Object.hasOwn(adapter, "prepareCall")).toBe(true); + for (const [route, wireUrl] of [ + ["opencode", "https://opencode.ai/zen/v1/chat/completions"], + ["opencode-go", "https://opencode.ai/zen/go/v1/chat/completions"], + ] as const) { + // Measured: the Go plane's repaired base is ZEN's. pi-ai serves both + // planes from ONE adapter instance, which decoratePlaneAdapters wraps + // once per route, so the Go wrapper runs over the Zen one and keeps the + // base it already filled. + const decorated = "https://opencode.ai/inference/openai/v1"; + const prepared = await adapter.prepareCall(route, "deepseek-v4-pro"); + expect(prepared).toMatchObject({ + api: "openai-completions", + baseUrl: decorated, + }); + const { drained, requests } = await sentBy(route, "deepseek-v4-pro"); + expect(requests.map((request) => request.url)).toEqual([wireUrl]); + // Measured, not compared as literals: the URL the turn sent does not + // start with the base the decorated adapter prepared. + expect(requests[0]?.url.startsWith(String(prepared.baseUrl))).toBe( + false + ); + expect(drained.finish).toEqual({ code: "AUTH", kind: "error" }); + } + }); + + // The plan expected INVALID_CONFIG for Go Gemini; that holds in the + // published host's order (the plugin-first block, where the case is + // [KNOWN_DEFECT]) and not in this one, where the wrapped plane adapter + // delegates to opencode-go-google (decoratePlaneAdapters in + // src/models-discovery.ts), so here it is [BASELINE]. todayRowsFor is the + // table the binary e2e picks when its probe infers this order; these cases + // measure every facet of it, and the apply-order case below pins that + // only that one facet moves. + it.each(facetCasesFor("pi-ai-first"))( + "[$tag] $key · $facet", + async ({ facet, key }) => { + expect(await measureFacet(facet, key)).toEqual( + expectedFacet(facet, key, todayRowsFor("pi-ai-first")) + ); + } + ); + } +); + +describe("characterization · the apply order (no host)", () => { + // Pure: it reads the two tables the binary e2e picks between, and boots + // nothing. Each order's generated facet cases measure its table; this case + // pins WHERE they differ today. The dependence on the order is the defect: + // when P-R1 flips this row it drops todayRowsFor's pi-ai-first override in + // the same diff, and this case becomes "[BASELINE] no facet depends on the + // apply order" (CONTRIBUTING.md, "Tags"). + it("[KNOWN_DEFECT] the apply order decides exactly one facet today: Go gemini-4-argon's callConfig is INVALID_CONFIG in the published order and \"ok\" with pi-ai first (P-R1 drops todayRowsFor's pi-ai-first override with this row's flip)", () => { + const piAiFirst = todayRowsFor("pi-ai-first"); + const moved = ROW_KEYS.flatMap((key) => + FACETS.filter( + (facet) => + JSON.stringify(expectedFacet(facet, key, piAiFirst)) !== + JSON.stringify(expectedFacet(facet, key)) + ).map((facet) => `${key} · ${facet}`) + ); + expect(moved).toEqual(["opencode-go/gemini-4-argon · callConfig"]); + const key = "opencode-go/gemini-4-argon"; + expect(expectedFacet("callConfig", key)).toBe("INVALID_CONFIG"); + expect(expectedFacet("callConfig", key, piAiFirst)).toBe("ok"); + }); +}); + +describe( + "characterization · a gap-free plane (tier B)", + { timeout: 30_000 }, + () => { + it("[KNOWN_DEFECT] the unknown id itself is what the save check names when the plane stores no other gap", async () => { + // Plain S0 (no declared unknown id) minus Zen's stored gap id. + const providers = s0Providers(); + const zen = providers.opencode as { models: { id: string }[] }; + zen.models = zen.models.filter( + (model) => model.id !== "gpt-5.3-codex-spark" + ); + // The plugin's row mounts too, so its fetch patch wraps the recording + // fetch, as in every other case here, and not the suite's offline guard. + wire = recordingFetch(); + vi.stubGlobal("fetch", wire.fetch); + tier = await bootTierB({ providers }); + const { ctx, piEntry, profile } = tier; + const before = readFileSync(profile.patchPath); + await expect( + ctx.configEditor.edit(piEntry, appendTo("opencode", SAVE_CANDIDATE)) + ).rejects.toThrow(S0_TODAY.saveRejectedUnknown); + expect(readFileSync(profile.patchPath)).toEqual(before); + }); + } +); diff --git a/test/host-credentials.test.ts b/test/host-credentials.test.ts new file mode 100644 index 00000000..504b8ab4 --- /dev/null +++ b/test/host-credentials.test.ts @@ -0,0 +1,257 @@ +/** + * pi-ai's credential paths, measured on the real package (tier A). + * + * D20 says the plane adapter resolves keys "by pi-ai's rule", and P-R1 builds + * its credential store from pi-ai's record mapping. Both are only as true as + * what pi-ai does today, so these cases drive a real turn through + * `LlmRuntime` and read the wire: which key the credentials service was asked + * for, whether the launch environment was consulted, and what authorization + * reached the request. Header values are recorded by LENGTH only, so no key + * text lands in an assertion or a log. + * + * Host references are into `deepseek-harness`'s `packages/llm/llm-pi-ai/src`. + * + * @module test/host-credentials.test + */ +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { + adaptersOf, + drainStream, + fakeCredentials, + fakeLaunchEnvironment, + mountTierA, + recordingFetch, + type TierA, +} from "./support/host.ts"; + +let tier: TierA | undefined; + +afterEach(async () => { + await tier?.dispose(); + tier = undefined; + vi.unstubAllGlobals(); +}); + +/** What pi's `Models.getAuth` answers, as far as these cases read it. */ +interface PiAuthResult { + readonly env?: Readonly>; + readonly source?: string; +} + +/** + * pi's own auth answer for a route, through the collection pi-ai's adapter + * built for its current snapshot (adapter.ts:251-259, private) and pi's public + * `Models.getAuth` (pi 1.1.0 `dist/models.d.ts`). It is the one place a stored + * record's `env` is observable on a plane: pi forwards it as the result's + * `env` (`dist/auth/helpers.js`), and no plane request reads it back. + * + * @param current - the tier whose pi-ai adapter to read. + * @param route - the route to resolve auth for. + * @returns the source and env, with the key left out. + */ +const piAuthOf = async ( + current: TierA, + route: string +): Promise => { + const adapter = adaptersOf(current.llm).get(route)?.adapter as unknown as { + current: () => { + models: { getAuth: (id: string) => Promise }; + }; + }; + const result = await adapter.current().models.getAuth(route); + return result === undefined + ? undefined + : { env: result.env, source: result.source }; +}; + +/** One completions model on Zen, which pi 1.1.0 serves at its legacy base. */ +const ZEN_COMPLETIONS = { models: [{ id: "deepseek-v4-pro" }] }; +const LEGACY_COMPLETIONS = "https://opencode.ai/zen/v1/chat/completions"; + +describe("host · pi-ai's credential paths (tier A)", () => { + it("maps an api-key record under llm-pi-ai/opencode onto the wire with no apiKeyEnv (auth.ts:74-84,141-148)", async () => { + const wire = recordingFetch(); + vi.stubGlobal("fetch", wire.fetch); + const credentials = fakeCredentials({ + records: { "llm-pi-ai/opencode": { key: "stored-k", kind: "api-key" } }, + }); + tier = await mountTierA({ + credentials, + providers: { opencode: ZEN_COMPLETIONS }, + }); + const drained = await drainStream(tier.llm, "opencode", "deepseek-v4-pro"); + expect(wire.requests.map((request) => request.url)).toEqual([ + LEGACY_COMPLETIONS, + ]); + expect(wire.requests[0]?.headers.authorization).toBe( + "Bearer stored-k".length + ); + expect(credentials.calls).toContainEqual([ + "readRecord", + "llm-pi-ai/opencode", + ]); + expect(drained.finish).toEqual({ code: "AUTH", kind: "error" }); + // A record with no env hands pi none (auth.ts:80 spreads it only when set). + expect(await piAuthOf(tier, "opencode")).toEqual({ + env: undefined, + source: "stored credential", + }); + }); + + it("maps an api-key record carrying env as well: pi's auth carries the stored env, the turn still authenticates with the stored key, and the launch environment is not read (auth.ts:74-84)", async () => { + const wire = recordingFetch(); + vi.stubGlobal("fetch", wire.fetch); + const credentials = fakeCredentials({ + records: { + "llm-pi-ai/opencode": { + env: { OPENCODE_REGION: "test" }, + key: "stored-k", + kind: "api-key", + }, + }, + }); + const launchEnvironment = fakeLaunchEnvironment({ + OPENCODE_API_KEY: "env-key-k", + }); + tier = await mountTierA({ + credentials, + launchEnvironment, + providers: { opencode: ZEN_COMPLETIONS }, + }); + const drained = await drainStream(tier.llm, "opencode", "deepseek-v4-pro"); + expect(wire.requests.map((request) => request.url)).toEqual([ + LEGACY_COMPLETIONS, + ]); + // The stored key's length, not the environment's (a different length). + expect(wire.requests[0]?.headers.authorization).toBe( + "Bearer stored-k".length + ); + expect(launchEnvironment.reads).toEqual([]); + expect(drained.finish).toEqual({ code: "AUTH", kind: "error" }); + // The env branch itself (auth.ts:80): nothing on a plane's wire reads the + // stored env, so it is asserted where pi hands it on, in its auth answer. + expect(await piAuthOf(tier, "opencode")).toEqual({ + env: { OPENCODE_REGION: "test" }, + source: "stored credential", + }); + }); + + it("hands a non-api-key record's payload to pi verbatim: on a plane, whose pi auth has no OAuth, the turn fails PI_AI_ERROR with no request and no fallback to the launch environment (auth.ts:74-84)", async () => { + // Not a record a plane's Sign-in writes (pi 1.1.0's planes offer an API + // key only); pinned so P-R1's store, which copies this mapping, answers + // the same for one a user or another plane left behind. + const wire = recordingFetch(); + vi.stubGlobal("fetch", wire.fetch); + const credentials = fakeCredentials({ + records: { + "llm-pi-ai/opencode": { + kind: "grant", + payload: { + access: "grant-access", + expires: Date.now() + 3_600_000, + refresh: "grant-refresh", + type: "oauth", + }, + }, + }, + }); + const launchEnvironment = fakeLaunchEnvironment({ + OPENCODE_API_KEY: "env-k", + }); + tier = await mountTierA({ + credentials, + launchEnvironment, + providers: { opencode: ZEN_COMPLETIONS }, + }); + const drained = await drainStream(tier.llm, "opencode", "deepseek-v4-pro"); + expect(credentials.calls).toContainEqual([ + "readRecord", + "llm-pi-ai/opencode", + ]); + expect(wire.requests).toEqual([]); + expect(launchEnvironment.reads).toEqual([]); + expect(drained.finish).toEqual({ code: "PI_AI_ERROR", kind: "error" }); + }); + + it("F18 — a named ref the credentials service misses is MISSING_CREDENTIAL with no request, and listModels answers [] (index.ts:188-211; adapter.ts:295-312)", async () => { + const wire = recordingFetch(); + vi.stubGlobal("fetch", wire.fetch); + const credentials = fakeCredentials(); + const launchEnvironment = fakeLaunchEnvironment({ + OPENCODE_API_KEY: "env-k", + }); + tier = await mountTierA({ + credentials, + launchEnvironment, + providers: { + opencode: { ...ZEN_COMPLETIONS, apiKeyEnv: "OPENCODE_API_KEY" }, + }, + }); + expect(await tier.llm.listModels("opencode")).toEqual([]); + const drained = await drainStream(tier.llm, "opencode", "deepseek-v4-pro"); + expect(drained.finish).toEqual({ + code: "MISSING_CREDENTIAL", + kind: "error", + }); + expect(wire.requests).toEqual([]); + expect(credentials.calls).toContainEqual(["resolve", "OPENCODE_API_KEY"]); + // With a credentials service mounted, the launch environment is never the + // fallback for a NAMED ref (index.ts:199-203). + expect(launchEnvironment.reads).not.toContain("OPENCODE_API_KEY"); + }); + + it("resolves a named ref through the credentials service and sends it (index.ts:199-205)", async () => { + const wire = recordingFetch(); + vi.stubGlobal("fetch", wire.fetch); + tier = await mountTierA({ + credentials: fakeCredentials({ refs: { OPENCODE_API_KEY: "ref-k" } }), + providers: { + opencode: { ...ZEN_COMPLETIONS, apiKeyEnv: "OPENCODE_API_KEY" }, + }, + }); + const listed = await tier.llm.listModels("opencode"); + expect(listed.map((model) => model.id)).toEqual(["deepseek-v4-pro"]); + await drainStream(tier.llm, "opencode", "deepseek-v4-pro"); + expect(wire.requests.map((request) => request.url)).toEqual([ + LEGACY_COMPLETIONS, + ]); + expect(wire.requests[0]?.headers.authorization).toBe("Bearer ref-k".length); + }); + + it("with no ref, pi's ambient auth asks the credentials service first, then the launch environment (auth.ts:203-215)", async () => { + const wire = recordingFetch(); + vi.stubGlobal("fetch", wire.fetch); + // One log for both fakes, so the service calls and the environment read + // are ordered on the same clock. + const log: string[] = []; + const credentials = fakeCredentials({ sharedLog: log }); + const launchEnvironment = fakeLaunchEnvironment( + { OPENCODE_API_KEY: "env-k" }, + log + ); + tier = await mountTierA({ + credentials, + launchEnvironment, + providers: { opencode: ZEN_COMPLETIONS }, + }); + await drainStream(tier.llm, "opencode", "deepseek-v4-pro"); + expect(wire.requests.map((request) => request.url)).toEqual([ + LEGACY_COMPLETIONS, + ]); + expect(wire.requests[0]?.headers.authorization).toBe("Bearer env-k".length); + // The stored record is read first, then the reference, then the + // environment: the credentials service is asked before the snapshot. + const at = (entry: string): number => { + const index = log.indexOf(entry); + expect(index, `${entry} in ${log.join(", ")}`).toBeGreaterThan(-1); + return index; + }; + expect(at("readRecord:llm-pi-ai/opencode")).toBeLessThan( + at("resolve:OPENCODE_API_KEY") + ); + expect(at("resolve:OPENCODE_API_KEY")).toBeLessThan( + at("env:OPENCODE_API_KEY") + ); + }); +}); diff --git a/test/host-editor-contracts.test.ts b/test/host-editor-contracts.test.ts new file mode 100644 index 00000000..2f2698f5 --- /dev/null +++ b/test/host-editor-contracts.test.ts @@ -0,0 +1,460 @@ +/** + * The config editor, the loader's volatile commit and the settings forms, + * pinned on the real host (tier B). + * + * The route-claim design keeps the user's file whole while pi-ai's resolved + * config loses the two planes (docs/improvement-plan.md, F12-F16, F20). That + * only works if the editor writes the raw value it was handed, if a commit can + * be told from the editor's validation run by identity, if a plane-only edit + * re-registers nothing, and if the settings page reads what these cases say + * it reads. Each case boots a fresh host and names the host file and line it + * pins; references are into `deepseek-harness`'s `packages/` and `vendor/`. + * + * These are host contracts, so the plugin's own row is left out (its hidden + * mount would add registry events of its own) except where the plugin's row is + * the subject (F16). + * + * @module test/host-editor-contracts.test + */ +import { readFileSync } from "node:fs"; + +import type { Context, Fiber } from "@deepseek-ai/cordis"; +import { readProfilePatches } from "@deepseek-ai/dsh-app-boot"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { + bootTierB, + fakeCredentials, + planeKeysOnly, + providerWithApiKey, + recordingFetch, + s0Providers, + type TierB, + waterfallSpy, + withoutPlanes, +} from "./support/host.ts"; + +/** The listener the claim design installs, prepended on the root. */ +function stripPlanesForPiAi( + this: Fiber, + raw: unknown, + next: () => unknown +): unknown { + if (this.runtime?.name !== "llm-pi-ai") { + return next(); + } + const copy = withoutPlanes(raw); + next(); + return copy; +} + +const prependStrip = (root: Context): void => { + root.on("internal/config", stripPlanesForPiAi, { prepend: true }); +}; + +/** S0 without the ids pi 1.1.0 cannot place, so pi-ai's save check passes. */ +const knownProviders = (): Record> => { + const providers = s0Providers(); + const gaps = new Set([ + "gpt-5.3-codex-spark", + "gemini-4-argon", + "minimax-m3.1-flash", + ]); + for (const plane of ["opencode", "opencode-go"]) { + const entry = providers[plane] as { models: { id: string }[] }; + entry.models = entry.models.filter((model) => !gaps.has(model.id)); + } + return providers; +}; + +type ProviderDict = Record>; + +/** An edit of one provider entry, the way a settings surface derives one. */ +const editProvider = + (route: string, patch: Record) => + (current: Record): Record => { + const providers = current.providers as ProviderDict; + return { + ...current, + providers: { ...providers, [route]: { ...providers[route], ...patch } }, + }; + }; + +/** The last pi-ai row with a config in the profile, read through the host. */ +const savedPiRow = (tier: TierB): { providers: ProviderDict } => { + const row = readProfilePatches("dsh", tier.profile).findLast( + (patch) => patch.id === "llm-pi-ai" && patch.config !== undefined + ); + return row?.config as { providers: ProviderDict }; +}; + +const resolvedKeys = (fiber: Fiber | undefined): string[] => { + const config = fiber?.config as + | { providers: { get: () => Record } } + | undefined; + return Object.keys(config?.providers.get() ?? {}); +}; + +/** The provider keys one side of a settings descriptor holds. */ +const providerKeys = (side: unknown): string[] => + Object.keys((side as { providers?: object } | undefined)?.providers ?? {}); + +/** The refs `credentials.describe` was asked about from `from` on. */ +const describedRefs = (tier: TierB, from: number): string[] => + tier.credentials.calls + .slice(from) + .filter(([method]) => method === "describe") + .map(([, ref]) => ref); + +const routeIds = (tier: TierB): string[] => + tier.llm.listProviders().map((provider) => provider.id); + +let tier: TierB | undefined; + +afterEach(async () => { + await tier?.dispose(); + tier = undefined; + vi.unstubAllGlobals(); +}); + +describe( + "host · the config editor and the volatile commit (tier B)", + { + timeout: 30_000, + }, + () => { + it("writes the raw next to the profile YAML, not the resolved value (config-editor/src/index.ts:100-128,142)", async () => { + tier = await bootTierB({ + plugin: false, + prepare: prependStrip, + providers: knownProviders(), + }); + expect(routeIds(tier)).toEqual(["deepseek"]); + await tier.ctx.configEditor.edit( + tier.piEntry, + editProvider("opencode", { displayName: "Zen (edited)" }) + ); + const saved = savedPiRow(tier); + expect(saved.providers.opencode?.displayName).toBe("Zen (edited)"); + expect(Object.keys(saved.providers).toSorted()).toEqual([ + "deepseek", + "opencode", + "opencode-go", + ]); + // The editor's own read of the file agrees. + const override = tier.ctx.configEditor + .configuration() + .find((row) => row.entry === tier?.piEntry)?.override as { + providers: ProviderDict; + }; + expect(override.providers.opencode?.displayName).toBe("Zen (edited)"); + // The fiber keeps the raw it was handed, and resolves the stripped copy. + expect(resolvedKeys(tier.piFiber)).toEqual(["deepseek"]); + const raw = ( + tier.piFiber as unknown as { _config: { providers: ProviderDict } } + )._config; + expect(raw.providers.opencode?.displayName).toBe("Zen (edited)"); + }); + + it("F14 — the editor's candidate is never fiber._config; _commitVolatile's run is (config-editor:103; loader entry.ts:149,171)", async () => { + tier = await bootTierB({ + plugin: false, + prepare: prependStrip, + providers: knownProviders(), + }); + const booted = tier; + const spy = waterfallSpy(booted.ctx, () => booted.piEntry.fiber); + await booted.ctx.configEditor.edit( + booted.piEntry, + editProvider("opencode", { displayName: "Zen (edited)" }) + ); + expect(spy.records.map((record) => record.isCommit)).toEqual([ + false, + true, + ]); + const [, commit] = spy.records; + expect(commit?.raw).toBe(booted.piEntry.options.config); + expect(commit?.raw).toBe( + (booted.piFiber as unknown as { _config: unknown })._config + ); + expect(spy.records[0]?.raw).not.toBe(commit?.raw); + }); + + it("F15 — a plane-only edit emits no loader/volatile-update and restarts nothing; a sibling's edit emits one (loader entry.ts:177-191)", async () => { + tier = await bootTierB({ + plugin: false, + prepare: prependStrip, + providers: knownProviders(), + }); + const volatile: unknown[] = []; + let adaptersUpdated = 0; + tier.ctx.on( + "loader/volatile-update", + (paths: unknown) => { + volatile.push(paths); + }, + { global: true } + ); + tier.ctx.on( + "llm/adapters-updated", + () => { + adaptersUpdated += 1; + }, + { global: true } + ); + const before = tier.piEntry.fiber; + await tier.ctx.configEditor.edit( + tier.piEntry, + editProvider("opencode", { displayName: "Zen (edited)" }) + ); + expect(volatile).toEqual([]); + expect(adaptersUpdated).toBe(0); + expect(tier.piEntry.fiber).toBe(before); + expect(tier.piFiber.state).toBe(2); + expect(routeIds(tier)).toEqual(["deepseek"]); + + // Control: an edit pi-ai DOES see commits one volatile update, and pi-ai + // re-registers in place — the route replace and the directory replace + // each announce (llm/src/index.ts:475,536) — without a restart. + await tier.ctx.configEditor.edit( + tier.piEntry, + editProvider("deepseek", { displayName: "DS (edited)" }) + ); + expect(volatile).toEqual([[["providers"]]]); + expect(adaptersUpdated).toBe(2); + expect(tier.piEntry.fiber).toBe(before); + expect(tier.llm.listProviders()).toEqual([ + { id: "deepseek", name: "DS (edited)" }, + ]); + }); + + it("F4 — the stripping listener does not shield pi-ai's own save check: with a gap id stored, a plane edit is refused and the file is untouched (llm-pi-ai/src/index.ts:177-186; config.ts:364-368)", async () => { + // Read for P-R1. An outermost listener's copy is what the fiber KEEPS, + // but pi-ai validates the value its inner chain returns — the editor's + // raw candidate — and against its stripped previous config every plane + // entry reads as changed, so the strict check runs on it. + tier = await bootTierB({ + plugin: false, + prepare: prependStrip, + providers: s0Providers(), + }); + const booted = tier; + const spy = waterfallSpy(booted.ctx, () => booted.piEntry.fiber); + const before = readFileSync(booted.profile.patchPath); + await expect( + booted.ctx.configEditor.edit( + booted.piEntry, + editProvider("opencode", { displayName: "Zen (edited)" }) + ) + ).rejects.toThrow( + /provider "opencode" model "gpt-5\.3-codex-spark" needs an api/u + ); + expect(spy.records.map((record) => record.isCommit)).toEqual([false]); + expect(readFileSync(booted.profile.patchPath)).toEqual(before); + }); + } +); + +describe("host · settings.describe (tier B)", { timeout: 30_000 }, () => { + it("host contract, F20 — builds value from the resolved fiber config while user keeps the YAML's entries (settings/src/index.ts:319-322)", async () => { + tier = await bootTierB({ + plugin: false, + prepare: prependStrip, + providers: s0Providers(), + }); + const stripped = tier.ctx.settings + .describe() + .find((descriptor) => descriptor.ns === "llm-pi-ai"); + expect(providerKeys(stripped?.value)).toEqual(["deepseek"]); + expect(providerKeys(stripped?.user).toSorted()).toEqual([ + "deepseek", + "opencode", + "opencode-go", + ]); + expect(stripped?.applies).toBe("live"); + }); + + it("host contract, F20 control — with no listener, value carries every entry the YAML holds", async () => { + tier = await bootTierB({ plugin: false, providers: s0Providers() }); + const plain = tier.ctx.settings + .describe() + .find((descriptor) => descriptor.ns === "llm-pi-ai"); + expect(providerKeys(plain?.value).toSorted()).toEqual([ + "deepseek", + "opencode", + "opencode-go", + ]); + }); +}); + +describe( + "host · session-controller's hasProviderApiKey, restated (tier B, no plugin row)", + { timeout: 30_000 }, + () => { + // Host contracts, measured without the plugin, so nothing in src/ can move + // them. The with-plugin row P-R1 must keep green is the [BASELINE] case in + // test/host-characterization.test.ts. + it("host contract — with only the plane keys stored, the Zen plane answers it: its describe().value carries apiKeyEnv and the key is configured", async () => { + tier = await bootTierB({ + credentials: planeKeysOnly(), + plugin: false, + providers: s0Providers(), + }); + const from = tier.credentials.calls.length; + expect(await providerWithApiKey(tier)).toBe("opencode"); + expect(describedRefs(tier, from)).toContain("OPENCODE_API_KEY"); + }); + + it("host contract, control — with no key stored it answers false, and with only DEEPSEEK_API_KEY the sibling answers it", async () => { + tier = await bootTierB({ plugin: false, providers: s0Providers() }); + expect(await providerWithApiKey(tier)).toBeUndefined(); + await tier.dispose(); + tier = await bootTierB({ + credentials: fakeCredentials({ refs: { DEEPSEEK_API_KEY: "ds" } }), + plugin: false, + providers: s0Providers(), + }); + expect(await providerWithApiKey(tier)).toBe("deepseek"); + }); + + it("restatement — an apiKeyEnv outside credentialRef's grammar throws upstream's TypeError instead of reading 'not configured' (credentials/src/index.ts:29-34)", async () => { + // pi-ai's resolveProfiles refuses such a ref today, so only a describe + // graft (P-R1's) could carry one; the restatement must then fail the + // way upstream's initializeDefaultModel would, not skip the provider. + tier = await bootTierB({ + credentials: planeKeysOnly(), + plugin: false, + providers: s0Providers(), + }); + const { settings } = tier.ctx; + const malformedDescribe: typeof settings.describe = (options) => + settings.describe(options).map((namespace) => { + if (namespace.ns !== "llm-pi-ai") { + return namespace; + } + const value = structuredClone(namespace.value) as { + providers: { opencode: Record }; + }; + value.providers.opencode.apiKeyEnv = "OPENCODE API KEY"; + return { ...namespace, value }; + }); + const view = { + credentials: tier.credentials, + ctx: { settings: { describe: malformedDescribe } }, + llm: tier.llm, + } as unknown as TierB; + const from = tier.credentials.calls.length; + await expect(providerWithApiKey(view)).rejects.toThrow( + new TypeError( + 'credential ref "OPENCODE API KEY" must match /^[A-Za-z_][A-Za-z0-9_]*$/' + ) + ); + // The malformed ref is never handed to the credentials service. + expect(describedRefs(tier, from)).toEqual(["DEEPSEEK_API_KEY"]); + // Control: the unmodified view answers the plane, so the throw is the ref's. + expect(await providerWithApiKey(tier)).toBe("opencode"); + }); + + it("host contract, F20 — under a test-local strip it answers FALSE with the plane keys stored: the directory still lists the planes, but their describe().value is gone, so their keys are never asked about", async () => { + // Why P-R1 needs the settings.describe wrapper (D18): upstream, false + // here makes initializeDefaultModel replace the user's default with the + // account's first model. The strip is this file's own listener, not the + // plugin, so this case cannot flip; what P-R1 must keep green while its + // own strip is active is the with-plugin [BASELINE] in + // test/host-characterization.test.ts. + tier = await bootTierB({ + credentials: planeKeysOnly(), + plugin: false, + prepare: prependStrip, + providers: s0Providers(), + }); + for (const plane of ["opencode", "opencode-go"]) { + expect(tier.llm.listConfigurableProviders()).toContainEqual( + expect.objectContaining({ + provider: plane, + settingsNs: "llm-pi-ai", + settingsPath: ["providers", plane], + }) + ); + } + const from = tier.credentials.calls.length; + expect(await providerWithApiKey(tier)).toBeUndefined(); + expect(describedRefs(tier, from)).toEqual(["DEEPSEEK_API_KEY"]); + }); + } +); + +describe( + "host · !!js and the outermost listener (tier B)", + { timeout: 30_000 }, + () => { + it("F16 — the outermost listener sees an un-interpolated {__jsExpr} node; the fiber's config holds the evaluated value (loader/src/index.ts:104-113)", async () => { + // The expression evaluates to the NON-default value (`debug` defaults + // to false), so a value that fell back to the default cannot pass. + const wire = recordingFetch(); + vi.stubGlobal("fetch", wire.fetch); + const seen: unknown[] = []; + tier = await bootTierB({ + prepare: (root) => { + root.on( + "internal/config", + function recordPluginRaw( + this: Fiber, + raw: unknown, + next: () => unknown + ) { + if (this.runtime?.name === "dsh-opencode-patch") { + seen.push(raw); + } + return next(); + }, + { prepend: true } + ); + }, + profilePatchYaml: [ + "- id: dsh-opencode-patch", + " config:", + ' debug: !!js "1 < 2"', + "", + ].join("\n"), + }); + expect(seen.length).toBeGreaterThan(0); + expect((seen[0] as { debug: unknown }).debug).toEqual({ + __jsExpr: "1 < 2", + }); + const pluginConfig = tier.pluginEntry?.fiber?.config as + | { debug: unknown } + | undefined; + expect(pluginConfig?.debug).toBe(true); + }); + + it("F16 on pi-ai's row — a !!js node inside a provider profile boots evaluated, but pi-ai's own check refuses every later save of the row (llm-pi-ai/src/index.ts:177-180)", async () => { + // Read for P-R1: pi-ai's listener runs `Config(raw)` on the value its + // inner chain returns, which the loader has not interpolated yet, so a + // `!!js` node inside `providers` survives boot and blocks the editor. + tier = await bootTierB({ + plugin: false, + profilePatchYaml: [ + "- id: llm-pi-ai", + " config:", + " providers:", + " deepseek:", + " apiKeyEnv: DEEPSEEK_API_KEY", + ' displayName: !!js \'"D" + "S"\'', + "", + ].join("\n"), + }); + expect(tier.llm.listProviders()).toEqual([ + { id: "deepseek", name: "DS" }, + ]); + await expect( + tier.ctx.configEditor.edit( + tier.piEntry, + editProvider("deepseek", { baseURL: "http://127.0.0.1:9/v1" }) + ) + ).rejects.toThrow( + "$.providers.deepseek.displayName expected string but got [object Object]" + ); + }); + } +); diff --git a/test/host-harness.test.ts b/test/host-harness.test.ts new file mode 100644 index 00000000..4d122a8a --- /dev/null +++ b/test/host-harness.test.ts @@ -0,0 +1,239 @@ +/** + * The harness's own seams, each exercised once (`test/support/host.ts`). + * + * P-R1 builds on options no contract or characterization case passes: this + * plugin mounted in tier A, a profile row for it in tier B, and a custom + * responder on the recording fetch. A builder nobody has run is a builder + * whose behaviour a later phase would assume, so each is run here and what it + * does is recorded — including that tier A's plugin is a DEGRADED one. So are + * tier B's two failure paths: a first row whose config the schema rejects (in + * each apply order), and a key the contributor's shell exports. + * + * @module test/host-harness.test + */ +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { + adaptersOf, + type ApplyOrder, + bootTierB, + characterizedProviders, + drainStream, + fakeCredentials, + mountTierA, + PLANES, + recordingFetch, + S0_TODAY, + s0Providers, + type TierA, + type TierB, + waitForRoutes, +} from "./support/host.ts"; +import { INTERNAL_ROUTES, loadPiAiAdapterHost } from "./support/src.ts"; + +let tierA: TierA | undefined; +let tierB: TierB | undefined; + +afterEach(async () => { + try { + await tierA?.dispose(); + await tierB?.dispose(); + } finally { + tierA = undefined; + tierB = undefined; + vi.restoreAllMocks(); + vi.unstubAllGlobals(); + vi.unstubAllEnvs(); + } +}); + +/** cordis `FiberState`: the two a booted row ends in here. */ +const ACTIVE = 2; +const FAILED = 3; + +/** Poll until `predicate` holds, or fail after `deadlineMs`. */ +const until = async ( + predicate: () => boolean, + what: string, + deadlineMs = 10_000 +): Promise => { + const end = Date.now() + deadlineMs; + while (!predicate()) { + if (Date.now() > end) { + throw new Error(`never happened: ${what}`); + } + await new Promise((resolve) => { + setTimeout(resolve, 10); + }); + } +}; + +/** A two-chunk chat-completions stream that ends with `stop`. */ +const completionStream = (model: string): Response => { + const chunk = (delta: object, finish: string | null) => + `data: ${JSON.stringify({ + choices: [{ delta, finish_reason: finish, index: 0 }], + created: 1, + id: "harness", + model, + object: "chat.completion.chunk", + })}\n\n`; + return new Response( + `${chunk({ content: "hello", role: "assistant" }, null)}${chunk({}, "stop")}data: [DONE]\n\n`, + { headers: { "content-type": "text/event-stream" }, status: 200 } + ); +}; + +describe( + "host harness · its seams (tiers A and B)", + { timeout: 30_000 }, + () => { + it("tier A with the plugin: it applies and patches fetch, but WITHOUT a loader it cannot find pi-ai, so no hidden route mounts; disposal restores fetch", async () => { + const info = vi.spyOn(console, "info").mockImplementation(() => { + // Silenced: the case reads the calls, not the console. + }); + const wire = recordingFetch(); + vi.stubGlobal("fetch", wire.fetch); + tierA = await mountTierA({ + plugin: {}, + providers: { opencode: { models: [{ id: "deepseek-v4-pro" }] } }, + }); + expect(tierA.pluginFiber?.state).toBe(2); + expect(globalThis.fetch).not.toBe(wire.fetch); + // The plugin finds pi-ai through the loader's entries + // (`piAiEntry` in src/responses-provider.ts), and tier A has no loader: the mount + // reports that and stands down. A P-R1 case that needs the real plugin + // beside pi-ai belongs in tier B. + await until( + () => + info.mock.calls.some((call) => + String(call[0]).includes("no loaded llm-pi-ai in this composition") + ), + "the mount's stand-down line" + ); + const registry = adaptersOf(tierA.llm); + for (const route of INTERNAL_ROUTES) { + expect(registry.has(route), route).toBe(false); + } + expect([...registry.keys()]).toEqual(["opencode"]); + // The Google bridge's absence is pinned by its CAUSE, not its outcome: + // this case declares no Google model, so an empty registry above would + // hold for that reason alone. The bridge builds its adapter from the + // record `loadPiAiAdapterHost` answers (src/responses-provider.ts), and + // registerGoogleBridge stands down without its `PiAiAdapter` + // (src/google-bridge.ts) — and without a loader there is no record. + await expect(loadPiAiAdapterHost(tierA.ctx)).resolves.toBeUndefined(); + await tierA.dispose(); + tierA = undefined; + expect(globalThis.fetch).toBe(wire.fetch); + }); + + it("tier B with pluginConfig: the profile row reaches the plugin's fiber config", async () => { + vi.stubGlobal("fetch", recordingFetch().fetch); + tierB = await bootTierB({ pluginConfig: { debug: true } }); + const config = tierB.pluginEntry?.fiber?.config as + | { debug?: unknown } + | undefined; + // `debug` defaults to false, so only the row can make it true. + expect(config?.debug).toBe(true); + }); + + it("recordingFetch with a responder: a turn is answered by it and still recorded, so a test can drive a stream to completion", async () => { + const wire = recordingFetch({ + respond: (request) => completionStream(request.model ?? ""), + }); + vi.stubGlobal("fetch", wire.fetch); + tierA = await mountTierA({ + credentials: fakeCredentials({ refs: { OPENCODE_API_KEY: "k" } }), + providers: { + opencode: { + apiKeyEnv: "OPENCODE_API_KEY", + models: [{ id: "deepseek-v4-pro" }], + }, + }, + }); + const drained = await drainStream( + tierA.llm, + "opencode", + "deepseek-v4-pro" + ); + expect(drained.error).toBeUndefined(); + expect(drained.finish).toEqual({ code: undefined, kind: "stop" }); + expect(drained.types).toContain("text-delta"); + expect(wire.requests.map((request) => request.model)).toEqual([ + "deepseek-v4-pro", + ]); + }); + it.each([ + [ + "plugin-first", + "an invalid plugin row", + { pluginConfig: { debug: "not-a-boolean" } }, + ], + [ + "pi-ai-first", + "an invalid pi-ai profile", + { providers: { deepseek: { displayName: 42 } } }, + ], + ] as const)( + "tier B, %s with %s: the first row FAILS without applying, the held row is still released, and the boot finishes promptly", + async (order, _what, rows) => { + // Before the fallback release, the held row waited for an `apply` that + // never came, and the boot hung until the hook timed out. + vi.spyOn(process.stderr, "write").mockImplementation(() => true); + vi.stubGlobal("fetch", recordingFetch().fetch); + tierB = await bootTierB({ order: order as ApplyOrder, ...rows }); + const [failedEntry, heldEntry] = + order === "plugin-first" + ? [tierB.pluginEntry, tierB.piEntry] + : [tierB.piEntry, tierB.pluginEntry]; + expect(failedEntry?.fiber?.state).toBe(FAILED); + // The real error is readable, not a hook timeout. + await expect(failedEntry?.fiber?.await()).rejects.toThrow( + order === "plugin-first" ? /debug/u : /displayName/u + ); + expect(heldEntry?.fiber?.state).toBe(ACTIVE); + }, + 15_000 + ); + + it("tier B with OPENCODE_API_KEY exported: the boot blanks it, so the deployment measured is still the keyless one, and dispose puts it back", async () => { + vi.stubEnv("OPENCODE_API_KEY", "sk-ambient-not-a-key"); + vi.stubEnv("OPENCODE_GO_API_KEY", "sk-ambient-go-not-a-key"); + vi.stubGlobal("fetch", recordingFetch().fetch); + tierB = await bootTierB({ + credentials: fakeCredentials({ + refs: { + OPENCODE_API_KEY: "zen-test", + OPENCODE_GO_API_KEY: "go-test", + }, + }), + providers: characterizedProviders(), + }); + await waitForRoutes(tierB.llm, ["opencode-responses"]); + expect(process.env.OPENCODE_API_KEY).toBe(""); + expect(process.env.OPENCODE_GO_API_KEY).toBe(""); + // The plugin reads the environment for its keyless listing filter: with + // the shell's key left in place, the paid models would be listed here. + for (const plane of PLANES) { + expect(await tierB.llm.listModels(plane), plane).toEqual( + S0_TODAY.keylessListing[plane] + ); + } + await tierB.dispose(); + tierB = undefined; + expect(process.env.OPENCODE_API_KEY).toBe("sk-ambient-not-a-key"); + expect(process.env.OPENCODE_GO_API_KEY).toBe("sk-ambient-go-not-a-key"); + }); + + it("tier B leaves an unset key unset", async () => { + // oxlint-disable-next-line unicorn/no-useless-undefined -- `stubEnv` deletes the variable only when handed `undefined`, and restores it after. + vi.stubEnv("OPENCODE_API_KEY", undefined); + tierB = await bootTierB({ plugin: false, providers: s0Providers() }); + expect(process.env.OPENCODE_API_KEY).toBeUndefined(); + await tierB.dispose(); + tierB = undefined; + expect(process.env.OPENCODE_API_KEY).toBeUndefined(); + }); + } +); diff --git a/test/host-shapes.test.ts b/test/host-shapes.test.ts new file mode 100644 index 00000000..2f2dc1f1 --- /dev/null +++ b/test/host-shapes.test.ts @@ -0,0 +1,532 @@ +/** + * The host shapes and exports the plane provider (P-R1) is built from, pinned + * on the installed packages. + * + * The design reaches into pi-ai's private profile shape, its exported `Config` + * and `recordKeyFor`, the loader's exported `interpolate` and pi's public + * per-protocol exports (docs/improvement-plan.md, "Target architecture"). None + * of that is a promise the host made, so each piece is pinned here against the + * real package, with the host file and line it reads. A host bump that moves + * one turns this file red before any routing code depends on it. + * + * `test/mount-protocols.test.ts` already pins the four `api/*.lazy` factories + * against the catalog's SDKs; this file does not repeat it. + * + * @module test/host-shapes.test + */ +import { readFileSync } from "node:fs"; + +import { Context } from "@deepseek-ai/cordis"; +import { interpolate } from "@deepseek-ai/cordis-plugin-loader"; +import { resolveRetryPolicy } from "@deepseek-ai/dsh-llm"; +import * as LlmPiAi from "@deepseek-ai/dsh-llm-pi-ai"; +import z from "@deepseek-ai/schemastery"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { + adaptersOf, + drainStream, + fakeCredentials, + mountTierA, + recordingFetch, + s0Providers, + type TierA, +} from "./support/host.ts"; + +/** A schemastery node, as far as the role walk reads it. */ +interface SchemaNode { + readonly type?: string; + readonly meta?: { readonly role?: string }; + readonly dict?: Readonly>; + readonly inner?: SchemaNode; + readonly list?: readonly SchemaNode[]; +} + +/** + * The node types the role walk knows. A host bump that introduces another + * (a wrapper carrying `inner`, say) fails the pin by name instead of being + * walked blind. + */ +const KNOWN_NODE_TYPES = new Set([ + "any", + "array", + "boolean", + "const", + "dict", + "intersect", + "number", + "object", + "string", + "transform", + "union", +]); + +/** The path segment an `inner` node is walked under: none for a wrapper. */ +const INNER_SEGMENT: Readonly> = { + array: ["[]"], + dict: ["*"], +}; + +/** + * Every node under `node` that declares a role, as `[...path, role]`, and + * every node type visited. A dict's value is walked under `*`, an array's item + * under `[]`, and any other node's `inner` (a transform, or a wrapper a later + * schemastery adds) at its parent's path; union and intersect members share + * their parent's path too. + */ +const rolesUnder = ( + node: SchemaNode, + path: string[], + types = new Set() +): string[][] => { + const found: string[][] = []; + types.add(String(node.type)); + if (node.meta?.role !== undefined) { + found.push([...path, node.meta.role]); + } + if (node.type === "object") { + for (const [key, child] of Object.entries(node.dict ?? {})) { + found.push(...rolesUnder(child, [...path, key], types)); + } + } + if (node.inner !== undefined) { + found.push( + ...rolesUnder( + node.inner, + [...path, ...(INNER_SEGMENT[String(node.type)] ?? [])], + types + ) + ); + } + for (const member of node.list ?? []) { + found.push(...rolesUnder(member, path, types)); + } + return found; +}; + +/** A resolved provider profile as pi-ai's adapter holds it (private). */ +const profileOf = (tier: TierA, route: string): Record => { + const adapter = adaptersOf(tier.llm).get(route)?.adapter as unknown as { + config: { profiles: () => ReadonlyMap> }; + }; + const profile = adapter.config.profiles().get(route); + if (profile === undefined) { + throw new Error(`no resolved profile for ${route}`); + } + return profile; +}; + +/** A pi model, as far as these pins read one. */ +interface PiModel { + readonly api: string; + readonly baseUrl: string; +} + +/** Per protocol, how many models a pi provider carries and on which bases. */ +const byProtocol = ( + provider: unknown +): Record => { + const models = ( + provider as { getModels: () => readonly PiModel[] } + ).getModels(); + const out: Record = {}; + for (const model of models) { + const row = out[model.api] ?? { bases: [], count: 0 }; + out[model.api] = row; + row.count += 1; + if (!row.bases.includes(model.baseUrl)) { + row.bases.push(model.baseUrl); + } + } + return out; +}; + +let tier: TierA | undefined; + +afterEach(async () => { + await tier?.dispose(); + tier = undefined; + vi.unstubAllGlobals(); +}); + +describe("host · pi-ai's Config (llm-pi-ai/src/config.ts)", () => { + it("resolves an empty profile to these defaults, and providers is volatile (config.ts:47-80,326-354)", () => { + const resolved = ( + LlmPiAi.Config({ providers: { x: {} } } as never) as unknown as { + providers: { get: () => Record }; + } + ).providers.get().x; + expect(resolved).toEqual({ + compat: { chatTemplateArgs: {}, chatTemplateKwargs: {} }, + defaultContextWindow: 262_144, + defaultInput: ["text"], + defaultMaxTokens: 32_768, + headers: {}, + maxRequestImageBytes: 20_971_520, + modelOverrides: {}, + models: [], + requestImageMaxBytes: 1_048_576, + requestImagePixelBudget: 4_194_304, + streamIdleTimeoutMs: 300_000, + thinkingBudgets: {}, + }); + const schema = LlmPiAi.Config as unknown as SchemaNode & { + dict: { providers: { meta: { volatile?: boolean } } }; + }; + expect(schema.dict.providers.meta.volatile).toBe(true); + }); + + it("resolves an absent retry policy to normal mode with five retries (dsh-llm retry-policy.ts:149)", () => { + expect(resolveRetryPolicy(undefined, "t")).toEqual({ + initialDelayMs: 500, + jitterRatio: 0.1, + maxDelayMs: 10_000, + maxRetries: 5, + mode: "normal", + retryableCodes: [ + "EMPTY_RESPONSE", + "RATE_LIMIT", + "SERVER", + "TIMEOUT", + "TRANSPORT", + ], + }); + }); + + it("carries no role('secret') under providers; apiKeyEnv is the only role, credential-ref (config.ts:327)", () => { + const schema = LlmPiAi.Config as unknown as SchemaNode; + const types = new Set(); + expect(rolesUnder(schema, [], types)).toEqual([ + ["providers", "*", "apiKeyEnv", "credential-ref"], + ]); + expect( + [...types].filter((type) => !KNOWN_NODE_TYPES.has(type)), + "a node type the walk does not know" + ).toEqual([]); + // The walk reads roles at every depth: a planted secret is reported. + const planted = z.object({ + providers: z.dict( + z.object({ + models: z.array(z.object({ token: z.string().role("secret") })), + }) + ), + }) as unknown as SchemaNode; + expect(rolesUnder(planted, [])).toEqual([ + ["providers", "*", "models", "[]", "token", "secret"], + ]); + // ...including one under a transform, whose `inner` is walked too. + const wrapped = z.object({ + providers: z.dict( + z.transform( + z.object({ token: z.string().role("secret") }), + (value: unknown) => value + ) + ), + }) as unknown as SchemaNode; + expect(rolesUnder(wrapped, [])).toEqual([ + ["providers", "*", "token", "secret"], + ]); + }); +}); + +describe("host · ResolvedPiAiProviderProfile (llm-pi-ai/src/config.ts:186-219,490-507)", () => { + it("declares exactly these fields in the published types", () => { + const text = readFileSync( + new URL( + "../node_modules/@deepseek-ai/dsh-llm-pi-ai/lib/types/config.d.ts", + import.meta.url + ), + "utf8" + ); + const start = text.indexOf("export interface ResolvedPiAiProviderProfile"); + expect(start).toBeGreaterThan(-1); + const block = text.slice(start, text.indexOf("\n}", start)); + // The heritage clause names what is inherited; the body what is resolved. + expect(block).toContain( + "extends Omit" + ); + const fields = [...block.matchAll(/^ {4}(\w+\??):/gmu)].map( + (match) => match[1] + ); + expect(fields).toEqual([ + "provider", + "displayName", + "apiKeyEnv?", + "streamIdleTimeoutMs", + "maxRequestImageBytes", + "requestImagePixelBudget", + "requestImageMaxBytes", + "retryPolicy", + "piProvider?", + "catalogError?", + "modelErrors", + "configuredMaxTokens", + ]); + }); + + it("resolves a profile with these keys at runtime, models destructured out; F4 — a stored id pi cannot place is deferred to modelErrors and catalogError (config.ts:487)", async () => { + tier = await mountTierA({ providers: s0Providers() }); + const profile = profileOf(tier, "opencode"); + expect(Object.keys(profile).toSorted()).toEqual([ + "apiKeyEnv", + "catalogError", + "compat", + "configuredMaxTokens", + "defaultContextWindow", + "defaultInput", + "defaultMaxTokens", + "displayName", + "headers", + "maxRequestImageBytes", + "modelErrors", + "modelOverrides", + "piProvider", + "provider", + "requestImageMaxBytes", + "requestImagePixelBudget", + "retryPolicy", + "streamIdleTimeoutMs", + "thinkingBudgets", + ]); + expect(profile.modelErrors).toBeInstanceOf(Map); + expect(profile.configuredMaxTokens).toBeInstanceOf(Map); + // A stored gap id is a deferred diagnostic, not a refusal (config.ts:487). + expect([...(profile.modelErrors as Map).keys()]).toEqual([ + "gpt-5.3-codex-spark", + ]); + expect(profile.catalogError).toMatch( + /model "gpt-5\.3-codex-spark" needs an api/u + ); + expect([ + ...( + profileOf(tier, "opencode-go").modelErrors as Map + ).keys(), + ]).toEqual(["gemini-4-argon", "minimax-m3.1-flash"]); + }); + + it("F4 — a stored id pi cannot place is not listed, and a turn on it is INVALID_CONFIG with no request (index.ts:171; adapter.ts:271-311)", async () => { + // Plugin-less, so this half of F4 outlives the characterization rows that + // also show it with the plugin mounted (P-R1 flips those). A key is stored + // for each plane: with none, F18's empty listing would mask the check. + const wire = recordingFetch(); + vi.stubGlobal("fetch", wire.fetch); + const mounted = await mountTierA({ + credentials: fakeCredentials({ + refs: { OPENCODE_API_KEY: "zen-k", OPENCODE_GO_API_KEY: "go-k" }, + }), + providers: s0Providers(), + }); + tier = mounted; + const listed = async (route: string): Promise => { + const models = await mounted.llm.listModels(route); + return models.map((model) => model.id); + }; + expect(await listed("opencode")).toEqual([ + "deepseek-v4-pro", + "gpt-5.4", + "claude-haiku-5-5", + "gemini-3-flash", + ]); + expect(await listed("opencode-go")).toEqual([ + "deepseek-v4-pro", + "muse-spark-1.2-contributor", + "minimax-m3", + ]); + for (const [route, model] of [ + ["opencode", "gpt-5.3-codex-spark"], + ["opencode-go", "gemini-4-argon"], + ["opencode-go", "minimax-m3.1-flash"], + ] as const) { + const drained = await drainStream(mounted.llm, route, model); + expect(drained.finish, `${route}/${model}`).toEqual({ + code: "INVALID_CONFIG", + kind: "error", + }); + } + expect(wire.requests).toEqual([]); + }); + + it("F1/F2 — a model-less plane resolves pi 1.1.0's whole catalog, per protocol, on the legacy bases", async () => { + tier = await mountTierA({ + providers: { opencode: {}, "opencode-go": {} }, + }); + const zen = profileOf(tier, "opencode"); + expect(zen.catalogError).toBeUndefined(); + expect(byProtocol(zen.piProvider)).toEqual({ + "anthropic-messages": { bases: ["https://opencode.ai/zen"], count: 18 }, + "google-generative-ai": { + bases: ["https://opencode.ai/zen/v1"], + count: 7, + }, + "openai-completions": { + bases: ["https://opencode.ai/zen/v1"], + count: 28, + }, + "openai-responses": { bases: ["https://opencode.ai/zen/v1"], count: 30 }, + }); + // F10: no Google model on Go. + expect(byProtocol(profileOf(tier, "opencode-go").piProvider)).toEqual({ + "anthropic-messages": { bases: ["https://opencode.ai/zen/go"], count: 5 }, + "openai-completions": { + bases: ["https://opencode.ai/zen/go/v1"], + count: 19, + }, + "openai-responses": { + bases: ["https://opencode.ai/zen/go/v1"], + count: 6, + }, + }); + }); + + it("F5 — a route-level api and baseURL override every model, collapsing pi's four protocols onto one (llm-pi-ai/src/catalog.ts:917,922)", async () => { + tier = await mountTierA({ + providers: { + opencode: { api: "openai-responses" }, + "opencode-go": { baseURL: "https://gateway.test/go" }, + }, + }); + const zen = profileOf(tier, "opencode"); + expect(zen.catalogError).toBeUndefined(); + // Every one of F2's 83 Zen models now speaks Responses; each keeps its own + // base, which only the route's baseURL would override. + expect(byProtocol(zen.piProvider)).toEqual({ + "openai-responses": { + bases: ["https://opencode.ai/zen", "https://opencode.ai/zen/v1"], + count: 83, + }, + }); + // Go's 30 models keep their protocols and all move to the route's base. + expect(byProtocol(profileOf(tier, "opencode-go").piProvider)).toEqual({ + "anthropic-messages": { bases: ["https://gateway.test/go"], count: 5 }, + "openai-completions": { bases: ["https://gateway.test/go"], count: 19 }, + "openai-responses": { bases: ["https://gateway.test/go"], count: 6 }, + }); + }); +}); + +describe("host · pi-ai's credential key (llm-pi-ai/src/auth.ts:29-39; credentials/src/index.ts:69-78)", () => { + it("addresses a route's record as llm-pi-ai/, and refuses a key outside the grammar", () => { + expect(LlmPiAi.recordKeyFor("opencode")).toBe("llm-pi-ai/opencode"); + expect(LlmPiAi.recordKeyFor("opencode-go")).toBe("llm-pi-ai/opencode-go"); + expect(() => LlmPiAi.recordKeyFor("Not Valid")).toThrow(TypeError); + }); +}); + +describe("host · the loader's interpolate (vendor/loader/src/config/utils.ts:12-22; index.ts:19)", () => { + it("replaces {__jsExpr} nodes at any depth and leaves the rest alone", () => { + expect( + interpolate(new Context(), { + a: { __jsExpr: "1 + 1" }, + b: [{ __jsExpr: '"x"' }], + c: "c", + }) + ).toEqual({ a: 2, b: ["x"], c: "c" }); + }); +}); + +describe("host · pi 1.1.0's public exports (package.json exports ./providers/*, ./api/*, ./models)", () => { + it("opencodeProvider/opencodeGoProvider resolve keys through pi's ambient auth (providers/opencode.js:10-24; opencode-go.js:8-20)", async () => { + const { opencodeProvider } = + await import("@earendil-works/pi-ai/providers/opencode"); + const { opencodeGoProvider } = + await import("@earendil-works/pi-ai/providers/opencode-go"); + for (const provider of [opencodeProvider(), opencodeGoProvider()]) { + const auth = provider.auth as { + apiKey?: { + resolve: (input: { + ctx: { env: (name: string) => Promise }; + credential?: { key?: string }; + signal: AbortSignal; + }) => Promise< + { auth: { apiKey: string }; source: string } | undefined + >; + }; + }; + expect(typeof auth.apiKey?.resolve).toBe("function"); + const asked: string[] = []; + const resolved = await auth.apiKey?.resolve({ + ctx: { + env: async (name) => { + asked.push(name); + return name === "OPENCODE_API_KEY" ? "ambient" : undefined; + }, + }, + signal: new AbortController().signal, + }); + // Both planes read the one Zen variable: Go has no variable of its own. + expect(asked).toEqual(["OPENCODE_API_KEY"]); + expect(resolved).toMatchObject({ + auth: { apiKey: "ambient" }, + source: "OPENCODE_API_KEY", + }); + } + }); + + it("F10 — the Go provider has no Google API: a Google model streams pi's own 'no API implementation' error", async () => { + const { opencodeProvider } = + await import("@earendil-works/pi-ai/providers/opencode"); + const { opencodeGoProvider } = + await import("@earendil-works/pi-ai/providers/opencode-go"); + const apisOf = (provider: { getModels: () => readonly PiModel[] }) => + [...new Set(provider.getModels().map((model) => model.api))].toSorted(); + expect(apisOf(opencodeProvider())).toEqual([ + "anthropic-messages", + "google-generative-ai", + "openai-completions", + "openai-responses", + ]); + expect(apisOf(opencodeGoProvider())).toEqual([ + "anthropic-messages", + "openai-completions", + "openai-responses", + ]); + const gemini = opencodeProvider() + .getModels() + .find((model) => model.api === "google-generative-ai"); + expect(gemini).toBeDefined(); + const events: unknown[] = []; + for await (const event of opencodeGoProvider().stream( + { ...gemini, provider: "opencode-go" } as never, + { messages: [] } as never, + {} + )) { + events.push(event); + } + expect(JSON.stringify(events)).toContain( + 'Provider opencode-go has no API implementation for \\"google-generative-ai\\"' + ); + }); + + it("exports createProvider and withOpenCodeSessionHeader (models.js:527; providers/opencode-headers.js:15)", async () => { + const { createProvider } = await import("@earendil-works/pi-ai/models"); + expect(typeof createProvider).toBe("function"); + const { withOpenCodeSessionHeader } = + await import("@earendil-works/pi-ai/providers/opencode-headers"); + const wrapped = withOpenCodeSessionHeader({ + stream: (_model: unknown, _context: unknown, options: unknown) => options, + streamSimple: (_model: unknown, _context: unknown, options: unknown) => + options, + } as never) as { + streamSimple: ( + model: unknown, + context: unknown, + options: unknown + ) => unknown; + }; + expect(wrapped.streamSimple({}, {}, { sessionId: "ses_x" })).toEqual({ + headers: { "x-opencode-session": "ses_x" }, + sessionId: "ses_x", + }); + // A caller-set header is kept, whatever its case. + expect( + wrapped.streamSimple( + {}, + {}, + { headers: { "X-OpenCode-Session": "mine" }, sessionId: "ses_x" } + ) + ).toEqual({ + headers: { "X-OpenCode-Session": "mine" }, + sessionId: "ses_x", + }); + }); +}); diff --git a/test/host-waterfall.test.ts b/test/host-waterfall.test.ts new file mode 100644 index 00000000..c2440720 --- /dev/null +++ b/test/host-waterfall.test.ts @@ -0,0 +1,285 @@ +/** + * The `internal/config` waterfall, pinned on the real host (tier A). + * + * The route-claim design (docs/improvement-plan.md, D13) rests on cordis and + * pi-ai behaving exactly as these cases say: pi-ai registers the keys of its + * RESOLVED config (F11), a listener another plugin prepends wraps pi-ai's + * fiber and runs outermost (F8), a commit is told from a validation by + * identity (F14), and the fiber keeps the outermost listener's return value + * without anyone mutating the input (F12). A pin that goes red here sends the + * design back to the plan. + * + * Host references are into `deepseek-harness` at the tag `host-versions.ts` + * names: `vendor/cordis/src/…` and `packages/llm/…`. + * + * @module test/host-waterfall.test + */ +import type { Context, Fiber } from "@deepseek-ai/cordis"; +import { LlmAdapter } from "@deepseek-ai/dsh-llm"; +import { afterEach, describe, expect, it } from "vitest"; + +import { + deepFreeze, + mountTierA, + s0Providers, + type TierA, + waterfallSpy, + withoutPlanes, +} from "./support/host.ts"; + +/** The resolved provider keys pi-ai's fiber holds. */ +const resolvedKeys = (fiber: Fiber): string[] => + Object.keys( + ( + fiber.config as { providers: { get: () => Record } } + ).providers.get() + ).toSorted(); + +/** The raw config the fiber stores (`fiber._config`). */ +const rawOf = (fiber: Fiber): unknown => + (fiber as unknown as { _config: unknown })._config; + +/** The registered route ids, sorted. */ +const routeIds = (tier: TierA): string[] => + tier.llm + .listProviders() + .map((provider) => provider.id) + .toSorted(); + +/** An adapter that registers and never streams: the claim's stand-in. */ +class StubAdapter extends LlmAdapter { + // oxlint-disable-next-line require-yield -- never streams; the registry is the subject. + async *stream(): AsyncIterable { + throw new Error("never streams"); + } +} + +/** + * The listener the claim design installs: for pi-ai's fiber only, the inner + * chain runs and the fiber keeps a fresh copy without the planes. + */ +function stripPlanesForPiAi( + this: Fiber, + raw: unknown, + next: () => unknown +): unknown { + if (this.runtime?.name !== "llm-pi-ai") { + return next(); + } + const copy = withoutPlanes(raw); + next(); + return copy; +} + +let tier: TierA | undefined; + +afterEach(async () => { + await tier?.dispose(); + tier = undefined; +}); + +describe("host · the internal/config waterfall (tier A)", () => { + it("F11 — pi-ai registers exactly the keys of its resolved config (llm-pi-ai/src/index.ts:168-176,299,307)", async () => { + tier = await mountTierA({ providers: s0Providers() }); + expect(routeIds(tier)).toEqual(resolvedKeys(tier.piFiber)); + expect(routeIds(tier)).toEqual(["deepseek", "opencode", "opencode-go"]); + }); + + it("F11 — the RESOLVED keys, not the raw ones: with a listener stripping the planes, the routes follow the resolved config while the raw still holds all three", async () => { + tier = await mountTierA({ + before: (ctx) => { + ctx.on("internal/config", stripPlanesForPiAi, { prepend: true }); + }, + providers: s0Providers(), + }); + expect(routeIds(tier)).toEqual(resolvedKeys(tier.piFiber)); + expect(routeIds(tier)).toEqual(["deepseek"]); + expect( + Object.keys( + (rawOf(tier.piFiber) as { providers: Record }) + .providers + ).toSorted() + ).toEqual(["deepseek", "opencode", "opencode-go"]); + }); + + it("F8/F12 — a listener another plugin prepends fires for pi-ai's fiber with this === fiber, outermost (cordis events.ts:234-243,254-260)", async () => { + tier = await mountTierA({ providers: s0Providers() }); + const order: string[] = []; + const seen: { name: string; self: unknown; raw: unknown }[] = []; + // A sibling plugin, mounted AFTER pi-ai: its plain listener lands after + // pi-ai's own (inner), its prepended one before everything (outer). + await tier.ctx.plugin({ + apply(c: Context) { + c.on( + "internal/config", + function listenerA(this: Fiber, raw: unknown, next: () => unknown) { + if (this.runtime?.name === "llm-pi-ai") { + order.push("A"); + seen.push({ name: "A", raw, self: this }); + } + return next(); + } + ); + c.on( + "internal/config", + function listenerB(this: Fiber, raw: unknown, next: () => unknown) { + if (this.runtime?.name !== "llm-pi-ai") { + return next(); + } + order.push("B:enter"); + seen.push({ name: "B", raw, self: this }); + const value = next(); + order.push("B:exit"); + return value; + }, + { prepend: true } + ); + }, + name: "claimer", + }); + const input = { providers: s0Providers() }; + tier.piFiber.update(input); + await tier.piFiber.await(); + // Two runs: `update` resolves while ACTIVE (fiber.ts:747), then the + // restart re-resolves the stored raw (fiber.ts:655). B brackets A both + // times, so B is outermost and A runs inside the chain B started. + expect(order).toEqual(["B:enter", "A", "B:exit", "B:enter", "A", "B:exit"]); + for (const record of seen) { + expect(record.self).toBe(tier.piFiber); + } + expect(seen[0]?.raw).toBe(input); + expect(tier.piFiber.state).toBe(2); + }); + + it("F8/F4 — pi-ai's own listener validates the value its inner chain returns, not what an outer listener will return (llm-pi-ai/src/index.ts:177-186; config.ts:364-368)", async () => { + // Read for P-R1: an outer listener that strips the planes and returns a + // copy does NOT shield pi-ai's save check, which runs on `next()`'s value. + // A listener INSIDE pi-ai's (registered after it, unprepended) is what + // that check sees. + tier = await mountTierA({ providers: s0Providers() }); + const order: string[] = []; + await tier.ctx.plugin({ + apply(c: Context) { + c.on( + "internal/config", + function addUnknownId( + this: Fiber, + raw: unknown, + next: () => unknown + ) { + if (this.runtime?.name !== "llm-pi-ai") { + return next(); + } + order.push("A"); + next(); + // A changed plane holding an id pi's catalog does not describe. + const candidate = structuredClone(raw) as { + providers: Record; + }; + candidate.providers.opencode?.models?.push({ + id: "not-a-real-model", + }); + return candidate; + } + ); + c.on( + "internal/config", + function bracket(this: Fiber, _raw: unknown, next: () => unknown) { + if (this.runtime?.name !== "llm-pi-ai") { + return next(); + } + order.push("B:enter"); + try { + return next(); + } catch (error) { + order.push("B:pi-ai threw"); + throw error; + } + }, + { prepend: true } + ); + }, + name: "claimer", + }); + expect(() => { + tier?.piFiber.update({ providers: s0Providers() }); + // The strict check covers the whole changed plane, so it names the + // first id it cannot place: S0's stored gap model, ahead of the one A + // added. An unchanged plane is not checked at all (the case above). + }).toThrow( + /provider "opencode" model "gpt-5\.3-codex-spark" needs an api/u + ); + expect(order).toEqual(["B:enter", "A", "B:pi-ai threw"]); + }); + + it("F14 — raw === fiber._config on _reload and on update (cordis fiber.ts:641-644,655,738,747)", async () => { + tier = await mountTierA({ providers: s0Providers() }); + const { piFiber } = tier; + const spy = waterfallSpy(tier.ctx, () => piFiber); + await piFiber.restart(); + expect(spy.records).toHaveLength(1); + expect(spy.records[0]?.isCommit).toBe(true); + expect(spy.records[0]?.raw).toBe(rawOf(piFiber)); + + spy.records.length = 0; + const next = structuredClone(rawOf(piFiber)); + piFiber.update(next); + await piFiber.await(); + expect(spy.records.map((record) => record.isCommit)).toEqual([true, true]); + for (const record of spy.records) { + expect(record.raw).toBe(next); + } + expect(rawOf(piFiber)).toBe(next); + }); + + it("F12 — a deep-frozen input is never mutated, and the fiber keeps the outermost listener's return value (cordis fiber.ts:642,655; utils.ts:117-125)", async () => { + const frozen = deepFreeze({ providers: s0Providers() }); + tier = await mountTierA({ + before: (ctx) => { + ctx.on("internal/config", stripPlanesForPiAi, { prepend: true }); + }, + piConfig: frozen, + providers: {}, + }); + const { piFiber } = tier; + expect(rawOf(piFiber)).toBe(frozen); + expect( + Object.keys((rawOf(piFiber) as typeof frozen).providers).toSorted() + ).toEqual(["deepseek", "opencode", "opencode-go"]); + expect(resolvedKeys(piFiber)).toEqual(["deepseek"]); + expect(routeIds(tier)).toEqual(["deepseek"]); + }); + + it("F13 (tier A half) — with the planes stripped, registerAdapter(planes) passes the duplicate check and the directory still lists both under llm-pi-ai (llm/src/index.ts:401-445; llm-pi-ai/src/index.ts:128-150)", async () => { + tier = await mountTierA({ + before: (ctx) => { + ctx.on("internal/config", stripPlanesForPiAi, { prepend: true }); + }, + providers: s0Providers(), + }); + expect(routeIds(tier)).toEqual(["deepseek"]); + const handle = tier.llm.registerAdapter( + ["opencode", "opencode-go"], + new StubAdapter() + ); + try { + expect(routeIds(tier)).toEqual(["deepseek", "opencode", "opencode-go"]); + const directory = tier.llm.listConfigurableProviders(); + for (const id of ["opencode", "opencode-go"]) { + expect(directory.find((entry) => entry.provider === id)).toMatchObject({ + settingsNs: "llm-pi-ai", + settingsPath: ["providers", id], + }); + } + } finally { + handle(); + } + }); + + it("F13 control — while pi-ai holds the planes, registering them is DUPLICATE_ADAPTER (llm/src/index.ts:443)", async () => { + tier = await mountTierA({ providers: s0Providers() }); + expect(() => + tier?.llm.registerAdapter(["opencode"], new StubAdapter()) + ).toThrow(expect.objectContaining({ code: "DUPLICATE_ADAPTER" })); + }); +}); diff --git a/test/keyless-matrix.test.ts b/test/keyless-matrix.test.ts new file mode 100644 index 00000000..f616e059 --- /dev/null +++ b/test/keyless-matrix.test.ts @@ -0,0 +1,352 @@ +/** + * The keyless probe matrix's cell table and classifier, offline. + * + * The live half (`test/e2e/keyless-matrix.e2e.ts`) cannot run in a sandbox + * without the vendor's hosts, so everything it decides is decided here: which + * sixteen URLs are probed, with which bodies, and how an answer is read. The + * URLs are written out from the vendor's endpoint layout, not derived from the + * module, and the bundled-catalog case checks invariants rather than model + * names, so the nightly catalog refresh cannot turn it red. + * + * @module test/keyless-matrix.test + */ +import { describe, expect, it } from "vitest"; + +import { PLANES, S0_TODAY } from "./support/host.ts"; +import { + cellKey, + classifyKeylessProbe, + isFreeSpec, + isTransientProbeAnswer, + keylessMatrix, + MATRIX_PROTOCOLS, + type MatrixCell, + matrixUrl, + NEXT_DISPATCH, + TODAY_DISPATCH, +} from "./support/keyless-matrix.ts"; +import { + type CatalogModelSpec, + OPENCODE_GO_CATALOG, + OPENCODE_ZEN_CATALOG, + protocolFor, +} from "./support/src.ts"; + +const spec = ( + id: string, + provider_npm?: string, + free = false +): CatalogModelSpec => ({ + context_window: 1000, + id, + input_modalities: ["text"], + max_output_tokens: 100, + name: id, + ...(free ? { is_free: true } : {}), + ...(provider_npm === undefined ? {} : { provider_npm }), +}); + +/** + * One paid model per protocol on each plane, each behind a free one (flagged, + * or only named `free`) and Zen's Responses model behind a retired one too. + * A free model answers a keyless request with a real completion, so the cells + * must never pick one. + */ +const ZEN = [ + spec("zen-beam", undefined, true), + spec("zen-chat-free"), + spec("zen-chat"), + spec("zen-resp-free", "@ai-sdk/openai", true), + spec("zen-old-resp", "@ai-sdk/openai"), + spec("zen-resp", "@ai-sdk/openai"), + spec("zen-claude-free", "@ai-sdk/anthropic", true), + spec("zen-claude", "@ai-sdk/anthropic"), + spec("zen-gemini-free", "@ai-sdk/google", true), + spec("zen-gemini", "@ai-sdk/google"), + spec("zen-mistral", "@ai-sdk/mistral"), +]; +const GO = [ + spec("go-beam", undefined, true), + spec("go-chat"), + spec("go-resp-free", "@ai-sdk/openai"), + spec("go-resp", "@ai-sdk/openai"), + spec("go-claude", "@ai-sdk/anthropic"), + spec("go-gemini-free", "@ai-sdk/google", true), + spec("go-gemini", "@ai-sdk/google"), +]; + +const key = (cell: MatrixCell): string => cellKey(cell); + +const CREDENTIAL_HEADERS = new Set([ + "authorization", + "x-api-key", + "x-goog-api-key", +]); + +describe("keyless matrix · the cells", () => { + it("is sixteen cells with exactly these URLs and models", () => { + const cells = keylessMatrix({ + go: GO, + isDeprecated: (plane, id) => + plane === "opencode" && id === "zen-old-resp", + zen: ZEN, + }); + expect(cells).toHaveLength(16); + expect( + Object.fromEntries( + cells.map((cell) => [key(cell), [cell.url, cell.model]]) + ) + ).toEqual({ + "opencode openai-completions legacy": [ + "https://opencode.ai/zen/v1/chat/completions", + "zen-chat", + ], + "opencode openai-completions modern": [ + "https://opencode.ai/inference/openai/v1/chat/completions", + "zen-chat", + ], + "opencode openai-responses legacy": [ + "https://opencode.ai/zen/v1/responses", + "zen-resp", + ], + "opencode openai-responses modern": [ + "https://opencode.ai/inference/openai/v1/responses", + "zen-resp", + ], + "opencode anthropic-messages legacy": [ + "https://opencode.ai/zen/v1/messages", + "zen-claude", + ], + "opencode anthropic-messages modern": [ + "https://opencode.ai/inference/anthropic/v1/messages", + "zen-claude", + ], + "opencode google-generative-ai legacy": [ + "https://opencode.ai/zen/v1/models/zen-gemini:streamGenerateContent?alt=sse", + "zen-gemini", + ], + "opencode google-generative-ai modern": [ + "https://opencode.ai/inference/google/v1beta/models/zen-gemini:streamGenerateContent?alt=sse", + "zen-gemini", + ], + "opencode-go openai-completions legacy": [ + "https://opencode.ai/zen/go/v1/chat/completions", + "go-chat", + ], + "opencode-go openai-completions modern": [ + "https://opencode.ai/inference/go/openai/v1/chat/completions", + "go-chat", + ], + "opencode-go openai-responses legacy": [ + "https://opencode.ai/zen/go/v1/responses", + "go-resp", + ], + "opencode-go openai-responses modern": [ + "https://opencode.ai/inference/go/openai/v1/responses", + "go-resp", + ], + "opencode-go anthropic-messages legacy": [ + "https://opencode.ai/zen/go/v1/messages", + "go-claude", + ], + "opencode-go anthropic-messages modern": [ + "https://opencode.ai/inference/go/anthropic/v1/messages", + "go-claude", + ], + "opencode-go google-generative-ai legacy": [ + "https://opencode.ai/zen/go/v1/models/go-gemini:streamGenerateContent", + "go-gemini", + ], + "opencode-go google-generative-ai modern": [ + "https://opencode.ai/inference/go/google/v1beta/models/go-gemini:streamGenerateContent?alt=sse", + "go-gemini", + ], + }); + }); + + it("carries the model in the body for the three body-addressed protocols, and the smallest output budget", () => { + const cells = keylessMatrix({ go: GO, zen: ZEN }); + const zen = (protocol: string, family: string) => + cells.find( + (cell) => + cell.plane === "opencode" && + cell.protocol === protocol && + cell.family === family + ); + expect(zen("openai-completions", "modern")?.body).toEqual({ + max_tokens: 1, + messages: [{ content: "hi", role: "user" }], + model: "zen-chat", + }); + expect(zen("openai-responses", "legacy")?.body).toEqual({ + input: "hi", + max_output_tokens: 1, + model: "zen-old-resp", + }); + expect(zen("anthropic-messages", "modern")?.body).toEqual({ + max_tokens: 1, + messages: [{ content: "hi", role: "user" }], + model: "zen-claude", + }); + expect(zen("google-generative-ai", "modern")?.body).toEqual({ + contents: [{ parts: [{ text: "hi" }], role: "user" }], + generationConfig: { maxOutputTokens: 1 }, + }); + expect(zen("anthropic-messages", "legacy")?.headers).toEqual({ + "anthropic-version": "2023-06-01", + "content-type": "application/json", + }); + expect(zen("openai-completions", "legacy")?.headers).toEqual({ + "content-type": "application/json", + }); + }); + + it("never picks a free model, flagged or only named so", () => { + const cells = keylessMatrix({ go: GO, zen: ZEN }); + const all = [...ZEN, ...GO]; + for (const cell of cells) { + const picked = all.find((entry) => entry.id === cell.model); + expect(picked, key(cell)).toBeDefined(); + expect(picked?.is_free, key(cell)).not.toBe(true); + expect(cell.model, key(cell)).not.toMatch(/free/iu); + } + // A plane whose only model for a protocol is free has no cell model. + const onlyFree = keylessMatrix({ + go: GO, + zen: [spec("zen-beam", undefined, true)], + }); + const zenCells = onlyFree.filter((cell) => cell.plane === "opencode"); + expect(zenCells).toHaveLength(8); + expect(zenCells.filter((cell) => cell.model !== undefined)).toEqual([]); + expect(isFreeSpec(spec("paid"))).toBe(false); + expect(isFreeSpec(spec("x", undefined, true))).toBe(true); + expect(isFreeSpec(spec("big-FREE-model"))).toBe(true); + }); + + it("reports a protocol a plane has no model for as a model-less, body-less cell", () => { + const cells = keylessMatrix({ + go: GO.filter((entry) => entry.provider_npm !== "@ai-sdk/google"), + zen: ZEN, + }); + const missing = cells.filter((cell) => cell.model === undefined); + expect(missing.map(key)).toEqual([ + "opencode-go google-generative-ai legacy", + "opencode-go google-generative-ai modern", + ]); + for (const cell of missing) { + expect(cell.body).toBeUndefined(); + } + }); + + it("on the bundled catalogs: sixteen cells, each the first PAID model of its protocol, and none can carry a key", () => { + const cells = keylessMatrix({ + go: OPENCODE_GO_CATALOG, + zen: OPENCODE_ZEN_CATALOG, + }); + expect(cells).toHaveLength(16); + for (const cell of cells) { + const catalog = + cell.plane === "opencode" ? OPENCODE_ZEN_CATALOG : OPENCODE_GO_CATALOG; + const first = catalog.find( + (entry) => + protocolFor(entry) === cell.protocol && + entry.is_free !== true && + !/free/iu.test(entry.id) + ); + expect(cell.model, key(cell)).toBe(first?.id); + expect(cell.model, key(cell)).toBeDefined(); + expect( + catalog.find((entry) => entry.id === cell.model)?.is_free, + key(cell) + ).not.toBe(true); + if (cell.protocol === "google-generative-ai") { + expect(cell.url).toContain( + `/models/${cell.model}:streamGenerateContent` + ); + } else { + expect(cell.body?.model).toBe(cell.model); + } + expect(cell.method).toBe("POST"); + expect( + Object.keys(cell.headers).filter((header) => + CREDENTIAL_HEADERS.has(header.toLowerCase()) + ) + ).toEqual([]); + expect(new URL(cell.url).hostname).toBe("opencode.ai"); + } + }); +}); + +describe("keyless matrix · the cells the live run asserts", () => { + it("TODAY_DISPATCH is exactly the cells S0_TODAY's measured turns were sent to", () => { + // Each URL a characterized turn sent is matched to the one cell whose PATH + // it is (the query aside: Messages adds `?beta=true`), with the row's own + // model in the Google path. A row that sent nothing adds nothing. + const reached = new Set(); + for (const [rowKey, row] of Object.entries(S0_TODAY.rows)) { + const slash = rowKey.indexOf("/"); + const plane = rowKey.slice(0, slash) as (typeof PLANES)[number]; + const model = rowKey.slice(slash + 1); + expect(PLANES, rowKey).toContain(plane); + for (const sent of row.sent) { + const matches = MATRIX_PROTOCOLS.flatMap((protocol) => + (["legacy", "modern"] as const) + .filter( + (family) => + new URL(matrixUrl(plane, protocol, family, model)).pathname === + new URL(sent).pathname + ) + .map((family) => cellKey({ family, plane, protocol })) + ); + expect(matches, `${rowKey} ${sent}`).toHaveLength(1); + reached.add(matches[0] ?? ""); + } + } + expect([...reached].toSorted()).toEqual([...TODAY_DISPATCH].toSorted()); + // Completions reach the legacy family today: the modern completions cells + // are P-R1's target, asserted live as a forward guard, never as today's. + expect([...TODAY_DISPATCH]).toContain("opencode openai-completions legacy"); + expect( + [...NEXT_DISPATCH].filter((cell) => TODAY_DISPATCH.has(cell)) + ).toEqual([]); + expect([...NEXT_DISPATCH].toSorted()).toEqual([ + "opencode openai-completions modern", + "opencode-go openai-completions modern", + ]); + const all = new Set(keylessMatrix({ go: GO, zen: ZEN }).map(key)); + for (const cell of [...TODAY_DISPATCH, ...NEXT_DISPATCH]) { + expect(all.has(cell), cell).toBe(true); + } + }); +}); + +describe("keyless matrix · reading an answer", () => { + it.each([ + [404, "", "path-gone"], + [401, '{"error":{"type":"AuthError"}}', "path-exists"], + [403, "", "path-exists"], + [410, "", "model-gone"], + [400, '{"error":{"message":"Model is unavailable"}}', "model-gone"], + [400, "model is UNAVAILABLE for this plan", "model-gone"], + [400, "Invalid API key", "path-exists"], + [200, "{}", "other"], + [500, "", "other"], + ] as const)("%i %s → %s", (status, bodyText, verdict) => { + expect(classifyKeylessProbe({ bodyText, status })).toBe(verdict); + }); +}); + +describe("keyless matrix · a transient answer is retried once", () => { + it.each([ + [undefined, true], + [429, true], + [500, true], + [503, true], + [401, false], + [404, false], + [400, false], + [200, false], + ] as const)("%s → %s", (status, retry) => { + expect(isTransientProbeAnswer(status)).toBe(retry); + }); +}); diff --git a/test/support/host.ts b/test/support/host.ts new file mode 100644 index 00000000..77222897 --- /dev/null +++ b/test/support/host.ts @@ -0,0 +1,1447 @@ +/** + * The real host, in-process: the harness the routing work is judged against. + * + * Two tiers, both built from the host packages this repository installs at the + * versions `host-versions.ts` names (pi 1.1.0 patched, the `@deepseek-ai` + * cluster at the published host's tag): + * + * - **Tier A** ({@link mountTierA}): a bare cordis `Context`, `LlmRuntime`, and + * `dsh-llm-pi-ai` mounted as a plugin with a `{ providers }` config, so its + * config is the schemastery instance and its fiber has `update()` and + * `restart()`. No loader, no profile, no files. It is the tier for the + * `internal/config` waterfall and pi-ai's credential paths: anything that + * needs a fiber and the registry, and nothing that reads a profile. Mirrors + * the harness's own `llm/llm-pi-ai/tests/egress.spec.ts:33-47`. + * - **Tier B** ({@link bootTierB}): dsh-app-boot's `boot()` with a temporary + * HOME, a test bundle and a profile patch, exactly as the harness's + * `experimental/speech-to-text/tests/configuration.spec.ts:12-33` boots one + * (`initProfile`, `readProfilePatches`, `root.provide("profileContext")`). + * It is the tier for the config editor, `settings.describe`, the loader's + * volatile commit, and the characterization of the plugin as it ships: + * everything that reads the loader's entries or writes the profile YAML. + * + * **Tier B's rows carry PACKAGE names, not `cordis:` builtins.** The five host + * rows (`llm`, `llm-pi-ai`, `config-editor`, `settings`, and this plugin) are + * named `@deepseek-ai/dsh-llm` … `dsh-opencode-patch`, and a stub module loader + * on `root.loader.internal` resolves those names to the statically imported + * namespaces — the harness's own pattern for a composition test + * (`llm/llm-pi-ai/tests/loader-composition.spec.ts:62-79`). A builtin row is + * named `cordis:` (the loader's `vendor/loader/src/config/tree.ts:112-115`), + * and the plugin finds pi-ai's entry by `options.name === + * "@deepseek-ai/dsh-llm-pi-ai"` (`piAiEntry` in `src/responses-provider.ts`), + * so a builtin-named row would silently drop today's hidden-route mount and the + * Google bridge from everything this harness characterizes. + * + * **Tier B fixes the apply order.** The loader starts rows in parallel, and the + * stub loader resolves every import at once, so which of pi-ai and the plugin + * applies first would otherwise be an accident of row order. It matters: the + * plugin's adapter decoration wraps only what is registered when it applies. + * The default is `plugin-first`, the order measured on the published binary + * ({@link ApplyOrder}); `pi-ai-first` is the other one. The held row is + * released when the first row's `apply` returns OR its fiber settles in any + * state, so a first row whose config fails to resolve leaves the boot to + * finish with that fiber FAILED instead of hanging it. + * + * **Nothing here reads the process environment.** Both tiers provide a fake + * `credentials` service ({@link fakeCredentials}) and a launch-environment + * snapshot ({@link fakeLaunchEnvironment}) on the root, which is where pi-ai + * looks first (`launchEnvironmentOf` reads `ctx.get("launchEnvironment")` + * before it falls back to `process.env`). The plugin itself still reads + * `process.env.OPENCODE_API_KEY`/`OPENCODE_GO_API_KEY` for its keyless listing + * filter (`hasPlaneCredential` in `src/lifecycle.ts`), so tier B blanks either + * one a contributor's shell exports for the life of the boot and restores it + * on dispose: the suite measures the keyless deployment whatever the shell + * holds. + * + * **Offline.** {@link recordingFetch} answers every `opencode.ai` request with a + * 401 and refuses every other host. Install it with `vi.stubGlobal("fetch", …)` + * BEFORE mounting the plugin — its fetch patch captures `globalThis.fetch` at + * effect time (`installFetchPatch` in `src/lifecycle.ts`) — and restore it with + * `vi.unstubAllGlobals()` AFTER disposal, or the patch's own restore check + * leaves the stub in place. + * + * Every builder returns a `dispose` that the caller runs in a `finally` or an + * `afterEach`. + * + * @module test/support/host + */ +import { + mkdirSync, + mkdtempSync, + readFileSync, + realpathSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { Context, type Fiber } from "@deepseek-ai/cordis"; +import type { Entry, ModuleLoaderV2 } from "@deepseek-ai/cordis-plugin-loader"; +import { + boot, + initProfile, + type ProfileContext, + readProfilePatches, +} from "@deepseek-ai/dsh-app-boot"; +import { ConfigEditor } from "@deepseek-ai/dsh-config-editor"; +import { LlmRuntime } from "@deepseek-ai/dsh-llm"; +import * as LlmPiAi from "@deepseek-ai/dsh-llm-pi-ai"; +import Settings from "@deepseek-ai/dsh-settings"; + +import { apply, Config, inject, name } from "./src.ts"; + +/** The package name the plugin looks pi-ai up by (`PI_AI_PACKAGE` in `src/responses-provider.ts`). */ +export const PI_AI_PACKAGE = "@deepseek-ai/dsh-llm-pi-ai"; + +/** The `llm` service as the tests read it, private registry included. */ +export type HostLlm = Context["llm"]; + +/** One registry row, as `LlmRuntime`'s private `adapters` map holds it. */ +export interface RegistryRow { + readonly adapter: { + readonly constructor: { readonly name: string }; + readonly [key: string]: unknown; + }; + readonly provider: { readonly id: string; readonly name: string }; + readonly retryPolicy: unknown; +} + +/** + * The registry map. Private in the type, readable at runtime, and the only way + * to see a route `listProviders()` hides. + */ +export const adaptersOf = (llm: HostLlm): Map => + (llm as unknown as { adapters: Map }).adapters; + +// --------------------------------------------------------------------------- +// Fakes +// --------------------------------------------------------------------------- + +/** One call the fake credentials service answered, values never included. */ +export type CredentialCall = readonly [method: string, key: string]; + +/** What {@link fakeCredentials} is seeded with. */ +export interface CredentialSeed { + /** + * A log shared with {@link fakeLaunchEnvironment}: each call appends + * `:` here too, so a test can order a credentials call against + * an environment read on one clock. + */ + readonly sharedLog?: string[]; + /** Reference values by name, e.g. `{ OPENCODE_API_KEY: "k" }`. */ + readonly refs?: Readonly>; + /** Records by joined key, e.g. `{ "llm-pi-ai/opencode": { kind: "api-key", key: "k" } }`. */ + readonly records?: Readonly>; +} + +/** The fake, with the log a test reads. */ +export interface FakeCredentials { + readonly calls: CredentialCall[]; + readonly service: Record unknown>; +} + +/** + * A `credentials` service over two in-memory maps: references (the + * `OPENCODE_API_KEY` form) and records (the `llm-pi-ai/opencode` form). It + * implements the surface pi-ai and the settings forms read, and logs every + * method with its key, so a test can assert which key the host asked for. + * + * @param seed - the references and records it starts with. + * @returns the service and its call log. + */ +export const fakeCredentials = (seed: CredentialSeed = {}): FakeCredentials => { + const refs = new Map(Object.entries(seed.refs ?? {})); + const records = new Map(Object.entries(seed.records ?? {})); + const calls: CredentialCall[] = []; + const log = (method: string, key: string): void => { + calls.push([method, key]); + seed.sharedLog?.push(`${method}:${key}`); + }; + const service = { + resolve: async (ref: string) => { + log("resolve", ref); + const value = refs.get(ref); + return value === undefined || value.length === 0 + ? undefined + : { source: "fake", value }; + }, + describe: async (ref: string) => { + log("describe", ref); + const configured = (refs.get(ref) ?? "").length > 0; + return configured + ? { configured, source: "fake", writable: true } + : { configured, writable: true }; + }, + set: async (ref: string, value: string) => { + log("set", ref); + refs.set(ref, value); + }, + unset: async (ref: string) => { + log("unset", ref); + refs.delete(ref); + }, + readRecord: async (key: string) => { + log("readRecord", key); + return records.get(key); + }, + describeRecord: async (key: string) => { + log("describeRecord", key); + const record = records.get(key) as { kind?: string } | undefined; + return record === undefined + ? { configured: false, writable: true } + : { configured: true, kind: record.kind, writable: true }; + }, + listRecords: async () => { + log("listRecords", "*"); + return [...records.entries()].map(([key, record]) => ({ + key, + kind: (record as { kind?: string }).kind, + })); + }, + modifyRecord: async ( + key: string, + mutate: (current: unknown) => Promise + ) => { + log("modifyRecord", key); + const next = await mutate(records.get(key)); + if (next !== undefined) { + records.set(key, next); + } + return records.get(key); + }, + deleteRecord: async (key: string) => { + log("deleteRecord", key); + records.delete(key); + }, + }; + return { + calls, + service: service as unknown as FakeCredentials["service"], + }; +}; + +/** The fake launch environment, with the names it was asked for. */ +export interface FakeLaunchEnvironment { + readonly reads: string[]; + readonly snapshot: { + readonly get: ( + name: string + ) => { value: string; source: "process" } | undefined; + readonly getFrom: ( + name: string, + sources: readonly string[] + ) => { value: string; source: "process" } | undefined; + }; +} + +/** + * A launch-environment snapshot over a plain record, in the shape + * `launchEnvironmentOf` hands back (`get`, `getFrom`). It stands in for the + * launcher's snapshot so the host never falls back to `process.env`. + * + * @param values - the variables the "process" layer holds. + * @param sharedLog - a log shared with {@link fakeCredentials}; each read + * appends `env:`. + * @returns the snapshot and the log of names read. + */ +export const fakeLaunchEnvironment = ( + values: Readonly> = {}, + sharedLog?: string[] +): FakeLaunchEnvironment => { + const reads: string[] = []; + const lookup = (key: string) => { + reads.push(key); + sharedLog?.push(`env:${key}`); + const value = values[key]; + return value === undefined + ? undefined + : { source: "process" as const, value }; + }; + return { + reads, + snapshot: { + get: lookup, + getFrom: (key, sources) => + sources.includes("process") ? lookup(key) : undefined, + }, + }; +}; + +// --------------------------------------------------------------------------- +// The wire +// --------------------------------------------------------------------------- + +/** One request the recording fetch saw. Credential header VALUES are never kept. */ +export interface RecordedRequest { + readonly url: string; + readonly method: string; + readonly model: string | undefined; + /** Lower-cased names; a credential header's value is replaced by its length. */ + readonly headers: Readonly>; +} + +/** The hosts the recording fetch answers; everything else is refused. */ +const GATEWAY_HOSTS = new Set(["opencode.ai", "models.opencode.ai"]); + +/** Headers whose value is a secret: recorded by length only. */ +const SECRET_HEADERS = new Set([ + "authorization", + "x-api-key", + "x-goog-api-key", +]); + +/** The 401 every gateway request gets unless the caller supplies a responder. */ +export const offlineAuthResponse = (): Response => + Response.json( + { error: { message: "probe: offline", type: "AuthError" } }, + { status: 401 } + ); + +/** A fetch replacement and what it recorded. */ +export interface RecordingFetch { + readonly fetch: typeof fetch; + readonly requests: RecordedRequest[]; +} + +const urlOf = (input: RequestInfo | URL): string => { + if (typeof input === "string") { + return input; + } + return input instanceof URL ? input.href : input.url; +}; + +const recordHeaders = ( + input: RequestInfo | URL, + init: RequestInit | undefined +): Record => { + const merged = new Headers(input instanceof Request ? input.headers : {}); + for (const [key, value] of new Headers(init?.headers ?? {})) { + merged.set(key, value); + } + const out: Record = {}; + for (const [key, value] of merged) { + out[key] = SECRET_HEADERS.has(key) ? value.length : value; + } + return out; +}; + +const bodyTextOf = async ( + input: RequestInfo | URL, + init: RequestInit | undefined +): Promise => { + if (typeof init?.body === "string") { + return init.body; + } + if (init?.body instanceof Uint8Array) { + return new TextDecoder().decode(init.body); + } + if (input instanceof Request) { + try { + return await input.clone().text(); + } catch { + return undefined; + } + } + return undefined; +}; + +/** + * A fetch that records every gateway request and answers it offline. + * + * @param options.respond - the answer for a gateway request (default: a 401). + * @returns the fetch and its log. + */ +export const recordingFetch = ( + options: { + readonly respond?: (request: RecordedRequest) => Response; + } = {} +): RecordingFetch => { + const requests: RecordedRequest[] = []; + const respond = options.respond ?? offlineAuthResponse; + const replacement = async ( + input: RequestInfo | URL, + init?: RequestInit + ): Promise => { + const url = urlOf(input); + let host = ""; + try { + ({ hostname: host } = new URL(url)); + } catch { + host = ""; + } + if (!GATEWAY_HOSTS.has(host)) { + throw new TypeError( + `host harness: refused a request to ${host || "a non-URL input"}` + ); + } + const text = await bodyTextOf(input, init); + let model: string | undefined; + try { + const parsed: unknown = JSON.parse(text ?? ""); + const candidate = (parsed as { model?: unknown } | null)?.model; + model = typeof candidate === "string" ? candidate : undefined; + } catch { + model = undefined; + } + const request: RecordedRequest = { + headers: recordHeaders(input, init), + method: ( + init?.method ?? (input instanceof Request ? input.method : "GET") + ).toUpperCase(), + model, + url, + }; + requests.push(request); + return respond(request); + }; + return { fetch: replacement as typeof fetch, requests }; +}; + +// --------------------------------------------------------------------------- +// The S0 fixture +// --------------------------------------------------------------------------- + +/** + * The profile run S0 measured: on each plane one known model per protocol plus + * the models pi 1.1.0 lacks, and a sibling pi-ai route that is not a plane. + * `deepseek`'s base is loopback port 9 (discard), so nothing reaches it. + */ +export const S0_PROFILE = { + deepseek: { + apiKeyEnv: "DEEPSEEK_API_KEY", + baseURL: "http://127.0.0.1:9/v1", + }, + opencode: { + apiKeyEnv: "OPENCODE_API_KEY", + models: [ + { id: "deepseek-v4-pro" }, + { id: "gpt-5.4" }, + { id: "claude-haiku-5-5" }, + { id: "gemini-3-flash" }, + { contextWindow: 400_000, id: "gpt-5.3-codex-spark", name: "My Spark" }, + ], + }, + "opencode-go": { + apiKeyEnv: "OPENCODE_GO_API_KEY", + models: [ + { id: "deepseek-v4-pro" }, + { id: "muse-spark-1.2-contributor" }, + { id: "minimax-m3" }, + { id: "gemini-4-argon" }, + { id: "minimax-m3.1-flash" }, + ], + }, +} as const; + +/** + * A DECLARED id that neither pi 1.1.0 nor the plugin's catalog describes (G1's + * second class, beside the gap ids the plugin's catalog still describes). + */ +export const UNKNOWN_ID = "not-in-any-catalog"; + +/** + * The characterization profile: {@link S0_PROFILE} with {@link UNKNOWN_ID} + * declared LAST on each plane, so the strict check still names each plane's + * stored gap id first. The contract tests keep plain S0, whose rows the + * declared unknown id would only blur. + */ +export const CHARACTERIZED_PROFILE = { + ...S0_PROFILE, + opencode: { + ...S0_PROFILE.opencode, + models: [...S0_PROFILE.opencode.models, { id: UNKNOWN_ID }], + }, + "opencode-go": { + ...S0_PROFILE["opencode-go"], + models: [...S0_PROFILE["opencode-go"].models, { id: UNKNOWN_ID }], + }, +} as const; + +/** The plane routes pi-ai serves today. */ +export const PLANES = ["opencode", "opencode-go"] as const; +export type Plane = (typeof PLANES)[number]; + +/** A fresh, mutable copy of {@link S0_PROFILE}. */ +export const s0Providers = (): Record> => + structuredClone(S0_PROFILE) as unknown as Record< + string, + Record + >; + +/** A fresh, mutable copy of {@link CHARACTERIZED_PROFILE}. */ +export const characterizedProviders = (): Record< + string, + Record +> => + structuredClone(CHARACTERIZED_PROFILE) as unknown as Record< + string, + Record + >; + +/** The characterized model ids per plane, in declaration order. */ +export const CHARACTERIZED_MODELS: Readonly> = + { + opencode: CHARACTERIZED_PROFILE.opencode.models.map((model) => model.id), + "opencode-go": CHARACTERIZED_PROFILE["opencode-go"].models.map( + (model) => model.id + ), + }; + +/** + * How a characterization row is tagged, so a later phase flips named rows: + * `KNOWN_DEFECT` is wrong today and a phase fixes it, `INTENDED_CHANGE` is + * right today and a phase changes it on purpose, `BASELINE` must survive. + */ +export type RowTag = "BASELINE" | "INTENDED_CHANGE" | "KNOWN_DEFECT"; + +/** + * The three facets a row records, each tagged on its own: one model can be + * right on the wire and wrong in validation (Zen gpt-5.3-codex-spark is). + * `info` is `context` + `efforts`, `callConfig` the verdict, `wire` is + * `sent` + `finish`. + */ +export type RowFacet = "callConfig" | "info" | "wire"; + +/** What TODAY's plugin does with one characterized model on pi 1.1.0, as measured. */ +export interface TodayRow { + readonly tags: Readonly>; + /** + * `resolveModelInfo`: its context window and effort ids in order, or the + * error code it rejects with. + */ + readonly info: + | { readonly context: number; readonly efforts: readonly string[] } + | { readonly error: string }; + /** `resolveCallConfig`: `"ok"`, or the error code it rejects with. */ + readonly callConfig: "ok" | "INVALID_CONFIG"; + /** The gateway URLs one turn sent, in order (the catalog read excluded). */ + readonly sent: readonly string[]; + /** The code on the turn's final `finish` chunk. */ + readonly finish: string; +} + +const ZEN = "https://opencode.ai"; + +/** Shorthand for a row's facet tags. */ +const tags = ( + info: RowTag, + callConfig: RowTag, + wire: RowTag +): Readonly> => ({ callConfig, info, wire }); + +const B = "BASELINE"; +const D = "KNOWN_DEFECT"; +const I = "INTENDED_CHANGE"; + +/** + * TODAY's plugin (hidden routes, listing patch, adapter decoration and all) on + * pi 1.1.0 with {@link CHARACTERIZED_PROFILE} and a key stored for each plane, + * measured in tier B with the published host's apply order (`plugin-first`, + * see {@link ApplyOrder}) and shared with the binary e2e so the two cannot + * drift. Each facet carries its own tag, and the characterization generates + * one test per row and facet titled with it: P-R1 flips the `KNOWN_DEFECT` and + * `INTENDED_CHANGE` facets its PR body names, and no `BASELINE` facet moves — + * except that P-R2's measured family flip, if a `modern` cell fails, may move + * the endpoint-family part of a `BASELINE` wire facet, re-tagging each one it + * moves and naming it in its PR body. + * + * `context` and `efforts` are copied from the generated catalog + * (`src/catalog-data.ts`), which the nightly refresh rewrites. A fixture-sanity + * case in `test/host-characterization.test.ts` compares each row's context AND + * effort ids to the bundled catalog's derivation, so a refresh that moves a + * fixture model fails there by name — update the row here. A phase that + * moves a non-`BASELINE` `info` facet off the catalog on purpose (D17's user + * row, D1's toggle efforts) adds that row's key to the case's + * `INFO_NOT_FROM_CATALOG` set in the same diff. The binary e2e + * reads the same table, but its job runs on a PR only when this file is in + * its path scope (`.github/workflows/ci.yml`, `e2e-host`), which it is. + */ +export const S0_TODAY = { + /** `listModels` before any turn: the plugin sees no key, so it hides paid models. */ + keylessListing: { opencode: [], "opencode-go": [] }, + /** `listModels` once a turn captured a key: pi-resolved ids only. */ + keyedListing: { + opencode: [ + "deepseek-v4-pro", + "gpt-5.4", + "claude-haiku-5-5", + "gemini-3-flash", + ], + "opencode-go": [ + "deepseek-v4-pro", + "muse-spark-1.2-contributor", + "minimax-m3", + ], + }, + rows: { + "opencode/deepseek-v4-pro": { + callConfig: "ok", + info: { context: 1_000_000, efforts: ["high", "max"] }, + finish: "AUTH", + sent: [`${ZEN}/zen/v1/chat/completions`], + tags: tags(B, B, I), + }, + "opencode/gpt-5.4": { + callConfig: "ok", + info: { context: 1_050_000, efforts: ["low", "medium", "high", "xhigh"] }, + finish: "AUTH", + sent: [`${ZEN}/inference/openai/v1/responses`], + tags: tags(B, B, B), + }, + "opencode/claude-haiku-5-5": { + callConfig: "ok", + info: { + context: 1_000_000, + efforts: ["low", "medium", "high", "xhigh", "max"], + }, + finish: "AUTH", + sent: [`${ZEN}/inference/anthropic/v1/messages?beta=true`], + tags: tags(B, B, B), + }, + "opencode/gemini-3-flash": { + callConfig: "ok", + info: { + context: 1_048_576, + efforts: ["minimal", "low", "medium", "high"], + }, + finish: "PI_AI_ERROR", + sent: [ + `${ZEN}/inference/google/v1beta/models/gemini-3-flash:streamGenerateContent?alt=sse`, + ], + tags: tags(B, B, B), + }, + // The user's row (name "My Spark", contextWindow 400000) is ignored. + "opencode/gpt-5.3-codex-spark": { + callConfig: "INVALID_CONFIG", + info: { context: 128_000, efforts: ["low", "medium", "high", "xhigh"] }, + finish: "AUTH", + sent: [`${ZEN}/inference/openai/v1/responses`], + tags: tags(D, D, B), + }, + "opencode/not-in-any-catalog": { + callConfig: "INVALID_CONFIG", + finish: "INVALID_CONFIG", + info: { error: "INVALID_CONFIG" }, + sent: [], + tags: tags(D, D, D), + }, + "opencode-go/deepseek-v4-pro": { + callConfig: "ok", + info: { context: 1_000_000, efforts: ["high", "max"] }, + finish: "AUTH", + sent: [`${ZEN}/zen/go/v1/chat/completions`], + tags: tags(B, B, I), + }, + "opencode-go/muse-spark-1.2-contributor": { + callConfig: "ok", + info: { + context: 1_048_576, + efforts: ["minimal", "low", "medium", "high", "xhigh"], + }, + finish: "AUTH", + sent: [`${ZEN}/inference/go/openai/v1/responses`], + tags: tags(B, B, B), + }, + // A toggle-only model offered Minimal/Xhigh/Max (D1). + "opencode-go/minimax-m3": { + callConfig: "ok", + info: { + context: 1_000_000, + efforts: ["minimal", "low", "medium", "high", "xhigh", "max"], + }, + finish: "AUTH", + sent: [`${ZEN}/inference/go/anthropic/v1/messages?beta=true`], + tags: tags(D, B, B), + }, + "opencode-go/gemini-4-argon": { + callConfig: "INVALID_CONFIG", + info: { context: 1_048_576, efforts: ["low", "medium", "high"] }, + finish: "PI_AI_ERROR", + sent: [ + `${ZEN}/inference/go/google/v1beta/models/gemini-4-argon:streamGenerateContent?alt=sse`, + ], + tags: tags(B, D, B), + }, + "opencode-go/minimax-m3.1-flash": { + callConfig: "INVALID_CONFIG", + info: { + context: 1_000_000, + efforts: ["low", "medium", "high", "xhigh", "max"], + }, + finish: "INVALID_CONFIG", + sent: [], + tags: tags(B, D, D), + }, + "opencode-go/not-in-any-catalog": { + callConfig: "INVALID_CONFIG", + finish: "INVALID_CONFIG", + info: { error: "INVALID_CONFIG" }, + sent: [], + tags: tags(D, D, D), + }, + } satisfies Record, + /** + * What appending a new id to `opencode` answers while the plane stores a gap + * id: the strict check names the STORED gap id, whatever is being saved. + */ + saveBlockedByStoredGap: + /provider "opencode" model "gpt-5\.3-codex-spark" needs an api/u, + /** What saving a catalog-unknown id answers once the plane stores no gap. */ + saveRejectedUnknown: + /provider "opencode" model "not-a-real-model" needs an api/u, +} as const; + +/** The id the save rows try to add: unknown to every catalog, and not declared. */ +export const SAVE_CANDIDATE = "not-a-real-model"; + +/** + * The ids on `opencode` that pi 1.1.0 cannot place: the stored gap id and the + * declared unknown id. The save sequence (in-process, and the probe's + * `PROBE_SAVE_DROP`) removes them before saving {@link SAVE_CANDIDATE}. + */ +export const ZEN_UNDESCRIBED = ["gpt-5.3-codex-spark", UNKNOWN_ID] as const; + +/** + * TODAY's rows for an apply order. `S0_TODAY.rows` is the `plugin-first` + * table; with `pi-ai-first` the adapter decoration wraps pi-ai's plane adapter + * and exactly one facet moves — Go gemini-4-argon validates — + * measured in `test/host-characterization.test.ts`, which generates one test + * per row and facet for each order, titled with that order's tags. The moved + * facet is `BASELINE` in this table: its `"ok"` is already what P-R1 keeps, so + * there is nothing for a phase to flip in this order. The binary e2e picks the + * table from the order its probe INFERRED — whether the decoration wrapped + * pi-ai's plane adapter, which is what the rows depend on — because on a real + * host the order is a race (F17) the suite must not pin. + * + * @param order - the apply order, as inferred from the decoration. + * @returns the rows, keyed `/`. + */ +export const todayRowsFor = ( + order: ApplyOrder +): Readonly> => { + if (order === "plugin-first") { + return S0_TODAY.rows; + } + // Exhaustive on purpose: a probe JSON with a missing or renamed field reads + // `undefined` here, and must not fall through to the pi-ai-first table. + if (order !== "pi-ai-first") { + throw new Error(`todayRowsFor: unknown apply order ${String(order)}`); + } + return { + ...S0_TODAY.rows, + "opencode-go/gemini-4-argon": { + ...S0_TODAY.rows["opencode-go/gemini-4-argon"], + callConfig: "ok", + tags: { + ...S0_TODAY.rows["opencode-go/gemini-4-argon"].tags, + callConfig: B, + }, + }, + }; +}; + +/** + * Every {@link ApplyOrder}, for iterating {@link todayRowsFor}'s tables. Built + * from a `Record` keyed by the union, so a member added to `ApplyOrder` and + * missing here fails to compile rather than going unchecked. + */ +export const APPLY_ORDERS = Object.keys({ + "pi-ai-first": true, + "plugin-first": true, +} satisfies Record) as readonly ApplyOrder[]; + +/** + * What appending a new id to `opencode` answers while the plane stores ANY of + * {@link ZEN_UNDESCRIBED}, whichever one the installed pi cannot describe + * first. The pinned pi always names `gpt-5.3-codex-spark` + * (`S0_TODAY.saveBlockedByStoredGap`); a floating pi that learned it names the + * next one, so the floating cell asserts this instead. + */ +export const SAVE_BLOCKED_BY_ANY_UNDESCRIBED = new RegExp( + `provider "opencode" model "(?:${ZEN_UNDESCRIBED.map((id) => + id.replaceAll(".", String.raw`\.`) + ).join("|")})" needs an api`, + "u" +); + +const PI_DATA = new URL( + "../../node_modules/@earendil-works/pi-ai/dist/providers/data/", + import.meta.url +); + +const piData = (plane: Plane): Record> => + JSON.parse( + readFileSync( + new URL( + plane === "opencode" ? "opencode.json" : "opencode-go.json", + PI_DATA + ), + "utf8" + ) + ) as Record>; + +/** + * The protocol the installed pi's catalog files a model under on a plane, or + * `undefined` when pi does not know it. The data is keyed by protocol, then + * `chat:`, so a fixture can check which of its ids are gaps on pi 1.1.0. + * + * @param plane - the plane route. + * @param id - the model id. + * @returns the protocol, or `undefined` for a gap. + */ +export const piKnows = (plane: Plane, id: string): string | undefined => + Object.entries(piData(plane)).find(([, models]) => + Object.hasOwn(models, `chat:${id}`) + )?.[0]; + +// --------------------------------------------------------------------------- +// Waterfall helpers +// --------------------------------------------------------------------------- + +/** + * A fresh deep copy of a raw pi-ai config without the two plane entries. Pure: + * it never touches its input, so it is safe on a frozen one. + * + * @param raw - the raw config, `{ providers }`. + * @returns the copy. + */ +export const withoutPlanes = (raw: T): T => { + const copy = structuredClone(raw) as { providers?: Record }; + if (copy.providers !== undefined && copy.providers !== null) { + delete copy.providers.opencode; + delete copy.providers["opencode-go"]; + } + return copy as T; +}; + +/** Deep-freeze a value and everything under it. */ +export const deepFreeze = (value: T): T => { + if (value !== null && typeof value === "object") { + for (const child of Object.values(value)) { + deepFreeze(child); + } + Object.freeze(value); + } + return value; +}; + +/** One `internal/config` run the spy saw. */ +export interface WaterfallRecord { + readonly self: Fiber; + readonly raw: unknown; + /** `raw === this._config` at the moment the listener ran (F14). */ + readonly isCommit: boolean; + readonly order: number; +} + +/** + * Record every `internal/config` run for one fiber, from a listener prepended + * on `ctx` (so it is outermost among those registered before it). + * + * @param ctx - the context to register on (the root, or another plugin's). + * @param fiber - the fiber whose runs are kept; omit to keep every run. + * @returns the records, and the disposer. + */ +export const waterfallSpy = ( + ctx: Context, + fiber?: () => Fiber | undefined +): { records: WaterfallRecord[]; dispose: ReturnType } => { + const records: WaterfallRecord[] = []; + const dispose = ctx.on( + "internal/config", + function recordRun(this: Fiber, raw: unknown, next: () => unknown) { + const target = fiber?.(); + if (target === undefined || this === target) { + records.push({ + isCommit: raw === (this as unknown as { _config: unknown })._config, + order: records.length, + raw, + self: this, + }); + } + return next(); + }, + { prepend: true } + ); + return { dispose, records }; +}; + +/** + * Poll the registry until every route is present: the plugin mounts its + * hidden routes asynchronously (`mountInternalRoutes` in + * `src/lifecycle.ts`). + * + * @param llm - the `llm` service. + * @param ids - the routes to wait for. + * @param deadlineMs - how long to wait before failing. The default sits well + * inside the 30 s a tier-B hook gets for the boot AND this wait, so a route + * that never registers fails by name rather than as a hook timeout. + */ +export const waitForRoutes = async ( + llm: HostLlm, + ids: readonly string[], + deadlineMs = 10_000 +): Promise => { + const until = Date.now() + deadlineMs; + for (;;) { + const present = adaptersOf(llm); + const missing = ids.filter((id) => !present.has(id)); + if (missing.length === 0) { + return; + } + if (Date.now() > until) { + throw new Error( + `host harness: routes never registered: ${missing.join(", ")}` + ); + } + await new Promise((resolve) => { + setTimeout(resolve, 20); + }); + } +}; + +/** One drained `llm.stream` call: the chunk types and the final finish. */ +export interface DrainedStream { + readonly types: string[]; + readonly finish: + | { readonly kind: string | undefined; readonly code: string | undefined } + | undefined; + readonly error: string | undefined; +} + +/** + * Drive one turn through the host's own stream path and keep what it said. + * + * @param llm - the `llm` service. + * @param provider - the route. + * @param model - the model id. + * @returns the drained stream. + */ +export const drainStream = async ( + llm: HostLlm, + provider: string, + model: string +): Promise => { + const types: string[] = []; + let finish: DrainedStream["finish"]; + let failed: string | undefined; + try { + for await (const chunk of llm.stream({ + messages: [{ content: [{ text: "hi", type: "text" }], role: "user" }], + model, + provider, + signal: AbortSignal.timeout(10_000), + } as never)) { + const typed = chunk as { + type: string; + reason?: { kind?: string; failure?: { code?: string } }; + }; + types.push(typed.type); + if (typed.type === "finish") { + finish = { + code: typed.reason?.failure?.code, + kind: typed.reason?.kind, + }; + } + } + } catch (error) { + const failure = error as { code?: unknown; message?: unknown }; + failed = `${typeof failure.code === "string" ? `${failure.code} ` : ""}${String(failure.message ?? error)}`; + } + return { error: failed, finish, types }; +}; + +// --------------------------------------------------------------------------- +// Tier A +// --------------------------------------------------------------------------- + +/** What {@link mountTierA} takes. */ +export interface TierAOptions { + /** pi-ai's `providers` dict. */ + readonly providers: Record; + /** pi-ai's whole raw config, used as given instead of `{ providers }` (a frozen input). */ + readonly piConfig?: object; + /** + * Mount this plugin too, with this row config. Stub `fetch` first (its fetch + * patch captures `globalThis.fetch` at effect time). Tier A has no loader, + * so the plugin cannot find pi-ai's entry: it applies and patches fetch, but + * mounts NO hidden route and no Google bridge: the bridge's `PiAiAdapter` + * comes from the loader's pi-ai entry (`loadPiAiAdapterHost`), which tier A + * cannot answer. `test/host-harness.test.ts` pins both — the stand-down line + * and empty hidden routes, and `loadPiAiAdapterHost` answering `undefined`. + * Characterize the real plugin in tier B. + */ + readonly plugin?: Record; + readonly credentials?: FakeCredentials; + readonly launchEnvironment?: FakeLaunchEnvironment; + /** Runs on the root before pi-ai mounts (a listener to register first). */ + readonly before?: (ctx: Context) => void; +} + +/** A mounted tier A. */ +export interface TierA { + readonly ctx: Context; + readonly llm: HostLlm; + readonly piFiber: Fiber; + readonly pluginFiber: Fiber | undefined; + readonly credentials: FakeCredentials; + readonly launchEnvironment: FakeLaunchEnvironment; + readonly dispose: () => Promise; +} + +/** + * A bare `Context` with `LlmRuntime` and pi-ai mounted as plugins. + * + * @param options - the providers, and the optional plugin config and fakes. + * @returns the mount; the caller disposes it. + */ +export const mountTierA = async (options: TierAOptions): Promise => { + const ctx = new Context(); + const credentials = options.credentials ?? fakeCredentials(); + const launchEnvironment = + options.launchEnvironment ?? fakeLaunchEnvironment(); + try { + ctx.provide("credentials", credentials.service); + ctx.provide("launchEnvironment", launchEnvironment.snapshot); + await ctx.plugin(LlmRuntime); + options.before?.(ctx); + const piFiber: Fiber = await ctx.plugin( + LlmPiAi as never, + (options.piConfig ?? { providers: options.providers }) as never + ); + let pluginFiber: Fiber | undefined; + if (options.plugin !== undefined) { + pluginFiber = await ctx.plugin( + { apply, Config, inject, name } as never, + options.plugin as never + ); + } + return { + credentials, + ctx, + dispose: async () => { + await ctx.fiber.dispose(); + }, + launchEnvironment, + llm: ctx.llm, + piFiber, + pluginFiber, + }; + } catch (error) { + await ctx.fiber.dispose(); + throw error; + } +}; + +// --------------------------------------------------------------------------- +// Tier B +// --------------------------------------------------------------------------- + +/** The modules the stub loader resolves, by package name. */ +const HOST_MODULES: ReadonlyMap = new Map([ + ["@deepseek-ai/dsh-llm", LlmRuntime], + [PI_AI_PACKAGE, LlmPiAi], + ["@deepseek-ai/dsh-config-editor", ConfigEditor], + ["@deepseek-ai/dsh-settings", Settings], + [name, { apply, Config, inject, name }], +]); + +/** + * The loader's module seam, answering only the five host rows. Any other + * import is a composition this harness did not intend, so it throws. A value + * may be a promise, which is how {@link orderedModules} holds one row back. + * + * @param modules - the namespaces (or promises of them) by specifier. + * @returns the `ModuleLoaderV2` to put on `root.loader.internal`. + */ +export const stubModuleLoader = ( + modules: ReadonlyMap = HOST_MODULES +): ModuleLoaderV2 => ({ + getOrCreateModuleJob(): never { + throw new Error("unexpected module job creation"); + }, + import: async (specifier: string) => { + if (!modules.has(specifier)) { + throw new Error(`unexpected Loader import: ${specifier}`); + } + return await modules.get(specifier); + }, + load(): never { + throw new Error("unexpected module load"); + }, + loadCache: new Map() as never, + register(): never { + throw new Error("unexpected module hook registration"); + }, + resolveSync(): never { + throw new Error("unexpected synchronous module resolution"); + }, + version: "v2", +}); + +/** + * Which of pi-ai and this plugin APPLIES first. The loader starts rows in + * parallel (F17), so on a real host the order is whatever module loading makes + * it. Measured on the published binary: the plugin applies first, about 70 ms + * before pi-ai registers its routes, because pi-ai's module is the heavier + * import. The stub loader resolves every import at once, so the harness makes + * the order explicit instead of inheriting row order. + */ +export type ApplyOrder = "pi-ai-first" | "plugin-first"; + +/** The fiber states a row settles in: ACTIVE, FAILED, DISPOSED (cordis `FiberState`). */ +const SETTLED_STATES: ReadonlySet = new Set([2, 3, 4]); + +/** {@link orderedModules}' answer: the modules, and the hook that arms the fallback release. */ +interface OrderedModules { + readonly modules: ReadonlyMap; + /** Register, on the root, the release for a first row that never applies. */ + readonly watch: (root: Context) => void; +} + +/** + * The host modules with one of the two rows held back until the other's + * `apply` has returned. + * + * A first row can also settle WITHOUT applying: cordis resolves a fiber's + * config before it runs `apply` (`fiber.ts` `_reload`), so a config the + * schema rejects leaves the fiber FAILED and `apply` never called. The held + * import would then never resolve, the loader would wait on it for ever, and + * `boot()` with it. So `watch` also releases the held row as soon as the first + * row's fiber reports a settled state (`internal/status`), and the boot + * finishes with that fiber FAILED — its error readable through + * `fiber.await()`. A row that never leaves PENDING (a required service that + * never appears) still holds the second one; no row here injects one. + * + * @param order - which applies first. + * @returns the modules for {@link stubModuleLoader}, and the fallback hook. + */ +const orderedModules = (order: ApplyOrder): OrderedModules => { + const { promise: released, resolve: release } = + Promise.withResolvers(); + const [first, second] = + order === "plugin-first" ? [name, PI_AI_PACKAGE] : [PI_AI_PACKAGE, name]; + const firstModule = HOST_MODULES.get(first) as { + apply: (ctx: unknown, config: unknown) => unknown; + }; + const held = async (): Promise => { + await released; + return HOST_MODULES.get(second); + }; + const firstApply = (ctx: unknown, config: unknown): unknown => { + try { + return firstModule.apply(ctx, config); + } finally { + release(true); + } + }; + return { + modules: new Map([ + ...HOST_MODULES, + [first, { ...firstModule, apply: firstApply }], + [second, held()], + ]), + watch: (root) => { + root.on("internal/status", (fiber: Fiber) => { + // The registry keys a runtime by its plugin's `apply`, so this is the + // first row's fiber and no other. + if ( + fiber.runtime?.callback === firstApply && + SETTLED_STATES.has(fiber.state) + ) { + release(true); + } + }); + }, + }; +}; + +/** The bundle rows, in the ids the real host's `dsh-base` patch names them. */ +const BUNDLE_ROWS = [ + { id: "llm", name: "@deepseek-ai/dsh-llm" }, + { id: "llm-pi-ai", name: PI_AI_PACKAGE }, + { id: "config-editor", name: "@deepseek-ai/dsh-config-editor" }, + { id: "settings", name: "@deepseek-ai/dsh-settings" }, + { id: name, name }, +]; + +/** What {@link bootTierB} takes. */ +export interface TierBOptions { + /** pi-ai's `providers` dict, written to the profile patch. */ + readonly providers?: Record; + /** Mount this plugin's row (default `true`); host-only contracts leave it out. */ + readonly plugin?: boolean; + /** A profile row for this plugin's config; omitted, the shipped defaults apply. */ + readonly pluginConfig?: Record; + /** The profile patch as raw YAML text, replacing the generated one (for `!!js`). */ + readonly profilePatchYaml?: string; + readonly credentials?: FakeCredentials; + readonly launchEnvironment?: FakeLaunchEnvironment; + /** Runs on the root after the fakes are provided and before any row mounts. */ + readonly prepare?: (root: Context) => void; + /** + * Which of pi-ai and the plugin applies first (default `"plugin-first"`, the + * order measured on the published binary). Ignored without the plugin row. + */ + readonly order?: ApplyOrder; +} + +/** A booted tier B. */ +export interface TierB { + readonly ctx: Context; + readonly home: string; + readonly profile: ProfileContext; + readonly llm: HostLlm; + readonly piEntry: Entry; + readonly piFiber: Fiber; + /** The plugin's entry; `undefined` when the boot left the row out. */ + readonly pluginEntry: Entry | undefined; + readonly credentials: FakeCredentials; + readonly dispose: () => Promise; +} + +/** The keys the plugin reads from the environment directly. */ +export const AMBIENT_KEYS = [ + "OPENCODE_API_KEY", + "OPENCODE_GO_API_KEY", +] as const; + +/** + * Blank the keys the plugin reads directly (`hasPlaneCredential` in + * `src/lifecycle.ts`) for the life of one boot, so a contributor's exported + * key cannot turn the keyless rows into keyed ones. An empty value is what the + * plugin reads as "no key". Only a key that IS set is touched, and the + * returned restore puts back exactly what was there. + * + * @returns the restore, which the boot's dispose (and its failure path) runs. + */ +const blankAmbientKeys = (): (() => void) => { + const saved = AMBIENT_KEYS.filter( + (key) => (process.env[key] ?? "").length > 0 + ).map((key) => [key, process.env[key]] as const); + for (const [key] of saved) { + process.env[key] = ""; + } + return () => { + for (const [key, value] of saved) { + process.env[key] = value; + } + }; +}; + +/** The profile patch the S0-style boots write, as JSON text (valid YAML). */ +const profilePatchText = ( + providers: Record | undefined, + pluginConfig: Record | undefined +): string => + `${JSON.stringify( + [ + ...(providers === undefined + ? [] + : [{ config: { providers }, id: "llm-pi-ai" }]), + ...(pluginConfig === undefined + ? [] + : [{ config: pluginConfig, id: name }]), + ], + null, + 2 + )}\n`; + +/** + * Boot the host the way the harness's own composition tests do, with this + * plugin's row mounted as the real host mounts it. + * + * @param options - the providers, the plugin row and the fakes. + * @returns the booted host; the caller disposes it. + */ +export const bootTierB = async (options: TierBOptions = {}): Promise => { + const credentials = options.credentials ?? fakeCredentials(); + const launchEnvironment = + options.launchEnvironment ?? fakeLaunchEnvironment(); + const home = realpathSync(mkdtempSync(join(tmpdir(), "dsh-opencode-host-"))); + const restoreKeys = blankAmbientKeys(); + let ctx: Context | undefined; + try { + const dir = join(home, "profiles", "test"); + initProfile(dir, ["test-bundle"]); + const bundle = join(dir, "node_modules", "test-bundle"); + mkdirSync(bundle, { recursive: true }); + writeFileSync(join(home, "package.json"), '{"name":"test-installation"}\n'); + writeFileSync( + join(bundle, "package.json"), + JSON.stringify({ + dsh: { bundle: { patch: "cordis.patch.yml" } }, + name: "test-bundle", + version: "1.0.0", + }) + ); + writeFileSync( + join(bundle, "cordis.patch.yml"), + `${JSON.stringify( + [ + { + insert: BUNDLE_ROWS.filter( + (row) => options.plugin !== false || row.id !== name + ), + }, + ], + null, + 2 + )}\n` + ); + writeFileSync(join(dir, "cordis.yml"), "[]\n"); + const patchPath = join(dir, "cordis.patch.yml"); + writeFileSync( + patchPath, + options.profilePatchYaml ?? + profilePatchText(options.providers, options.pluginConfig) + ); + const profile: ProfileContext = { + cwd: home, + dir, + home, + installAnchor: join(home, "package.json"), + name: "test", + overlays: [], + patchPath, + startedBundles: ["test-bundle"], + telemetryDisabledEnv: undefined, + }; + ctx = await boot( + "dsh", + join(dir, "cordis.yml"), + readProfilePatches("dsh", profile), + (root) => { + root.provide("profileContext", profile); + root.provide("credentials", credentials.service); + root.provide("launchEnvironment", launchEnvironment.snapshot); + if (options.plugin === false) { + root.loader.internal = stubModuleLoader(HOST_MODULES); + } else { + const ordered = orderedModules(options.order ?? "plugin-first"); + ordered.watch(root); + root.loader.internal = stubModuleLoader(ordered.modules); + } + options.prepare?.(root); + } + ); + const booted = ctx; + const entries = booted.configEditor.entries(); + const entryFor = (id: string): Entry => { + const found = entries.find((entry) => entry.options.id === id); + if (found?.fiber === undefined) { + throw new Error(`host harness: no mounted entry ${id}`); + } + return found; + }; + const piEntry = entryFor("llm-pi-ai"); + const pluginEntry = options.plugin === false ? undefined : entryFor(name); + return { + credentials, + ctx: booted, + dispose: async () => { + try { + await booted.fiber.dispose(); + } finally { + rmSync(home, { force: true, recursive: true }); + restoreKeys(); + } + }, + home, + llm: booted.llm, + piEntry, + get piFiber() { + return piEntry.fiber!; + }, + pluginEntry, + profile, + }; + } catch (error) { + try { + await ctx?.fiber.dispose(); + } finally { + rmSync(home, { force: true, recursive: true }); + restoreKeys(); + } + throw error; + } +}; + +/** Only the two plane keys stored: what a user who pasted them into Settings has. */ +export const planeKeysOnly = (): FakeCredentials => + fakeCredentials({ + refs: { OPENCODE_API_KEY: "zen-test", OPENCODE_GO_API_KEY: "go-test" }, + }); + +/** + * `credentialRef`'s grammar (`credentials/credentials/src/index.ts:19,29-34`): + * a POSIX shell identifier. Restated because `@deepseek-ai/dsh-credentials` is + * only a transitive dependency here. + */ +const CREDENTIAL_REF_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/; + +/** + * session-controller's `hasProviderApiKey` (`api/session-controller/src/catalog.ts:97-116`), + * restated from the real parts because no installed copy reaches it: the + * published `@deepseek-ai/dsh-api-session-controller` entry exports + * `SessionController`, `buildModelCatalog` and a few classes but not this + * function, its tarball ships `lib/` and no `src/` (so the `./src/*` export + * resolves nothing), and the package is not a dependency here in any case. + * `initializeDefaultModel` (`index.ts:320-331`) calls it after a Sign-in and, + * when it answers false, replaces the user's default model with the account's + * first one. It answers the FIRST configurable provider whose profile, read + * from `settings.describe`'s VALUE at the provider's `settingsPath`, names an + * `apiKeyEnv` the credentials service reports as configured — so it is the + * reader F20 changes. + * + * It keeps upstream's `credentialRef(ref)` brand as a grammar check that + * throws the same `TypeError`, so a profile whose `apiKeyEnv` is outside the + * grammar fails here as it fails upstream rather than reading "not + * configured". The deliberate differences: it answers the provider id instead + * of `true`, and it omits upstream's `RemoteError` for a missing `settings` or + * `credentials` service, which tier B always provides. + * + * @param tier - the booted host (only `ctx.settings`, `llm` and + * `credentials.service` are read). + * @returns that provider's id (`true` upstream), or `undefined` (`false`). + * @throws {TypeError} for an `apiKeyEnv` outside `credentialRef`'s grammar. + */ +export const providerWithApiKey = async ( + tier: Pick +): Promise => { + const describeRef = tier.credentials.service.describe as unknown as ( + ref: string + ) => Promise<{ configured: boolean }>; + const namespaces = tier.ctx.settings.describe({ redactSecrets: true }); + for (const provider of tier.llm.listConfigurableProviders()) { + if (provider.provider === "deepseek-account") { + continue; + } + let profile: unknown = namespaces.find( + (namespace) => namespace.ns === provider.settingsNs + )?.value; + for (const key of provider.settingsPath) { + profile = + typeof profile === "object" && profile !== null + ? Reflect.get(profile, key) + : undefined; + } + if (typeof profile !== "object" || profile === null) { + continue; + } + const ref: unknown = Reflect.get(profile, "apiKeyEnv"); + if (typeof ref === "string" && ref.length > 0) { + if (!CREDENTIAL_REF_PATTERN.test(ref)) { + throw new TypeError( + `credential ref "${ref}" must match ${String(CREDENTIAL_REF_PATTERN)}` + ); + } + const described = await describeRef(ref); + if (described.configured) { + return provider.provider; + } + } + } + return undefined; +}; diff --git a/test/support/keyless-matrix.ts b/test/support/keyless-matrix.ts new file mode 100644 index 00000000..d83d6235 --- /dev/null +++ b/test/support/keyless-matrix.ts @@ -0,0 +1,295 @@ +/** + * The keyless 401-vs-404 probe matrix: which gateway paths still exist. + * + * Two planes × four protocols × two endpoint families, one representative + * PAID model per plane and protocol, every request sent WITHOUT a key. The gateway + * authenticates before it routes a model, so a keyless request tells the paths + * apart without spending anything: a 404 means the path is gone, a 401/403 + * that it exists, a 410 or a 400 "Model is unavailable" that the model is. + * `test/e2e/keyless-matrix.e2e.ts` sends the cells; P-R2 decides the default + * family from what they answer. + * + * Pure: every function here is a function of its arguments — no fetch, no + * environment, no module state — so the cell table and the classifier are + * unit-tested offline (`test/keyless-matrix.test.ts`). There is no code path + * here that can set `authorization`, `x-api-key` or `x-goog-api-key`. + * + * @module test/support/keyless-matrix + */ +import { + ANTHROPIC_VERSION, + type CatalogModelSpec, + LEGACY_PLANE_API, + OPENCODE_FAMILY_API, + protocolFor, +} from "./src.ts"; + +export type MatrixPlane = "opencode" | "opencode-go"; +export type MatrixFamily = "legacy" | "modern"; + +/** The four protocols, in the order cells are produced. */ +export const MATRIX_PROTOCOLS = [ + "openai-completions", + "openai-responses", + "anthropic-messages", + "google-generative-ai", +] as const; +export type MatrixProtocol = (typeof MATRIX_PROTOCOLS)[number]; + +/** One probe: what to send, and to where. */ +export interface MatrixCell { + readonly plane: MatrixPlane; + readonly protocol: MatrixProtocol; + readonly family: MatrixFamily; + /** `undefined` when the plane's catalog has no model for the protocol. */ + readonly model: string | undefined; + readonly url: string; + readonly method: "POST"; + readonly headers: Readonly>; + /** `undefined` exactly when `model` is. */ + readonly body: Readonly> | undefined; +} + +/** A cell's name: ` `, as the live table prints it. */ +export const cellKey = ( + cell: Pick +): string => `${cell.plane} ${cell.protocol} ${cell.family}`; + +/** + * The cells TODAY's plugin sends turns to with the default endpoint (`auto`, + * an alias of `modern`), on both planes: completions on pi 1.1.0's LEGACY + * bases (`/zen(/go)/v1/chat/completions`, pi-ai's own plane route) and the + * three hidden routes' MODERN paths (Responses, Messages, Google). A path-gone + * verdict on one of these breaks a shipped model in the default + * configuration, so the live matrix asserts them. `test/keyless-matrix.test.ts` + * derives the same set from the URLs `S0_TODAY` measured (with the default + * endpoint), so the two cannot drift. + * + * Not covered: a user who sets `endpoint: legacy` dispatches Responses and + * Messages on both planes, and Zen's Google, to the LEGACY cells, which the + * matrix records for P-R2 and does not assert; P-R2 decides whether that + * configuration is guarded. Go's Google is the exception: it stays on + * `/inference/go/google/v1beta` under every family (`GO_GOOGLE_BASE_URL` in + * `src/google-bridge.ts`), which is the modern cell this set already holds. + */ +export const TODAY_DISPATCH: ReadonlySet = new Set( + (["opencode", "opencode-go"] as const).flatMap((plane) => [ + cellKey({ family: "legacy", plane, protocol: "openai-completions" }), + cellKey({ family: "modern", plane, protocol: "openai-responses" }), + cellKey({ family: "modern", plane, protocol: "anthropic-messages" }), + cellKey({ family: "modern", plane, protocol: "google-generative-ai" }), + ]) +); + +/** + * The cells P-R1 moves traffic to that nothing dispatches to yet: completions + * on the modern family (`/inference(/go)/openai/v1`). The live matrix asserts + * them too, as P-R1's forward guard, and labels them so in its failure. + */ +export const NEXT_DISPATCH: ReadonlySet = new Set( + (["opencode", "opencode-go"] as const).map((plane) => + cellKey({ family: "modern", plane, protocol: "openai-completions" }) + ) +); + +/** What a probe answered, as the matrix reads it. */ +export type KeylessVerdict = + | "model-gone" + | "no-model" + | "other" + | "path-exists" + | "path-gone"; + +/** The catalog SDK each protocol is served for, for the modern family table. */ +const SDK_FOR: Readonly> = { + "anthropic-messages": "@ai-sdk/anthropic", + "google-generative-ai": "@ai-sdk/google", + "openai-completions": "", + "openai-responses": "@ai-sdk/openai", +}; + +/** The path each body-addressed protocol appends to its base. */ +const BODY_PATH: Readonly< + Record, string> +> = { + "anthropic-messages": "/messages", + "openai-completions": "/chat/completions", + "openai-responses": "/responses", +}; + +/** + * The path a protocol appends to its base. Google names the model in the path + * and streams; the three others name it in the body. + */ +const suffixFor = ( + protocol: MatrixProtocol, + model: string | undefined +): string => + protocol === "google-generative-ai" + ? `/models/${model ?? ""}:streamGenerateContent?alt=sse` + : BODY_PATH[protocol]; + +/** + * The URL a cell probes. + * + * The Go plane's legacy Google cell has no `?alt=sse`: pi 1.1.0 ships no Go + * Google provider, so that path is the bare one the plan names + * (`/zen/go/v1/models/:streamGenerateContent`), not a request pi makes. + */ +export const matrixUrl = ( + plane: MatrixPlane, + protocol: MatrixProtocol, + family: MatrixFamily, + model: string | undefined +): string => { + if (family === "legacy") { + // The legacy family: one base per plane, every protocol under its `/v1`. + const url = `${LEGACY_PLANE_API[plane]}${suffixFor(protocol, model)}`; + return plane === "opencode-go" && protocol === "google-generative-ai" + ? url.replace("?alt=sse", "") + : url; + } + const table = OPENCODE_FAMILY_API[plane === "opencode-go" ? "go" : "zen"]; + return `${table[SDK_FOR[protocol]] ?? ""}${suffixFor(protocol, model)}`; +}; + +/** The smallest valid request body for a protocol, carrying the model. */ +export const matrixBody = ( + protocol: MatrixProtocol, + model: string +): Readonly> => { + if (protocol === "openai-responses") { + return { input: "hi", max_output_tokens: 1, model }; + } + if (protocol === "google-generative-ai") { + return { + contents: [{ parts: [{ text: "hi" }], role: "user" }], + generationConfig: { maxOutputTokens: 1 }, + }; + } + // Chat completions and Messages share the minimal shape. + return { + max_tokens: 1, + messages: [{ content: "hi", role: "user" }], + model, + }; +}; + +/** The only headers a cell carries: never a credential. */ +const headersFor = (protocol: MatrixProtocol): Record => + protocol === "anthropic-messages" + ? { + "anthropic-version": ANTHROPIC_VERSION, + "content-type": "application/json", + } + : { "content-type": "application/json" }; + +/** + * Whether a catalog row is a free model. A keyless probe must never pick one: + * the gateway serves an unmetered model WITHOUT a key (measured: `beam-free` + * answers `200` with a real completion), so its cell would spend a completion + * and read as "other" instead of telling the path apart, and a free-tier + * model answers a wrong endpoint with a 500. The id test is defence in depth + * for a slice whose `is_free` was not derived (`parseModelsDevCatalog` sets it + * from the id or a zero cost). + */ +export const isFreeSpec = (spec: CatalogModelSpec): boolean => + spec.is_free === true || /free/iu.test(spec.id); + +/** + * The representative model for a protocol: the first catalog row that speaks + * it, is not free, and is not deprecated. A paid model is the class the + * gateway answers with 401 before it routes, which is what the matrix reads. + */ +const representative = ( + specs: readonly CatalogModelSpec[], + protocol: MatrixProtocol, + isDeprecated: (id: string) => boolean +): string | undefined => + specs.find( + (spec) => + protocolFor(spec) === protocol && + !isFreeSpec(spec) && + !isDeprecated(spec.id) + )?.id; + +/** + * Build the sixteen cells. + * + * @param catalogs.zen - the Zen plane's catalog slice. + * @param catalogs.go - the Go plane's catalog slice. + * @param catalogs.isDeprecated - which ids the catalog retired (default: none). + * A slice from `parseModelsDevCatalog` has already dropped its deprecated + * rows, so the live matrix passes none; this guards an unparsed slice. + * @returns the cells, plane by plane, protocol by protocol, legacy first. + */ +export const keylessMatrix = (catalogs: { + readonly zen: readonly CatalogModelSpec[]; + readonly go: readonly CatalogModelSpec[]; + readonly isDeprecated?: (plane: MatrixPlane, id: string) => boolean; +}): MatrixCell[] => { + const cells: MatrixCell[] = []; + for (const plane of ["opencode", "opencode-go"] as const) { + const specs = plane === "opencode" ? catalogs.zen : catalogs.go; + for (const protocol of MATRIX_PROTOCOLS) { + const model = representative( + specs, + protocol, + (id) => catalogs.isDeprecated?.(plane, id) ?? false + ); + for (const family of ["legacy", "modern"] as const) { + cells.push({ + body: model === undefined ? undefined : matrixBody(protocol, model), + family, + headers: headersFor(protocol), + method: "POST", + model, + plane, + protocol, + url: matrixUrl(plane, protocol, family, model), + }); + } + } + } + return cells; +}; + +/** + * Whether a probe's answer is transient and worth one retry before it is + * classified: a rate limit, a server error, or no answer at all. The live + * matrix sends sixteen requests on every pull request, and one 429 must not + * turn the job red. + * + * @param status - the HTTP status, or `undefined` for a transport error. + * @returns `true` to retry once. + */ +export const isTransientProbeAnswer = (status: number | undefined): boolean => + status === undefined || status === 429 || status >= 500; + +/** + * Read one keyless answer. + * + * @param answer.status - the HTTP status. + * @param answer.bodyText - the response body, as text. + * @returns the verdict. + */ +export const classifyKeylessProbe = (answer: { + readonly status: number; + readonly bodyText: string; +}): Exclude => { + if (answer.status === 404) { + return "path-gone"; + } + if (answer.status === 410) { + return "model-gone"; + } + if (answer.status === 400) { + return /model is unavailable/iu.test(answer.bodyText) + ? "model-gone" + : "path-exists"; + } + if (answer.status === 401 || answer.status === 403) { + return "path-exists"; + } + return "other"; +}; diff --git a/test/support/src.ts b/test/support/src.ts index a8ea1fa2..b42ae157 100644 --- a/test/support/src.ts +++ b/test/support/src.ts @@ -168,6 +168,7 @@ export { isDeprecatedModelOn, isGoModelsListingUrl, isModelsListingUrl, + MODELS_OPENCODE_URL, parseModelsDevCatalog, refreshCatalog, resetCatalogState, @@ -209,6 +210,7 @@ export { export { BASE_FOR_PROTOCOL, baseForProtocol, + LEGACY_PLANE_API, piModelFor, planeBaseFor, protocolFor, diff --git a/vite.config.ts b/vite.config.ts index 5f2fff6e..c9ff8205 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -25,7 +25,12 @@ export default defineConfig({ }, overrides: [ { - files: ["test/**/*.ts", "test/**/*.tsx", "scripts/**/*.ts"], + files: [ + "test/**/*.ts", + "test/**/*.tsx", + "test/**/*.mjs", + "scripts/**/*.ts", + ], rules: { "class-methods-use-this": "off", "func-style": "off",